├── deploy ├── csi-storageclass.yaml ├── csi-driver.yaml ├── csi-provisioner.yaml └── csi-3fs.yaml ├── examples ├── pvc.yaml └── pod.yaml ├── .gitignore ├── Dockerfile ├── internal └── endpoint │ └── endpoint.go ├── pkg ├── driver │ ├── options.go │ ├── flag.go │ ├── identityserver.go │ ├── server.go │ ├── healthcheck.go │ ├── 3fs.go │ ├── nodeserver.go │ └── controllerserver.go └── state │ ├── strings.go │ └── state.go ├── cmd └── csi-driver-3fs │ └── main.go ├── Makefile ├── README.md ├── go.mod ├── LICENSE └── go.sum /deploy/csi-storageclass.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: StorageClass 3 | apiVersion: storage.k8s.io/v1 4 | metadata: 5 | name: csi-3fs 6 | provisioner: 3fs.csi.mthreads.com 7 | parameters: {} -------------------------------------------------------------------------------- /examples/pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: demo0 5 | namespace: default 6 | spec: 7 | accessModes: 8 | - ReadWriteMany 9 | resources: 10 | requests: 11 | storage: 1Gi 12 | storageClassName: csi-3fs 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | dist 3 | .idea 4 | .DS_Store 5 | .vscode/ 6 | charts/*/charts/** 7 | test-charts/*/charts/** 8 | test-charts/*/Chart.lock 9 | .cr-release-packages 10 | .cr-release-packages/** 11 | !charts/*/charts/crds 12 | !charts/*/charts/crds/** 13 | .helm/ 14 | .local/ 15 | tmp/ 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23-bullseye AS gobuild 2 | ENV GOPROXY='https://goproxy.cn,direct' 3 | WORKDIR /build 4 | ADD . /build/ 5 | RUN go mod download -x 6 | RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o ./csi-3fs ./cmd/csi-driver-3fs 7 | 8 | FROM alpine:3.20 9 | RUN apk add --no-cache fuse 10 | COPY --from=gobuild /build/csi-3fs /csi-3fs 11 | ENTRYPOINT ["/csi-3fs"] -------------------------------------------------------------------------------- /examples/pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: nginx-test0 5 | namespace: default 6 | spec: 7 | containers: 8 | - name: csi-3fs-test-nginx 9 | image: nginx:latest 10 | volumeMounts: 11 | - mountPath: /usr/share/nginx/html/ 12 | name: webroot 13 | volumes: 14 | - name: webroot 15 | persistentVolumeClaim: 16 | claimName: demo0 17 | readOnly: false 18 | -------------------------------------------------------------------------------- /deploy/csi-driver.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: CSIDriver 3 | metadata: 4 | name: 3fs.csi.mthreads.com 5 | labels: 6 | app.kubernetes.io/instance: 3fs.csi.mthreads.com 7 | app.kubernetes.io/part-of: csi-driver-3fs 8 | app.kubernetes.io/name: 3fs.csi.mthreads.com 9 | app.kubernetes.io/component: csi-driver 10 | spec: 11 | volumeLifecycleModes: 12 | - Persistent 13 | podInfoOnMount: true 14 | fsGroupPolicy: File 15 | -------------------------------------------------------------------------------- /internal/endpoint/endpoint.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func Parse(ep string) (string, string, error) { 11 | if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") { 12 | s := strings.SplitN(ep, "://", 2) 13 | if s[1] != "" { 14 | return s[0], s[1], nil 15 | } 16 | return "", "", fmt.Errorf("Invalid endpoint: %v", ep) 17 | } 18 | 19 | return "unix", ep, nil 20 | } 21 | 22 | func Listen(endpoint string) (net.Listener, func(), error) { 23 | proto, addr, err := Parse(endpoint) 24 | if err != nil { 25 | return nil, nil, err 26 | } 27 | 28 | cleanup := func() {} 29 | if proto == "unix" { 30 | addr = "/" + addr 31 | if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { //nolint: vetshadow 32 | return nil, nil, fmt.Errorf("%s: %q", addr, err) 33 | } 34 | cleanup = func() { 35 | os.Remove(addr) 36 | } 37 | } 38 | 39 | l, err := net.Listen(proto, addr) 40 | return l, cleanup, err 41 | } 42 | -------------------------------------------------------------------------------- /pkg/driver/options.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | 7 | "github.com/MooreThreads/csi-driver-3fs/pkg/state" 8 | ) 9 | 10 | // ignoreFailedReadParameterName is a parameter that, when set to true, 11 | // causes the `--ignore-failed-read` option to be passed to `tar`. 12 | const ignoreFailedReadParameterName = "ignoreFailedRead" 13 | 14 | func optionsFromParameters(vol state.Volume, parameters map[string]string) ([]string, error) { 15 | // We do not support options for snapshots of block volumes 16 | if vol.VolAccessType == state.BlockAccess { 17 | return nil, nil 18 | } 19 | 20 | ignoreFailedReadString := parameters[ignoreFailedReadParameterName] 21 | if len(ignoreFailedReadString) == 0 { 22 | return nil, nil 23 | } 24 | 25 | if ok, err := strconv.ParseBool(ignoreFailedReadString); err != nil { 26 | return nil, fmt.Errorf( 27 | "invalid value for %q, expected boolean but was %q", 28 | ignoreFailedReadParameterName, 29 | ignoreFailedReadString, 30 | ) 31 | } else if ok { 32 | return []string{"--ignore-failed-read"}, nil 33 | } 34 | 35 | return nil, nil 36 | } 37 | -------------------------------------------------------------------------------- /pkg/state/strings.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package state 18 | 19 | // Strings is an ordered set of strings with helper functions for 20 | // adding, searching and removing entries. 21 | type Strings []string 22 | 23 | // Add appends at the end. 24 | func (s *Strings) Add(str string) { 25 | *s = append(*s, str) 26 | } 27 | 28 | // Has checks whether the string is already present. 29 | func (s *Strings) Has(str string) bool { 30 | for _, str2 := range *s { 31 | if str == str2 { 32 | return true 33 | } 34 | } 35 | return false 36 | } 37 | 38 | // Empty returns true if the list is empty. 39 | func (s *Strings) Empty() bool { 40 | return len(*s) == 0 41 | } 42 | 43 | // Remove removes the first occurrence of the string, if present. 44 | func (s *Strings) Remove(str string) { 45 | for i, str2 := range *s { 46 | if str == str2 { 47 | *s = append((*s)[:i], (*s)[i+1:]...) 48 | return 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pkg/driver/flag.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "strings" 8 | 9 | "k8s.io/apimachinery/pkg/api/resource" 10 | ) 11 | 12 | type Capacity map[string]resource.Quantity 13 | 14 | // Set is an implementation of flag.Value.Set. 15 | func (c *Capacity) Set(arg string) error { 16 | parts := strings.SplitN(arg, "=", 2) 17 | if len(parts) != 2 { 18 | return errors.New("must be of format =") 19 | } 20 | quantity, err := resource.ParseQuantity(parts[1]) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | // We overwrite any previous value. 26 | if *c == nil { 27 | *c = Capacity{} 28 | } 29 | (*c)[parts[0]] = quantity 30 | return nil 31 | } 32 | 33 | func (c *Capacity) String() string { 34 | return fmt.Sprintf("%v", map[string]resource.Quantity(*c)) 35 | } 36 | 37 | var _ flag.Value = &Capacity{} 38 | 39 | // Enabled returns true if capacities are configured. 40 | func (c *Capacity) Enabled() bool { 41 | return len(*c) > 0 42 | } 43 | 44 | // StringArray is a flag.Value implementation that allows to specify 45 | // a comma-separated list of strings on the command line. 46 | type StringArray []string 47 | 48 | // Set is an implementation of flag.Value.Set. 49 | func (s *StringArray) Set(value string) error { 50 | parts := strings.Split(value, ",") 51 | for _, part := range parts { 52 | *s = append(*s, strings.TrimSpace(part)) 53 | } 54 | return nil 55 | } 56 | 57 | // String is an implementation of flag.Value.String. 58 | func (s *StringArray) String() string { 59 | return fmt.Sprintf("%v", []string(*s)) 60 | } 61 | 62 | var _ flag.Value = &StringArray{} 63 | -------------------------------------------------------------------------------- /cmd/csi-driver-3fs/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "path" 8 | 9 | "k8s.io/klog/v2" 10 | 11 | "github.com/MooreThreads/csi-driver-3fs/pkg/driver" 12 | ) 13 | 14 | var ( 15 | conf driver.Config 16 | showVersion bool 17 | ) 18 | 19 | func init() { 20 | flag.StringVar(&conf.VendorVersion, "version", "v0.1.0", "CSI vendor version") 21 | flag.StringVar(&conf.Endpoint, "endpoint", "unix://tmp/csi.sock", "CSI endpoint") 22 | flag.StringVar(&conf.DriverName, "drivername", "3fs.csi.mthreads.com", "name of the driver") 23 | flag.StringVar(&conf.StateDir, "statedir", "/3fs/stage", "directory for storing state and volumes") 24 | flag.StringVar(&conf.NodeID, "nodeid", "", "node id") 25 | flag.BoolVar(&conf.EnableAttach, "enable-attach", true, "Enables RPC_PUBLISH_UNPUBLISH_VOLUME capability.") 26 | flag.Int64Var(&conf.MaxVolumeSize, "max-volume-size", 1024*1024*1024*1024, "maximum size of volumes in bytes") 27 | flag.BoolVar(&showVersion, "showVersion", false, "Show version.") 28 | } 29 | 30 | func main() { 31 | klog.InitFlags(nil) 32 | flag.Parse() 33 | 34 | if showVersion { 35 | baseName := path.Base(os.Args[0]) 36 | fmt.Println(baseName, conf.VendorVersion) 37 | return 38 | } 39 | 40 | if conf.MaxVolumeExpansionSizeNode == 0 { 41 | conf.MaxVolumeExpansionSizeNode = conf.MaxVolumeSize 42 | } 43 | 44 | driver, err := driver.NewCsi3fsDriver(conf) 45 | if err != nil { 46 | fmt.Printf("Failed to initialize driver: %s", err.Error()) 47 | os.Exit(1) 48 | } 49 | 50 | if err := driver.Run(); err != nil { 51 | fmt.Printf("Failed to run driver: %s", err.Error()) 52 | os.Exit(1) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pkg/driver/identityserver.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "github.com/container-storage-interface/spec/lib/go/csi" 5 | "golang.org/x/net/context" 6 | "google.golang.org/grpc/codes" 7 | "google.golang.org/grpc/status" 8 | "k8s.io/klog/v2" 9 | ) 10 | 11 | func (c3 *csi3fs) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { 12 | klog.V(5).Infof("Using default GetPluginInfo") 13 | 14 | if c3.config.DriverName == "" { 15 | return nil, status.Error(codes.Unavailable, "Driver name not configured") 16 | } 17 | 18 | if c3.config.VendorVersion == "" { 19 | return nil, status.Error(codes.Unavailable, "Driver is missing version") 20 | } 21 | 22 | return &csi.GetPluginInfoResponse{ 23 | Name: c3.config.DriverName, 24 | VendorVersion: c3.config.VendorVersion, 25 | }, nil 26 | } 27 | 28 | func (c3 *csi3fs) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { 29 | return &csi.ProbeResponse{}, nil 30 | } 31 | 32 | func (c3 *csi3fs) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { 33 | klog.V(5).Infof("Using default capabilities") 34 | caps := []*csi.PluginCapability{ 35 | { 36 | Type: &csi.PluginCapability_Service_{ 37 | Service: &csi.PluginCapability_Service{ 38 | Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, 39 | }, 40 | }, 41 | }, 42 | { 43 | Type: &csi.PluginCapability_Service_{ 44 | Service: &csi.PluginCapability_Service{ 45 | Type: csi.PluginCapability_Service_GROUP_CONTROLLER_SERVICE, 46 | }, 47 | }, 48 | }, 49 | } 50 | 51 | return &csi.GetPluginCapabilitiesResponse{Capabilities: caps}, nil 52 | } 53 | -------------------------------------------------------------------------------- /pkg/driver/server.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/container-storage-interface/spec/lib/go/csi" 7 | "google.golang.org/grpc" 8 | "k8s.io/klog/v2" 9 | 10 | "github.com/MooreThreads/csi-driver-3fs/internal/endpoint" 11 | ) 12 | 13 | func NewNonBlockingGRPCServer() *nonBlockingGRPCServer { 14 | return &nonBlockingGRPCServer{} 15 | } 16 | 17 | // NonBlocking server 18 | type nonBlockingGRPCServer struct { 19 | wg sync.WaitGroup 20 | server *grpc.Server 21 | cleanup func() 22 | } 23 | 24 | func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { 25 | s.wg.Add(1) 26 | 27 | go s.serve(endpoint, ids, cs, ns) 28 | 29 | return 30 | } 31 | 32 | func (s *nonBlockingGRPCServer) Wait() { 33 | s.wg.Wait() 34 | } 35 | 36 | func (s *nonBlockingGRPCServer) Stop() { 37 | s.server.GracefulStop() 38 | s.cleanup() 39 | } 40 | 41 | func (s *nonBlockingGRPCServer) ForceStop() { 42 | s.server.Stop() 43 | s.cleanup() 44 | } 45 | 46 | func (s *nonBlockingGRPCServer) serve(ep string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { 47 | listener, cleanup, err := endpoint.Listen(ep) 48 | if err != nil { 49 | klog.Fatalf("Failed to listen: %v", err) 50 | } 51 | 52 | opts := []grpc.ServerOption{} 53 | server := grpc.NewServer(opts...) 54 | s.server = server 55 | s.cleanup = cleanup 56 | 57 | if ids != nil { 58 | csi.RegisterIdentityServer(server, ids) 59 | } 60 | if cs != nil { 61 | csi.RegisterControllerServer(server, cs) 62 | } 63 | if ns != nil { 64 | csi.RegisterNodeServer(server, ns) 65 | } 66 | 67 | klog.Infof("Listening for connections on address: %#v", listener.Addr()) 68 | 69 | server.Serve(listener) 70 | } 71 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: run build image push vendor client-gen clean 2 | 3 | dockerhubUser = xxx/xxx 4 | tag = latest 5 | app = csi-3fs 6 | targetDir ?= bin 7 | buildx ?= false 8 | dualPlatform ?= linux/amd64,linux/arm64 9 | 10 | # build all apps 11 | build: 12 | @echo "Building $(app)" 13 | CGO_ENABLED=0 go build -ldflags "-s -w" -o $(targetDir)/$(app) ./cmd/csi-driver-3fs 14 | 15 | # build all images 16 | image: 17 | ifeq ($(buildx), false) 18 | echo "Building $(app) image" 19 | docker build -t $(dockerhubUser)/$(app):$(tag) --no-cache --build-arg APP=$(app) . 20 | else ifeq ($(buildx), true) 21 | echo "Building $(app) multi-arch image" 22 | docker buildx build -t $(dockerhubUser)/$(app):$(tag) --no-cache --platform $(dualPlatform) --push --build-arg APP=$(app) . 23 | endif 24 | 25 | # push all images 26 | push: image 27 | echo "Pushing $(app) image" 28 | docker push $(dockerhubUser)/$(app):$(tag) 29 | 30 | PHONY: golang-vet 31 | # golang vet 32 | golang-vet: 33 | go vet ./... 34 | 35 | .PHONY: golang-fmt 36 | # golang code formatting 37 | golang-fmt: 38 | golines --ignore-generated --ignored-dirs=vendor -w --max-len=150 --base-formatter=gofumpt . 39 | gci write -s standard -s default -s 'prefix(github.com/MooreThreads/csi-driver-3fs)' --skip-generated --skip-vendor . 40 | 41 | # show help 42 | help: 43 | @echo '' 44 | @echo 'Usage:' 45 | @echo ' make [target]' 46 | @echo '' 47 | @echo 'Targets:' 48 | @awk '/^[a-zA-Z\-0-9]+:/ { \ 49 | helpMessage = match(lastLine, /^# (.*)/); \ 50 | if (helpMessage) { \ 51 | helpCommand = substr($$1, 0, index($$1, ":")-1); \ 52 | helpMessage = substr(lastLine, RSTART + 2, RLENGTH); \ 53 | printf "\033[36m%-22s\033[0m %s\n", helpCommand,helpMessage; \ 54 | } \ 55 | } \ 56 | { lastLine = $$0 }' $(MAKEFILE_LIST) 57 | 58 | .DEFAULT_GOAL := help -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MT CSI Driver for 3FS 2 | 3 | This is a Container Storage Interface ([CSI](https://github.com/container-storage-interface/spec/blob/master/spec.md)) for [3FS](https://github.com/deepseek-ai/3FS) storage. 4 | 5 | ## Kubernetes installation 6 | 7 | ### Requirements 8 | 9 | * Kubernetes 1.23+ 10 | * Kubernetes has to allow privileged containers 11 | * Docker daemon must allow shared mounts (systemd flag `MountFlags=shared`) 12 | * Deepseek 3FS is deployed and the client is complete 13 | 14 | ### Manual installation 15 | 16 | #### 1. Build csi driver image 17 | 18 | ```sh 19 | make image 20 | ``` 21 | 22 | #### 2. Deploy the driver 23 | 24 | ```bash 25 | cd deploy 26 | kubectl apply -f csi-provisioner.yaml 27 | kubectl apply -f csi-driver.yaml 28 | kubectl apply -f csi-3fs.yaml 29 | kubectl apply -f csi-storageclass.yaml 30 | ``` 31 | 32 | #### 3. Deploy the pvc and pod 33 | 34 | ```bash 35 | cd examples 36 | kubectl apply -f pvc.yaml 37 | kubectl apply -f pod.yaml 38 | ``` 39 | 40 | #### 4. Check the results 41 | 42 | ```bash 43 | $ kubectl get csidriver 44 | NAME ATTACHREQUIRED PODINFOONMOUNT STORAGECAPACITY TOKENREQUESTS REQUIRESREPUBLISH MODES AGE 45 | 3fs.csi.mthreads.com true true false false Persistent 73m 46 | 47 | $ kubectl get sc 48 | NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE 49 | csi-3fs 3fs.csi.mthreads.com Delete Immediate false 59m 50 | 51 | $ kubectl get pvc -n default 52 | NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE 53 | default demo0 Bound pvc-52b8d8bc-f2e4-4a19-8aab-cb697e46de8f 1Gi RWX csi-3fs 47m 54 | ``` 55 | 56 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/MooreThreads/csi-driver-3fs 2 | 3 | go 1.23.1 4 | 5 | toolchain go1.24.0 6 | 7 | require ( 8 | github.com/container-storage-interface/spec v1.11.0 9 | github.com/kubernetes-csi/csi-lib-utils v0.20.0 10 | github.com/pborman/uuid v1.2.1 11 | github.com/stretchr/testify v1.10.0 12 | golang.org/x/net v0.32.0 13 | google.golang.org/grpc v1.69.0 14 | google.golang.org/protobuf v1.36.0 15 | k8s.io/apimachinery v0.32.0 16 | k8s.io/klog/v2 v2.130.1 17 | k8s.io/kubernetes v1.32.0 18 | k8s.io/utils v0.0.0-20241210054802-24370beab758 19 | ) 20 | 21 | require ( 22 | github.com/beorn7/perks v1.0.1 // indirect 23 | github.com/blang/semver/v4 v4.0.0 // indirect 24 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 25 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 26 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 27 | github.com/go-logr/logr v1.4.2 // indirect 28 | github.com/gogo/protobuf v1.3.2 // indirect 29 | github.com/google/uuid v1.6.0 // indirect 30 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 31 | github.com/klauspost/compress v1.17.11 // indirect 32 | github.com/moby/sys/mountinfo v0.7.2 // indirect 33 | github.com/moby/sys/userns v0.1.0 // indirect 34 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 35 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 36 | github.com/prometheus/client_golang v1.20.5 // indirect 37 | github.com/prometheus/client_model v0.6.1 // indirect 38 | github.com/prometheus/common v0.61.0 // indirect 39 | github.com/prometheus/procfs v0.15.1 // indirect 40 | github.com/spf13/cobra v1.8.1 // indirect 41 | github.com/spf13/pflag v1.0.5 // indirect 42 | github.com/x448/float16 v0.8.4 // indirect 43 | go.opentelemetry.io/otel v1.33.0 // indirect 44 | go.opentelemetry.io/otel/trace v1.33.0 // indirect 45 | golang.org/x/sys v0.28.0 // indirect 46 | golang.org/x/text v0.21.0 // indirect 47 | google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 // indirect 48 | gopkg.in/inf.v0 v0.9.1 // indirect 49 | gopkg.in/yaml.v3 v3.0.1 // indirect 50 | k8s.io/apiextensions-apiserver v0.32.0 // indirect 51 | k8s.io/apiserver v0.32.0 // indirect 52 | k8s.io/client-go v0.32.0 // indirect 53 | k8s.io/component-base v0.32.0 // indirect 54 | k8s.io/controller-manager v0.32.0 // indirect 55 | k8s.io/mount-utils v0.32.0 // indirect 56 | sigs.k8s.io/yaml v1.4.0 // indirect 57 | ) 58 | -------------------------------------------------------------------------------- /deploy/csi-provisioner.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: csi-3fs-provisioner-sa 5 | namespace: csi-3fs-system 6 | --- 7 | kind: ClusterRole 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | metadata: 10 | name: csi-3fs-external-provisioner-runner 11 | rules: 12 | - apiGroups: [""] 13 | resources: ["secrets"] 14 | verbs: ["get", "list"] 15 | - apiGroups: [""] 16 | resources: ["persistentvolumes"] 17 | verbs: ["get", "list", "watch", "create", "patch", "delete"] 18 | - apiGroups: [""] 19 | resources: ["persistentvolumeclaims"] 20 | verbs: ["get", "list", "watch", "update"] 21 | - apiGroups: ["storage.k8s.io"] 22 | resources: ["storageclasses", "volumeattachments"] 23 | verbs: ["get", "list", "watch"] 24 | - apiGroups: [""] 25 | resources: ["events"] 26 | verbs: ["list", "watch", "create", "update", "patch"] 27 | --- 28 | kind: ClusterRoleBinding 29 | apiVersion: rbac.authorization.k8s.io/v1 30 | metadata: 31 | name: csi-3fs-provisioner-role 32 | subjects: 33 | - kind: ServiceAccount 34 | name: csi-3fs-provisioner-sa 35 | namespace: csi-3fs-system 36 | roleRef: 37 | kind: ClusterRole 38 | name: csi-3fs-external-provisioner-runner 39 | apiGroup: rbac.authorization.k8s.io 40 | --- 41 | kind: Service 42 | apiVersion: v1 43 | metadata: 44 | name: csi-3fs-provisioner 45 | namespace: csi-3fs-system 46 | labels: 47 | app: csi-3fs-provisioner 48 | spec: 49 | selector: 50 | app: csi-3fs-provisioner 51 | ports: 52 | - name: csi-3fs-dummy 53 | port: 65535 54 | --- 55 | kind: StatefulSet 56 | apiVersion: apps/v1 57 | metadata: 58 | name: csi-3fs-provisioner 59 | namespace: csi-3fs-system 60 | spec: 61 | serviceName: "csi-3fs-provisioner" 62 | replicas: 1 63 | selector: 64 | matchLabels: 65 | app: csi-3fs-provisioner 66 | template: 67 | metadata: 68 | labels: 69 | app: csi-3fs-provisioner 70 | spec: 71 | serviceAccount: csi-3fs-provisioner-sa 72 | tolerations: 73 | - operator: "Exists" 74 | containers: 75 | - name: csi-provisioner 76 | image: csi-provisioner:v3.4.0 77 | args: 78 | - "--csi-address=$(ADDRESS)" 79 | - "--v=4" 80 | env: 81 | - name: ADDRESS 82 | value: /var/lib/kubelet/plugins/3fs.csi.mthreads.com/csi.sock 83 | imagePullPolicy: "IfNotPresent" 84 | volumeMounts: 85 | - name: socket-dir 86 | mountPath: /var/lib/kubelet/plugins/3fs.csi.mthreads.com 87 | - name: csi-3fs 88 | image: xxx/xxx/csi-3fs:latest 89 | imagePullPolicy: IfNotPresent 90 | args: 91 | - "--endpoint=$(CSI_ENDPOINT)" 92 | - "--nodeid=$(NODE_ID)" 93 | - "--v=4" 94 | env: 95 | - name: CSI_ENDPOINT 96 | value: unix:///var/lib/kubelet/plugins/3fs.csi.mthreads.com/csi.sock 97 | - name: NODE_ID 98 | valueFrom: 99 | fieldRef: 100 | fieldPath: spec.nodeName 101 | volumeMounts: 102 | - name: socket-dir 103 | mountPath: /var/lib/kubelet/plugins/3fs.csi.mthreads.com 104 | volumes: 105 | - name: socket-dir 106 | emptyDir: {} 107 | -------------------------------------------------------------------------------- /deploy/csi-3fs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: csi-3fs 5 | namespace: csi-3fs-system 6 | --- 7 | kind: ClusterRole 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | metadata: 10 | name: csi-3fs 11 | --- 12 | kind: ClusterRoleBinding 13 | apiVersion: rbac.authorization.k8s.io/v1 14 | metadata: 15 | name: csi-3fs 16 | subjects: 17 | - kind: ServiceAccount 18 | name: csi-3fs 19 | namespace: csi-3fs-system 20 | roleRef: 21 | kind: ClusterRole 22 | name: csi-3fs 23 | apiGroup: rbac.authorization.k8s.io 24 | --- 25 | kind: DaemonSet 26 | apiVersion: apps/v1 27 | metadata: 28 | name: csi-3fs 29 | namespace: csi-3fs-system 30 | spec: 31 | selector: 32 | matchLabels: 33 | app: csi-3fs 34 | template: 35 | metadata: 36 | labels: 37 | app: csi-3fs 38 | spec: 39 | tolerations: 40 | - operator: "Exists" 41 | serviceAccount: csi-3fs 42 | containers: 43 | - name: driver-registrar 44 | image: csi-node-driver-registrar:v2.10.1 45 | args: 46 | - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" 47 | - "--v=4" 48 | - "--csi-address=$(ADDRESS)" 49 | env: 50 | - name: ADDRESS 51 | value: /csi/csi.sock 52 | - name: DRIVER_REG_SOCK_PATH 53 | value: /var/lib/kubelet/plugins/3fs.csi.mthreads.com/csi.sock 54 | - name: KUBE_NODE_NAME 55 | valueFrom: 56 | fieldRef: 57 | fieldPath: spec.nodeName 58 | volumeMounts: 59 | - name: plugin-dir 60 | mountPath: /csi 61 | - name: registration-dir 62 | mountPath: /registration/ 63 | - name: csi-3fs 64 | securityContext: 65 | privileged: true 66 | capabilities: 67 | add: ["SYS_ADMIN"] 68 | allowPrivilegeEscalation: true 69 | image: xxx/xxx/csi-3fs:latest 70 | imagePullPolicy: IfNotPresent 71 | args: 72 | - "--endpoint=$(CSI_ENDPOINT)" 73 | - "--nodeid=$(NODE_ID)" 74 | - "--v=4" 75 | env: 76 | - name: CSI_ENDPOINT 77 | value: unix:///csi/csi.sock 78 | - name: NODE_ID 79 | valueFrom: 80 | fieldRef: 81 | fieldPath: spec.nodeName 82 | - name: DATA_DIR 83 | value: /var/lib/kubelet/plugins/kubernetes.io/csi 84 | volumeMounts: 85 | - name: plugin-dir 86 | mountPath: /csi 87 | - name: stage-dir 88 | mountPath: /3fs/stage 89 | mountPropagation: "Bidirectional" 90 | - name: pods-mount-dir 91 | mountPath: /var/lib/kubelet/pods 92 | mountPropagation: "Bidirectional" 93 | - name: fuse-device 94 | mountPath: /dev/fuse 95 | - name: systemd-control 96 | mountPath: /run/systemd 97 | volumes: 98 | - name: registration-dir 99 | hostPath: 100 | path: /var/lib/kubelet/plugins_registry/ 101 | type: DirectoryOrCreate 102 | - name: plugin-dir 103 | hostPath: 104 | path: /var/lib/kubelet/plugins/3fs.csi.mthreads.com 105 | type: DirectoryOrCreate 106 | - name: stage-dir 107 | hostPath: 108 | path: /3fs/stage 109 | type: DirectoryOrCreate 110 | - name: pods-mount-dir 111 | hostPath: 112 | path: /var/lib/kubelet/pods 113 | type: Directory 114 | - name: fuse-device 115 | hostPath: 116 | path: /dev/fuse 117 | - name: systemd-control 118 | hostPath: 119 | path: /run/systemd 120 | type: DirectoryOrCreate 121 | -------------------------------------------------------------------------------- /pkg/driver/healthcheck.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "strings" 9 | 10 | "k8s.io/klog/v2" 11 | fs "k8s.io/kubernetes/pkg/volume/util/fs" 12 | ) 13 | 14 | const ( 15 | podVolumeTargetPath = "/var/lib/kubelet/pods" 16 | csiSignOfVolumeTargetPath = "kubernetes.io~csi/pvc" 17 | ) 18 | 19 | type MountPointInfo struct { 20 | Target string `json:"target"` 21 | Source string `json:"source"` 22 | FsType string `json:"fstype"` 23 | Options string `json:"options"` 24 | ContainerFileSystem []MountPointInfo `json:"children,omitempty"` 25 | } 26 | 27 | type ContainerFileSystem struct { 28 | Children []MountPointInfo `json:"children"` 29 | } 30 | 31 | type FileSystems struct { 32 | Filsystem []ContainerFileSystem `json:"filesystems"` 33 | } 34 | 35 | func checkPathExist(path string) (bool, error) { 36 | _, err := os.Stat(path) 37 | if err != nil { 38 | if os.IsNotExist(err) { 39 | return false, nil 40 | } 41 | 42 | return false, err 43 | } 44 | 45 | return true, nil 46 | } 47 | 48 | func parseMountInfo(originalMountInfo []byte) ([]MountPointInfo, error) { 49 | fs := FileSystems{ 50 | Filsystem: make([]ContainerFileSystem, 0), 51 | } 52 | 53 | if err := json.Unmarshal(originalMountInfo, &fs); err != nil { 54 | return nil, err 55 | } 56 | 57 | if len(fs.Filsystem) <= 0 { 58 | return nil, fmt.Errorf("failed to get mount info") 59 | } 60 | 61 | return fs.Filsystem[0].Children, nil 62 | } 63 | 64 | func checkMountPointExist(volumePath string) (bool, error) { 65 | cmdPath, err := exec.LookPath("findmnt") 66 | if err != nil { 67 | return false, fmt.Errorf("findmnt not found: %w", err) 68 | } 69 | 70 | out, err := exec.Command(cmdPath, "--json").CombinedOutput() 71 | if err != nil { 72 | klog.V(3).Infof("failed to execute command: %+v", cmdPath) 73 | return false, err 74 | } 75 | 76 | if len(out) < 1 { 77 | return false, fmt.Errorf("mount point info is nil") 78 | } 79 | 80 | mountInfos, err := parseMountInfo([]byte(out)) 81 | if err != nil { 82 | return false, fmt.Errorf("failed to parse the mount infos: %+v", err) 83 | } 84 | 85 | mountInfosOfPod := MountPointInfo{} 86 | for _, mountInfo := range mountInfos { 87 | if mountInfo.Target == podVolumeTargetPath { 88 | mountInfosOfPod = mountInfo 89 | break 90 | } 91 | } 92 | 93 | for _, mountInfo := range mountInfosOfPod.ContainerFileSystem { 94 | if !strings.Contains(mountInfo.Source, volumePath) { 95 | continue 96 | } 97 | 98 | _, err = os.Stat(mountInfo.Target) 99 | if err != nil { 100 | if os.IsNotExist(err) { 101 | return false, nil 102 | } 103 | 104 | return false, err 105 | } 106 | 107 | return true, nil 108 | } 109 | 110 | return false, nil 111 | } 112 | 113 | func (c3 *csi3fs) checkPVCapacityValid(volID string) (bool, error) { 114 | volumePath := c3.getVolumePath(volID) 115 | _, fscapacity, _, _, _, _, err := fs.Info(volumePath) 116 | if err != nil { 117 | return false, fmt.Errorf("failed to get capacity info: %+v", err) 118 | } 119 | 120 | volume, err := c3.state.GetVolumeByID(volID) 121 | if err != nil { 122 | return false, err 123 | } 124 | volumeCapacity := volume.VolSize 125 | klog.V(3).Infof("volume capacity: %+v fs capacity:%+v", volumeCapacity, fscapacity) 126 | return fscapacity >= volumeCapacity, nil 127 | } 128 | 129 | func getPVStats(volumePath string) (available int64, capacity int64, used int64, inodes int64, inodesFree int64, inodesUsed int64, err error) { 130 | return fs.Info(volumePath) 131 | } 132 | 133 | func (c3 *csi3fs) checkPVUsage(volID string) (bool, error) { 134 | volumePath := c3.getVolumePath(volID) 135 | fsavailable, _, _, _, _, _, err := fs.Info(volumePath) 136 | if err != nil { 137 | return false, err 138 | } 139 | 140 | klog.V(3).Infof("fs available: %+v", fsavailable) 141 | return fsavailable > 0, nil 142 | } 143 | 144 | func (c3 *csi3fs) doHealthCheckInControllerSide(volID string) (bool, string) { 145 | volumePath := c3.getVolumePath(volID) 146 | klog.V(3).Infof("Volume with ID %s has path %s.", volID, volumePath) 147 | spExist, err := checkPathExist(volumePath) 148 | if err != nil { 149 | return false, err.Error() 150 | } 151 | 152 | if !spExist { 153 | return false, "The source path of the volume doesn't exist" 154 | } 155 | 156 | capValid, err := c3.checkPVCapacityValid(volID) 157 | if err != nil { 158 | return false, err.Error() 159 | } 160 | 161 | if !capValid { 162 | return false, "The capacity of volume is greater than actual storage" 163 | } 164 | 165 | available, err := c3.checkPVUsage(volID) 166 | if err != nil { 167 | return false, err.Error() 168 | } 169 | 170 | if !available { 171 | return false, "The free space of the volume is insufficient" 172 | } 173 | 174 | return true, "" 175 | } 176 | 177 | func (c3 *csi3fs) doHealthCheckInNodeSide(volID string) (bool, string) { 178 | volumePath := c3.getVolumePath(volID) 179 | mpExist, err := checkMountPointExist(volumePath) 180 | if err != nil { 181 | return false, err.Error() 182 | } 183 | 184 | if !mpExist { 185 | return false, "The volume isn't mounted" 186 | } 187 | 188 | return true, "" 189 | } 190 | -------------------------------------------------------------------------------- /pkg/driver/3fs.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | "sync" 10 | 11 | "github.com/container-storage-interface/spec/lib/go/csi" 12 | "google.golang.org/grpc/codes" 13 | "google.golang.org/grpc/status" 14 | "k8s.io/apimachinery/pkg/api/resource" 15 | "k8s.io/klog/v2" 16 | 17 | "github.com/MooreThreads/csi-driver-3fs/pkg/state" 18 | ) 19 | 20 | const ( 21 | storageKind = "kind" 22 | ) 23 | 24 | type csi3fs struct { 25 | csi.UnimplementedIdentityServer 26 | csi.UnimplementedControllerServer 27 | csi.UnimplementedNodeServer 28 | csi.UnimplementedGroupControllerServer 29 | csi.UnimplementedSnapshotMetadataServer 30 | 31 | config Config 32 | mutex sync.Mutex 33 | state state.State 34 | } 35 | 36 | type Config struct { 37 | DriverName string 38 | Endpoint string 39 | ProxyEndpoint string 40 | NodeID string 41 | VendorVersion string 42 | StateDir string 43 | MaxVolumesPerNode int64 44 | MaxVolumeSize int64 45 | AttachLimit int64 46 | Capacity Capacity 47 | Ephemeral bool 48 | ShowVersion bool 49 | EnableAttach bool 50 | EnableVolumeExpansion bool 51 | EnableControllerModifyVolume bool 52 | AcceptedMutableParameterNames StringArray 53 | DisableControllerExpansion bool 54 | DisableNodeExpansion bool 55 | MaxVolumeExpansionSizeNode int64 56 | CheckVolumeLifecycle bool 57 | } 58 | 59 | func NewCsi3fsDriver(cfg Config) (*csi3fs, error) { 60 | if cfg.DriverName == "" { 61 | return nil, errors.New("no driver name provided") 62 | } 63 | 64 | if cfg.NodeID == "" { 65 | return nil, errors.New("no node id provided") 66 | } 67 | 68 | if cfg.Endpoint == "" { 69 | return nil, errors.New("no driver endpoint provided") 70 | } 71 | 72 | if err := os.MkdirAll(cfg.StateDir, 0o750); err != nil { 73 | return nil, fmt.Errorf("failed to create dataRoot: %v", err) 74 | } 75 | 76 | klog.Infof("Driver: %v ", cfg.DriverName) 77 | klog.Infof("Version: %s", cfg.VendorVersion) 78 | 79 | s, err := state.New(path.Join(cfg.StateDir, "state.json")) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | c3 := &csi3fs{config: cfg, state: s} 85 | return c3, nil 86 | } 87 | 88 | func (c3 *csi3fs) Run() error { 89 | s := NewNonBlockingGRPCServer() 90 | s.Start(c3.config.Endpoint, c3, c3, c3) 91 | s.Wait() 92 | 93 | return nil 94 | } 95 | 96 | func (c3 *csi3fs) getVolumePath(volID string) string { 97 | return filepath.Join(c3.config.StateDir, volID) 98 | } 99 | 100 | // createVolume creates the directory for the 3fs volume. 101 | func (c3 *csi3fs) createVolume(volID, name string, cap int64, volAccessType state.AccessType, ephemeral bool, kind string) (*state.Volume, error) { 102 | // Check for maximum available capacity 103 | if cap > c3.config.MaxVolumeSize { 104 | return nil, status.Errorf(codes.OutOfRange, "Requested capacity %d exceeds maximum allowed %d", cap, c3.config.MaxVolumeSize) 105 | } 106 | 107 | if c3.config.Capacity.Enabled() { 108 | if kind == "" { 109 | // Pick some kind with sufficient remaining capacity. 110 | for k, c := range c3.config.Capacity { 111 | if c3.sumVolumeSizes(k)+cap <= c.Value() { 112 | kind = k 113 | break 114 | } 115 | } 116 | } 117 | if kind == "" { 118 | // Still nothing?! 119 | return nil, status.Errorf(codes.ResourceExhausted, "requested capacity %d of arbitrary storage exceeds all remaining capacity", cap) 120 | } 121 | used := c3.sumVolumeSizes(kind) 122 | available := c3.config.Capacity[kind] 123 | if used+cap > available.Value() { 124 | return nil, status.Errorf(codes.ResourceExhausted, "requested capacity %d exceeds remaining capacity for %q, %s out of %s already used", 125 | cap, kind, resource.NewQuantity(used, resource.BinarySI).String(), available.String()) 126 | } 127 | } else if kind != "" { 128 | return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("capacity tracking disabled, specifying kind %q is invalid", kind)) 129 | } 130 | 131 | path := c3.getVolumePath(volID) 132 | 133 | switch volAccessType { 134 | case state.MountAccess: 135 | err := os.MkdirAll(path, 0o777) 136 | if err != nil { 137 | return nil, err 138 | } 139 | default: 140 | return nil, fmt.Errorf("unsupported access type %v", volAccessType) 141 | } 142 | 143 | volume := state.Volume{ 144 | VolID: volID, 145 | VolName: name, 146 | VolSize: cap, 147 | VolPath: path, 148 | VolAccessType: volAccessType, 149 | Kind: kind, 150 | } 151 | 152 | klog.V(4).Infof("adding 3fs volume: %s = %+v", volID, volume) 153 | if err := c3.state.UpdateVolume(volume); err != nil { 154 | return nil, err 155 | } 156 | return &volume, nil 157 | } 158 | 159 | // deleteVolume deletes the directory for the 3fs volume. 160 | func (c3 *csi3fs) deleteVolume(volID string) error { 161 | klog.V(4).Infof("starting to delete 3fs volume: %s", volID) 162 | 163 | vol, err := c3.state.GetVolumeByID(volID) 164 | if err != nil { 165 | return nil 166 | } 167 | 168 | path := c3.getVolumePath(volID) 169 | if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { 170 | return err 171 | } 172 | if err := c3.state.DeleteVolume(volID); err != nil { 173 | return err 174 | } 175 | 176 | klog.V(4).Infof("deleted 3fs volume: %s = %+v", volID, vol) 177 | return nil 178 | } 179 | 180 | func (c3 *csi3fs) sumVolumeSizes(kind string) int64 { 181 | var sum int64 182 | for _, volume := range c3.state.GetVolumes() { 183 | if volume.Kind == kind { 184 | sum += volume.VolSize 185 | } 186 | } 187 | return sum 188 | } 189 | 190 | func (c3 *csi3fs) getAttachCount() int64 { 191 | var count int64 192 | for _, vol := range c3.state.GetVolumes() { 193 | if vol.Attached { 194 | count++ 195 | } 196 | } 197 | return count 198 | } 199 | -------------------------------------------------------------------------------- /pkg/state/state.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package state manages the internal state of the driver which needs to be maintained 18 | // across driver restarts. 19 | package state 20 | 21 | import ( 22 | "encoding/json" 23 | "errors" 24 | "os" 25 | "sort" 26 | 27 | "google.golang.org/grpc/codes" 28 | "google.golang.org/grpc/status" 29 | "google.golang.org/protobuf/types/known/timestamppb" 30 | ) 31 | 32 | type AccessType int 33 | 34 | const ( 35 | MountAccess AccessType = iota 36 | BlockAccess 37 | 38 | // BlockSizeBytes represents the default block size. 39 | BlockSizeBytes = 4096 40 | ) 41 | 42 | type Volume struct { 43 | VolName string 44 | VolID string 45 | VolSize int64 46 | VolPath string 47 | VolAccessType AccessType 48 | ParentVolID string 49 | ParentSnapID string 50 | Ephemeral bool 51 | NodeID string 52 | Kind string 53 | ReadOnlyAttach bool 54 | Attached bool 55 | // Staged contains the staging target path at which the volume 56 | // was staged. A set of paths is used for consistency 57 | // with Published. 58 | Staged Strings 59 | // Published contains the target paths where the volume 60 | // was published. 61 | Published Strings 62 | } 63 | 64 | type Snapshot struct { 65 | Name string 66 | Id string 67 | VolID string 68 | Path string 69 | CreationTime *timestamppb.Timestamp 70 | SizeBytes int64 71 | ReadyToUse bool 72 | GroupSnapshotID string 73 | } 74 | 75 | type GroupSnapshot struct { 76 | Name string 77 | Id string 78 | SnapshotIDs []string 79 | SourceVolumeIDs []string 80 | CreationTime *timestamppb.Timestamp 81 | ReadyToUse bool 82 | } 83 | 84 | // State is the interface that the rest of the code has to use to 85 | // access and change state. All error messages contain gRPC 86 | // status codes and can be returned without wrapping. 87 | type State interface { 88 | // GetVolumeByID retrieves a volume by its unique ID or returns 89 | // an error including that ID when not found. 90 | GetVolumeByID(volID string) (Volume, error) 91 | 92 | // GetVolumeByName retrieves a volume by its name or returns 93 | // an error including that name when not found. 94 | GetVolumeByName(volName string) (Volume, error) 95 | 96 | // GetVolumes returns all currently existing volumes. 97 | GetVolumes() []Volume 98 | 99 | // UpdateVolume updates the existing 3fs volume, 100 | // identified by its volume ID, or adds it if it does 101 | // not exist yet. 102 | UpdateVolume(volume Volume) error 103 | 104 | // DeleteVolume deletes the volume with the given 105 | // volume ID. It is not an error when such a volume 106 | // does not exist. 107 | DeleteVolume(volID string) error 108 | 109 | // GetSnapshotByID retrieves a snapshot by its unique ID or returns 110 | // an error including that ID when not found. 111 | GetSnapshotByID(snapshotID string) (Snapshot, error) 112 | 113 | // GetSnapshotByName retrieves a snapshot by its name or returns 114 | // an error including that name when not found. 115 | GetSnapshotByName(volName string) (Snapshot, error) 116 | 117 | // GetSnapshots returns all currently existing snapshots. 118 | GetSnapshots() []Snapshot 119 | 120 | // UpdateSnapshot updates the existing 3fs snapshot, 121 | // identified by its snapshot ID, or adds it if it does 122 | // not exist yet. 123 | UpdateSnapshot(snapshot Snapshot) error 124 | 125 | // DeleteSnapshot deletes the snapshot with the given 126 | // snapshot ID. It is not an error when such a snapshot 127 | // does not exist. 128 | DeleteSnapshot(snapshotID string) error 129 | 130 | // GetGroupSnapshotByID retrieves a groupsnapshot by its unique ID or 131 | // returns an error including that ID when not found. 132 | GetGroupSnapshotByID(vgsID string) (GroupSnapshot, error) 133 | 134 | // GetGroupSnapshotByName retrieves a groupsnapshot by its name or 135 | // returns an error including that name when not found. 136 | GetGroupSnapshotByName(volName string) (GroupSnapshot, error) 137 | 138 | // GetGroupSnapshots returns all currently existing groupsnapshots. 139 | GetGroupSnapshots() []GroupSnapshot 140 | 141 | // UpdateGroupSnapshot updates the existing 3fs groupsnapshot, 142 | // identified by its snapshot ID, or adds it if it does not exist yet. 143 | UpdateGroupSnapshot(snapshot GroupSnapshot) error 144 | 145 | // DeleteGroupSnapshot deletes the groupsnapshot with the given 146 | // groupsnapshot ID. It is not an error when such a groupsnapshot does 147 | // not exist. 148 | DeleteGroupSnapshot(groupSnapshotID string) error 149 | } 150 | 151 | type resources struct { 152 | Volumes []Volume 153 | Snapshots []Snapshot 154 | GroupSnapshots []GroupSnapshot 155 | } 156 | 157 | type state struct { 158 | resources 159 | 160 | statefilePath string 161 | } 162 | 163 | var _ State = &state{} 164 | 165 | // New retrieves the complete state of the driver from the file if given 166 | // and then ensures that all changes are mirrored immediately in the 167 | // given file. If not given, the initial state is empty and changes 168 | // are not saved. 169 | func New(statefilePath string) (State, error) { 170 | s := &state{ 171 | statefilePath: statefilePath, 172 | } 173 | 174 | return s, s.restore() 175 | } 176 | 177 | func (s *state) dump() error { 178 | data, err := json.Marshal(&s.resources) 179 | if err != nil { 180 | return status.Errorf(codes.Internal, "error encoding volumes and snapshots: %v", err) 181 | } 182 | if err := os.WriteFile(s.statefilePath, data, 0o600); err != nil { 183 | return status.Errorf(codes.Internal, "error writing state file: %v", err) 184 | } 185 | return nil 186 | } 187 | 188 | func (s *state) restore() error { 189 | s.Volumes = nil 190 | s.Snapshots = nil 191 | 192 | data, err := os.ReadFile(s.statefilePath) 193 | switch { 194 | case errors.Is(err, os.ErrNotExist): 195 | // Nothing to do. 196 | return nil 197 | case err != nil: 198 | return status.Errorf(codes.Internal, "error reading state file: %v", err) 199 | } 200 | if err := json.Unmarshal(data, &s.resources); err != nil { 201 | return status.Errorf(codes.Internal, "error encoding volumes and snapshots from state file %q: %v", s.statefilePath, err) 202 | } 203 | return nil 204 | } 205 | 206 | func (s *state) GetVolumeByID(volID string) (Volume, error) { 207 | for _, volume := range s.Volumes { 208 | if volume.VolID == volID { 209 | return volume, nil 210 | } 211 | } 212 | return Volume{}, status.Errorf(codes.NotFound, "volume id %s does not exist in the volumes list", volID) 213 | } 214 | 215 | func (s *state) GetVolumeByName(volName string) (Volume, error) { 216 | for _, volume := range s.Volumes { 217 | if volume.VolName == volName { 218 | return volume, nil 219 | } 220 | } 221 | return Volume{}, status.Errorf(codes.NotFound, "volume name %s does not exist in the volumes list", volName) 222 | } 223 | 224 | func (s *state) GetVolumes() []Volume { 225 | volumes := make([]Volume, len(s.Volumes)) 226 | copy(volumes, s.Volumes) 227 | return volumes 228 | } 229 | 230 | func (s *state) UpdateVolume(update Volume) error { 231 | for i, volume := range s.Volumes { 232 | if volume.VolID == update.VolID { 233 | s.Volumes[i] = update 234 | return s.dump() 235 | } 236 | } 237 | s.Volumes = append(s.Volumes, update) 238 | return s.dump() 239 | } 240 | 241 | func (s *state) DeleteVolume(volID string) error { 242 | for i, volume := range s.Volumes { 243 | if volume.VolID == volID { 244 | s.Volumes = append(s.Volumes[:i], s.Volumes[i+1:]...) 245 | return s.dump() 246 | } 247 | } 248 | return nil 249 | } 250 | 251 | func (s *state) GetSnapshotByID(snapshotID string) (Snapshot, error) { 252 | for _, snapshot := range s.Snapshots { 253 | if snapshot.Id == snapshotID { 254 | return snapshot, nil 255 | } 256 | } 257 | return Snapshot{}, status.Errorf(codes.NotFound, "snapshot id %s does not exist in the snapshots list", snapshotID) 258 | } 259 | 260 | func (s *state) GetSnapshotByName(name string) (Snapshot, error) { 261 | for _, snapshot := range s.Snapshots { 262 | if snapshot.Name == name { 263 | return snapshot, nil 264 | } 265 | } 266 | return Snapshot{}, status.Errorf(codes.NotFound, "snapshot name %s does not exist in the snapshots list", name) 267 | } 268 | 269 | func (s *state) GetSnapshots() []Snapshot { 270 | snapshots := make([]Snapshot, len(s.Snapshots)) 271 | copy(snapshots, s.Snapshots) 272 | return snapshots 273 | } 274 | 275 | func (s *state) UpdateSnapshot(update Snapshot) error { 276 | for i, snapshot := range s.Snapshots { 277 | if snapshot.Id == update.Id { 278 | s.Snapshots[i] = update 279 | return s.dump() 280 | } 281 | } 282 | s.Snapshots = append(s.Snapshots, update) 283 | return s.dump() 284 | } 285 | 286 | func (s *state) DeleteSnapshot(snapshotID string) error { 287 | for i, snapshot := range s.Snapshots { 288 | if snapshot.Id == snapshotID { 289 | s.Snapshots = append(s.Snapshots[:i], s.Snapshots[i+1:]...) 290 | return s.dump() 291 | } 292 | } 293 | return nil 294 | } 295 | 296 | func (s *state) GetGroupSnapshotByID(groupSnapshotID string) (GroupSnapshot, error) { 297 | for _, groupSnapshot := range s.GroupSnapshots { 298 | if groupSnapshot.Id == groupSnapshotID { 299 | return groupSnapshot, nil 300 | } 301 | } 302 | return GroupSnapshot{}, status.Errorf(codes.NotFound, "groupsnapshot id %s does not exist in the groupsnapshots list", groupSnapshotID) 303 | } 304 | 305 | func (s *state) GetGroupSnapshotByName(name string) (GroupSnapshot, error) { 306 | for _, groupSnapshot := range s.GroupSnapshots { 307 | if groupSnapshot.Name == name { 308 | return groupSnapshot, nil 309 | } 310 | } 311 | return GroupSnapshot{}, status.Errorf(codes.NotFound, "groupsnapshot name %s does not exist in the groupsnapshots list", name) 312 | } 313 | 314 | func (s *state) GetGroupSnapshots() []GroupSnapshot { 315 | groupSnapshots := make([]GroupSnapshot, len(s.GroupSnapshots)) 316 | for i, groupSnapshot := range s.GroupSnapshots { 317 | groupSnapshots[i] = groupSnapshot 318 | } 319 | return groupSnapshots 320 | } 321 | 322 | func (s *state) UpdateGroupSnapshot(update GroupSnapshot) error { 323 | for i, groupSnapshot := range s.GroupSnapshots { 324 | if groupSnapshot.Id == update.Id { 325 | s.GroupSnapshots[i] = update 326 | return s.dump() 327 | } 328 | } 329 | s.GroupSnapshots = append(s.GroupSnapshots, update) 330 | return s.dump() 331 | } 332 | 333 | func (s *state) DeleteGroupSnapshot(groupSnapshotID string) error { 334 | for i, groupSnapshot := range s.GroupSnapshots { 335 | if groupSnapshot.Id == groupSnapshotID { 336 | s.GroupSnapshots = append(s.GroupSnapshots[:i], s.GroupSnapshots[i+1:]...) 337 | return s.dump() 338 | } 339 | } 340 | return nil 341 | } 342 | 343 | func (gs *GroupSnapshot) MatchesSourceVolumeIDs(sourceVolumeIDs []string) bool { 344 | return equalIDs(gs.SourceVolumeIDs, sourceVolumeIDs) 345 | } 346 | 347 | func (gs *GroupSnapshot) MatchesSnapshotIDs(snapshotIDs []string) bool { 348 | return equalIDs(gs.SnapshotIDs, snapshotIDs) 349 | } 350 | 351 | func equalIDs(a, b []string) bool { 352 | if len(a) != len(b) { 353 | return false 354 | } 355 | 356 | // sort slices so that values are at the same location 357 | sort.Strings(a) 358 | sort.Strings(b) 359 | 360 | for i, v := range a { 361 | if v != b[i] { 362 | return false 363 | } 364 | } 365 | 366 | return true 367 | } 368 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /pkg/driver/nodeserver.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/container-storage-interface/spec/lib/go/csi" 8 | "golang.org/x/net/context" 9 | "google.golang.org/grpc/codes" 10 | "google.golang.org/grpc/status" 11 | "k8s.io/klog/v2" 12 | "k8s.io/utils/mount" 13 | 14 | "github.com/MooreThreads/csi-driver-3fs/pkg/state" 15 | ) 16 | 17 | const ( 18 | failedPreconditionAccessModeConflict = "volume uses SINGLE_NODE_SINGLE_WRITER access mode and is already mounted at a different target path" 19 | ) 20 | 21 | func (c3 *csi3fs) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { 22 | // Check arguments 23 | if req.GetVolumeCapability() == nil { 24 | return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request") 25 | } 26 | if len(req.GetVolumeId()) == 0 { 27 | return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") 28 | } 29 | if len(req.GetTargetPath()) == 0 { 30 | return nil, status.Error(codes.InvalidArgument, "Target path missing in request") 31 | } 32 | 33 | targetPath := req.GetTargetPath() 34 | 35 | c3.mutex.Lock() 36 | defer c3.mutex.Unlock() 37 | 38 | mounter := mount.New("") 39 | 40 | vol, err := c3.state.GetVolumeByID(req.GetVolumeId()) 41 | if err != nil { 42 | return nil, status.Error(codes.NotFound, err.Error()) 43 | } 44 | 45 | if hasSingleNodeSingleWriterAccessMode(req) && isMountedElsewhere(req, vol) { 46 | return nil, status.Error(codes.FailedPrecondition, failedPreconditionAccessModeConflict) 47 | } 48 | if vol.Staged.Empty() { 49 | return nil, status.Errorf(codes.FailedPrecondition, "volume %q must be staged before publishing", vol.VolID) 50 | } 51 | if !vol.Staged.Has(req.GetStagingTargetPath()) { 52 | return nil, status.Errorf(codes.InvalidArgument, "volume %q was staged at %v, not %q", vol.VolID, vol.Staged, req.GetStagingTargetPath()) 53 | } 54 | 55 | if vol.VolAccessType != state.MountAccess { 56 | return nil, status.Error(codes.InvalidArgument, "cannot publish a non-mount volume as mount volume") 57 | } 58 | 59 | notMnt, err := mount.IsNotMountPoint(mounter, targetPath) 60 | if err != nil { 61 | if os.IsNotExist(err) { 62 | if err = os.Mkdir(targetPath, 0o750); err != nil { 63 | return nil, fmt.Errorf("create target path: %w", err) 64 | } 65 | notMnt = true 66 | } else { 67 | return nil, fmt.Errorf("check target path: %w", err) 68 | } 69 | } 70 | 71 | if !notMnt { 72 | return &csi.NodePublishVolumeResponse{}, nil 73 | } 74 | 75 | readOnly := req.GetReadonly() 76 | volumeId := req.GetVolumeId() 77 | 78 | options := []string{"bind"} 79 | if readOnly { 80 | options = append(options, "ro") 81 | } 82 | path := c3.getVolumePath(volumeId) 83 | 84 | if err := mounter.Mount(path, targetPath, "", options); err != nil { 85 | return nil, fmt.Errorf("failed to mount device: %s at %s: %s", path, targetPath, err.Error()) 86 | } 87 | 88 | vol.NodeID = c3.config.NodeID 89 | vol.Published.Add(targetPath) 90 | if err := c3.state.UpdateVolume(vol); err != nil { 91 | return nil, err 92 | } 93 | return &csi.NodePublishVolumeResponse{}, nil 94 | } 95 | 96 | func (c3 *csi3fs) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { 97 | // Check arguments 98 | if len(req.GetVolumeId()) == 0 { 99 | return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") 100 | } 101 | if len(req.GetTargetPath()) == 0 { 102 | return nil, status.Error(codes.InvalidArgument, "Target path missing in request") 103 | } 104 | targetPath := req.GetTargetPath() 105 | volumeID := req.GetVolumeId() 106 | 107 | c3.mutex.Lock() 108 | defer c3.mutex.Unlock() 109 | 110 | vol, err := c3.state.GetVolumeByID(volumeID) 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | if !vol.Published.Has(targetPath) { 116 | klog.V(4).Infof("Volume %q is not published at %q, nothing to do.", volumeID, targetPath) 117 | return &csi.NodeUnpublishVolumeResponse{}, nil 118 | } 119 | 120 | // Unmount only if the target path is really a mount point. 121 | if notMnt, err := mount.IsNotMountPoint(mount.New(""), targetPath); err != nil { 122 | if !os.IsNotExist(err) { 123 | return nil, fmt.Errorf("check target path: %w", err) 124 | } 125 | } else if !notMnt { 126 | // Unmounting the image or filesystem. 127 | err = mount.New("").Unmount(targetPath) 128 | if err != nil { 129 | return nil, fmt.Errorf("unmount target path: %w", err) 130 | } 131 | } 132 | 133 | // Delete the mount point. 134 | if err = os.RemoveAll(targetPath); err != nil { 135 | return nil, fmt.Errorf("remove target path: %w", err) 136 | } 137 | klog.V(4).Infof("3fs: volume %s has been unpublished.", targetPath) 138 | 139 | vol.Published.Remove(targetPath) 140 | if err := c3.state.UpdateVolume(vol); err != nil { 141 | return nil, err 142 | } 143 | 144 | return &csi.NodeUnpublishVolumeResponse{}, nil 145 | } 146 | 147 | func (c3 *csi3fs) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { 148 | // Check arguments 149 | if len(req.GetVolumeId()) == 0 { 150 | return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") 151 | } 152 | stagingTargetPath := req.GetStagingTargetPath() 153 | if stagingTargetPath == "" { 154 | return nil, status.Error(codes.InvalidArgument, "Target path missing in request") 155 | } 156 | if req.GetVolumeCapability() == nil { 157 | return nil, status.Error(codes.InvalidArgument, "Volume Capability missing in request") 158 | } 159 | 160 | c3.mutex.Lock() 161 | defer c3.mutex.Unlock() 162 | 163 | vol, err := c3.state.GetVolumeByID(req.VolumeId) 164 | if err != nil { 165 | return nil, err 166 | } 167 | 168 | if c3.config.EnableAttach && !vol.Attached { 169 | return nil, status.Errorf(codes.Internal, "ControllerPublishVolume must be called on volume '%s' before staging on node", 170 | vol.VolID) 171 | } 172 | 173 | if vol.Staged.Has(stagingTargetPath) { 174 | klog.V(4).Infof("Volume %q is already staged at %q, nothing to do.", req.VolumeId, stagingTargetPath) 175 | return &csi.NodeStageVolumeResponse{}, nil 176 | } 177 | 178 | if !vol.Staged.Empty() { 179 | return nil, status.Errorf(codes.FailedPrecondition, "volume %q is already staged at %v", req.VolumeId, vol.Staged) 180 | } 181 | 182 | vol.Staged.Add(stagingTargetPath) 183 | if err := c3.state.UpdateVolume(vol); err != nil { 184 | return nil, err 185 | } 186 | 187 | return &csi.NodeStageVolumeResponse{}, nil 188 | } 189 | 190 | func (c3 *csi3fs) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { 191 | // Check arguments 192 | if len(req.GetVolumeId()) == 0 { 193 | return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") 194 | } 195 | stagingTargetPath := req.GetStagingTargetPath() 196 | if stagingTargetPath == "" { 197 | return nil, status.Error(codes.InvalidArgument, "Target path missing in request") 198 | } 199 | 200 | c3.mutex.Lock() 201 | defer c3.mutex.Unlock() 202 | 203 | vol, err := c3.state.GetVolumeByID(req.VolumeId) 204 | if err != nil { 205 | return nil, err 206 | } 207 | 208 | if !vol.Staged.Has(stagingTargetPath) { 209 | klog.V(4).Infof("Volume %q is not staged at %q, nothing to do.", req.VolumeId, stagingTargetPath) 210 | return &csi.NodeUnstageVolumeResponse{}, nil 211 | } 212 | 213 | if !vol.Published.Empty() { 214 | return nil, status.Errorf(codes.Internal, "volume %q is still published at %q on node %q", vol.VolID, vol.Published, vol.NodeID) 215 | } 216 | vol.Staged.Remove(stagingTargetPath) 217 | if err := c3.state.UpdateVolume(vol); err != nil { 218 | return nil, err 219 | } 220 | 221 | return &csi.NodeUnstageVolumeResponse{}, nil 222 | } 223 | 224 | func (c3 *csi3fs) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { 225 | resp := &csi.NodeGetInfoResponse{ 226 | NodeId: c3.config.NodeID, 227 | MaxVolumesPerNode: c3.config.MaxVolumesPerNode, 228 | } 229 | 230 | if c3.config.AttachLimit > 0 { 231 | resp.MaxVolumesPerNode = c3.config.AttachLimit 232 | } 233 | 234 | return resp, nil 235 | } 236 | 237 | func (c3 *csi3fs) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { 238 | caps := []*csi.NodeServiceCapability{ 239 | { 240 | Type: &csi.NodeServiceCapability_Rpc{ 241 | Rpc: &csi.NodeServiceCapability_RPC{ 242 | Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, 243 | }, 244 | }, 245 | }, 246 | { 247 | Type: &csi.NodeServiceCapability_Rpc{ 248 | Rpc: &csi.NodeServiceCapability_RPC{ 249 | Type: csi.NodeServiceCapability_RPC_VOLUME_CONDITION, 250 | }, 251 | }, 252 | }, 253 | { 254 | Type: &csi.NodeServiceCapability_Rpc{ 255 | Rpc: &csi.NodeServiceCapability_RPC{ 256 | Type: csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, 257 | }, 258 | }, 259 | }, 260 | { 261 | Type: &csi.NodeServiceCapability_Rpc{ 262 | Rpc: &csi.NodeServiceCapability_RPC{ 263 | Type: csi.NodeServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, 264 | }, 265 | }, 266 | }, 267 | } 268 | if c3.config.EnableVolumeExpansion && !c3.config.DisableNodeExpansion { 269 | caps = append(caps, &csi.NodeServiceCapability{ 270 | Type: &csi.NodeServiceCapability_Rpc{ 271 | Rpc: &csi.NodeServiceCapability_RPC{ 272 | Type: csi.NodeServiceCapability_RPC_EXPAND_VOLUME, 273 | }, 274 | }, 275 | }) 276 | } 277 | 278 | return &csi.NodeGetCapabilitiesResponse{Capabilities: caps}, nil 279 | } 280 | 281 | func (c3 *csi3fs) NodeGetVolumeStats(ctx context.Context, in *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { 282 | if len(in.GetVolumeId()) == 0 { 283 | return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") 284 | } 285 | if len(in.GetVolumePath()) == 0 { 286 | return nil, status.Error(codes.InvalidArgument, "Volume Path not provided") 287 | } 288 | 289 | c3.mutex.Lock() 290 | defer c3.mutex.Unlock() 291 | 292 | volume, err := c3.state.GetVolumeByID(in.GetVolumeId()) 293 | if err != nil { 294 | return nil, err 295 | } 296 | 297 | if _, err := os.Stat(in.GetVolumePath()); err != nil { 298 | return nil, status.Errorf(codes.NotFound, "Could not get file information from %s: %+v", in.GetVolumePath(), err) 299 | } 300 | 301 | healthy, msg := c3.doHealthCheckInNodeSide(in.GetVolumeId()) 302 | klog.V(3).Infof("Healthy state: %+v Volume: %+v", volume.VolName, healthy) 303 | available, capacity, used, inodes, inodesFree, inodesUsed, err := getPVStats(in.GetVolumePath()) 304 | if err != nil { 305 | return nil, fmt.Errorf("get volume stats failed: %w", err) 306 | } 307 | 308 | klog.V(3). 309 | Infof("Capacity: %+v Used: %+v Available: %+v Inodes: %+v Free inodes: %+v Used inodes: %+v", capacity, used, available, inodes, inodesFree, inodesUsed) 310 | return &csi.NodeGetVolumeStatsResponse{ 311 | Usage: []*csi.VolumeUsage{ 312 | { 313 | Available: available, 314 | Used: used, 315 | Total: capacity, 316 | Unit: csi.VolumeUsage_BYTES, 317 | }, { 318 | Available: inodesFree, 319 | Used: inodesUsed, 320 | Total: inodes, 321 | Unit: csi.VolumeUsage_INODES, 322 | }, 323 | }, 324 | VolumeCondition: &csi.VolumeCondition{ 325 | Abnormal: !healthy, 326 | Message: msg, 327 | }, 328 | }, nil 329 | } 330 | 331 | // NodeExpandVolume is only implemented so the driver can be used for e2e testing. 332 | func (c3 *csi3fs) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { 333 | return &csi.NodeExpandVolumeResponse{}, status.Error(codes.Unimplemented, "Not implemented") 334 | } 335 | 336 | // hasSingleNodeSingleWriterAccessMode checks if the publish request uses the 337 | // SINGLE_NODE_SINGLE_WRITER access mode. 338 | func hasSingleNodeSingleWriterAccessMode(req *csi.NodePublishVolumeRequest) bool { 339 | accessMode := req.GetVolumeCapability().GetAccessMode() 340 | return accessMode != nil && accessMode.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER 341 | } 342 | 343 | // isMountedElsewhere checks if the volume to publish is mounted elsewhere on 344 | // the node. 345 | func isMountedElsewhere(req *csi.NodePublishVolumeRequest, vol state.Volume) bool { 346 | for _, targetPath := range vol.Published { 347 | if targetPath != req.GetTargetPath() { 348 | return true 349 | } 350 | } 351 | return false 352 | } 353 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 2 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 3 | github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= 4 | github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= 5 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 6 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/container-storage-interface/spec v1.11.0 h1:H/YKTOeUZwHtyPOr9raR+HgFmGluGCklulxDYxSdVNM= 8 | github.com/container-storage-interface/spec v1.11.0/go.mod h1:DtUvaQszPml1YJfIK7c00mlv6/g4wNMLanLgiUbKFRI= 9 | github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 10 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 11 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= 13 | github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= 14 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 15 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 16 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 17 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 18 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 19 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 20 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 21 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 22 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 23 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 24 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 25 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 26 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 27 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 28 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 29 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 30 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 31 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 32 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 33 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 34 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 35 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 36 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 37 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 38 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 39 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 40 | github.com/kubernetes-csi/csi-lib-utils v0.20.0 h1:JTvHRJugn+cByMnIU4nCnqPqOOUhuPzhlLqRvenwjDA= 41 | github.com/kubernetes-csi/csi-lib-utils v0.20.0/go.mod h1:3b/HFVURW11oxV/gUAKyhhkvFpxXO/zRdvh1wdEfCZY= 42 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 43 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 44 | github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= 45 | github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= 46 | github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= 47 | github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= 48 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 49 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 50 | github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= 51 | github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= 52 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 53 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 54 | github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= 55 | github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= 56 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 57 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 58 | github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= 59 | github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= 60 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 61 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 62 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 63 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 64 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 65 | github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= 66 | github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= 67 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 68 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 69 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 70 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 71 | github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= 72 | github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= 73 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 74 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 75 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 76 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 77 | go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= 78 | go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= 79 | go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= 80 | go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= 81 | go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= 82 | go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= 83 | go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= 84 | go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= 85 | go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= 86 | go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= 87 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 88 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 89 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 90 | golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= 91 | golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= 92 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 93 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 94 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 95 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 96 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 97 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 98 | golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= 99 | golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= 100 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 101 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 102 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 103 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 104 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 105 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 106 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= 107 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 108 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 109 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 110 | golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= 111 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= 112 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 113 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 114 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 115 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 116 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 117 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 118 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 119 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 120 | google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 h1:Z7FRVJPSMaHQxD0uXU8WdgFh8PseLM8Q8NzhnpMrBhQ= 121 | google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA= 122 | google.golang.org/grpc v1.69.0 h1:quSiOM1GJPmPH5XtU+BCoVXcDVJJAzNcoyfC2cCjGkI= 123 | google.golang.org/grpc v1.69.0/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= 124 | google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ= 125 | google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 126 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 127 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 128 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 129 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 130 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 131 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 132 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 133 | k8s.io/apiextensions-apiserver v0.32.0 h1:S0Xlqt51qzzqjKPxfgX1xh4HBZE+p8KKBq+k2SWNOE0= 134 | k8s.io/apiextensions-apiserver v0.32.0/go.mod h1:86hblMvN5yxMvZrZFX2OhIHAuFIMJIZ19bTvzkP+Fmw= 135 | k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg= 136 | k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= 137 | k8s.io/apiserver v0.32.0 h1:VJ89ZvQZ8p1sLeiWdRJpRD6oLozNZD2+qVSLi+ft5Qs= 138 | k8s.io/apiserver v0.32.0/go.mod h1:HFh+dM1/BE/Hm4bS4nTXHVfN6Z6tFIZPi649n83b4Ag= 139 | k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= 140 | k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= 141 | k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU= 142 | k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM= 143 | k8s.io/controller-manager v0.32.0 h1:tpQl1rvH4huFB6Avl1nhowZHtZoCNWqn6OYdZPl7Ybc= 144 | k8s.io/controller-manager v0.32.0/go.mod h1:JRuYnYCkKj3NgBTy+KNQKIUm/lJRoDAvGbfdEmk9LhY= 145 | k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= 146 | k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= 147 | k8s.io/kubernetes v1.32.0 h1:4BDBWSolqPrv8GC3YfZw0CJvh5kA1TPnoX0FxDVd+qc= 148 | k8s.io/kubernetes v1.32.0/go.mod h1:tiIKO63GcdPRBHW2WiUFm3C0eoLczl3f7qi56Dm1W8I= 149 | k8s.io/mount-utils v0.32.0 h1:KOQAhPzJICATXnc6XCkWoexKbkOexRnMCUW8APFfwg4= 150 | k8s.io/mount-utils v0.32.0/go.mod h1:Kun5c2svjAPx0nnvJKYQWhfeNW+O0EpzHgRhDcYoSY0= 151 | k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= 152 | k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 153 | sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= 154 | sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= 155 | -------------------------------------------------------------------------------- /pkg/driver/controllerserver.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strconv" 7 | 8 | "github.com/container-storage-interface/spec/lib/go/csi" 9 | "github.com/pborman/uuid" 10 | "golang.org/x/net/context" 11 | "google.golang.org/grpc/codes" 12 | "google.golang.org/grpc/status" 13 | "google.golang.org/protobuf/types/known/wrapperspb" 14 | "k8s.io/apimachinery/pkg/util/sets" 15 | "k8s.io/klog/v2" 16 | 17 | "github.com/MooreThreads/csi-driver-3fs/pkg/state" 18 | ) 19 | 20 | const ( 21 | deviceID = "deviceID" 22 | ) 23 | 24 | func (c3 *csi3fs) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (resp *csi.CreateVolumeResponse, finalErr error) { 25 | if err := c3.validateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { 26 | klog.V(3).Infof("invalid create volume req: %v", req) 27 | return nil, err 28 | } 29 | 30 | if len(req.GetMutableParameters()) > 0 { 31 | if err := c3.validateControllerServiceRequest(csi.ControllerServiceCapability_RPC_MODIFY_VOLUME); err != nil { 32 | klog.V(3).Infof("invalid create volume req: %v", req) 33 | return nil, err 34 | } 35 | 36 | if err := c3.validateVolumeMutableParameters(req.MutableParameters); err != nil { 37 | return nil, err 38 | } 39 | } 40 | 41 | // Check arguments 42 | if len(req.GetName()) == 0 { 43 | return nil, status.Error(codes.InvalidArgument, "Name missing in request") 44 | } 45 | caps := req.GetVolumeCapabilities() 46 | if caps == nil { 47 | return nil, status.Error(codes.InvalidArgument, "Volume Capabilities missing in request") 48 | } 49 | 50 | requestedAccessType := state.MountAccess 51 | 52 | c3.mutex.Lock() 53 | defer c3.mutex.Unlock() 54 | 55 | capacity := int64(req.GetCapacityRange().GetRequiredBytes()) 56 | 57 | volumeID := uuid.NewUUID().String() 58 | kind := req.GetParameters()[storageKind] 59 | vol, err := c3.createVolume(volumeID, req.GetName(), capacity, requestedAccessType, false /* ephemeral */, kind) 60 | if err != nil { 61 | return nil, err 62 | } 63 | klog.V(4).Infof("created volume %s at path %s", vol.VolID, vol.VolPath) 64 | 65 | return &csi.CreateVolumeResponse{ 66 | Volume: &csi.Volume{ 67 | VolumeId: volumeID, 68 | CapacityBytes: req.GetCapacityRange().GetRequiredBytes(), 69 | VolumeContext: req.GetParameters(), 70 | ContentSource: req.GetVolumeContentSource(), 71 | }, 72 | }, nil 73 | } 74 | 75 | func (c3 *csi3fs) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { 76 | // Check arguments 77 | if len(req.GetVolumeId()) == 0 { 78 | return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") 79 | } 80 | 81 | if err := c3.validateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { 82 | klog.V(3).Infof("invalid delete volume req: %v", req) 83 | return nil, err 84 | } 85 | 86 | c3.mutex.Lock() 87 | defer c3.mutex.Unlock() 88 | 89 | volId := req.GetVolumeId() 90 | vol, err := c3.state.GetVolumeByID(volId) 91 | if err != nil { 92 | return &csi.DeleteVolumeResponse{}, nil 93 | } 94 | 95 | if vol.Attached || !vol.Published.Empty() || !vol.Staged.Empty() { 96 | msg := fmt.Sprintf("Volume '%s' is still used (attached: %v, staged: %v, published: %v) by '%s' node", 97 | vol.VolID, 98 | vol.Attached, 99 | vol.Staged, 100 | vol.Published, 101 | vol.NodeID, 102 | ) 103 | if c3.config.CheckVolumeLifecycle { 104 | return nil, status.Error(codes.Internal, msg) 105 | } 106 | klog.Warning(msg) 107 | } 108 | 109 | if err := c3.deleteVolume(volId); err != nil { 110 | return nil, fmt.Errorf("failed to delete volume %v: %w", volId, err) 111 | } 112 | 113 | klog.V(4).Infof("volume %v successfully deleted", volId) 114 | return &csi.DeleteVolumeResponse{}, nil 115 | } 116 | 117 | func (c3 *csi3fs) ControllerGetCapabilities( 118 | ctx context.Context, 119 | req *csi.ControllerGetCapabilitiesRequest, 120 | ) (*csi.ControllerGetCapabilitiesResponse, error) { 121 | return &csi.ControllerGetCapabilitiesResponse{Capabilities: c3.getControllerServiceCapabilities()}, nil 122 | } 123 | 124 | func (c3 *csi3fs) ValidateVolumeCapabilities( 125 | ctx context.Context, 126 | req *csi.ValidateVolumeCapabilitiesRequest, 127 | ) (*csi.ValidateVolumeCapabilitiesResponse, error) { 128 | // Check arguments 129 | if len(req.GetVolumeId()) == 0 { 130 | return nil, status.Error(codes.InvalidArgument, "Volume ID cannot be empty") 131 | } 132 | if len(req.VolumeCapabilities) == 0 { 133 | return nil, status.Error(codes.InvalidArgument, req.VolumeId) 134 | } 135 | 136 | c3.mutex.Lock() 137 | defer c3.mutex.Unlock() 138 | 139 | if _, err := c3.state.GetVolumeByID(req.GetVolumeId()); err != nil { 140 | return nil, err 141 | } 142 | 143 | for _, cap := range req.GetVolumeCapabilities() { 144 | if cap.GetMount() == nil { 145 | return nil, status.Error(codes.InvalidArgument, "cannot have mount access type be undefined") 146 | } 147 | } 148 | 149 | return &csi.ValidateVolumeCapabilitiesResponse{ 150 | Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ 151 | VolumeContext: req.GetVolumeContext(), 152 | VolumeCapabilities: req.GetVolumeCapabilities(), 153 | Parameters: req.GetParameters(), 154 | }, 155 | }, nil 156 | } 157 | 158 | func (c3 *csi3fs) ControllerPublishVolume( 159 | ctx context.Context, 160 | req *csi.ControllerPublishVolumeRequest, 161 | ) (*csi.ControllerPublishVolumeResponse, error) { 162 | if !c3.config.EnableAttach { 163 | return nil, status.Error(codes.Unimplemented, "ControllerPublishVolume is not supported") 164 | } 165 | 166 | if len(req.VolumeId) == 0 { 167 | return nil, status.Error(codes.InvalidArgument, "Volume ID cannot be empty") 168 | } 169 | if len(req.NodeId) == 0 { 170 | return nil, status.Error(codes.InvalidArgument, "Node ID cannot be empty") 171 | } 172 | if req.VolumeCapability == nil { 173 | return nil, status.Error(codes.InvalidArgument, "Volume Capabilities cannot be empty") 174 | } 175 | 176 | if req.NodeId != c3.config.NodeID { 177 | return nil, status.Errorf(codes.NotFound, "Not matching Node ID %s to 3fs Node ID %s", req.NodeId, c3.config.NodeID) 178 | } 179 | 180 | c3.mutex.Lock() 181 | defer c3.mutex.Unlock() 182 | 183 | vol, err := c3.state.GetVolumeByID(req.VolumeId) 184 | if err != nil { 185 | return nil, status.Error(codes.NotFound, err.Error()) 186 | } 187 | 188 | // Check to see if the volume is already published. 189 | if vol.Attached { 190 | if req.GetReadonly() != vol.ReadOnlyAttach { 191 | return nil, status.Error(codes.AlreadyExists, "Volume published but has incompatible readonly flag") 192 | } 193 | 194 | return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{}}, nil 195 | } 196 | 197 | // Check attach limit before publishing. 198 | if c3.config.AttachLimit > 0 && c3.getAttachCount() >= c3.config.AttachLimit { 199 | return nil, status.Errorf(codes.ResourceExhausted, "Cannot attach any more volumes to this node ('%s')", c3.config.NodeID) 200 | } 201 | 202 | vol.Attached = true 203 | vol.ReadOnlyAttach = req.GetReadonly() 204 | if err := c3.state.UpdateVolume(vol); err != nil { 205 | return nil, err 206 | } 207 | 208 | return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{}}, nil 209 | } 210 | 211 | func (c3 *csi3fs) ControllerUnpublishVolume( 212 | ctx context.Context, 213 | req *csi.ControllerUnpublishVolumeRequest, 214 | ) (*csi.ControllerUnpublishVolumeResponse, error) { 215 | if !c3.config.EnableAttach { 216 | return nil, status.Error(codes.Unimplemented, "ControllerUnpublishVolume is not supported") 217 | } 218 | 219 | if len(req.VolumeId) == 0 { 220 | return nil, status.Error(codes.InvalidArgument, "Volume ID cannot be empty") 221 | } 222 | 223 | // Empty node id is not a failure as per Spec 224 | if req.NodeId != "" && req.NodeId != c3.config.NodeID { 225 | return nil, status.Errorf(codes.NotFound, "Node ID %s does not match to expected Node ID %s", req.NodeId, c3.config.NodeID) 226 | } 227 | 228 | c3.mutex.Lock() 229 | defer c3.mutex.Unlock() 230 | 231 | vol, err := c3.state.GetVolumeByID(req.VolumeId) 232 | if err != nil { 233 | return &csi.ControllerUnpublishVolumeResponse{}, nil 234 | } 235 | 236 | // Check to see if the volume is staged/published on a node 237 | if !vol.Published.Empty() || !vol.Staged.Empty() { 238 | msg := fmt.Sprintf("Volume '%s' is still used (staged: %v, published: %v) by '%s' node", 239 | vol.VolID, 240 | vol.Staged, 241 | vol.Published, 242 | vol.NodeID, 243 | ) 244 | if c3.config.CheckVolumeLifecycle { 245 | return nil, status.Error(codes.Internal, msg) 246 | } 247 | klog.Warning(msg) 248 | } 249 | 250 | vol.Attached = false 251 | if err := c3.state.UpdateVolume(vol); err != nil { 252 | return nil, status.Errorf(codes.Internal, "could not update volume %s: %v", vol.VolID, err) 253 | } 254 | 255 | return &csi.ControllerUnpublishVolumeResponse{}, nil 256 | } 257 | 258 | func (c3 *csi3fs) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { 259 | c3.mutex.Lock() 260 | defer c3.mutex.Unlock() 261 | 262 | available := c3.config.MaxVolumeSize 263 | if c3.config.Capacity.Enabled() { 264 | kind := req.GetParameters()[storageKind] 265 | quantity := c3.config.Capacity[kind] 266 | allocated := c3.sumVolumeSizes(kind) 267 | available = quantity.Value() - allocated 268 | } 269 | maxVolumeSize := c3.config.MaxVolumeSize 270 | if maxVolumeSize > available { 271 | maxVolumeSize = available 272 | } 273 | 274 | return &csi.GetCapacityResponse{ 275 | AvailableCapacity: available, 276 | MaximumVolumeSize: &wrapperspb.Int64Value{Value: maxVolumeSize}, 277 | MinimumVolumeSize: &wrapperspb.Int64Value{Value: 0}, 278 | }, nil 279 | } 280 | 281 | func (c3 *csi3fs) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { 282 | volumeRes := &csi.ListVolumesResponse{ 283 | Entries: []*csi.ListVolumesResponse_Entry{}, 284 | } 285 | 286 | var ( 287 | startIdx, volumesLength, maxLength int64 288 | c3Volume state.Volume 289 | ) 290 | c3.mutex.Lock() 291 | defer c3.mutex.Unlock() 292 | 293 | // Sort by volume ID. 294 | volumes := c3.state.GetVolumes() 295 | sort.Slice(volumes, func(i, j int) bool { 296 | return volumes[i].VolID < volumes[j].VolID 297 | }) 298 | 299 | if req.StartingToken == "" { 300 | req.StartingToken = "1" 301 | } 302 | 303 | startIdx, err := strconv.ParseInt(req.StartingToken, 10, 32) 304 | if err != nil { 305 | return nil, status.Error(codes.Aborted, "The type of startingToken should be integer") 306 | } 307 | 308 | volumesLength = int64(len(volumes)) 309 | maxLength = int64(req.MaxEntries) 310 | 311 | if maxLength > volumesLength || maxLength <= 0 { 312 | maxLength = volumesLength 313 | } 314 | 315 | for index := startIdx - 1; index < volumesLength && index < maxLength; index++ { 316 | c3Volume = volumes[index] 317 | healthy, msg := c3.doHealthCheckInControllerSide(c3Volume.VolID) 318 | klog.V(3).Infof("Healthy state: %s Volume: %t", c3Volume.VolName, healthy) 319 | volumeRes.Entries = append(volumeRes.Entries, &csi.ListVolumesResponse_Entry{ 320 | Volume: &csi.Volume{ 321 | VolumeId: c3Volume.VolID, 322 | CapacityBytes: c3Volume.VolSize, 323 | }, 324 | Status: &csi.ListVolumesResponse_VolumeStatus{ 325 | PublishedNodeIds: []string{c3Volume.NodeID}, 326 | VolumeCondition: &csi.VolumeCondition{ 327 | Abnormal: !healthy, 328 | Message: msg, 329 | }, 330 | }, 331 | }) 332 | } 333 | 334 | klog.V(5).Infof("Volumes are: %+v", volumeRes) 335 | return volumeRes, nil 336 | } 337 | 338 | func (c3 *csi3fs) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { 339 | c3.mutex.Lock() 340 | defer c3.mutex.Unlock() 341 | 342 | var abnormal bool 343 | var err error 344 | 345 | volume, err := c3.state.GetVolumeByID(req.GetVolumeId()) 346 | if err != nil { 347 | abnormal = true 348 | } 349 | 350 | return &csi.ControllerGetVolumeResponse{ 351 | Volume: &csi.Volume{ 352 | VolumeId: volume.VolID, 353 | CapacityBytes: volume.VolSize, 354 | }, 355 | Status: &csi.ControllerGetVolumeResponse_VolumeStatus{ 356 | PublishedNodeIds: []string{volume.NodeID}, 357 | VolumeCondition: &csi.VolumeCondition{ 358 | Abnormal: abnormal, 359 | Message: err.Error(), 360 | }, 361 | }, 362 | }, nil 363 | } 364 | 365 | func (c3 *csi3fs) ControllerModifyVolume(ctx context.Context, req *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) { 366 | if err := c3.validateControllerServiceRequest(csi.ControllerServiceCapability_RPC_MODIFY_VOLUME); err != nil { 367 | return nil, err 368 | } 369 | 370 | // Check arguments 371 | if len(req.VolumeId) == 0 { 372 | return nil, status.Error(codes.InvalidArgument, "Volume ID cannot be empty") 373 | } 374 | if len(req.MutableParameters) == 0 { 375 | return nil, status.Error(codes.InvalidArgument, "Mutable parameters cannot be empty") 376 | } 377 | 378 | c3.mutex.Lock() 379 | defer c3.mutex.Unlock() 380 | 381 | _, err := c3.state.GetVolumeByID(req.VolumeId) 382 | if err != nil { 383 | return nil, status.Error(codes.NotFound, err.Error()) 384 | } 385 | 386 | return &csi.ControllerModifyVolumeResponse{}, nil 387 | } 388 | 389 | // validateVolumeMutableParameters is a helper function to check if the mutable parameters are in the accepted list 390 | func (c3 *csi3fs) validateVolumeMutableParameters(params map[string]string) error { 391 | if len(c3.config.AcceptedMutableParameterNames) == 0 { 392 | return nil 393 | } 394 | 395 | accepts := sets.New(c3.config.AcceptedMutableParameterNames...) 396 | unsupported := []string{} 397 | for k := range params { 398 | if !accepts.Has(k) { 399 | unsupported = append(unsupported, k) 400 | } 401 | } 402 | if len(unsupported) > 0 { 403 | return status.Errorf(codes.InvalidArgument, "invalid parameters: %v", unsupported) 404 | } 405 | return nil 406 | } 407 | 408 | func (c3 *csi3fs) validateControllerServiceRequest(c csi.ControllerServiceCapability_RPC_Type) error { 409 | if c == csi.ControllerServiceCapability_RPC_UNKNOWN { 410 | return nil 411 | } 412 | 413 | for _, cap := range c3.getControllerServiceCapabilities() { 414 | if c == cap.GetRpc().GetType() { 415 | return nil 416 | } 417 | } 418 | return status.Errorf(codes.InvalidArgument, "unsupported capability %s", c) 419 | } 420 | 421 | func (c3 *csi3fs) getControllerServiceCapabilities() []*csi.ControllerServiceCapability { 422 | cl := []csi.ControllerServiceCapability_RPC_Type{ 423 | csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, 424 | csi.ControllerServiceCapability_RPC_GET_VOLUME, 425 | csi.ControllerServiceCapability_RPC_GET_CAPACITY, 426 | csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, 427 | csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, 428 | csi.ControllerServiceCapability_RPC_LIST_VOLUMES, 429 | csi.ControllerServiceCapability_RPC_CLONE_VOLUME, 430 | csi.ControllerServiceCapability_RPC_VOLUME_CONDITION, 431 | csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, 432 | } 433 | if c3.config.EnableVolumeExpansion && !c3.config.DisableControllerExpansion { 434 | cl = append(cl, csi.ControllerServiceCapability_RPC_EXPAND_VOLUME) 435 | } 436 | if c3.config.EnableAttach { 437 | cl = append(cl, csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME) 438 | } 439 | if c3.config.EnableControllerModifyVolume { 440 | cl = append(cl, csi.ControllerServiceCapability_RPC_MODIFY_VOLUME) 441 | } 442 | 443 | csc := make([]*csi.ControllerServiceCapability, 0, len(cl)) 444 | 445 | for _, cap := range cl { 446 | csc = append(csc, &csi.ControllerServiceCapability{ 447 | Type: &csi.ControllerServiceCapability_Rpc{ 448 | Rpc: &csi.ControllerServiceCapability_RPC{ 449 | Type: cap, 450 | }, 451 | }, 452 | }) 453 | } 454 | 455 | return csc 456 | } 457 | --------------------------------------------------------------------------------