├── .github └── workflows │ └── build-docker-images.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── cmd └── moosefs-csi-plugin │ ├── Dockerfile │ └── main.go ├── deploy ├── csi-moosefs-config.yaml └── csi-moosefs.yaml ├── driver ├── controller.go ├── identity.go ├── mfs_handler.go ├── mounter.go ├── node.go └── service.go ├── examples ├── dynamic-provisioning │ ├── pod.yaml │ └── pvc.yaml ├── mount-volume │ ├── pod.yaml │ ├── pv.yaml │ └── pvc.yaml └── static-provisioning │ ├── pod.yaml │ ├── pv.yaml │ └── pvc.yaml ├── go.mod └── go.sum /.github/workflows/build-docker-images.yml: -------------------------------------------------------------------------------- 1 | name: Build and publish moosefs-csi driver Docker image 2 | 3 | on: 4 | push: 5 | branches: 6 | - dev 7 | 8 | env: 9 | REGISTRY_IMAGE: moosefs/moosefs-csi 10 | MFS_CLIENT: 4.57.6 11 | CSI_VERSION: 0.9.8 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | platform: 20 | - linux/amd64 21 | - linux/arm64 22 | - linux/arm/v7 23 | steps: 24 | - name: Prepare 25 | run: | 26 | platform=${{ matrix.platform }} 27 | echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV 28 | 29 | - name: Docker meta 30 | id: meta 31 | uses: docker/metadata-action@v5 32 | with: 33 | images: ${{ env.REGISTRY_IMAGE }} 34 | tags: | 35 | type=ref,event=branch 36 | type=raw,value=${{ env.CSI_VERSION }}-${{ env.MFS_CLIENT }} 37 | type=raw,value=${{ env.CSI_VERSION }}-{{branch}} 38 | 39 | - name: Set up QEMU 40 | uses: docker/setup-qemu-action@v3 41 | 42 | - name: Set up Docker Buildx 43 | uses: docker/setup-buildx-action@v3 44 | 45 | - name: Login to GitHub Container Registry 46 | uses: docker/login-action@v3 47 | with: 48 | registry: ghcr.io 49 | username: ${{ github.actor }} 50 | password: ${{ secrets.GHCR_TOKEN }} 51 | 52 | - name: Build and push by digest MooseFS Client v4 53 | id: build 54 | uses: docker/build-push-action@v6 55 | with: 56 | file: ./cmd/moosefs-csi-plugin/Dockerfile 57 | platforms: ${{ matrix.platform }} 58 | labels: ${{ steps.meta.outputs.labels }} 59 | outputs: type=image,name=ghcr.io/${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true 60 | build-args: | 61 | MFS_TAG=v${{ env.MFS_CLIENT }} 62 | CSI_TAG=dev 63 | 64 | - name: Export digest 65 | run: | 66 | mkdir -p /tmp/digests 67 | digest="${{ steps.build.outputs.digest }}" 68 | touch "/tmp/digests/${digest#sha256:}" 69 | 70 | - name: Upload digest 71 | uses: actions/upload-artifact@v4 72 | with: 73 | name: digests-${{ env.PLATFORM_PAIR }} 74 | path: /tmp/digests/* 75 | if-no-files-found: error 76 | retention-days: 1 77 | 78 | merge: 79 | runs-on: ubuntu-latest 80 | needs: 81 | - build 82 | steps: 83 | - name: Download digests 84 | uses: actions/download-artifact@v4 85 | with: 86 | path: /tmp/digests 87 | pattern: digests-* 88 | merge-multiple: true 89 | 90 | - name: Set up Docker Buildx 91 | uses: docker/setup-buildx-action@v3 92 | 93 | - name: Docker meta 94 | id: meta 95 | uses: docker/metadata-action@v5 96 | with: 97 | images: ghcr.io/${{ env.REGISTRY_IMAGE }} 98 | tags: | 99 | type=ref,event=branch 100 | type=raw,value=${{ env.CSI_VERSION }}-${{ env.MFS_CLIENT }} 101 | type=raw,value=${{ env.CSI_VERSION }}-{{branch}} 102 | 103 | - name: Login to GitHub Container Registry 104 | uses: docker/login-action@v3 105 | with: 106 | registry: ghcr.io 107 | username: ${{ github.actor }} 108 | password: ${{ secrets.GHCR_TOKEN }} 109 | 110 | - name: Create manifest list and push 111 | working-directory: /tmp/digests 112 | run: | 113 | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ 114 | $(printf 'ghcr.io/${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) 115 | 116 | - name: Inspect image 117 | run: | 118 | docker buildx imagetools inspect ghcr.io/${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} 119 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Gopkg.lock 2 | Gopkg.toml 3 | /cmd/moosefs-csi-plugin/moosefs-csi-plugin 4 | /vendor 5 | /.idea 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright (c) 2025 Saglabs SA. All Rights Reserved. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2025 Saglabs SA. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | MFS_VERSION = "4.57.6" 16 | CSI_VERSION ?= "0.9.8" 17 | 18 | MFS_TAG=$(CSI_VERSION)-$(MFS_VERSION) 19 | DEV_TAG=$(CSI_VERSION)-dev 20 | 21 | NAME=moosefs-csi 22 | 23 | csi: clean compile 24 | dev: build-dev push-dev 25 | prod: build-prod push-prod 26 | 27 | compile: 28 | @echo "==> Building the CSI driver" 29 | @env CGO_ENABLED=0 GOCACHE=/tmp/go-cache GOOS=linux GOARCH=amd64 go build -a -o cmd/moosefs-csi-plugin/${NAME} cmd/moosefs-csi-plugin/main.go 30 | 31 | build-dev: 32 | @echo "==> Building DEV CSI images" 33 | @docker build --no-cache -t moosefs/$(NAME):dev -t moosefs/$(NAME):$(DEV_TAG) --build-arg MFS_TAG=v$(MFS_VERSION) --build-arg CSI_TAG=dev cmd/moosefs-csi-plugin 34 | 35 | push-dev: 36 | @echo "==> Publishing DEV CSI image on hub.docker.com: moosefs/$(NAME):$(DEV_TAG)" 37 | @docker push moosefs/$(NAME):$(DEV_TAG) 38 | @docker push moosefs/$(NAME):dev 39 | 40 | build-prod: 41 | @echo "==> Building Production CSI images" 42 | @docker build --no-cache -t moosefs/$(NAME):$(MFS_TAG) --build-arg MFS_TAG=v$(MFS_VERSION) --build-arg CSI_TAG=$(CSI_VERSION) cmd/moosefs-csi-plugin 43 | 44 | push-prod: 45 | @echo "==> Publishing PRODUCTION CSI image on hub.docker.com: moosefs/$(NAME):$(MFS_TAG)" 46 | @docker push moosefs/$(NAME):$(MFS_TAGCE) 47 | 48 | dev-buildx: 49 | @echo "==> Using buildx to build and publish dev image" 50 | @docker buildx build --no-cache --push --platform linux/amd64,linux/arm64,linux/arm/v7 --build-arg MFS_TAG=v$(MFS_VERSION) --build-arg CSI_TAG=dev -t moosefs/$(NAME):dev -t moosefs/$(NAME):$(DEV_TAG) cmd/moosefs-csi-plugin 51 | 52 | prod-buildx: 53 | @echo "==> Using buildx to build and publish production image" 54 | @docker buildx build --push --platform linux/amd64,linux/arm64,linux/arm/v7 --build-arg MFS_TAG=v$(MFS_VERSION) --build-arg CSI_TAG=dev -t moosefs/$(NAME):$(MFS_TAG) cmd/moosefs-csi-plugin 55 | 56 | clean: 57 | @echo "==> Cleaning releases" 58 | @GOOS=linux go clean -i -x ./... 59 | 60 | .PHONY: clean 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Container Storage Interface (CSI) driver for MooseFS 2 | 3 | [Container storage interface](https://github.com/container-storage-interface/spec) is an [industry standard](https://github.com/container-storage-interface/spec/blob/master/spec.md) that enables storage vendors to develop a plugin once and have it work across a number of container orchestration systems. 4 | 5 | [MooseFS](https://moosefs.com) is a petabyte Open-Source distributed file system. It aims to be fault-tolerant, highly available, highly performing, scalable general-purpose network distributed file system for data centers. 6 | 7 | MooseFS source code can be found [on GitHub](https://github.com/moosefs/moosefs). 8 | 9 | --- 10 | 11 | *Note that a pool of MooseFS Clients that are available for use by containers is created on each node. By default the number of MooseFS Clients in the pool is `1`.* 12 | 13 | ## Changelog 14 | 15 | Driver verson 0.9.8 16 | * MooseFS client updated to version 4.57.6. 17 | * The provisioner and registrar images entries have been updated. 18 | * Update the Dockerfile to build only mfsmount and mfscli. 19 | 20 | Driver verson 0.9.7 21 | * Added support for MooseFS 4 client. 22 | * Enabled passing additional mfsmount parameters during the mount process (password and more). 23 | * Support for cross-platform compilation has been enabled. 24 | * Repository images support AMD64, ARM64 and ARMv7 architectures by default. 25 | 26 | ## Installation on Kubernetes 27 | 28 | ### Prerequisites 29 | 30 | * MooseFS Cluster up and running 31 | 32 | * `--allow-privileged=true` flag set for both API server and kubelet (default value for kubelet is `true`) 33 | 34 | ### **Deployment** 35 | 36 | 1. Edit `deploy/kubernetes/csi-moosefs-config.yaml` config map file with your settings: 37 | 38 | * `master_host` – domain name (**recommended**) or IP address of your MooseFS Master Server(s). It is an equivalent to `-H master_host` or `-o mfsmaster=master_host` passed to MooseFS Client. 39 | * `master_port` – port number of your MooseFS Master Server. It is an equivalent to `-P master_port` or `-o mfsport=master_port` passed to MooseFS Client. 40 | * `k8s_root_dir` – each mount's root directory on MooseFS. Each path is relative to this one. Equivalent to `-S k8s_root_dir` or `-o mfssubfolder=k8s_root_dir` passed to MooseFS Client. 41 | * `driver_working_dir` – a driver working directory inside MooseFS where persistent volumes, logs and metadata is stored (actual path is: `k8s_root_dir/driver_working_dir`) 42 | * `mount_count` – number of pre created MooseFS clients running on each node 43 | * `mfs_logging` – driver can create logs from each component in `k8s_root_dir/driver_working_dir/logs` directory. Boolean `"true"`/`"false"` value. 44 | 45 | 2. Apply csi-moosefs-config config map to the cluster: 46 | 47 | ``` 48 | $ kubectl apply -f deploy/kubernetes/csi-moosefs-config.yaml 49 | ``` 50 | 51 | Check the config map status: 52 | 53 | ``` 54 | $ kubectl get configmap -n kube-system 55 | NAME DATA AGE 56 | csi-moosefs-config 6 42s 57 | ``` 58 | 59 | 3. Update `deploy/csi-moosefs.yaml` file with the aproprieate image: 60 | 61 | The default image consists of the latest version of the CSI plug-in and the latest version of the MooseFS Community Edition client: 62 | 63 | * Locate image definition under the `csi-moosefs-plugin` plugin name(line 230 and line 329) 64 | `mage: ghcr.io/moosefs/moosefs-csi:dev` 65 | * Update the `image` version suffix in the plugin's section accordingly: 66 | * `0.9.8-4.57.6` – plugin version 0.9.7 and MooseFS CE 4.57.6 67 | * `0.9.7-4.57.5` – plugin version 0.9.7 and MooseFS CE 4.57.5 68 | * `0.9.7-4.56.6` – plugin version 0.9.7 and MooseFS CE 4.56.6 69 | 70 | You can find a complete list of available images at: \ 71 | https://github.com/moosefs/moosefs-csi/pkgs/container/moosefs-csi 72 | 73 | Fot driver with MoosreFS client PRO version: https://registry.moosefs.com/v2/moosefs-csi-plugin/tags/list. 74 | * `0.9.7-4.56.6-pro` – plugin version 0.9.7 and MooseFS PRO 4.56.6 75 | 76 | 77 | **Note there are two occurrences of `csi-moosefs-plugin` in `csi-moosefs.yaml` file and it is necessary to update the image version in both places of the file.** 78 | 79 | 4. Deploy CSI MooseFS plugin along with CSI Sidecar Containers: 80 | 81 | ``` 82 | $ kubectl apply -f deploy/kubernetes/csi-moosefs.yaml 83 | ``` 84 | 85 | 5. Ensure that all the containers are ready, up and running 86 | 87 | ``` 88 | kube@k-master:~$ kubectl get pods -n kube-system | grep csi-moosefs 89 | csi-moosefs-controller-0 4/4 Running 0 44m 90 | csi-moosefs-node-7h4pj 2/2 Running 0 44m 91 | csi-moosefs-node-8n5hj 2/2 Running 0 44m 92 | csi-moosefs-node-n4prg 2/2 Running 0 44m 93 | ``` 94 | 95 | You should see a single `csi-moosefs-controller-x` running and `csi-moosefs-node-xxxxx` one per each node. 96 | 97 | You may also take a look at your MooseFS CGI Monitoring Interface ("Mounts" tab) to check if new Clients are connected – mount points: `/mnt/controller` and `/mnt/${nodeId}[_${mountId}]`. 98 | 99 | ### **Verification** 100 | 101 | 1. Create a persistent volume claim for 5 GiB: 102 | 103 | ``` 104 | $ kubectl apply -f examples/kubernetes/dynamic-provisioning/pvc.yaml 105 | ``` 106 | 107 | 2. Verify if the persistent volume claim exists and wait until it's STATUS is `Bound`: 108 | 109 | ``` 110 | $ kubectl get pvc 111 | NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE 112 | my-moosefs-pvc Bound pvc-a62451d4-0d75-4f81-bfb3-8402c59bfc25 5Gi RWX moosefs-storage 69m 113 | ``` 114 | 115 | 3. After its in `Bound` state, create a sample workload that mounts the volume: 116 | 117 | ``` 118 | $ kubectl apply -f examples/kubernetes/dynamic-provisioning/pod.yaml 119 | ``` 120 | 121 | 4. Verify the storage is mounted: 122 | 123 | ``` 124 | $ kubectl exec my-moosefs-pod -- df -h /data 125 | Filesystem Size Used Available Use% Mounted on 126 | 172.17.2.80:9421 4.2T 1.4T 2.8T 33% /data 127 | ``` 128 | 129 | You may take a look at MooseFS GUI Monitoring Interface ("Quotas" tab) to check if a quota for 5 GiB on a newly created volume directory has been set. 130 | Dynamically provisioned volumes are stored on MooseFS in `k8s_root_dir/driver_working_dir/volumes` directory. 131 | 132 | 5. Clean up: 133 | 134 | ``` 135 | $ kubectl delete -f examples/kubernetes/dynamic-provisioning/pod.yaml 136 | $ kubectl delete -f examples/kubernetes/dynamic-provisioning/pvc.yaml 137 | ``` 138 | 139 | ## More examples and capabilities 140 | 141 | ### Volume Expansion 142 | 143 | Volume expansion can be done by updating and applying corresponding PVC specification. 144 | 145 | **Note:** the volume size can only be increased. Any attempts to decrease it will result in an error. It is not recommended to resize Persistent Volume MooseFS-allocated quotas via MooseFS native tools, as such changes will not be visible in your Container Orchestrator. 146 | 147 | ### Static provisioning 148 | 149 | Volumes can be also provisioned statically by creating or using a existing directory in `k8s_root_dir/driver_working_dir/volumes`. Example PersistentVolume `examples/kubernetes/static-provisioning/pv.yaml` definition, requires existing volume in volumes directory. 150 | 151 | ### Mount MooseFS inside containers 152 | 153 | It is possible to mount any MooseFS directory inside containers using static provisioning. 154 | 155 | 1. Create a Persistent Volume (`examples/kubernetes/mount-volume/pv.yaml`): 156 | 157 | ``` 158 | kind: PersistentVolume 159 | apiVersion: v1 160 | metadata: 161 | name: my-moosefs-pv-mount 162 | spec: 163 | storageClassName: "" # empty Storage Class 164 | capacity: 165 | storage: 1Gi # required, however does not have any effect 166 | accessModes: 167 | - ReadWriteMany 168 | csi: 169 | driver: csi.moosefs.com 170 | volumeHandle: my-mount-volume # unique volume name 171 | volumeAttributes: 172 | mfsSubDir: "/" # subdirectory to be mounted as a rootdir (inside k8s_root_dir) 173 | ``` 174 | 175 | 2. Create corresponding Persistent Volume Claim (`examples/kubernetes/mount-volume/pvc.yaml`): 176 | 177 | ``` 178 | kind: PersistentVolumeClaim 179 | apiVersion: v1 180 | metadata: 181 | name: my-moosefs-pvc-mount 182 | spec: 183 | storageClassName: "" # empty Storage Class 184 | volumeName: my-moosefs-pv-mount 185 | accessModes: 186 | - ReadWriteMany 187 | resources: 188 | requests: 189 | storage: 1Gi # at least as much as in PV, does not have any effect 190 | ``` 191 | 192 | 3. Apply both configurations: 193 | 194 | ``` 195 | $ kubectl apply -f examples/kubernetes/mount-volume/pv.yaml 196 | $ kubectl apply -f examples/kubernetes/mount-volume/pvc.yaml 197 | ``` 198 | 199 | 4. Verify that PVC exists and wait until it is bound to the previously created PV: 200 | 201 | ``` 202 | $ kubectl get pvc 203 | NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE 204 | my-moosefs-pvc-mount Bound my-moosefs-pv-mount 1Gi RWX 23m 205 | ``` 206 | 207 | 5. Create a sample workload that mounts the volume: 208 | 209 | ``` 210 | $ kubectl apply -f examples/kubernetes/mount-volume/pod.yaml 211 | ``` 212 | 213 | 6. Verify that the storage is mounted: 214 | 215 | ``` 216 | $ kubectl exec -it my-moosefs-pod-mount -- ls /data 217 | ``` 218 | 219 | You should see the content of `k8s_root_dir/mfsSubDir`. 220 | 221 | 7. Clean up: 222 | 223 | ``` 224 | $ kubectl delete -f examples/kubernetes/mount-volume/pod.yaml 225 | $ kubectl delete -f examples/kubernetes/mount-volume/pvc.yaml 226 | $ kubectl delete -f examples/kubernetes/mount-volume/pv.yaml 227 | ``` 228 | 229 | By using `containers[*].volumeMounts[*].subPath` field of `PodSpec` it is possible to specify a proper MooseFS subdirectory using only one PV/PVC pair, without creating a new one for each subdirectory: 230 | 231 | ``` 232 | kind: Deployment 233 | apiVersion: apps/v1 234 | metadata: 235 | name: my-site-app 236 | spec: 237 | template: 238 | spec: 239 | containers: 240 | - name: my-frontend 241 | # ... 242 | volumeMounts: 243 | - name: my-moosefs-mount 244 | mountPath: "/var/www/my-site/assets/images" 245 | subPath: "resources/my-site/images" 246 | - name: my-moosefs-mount 247 | mountPath: "/var/www/my-site/assets/css" 248 | subPath: "resources/my-site/css" 249 | volumes: 250 | - name: my-moosefs-mount 251 | persistentVolumeClaim: 252 | claimName: my-moosefs-pvc-mount 253 | ``` 254 | 255 | ## Version Compatibility 256 | 257 | | Kubernetes | MooseFS CSI Driver | 258 | |:----------:|:------------------:| 259 | | `v1.26` | `v0.9.7` | 260 | | `-----` | `------` | 261 | | `v1.32` | `v0.9.8` | 262 | 263 | ## Copyright 264 | 265 | Copyright (c) 2020-2025 Saglabs SA 266 | 267 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](https://www.apache.org/licenses/LICENSE-2.0). 268 | 269 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 270 | 271 | See the License for the specific language governing permissions and limitations under the License. 272 | 273 | ## License 274 | 275 | [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0) 276 | 277 | ## Code of conduct 278 | 279 | Participation in this project is governed by [Kubernetes/CNCF code of conduct](https://github.com/kubernetes/community/blob/master/code-of-conduct.md) 280 | -------------------------------------------------------------------------------- /cmd/moosefs-csi-plugin/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2025 Saglabs SA. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ARG CSI_TAG="v0.9.8" 16 | ARG MFS_TAG="v4.57.6" 17 | 18 | #Build MooseFS CSI driver from source 19 | FROM golang:1.23-bookworm AS csibuilder 20 | WORKDIR /build 21 | ARG CSI_TAG 22 | RUN git clone --depth 1 --branch ${CSI_TAG} https://github.com/moosefs/moosefs-csi.git 23 | RUN cd moosefs-csi && CGO_ENABLED=0 GOCACHE=/tmp/go-cache GOOS=linux go build -a -o /build/moosefs-csi-plugin cmd/moosefs-csi-plugin/main.go 24 | 25 | #Build MooseFS Client from source Debian 12 Bookworm 26 | # MooseFS client is required for the CSI driver to mount volumes 27 | FROM ghcr.io/moosefs/mfsbuilder:latest AS mfsbuilder 28 | WORKDIR /moosefs 29 | ARG MFS_TAG 30 | RUN git clone --depth 1 --branch ${MFS_TAG} https://github.com/moosefs/moosefs.git /moosefs 31 | RUN autoreconf -f -i 32 | RUN ./configure --prefix=/usr --mandir=/share/man --sysconfdir=/etc --localstatedir=/var/lib --with-default-user=mfs --with-default-group=mfs --disable-mfsbdev --disable-mfsmaster --disable-mfschunkserver --disable-mfsmetalogger --disable-mfsnetdump --disable-mfscgi --disable-mfscgiserv --disable-mfscli 33 | RUN cd /moosefs/mfsclient && make DESTDIR=/tmp/ install 34 | 35 | #Build CSI plugin container 36 | FROM debian:bookworm 37 | RUN apt update && apt install -y libfuse3-3 38 | COPY --from=csibuilder /build/moosefs-csi-plugin /bin/moosefs-csi-plugin 39 | COPY --from=mfsbuilder /tmp/usr/bin /usr/bin 40 | RUN ["ln", "-s", "/usr/bin/mfsmount", "/usr/sbin/mount.moosefs"] 41 | ENTRYPOINT ["/bin/moosefs-csi-plugin"] 42 | -------------------------------------------------------------------------------- /cmd/moosefs-csi-plugin/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2025 Saglabs SA. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | 22 | "github.com/moosefs/moosefs-csi/driver" 23 | log "github.com/sirupsen/logrus" 24 | ) 25 | 26 | func main() { 27 | var ( 28 | mode = flag.String("mode", "value", "") 29 | csiEndpoint = flag.String("csi-endpoint", "unix:///var/lib/csi/sockets/pluginproxy/csi.sock", "CSI endpoint") 30 | mfsmaster = flag.String("master-host", "mfsmaster", "MooseFS master hostname or IP(Community Edition only)") 31 | mfsmaster_port = flag.Int("master-port", 9421, "MooseFS master communication port. Default 9421") 32 | nodeId = flag.String("node-id", "", "") 33 | rootDir = flag.String("root-dir", "/", "") 34 | pluginDataDir = flag.String("plugin-data-dir", "/", "") 35 | mountPointsCount = flag.Int("mount-points-count", 1, "") 36 | mfsMountOptions = flag.String("mfs-mount-options", "", "extra options passed to mfsmount command. For example mfsmd5pass=MD5") 37 | sanityTestRun = flag.Bool("sanity-test-run", false, "") 38 | logLevel = flag.Int("log-level", 5, "") 39 | mfsLog = flag.Bool("mfs-logging", true, "") 40 | ) 41 | flag.Parse() 42 | 43 | driver.Init(*sanityTestRun, *logLevel, *mfsLog) 44 | 45 | if *sanityTestRun { 46 | log.Infof("=============== SANITY TEST ===============") 47 | } 48 | // this won't be logged to mfs log file 49 | log.Infof("Starting new service (mode: %s; master-host: %s; master-port: %d; mfs-mount-options %s; node-id: %s; root-dir: %s; plugin-data-dir: %s)", 50 | *mode, *mfsmaster, *mfsmaster_port, *mfsMountOptions, *nodeId, *rootDir, *pluginDataDir) 51 | 52 | var srv driver.Service 53 | var err error 54 | switch *mode { 55 | case "node": 56 | srv, err = driver.NewNodeService(*mfsmaster, *mfsmaster_port, *rootDir, *pluginDataDir, *nodeId, *mfsMountOptions, *mountPointsCount) 57 | if err != nil { 58 | log.Errorf("main - couldn't create node service. Error: %s", err.Error()) 59 | return 60 | } 61 | case "controller": 62 | srv, err = driver.NewControllerService(*mfsmaster, *mfsmaster_port, *rootDir, *pluginDataDir, *mfsMountOptions) 63 | if err != nil { 64 | log.Errorf("main - couldn't create controller service. Error: %s", err.Error()) 65 | return 66 | } 67 | default: 68 | log.Errorf("main - unrecognized mode = %s", *mode) 69 | return 70 | } 71 | 72 | if err = driver.StartService(&srv, *mode, *csiEndpoint); err != nil { 73 | log.Errorf("main - couldn't start service %s", err.Error()) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /deploy/csi-moosefs-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | namespace: kube-system 5 | name: csi-moosefs-config 6 | data: 7 | # MooseFS master hostname or IP 8 | master_host: "mfsmaster" 9 | # MooseFS master port 10 | master_port: "9421" 11 | # MooseFS root directory for all claims within this Kubernetes cluster 12 | k8s_root_dir: "/" 13 | # MooseFS directory (relative to csi_root_dir) for all driver data 14 | # (effective working dir will be calculated as k8s_root_dir/driver_working_dir) 15 | driver_working_dir: "pv_data" 16 | # Number of pre-created MooseFS mounts on each node 17 | mount_count: "1" 18 | # Should driver log to k8s_root_dir/driver_working_dir/logs directory 19 | mfs_logging: "true" 20 | # Additional mfsmount options 21 | mfs_mount_options: "mfsmd5pass=[MD5 hash]" -------------------------------------------------------------------------------- /deploy/csi-moosefs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: storage.k8s.io/v1 3 | kind: CSIDriver 4 | metadata: 5 | name: csi.moosefs.com 6 | spec: 7 | attachRequired: true 8 | podInfoOnMount: false 9 | 10 | --- 11 | kind: StorageClass 12 | apiVersion: storage.k8s.io/v1 13 | metadata: 14 | name: moosefs-storage 15 | namespace: kube-system 16 | annotations: 17 | storageclass.kubernetes.io/is-default-class: "true" 18 | provisioner: csi.moosefs.com 19 | allowVolumeExpansion: true 20 | 21 | --- 22 | apiVersion: v1 23 | kind: ServiceAccount 24 | metadata: 25 | name: csi-moosefs-controller-sa 26 | namespace: kube-system 27 | 28 | --- 29 | apiVersion: v1 30 | kind: ServiceAccount 31 | metadata: 32 | name: csi-moosefs-node-sa 33 | namespace: kube-system 34 | 35 | --- 36 | kind: ClusterRole 37 | apiVersion: rbac.authorization.k8s.io/v1 38 | metadata: 39 | name: csi-moosefs-provisioner-role 40 | rules: 41 | - apiGroups: [""] 42 | resources: ["secrets"] 43 | verbs: ["get", "list"] 44 | - apiGroups: [""] 45 | resources: ["persistentvolumes"] 46 | verbs: ["get", "list", "watch", "create", "delete"] 47 | - apiGroups: [""] 48 | resources: ["persistentvolumeclaims"] 49 | verbs: ["get", "list", "watch", "update"] 50 | - apiGroups: ["storage.k8s.io"] 51 | resources: ["storageclasses"] 52 | verbs: ["get", "list", "watch"] 53 | - apiGroups: [""] 54 | resources: ["events"] 55 | verbs: ["get", "list", "watch", "create", "update", "patch"] 56 | 57 | --- 58 | kind: ClusterRoleBinding 59 | apiVersion: rbac.authorization.k8s.io/v1 60 | metadata: 61 | name: csi-moosefs-provisioner-binding 62 | subjects: 63 | - kind: ServiceAccount 64 | name: csi-moosefs-controller-sa 65 | namespace: kube-system 66 | roleRef: 67 | kind: ClusterRole 68 | name: csi-moosefs-provisioner-role 69 | apiGroup: rbac.authorization.k8s.io 70 | 71 | --- 72 | kind: ClusterRole 73 | apiVersion: rbac.authorization.k8s.io/v1 74 | metadata: 75 | name: csi-moosefs-attacher-role 76 | rules: 77 | - apiGroups: [""] 78 | resources: ["persistentvolumes"] 79 | verbs: ["get", "list", "watch", "patch"] 80 | - apiGroups: ["storage.k8s.io"] 81 | resources: ["csinodes"] 82 | verbs: ["get", "list", "watch"] 83 | - apiGroups: ["storage.k8s.io"] 84 | resources: ["volumeattachments"] 85 | verbs: ["get", "list", "watch", "patch"] 86 | - apiGroups: ["storage.k8s.io"] 87 | resources: ["volumeattachments/status"] 88 | verbs: ["patch"] 89 | 90 | --- 91 | kind: ClusterRoleBinding 92 | apiVersion: rbac.authorization.k8s.io/v1 93 | metadata: 94 | name: csi-moosefs-attacher-binding 95 | subjects: 96 | - kind: ServiceAccount 97 | name: csi-moosefs-controller-sa 98 | namespace: kube-system 99 | roleRef: 100 | kind: ClusterRole 101 | name: csi-moosefs-attacher-role 102 | apiGroup: rbac.authorization.k8s.io 103 | 104 | --- 105 | kind: ClusterRole 106 | apiVersion: rbac.authorization.k8s.io/v1 107 | metadata: 108 | name: csi-moosefs-resizer-role 109 | rules: 110 | - apiGroups: [""] 111 | resources: ["persistentvolumes"] 112 | verbs: ["get", "list", "watch", "update", "patch"] 113 | - apiGroups: [""] 114 | resources: ["persistentvolumeclaims"] 115 | verbs: ["get", "list", "watch"] 116 | - apiGroups: [""] 117 | resources: ["persistentvolumeclaims/status"] 118 | verbs: ["update", "patch"] 119 | - apiGroups: [""] 120 | resources: ["events"] 121 | verbs: ["get", "list", "watch", "create", "update", "patch"] 122 | 123 | --- 124 | kind: ClusterRoleBinding 125 | apiVersion: rbac.authorization.k8s.io/v1 126 | metadata: 127 | name: csi-moosefs-resizer-binding 128 | subjects: 129 | - kind: ServiceAccount 130 | name: csi-moosefs-controller-sa 131 | namespace: kube-system 132 | roleRef: 133 | kind: ClusterRole 134 | name: csi-moosefs-resizer-role 135 | apiGroup: rbac.authorization.k8s.io 136 | 137 | --- 138 | kind: ClusterRole 139 | apiVersion: rbac.authorization.k8s.io/v1 140 | metadata: 141 | name: csi-moosefs-driver-registrar-node-role 142 | rules: 143 | - apiGroups: [""] 144 | resources: ["events"] 145 | verbs: ["get", "list", "watch", "create", "update", "patch"] 146 | 147 | --- 148 | kind: ClusterRoleBinding 149 | apiVersion: rbac.authorization.k8s.io/v1 150 | metadata: 151 | name: csi-moosefs-driver-registrar-node-binding 152 | subjects: 153 | - kind: ServiceAccount 154 | name: csi-moosefs-node-sa 155 | namespace: kube-system 156 | roleRef: 157 | kind: ClusterRole 158 | name: csi-moosefs-driver-registrar-node-role 159 | apiGroup: rbac.authorization.k8s.io 160 | 161 | --- 162 | kind: StatefulSet 163 | apiVersion: apps/v1 164 | metadata: 165 | name: csi-moosefs-controller 166 | namespace: kube-system 167 | spec: 168 | serviceName: "csi-moosefs" 169 | replicas: 1 170 | selector: 171 | matchLabels: 172 | app: csi-moosefs-controller 173 | role: csi-moosefs 174 | template: 175 | metadata: 176 | labels: 177 | app: csi-moosefs-controller 178 | role: csi-moosefs 179 | spec: 180 | priorityClassName: system-cluster-critical 181 | serviceAccount: csi-moosefs-controller-sa 182 | hostNetwork: true 183 | containers: 184 | # provisioner 185 | - name: csi-provisioner 186 | image: quay.io/k8scsi/csi-provisioner:v2.1.0 187 | args: 188 | - "--csi-address=$(ADDRESS)" 189 | - "--v=5" 190 | env: 191 | - name: ADDRESS 192 | value: /var/lib/csi/sockets/pluginproxy/csi.sock 193 | imagePullPolicy: "Always" 194 | volumeMounts: 195 | - name: socket-dir 196 | mountPath: /var/lib/csi/sockets/pluginproxy/ 197 | # attacher 198 | - name: csi-attacher 199 | image: quay.io/k8scsi/csi-attacher:v3.1.0 200 | args: 201 | - "--v=5" 202 | - "--csi-address=$(ADDRESS)" 203 | env: 204 | - name: ADDRESS 205 | value: /var/lib/csi/sockets/pluginproxy/csi.sock 206 | imagePullPolicy: "Always" 207 | volumeMounts: 208 | - name: socket-dir 209 | mountPath: /var/lib/csi/sockets/pluginproxy/ 210 | # resizer 211 | - name: csi-resizer 212 | image: quay.io/k8scsi/csi-resizer:v0.5.0 213 | args: 214 | - "--v=5" 215 | - "--csi-address=$(ADDRESS)" 216 | env: 217 | - name: ADDRESS 218 | value: /var/lib/csi/sockets/pluginproxy/csi.sock 219 | imagePullPolicy: "Always" 220 | volumeMounts: 221 | - name: socket-dir 222 | mountPath: /var/lib/csi/sockets/pluginproxy/ 223 | # MooseFS CSI Plugin 224 | - name: csi-moosefs-plugin 225 | securityContext: 226 | privileged: true 227 | capabilities: 228 | add: ["SYS_ADMIN"] 229 | allowPrivilegeEscalation: true 230 | image: ghcr.io/moosefs/moosefs-csi:0.9.8-4.57.6 231 | args: 232 | - "--mode=controller" 233 | - "--csi-endpoint=$(CSI_ENDPOINT)" 234 | - "--master-host=$(MASTER_HOST)" 235 | - "--master-port=$(MASTER_PORT)" 236 | - "--root-dir=$(ROOT_DIR)" 237 | - "--plugin-data-dir=$(WORKING_DIR)" 238 | - "--mfs-logging=$(MFS_LOGGING)" 239 | - "--mfs-mount-options=$(MFS_MOUNT_OPTIONS)" 240 | env: 241 | - name: CSI_ENDPOINT 242 | value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock 243 | - name: MASTER_HOST 244 | valueFrom: 245 | configMapKeyRef: 246 | name: csi-moosefs-config 247 | key: master_host 248 | - name: MASTER_PORT 249 | valueFrom: 250 | configMapKeyRef: 251 | name: csi-moosefs-config 252 | key: master_port 253 | - name: ROOT_DIR 254 | valueFrom: 255 | configMapKeyRef: 256 | name: csi-moosefs-config 257 | key: k8s_root_dir 258 | - name: WORKING_DIR 259 | valueFrom: 260 | configMapKeyRef: 261 | name: csi-moosefs-config 262 | key: driver_working_dir 263 | - name: MFS_LOGGING 264 | valueFrom: 265 | configMapKeyRef: 266 | name: csi-moosefs-config 267 | key: mfs_logging 268 | - name: MFS_MOUNT_OPTIONS 269 | valueFrom: 270 | configMapKeyRef: 271 | name: csi-moosefs-config 272 | key: mfs_mount_options 273 | imagePullPolicy: "Always" 274 | volumeMounts: 275 | - name: socket-dir 276 | mountPath: /var/lib/csi/sockets/pluginproxy/ 277 | volumes: 278 | - name: socket-dir 279 | emptyDir: {} 280 | 281 | --- 282 | kind: DaemonSet 283 | apiVersion: apps/v1 284 | metadata: 285 | name: csi-moosefs-node 286 | namespace: kube-system 287 | spec: 288 | selector: 289 | matchLabels: 290 | app: csi-moosefs-node 291 | template: 292 | metadata: 293 | labels: 294 | app: csi-moosefs-node 295 | role: csi-moosefs 296 | spec: 297 | priorityClassName: system-node-critical 298 | serviceAccount: csi-moosefs-node-sa 299 | hostNetwork: true 300 | containers: 301 | # registrar 302 | - name: driver-registrar 303 | image: quay.io/k8scsi/csi-node-driver-registrar:v2.1.0 304 | args: 305 | - "--v=5" 306 | - "--csi-address=$(ADDRESS)" 307 | - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" 308 | env: 309 | - name: ADDRESS 310 | value: /csi/csi.sock 311 | - name: DRIVER_REG_SOCK_PATH 312 | value: /var/lib/kubelet/plugins/csi.moosefs.com/csi.sock 313 | - name: KUBE_NODE_NAME 314 | valueFrom: 315 | fieldRef: 316 | fieldPath: spec.nodeName 317 | volumeMounts: 318 | - name: plugin-dir 319 | mountPath: /csi/ 320 | - name: registration-dir 321 | mountPath: /registration/ 322 | # MooseFS CSI Plugin 323 | - name: csi-moosefs-plugin 324 | securityContext: 325 | privileged: true 326 | capabilities: 327 | add: ["SYS_ADMIN"] 328 | allowPrivilegeEscalation: true 329 | image: ghcr.io/moosefs/moosefs-csi:0.9.8-4.57.6 330 | args: 331 | - "--mode=node" 332 | - "--csi-endpoint=$(CSI_ENDPOINT)" 333 | - "--master-host=$(MASTER_HOST)" 334 | - "--master-port=$(MASTER_PORT)" 335 | - "--node-id=$(NODE_ID)" 336 | - "--root-dir=$(ROOT_DIR)" 337 | - "--plugin-data-dir=$(WORKING_DIR)" 338 | - "--mount-points-count=$(MOUNT_COUNT)" 339 | - "--mfs-logging=$(MFS_LOGGING)" 340 | - "--mfs-mount-options=$(MFS_MOUNT_OPTIONS)" 341 | env: 342 | - name: CSI_ENDPOINT 343 | value: unix:///csi/csi.sock 344 | - name: MASTER_HOST 345 | valueFrom: 346 | configMapKeyRef: 347 | name: csi-moosefs-config 348 | key: master_host 349 | - name: MASTER_PORT 350 | valueFrom: 351 | configMapKeyRef: 352 | name: csi-moosefs-config 353 | key: master_port 354 | - name: NODE_ID 355 | valueFrom: 356 | fieldRef: 357 | fieldPath: spec.nodeName 358 | - name: ROOT_DIR 359 | valueFrom: 360 | configMapKeyRef: 361 | name: csi-moosefs-config 362 | key: k8s_root_dir 363 | - name: WORKING_DIR 364 | valueFrom: 365 | configMapKeyRef: 366 | name: csi-moosefs-config 367 | key: driver_working_dir 368 | - name: MOUNT_COUNT 369 | valueFrom: 370 | configMapKeyRef: 371 | name: csi-moosefs-config 372 | key: mount_count 373 | - name: MFS_LOGGING 374 | valueFrom: 375 | configMapKeyRef: 376 | name: csi-moosefs-config 377 | key: mfs_logging 378 | - name: MFS_MOUNT_OPTIONS 379 | valueFrom: 380 | configMapKeyRef: 381 | name: csi-moosefs-config 382 | key: mfs_mount_options 383 | imagePullPolicy: "Always" 384 | volumeMounts: 385 | - name: plugin-dir 386 | mountPath: /csi 387 | - name: pods-mount-dir 388 | mountPath: /var/lib/kubelet 389 | mountPropagation: "Bidirectional" 390 | - mountPath: /dev 391 | name: device-dir 392 | volumes: 393 | - name: registration-dir 394 | hostPath: 395 | path: /var/lib/kubelet/plugins_registry/ 396 | type: Directory 397 | - name: plugin-dir 398 | hostPath: 399 | path: /var/lib/kubelet/plugins/csi.moosefs.com 400 | type: DirectoryOrCreate 401 | - name: pods-mount-dir 402 | hostPath: 403 | path: /var/lib/kubelet 404 | type: Directory 405 | - name: device-dir 406 | hostPath: 407 | path: /dev 408 | -------------------------------------------------------------------------------- /driver/controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2025 Saglabs SA. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package driver 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "github.com/container-storage-interface/spec/lib/go/csi" 24 | "google.golang.org/grpc/codes" 25 | "google.golang.org/grpc/status" 26 | ) 27 | 28 | var ( 29 | supportedAccessMode = []*csi.VolumeCapability_AccessMode{ 30 | { 31 | Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, 32 | }, 33 | } 34 | supportedAccessModeMode = []csi.VolumeCapability_AccessMode_Mode{ 35 | csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, 36 | csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY, 37 | csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, 38 | csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER, 39 | csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, 40 | } 41 | moosefsVolumeCapability = &csi.VolumeCapability_MountVolume{ 42 | FsType: "moosefs", 43 | } 44 | ) 45 | 46 | var controllerCapabilities = []csi.ControllerServiceCapability_RPC_Type{ 47 | csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, 48 | csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, 49 | csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, 50 | // csi.ControllerServiceCapability_RPC_PUBLISH_READONLY, 51 | } 52 | 53 | type ControllerService struct { 54 | // csi.UnimplementedControllerServer 55 | csi.ControllerServer 56 | Service 57 | 58 | ctlMount *mfsHandler 59 | } 60 | 61 | var _ csi.ControllerServer = &ControllerService{} 62 | 63 | func NewControllerService(mfsmaster string, mfsmaster_port int, rootPath, pluginDataPath, mfsMountOptions string) (*ControllerService, error) { 64 | log.Infof("NewControllerService creation - mfsmaster %s, rootDir %s, pluginDataDir %s)", mfsmaster, rootPath, pluginDataPath) 65 | 66 | ctlMount := NewMfsHandler(mfsmaster, mfsmaster_port, rootPath, pluginDataPath, "controller", mfsMountOptions) 67 | if err := ctlMount.MountMfs(); err != nil { 68 | return nil, err 69 | } 70 | if MfsLog { 71 | ctlMount.SetMfsLogging() 72 | } 73 | return &ControllerService{ctlMount: ctlMount}, nil 74 | } 75 | 76 | // CreateVolume creates a new volume from the given request. The function is idempotent. 77 | func (cs *ControllerService) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { 78 | log.Infof("CreateVolume - Name: %s", req.Name) 79 | if req.Name == "" { 80 | return nil, status.Error(codes.InvalidArgument, "CreateVolume: Name must be provided") 81 | } 82 | if req.VolumeCapabilities == nil || len(req.VolumeCapabilities) == 0 { 83 | return nil, status.Error(codes.InvalidArgument, "CreateVolume: Volume capabilities must be provided") 84 | } 85 | requestedQuota, err := getRequestCapacity(req.CapacityRange) 86 | if err != nil { 87 | return nil, status.Error(codes.InvalidArgument, err.Error()) 88 | } 89 | 90 | for _, cap := range req.VolumeCapabilities { 91 | if cap.GetBlock() != nil { 92 | return nil, status.Error(codes.InvalidArgument, "CreateVolume: Block storage not supported") 93 | } 94 | } 95 | 96 | if req.VolumeContentSource != nil { 97 | return nil, status.Error(codes.InvalidArgument, "CreateVolume: VolumeContentSource not supported") 98 | } 99 | 100 | volumeId := req.Name 101 | exists, err := cs.ctlMount.VolumeExist(volumeId) 102 | if err != nil { 103 | return nil, status.Error(codes.Internal, err.Error()) 104 | } 105 | var acquiredSize int64 106 | if exists { 107 | currQuota, err := cs.ctlMount.GetQuota(volumeId) 108 | if err != nil { 109 | return nil, status.Error(codes.Internal, err.Error()) 110 | } 111 | if currQuota != requestedQuota { 112 | return nil, status.Errorf(codes.AlreadyExists, "CreateVolume: volume %s already exists and has different capacity from requested (current %d, requested %d)", 113 | volumeId, currQuota, requestedQuota) 114 | } 115 | } else { 116 | acquiredSize, err = cs.ctlMount.CreateVolume(volumeId, requestedQuota) 117 | if err != nil { 118 | return nil, status.Error(codes.Internal, err.Error()) 119 | } 120 | if acquiredSize != requestedQuota { 121 | log.Warningf("CreateVolume - requested %d bytes, got %d", requestedQuota, acquiredSize) 122 | } 123 | } 124 | if len(req.Parameters) != 0 { 125 | return nil, status.Errorf(codes.Internal, "CreateVolume: Plugin parameters are not supported") 126 | } 127 | /* 128 | volumeContext := req.GetParameters() 129 | if volumeContext == nil { 130 | volumeContext = make(map[string]string) 131 | } 132 | mfsVolumePath := cs.ctlMount.MfsPathToVolume(volumeId) 133 | volumeContext["mfsVolumePath"] = mfsVolumePath 134 | 135 | resp := &csi.CreateVolumeResponse{ 136 | Volume: &csi.Volume{ 137 | VolumeId: req.GetName(), 138 | CapacityBytes: acquiredSize, 139 | VolumeContext: volumeContext, 140 | }, 141 | } 142 | return resp, nil 143 | 144 | */ 145 | resp := &csi.CreateVolumeResponse{ 146 | Volume: &csi.Volume{ 147 | VolumeId: req.GetName(), 148 | CapacityBytes: acquiredSize, 149 | }, 150 | } 151 | return resp, nil 152 | } 153 | 154 | func (cs *ControllerService) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { 155 | log.Infof("DeleteVolume - VolumeId: %s", req.VolumeId) 156 | 157 | if req.VolumeId == "" { 158 | return nil, status.Error(codes.InvalidArgument, "DeleteVolume: VolumeId must be provided") 159 | } 160 | 161 | exists, err := cs.ctlMount.VolumeExist(req.VolumeId) 162 | if err != nil { 163 | return nil, status.Error(codes.Internal, err.Error()) 164 | } 165 | if !exists { 166 | return &csi.DeleteVolumeResponse{}, nil 167 | } 168 | if err := cs.ctlMount.DeleteVolume(req.VolumeId); err != nil { 169 | return nil, status.Error(codes.Internal, err.Error()) 170 | } 171 | return &csi.DeleteVolumeResponse{}, nil 172 | } 173 | 174 | func (cs *ControllerService) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { 175 | if req.VolumeId == "" { 176 | return nil, status.Error(codes.InvalidArgument, "ControllerExpandVolume: VolumeId must be provided") 177 | } 178 | size, err := getRequestCapacity(req.CapacityRange) 179 | if err != nil { 180 | return nil, status.Error(codes.InvalidArgument, err.Error()) 181 | } 182 | log.Infof("ControllerExpandVolume - VolumeId: %s, size: %d)", req.VolumeId, size) 183 | 184 | exists, err := cs.ctlMount.VolumeExist(req.VolumeId) 185 | if err != nil { 186 | return nil, status.Error(codes.Internal, err.Error()) 187 | } 188 | if !exists { 189 | return nil, status.Error(codes.NotFound, "ControllerExpandVolume: Volume not found") 190 | } 191 | 192 | acquiredSize, err := cs.ctlMount.SetQuota(req.VolumeId, size) 193 | if err != nil { 194 | return nil, status.Error(codes.Internal, err.Error()) 195 | } 196 | 197 | if acquiredSize != size { 198 | log.Warningf("ControllerExpandVolume - requested %d bytes, got %d", size, acquiredSize) 199 | } 200 | 201 | return &csi.ControllerExpandVolumeResponse{ 202 | CapacityBytes: acquiredSize, 203 | NodeExpansionRequired: false, 204 | }, nil 205 | } 206 | 207 | func (cs *ControllerService) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { 208 | log.Infof("ValidateVolumeCapabilities - VolumeId: %s", req.VolumeId) 209 | 210 | if req.VolumeId == "" { 211 | return nil, status.Error(codes.InvalidArgument, "ValidateVolumeCapabilities: VolumeId must be provided") 212 | } 213 | if len(req.VolumeCapabilities) == 0 || req.VolumeCapabilities == nil { 214 | return nil, status.Error(codes.InvalidArgument, "ValidateVolumeCapabilities: VolumeCapabilities must be provided") 215 | } 216 | 217 | if exists, err := cs.ctlMount.VolumeExist(req.VolumeId); err != nil { 218 | return nil, err 219 | } else if !exists { 220 | return nil, status.Errorf(codes.NotFound, "ValidateVolumeCapabilities: Volume %s not found", req.VolumeId) 221 | } 222 | 223 | resp := &csi.ValidateVolumeCapabilitiesResponse{} 224 | // resp := &csi.ValidateVolumeCapabilitiesResponse{ 225 | // Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ 226 | // VolumeCapabilities: volCap, 227 | // }, 228 | // } 229 | 230 | ok := true 231 | for _, cap := range req.VolumeCapabilities { 232 | if cap.GetBlock() != nil { 233 | ok = false 234 | } 235 | } 236 | if ok { 237 | resp.Confirmed = &csi.ValidateVolumeCapabilitiesResponse_Confirmed{VolumeCapabilities: req.GetVolumeCapabilities()} 238 | } 239 | return resp, nil 240 | } 241 | 242 | func (cs *ControllerService) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { 243 | log.Infof("ControllerGetCapabilities") 244 | var caps []*csi.ControllerServiceCapability 245 | for _, capa := range controllerCapabilities { 246 | caps = append(caps, &csi.ControllerServiceCapability{ 247 | Type: &csi.ControllerServiceCapability_Rpc{ 248 | Rpc: &csi.ControllerServiceCapability_RPC{ 249 | Type: capa, 250 | }, 251 | }, 252 | }) 253 | } 254 | 255 | return &csi.ControllerGetCapabilitiesResponse{ 256 | Capabilities: caps, 257 | }, nil 258 | } 259 | 260 | func (cs *ControllerService) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { 261 | log.Infof("ControllerPublishVolume - VolumeId: %s NodeId: %s VolumeContext: %v", req.VolumeId, req.NodeId, req.VolumeContext) 262 | if req.VolumeId == "" { 263 | return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume: VolumeId must be provided") 264 | } 265 | if req.NodeId == "" { 266 | return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume: NodeId must be provided") 267 | } 268 | if req.VolumeCapability == nil { 269 | return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume: VolumeCapability capabilities must be provided") 270 | } 271 | 272 | if req.VolumeCapability.GetBlock() != nil { 273 | return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume: Block storage not supported") 274 | } 275 | publishContext := make(map[string]string) 276 | if req.Readonly { 277 | publishContext["readonly"] = "true" 278 | } 279 | // dynamic or static and existing volume 280 | if len(req.GetVolumeContext()) == 0 { 281 | exists, err := cs.ctlMount.VolumeExist(req.VolumeId) 282 | if err != nil { 283 | return nil, status.Error(codes.Internal, err.Error()) 284 | } 285 | if !exists { 286 | return nil, status.Errorf(codes.NotFound, "ControllerPublishVolume: Volume %s not found", req.VolumeId) 287 | } else { 288 | return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil 289 | } 290 | } 291 | create, found := req.VolumeContext["create_on_publish"] 292 | do_create := (found && create == "true") 293 | _, found = req.VolumeContext["mfsSubDir"] 294 | if found { 295 | if do_create { 296 | return nil, status.Errorf(codes.InvalidArgument, "ControllerPublishVolume: VolumeContext contain both 'create' and 'mfsSubDir'") 297 | } else { 298 | cs.ctlMount.CreateMountVolume(req.VolumeId) 299 | return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil 300 | } 301 | } 302 | 303 | if exists, err := cs.ctlMount.VolumeExist(req.VolumeId); err != nil { 304 | return nil, status.Error(codes.Internal, err.Error()) 305 | } else if exists { 306 | return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil 307 | } 308 | 309 | if do_create { 310 | if _, err := cs.ctlMount.CreateVolume(req.VolumeId, 0); err != nil { 311 | return nil, status.Error(codes.Internal, err.Error()) 312 | } 313 | return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil 314 | } else { 315 | return nil, status.Errorf(codes.NotFound, "ControllerPublishVolume: Volume %s not found, 'create_on_publish' set to false'", req.VolumeId) 316 | } 317 | } 318 | 319 | func (cs *ControllerService) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { 320 | log.Infof("ControllerUnpublishVolume - VolumeId: %s, NodeId: %s", req.VolumeId, req.NodeId) 321 | if req.VolumeId == "" { 322 | return nil, status.Error(codes.InvalidArgument, "ControllerUnpublishVolume: VolumeId must be provided") 323 | } 324 | return &csi.ControllerUnpublishVolumeResponse{}, nil 325 | } 326 | 327 | ////////////////////// 328 | 329 | // getRequestCapacity extracts the storage size from the given capacity 330 | // range. If the capacity range is not satisfied it returns the default volume 331 | // size. 332 | func getRequestCapacity(capRange *csi.CapacityRange) (int64, error) { 333 | // todo(ad): fix default value 334 | if capRange == nil { 335 | return 1 << 31, nil 336 | } 337 | reqSize := capRange.RequiredBytes 338 | maxSize := capRange.LimitBytes 339 | var capacity int64 = 0 340 | 341 | if reqSize == 0 && maxSize == 0 { 342 | return 0, fmt.Errorf("getRequestCapacity: RequredBytes or LimitBytes must be provided") 343 | } 344 | if reqSize < 0 || maxSize < 0 { 345 | return 0, fmt.Errorf("getRequestCapacity: RequredBytes and LimitBytes can't be negative") 346 | } 347 | if reqSize == 0 { 348 | capacity = maxSize 349 | } else { 350 | capacity = reqSize 351 | } 352 | return capacity, nil 353 | } 354 | 355 | ////////// 356 | /* 357 | func (cs *ControllerService) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { 358 | log.Infof("ControllerService::ListVolumes") 359 | return nil, status.Errorf(codes.Unimplemented, "method ListVolumes not implemented") 360 | } 361 | 362 | func (cs *ControllerService) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { 363 | return nil, status.Errorf(codes.Unimplemented, "method CreateSnapshot not implemented") 364 | } 365 | 366 | func (cs *ControllerService) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { 367 | return nil, status.Errorf(codes.Unimplemented, "method DeleteSnapshot not implemented") 368 | } 369 | 370 | func (cs *ControllerService) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { 371 | return nil, status.Errorf(codes.Unimplemented, "method ListSnapshots not implemented") 372 | } 373 | */ 374 | -------------------------------------------------------------------------------- /driver/identity.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2025 Saglabs SA. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package driver 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/container-storage-interface/spec/lib/go/csi" 23 | "github.com/golang/protobuf/ptypes/wrappers" 24 | ) 25 | 26 | type IdentityService struct { 27 | csi.UnimplementedIdentityServer 28 | Service 29 | } 30 | 31 | var _ csi.IdentityServer = &IdentityService{} 32 | 33 | func (is *IdentityService) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { 34 | log.Infof("GetPluginInfo") 35 | 36 | return &csi.GetPluginInfoResponse{ 37 | Name: driverName, 38 | VendorVersion: driverVersion, 39 | }, nil 40 | } 41 | 42 | func (is *IdentityService) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { 43 | log.Infof("GetPluginCapabilities") 44 | 45 | return &csi.GetPluginCapabilitiesResponse{ 46 | Capabilities: []*csi.PluginCapability{ 47 | { 48 | Type: &csi.PluginCapability_Service_{ 49 | Service: &csi.PluginCapability_Service{ 50 | Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, 51 | }, 52 | }, 53 | }, 54 | { 55 | Type: &csi.PluginCapability_VolumeExpansion_{ 56 | VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ 57 | Type: csi.PluginCapability_VolumeExpansion_ONLINE, 58 | }, 59 | }, 60 | }, 61 | }, 62 | }, nil 63 | } 64 | 65 | func (is *IdentityService) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { 66 | log.Infof("Probe") 67 | 68 | return &csi.ProbeResponse{ 69 | Ready: &wrappers.BoolValue{ 70 | Value: true, 71 | }, 72 | }, nil 73 | } 74 | -------------------------------------------------------------------------------- /driver/mfs_handler.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2025 Saglabs SA. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package driver 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "io" 23 | "os" 24 | "os/exec" 25 | "path" 26 | "strconv" 27 | "strings" 28 | 29 | "gopkg.in/natefinch/lumberjack.v2" 30 | ) 31 | 32 | const ( 33 | fsType = "moosefs" 34 | newVolumeMode = 0755 35 | getQuotaCmd = "mfsgetquota" 36 | setQuotaCmd = "mfssetquota" 37 | quotaLimitType = "-L" 38 | quotaLimitRow = 2 39 | quotaLimitCol = 3 40 | logsDirName = "logs" 41 | volumesDirName = "volumes" 42 | mvolumesDirName = "mount_volumes" 43 | mntDir = "/mnt" 44 | ) 45 | 46 | // todo(ad): in future possibly add more options (mount options?) 47 | type mfsHandler struct { 48 | mfsmaster string // mfsmaster address 49 | mfsmaster_port int // mfsmaster port 50 | rootPath string // mfs root path 51 | pluginDataPath string // plugin data path (inside rootPath) 52 | name string // handler name 53 | hostMountPath string // host mfs mount path 54 | mfsMountOptions string // mfsmount additional options 55 | } 56 | 57 | func NewMfsHandler(mfsmaster string, mfsmaster_port int, rootPath, pluginDataPath, name, mfsMountOptions string, num ...int) *mfsHandler { 58 | var numSufix = "" 59 | var mountOptions = "" 60 | 61 | if len(num) == 2 { 62 | if num[0] == 0 && num[1] == 1 { 63 | numSufix = "" 64 | } else { 65 | numSufix = fmt.Sprintf("_%02d", num[0]) 66 | } 67 | } else if len(num) != 0 { 68 | log.Errorf("NewMfsHandler - Unexpected number of arguments: %d; expected 0 or 2", len(num)) 69 | } 70 | 71 | if len(mfsMountOptions) != 0 { 72 | mountOptions = mfsMountOptions 73 | } 74 | 75 | return &mfsHandler{ 76 | mfsmaster: mfsmaster, 77 | mfsmaster_port: mfsmaster_port, 78 | rootPath: rootPath, 79 | pluginDataPath: pluginDataPath, 80 | name: name, 81 | hostMountPath: path.Join(mntDir, fmt.Sprintf("%s%s", name, numSufix)), 82 | mfsMountOptions: mountOptions, 83 | } 84 | } 85 | 86 | func (mnt *mfsHandler) SetMfsLogging() { 87 | log.Infof("Setting up MooseFS Logging - path: %s", path.Join(mnt.rootPath, mnt.pluginDataPath, logsDirName)) 88 | mfsLogFile := &lumberjack.Logger{ 89 | Filename: path.Join(mnt.HostPathToLogs(), fmt.Sprintf("%s.log", mnt.name)), 90 | MaxSize: 100, 91 | MaxBackups: 3, 92 | MaxAge: 0, 93 | Compress: true, 94 | } 95 | mw := io.MultiWriter(os.Stderr, mfsLogFile) 96 | log.SetOutput(mw) 97 | log.Info("MooseFS Logging set up!") 98 | } 99 | 100 | func (mnt *mfsHandler) VolumeExist(volumeId string) (bool, error) { 101 | path := mnt.HostPathToVolume(volumeId) 102 | _, err := os.Stat(path) 103 | if err == nil { 104 | return true, nil 105 | } 106 | if os.IsNotExist(err) { 107 | return false, nil 108 | } 109 | return false, err 110 | } 111 | 112 | func (mnt *mfsHandler) MountVolumeExist(volumeId string) (bool, error) { 113 | path := mnt.HostPathToMountVolume(volumeId) 114 | _, err := os.Stat(path) 115 | if err == nil { 116 | return true, nil 117 | } 118 | if os.IsNotExist(err) { 119 | return false, nil 120 | } 121 | return false, err 122 | } 123 | 124 | func (mnt *mfsHandler) CreateMountVolume(volumeId string) error { 125 | path := mnt.HostPathToMountVolume(volumeId) 126 | if err := os.MkdirAll(path, newVolumeMode); err != nil { 127 | return err 128 | } 129 | return nil 130 | } 131 | 132 | func (mnt *mfsHandler) CreateVolume(volumeId string, size int64) (int64, error) { 133 | path := mnt.HostPathToVolume(volumeId) 134 | if err := os.MkdirAll(path, newVolumeMode); err != nil { 135 | return 0, err 136 | } 137 | if size == 0 { 138 | return 0, nil 139 | } 140 | acquiredSize, err := mnt.SetQuota(volumeId, size) 141 | if err != nil { 142 | return 0, err 143 | } 144 | return acquiredSize, nil 145 | } 146 | 147 | func (mnt *mfsHandler) DeleteVolume(volumeId string) error { 148 | path := mnt.HostPathToVolume(volumeId) 149 | if err := os.RemoveAll(path); err != nil { 150 | // todo(ad): fix msg 151 | log.Errorf("-------------------ControllerService::DeleteVolume -- Couldn't remove volume %s in directory %s. Error: %s", 152 | volumeId, path, err.Error()) 153 | return err 154 | } 155 | 156 | return nil 157 | } 158 | 159 | func (mnt *mfsHandler) GetQuota(volumeId string) (int64, error) { 160 | log.Infof("GetQuota - volumeId: %s", volumeId) 161 | 162 | //path := mnt.MfsPathToVolume(volumeId) 163 | path := mnt.HostPathToVolume(volumeId) 164 | 165 | cmd := exec.Command(getQuotaCmd, path) 166 | //cmd.Dir = mnt.hostMountPath 167 | out, err := cmd.CombinedOutput() 168 | 169 | if err != nil { 170 | return 0, fmt.Errorf("GetQuota: Error while executing command %s %s. Error: %s output: %v", getQuotaCmd, path, err.Error(), string(out)) 171 | } 172 | if quotaLimit, err := parseMfsQuotaToolsOutput(string(out)); err != nil { 173 | return 0, err 174 | } else if quotaLimit == -1 { 175 | return 0, fmt.Errorf("GetQuota: Quota for volume %s is not set or %s output is incorrect. Output: %s", volumeId, getQuotaCmd, string(out)) 176 | } else { 177 | return quotaLimit, nil 178 | } 179 | } 180 | 181 | func (mnt *mfsHandler) SetQuota(volumeId string, size int64) (int64, error) { 182 | log.Infof("SetQuota - volumeId: %s, size: %d", volumeId, size) 183 | 184 | //path := mnt.MfsPathToVolume(volumeId) 185 | path := mnt.HostPathToVolume(volumeId) 186 | if size <= 0 { 187 | return 0, errors.New("SetQuota: size must be positive") 188 | } 189 | setQuotaArgs := []string{quotaLimitType, strconv.FormatInt(size, 10), path} 190 | cmd := exec.Command(setQuotaCmd, setQuotaArgs...) 191 | //cmd.Dir = mnt.hostMountPath 192 | out, err := cmd.CombinedOutput() 193 | 194 | if err != nil { 195 | return 0, fmt.Errorf("SetQuota: Error while executing command %s %v. Error: %s output: %v", setQuotaCmd, setQuotaArgs, err.Error(), string(out)) 196 | } 197 | if quotaLimit, err := parseMfsQuotaToolsOutput(string(out)); err != nil { 198 | return 0, err 199 | } else if quotaLimit == -1 { 200 | return 0, fmt.Errorf("SetQuota: Quota for volume %s is not set or %s output is incorrect. Output: %s", volumeId, setQuotaCmd, string(out)) 201 | } else { 202 | return quotaLimit, nil 203 | } 204 | } 205 | 206 | func parseMfsQuotaToolsOutput(output string) (int64, error) { 207 | var cols []string 208 | var s string 209 | 210 | lines := strings.Split(output, "\n") 211 | ll := len(lines) 212 | 213 | if ll == 8 { 214 | // new mfsgetquota output format 215 | cols = strings.Split(lines[ll-4], "|") 216 | s = strings.TrimSpace(cols[4]) 217 | } else if ll == 6 { 218 | // old mfsgetquota output format 219 | cols := strings.Split(lines[ll-4], "|") 220 | s = strings.TrimSpace(cols[3]) 221 | } else { 222 | return -1, fmt.Errorf("error while parsing mfsgetquota tool output (unexpected number of lines); output: %s", output) 223 | } 224 | 225 | if s == "-" { 226 | // no quota set 227 | return -1, nil 228 | } 229 | 230 | quotaLimit, err := strconv.ParseInt(s, 10, 64) 231 | 232 | if err != nil { 233 | return -1, err 234 | } 235 | 236 | return quotaLimit, nil 237 | } 238 | 239 | // Mount mounts mfsclient at speciefied earlier point 240 | func (mnt *mfsHandler) MountMfs() error { 241 | var mountOptions []string 242 | mounter := Mounter{} 243 | mountSource := fmt.Sprintf("%s:%d:%s", mnt.mfsmaster, mnt.mfsmaster_port, mnt.rootPath) 244 | 245 | if len(mnt.mfsMountOptions) != 0 { 246 | mountOptions = strings.Split(mnt.mfsMountOptions, ",") 247 | } else { 248 | mountOptions = make([]string, 0) 249 | } 250 | 251 | log.Infof("MountMfs - source: %s, target: %s, options: %v", mountSource, mnt.hostMountPath, mountOptions) 252 | 253 | if isMounted, err := mounter.IsMounted(mnt.hostMountPath); err != nil { 254 | return err 255 | } else if isMounted { 256 | log.Warnf("MountMfs - Mount found in %s. Unmounting...", mnt.hostMountPath) 257 | if err = mounter.UMount(mnt.hostMountPath); err != nil { 258 | return err 259 | } 260 | } 261 | if err := os.RemoveAll(mnt.hostMountPath); err != nil { 262 | return err 263 | } 264 | if err := mounter.Mount(mountSource, mnt.hostMountPath, fsType, mountOptions...); err != nil { 265 | return err 266 | } 267 | log.Infof("MountMfs - Successfully mounted %s to %s", mountSource, mnt.hostMountPath) 268 | return nil 269 | } 270 | 271 | func (mnt *mfsHandler) BindMount(mfsSource string, target string, options ...string) error { 272 | mounter := Mounter{} 273 | source := mnt.HostPathTo(mfsSource) 274 | log.Infof("BindMount - source: %s, target: %s, options: %v", source, target, options) 275 | if isMounted, err := mounter.IsMounted(target); err != nil { 276 | return err 277 | } else if !isMounted { 278 | if err := mounter.Mount(source, target, fsType, append(options, "bind")...); err != nil { 279 | return err 280 | } 281 | } else { 282 | log.Infof("BindMount - target %s is already mounted", target) 283 | } 284 | return nil 285 | } 286 | 287 | func (mnt *mfsHandler) BindUMount(target string) error { 288 | mounter := Mounter{} 289 | log.Infof("BindUMount - target: %s", target) 290 | if mounted, err := mounter.IsMounted(target); err != nil { 291 | return err 292 | } else if mounted { 293 | if err := mounter.UMount(target); err != nil { 294 | return err 295 | } 296 | } else { 297 | log.Infof("BindUMount - target %s was already unmounted", target) 298 | } 299 | return nil 300 | } 301 | 302 | // HostPathToVolume returns absoluthe path to given volumeId on host mfsclient mountpoint 303 | func (mnt *mfsHandler) HostPathToVolume(volumeId string) string { 304 | return path.Join(mnt.hostMountPath, mnt.pluginDataPath, volumesDirName, volumeId) 305 | } 306 | 307 | func (mnt *mfsHandler) MfsPathToVolume(volumeId string) string { 308 | return path.Join(mnt.pluginDataPath, volumesDirName, volumeId) 309 | } 310 | 311 | func (mnt *mfsHandler) HostPathToMountVolume(volumeId string) string { 312 | return path.Join(mnt.hostMountPath, mnt.pluginDataPath, "mount_volumes", volumeId) 313 | } 314 | 315 | func (mnt *mfsHandler) HostPathToLogs() string { 316 | return path.Join(mnt.hostMountPath, mnt.pluginDataPath, logsDirName) 317 | } 318 | 319 | func (mnt *mfsHandler) HostPluginDataPath() string { 320 | return path.Join(mnt.hostMountPath, mnt.pluginDataPath) 321 | } 322 | 323 | func (mnt *mfsHandler) HostPathTo(to string) string { 324 | return path.Join(mnt.hostMountPath, to) 325 | } 326 | -------------------------------------------------------------------------------- /driver/mounter.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2025 Saglabs SA. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | */ 17 | 18 | /* 19 | * courtesy: https://github.com/digitalocean/csi-digitalocean/blob/master/driver/mounter.go 20 | */ 21 | 22 | package driver 23 | 24 | import ( 25 | "encoding/json" 26 | "errors" 27 | "fmt" 28 | "os" 29 | "os/exec" 30 | "strings" 31 | ) 32 | 33 | type MounterInterface interface { 34 | // Mount a volume 35 | Mount(sourcePath string, destPath, mountType string, opts ...string) error 36 | 37 | // Unmount a volume 38 | UMount(destPath string) error 39 | 40 | // Verify mount 41 | IsMounted(destPath string) (bool, error) 42 | } 43 | 44 | type Mounter struct { 45 | MounterInterface 46 | } 47 | 48 | var _ MounterInterface = &Mounter{} 49 | 50 | type findmntResponse struct { 51 | FileSystems []fileSystem `json:"filesystems"` 52 | } 53 | 54 | type fileSystem struct { 55 | Target string `json:"target"` 56 | Propagation string `json:"propagation"` 57 | FsType string `json:"fstype"` 58 | Options string `json:"options"` 59 | } 60 | 61 | const ( 62 | mountCmd = "mount" 63 | umountCmd = "umount" 64 | findmntCmd = "findmnt" 65 | newDirMode = 0750 66 | ) 67 | 68 | func (m *Mounter) Mount(sourcePath, destPath, mountType string, opts ...string) error { 69 | mountArgs := []string{} 70 | if sourcePath == "" { 71 | return errors.New("Mounter::Mount -- sourcePath must be provided") 72 | } 73 | 74 | if destPath == "" { 75 | return errors.New("Mounter::Mount -- Destination path must be provided") 76 | } 77 | 78 | mountArgs = append(mountArgs, "-t", mountType) 79 | if len(opts) > 0 { 80 | mountArgs = append(mountArgs, "-o", strings.Join(opts, ",")) 81 | } 82 | 83 | mountArgs = append(mountArgs, sourcePath) 84 | mountArgs = append(mountArgs, destPath) 85 | 86 | // create target, os.Mkdirall is noop if it exists 87 | err := os.MkdirAll(destPath, newDirMode) 88 | if err != nil { 89 | return err 90 | } 91 | out, err := exec.Command(mountCmd, mountArgs...).CombinedOutput() 92 | if err != nil { 93 | return fmt.Errorf("Mounter::Mount -- mounting failed: %v cmd: '%s %s' output: %q", 94 | err, mountCmd, strings.Join(mountArgs, " "), string(out)) 95 | } 96 | return nil 97 | } 98 | 99 | func (m *Mounter) UMount(destPath string) error { 100 | umountArgs := []string{} 101 | 102 | if destPath == "" { 103 | return errors.New("Mounter::UMount -- Destination path must be provided") 104 | } 105 | // todo(ad): sprawdzanie czy istnieje katalog 106 | umountArgs = append(umountArgs, destPath) 107 | 108 | out, err := exec.Command(umountCmd, umountArgs...).CombinedOutput() 109 | if err != nil { 110 | return fmt.Errorf("Mounter::UMount -- mounting failed: %v cmd: '%s %s' output: %q", 111 | err, umountCmd, strings.Join(umountArgs, " "), string(out)) 112 | } 113 | 114 | return nil 115 | } 116 | 117 | func (m *Mounter) IsMounted(destPath string) (bool, error) { 118 | if destPath == "" { 119 | return false, errors.New("Mounter::IsMounted -- target must be provided") 120 | } 121 | 122 | _, err := exec.LookPath(findmntCmd) 123 | if err != nil { 124 | if err == exec.ErrNotFound { 125 | return false, fmt.Errorf("Mounter::IsMounted -- %q executable not found in $PATH", findmntCmd) 126 | } 127 | return false, err 128 | } 129 | 130 | findmntArgs := []string{"-o", "TARGET,PROPAGATION,FSTYPE,OPTIONS", "-M", destPath, "-J"} 131 | out, err := exec.Command(findmntCmd, findmntArgs...).CombinedOutput() 132 | if err != nil { 133 | // findmnt exits with non zero exit status if it couldn't find anything 134 | if strings.TrimSpace(string(out)) == "" { 135 | return false, nil 136 | } 137 | return false, fmt.Errorf("Mounter::IsMounted -- checking mounted failed: %v cmd: %q output: %q", 138 | err, findmntCmd, string(out)) 139 | } 140 | 141 | if string(out) == "" { 142 | log.Warningf("Mounter::IsMounted -- %s returns no output while returning status 0 - unexpected behaviour but not an actual error", findmntCmd) 143 | return false, nil 144 | } 145 | 146 | var resp *findmntResponse 147 | err = json.Unmarshal(out, &resp) 148 | if err != nil { 149 | return false, fmt.Errorf("Mounter::IsMounted -- couldn't unmarshal data: %q: %s", string(out), err) 150 | } 151 | 152 | for _, fs := range resp.FileSystems { 153 | // check if the mount is propagated correctly. It should be set to shared, unless we run sanity tests 154 | if fs.Propagation != "shared" && !SanityTestRun { 155 | return true, fmt.Errorf("Mounter::IsMounted -- mount propagation for target %q is not enabled (%s instead of shared)", destPath, fs.Propagation) 156 | } 157 | // the mountpoint should match as well 158 | if fs.Target == destPath { 159 | return true, nil 160 | } 161 | } 162 | return false, nil 163 | } 164 | -------------------------------------------------------------------------------- /driver/node.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2025 Saglabs SA. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package driver 18 | 19 | import ( 20 | "context" 21 | "math/rand" 22 | 23 | "github.com/container-storage-interface/spec/lib/go/csi" 24 | "google.golang.org/grpc/codes" 25 | "google.golang.org/grpc/status" 26 | ) 27 | 28 | type NodeService struct { 29 | csi.UnimplementedNodeServer 30 | Service 31 | 32 | mountPointsCount int 33 | mountPoints []*mfsHandler 34 | nodeId string 35 | } 36 | 37 | var _ csi.NodeServer = &NodeService{} 38 | 39 | var nodeCapabilities = []csi.NodeServiceCapability_RPC_Type{ 40 | //csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, 41 | // csi.NodeServiceCapability_RPC_VOLUME_CONDITION, 42 | // csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, 43 | } 44 | 45 | func NewNodeService(mfsmaster string, mfsmaster_port int, rootPath, pluginDataPath, nodeId, mfsMountOptions string, mountPointsCount int) (*NodeService, error) { 46 | log.Infof("NewNodeService creation (mfsmaster %s, rootDir %s, pluginDataDir %s, nodeId %s, mountPointsCount %d)", mfsmaster, rootPath, pluginDataPath, nodeId, mountPointsCount) 47 | 48 | mountPoints := make([]*mfsHandler, mountPointsCount) 49 | for i := 0; i < mountPointsCount; i++ { 50 | mountPoints[i] = NewMfsHandler(mfsmaster, mfsmaster_port, rootPath, pluginDataPath, nodeId, mfsMountOptions, i, mountPointsCount) 51 | if err := mountPoints[i].MountMfs(); err != nil { 52 | return nil, err 53 | } 54 | } 55 | if MfsLog { 56 | mountPoints[0].SetMfsLogging() 57 | } 58 | 59 | ns := &NodeService{ 60 | mountPointsCount: mountPointsCount, 61 | mountPoints: mountPoints, 62 | nodeId: nodeId, 63 | } 64 | return ns, nil 65 | } 66 | 67 | func (ns *NodeService) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { 68 | log.Infof("NodePublishVolume - VolumeId: %s, Readonly: %v, VolumeContext %v, PublishContext %v, VolumeCapability %v TargetPath %s", req.GetVolumeId(), req.GetReadonly(), req.GetVolumeContext(), req.GetPublishContext(), req.GetVolumeCapability(), req.GetTargetPath()) 69 | if req.VolumeId == "" { 70 | return nil, status.Error(codes.InvalidArgument, "NodePublishVolume: VolumeId must be provided") 71 | } 72 | if req.TargetPath == "" { 73 | return nil, status.Error(codes.InvalidArgument, "NodePublishVolume: TargetPath must be provided") 74 | } 75 | if req.VolumeCapability == nil { 76 | return nil, status.Error(codes.InvalidArgument, "NodePublishVolume: VolumeCapability must be provided") 77 | } 78 | 79 | var source string 80 | if subDir, found := req.GetVolumeContext()["mfsSubDir"]; found { 81 | source = subDir 82 | } else { 83 | source = ns.mountPoints[0].MfsPathToVolume(req.VolumeId) 84 | } 85 | target := req.TargetPath 86 | options := req.VolumeCapability.GetMount().MountFlags 87 | if req.GetReadonly() { 88 | options = append(options, "ro") 89 | } 90 | if handler, err := ns.pickHandler(req.GetVolumeContext(), req.GetPublishContext()); err != nil { 91 | return nil, err 92 | } else { 93 | if err := handler.BindMount(source, target, options...); err != nil { 94 | return nil, status.Error(codes.Internal, err.Error()) 95 | } 96 | } 97 | return &csi.NodePublishVolumeResponse{}, nil 98 | } 99 | 100 | func (ns *NodeService) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { 101 | log.Infof("NodeUnpublishVolume - VolumeId: %s, TargetPath: %s)", req.GetVolumeId(), req.GetTargetPath()) 102 | if req.VolumeId == "" { 103 | return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume: Volume Id must be provided") 104 | } 105 | if req.TargetPath == "" { 106 | return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume: Target Path must be provided") 107 | } 108 | 109 | found, err := ns.mountPoints[0].VolumeExist(req.VolumeId) 110 | if err != nil { 111 | return nil, status.Error(codes.Internal, err.Error()) 112 | } else if !found { 113 | found, err = ns.mountPoints[0].MountVolumeExist(req.VolumeId) 114 | if err != nil { 115 | return nil, status.Error(codes.Internal, err.Error()) 116 | } 117 | if !found { 118 | return nil, status.Errorf(codes.NotFound, "NodeUnpublishVolume: volume %s not found", req.VolumeId) 119 | } 120 | } 121 | if err = ns.mountPoints[0].BindUMount(req.TargetPath); err != nil { 122 | return nil, status.Error(codes.Internal, err.Error()) 123 | } 124 | return &csi.NodeUnpublishVolumeResponse{}, nil 125 | } 126 | 127 | func (ns *NodeService) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { 128 | log.Infof("NodeGetInfo") 129 | return &csi.NodeGetInfoResponse{ 130 | NodeId: ns.nodeId, 131 | }, nil 132 | } 133 | 134 | func (ns *NodeService) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { 135 | log.Infof("NodeGetCapabilities") 136 | var caps []*csi.NodeServiceCapability 137 | for _, capa := range nodeCapabilities { 138 | caps = append(caps, &csi.NodeServiceCapability{ 139 | Type: &csi.NodeServiceCapability_Rpc{ 140 | Rpc: &csi.NodeServiceCapability_RPC{ 141 | Type: capa, 142 | }, 143 | }, 144 | }) 145 | } 146 | return &csi.NodeGetCapabilitiesResponse{ 147 | Capabilities: caps, 148 | }, nil 149 | } 150 | 151 | /* 152 | func (ns *NodeService) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { 153 | log.Infof("NodeService::NodeGetVolumeStats (volume_id %s, volume_path %s, staging_path %s)", 154 | req.VolumeId, req.VolumePath, req.StagingTargetPath) 155 | 156 | if req.VolumeId == "" { 157 | return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats: VolumeId must be provided") 158 | } 159 | if req.VolumePath == "" { 160 | return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats: VolumePath must be provided") 161 | } 162 | 163 | cond := false 164 | _, err := ioutil.ReadDir(req.VolumePath) 165 | if err != nil { 166 | log.Infof("%s %s corrupted", req.VolumeId, req.VolumePath) 167 | cond = true 168 | } else { 169 | log.Infof("%s %s NOT corrupted", req.VolumeId, req.VolumePath) 170 | } 171 | return &csi.NodeGetVolumeStatsResponse{VolumeCondition: &csi.VolumeCondition{ 172 | Abnormal: cond, 173 | Message: "", 174 | }}, nil 175 | } 176 | */ 177 | ////////////// 178 | 179 | // pickHandler - Returns proper handler. Currently picks random mfs handler. 180 | func (ns *NodeService) pickHandler(volumeContext map[string]string, publishContext map[string]string) (*mfsHandler, error) { 181 | if ns.mountPointsCount <= 0 { 182 | return nil, status.Error(codes.Internal, "pickHandler: there is no mfs handlers") 183 | } 184 | return ns.mountPoints[rand.Uint32()%uint32(ns.mountPointsCount)], nil 185 | } 186 | 187 | // pickHandlerFromVolumeId - Unimplemented, always picks first handler. 188 | func (ns *NodeService) pickHandlerFromVolumeId(volumeId string) (*mfsHandler, error) { 189 | if ns.mountPointsCount <= 0 { 190 | return nil, status.Error(codes.Internal, "pickHandlerFromVolumeId: there is no mfs handlers") 191 | } 192 | return ns.mountPoints[0], nil 193 | } 194 | -------------------------------------------------------------------------------- /driver/service.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2025 Saglabs SA. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package driver 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "net" 23 | "net/url" 24 | "os" 25 | "path" 26 | "path/filepath" 27 | 28 | "github.com/container-storage-interface/spec/lib/go/csi" 29 | "github.com/sirupsen/logrus" 30 | "google.golang.org/grpc" 31 | "google.golang.org/grpc/status" 32 | ) 33 | 34 | const ( 35 | driverName = "csi.moosefs.com" 36 | driverVersion = "0.9.8" 37 | ) 38 | 39 | type Service interface{} 40 | 41 | var SanityTestRun bool 42 | var MfsLog bool 43 | var log logrus.Logger 44 | 45 | func Init(sanityTestRun bool, logLevel int, mfsLog bool) error { 46 | log = *logrus.New() 47 | SanityTestRun = sanityTestRun 48 | log.SetLevel(logrus.Level(logLevel)) 49 | MfsLog = mfsLog 50 | return nil 51 | } 52 | 53 | func StartService(service *Service, mode, csiEndpoint string) error { 54 | log.Infof("StartService - endpoint %s", csiEndpoint) 55 | gRPCServer := CreategRPCServer() 56 | listener, err := CreateListener(csiEndpoint) 57 | if err != nil { 58 | return err 59 | } 60 | csi.RegisterIdentityServer(gRPCServer, &IdentityService{}) 61 | 62 | switch (*service).(type) { 63 | case *NodeService: 64 | log.Infof("StartService - Registering node service") 65 | csi.RegisterNodeServer(gRPCServer, (*service).(csi.NodeServer)) 66 | case *ControllerService: 67 | log.Infof("StartService - Registering controller service") 68 | csi.RegisterControllerServer(gRPCServer, (*service).(csi.ControllerServer)) 69 | default: 70 | return fmt.Errorf("StartService: Unrecognized service type: %T", service) 71 | } 72 | 73 | log.Info("StartService - Starting to serve!") 74 | err = gRPCServer.Serve(listener) 75 | if err != nil { 76 | return err 77 | } 78 | log.Info("StartService - gRPCServer stopped without an error!") 79 | return nil 80 | } 81 | 82 | // CreateListener create listener ready for communication over given csi endpoint 83 | func CreateListener(csiEndpoint string) (net.Listener, error) { 84 | log.Infof("CreateListener - endpoint %s", csiEndpoint) 85 | 86 | u, err := url.Parse(csiEndpoint) 87 | if err != nil { 88 | return nil, fmt.Errorf("CreateListener - Unable to parse address: %q", err) 89 | } 90 | 91 | addr := path.Join(u.Host, filepath.FromSlash(u.Path)) 92 | if u.Host == "" { 93 | addr = filepath.FromSlash(u.Path) 94 | } 95 | 96 | // CSI plugins talk only over UNIX sockets currently 97 | if u.Scheme != "unix" { 98 | return nil, fmt.Errorf("CreateListener - Currently only unix domain sockets are supported, have: %s", u.Scheme) 99 | } else { 100 | // remove the socket if it's already there. This can happen if we 101 | // deploy a new version and the socket was created from the old running 102 | // plugin. 103 | log.Infof("CreateListener - Removing socket %s", addr) 104 | if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { 105 | return nil, fmt.Errorf("CreateListener - Failed to remove unix domain socket file %s, error: %s", addr, err) 106 | } 107 | } 108 | 109 | listener, err := net.Listen(u.Scheme, addr) 110 | if err != nil { 111 | return nil, fmt.Errorf("CreateListener - Failed to listen: %v", err) 112 | } 113 | 114 | return listener, nil 115 | } 116 | 117 | func CreategRPCServer() *grpc.Server { 118 | log.Info("CreategRPCServer") 119 | // log response errors for better observability 120 | errHandler := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 121 | resp, err := handler(ctx, req) 122 | if err != nil { 123 | stat, rpcErr := status.FromError(err) 124 | if rpcErr { 125 | log.Errorf("rpc error: %s - %s", stat.Code(), stat.Message()) 126 | } else { 127 | log.Errorf("unexpected error type - %s", err.Error()) 128 | } 129 | } 130 | return resp, err 131 | } 132 | return grpc.NewServer(grpc.UnaryInterceptor(errHandler)) 133 | } 134 | -------------------------------------------------------------------------------- /examples/dynamic-provisioning/pod.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: my-moosefs-pod 5 | spec: 6 | containers: 7 | - name: my-frontend 8 | image: busybox 9 | volumeMounts: 10 | - mountPath: "/data" 11 | name: moosefs-volume 12 | command: [ "sleep", "1000000" ] 13 | volumes: 14 | - name: moosefs-volume 15 | persistentVolumeClaim: 16 | claimName: my-moosefs-pvc 17 | -------------------------------------------------------------------------------- /examples/dynamic-provisioning/pvc.yaml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | metadata: 4 | name: my-moosefs-pvc 5 | spec: 6 | storageClassName: moosefs-storage 7 | accessModes: 8 | - ReadWriteMany 9 | resources: 10 | requests: 11 | storage: 5Gi 12 | -------------------------------------------------------------------------------- /examples/mount-volume/pod.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: my-moosefs-pod-mount 5 | spec: 6 | containers: 7 | - name: my-frontend 8 | image: busybox 9 | volumeMounts: 10 | - mountPath: "/data" 11 | name: moosefs-volume 12 | command: [ "sleep", "1000000" ] 13 | volumes: 14 | - name: moosefs-volume 15 | persistentVolumeClaim: 16 | claimName: my-moosefs-pvc-mount 17 | -------------------------------------------------------------------------------- /examples/mount-volume/pv.yaml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolume 2 | apiVersion: v1 3 | metadata: 4 | name: my-moosefs-pv-mount 5 | spec: 6 | storageClassName: "" # empty Storage Class 7 | capacity: 8 | storage: 1Gi # required, however does not have any effect 9 | accessModes: 10 | - ReadWriteMany 11 | csi: 12 | driver: csi.moosefs.com 13 | volumeHandle: my-mount-volume # unique volume name 14 | volumeAttributes: 15 | mfsSubDir: "/" # subfolder to be mounted as a rootdir (inside k8s_root_dir) 16 | -------------------------------------------------------------------------------- /examples/mount-volume/pvc.yaml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | metadata: 4 | name: my-moosefs-pvc-mount 5 | spec: 6 | storageClassName: "" # empty Storage Class 7 | volumeName: my-moosefs-pv-mount 8 | accessModes: 9 | - ReadWriteMany 10 | resources: 11 | requests: 12 | storage: 1Gi # at least as much as in PV, does not have any effect 13 | -------------------------------------------------------------------------------- /examples/static-provisioning/pod.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: my-moosefs-pod-static 5 | spec: 6 | containers: 7 | - name: my-frontend 8 | image: busybox 9 | volumeMounts: 10 | - mountPath: "/data" 11 | name: moosefs-volume 12 | command: [ "sleep", "1000000" ] 13 | volumes: 14 | - name: moosefs-volume 15 | persistentVolumeClaim: 16 | claimName: my-moosefs-pvc-static 17 | -------------------------------------------------------------------------------- /examples/static-provisioning/pv.yaml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolume 2 | apiVersion: v1 3 | metadata: 4 | name: my-moosefs-pv-static 5 | spec: 6 | storageClassName: moosefs-storage 7 | capacity: 8 | storage: 5Gi 9 | accessModes: 10 | - ReadWriteMany 11 | csi: 12 | driver: csi.moosefs.com 13 | volumeHandle: my-volume-0000 14 | -------------------------------------------------------------------------------- /examples/static-provisioning/pvc.yaml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | metadata: 4 | name: my-moosefs-pvc-static 5 | spec: 6 | storageClassName: moosefs-storage 7 | accessModes: 8 | - ReadWriteMany 9 | resources: 10 | requests: 11 | storage: 5Gi 12 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/moosefs/moosefs-csi 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/container-storage-interface/spec v1.4.0 7 | github.com/golang/protobuf v1.4.3 8 | github.com/sirupsen/logrus v1.8.0 9 | golang.org/x/net v0.0.0-20200320220750-118fecf932d8 // indirect 10 | golang.org/x/sys v0.0.0-20200321134203-328b4cd54aae // indirect 11 | golang.org/x/text v0.3.2 // indirect 12 | google.golang.org/grpc v1.36.0 13 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 14 | gopkg.in/yaml.v2 v2.2.5 // indirect 15 | ) 16 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 5 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 6 | github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= 7 | github.com/container-storage-interface/spec v1.4.0 h1:ozAshSKxpJnYUfmkpZCTYyF/4MYeYlhdXbAvPvfGmkg= 8 | github.com/container-storage-interface/spec v1.4.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= 9 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 13 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 14 | github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= 15 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 16 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 17 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 18 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 19 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 20 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 21 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 22 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 23 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 24 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 25 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 26 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 27 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 28 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 29 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 30 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 31 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 32 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 33 | github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= 34 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 35 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 36 | github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= 37 | github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= 38 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 39 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 40 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 41 | github.com/sirupsen/logrus v1.8.0 h1:nfhvjKcUMhBMVqbKHJlk5RPrrfYr/NMo3692g0dwfWU= 42 | github.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A= 43 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 44 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 45 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 46 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 47 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 48 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 49 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 50 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 51 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 52 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 53 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 54 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 55 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 56 | golang.org/x/net v0.0.0-20200320220750-118fecf932d8 h1:1+zQlQqEEhUeStBTi653GZAnAuivZq/2hz+Iz+OP7rg= 57 | golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 58 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 59 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 60 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 61 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 62 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 63 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 64 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 65 | golang.org/x/sys v0.0.0-20200321134203-328b4cd54aae h1:3tcmuaB7wwSZtelmiv479UjUB+vviwABz7a133ZwOKQ= 66 | golang.org/x/sys v0.0.0-20200321134203-328b4cd54aae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 67 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 68 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 69 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 70 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 71 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 72 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 73 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 74 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 75 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 76 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 77 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 78 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 79 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 80 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 81 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= 82 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 83 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 84 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 85 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 86 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 87 | google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= 88 | google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= 89 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 90 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 91 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 92 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 93 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 94 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 95 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 96 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 97 | google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= 98 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 99 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 100 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 101 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= 102 | gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= 103 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 104 | gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= 105 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 106 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 107 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 108 | --------------------------------------------------------------------------------