├── .dockerignore ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── build └── docker │ ├── driverd │ └── Dockerfile │ └── lvmctrld │ └── Dockerfile ├── cmd ├── driverd │ ├── driverd.bin │ └── main.go └── lvmctrld │ ├── lvmctrld.bin │ └── main.go ├── conf ├── csi-sanlock-lvm-config.yaml ├── csi-sanlock-lvm-storageclass.yaml └── csi-sanlock-lvm-volumesnapshotclass.yaml ├── configs └── lvmctrld │ └── lvm.conf ├── csi-sanity-test.sh ├── deploy └── kubernetes │ ├── crd-snapshot.storage.k8s.io_volumesnapshotclasses.url │ ├── crd-snapshot.storage.k8s.io_volumesnapshotclasses.url.yaml │ ├── crd-snapshot.storage.k8s.io_volumesnapshotcontents.url │ ├── crd-snapshot.storage.k8s.io_volumesnapshotcontents.url.yaml │ ├── crd-snapshot.storage.k8s.io_volumesnapshots.url │ ├── crd-snapshot.storage.k8s.io_volumesnapshots.url.yaml │ ├── csi-sanlock-lvm-attacher.var │ ├── csi-sanlock-lvm-attacher.var.yaml │ ├── csi-sanlock-lvm-driverinfo.var │ ├── csi-sanlock-lvm-driverinfo.var.yaml │ ├── csi-sanlock-lvm-init.var │ ├── csi-sanlock-lvm-init.var.yaml │ ├── csi-sanlock-lvm-plugin.var │ ├── csi-sanlock-lvm-plugin.var.yaml │ ├── csi-sanlock-lvm-provisioner.var │ ├── csi-sanlock-lvm-provisioner.var.yaml │ ├── csi-sanlock-lvm-resizer.var │ ├── csi-sanlock-lvm-resizer.var.yaml │ ├── csi-sanlock-lvm-snapshotter.var │ ├── csi-sanlock-lvm-snapshotter.var.yaml │ ├── csi-sanlock-lvm-socat.var │ ├── csi-sanlock-lvm-socat.var.yaml │ ├── kustomization.yaml │ ├── rbac-attacher.url │ ├── rbac-attacher.url.yaml │ ├── rbac-provisioner.url │ ├── rbac-provisioner.url.yaml │ ├── rbac-resizer.url │ ├── rbac-resizer.url.yaml │ ├── rbac-snapshot-controller.url │ ├── rbac-snapshot-controller.url.yaml │ ├── rbac-snapshotter.url │ ├── rbac-snapshotter.url.yaml │ ├── setup-snapshot-controller.url │ └── setup-snapshot-controller.url.yaml ├── examples ├── pod-block.yaml ├── pod.yaml ├── pvc-block.yaml ├── pvc.yaml └── snap.yaml ├── go.mod ├── go.sum └── pkg ├── diskrpc ├── allocator.go ├── allocator_test.go ├── diskrpc.go ├── diskrpc_test.go ├── mailbox.go └── mailbox_test.go ├── driverd ├── baseserver.go ├── cmpmatcher_test.go ├── controllerserver.go ├── controllerserver_test.go ├── diskrpcservice.go ├── filesystem.go ├── identityserver.go ├── listener.go ├── lvmctrldclient.go ├── nodeserver.go ├── nodeserver_test.go ├── tagencoder.go ├── tagencoder_test.go ├── tags.go ├── volumeinfo.go ├── volumelock.go ├── volumelock_test.go └── volumeref.go ├── grpclogger └── grpclogger.go ├── lvmctrld ├── fakerunner_test.go ├── listener.go ├── lock.go ├── lvmctrldserver.go ├── lvmctrldserver_test.go └── runner.go ├── mock ├── diskrpc.mock ├── filesystem.mock ├── filesystemregistry.mock ├── lvmctrldclient.mock ├── mount.mock └── volumelocker.mock └── proto ├── diskrpc.proto ├── lvmctrld.proto └── prototest └── helpers.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # Scm, ide. 2 | .git 3 | .idea 4 | 5 | # Build artifacts. 6 | .makeargs 7 | coverage.txt 8 | cmd/*/* 9 | !cmd/*/*.* 10 | pkg/proto/*.go 11 | pkg/mock/*.go 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Scm, ide. 2 | .git 3 | .idea 4 | 5 | # Build artifacts. 6 | .makeargs 7 | coverage.txt 8 | cmd/*/* 9 | !cmd/*/*.* 10 | pkg/proto/*.go 11 | pkg/mock/*.go 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | language: golang 16 | dist: bionic 17 | 18 | services: 19 | - docker 20 | 21 | go: 22 | - 1.16 23 | 24 | env: 25 | global: 26 | - PROTOBUF_VERSION=3.11.4 27 | - PATH="$PATH:$HOME/protoc/bin:$HOME/go/bin" 28 | 29 | before_install: 30 | - curl -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -o /tmp/protoc.zip 31 | - unzip /tmp/protoc.zip -d "$HOME"/protoc 32 | - go get -v github.com/golang/protobuf/protoc-gen-go 33 | - go get -v github.com/golang/mock/mockgen 34 | 35 | script: 36 | - make -j2 all build-image 37 | 38 | after_success: 39 | - bash <(curl -s https://codecov.io/bash) 40 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows 28 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/). 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: args build clean mock proto test %.image %.push 2 | 3 | # Recursive wildcard from https://stackoverflow.com/questions/2483182/recursive-wildcards-in-gnu-make. 4 | rwildcard=$(foreach d,$(wildcard $(1:=/*)),$(call rwildcard,$d,$2) $(filter $(subst *,%,$2),$d)) 5 | 6 | BIN=cmd/lvmctrld/lvmctrld cmd/driverd/driverd 7 | GO=$(call rwildcard,.,*.go) 8 | MOCK=$(addprefix pkg/mock/, diskrpc.mock.go filesystem.mock.go filesystemregistry.mock.go lvmctrldclient.mock.go mount.mock.go volumelocker.mock.go) 9 | PROTO=$(addprefix pkg/proto/, lvmctrld.pb.go lvmctrld_grpc.pb.go diskrpc.pb.go) 10 | MANIFEST=$(addsuffix .yaml, $(wildcard deploy/kubernetes/*.url) $(wildcard deploy/kubernetes/*.var)) 11 | IMAGE=lvmctrld.image driverd.image 12 | PUSH=lvmctrld.push driverd.push 13 | 14 | # https://github.com/kubernetes-csi/external-snapshotter/tags, k8s >= 1.20 15 | export EXTERNAL_SNAPSHOTTER_VERSION=v6.2.2 16 | # https://github.com/kubernetes-csi/external-attacher/tags, k8s >= 1.17 17 | export EXTERNAL_ATTACHER_VERSION=v3.5.0 18 | # https://github.com/kubernetes-csi/external-provisioner/tags, k8s >= 1.20 19 | export EXTERNAL_PROVISIONER_VERSION=v3.4.0 20 | # https://github.com/kubernetes-csi/external-resizer/tags, k8s >= 1.16 21 | export EXTERNAL_RESIZER_VERSION=v1.7.0 22 | 23 | VERSION?=$(shell git describe --tags 2>/dev/null || (printf commit-; git rev-parse --short HEAD)) 24 | export VERSION 25 | COMMIT?=$(shell git rev-parse --short HEAD) 26 | export COMMIT 27 | 28 | # Ensure build parameter changes causes the precedent build to be discarded. 29 | -include .makeargs 30 | ARGS_CURR=commit=$(COMMIT),version=$(VERSION) 31 | ifneq ($(ARGS_CURR),$(ARGS_PREV)) 32 | ARGS_DEP=args 33 | endif 34 | 35 | ifeq ($(VERSION), test) 36 | IMAGE_PULL_POLICY=Always 37 | else 38 | IMAGE_PULL_POLICY=IfNotPresent 39 | endif 40 | export IMAGE_PULL_POLICY 41 | 42 | build: $(BIN) $(MANIFEST) 43 | proto: $(PROTO) 44 | mock: $(MOCK) 45 | image: $(IMAGE) 46 | push: $(PUSH) 47 | 48 | args: $(ARGS_CLEAN) 49 | printf "ARGS_PREV=%s\nARGS_CLEAN=clean\n" $(ARGS_CURR) > .makeargs 50 | 51 | clean: 52 | $(RM) $(BIN) $(PROTO) $(MOCK) $(MANIFEST) 53 | 54 | test coverage.txt: mock 55 | go test -race -coverprofile=coverage.txt -covermode=atomic ./cmd/* ./pkg/* 56 | 57 | %: %.bin $(GO) $(ARGS_DEP) | proto 58 | CGO_ENABLED=0 go build -a -ldflags '-extldflags "-static" -X main.version=$(VERSION) -X main.commit=$(COMMIT)' -o $@ ./$(@D) 59 | 60 | %.pb.go %_grpc.pb.go: %.proto 61 | protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=require_unimplemented_servers=false:. --go-grpc_opt=paths=source_relative $< 62 | 63 | %.mock.go: %.mock 64 | mockgen -package mock -destination $@ `cat $<` 65 | 66 | %.url.yaml: %.url $(ARGS_DEP) 67 | curl -s -o $@ `cat $< | sed 's/@@EXTERNAL_SNAPSHOTTER_VERSION@@/$(EXTERNAL_SNAPSHOTTER_VERSION)/g;s/@@EXTERNAL_ATTACHER_VERSION@@/$(EXTERNAL_ATTACHER_VERSION)/g;s/@@EXTERNAL_PROVISIONER_VERSION@@/$(EXTERNAL_PROVISIONER_VERSION)/g;s/@@EXTERNAL_RESIZER_VERSION@@/$(EXTERNAL_RESIZER_VERSION)/g;'` 68 | 69 | %.var.yaml: %.var $(ARGS_DEP) 70 | envsubst < $< > $@ 71 | 72 | %.image: 73 | docker build --platform linux/x86_64 --build-arg VERSION=$(VERSION) --build-arg COMMIT=$(COMMIT) -t quay.io/aleofreddi/csi-sanlock-lvm-$*:$(VERSION) -f build/docker/$*/Dockerfile . 74 | 75 | %.push: %.image 76 | docker push quay.io/aleofreddi/csi-sanlock-lvm-$*:$(VERSION) 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CSI Sanlock-LVM Driver 2 | 3 | [![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 4 | [![Build Status](https://travis-ci.com/aleofreddi/csi-sanlock-lvm.svg?branch=master)](https://travis-ci.com/aleofreddi/csi-sanlock-lvm) 5 | [![Test Coverage](https://codecov.io/gh/aleofreddi/csi-sanlock-lvm/branch/master/graph/badge.svg)](https://codecov.io/gh/aleofreddi/csi-sanlock-lvm) 6 | 7 | `csi-sanlock-lvm` is a CSI driver for LVM and Sanlock. 8 | 9 | It comes in handy when you want your nodes to access data on a shared block 10 | device - a typical example being Kubernetes on bare metal on a SAN (storage area 11 | network). 12 | 13 | ## Maturity 14 | 15 | This project is in alpha state, YMMV. 16 | 17 | ## Features 18 | 19 | - Dynamic volume provisioning 20 | - Support both filesystem and block devices 21 | - Support different filesystems 22 | - Support single node read/write access 23 | - Online volume extension 24 | - Online snapshot support 25 | - Volume groups (fs groups) 26 | - ~~Ephemeral volumes~~ (TODO) 27 | 28 | ## Prerequisite 29 | 30 | - Kubernetes 1.20+ 31 | - `kubectl` 32 | 33 | ## Limitations 34 | 35 | Sanlock might require up to 3 minutes to start, so bringing up a node can take 36 | some time. 37 | 38 | Also, Sanlock requires every cluster node to get an unique integer in the range 39 | 1-2000. This is implemented using least significant bits of the node ip address, 40 | which works as long as all the nodes reside in a subnet that contains at most 41 | 2000 addresses (a `/22` subnet or larger). 42 | 43 | ## Installation 44 | 45 | This chapter describes a step-by-step procedure to get csi-sanlock-lvm running 46 | on your cluster. 47 | 48 | ### Initialize a shared volume group 49 | 50 | Before deploying the driver, you need to have at least a shared volume group set 51 | up, as well as some logical volumes dedicated for csi-sanlock-lvm rpc mechanism. 52 | You can use the provided `csi-sanlock-lvm-init` pod to initialize lvm as 53 | follows: 54 | 55 | ```shell 56 | kubectl apply -f "https://raw.githubusercontent.com/aleofreddi/csi-sanlock-lvm/v0.4.5/deploy/kubernetes/csi-sanlock-lvm-init.var.yaml" 57 | ``` 58 | 59 | Then attach the init pod as follows and initialize the VG. 60 | 61 | ```shell 62 | kubectl attach -it csi-sanlock-lvm-init 63 | ``` 64 | 65 | Within the init shell, initialize the shared volume group and the rpc logical 66 | volumes: 67 | 68 | ```shell 69 | # Adjust your devices before running this! 70 | vgcreate --shared vg01 [/dev/device1 ... /dev/deviceN] 71 | 72 | # Create the csl-rpc-data logical volume. 73 | lvcreate -L 8m -n csl-rpc-data \ 74 | --add-tag csi-sanlock-lvm.vleo.net/rpcRole=data vg01 && 75 | lvchange -a n vg01/csl-rpc-data 76 | 77 | # Create the csl-rpc-lock logical volume. 78 | lvcreate -L 512b -n csl-rpc-lock \ 79 | --add-tag csi-sanlock-lvm.vleo.net/rpcRole=lock vg01 && 80 | lvchange -a n vg01/csl-rpc-lock 81 | 82 | # Initialization complete, terminate the pod successfully. 83 | exit 0 84 | ```` 85 | 86 | Now cleanup the init pod: 87 | 88 | ```shell 89 | kubectl delete po csi-sanlock-lvm-init 90 | ``` 91 | 92 | ### Deploy the driver 93 | 94 | When the volume group setup is complete, go ahead and create a namespace to 95 | accommodate the driver: 96 | 97 | ```shell 98 | kubectl create namespace csi-sanlock-lvm-system 99 | ``` 100 | 101 | And then deploy using `kustomization`: 102 | 103 | ```shell 104 | # Install the csi-sanlock-lvm driver. 105 | kubectl apply -k "https://github.com/aleofreddi/csi-sanlock-lvm/deploy/kubernetes?ref=v0.4.5" 106 | ``` 107 | 108 | It might take up to 3 minutes for the csi plugin to become `Running` on each 109 | node, and all the containers should be ready (for the plugin ones that would 110 | be `4/4`). To check the current status you can use the following command: 111 | 112 | ```shell 113 | kubectl -n csi-sanlock-lvm-system get pod 114 | ``` 115 | 116 | Each node will should have its own `csi-sanlock-lvm-plugin` pod. You should get 117 | an output similar to: 118 | 119 | ``` 120 | NAME READY STATUS RESTARTS AGE 121 | csi-sanlock-lvm-attacher-0 1/1 Running 0 2m15s 122 | csi-sanlock-lvm-plugin-cm7h6 4/4 Running 0 2m13s 123 | csi-sanlock-lvm-plugin-zkw84 4/4 Running 0 2m13s 124 | csi-sanlock-lvm-provisioner-0 1/1 Running 0 2m14s 125 | csi-sanlock-lvm-resizer-0 1/1 Running 0 2m14s 126 | csi-sanlock-lvm-snapshotter-0 1/1 Running 0 2m14s 127 | snapshot-controller-0 1/1 Running 0 2m14s 128 | ``` 129 | 130 | ### Setup storage and snapshot classes 131 | 132 | To enable the csi-sanlock-lvm driver you need to refer it from a storage class. 133 | 134 | In particular, you need to set up a `StorageClass` object to manage volumes, and 135 | optionally a `VolumeSnapshotClass` object to manage snapshots. 136 | 137 | Configuration examples are provided at `conf/csi-sanlock-lvm-storageclass.yaml` 138 | and `conf/csi-sanlock-lvm-volumesnapshotclass.yaml`. 139 | 140 | The following storage class parameters are supported: 141 | 142 | - `volumeGroup` _(required)_: the volume group to use when provisioning logical 143 | volumes. 144 | 145 | The following volume snapshot class parameters are supported: 146 | 147 | - `maxSizePct` _(optional)_: maximum snapshot size as percentage of its origin 148 | size; 149 | - `maxSize` _(optional)_: maximum snapshot size. 150 | 151 | If both `maxSizePct` and `maxSize` are set, the strictest is applied. 152 | 153 | ## Example application 154 | 155 | The `examples` directory contains an example configuration that will spin up a 156 | pvc and pod using it: 157 | 158 | ```shell 159 | kubectl apply -f examples/pvc.yaml examples/pod.yaml 160 | ``` 161 | 162 | You can also create a snapshot of the volume using the `snap.yaml`: 163 | 164 | ```shell 165 | kubectl apply -f examples/snap.yaml 166 | ``` 167 | 168 | ## Building the binaries 169 | 170 | ### Requirements 171 | 172 | To build the project, you need: 173 | 174 | * Golang ≥1.20; 175 | * GNU make; 176 | * protoc compiler; 177 | * protoc-gen-go and protoc-gen-go-grpc; 178 | * [gomock](https://github.com/golang/mock). 179 | 180 | You can install protoc-gen-go and gomock as follows: 181 | 182 | ```shell 183 | go install github.com/golang/protobuf/protoc-gen-go google.golang.org/grpc/cmd/protoc-gen-go-grpc github.com/golang/mock/mockgen 184 | ``` 185 | 186 | ### Build binaries 187 | 188 | If you want to build the driver yourself, you can do so invoking make: 189 | 190 | ```shell 191 | make 192 | ``` 193 | 194 | ### Build docker images 195 | 196 | Similarly, you want to build docker images for the driver using 197 | the `build-image` target: 198 | 199 | ```shell 200 | make build-image 201 | ``` 202 | 203 | ## Security 204 | 205 | Each node exposes a CSI server via a socket to Kubernetes: access to such a 206 | socket grants direct access to any cluster volume. The same holds true for the 207 | RPC data volume which is used for inter-node communication. 208 | 209 | ## Disclaimer 210 | 211 | This is not an officially supported Google product. 212 | -------------------------------------------------------------------------------- /build/docker/driverd/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.20 AS builder 16 | 17 | RUN apt-get update \ 18 | && apt-get install -y protobuf-compiler gettext-base 19 | 20 | ADD . /src 21 | WORKDIR /src 22 | RUN go get github.com/golang/protobuf/protoc-gen-go google.golang.org/grpc/cmd/protoc-gen-go-grpc github.com/golang/mock/mockgen \ 23 | && go install github.com/golang/protobuf/protoc-gen-go google.golang.org/grpc/cmd/protoc-gen-go-grpc github.com/golang/mock/mockgen 24 | ARG VERSION COMMIT 25 | RUN make VERSION=$VERSION COMMIT=$COMMIT cmd/driverd/driverd 26 | 27 | FROM ubuntu:18.04 28 | 29 | RUN apt-get update \ 30 | && apt-get install -y file util-linux e2fsprogs lvm2 \ 31 | && apt-get clean 32 | 33 | RUN ln -s /proc/mounts /etc/mtab 34 | COPY --from=builder /src/cmd/driverd/driverd /driverd 35 | 36 | ENTRYPOINT ["/driverd"] 37 | -------------------------------------------------------------------------------- /build/docker/lvmctrld/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.20 AS builder 16 | 17 | RUN apt-get update \ 18 | && apt-get install -y protobuf-compiler gettext-base 19 | 20 | ADD . /src 21 | WORKDIR /src 22 | RUN go get github.com/golang/protobuf/protoc-gen-go google.golang.org/grpc/cmd/protoc-gen-go-grpc github.com/golang/mock/mockgen \ 23 | && go install github.com/golang/protobuf/protoc-gen-go google.golang.org/grpc/cmd/protoc-gen-go-grpc github.com/golang/mock/mockgen 24 | ARG VERSION COMMIT 25 | RUN make VERSION=$VERSION COMMIT=$COMMIT cmd/lvmctrld/lvmctrld 26 | 27 | FROM ubuntu:18.04 28 | 29 | RUN apt-get update \ 30 | && apt-get install -y file util-linux e2fsprogs lvm2 sanlock lvm2-lockd \ 31 | && apt-get clean 32 | 33 | COPY ./configs/lvmctrld/lvm.conf etc/lvm/lvm.conf 34 | RUN ln -s /proc/mounts /etc/mtab 35 | COPY --from=builder /src/cmd/lvmctrld/lvmctrld /lvmctrld 36 | 37 | ENTRYPOINT ["/lvmctrld"] 38 | -------------------------------------------------------------------------------- /cmd/driverd/driverd.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aleofreddi/csi-sanlock-lvm/a6595c97f862ae7aeaee4bd39fb2719de071339b/cmd/driverd/driverd.bin -------------------------------------------------------------------------------- /cmd/driverd/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "flag" 19 | "fmt" 20 | "os" 21 | 22 | driverd "github.com/aleofreddi/csi-sanlock-lvm/pkg/driverd" 23 | "k8s.io/klog" 24 | ) 25 | 26 | var ( 27 | drvName = flag.String("driver-name", "csi-lvm-sanlock.vleo.net", "driverName of the driver") 28 | listen = flag.String("listen", "unix:///var/run/csi.sock", "listen address") 29 | lvmctrld = flag.String("lvmctrld", "unix:///var/run/lvmctrld.sock", "lvmctrld address") 30 | nodeName = flag.String("node-name", "", "node name") 31 | defaultFs = flag.String("default-fs", "ext4", "default filesystem to use when none is specified") 32 | version string 33 | commit string 34 | ) 35 | 36 | func main() { 37 | klog.InitFlags(nil) 38 | flag.Parse() 39 | klog.Infof("Starting driverd %s (%s)", version, commit) 40 | 41 | listener, err := bootstrap() 42 | if err != nil { 43 | klog.Errorf("Bootstrap failed: %v", err) 44 | os.Exit(2) 45 | } 46 | if err = listener.Run(); err != nil { 47 | klog.Errorf("Execution failed: %v", err) 48 | os.Exit(3) 49 | } 50 | os.Exit(0) 51 | } 52 | 53 | func bootstrap() (*driverd.Listener, error) { 54 | // Start lvmctrld client. 55 | client, err := driverd.NewLvmCtrldClient(*lvmctrld) 56 | if err != nil { 57 | return nil, fmt.Errorf("failed to instance lvmctrld client: %v", err) 58 | } 59 | // Wait for lvmctrld to be ready. 60 | if err := client.Wait(); err != nil { 61 | return nil, fmt.Errorf("lvmctrld startup failed: %v", err) 62 | } 63 | 64 | // Retrieve hostname. 65 | var node string 66 | if *nodeName != "" { 67 | node = *nodeName 68 | } else { 69 | node, err = os.Hostname() 70 | if err != nil { 71 | return nil, fmt.Errorf("failed to retrieve hostname: %v", err) 72 | } 73 | } 74 | // Start lock. 75 | vl, err := driverd.NewVolumeLocker(client, node) 76 | if err != nil { 77 | return nil, fmt.Errorf("failed to instance volume lock: %v", err) 78 | } 79 | // Start DiskRPC 80 | drpc, err := driverd.NewDiskRpcService(client, vl) 81 | if err != nil { 82 | return nil, fmt.Errorf("failed to instance disk rpc service: %v", err) 83 | } 84 | // Instance servers. 85 | is, err := driverd.NewIdentityServer(*drvName, version) 86 | if err != nil { 87 | return nil, fmt.Errorf("failed to instance identity server: %v", err) 88 | } 89 | fsr, err := driverd.NewFileSystemRegistry() 90 | if err != nil { 91 | return nil, fmt.Errorf("failed to instance filesystem registry: %v", err) 92 | } 93 | ns, err := driverd.NewNodeServer(client, vl, fsr) 94 | if err != nil { 95 | return nil, fmt.Errorf("failed to instance identity server: %v", err) 96 | } 97 | cs, err := driverd.NewControllerServer(client, vl, drpc, fsr, *defaultFs) 98 | if err != nil { 99 | return nil, fmt.Errorf("failed to instance controller server: %v", err) 100 | } 101 | // Start DiskRPC 102 | if err := drpc.Start(); err != nil { 103 | return nil, fmt.Errorf("failed to start disk rpc: %v", err) 104 | } 105 | // Start server 106 | listener, err := driverd.NewListener(*listen, is, ns, cs) 107 | if err != nil { 108 | return nil, fmt.Errorf("failed to instance listener: %s", err.Error()) 109 | } 110 | return listener, nil 111 | } 112 | -------------------------------------------------------------------------------- /cmd/lvmctrld/lvmctrld.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aleofreddi/csi-sanlock-lvm/a6595c97f862ae7aeaee4bd39fb2719de071339b/cmd/lvmctrld/lvmctrld.bin -------------------------------------------------------------------------------- /cmd/lvmctrld/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "encoding/binary" 19 | "flag" 20 | "fmt" 21 | "net" 22 | "os" 23 | 24 | "github.com/aleofreddi/csi-sanlock-lvm/pkg/lvmctrld" 25 | "k8s.io/klog" 26 | ) 27 | 28 | var ( 29 | noLock = flag.Bool("no-lock", false, "disable locking, use the given host id. This option is mutually exclusive with lock-host-addr and lock-host-id") 30 | lockAddr = flag.String("lock-host-addr", "", "enable locking, compute host id from the given ip address. This options is mutually exclusive with lock-host-id and no-lock") 31 | lockId = flag.Uint("lock-host-id", 0, "enable locking, use the given host id. This option is mutually exclusive with lock-host-addr and no-lock") 32 | listen = flag.String("listen", "tcp://0.0.0.0:9000", "listen address") 33 | version string 34 | commit string 35 | ) 36 | 37 | func main() { 38 | klog.InitFlags(nil) 39 | flag.Parse() 40 | klog.Infof("Starting lvmctrld %s (%s)", version, commit) 41 | 42 | listener, err := bootstrap() 43 | if err != nil { 44 | klog.Errorf("Bootstrap failed: %v", err) 45 | os.Exit(2) 46 | } 47 | if err = listener.Run(); err != nil { 48 | klog.Errorf("Execution failed: %v", err) 49 | os.Exit(3) 50 | } 51 | os.Exit(0) 52 | } 53 | 54 | func bootstrap() (*lvmctrld.Listener, error) { 55 | if (*noLock == (*lockId != 0)) == ((*lockId != 0) == (*lockAddr != "")) { 56 | return nil, fmt.Errorf("invalid lock configuration, expected one of: no-lock|lock-host-addr|lock-host-id flag") 57 | } 58 | // Start global lock if needed. 59 | var id uint16 60 | var err error 61 | if *noLock { 62 | id = 1 63 | } else { 64 | id, err = parseHostId(*lockId, *lockAddr) 65 | if err != nil { 66 | return nil, fmt.Errorf("failed to parse host id: %v", err) 67 | } 68 | if err = lvmctrld.StartLock(id, []string{}); err != nil { 69 | return nil, fmt.Errorf("failed to start lock: %v", err) 70 | } 71 | } 72 | // Start server. 73 | listener, err := lvmctrld.NewListener(*listen, id) 74 | if err != nil { 75 | return nil, fmt.Errorf("failed to instance listener: %v", err) 76 | } 77 | if err = listener.Init(); err != nil { 78 | return nil, fmt.Errorf("failed to initialize listener: %v", err) 79 | } 80 | return listener, nil 81 | } 82 | 83 | func parseHostId(hostId uint, hostAddr string) (uint16, error) { 84 | if hostId != 0 { 85 | if hostId > 2000 { 86 | return 0, fmt.Errorf("invalid host id %d, expected a value in range [1, 2000]", hostId) 87 | } 88 | return uint16(hostId), nil 89 | } 90 | id, err := addressToHostId(hostAddr) 91 | if err != nil { 92 | return 0, fmt.Errorf("invalid address: %v", err) 93 | } 94 | return id, nil 95 | } 96 | 97 | func addressToHostId(ip string) (uint16, error) { 98 | address := net.ParseIP(ip) 99 | if address == nil { 100 | return 0, fmt.Errorf("invalid ip address: %s", ip) 101 | } 102 | return uint16((binary.BigEndian.Uint32(address[len(address)-4:])&0x7ff)%1999 + 1), nil 103 | } 104 | -------------------------------------------------------------------------------- /conf/csi-sanlock-lvm-config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: csi-sanlock-lvm 19 | -------------------------------------------------------------------------------- /conf/csi-sanlock-lvm-storageclass.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: storage.k8s.io/v1 16 | kind: StorageClass 17 | metadata: 18 | name: csi-sanlock-lvm-storageclass 19 | annotations: 20 | # Use this class as default one. 21 | storageclass.kubernetes.io/is-default-class: "true" 22 | provisioner: csi-sanlock-lvm.csi.vleo.net 23 | parameters: 24 | # Default volume group to use. 25 | volumeGroup: vg01 26 | fsType: ext4 27 | reclaimPolicy: Delete 28 | volumeBindingMode: Immediate 29 | allowVolumeExpansion: true 30 | -------------------------------------------------------------------------------- /conf/csi-sanlock-lvm-volumesnapshotclass.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: snapshot.storage.k8s.io/v1beta1 16 | kind: VolumeSnapshotClass 17 | metadata: 18 | name: csi-sanlock-lvm-snapclass 19 | annotations: 20 | # Use this class as default one. 21 | snapshot.storage.kubernetes.io/is-default-class: "true" 22 | driver: csi-sanlock-lvm.csi.vleo.net 23 | parameters: 24 | # Snapshot maximum size in bytes. 25 | #maxSize: "21474836480" # 20 GiB 26 | # Snapshot maximum size as a percentage of the origin size as a value in (0, 1]. 27 | maxSizePct: "0.2" 28 | deletionPolicy: Delete 29 | -------------------------------------------------------------------------------- /csi-sanity-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | die() { 18 | echo "ERROR: $@" >&2 19 | exit 10 20 | } 21 | 22 | me="$(basename $0)" 23 | rootdir="$(dirname $0)" 24 | rollback=: 25 | verbosity=3 26 | 27 | while getopts "hv:" arg; do 28 | case $arg in 29 | v) 30 | verbosity="$OPTARG" 31 | ;; 32 | *) 33 | echo Usage: "$(basename "$0")" [-v verbosity] [-- csi-sanity-params] >&2 34 | exit 1 35 | ;; 36 | esac 37 | done 38 | shift $((OPTIND-1)) 39 | 40 | [[ "$(uname | tr '[A-Z]' '[a-z]')" = linux ]] || die "$me requires a Linux machine" 41 | 42 | waitForSocket() { 43 | while [[ ! -r "$1" ]]; do 44 | printf . 45 | sleep 1 46 | done 47 | echo 48 | } 49 | 50 | for i in \ 51 | 'csi-sanity:install using `go get github.com/kubernetes-csi/csi-test/cmd/csi-sanity`' \ 52 | 'fallocate:install the proper package' \ 53 | 'losetup:install the proper package' \ 54 | 'pvcreate:install lvm tools' \ 55 | 'vgcreate:install lvm tools' \ 56 | './cmd/lvmctrld/lvmctrld:run "make" to build the driver' \ 57 | './cmd/driverd/driverd:run "make" to build the driver' \ 58 | ; do 59 | IFS=: read f d <<<"$i" 60 | which "$f" >/dev/null 2>&1 || die "$me requires $f, $d" 61 | done 62 | 63 | tmpdir="$(mktemp -d /tmp/csi-sanity-$$.XXXXX)" || die Failed to allocate a temporary directory 64 | rollback="echo Removing temporary directory \"$tmpdir\"; rm -rf \"$tmpdir\"; $rollback" 65 | trap "(trap '' INT; $rollback)" EXIT 66 | 67 | imgfile="$tmpdir/img" 68 | fallocate -l 1GiB "$imgfile" || die Failed to allocate test image file 69 | 70 | device="$(losetup --show -f "$imgfile")" || die Failed to setup loopback device 71 | rollback="echo Detaching $device; losetup -d $device; $rollback" 72 | trap "(trap '' INT; $rollback)" EXIT 73 | 74 | pvcreate -f "$device" || die Failed to create physical device 75 | 76 | vgcreate -s $((1024*1024))b vg_csi_sanity_$$ "$device" || die Failed to create volume group 77 | rollback="echo Bringing down vg_csi_sanity_$$; vgchange -a n vg_csi_sanity_$$; $rollback" 78 | 79 | lvcreate -L 512b -n rpc-lock --addtag csi-sanlock-lvm.vleo.net/rpcRole=lock vg_csi_sanity_$$ || die Failed to create rpc lock logical volume 80 | lvcreate -L 8m -n rpc-data --addtag csi-sanlock-lvm.vleo.net/rpcRole=data vg_csi_sanity_$$ || die Failed to create rpc data logical volume 81 | 82 | lvmctrld_sock="unix://$tmpdir/lvmctrld.sock" 83 | "$rootdir"/cmd/lvmctrld/lvmctrld --listen "$lvmctrld_sock" --no-lock -v "$verbosity" & 84 | rollback="echo Killing lvmctrld pid $!; kill $! 2>/dev/null; sleep 1; kill -9 $! 2>/dev/null; $rollback" 85 | trap "(trap '' INT; $rollback)" EXIT 86 | echo Waiting for lvmctrld to spin up... 87 | waitForSocket "$tmpdir/lvmctrld.sock" 88 | 89 | driverd_sock="unix://$tmpdir/driverd.sock" 90 | "$rootdir"/cmd/driverd/driverd --lvmctrld "$lvmctrld_sock" --listen "$driverd_sock" -v "$verbosity" & 91 | rollback="echo Killing driverd pid $!; kill $! 2>/dev/null; sleep 1; kill -9 $! 2>/dev/null; $rollback" 92 | trap "(trap '' INT; $rollback)" EXIT 93 | echo Waiting for driverd to spin up... 94 | waitForSocket "$tmpdir/driverd.sock" 95 | 96 | param_file="$tmpdir/params" 97 | cat > "$param_file" < 0; i = i >> 1 { 41 | depth++ 42 | } 43 | nodes := make([]*node, (1< b { 75 | return a 76 | } 77 | return b 78 | } 79 | 80 | func left(pos int) int { 81 | return 2*pos + 1 82 | } 83 | 84 | func right(pos int) int { 85 | return 2*pos + 2 86 | } 87 | 88 | func level(pos int) int { 89 | return int(math.Log2(float64(pos + 1))) 90 | } 91 | 92 | func address(nodes []*node, pos int) (Addr, int32) { 93 | l := level(pos) 94 | size := (len(nodes) + 1) / 2 / (1 << l) 95 | off := (pos - (1<= len(nodes) { 101 | return 102 | } 103 | nodes[pos] = &node{free: free} 104 | initialize(nodes, left(pos), free/2) 105 | initialize(nodes, right(pos), free/2) 106 | } 107 | 108 | func alloc(nodes []*node, pos int, size int32) (Addr, error) { 109 | l := len(nodes) 110 | n := nodes[pos] 111 | if size <= 0 || n.free < size { 112 | return -1, fmt.Errorf("not enough space") 113 | } 114 | rightPos := right(pos) 115 | leftPos := left(pos) 116 | if n.free >= size && (leftPos >= l || nodes[leftPos].free < size) && (rightPos >= l || nodes[rightPos].free < size) { 117 | // Allocate the current node. 118 | n.free = -n.free 119 | v, _ := address(nodes, pos) 120 | return v, nil 121 | } 122 | var v Addr 123 | if nodes[leftPos].free >= size { 124 | v, _ = alloc(nodes, leftPos, size) 125 | } else { 126 | v, _ = alloc(nodes, rightPos, size) 127 | } 128 | n.free = max(max(nodes[leftPos].free, nodes[rightPos].free), 0) 129 | return v, nil 130 | } 131 | 132 | func free(nodes []*node, pos int, size int32, addr Addr) error { 133 | if pos >= len(nodes) { 134 | return fmt.Errorf("invalid address") 135 | } 136 | n := nodes[pos] 137 | if n.free < 0 { 138 | if off, _ := address(nodes, pos); off == addr { 139 | n.free = -n.free 140 | return nil 141 | } 142 | return fmt.Errorf("invalid address") 143 | } 144 | rPos := right(pos) 145 | lPos := left(pos) 146 | if rPos > len(nodes) { 147 | return fmt.Errorf("invalid address") 148 | } 149 | var res error 150 | if off, _ := address(nodes, pos); int32(addr-off) < size/2 { 151 | res = free(nodes, lPos, size>>1, addr) 152 | } else { 153 | res = free(nodes, rPos, size>>1, addr) 154 | } 155 | if res == nil { 156 | rFree := nodes[rPos].free 157 | lFree := nodes[lPos].free 158 | if lFree+rFree == size { 159 | n.free = size 160 | } else { 161 | n.free = max(max(lFree, rFree), 0) 162 | } 163 | } 164 | return res 165 | } 166 | 167 | func dump(nodes []*node, pos int, lvl int, sb *strings.Builder) { 168 | for i := 0; i < lvl; i++ { 169 | sb.WriteRune(' ') 170 | } 171 | addr, size := address(nodes, pos) 172 | sb.WriteString(fmt.Sprintf("%d-%d: #%d %+v", addr, size, pos, nodes[pos])) 173 | sb.WriteRune('\n') 174 | lp := left(pos) 175 | if lp >= len(nodes) { 176 | return 177 | } 178 | dump(nodes, lp, lvl+1, sb) 179 | dump(nodes, right(pos), lvl+1, sb) 180 | } 181 | -------------------------------------------------------------------------------- /pkg/diskrpc/allocator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package diskrpc 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/golang/protobuf/proto" 21 | ) 22 | 23 | func TestAllocator(t *testing.T) { 24 | type allocOp struct { 25 | alloc *int32 26 | free *Addr 27 | want int32 28 | wantErr bool 29 | } 30 | type args struct { 31 | size int32 32 | } 33 | tests := []struct { 34 | name string 35 | args args 36 | ops []allocOp 37 | }{ 38 | { 39 | "Alloc all the space", 40 | args{8}, 41 | []allocOp{ 42 | {alloc: proto.Int32(8), free: nil, want: 0, wantErr: false}, 43 | {alloc: proto.Int32(1), free: nil, want: -1, wantErr: true}, 44 | }, 45 | }, 46 | { 47 | "Fill the tree and free everything", 48 | args{8}, 49 | []allocOp{ 50 | {alloc: proto.Int32(3), free: nil, want: 0, wantErr: false}, 51 | {alloc: proto.Int32(2), free: nil, want: 4, wantErr: false}, 52 | {alloc: proto.Int32(2), free: nil, want: 6, wantErr: false}, 53 | {alloc: proto.Int32(1), free: nil, want: -1, wantErr: true}, 54 | {alloc: nil, free: addrPtr(4), want: 0, wantErr: false}, 55 | {alloc: nil, free: addrPtr(0), want: 0, wantErr: false}, 56 | {alloc: nil, free: addrPtr(6), want: 0, wantErr: false}, 57 | }, 58 | }, 59 | { 60 | "Double Free", 61 | args{8}, 62 | []allocOp{ 63 | {alloc: proto.Int32(3), free: nil, want: 0, wantErr: false}, 64 | {alloc: proto.Int32(2), free: nil, want: 4, wantErr: false}, 65 | {alloc: nil, free: addrPtr(4), want: 0, wantErr: false}, 66 | {alloc: nil, free: addrPtr(4), want: 0, wantErr: true}, 67 | }, 68 | }, 69 | { 70 | "Invalid Free", 71 | args{8}, 72 | []allocOp{ 73 | {alloc: proto.Int32(3), free: nil, want: 0, wantErr: false}, 74 | {alloc: proto.Int32(2), free: nil, want: 4, wantErr: false}, 75 | {alloc: nil, free: addrPtr(1), want: 0, wantErr: true}, 76 | {alloc: nil, free: addrPtr(3), want: 0, wantErr: true}, 77 | }, 78 | }, 79 | } 80 | for _, tt := range tests { 81 | t.Run(tt.name, func(t *testing.T) { 82 | alloc, err := NewAllocatorBySize(tt.args.size) 83 | if err != nil { 84 | t.Fatalf("Setup() error = %v", err) 85 | } 86 | for _, op := range tt.ops { 87 | var res int32 88 | var resAddr Addr 89 | var err error 90 | if op.alloc != nil { 91 | resAddr, err = alloc.Alloc(*op.alloc) 92 | } else { 93 | err = alloc.Free(*op.free) 94 | } 95 | res = int32(resAddr) 96 | if (err != nil) != op.wantErr { 97 | t.Fatalf("Allocation(%+v) error = %v, wantErr %v", op, err, op.wantErr) 98 | } 99 | if res != op.want { 100 | t.Fatalf("Allocation(%+v) got = %v, want %d", op, res, op.want) 101 | } 102 | } 103 | }) 104 | } 105 | } 106 | 107 | func addrPtr(addr int) *Addr { 108 | t := Addr(addr) 109 | return &t 110 | } 111 | -------------------------------------------------------------------------------- /pkg/diskrpc/diskrpc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package diskrpc 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "math" 21 | "reflect" 22 | "sync" 23 | "time" 24 | 25 | pb "github.com/aleofreddi/csi-sanlock-lvm/pkg/proto" 26 | "github.com/golang/protobuf/proto" 27 | "github.com/golang/protobuf/ptypes" 28 | "github.com/google/uuid" 29 | "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" 30 | "google.golang.org/grpc/codes" 31 | "google.golang.org/grpc/status" 32 | "k8s.io/klog" 33 | ) 34 | 35 | // Multiplexer channel for RPC handlers. 36 | type Channel uint8 37 | 38 | // DiskRpc implements a synchronous RPC mechanism. 39 | type DiskRpc interface { 40 | // Register a handler for the given channel. Use a nil handler to unregister 41 | // a target. 42 | Register(channel Channel, handler interface{}) error 43 | 44 | // Handle incoming and outgoing messages. 45 | Handle(ctx context.Context) error 46 | 47 | // Execute an RPC call. 48 | Invoke(ctx context.Context, nodeID MailBoxID, targetID Channel, method string, req proto.Message, res proto.Message) error 49 | } 50 | 51 | // diskRpc implements the DiskRpc semantics on top of a MailBox. 52 | type diskRpc struct { 53 | mailBox MailBox 54 | 55 | // Mutex protects all the fields below. 56 | mutex sync.Mutex 57 | // A map that holds a handler object by channel. 58 | handlers map[Channel]interface{} 59 | // Pending requests, indexed by id. 60 | pending map[uuid.UUID]chan *pb.DiskRpcMessage 61 | // A queue of outgoing messages. 62 | outgoing []*Message 63 | } 64 | 65 | func NewDiskRpc(mb MailBox) (DiskRpc, error) { 66 | dr := &diskRpc{ 67 | mailBox: mb, 68 | handlers: make(map[Channel]interface{}), 69 | pending: make(map[uuid.UUID]chan *pb.DiskRpcMessage), 70 | } 71 | return dr, nil 72 | } 73 | 74 | func (s *diskRpc) Register(channel Channel, handler interface{}) error { 75 | s.mutex.Lock() 76 | defer s.mutex.Unlock() 77 | if handler == nil { 78 | delete(s.handlers, channel) 79 | } else { 80 | s.handlers[channel] = handler 81 | } 82 | return nil 83 | } 84 | 85 | func (s *diskRpc) Invoke(ctx context.Context, nodeID MailBoxID, targetID Channel, method string, req proto.Message, res proto.Message) error { 86 | // Generate request ID. 87 | id, err := uuid.NewRandom() 88 | if err != nil { 89 | return fmt.Errorf("failed to generate a random uuid: %v", err) 90 | } 91 | defer func() { 92 | klog.V(5).Infof("[req=%s] Remote RPC to node %d: %s(%+v) returned", id, nodeID, method, req) 93 | }() 94 | klog.V(5).Infof("[req=%s] Issuing remote RPC to node %d: %s(%+v)", id, nodeID, method, req) 95 | 96 | // Marshal request. 97 | var reqBuf []byte 98 | if req != nil { 99 | if reqBuf, err = proto.Marshal(req); err != nil { 100 | return fmt.Errorf("failed to serialize request to message: %v", err) 101 | } 102 | } 103 | idBuf, err := id.MarshalBinary() 104 | if err != nil { 105 | return fmt.Errorf("failed to serialize request uuid: %v", err) 106 | } 107 | dMsgBuf, err := proto.Marshal(&pb.DiskRpcMessage{ 108 | Time: ptypes.TimestampNow(), 109 | Type: pb.DiskRpcType_DISK_RPC_TYPE_REQUEST, 110 | Uuid: idBuf, 111 | Method: method, 112 | Request: reqBuf, 113 | }) 114 | if err != nil { 115 | return fmt.Errorf("failed to serialize request: %v", err) 116 | } 117 | 118 | // Schedule request. 119 | resCh := make(chan *pb.DiskRpcMessage) 120 | err = func() error { 121 | s.mutex.Lock() 122 | defer s.mutex.Unlock() 123 | // TODO: hardcoded limit! 124 | if len(s.outgoing) > 64 { 125 | return status.Errorf(codes.ResourceExhausted, "too many queued request") 126 | } 127 | s.pending[id] = resCh 128 | s.outgoing = append(s.outgoing, &Message{ 129 | Sender: 0, 130 | Recipient: MailBoxID(nodeID), 131 | Payload: dMsgBuf, 132 | }) 133 | return nil 134 | }() 135 | if err != nil { 136 | return err 137 | } 138 | defer func() { 139 | s.mutex.Lock() 140 | delete(s.pending, id) 141 | s.mutex.Unlock() 142 | }() 143 | 144 | // Get wait timeout duration. 145 | var wTime time.Duration 146 | if tout, ok := ctx.Deadline(); !ok { 147 | wTime = time.Now().Sub(tout) 148 | } else { 149 | // If no explicit deadline is set, we use a standard timeout value. 150 | // TODO: hardcoded timeout! 151 | wTime = 30 * time.Second 152 | } 153 | // Wait for the response. 154 | var dMsg *pb.DiskRpcMessage 155 | select { 156 | case dMsg = <-resCh: 157 | case <-time.After(wTime): 158 | klog.Errorf("[req=%s] Deadline exceeded", id) 159 | return context.DeadlineExceeded 160 | } 161 | 162 | // Unmarshal the message. 163 | var resErr error 164 | if dMsg.GetErrorMsg() != "" || dMsg.GetErrorCode() != 0 { 165 | resErr = status.Error(codes.Code(dMsg.GetErrorCode()), dMsg.GetErrorMsg()) 166 | } 167 | // Unmarshal the response. 168 | if dMsg.GetResponse() != nil { 169 | if err = proto.Unmarshal(dMsg.Response, res); err != nil { 170 | return fmt.Errorf("failed to deserialize response from message: %v", err) 171 | } 172 | } 173 | return resErr 174 | } 175 | 176 | func (s *diskRpc) Handle(ctx context.Context) error { 177 | msgs, err := s.mailBox.Recv() 178 | if err != nil { 179 | return fmt.Errorf("failed to receive messages: %v", err) 180 | } 181 | if len(msgs) > 0 { 182 | klog.V(3).Infof("Handling %d incoming messages", len(msgs)) 183 | } 184 | for _, msg := range msgs { 185 | // Unmarshal the response. 186 | var dMsg pb.DiskRpcMessage 187 | if err = proto.Unmarshal(msg.Payload, &dMsg); err != nil { 188 | return fmt.Errorf("failed to deserialize response from message: %v", err) 189 | } 190 | klog.V(6).Infof("\tIncoming message %+v", decode(msg)) 191 | switch dMsg.Type { 192 | case pb.DiskRpcType_DISK_RPC_TYPE_RESPONSE: 193 | if err := s.handleResponse(msg.Sender, &dMsg); err != nil { 194 | klog.Errorf("Failed to process incoming response: %v", err) 195 | } 196 | case pb.DiskRpcType_DISK_RPC_TYPE_REQUEST: 197 | if err := s.handleRequest(ctx, msg.Sender, &dMsg); err != nil { 198 | klog.Errorf("Failed to process incoming request: %v", err) 199 | } 200 | } 201 | } 202 | if len(msgs) > 0 { 203 | klog.V(3).Infof("Handling %d outgoing messages", len(msgs)) 204 | } 205 | s.mutex.Lock() 206 | defer s.mutex.Unlock() 207 | msgs = s.outgoing 208 | s.outgoing = nil 209 | for _, msg := range msgs { 210 | klog.V(6).Infof("\tOutgoing message %+v", decode(msg)) 211 | if err := s.mailBox.Send(msg); err != nil { 212 | klog.Errorf("Failed to process outgoing message: %v", err) 213 | } 214 | } 215 | return nil 216 | } 217 | 218 | func (s *diskRpc) handleResponse(sender MailBoxID, dMsg *pb.DiskRpcMessage) error { 219 | id, err := uuid.FromBytes(dMsg.Uuid) 220 | if err != nil { 221 | return fmt.Errorf("failed to deserialize response id: %v", err) 222 | } 223 | ch, err := func() (chan *pb.DiskRpcMessage, error) { 224 | s.mutex.Lock() 225 | defer s.mutex.Unlock() 226 | ch, ok := s.pending[id] 227 | if !ok { 228 | return nil, fmt.Errorf("response %s: request not found", id) 229 | } 230 | delete(s.pending, id) 231 | return ch, nil 232 | }() 233 | if err != nil { 234 | return err 235 | } 236 | ch <- dMsg 237 | return nil 238 | } 239 | 240 | func (s *diskRpc) handleRequest(ctx context.Context, sender MailBoxID, dMsg *pb.DiskRpcMessage) error { 241 | id, err := uuid.FromBytes(dMsg.Uuid) 242 | if err != nil { 243 | return fmt.Errorf("failed to deserialize request id: %v", err) 244 | } 245 | ch := dMsg.Channel 246 | if ch < 0 || ch > math.MaxUint8 { 247 | return fmt.Errorf("request %s: invalid channel %d", id, ch) 248 | } 249 | // Resolve method and instance request. 250 | target, ok := s.handlers[Channel(ch)] 251 | if !ok { 252 | return fmt.Errorf("request %s: unknown channel %d", id, ch) 253 | } 254 | m := reflect.ValueOf(target).MethodByName(dMsg.GetMethod()) 255 | if !m.IsValid() { 256 | return fmt.Errorf("request %s: unknown method %q", id, dMsg.GetMethod()) 257 | } 258 | reqType := m.Type().In(1).Elem() 259 | reqObj := reflect.New(reqType).Interface().(proto.Message) 260 | // Unmarshal request. 261 | if err := proto.Unmarshal(dMsg.GetRequest(), reqObj); err != nil { 262 | return fmt.Errorf("failed to deserialize request from message: %v", err) 263 | } 264 | // Invoke method. 265 | klog.V(6).Infof("DiskRpc: calling %s(%+v)...", dMsg.Method, protosanitizer.StripSecrets(reqObj)) 266 | resVal := m.Call([]reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(reqObj)}) 267 | // Encode response. 268 | var resObj proto.Message 269 | var resBuf []byte 270 | if !resVal[0].IsNil() { 271 | resObj := resVal[0].Interface().(proto.Message) 272 | if resBuf, err = proto.Marshal(resObj); err != nil { 273 | return fmt.Errorf("failed to serialize response to message: %v", err) 274 | } 275 | } 276 | var resErrMsg string 277 | var resErrCode uint32 278 | if !resVal[1].IsNil() { 279 | err := resVal[1].Interface().(error) 280 | resErrMsg = err.Error() 281 | resErrCode = uint32(status.Code(err)) 282 | klog.Errorf("DiskRpc: call %s(%+v) returned error %v", dMsg.Method, protosanitizer.StripSecrets(reqObj), err) 283 | } else { 284 | klog.V(5).Infof("DiskRpc: call %s(%+v) returned %+v", dMsg.Method, protosanitizer.StripSecrets(reqObj), protosanitizer.StripSecrets(resObj)) 285 | } 286 | dMsgBuf, err := proto.Marshal(&pb.DiskRpcMessage{ 287 | Time: ptypes.TimestampNow(), 288 | Type: pb.DiskRpcType_DISK_RPC_TYPE_RESPONSE, 289 | Uuid: dMsg.Uuid, 290 | Channel: dMsg.Channel, 291 | Response: resBuf, 292 | ErrorCode: resErrCode, 293 | ErrorMsg: resErrMsg, 294 | }) 295 | if err != nil { 296 | return fmt.Errorf("failed to marshal request: %v", err) 297 | } 298 | s.mutex.Lock() 299 | defer s.mutex.Unlock() 300 | // TODO: hardcoded limit! 301 | if len(s.outgoing) > 4*64 { 302 | return status.Errorf(codes.ResourceExhausted, "too many queued request") 303 | } 304 | s.outgoing = append(s.outgoing, &Message{ 305 | Sender: 0, 306 | Recipient: sender, 307 | Payload: dMsgBuf, 308 | }) 309 | return nil 310 | } 311 | 312 | func decode(msg *Message) string { 313 | var dMsg pb.DiskRpcMessage 314 | if err := proto.Unmarshal(msg.Payload, &dMsg); err != nil { 315 | return "broken message" 316 | } 317 | id, _ := uuid.FromBytes(dMsg.Uuid) 318 | return fmt.Sprintf("%d->%d %s(%s)", msg.Sender, msg.Recipient, dMsg.Type, id) 319 | } 320 | -------------------------------------------------------------------------------- /pkg/diskrpc/diskrpc_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package diskrpc 16 | 17 | // TODO: add tests! 18 | -------------------------------------------------------------------------------- /pkg/diskrpc/mailbox_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package diskrpc 16 | 17 | import ( 18 | "io/ioutil" 19 | "os" 20 | "reflect" 21 | "sync" 22 | "testing" 23 | ) 24 | 25 | func TestMailBoxBasic(t *testing.T) { 26 | file := makeTmpFile(t, "TestMailBox", 1024*1024) 27 | defer os.Remove(file.Name()) 28 | var mutex sync.Mutex 29 | mbox1, err := NewMailBox(1, &mutex, file.Name()) 30 | if err != nil { 31 | t.Fatalf("failed to initialize mailbox: %v", err) 32 | } 33 | mbox2, err := NewMailBox(2, &mutex, file.Name()) 34 | if err != nil { 35 | t.Fatalf("failed to initialize mailbox: %v", err) 36 | } 37 | 38 | err = mbox1.Send(&Message{Recipient: 2, Payload: []byte{1, 2, 3, 4, 5}}) 39 | if err != nil { 40 | t.Fatalf("failed to send message: %v", err) 41 | } 42 | err = mbox1.Send(&Message{Recipient: 2, Payload: []byte{5, 4, 3, 2, 1}}) 43 | if err != nil { 44 | t.Fatalf("failed to send message: %v", err) 45 | } 46 | msgs, err := mbox1.Recv() 47 | if err != nil { 48 | t.Fatalf("failed to receive messages: %v", err) 49 | } 50 | if len(msgs) > 0 { 51 | t.Fatalf("got = %v, want 0", len(msgs)) 52 | } 53 | msgs, err = mbox2.Recv() 54 | if err != nil { 55 | t.Fatalf("failed to receive messages: %v", err) 56 | } 57 | if len(msgs) != 2 { 58 | t.Fatalf("got = %v, want 0", len(msgs)) 59 | } 60 | } 61 | 62 | var testPayload = []byte{1, 2, 3, 4, 5} 63 | 64 | func TestMailBoxSelf(t *testing.T) { 65 | file := makeTmpFile(t, "TestMailBox", 1024*1024) 66 | defer os.Remove(file.Name()) 67 | var mutex sync.Mutex 68 | mbox1, err := NewMailBox(1, &mutex, file.Name()) 69 | if err != nil { 70 | t.Fatalf("failed to initialize mailbox: %v", err) 71 | } 72 | 73 | err = mbox1.Send(&Message{Recipient: 1, Payload: []byte{1, 2, 3, 4, 5}}) 74 | if err != nil { 75 | t.Fatalf("failed to send message: %v", err) 76 | } 77 | msgs, err := mbox1.Recv() 78 | if err != nil { 79 | t.Fatalf("failed to receive messages: %v", err) 80 | } 81 | 82 | want := []*Message{ 83 | &Message{Sender: 1, Recipient: 1, Payload: testPayload}, 84 | } 85 | if !reflect.DeepEqual(msgs, want) { 86 | t.Fatalf("got = %+v, want %+v", msgs, want) 87 | } 88 | } 89 | 90 | func TestMailBoxFill(t *testing.T) { 91 | file := makeTmpFile(t, "TestMailBox", 1024*1024) 92 | defer os.Remove(file.Name()) 93 | var mutex sync.Mutex 94 | mbox1, err := NewMailBox(1, &mutex, file.Name()) 95 | if err != nil { 96 | t.Fatalf("failed to initialize mailbox: %v", err) 97 | } 98 | mbox2, err := NewMailBox(2, &mutex, file.Name()) 99 | if err != nil { 100 | t.Fatalf("failed to initialize mailbox: %v", err) 101 | } 102 | 103 | // Fill the mailbox. 104 | cnt := 0 105 | for { 106 | err = mbox1.Send(&Message{Recipient: 2, Payload: []byte{1, 2, 3, 4, 5}}) 107 | if err != nil { 108 | break 109 | } 110 | cnt++ 111 | } 112 | // Expect mbox2 to fail message Send: mailbox is full. 113 | err = mbox2.Send(&Message{Recipient: 2, Payload: []byte{1, 2, 3, 4, 5}}) 114 | if err == nil { 115 | t.Fatalf("got = %v, want an error", err) 116 | } 117 | // Consume messages. 118 | msgs, err := mbox2.Recv() 119 | if err != nil { 120 | t.Fatalf("failed to receive messages: %v", err) 121 | } 122 | if len(msgs) == 0 { 123 | t.Fatalf("got = %v, want >0", len(msgs)) 124 | } 125 | // Expect mbox1 Send to succed. 126 | err = mbox1.Send(&Message{Recipient: 2, Payload: []byte{1, 2, 3, 4, 5}}) 127 | if err != nil { 128 | t.Fatalf("got = %v, want no error", err) 129 | } 130 | } 131 | 132 | func makeTmpFile(t *testing.T, id string, size int) *os.File { 133 | file, err := ioutil.TempFile("", "csi-sanlock-lvm."+id+".*.data") 134 | if err != nil { 135 | t.Fatalf("failed to get temp file: %v", err) 136 | } 137 | if err = file.Truncate(int64(size)); err != nil { 138 | t.Fatalf("failed to initialize temporary file: %v", err) 139 | } 140 | return file 141 | } 142 | -------------------------------------------------------------------------------- /pkg/driverd/baseserver.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | pb "github.com/aleofreddi/csi-sanlock-lvm/pkg/proto" 22 | "google.golang.org/grpc/codes" 23 | "google.golang.org/grpc/status" 24 | ) 25 | 26 | // Common functionalities for all servers. 27 | type baseServer struct { 28 | lvmctrld pb.LvmCtrldClient 29 | nodeID uint16 30 | } 31 | 32 | func newBaseServer(lvmctrld pb.LvmCtrldClient) (*baseServer, error) { 33 | st, err := lvmctrld.GetStatus(context.Background(), &pb.GetStatusRequest{}) 34 | if err != nil { 35 | return nil, fmt.Errorf("failed to retrieve status from lvmctrld: %v", err) 36 | } 37 | return &baseServer{ 38 | nodeID: uint16(st.NodeId), 39 | lvmctrld: lvmctrld, 40 | }, nil 41 | } 42 | 43 | // Fetch volume info from a volume reference. 44 | func (bs *baseServer) fetch(ctx context.Context, ref *VolumeRef) (*VolumeInfo, error) { 45 | lvs, err := bs.lvmctrld.Lvs(ctx, &pb.LvsRequest{ 46 | Target: []string{ref.VgLv()}, 47 | }) 48 | if status.Code(err) == codes.NotFound { 49 | return nil, status.Errorf(codes.NotFound, "volume %s not found", ref) 50 | } else if err != nil { 51 | return nil, status.Errorf(codes.Internal, "failed to fetch volume %s: %v", ref, err) 52 | } 53 | return NewVolumeInfoFromLv(lvs.Lvs[0]), nil 54 | } 55 | -------------------------------------------------------------------------------- /pkg/driverd/cmpmatcher_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd_test 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/golang/mock/gomock" 22 | "github.com/google/go-cmp/cmp" 23 | ) 24 | 25 | type cmpMatcher struct { 26 | t *testing.T 27 | want interface{} 28 | opts []cmp.Option 29 | } 30 | 31 | func CmpMatcher(t *testing.T, want interface{}, opts ...cmp.Option) gomock.Matcher { 32 | return &cmpMatcher{t, want, opts} 33 | } 34 | 35 | func (m *cmpMatcher) Matches(actual interface{}) bool { 36 | return cmp.Equal(m.want, actual, m.opts...) 37 | } 38 | 39 | func (m *cmpMatcher) String() string { 40 | return fmt.Sprintf("is equal to %v using %s", m.want, cmp.Options(m.opts).String()) 41 | } 42 | -------------------------------------------------------------------------------- /pkg/driverd/diskrpcservice.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | diskrpc "github.com/aleofreddi/csi-sanlock-lvm/pkg/diskrpc" 23 | pb "github.com/aleofreddi/csi-sanlock-lvm/pkg/proto" 24 | "k8s.io/klog" 25 | ) 26 | 27 | // DiskRpcService adapts DiskRpc to use volume locking. 28 | type DiskRpcService struct { 29 | baseServer 30 | diskrpc.DiskRpc 31 | mailBox diskrpc.MailBox 32 | } 33 | 34 | type RpcRole string 35 | 36 | const ( 37 | rpcRoleLock RpcRole = "lock" 38 | rpcRoleData RpcRole = "data" 39 | 40 | diskRpcOp = "diskRpc" 41 | 42 | maxRpcRetryDelay = 120 * time.Second 43 | ) 44 | 45 | type volumeLockerAdapter struct { 46 | locker VolumeLocker 47 | ctx context.Context 48 | vol VolumeRef 49 | } 50 | 51 | func (vl *volumeLockerAdapter) Lock() { 52 | for delay := 1 * time.Second; ; { 53 | err := vl.locker.LockVolume(vl.ctx, vl.vol, diskRpcOp) 54 | if err == nil { 55 | return 56 | } 57 | klog.Warningf("Failed to acquire RPC lock (retry in %s): %v", delay, err) 58 | time.Sleep(delay) 59 | if delay < maxRpcRetryDelay { 60 | delay *= 2 61 | } 62 | } 63 | } 64 | 65 | func (vl *volumeLockerAdapter) Unlock() { 66 | for delay := time.Duration(1); ; { 67 | err := vl.locker.UnlockVolume(vl.ctx, vl.vol, diskRpcOp) 68 | if err == nil { 69 | return 70 | } 71 | klog.Warningf("Failed to release RPC lock (retry in %s): %v", delay, err) 72 | time.Sleep(delay) 73 | if delay < maxRpcRetryDelay { 74 | delay *= 2 75 | } 76 | } 77 | } 78 | 79 | func NewDiskRpcService(lvmctrld pb.LvmCtrldClient, locker VolumeLocker) (*DiskRpcService, error) { 80 | bs, err := newBaseServer(lvmctrld) 81 | if err != nil { 82 | return nil, err 83 | } 84 | // Find rpc lock and data logical volumes. 85 | ctx := context.Background() 86 | lvs, err := lvmctrld.Lvs(ctx, &pb.LvsRequest{ 87 | Select: fmt.Sprintf("lv_tags=%s || lv_tags=%s", 88 | encodeTagKV(rpcRoleTagKey, string(rpcRoleData)), 89 | encodeTagKV(rpcRoleTagKey, string(rpcRoleLock))), 90 | }) 91 | if err != nil { 92 | return nil, fmt.Errorf("failed to list logical volumes: %v", err) 93 | } 94 | var data *VolumeInfo 95 | var lock *VolumeInfo 96 | for _, lv := range lvs.Lvs { 97 | lvRef := NewVolumeInfoFromLv(lv) 98 | tags, err := lvRef.Tags() 99 | if err != nil { 100 | return nil, fmt.Errorf("failed to decode tags for volume %q: %v", lvRef, err) 101 | } 102 | role := RpcRole(tags[rpcRoleTagKey]) 103 | if role == rpcRoleLock { 104 | if lock != nil { 105 | return nil, fmt.Errorf("found multiple logical volumes tagged with the %s RPC role", rpcRoleLock) 106 | } 107 | lock = lvRef 108 | } else if role == rpcRoleData { 109 | if data != nil { 110 | return nil, fmt.Errorf("found multiple logical volumes tagged with the %s RPC role", rpcRoleData) 111 | } 112 | data = lvRef 113 | } 114 | } 115 | if lock == nil || data == nil { 116 | return nil, fmt.Errorf("missing lock or data logical volumes (expected [%s%s, %s%s] tags)", 117 | encodeTagKeyPrefix(rpcRoleTagKey), rpcRoleData, 118 | encodeTagKeyPrefix(rpcRoleTagKey), rpcRoleLock) 119 | } 120 | // Activate data as shared. 121 | _, err = lvmctrld.LvChange(ctx, &pb.LvChangeRequest{ 122 | Target: []string{data.VgLv()}, 123 | Activate: pb.LvActivationMode_LV_ACTIVATION_MODE_ACTIVE_SHARED, 124 | }) 125 | if err != nil { 126 | return nil, fmt.Errorf("failed to activate rpc data logical volume: %v", err) 127 | } 128 | // Instance mailbox. 129 | mb, err := diskrpc.NewMailBox( 130 | diskrpc.MailBoxID(bs.nodeID), 131 | &volumeLockerAdapter{ 132 | ctx: ctx, 133 | locker: locker, 134 | vol: lock.VolumeRef, 135 | }, 136 | data.DevPath(), 137 | ) 138 | if err != nil { 139 | return nil, fmt.Errorf("failed to instance mailbox: %v", err) 140 | } 141 | dRpc, err := diskrpc.NewDiskRpc(mb) 142 | if err != nil { 143 | return nil, fmt.Errorf("failed to instance diskrpc: %v", err) 144 | } 145 | return &DiskRpcService{ 146 | baseServer: *bs, 147 | DiskRpc: dRpc, 148 | mailBox: mb, 149 | }, nil 150 | } 151 | 152 | func (s *DiskRpcService) Start() error { 153 | go func() { 154 | ctx := context.Background() 155 | for { 156 | err := s.Handle(ctx) 157 | if err != nil { 158 | klog.Errorf("DiskRpc failed to handle messages: %v", err) 159 | } 160 | select { 161 | //case <-done: return // FIXME! 162 | case <-time.After(1 * time.Second): 163 | } 164 | } 165 | }() 166 | return nil 167 | } 168 | -------------------------------------------------------------------------------- /pkg/driverd/filesystem.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "bytes" 19 | "google.golang.org/grpc/codes" 20 | "google.golang.org/grpc/status" 21 | "k8s.io/utils/mount" 22 | "os" 23 | "os/exec" 24 | "path/filepath" 25 | ) 26 | 27 | // Action to be taken when mountpoint does not exist. 28 | type mountPointAction int 29 | 30 | const ( 31 | requireExisting mountPointAction = 0 32 | createFile = 1 33 | createDirectory = 2 34 | ) 35 | 36 | type FileSystemRegistry interface { 37 | GetFileSystem(filesystem string) (FileSystem, error) 38 | } 39 | 40 | type FileSystem interface { 41 | Accepts(accessType VolumeAccessType) bool 42 | Make(device string) error 43 | Grow(device string) error 44 | Stage(device, stagePoint string, flags []string, grpID *int) error 45 | Unstage(stagePoint string) error 46 | Publish(device, stagePoint, mountPoint string, readOnly bool) error 47 | Unpublish(mountPoint string) error 48 | } 49 | 50 | type fileSystemRegistry struct { 51 | } 52 | 53 | type rawFilesystem struct { 54 | } 55 | 56 | type fileSystem struct { 57 | fileSystem string 58 | } 59 | 60 | func NewFileSystemRegistry() (*fileSystemRegistry, error) { 61 | return &fileSystemRegistry{}, nil 62 | } 63 | 64 | func (fr *fileSystemRegistry) GetFileSystem(filesystem string) (FileSystem, error) { 65 | return NewFileSystem(filesystem) 66 | } 67 | 68 | func NewFileSystem(fs string) (FileSystem, error) { 69 | if fs == BlockAccessFsName { 70 | return &rawFilesystem{}, nil 71 | } 72 | return &fileSystem{fs}, nil 73 | } 74 | 75 | func (fs *fileSystem) Make(device string) error { 76 | mkfs := exec.Command("mkfs", "-t", fs.fileSystem, device) 77 | stdout, stderr := new(bytes.Buffer), new(bytes.Buffer) 78 | mkfs.Stdout = stdout 79 | mkfs.Stderr = stderr 80 | if mkfs.Run() != nil { 81 | return status.Errorf(codes.Internal, "failed to format volume %s: %s %s", device, stdout.String(), stderr.String()) 82 | } 83 | return nil 84 | } 85 | 86 | func (fs *fileSystem) Grow(device string) error { 87 | checkfs := exec.Command("fsadm", "check", device) 88 | stdout, stderr := new(bytes.Buffer), new(bytes.Buffer) 89 | checkfs.Stdout = stdout 90 | checkfs.Stderr = stderr 91 | // 'fsadm check' can return code 3 when the requested check operation could 92 | // not be performed because the filesystem is mounted and does not support an 93 | // online fsck. 94 | if err := checkfs.Run(); err != nil && checkfs.ProcessState.ExitCode() != 3 { 95 | return status.Errorf(codes.Internal, "failed to check volume %s: %v (%s %s)", device, err, stdout.String(), stderr.String()) 96 | } 97 | resize := exec.Command("fsadm", "resize", device) 98 | stdout, stderr = new(bytes.Buffer), new(bytes.Buffer) 99 | resize.Stdout = stdout 100 | resize.Stderr = stderr 101 | if err := resize.Run(); err != nil { 102 | return status.Errorf(codes.Internal, "failed to resize volume %s: %v (%s %s)", device, err, stdout.String(), stderr.String()) 103 | } 104 | return nil 105 | } 106 | 107 | func (fs *fileSystem) Accepts(accessType VolumeAccessType) bool { 108 | return accessType == MountAccessType 109 | } 110 | 111 | func (fs *fileSystem) Stage(device, stagePoint string, flags []string, grpID *int) error { 112 | err := mountFs(device, stagePoint, fs.fileSystem, flags, requireExisting) 113 | if err != nil { 114 | return err 115 | } 116 | if grpID != nil { 117 | if err := grantGroupAccess(stagePoint, *grpID); err != nil { 118 | return status.Errorf(codes.Internal, "failed to grant group access for volume %s: %v", device, err) 119 | } 120 | } 121 | return nil 122 | } 123 | 124 | func (fs *fileSystem) Unstage(mountPoint string) error { 125 | return umountFs(mountPoint, false) 126 | } 127 | 128 | func (fs *fileSystem) Publish(device, stagePoint, mountPoint string, readOnly bool) error { 129 | flags := []string{"bind"} 130 | if readOnly { 131 | flags = append(flags, "ro") 132 | } 133 | return mountFs(stagePoint, mountPoint, "", flags, createDirectory) 134 | } 135 | 136 | func (fs *fileSystem) Unpublish(mountPoint string) error { 137 | return umountFs(mountPoint, true) 138 | } 139 | 140 | func (fs *rawFilesystem) Make(_ string) error { 141 | return nil 142 | } 143 | 144 | func (fs *rawFilesystem) Grow(_ string) error { 145 | return nil 146 | } 147 | 148 | func (fs *rawFilesystem) Accepts(accessType VolumeAccessType) bool { 149 | return accessType == BlockAccessType 150 | } 151 | 152 | func (fs *rawFilesystem) Stage(device, stagePoint string, flags []string, grpID *int) error { 153 | if grpID != nil { 154 | if err := grantGroupAccess(device, *grpID); err != nil { 155 | return status.Errorf(codes.Internal, "failed to grant group access for volume %s: %v", device, err) 156 | } 157 | } 158 | return nil 159 | } 160 | 161 | func (fs *rawFilesystem) Unstage(mountPoint string) error { 162 | return nil 163 | } 164 | 165 | func (fs *rawFilesystem) Publish(device, stagePoint, mountPoint string, readOnly bool) error { 166 | flags := []string{"bind"} 167 | if readOnly { 168 | flags = append(flags, "ro") 169 | } 170 | return mountFs(device, mountPoint, "", flags, createFile) 171 | } 172 | 173 | func (fs *rawFilesystem) Unpublish(mountPoint string) error { 174 | return umountFs(mountPoint, true) 175 | } 176 | 177 | func grantGroupAccess(path string, groupID int) error { 178 | // Search for files 179 | err := filepath.WalkDir(path, func(file string, d os.DirEntry, err error) error { 180 | return os.Chown(file, -1, groupID) 181 | }) 182 | if err != nil { 183 | return err 184 | } 185 | // Ensure root has full group access. 186 | if err := os.Chmod(path, os.FileMode(0770)); err != nil { 187 | return err 188 | } 189 | return nil 190 | } 191 | 192 | func mountFs(source, mountPoint, fsName string, flags []string, mpAction mountPointAction) error { 193 | mounter := mount.New("") 194 | notMounted, err := mounter.IsLikelyNotMountPoint(mountPoint) 195 | if err != nil { 196 | if !os.IsNotExist(err) { 197 | return status.Errorf(codes.Internal, "failed to determine if %s is mounted: %v", mountPoint, err) 198 | } 199 | switch mpAction { 200 | case requireExisting: 201 | return status.Errorf(codes.Internal, "%s does not exist", mountPoint) 202 | case createFile: 203 | file, err := os.OpenFile(mountPoint, os.O_CREATE, os.FileMode(0640)) 204 | if err = file.Close(); err != nil { 205 | return err 206 | } 207 | if err != nil { 208 | if !os.IsExist(err) { 209 | return status.Errorf(codes.Internal, "failed to create file %s: %v", mountPoint, err) 210 | } 211 | } 212 | case createDirectory: 213 | if err := os.MkdirAll(mountPoint, 0750); err != nil { 214 | return status.Errorf(codes.Internal, "failed to mkdir %s: %v", mountPoint, err) 215 | } 216 | } 217 | notMounted = true 218 | } 219 | 220 | if notMounted { 221 | // Mount the filesystem. 222 | err = mounter.Mount(source, mountPoint, fsName, flags) 223 | if err != nil { 224 | return status.Errorf(codes.Internal, "failed to mount: %v", err) 225 | } 226 | } 227 | return nil 228 | } 229 | 230 | func umountFs(targetPath string, deleteMountPoint bool) error { 231 | mounter := mount.New("") 232 | notMounted, err := mounter.IsLikelyNotMountPoint(targetPath) 233 | if err != nil && err == os.ErrNotExist { 234 | return nil 235 | } 236 | if !notMounted { 237 | err = mounter.Unmount(targetPath) 238 | if err != nil { 239 | return status.Errorf(codes.Internal, "failed to unmount %q: %s", targetPath, err.Error()) 240 | } 241 | } 242 | if deleteMountPoint { 243 | if err = os.RemoveAll(targetPath); err != nil { 244 | return status.Errorf(codes.Internal, "failed to remove %q: %s", targetPath, err.Error()) 245 | } 246 | } 247 | return nil 248 | } 249 | -------------------------------------------------------------------------------- /pkg/driverd/identityserver.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "github.com/container-storage-interface/spec/lib/go/csi" 19 | "golang.org/x/net/context" 20 | "google.golang.org/grpc/codes" 21 | "google.golang.org/grpc/status" 22 | ) 23 | 24 | type identityServer struct { 25 | drvName string 26 | version string 27 | } 28 | 29 | func NewIdentityServer(drvName, version string) (*identityServer, error) { 30 | if drvName == "" { 31 | return nil, status.Error(codes.Unavailable, "missing driver name") 32 | } 33 | if version == "" { 34 | return nil, status.Error(codes.Unavailable, "missing driver version") 35 | } 36 | return &identityServer{ 37 | drvName: drvName, 38 | version: version, 39 | }, nil 40 | } 41 | 42 | func (is *identityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { 43 | return &csi.GetPluginInfoResponse{ 44 | Name: is.drvName, 45 | VendorVersion: is.version, 46 | }, nil 47 | } 48 | 49 | func (is *identityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { 50 | return &csi.GetPluginCapabilitiesResponse{ 51 | Capabilities: []*csi.PluginCapability{ 52 | { 53 | Type: &csi.PluginCapability_Service_{ 54 | Service: &csi.PluginCapability_Service{ 55 | Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, 56 | }, 57 | }, 58 | }, 59 | { 60 | Type: &csi.PluginCapability_Service_{ 61 | Service: &csi.PluginCapability_Service{ 62 | Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS, 63 | }, 64 | }, 65 | }, 66 | { 67 | Type: &csi.PluginCapability_VolumeExpansion_{ 68 | VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ 69 | Type: csi.PluginCapability_VolumeExpansion_ONLINE, 70 | }, 71 | }, 72 | }, 73 | }, 74 | }, nil 75 | } 76 | 77 | func (is *identityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { 78 | return &csi.ProbeResponse{}, nil 79 | } 80 | -------------------------------------------------------------------------------- /pkg/driverd/listener.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "fmt" 19 | "net" 20 | "net/url" 21 | "os" 22 | 23 | logger "github.com/aleofreddi/csi-sanlock-lvm/pkg/grpclogger" 24 | "github.com/container-storage-interface/spec/lib/go/csi" 25 | "google.golang.org/grpc" 26 | "k8s.io/klog" 27 | ) 28 | 29 | type Listener struct { 30 | addr string 31 | 32 | is *identityServer 33 | ns *nodeServer 34 | cs *controllerServer 35 | } 36 | 37 | func NewListener(addr string, is *identityServer, ns *nodeServer, cs *controllerServer) (*Listener, error) { 38 | if addr == "" { 39 | return nil, fmt.Errorf("missing listen address") 40 | } 41 | return &Listener{ 42 | addr: addr, 43 | is: is, 44 | ns: ns, 45 | cs: cs, 46 | }, nil 47 | } 48 | 49 | func (l *Listener) Run() error { 50 | // Start gRPC server 51 | lsProto, lsAddr, err := parseAddress(l.addr) 52 | if err != nil { 53 | return fmt.Errorf("invalid listen address: %s", err.Error()) 54 | } 55 | if lsProto == "unix" { 56 | if err := os.Remove(lsAddr); err != nil && !os.IsNotExist(err) { 57 | return fmt.Errorf("failed to remove %s: %s", lsAddr, err.Error()) 58 | } 59 | } 60 | klog.Infof("Binding proto %s, address %s", lsProto, lsAddr) 61 | listener, err := net.Listen(lsProto, lsAddr) 62 | if err != nil { 63 | return fmt.Errorf("failed to listen %s://%s: %s", lsProto, lsAddr, err.Error()) 64 | } 65 | opts := []grpc.ServerOption{ 66 | grpc.UnaryInterceptor(logger.GrpcLogger), 67 | } 68 | grpcServer := grpc.NewServer(opts...) 69 | csi.RegisterIdentityServer(grpcServer, l.is) 70 | csi.RegisterNodeServer(grpcServer, l.ns) 71 | csi.RegisterControllerServer(grpcServer, l.cs) 72 | klog.Infof("Starting gRPC server") 73 | if err := grpcServer.Serve(listener); err != nil { 74 | return fmt.Errorf("failed to start server: %s", err.Error()) 75 | } 76 | return nil 77 | } 78 | 79 | func parseAddress(addr string) (string, string, error) { 80 | u, err := url.Parse(addr) 81 | if err != nil { 82 | return "", "", fmt.Errorf("failed to parse listen address: %s", err.Error()) 83 | } 84 | if u.Host != "" && u.Path != "" { 85 | return "", "", fmt.Errorf("failed to parse listen address: invalid address") 86 | } 87 | if u.Host != "" { 88 | return u.Scheme, u.Host, nil 89 | } 90 | return u.Scheme, u.Path, nil 91 | } 92 | -------------------------------------------------------------------------------- /pkg/driverd/lvmctrldclient.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "context" 19 | "net" 20 | "time" 21 | 22 | pb "github.com/aleofreddi/csi-sanlock-lvm/pkg/proto" 23 | "google.golang.org/grpc" 24 | "google.golang.org/grpc/connectivity" 25 | "k8s.io/klog" 26 | ) 27 | 28 | type LvmCtrldClientConnection struct { 29 | pb.LvmCtrldClient 30 | conn *grpc.ClientConn 31 | } 32 | 33 | func NewLvmCtrldClient(address string) (*LvmCtrldClientConnection, error) { 34 | conn, err := connect(address, 5*time.Minute) 35 | if err != nil { 36 | return nil, err 37 | } 38 | return &LvmCtrldClientConnection{ 39 | LvmCtrldClient: pb.NewLvmCtrldClient(conn), 40 | conn: conn, 41 | }, nil 42 | } 43 | 44 | func (c *LvmCtrldClientConnection) Wait() error { // FIXME: add a timeout here! 45 | for { 46 | vgs, err := c.Vgs(context.Background(), &pb.VgsRequest{}) 47 | if err != nil { 48 | klog.Infof("Failed to connect to lvmctrld (%s), retrying...", err.Error()) 49 | time.Sleep(1 * time.Second) 50 | continue 51 | } 52 | klog.Infof("lvmctrld startup complete, found %d volume group(s)", len(vgs.Vgs)) 53 | break 54 | } 55 | klog.Infof("Connected to lvmctrld") 56 | return nil 57 | } 58 | 59 | func (c *LvmCtrldClientConnection) Close() error { 60 | if c.conn == nil { 61 | return nil 62 | } 63 | return c.conn.Close() 64 | } 65 | 66 | func connect(address string, timeout time.Duration) (*grpc.ClientConn, error) { 67 | klog.V(2).Infof("Connecting to %s", address) 68 | dialOptions := []grpc.DialOption{ 69 | grpc.WithInsecure(), 70 | grpc.WithBackoffMaxDelay(time.Second), 71 | //grpc.WithUnaryInterceptor(LvmctrldLog), 72 | } 73 | dialOptions = append(dialOptions, grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) { 74 | protocol, target, err := parseAddress(addr) 75 | if err != nil { 76 | return nil, err 77 | } 78 | return net.DialTimeout(protocol, target, timeout) 79 | })) 80 | conn, err := grpc.Dial(address, dialOptions...) 81 | if err != nil { 82 | return nil, err 83 | } 84 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 85 | defer cancel() 86 | for { 87 | if !conn.WaitForStateChange(ctx, conn.GetState()) { 88 | klog.V(4).Infof("Connection timed out (%s)", conn.GetState()) 89 | return conn, nil 90 | } 91 | if conn.GetState() == connectivity.Ready { 92 | klog.V(3).Infof("Connected") 93 | return conn, nil 94 | } 95 | klog.V(4).Infof("Still trying, connection %s", conn.GetState()) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /pkg/driverd/tagencoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "strings" 21 | ) 22 | 23 | func encodeTag(value string) string { 24 | var sb strings.Builder 25 | for i, b := range []byte(value) { 26 | if isPlain(b, i) { 27 | sb.WriteByte(b) 28 | } else { 29 | sb.WriteString(fmt.Sprintf("&%2x", b)) 30 | } 31 | } 32 | return sb.String() 33 | } 34 | 35 | func decodeTag(encodedTag string) (string, error) { 36 | type State int 37 | const ( 38 | Normal State = iota 39 | Quote1 40 | Quote2 41 | ) 42 | var sb strings.Builder 43 | qState := Normal 44 | var qValue byte 45 | for i, b := range []byte(encodedTag) { 46 | if qState != Normal { 47 | v, err := hexDecode(b) 48 | if err != nil { 49 | return "", errors.New("encoded tag contains invalid quote sequence") 50 | } 51 | if qState == Quote1 { 52 | qValue = v << 4 53 | qState = Quote2 54 | } else { 55 | sb.WriteByte(qValue | v) 56 | qValue = 0 57 | qState = Normal 58 | } 59 | } else if b == '&' { 60 | qState = Quote1 61 | } else if isPlain(b, i) { 62 | sb.WriteByte(b) 63 | } else { 64 | return "", errors.New("encoded tag contains an invalid character") 65 | } 66 | } 67 | if qState != 0 { 68 | return "", errors.New("encoded tag contains truncated quote sequence") 69 | } 70 | return sb.String(), nil 71 | } 72 | 73 | func isPlain(b byte, pos int) bool { 74 | return b >= 'a' && b <= 'z' || 75 | b >= 'A' && b <= 'Z' || 76 | b >= '0' && b <= '9' || 77 | b == '_' || 78 | b == '+' || 79 | b == '.' || 80 | (b == '-' && pos > 0) || // LVM doesn't allow hyphen starting tags 81 | b == '/' || 82 | b == '=' || 83 | b == '!' || 84 | b == ':' || 85 | b == '#' 86 | } 87 | 88 | func hexDecode(hex byte) (byte, error) { 89 | if hex >= '0' && hex <= '9' { 90 | return hex - '0', nil 91 | } 92 | if hex >= 'A' && hex <= 'F' { 93 | return hex - 'A' + 10, nil 94 | } 95 | if hex >= 'a' && hex <= 'f' { 96 | return hex - 'a' + 10, nil 97 | } 98 | return 0, errors.New("invalid hexadecimal sequence") 99 | } 100 | -------------------------------------------------------------------------------- /pkg/driverd/tagencoder_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func Test_encodeTag(t *testing.T) { 22 | type args struct { 23 | value string 24 | } 25 | tests := []struct { 26 | name string 27 | args args 28 | want string 29 | }{ 30 | {"Unquoted characters", args{"unquoted.chars=01234456789_+.-/=!:#"}, "unquoted.chars=01234456789_+.-/=!:#"}, 31 | {"Quoted characters", args{"quoted_chars= {}$`"}, "quoted_chars=&20&7b&7d&24&60"}, 32 | {"Quote hyphen only when is the first character", args{"-_should_be_quoted._while_-_not"}, "&2d_should_be_quoted._while_-_not"}, 33 | } 34 | for _, tt := range tests { 35 | t.Run(tt.name, func(t *testing.T) { 36 | if got := encodeTag(tt.args.value); got != tt.want { 37 | t.Errorf("encodeTag() = %v, want %v", got, tt.want) 38 | } 39 | }) 40 | } 41 | } 42 | 43 | func Test_decodeTag(t *testing.T) { 44 | type args struct { 45 | encodedTag string 46 | } 47 | tests := []struct { 48 | name string 49 | args args 50 | want string 51 | wantErr bool 52 | }{ 53 | {"Unquoted characters", args{"unquoted.chars=01234456789"}, "unquoted.chars=01234456789", false}, 54 | {"Quoted characters", args{"quoted.chars=&20&2b&2d&7b&7d&24&60"}, "quoted.chars= +-{}$`", false}, 55 | {"Invalid characters", args{"invalid +"}, "", true}, 56 | {"Decode quoted and unquoted hyphen", args{"&2d-&2d"}, "---", false}, 57 | {"Reject unquoted initial hyphen", args{"--"}, "", true}, 58 | } 59 | for _, tt := range tests { 60 | t.Run(tt.name, func(t *testing.T) { 61 | got, err := decodeTag(tt.args.encodedTag) 62 | if (err != nil) != tt.wantErr { 63 | t.Errorf("decodeTag() error = %v, wantErr %v", err, tt.wantErr) 64 | return 65 | } 66 | if got != tt.want { 67 | t.Errorf("decodeTag() got = %v, want %v", got, tt.want) 68 | } 69 | }) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/driverd/tags.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "fmt" 19 | "sort" 20 | "strings" 21 | 22 | "golang.org/x/exp/maps" 23 | ) 24 | 25 | type TagKey string 26 | 27 | const ( 28 | tagPrefix = "csi-sanlock-lvm.vleo.net/" 29 | 30 | nameTagKey TagKey = "name" 31 | fsTagKey TagKey = "fs" 32 | ownerIdTagKey TagKey = "ownerId" 33 | ownerNodeTagKey TagKey = "ownerNode" 34 | sourceTagKey TagKey = "src" 35 | 36 | rpcRoleTagKey TagKey = "rpcRole" 37 | ) 38 | 39 | type LogVolType string 40 | 41 | const ( 42 | VolumeVolType LogVolType = "volume" 43 | SnapshotVolType LogVolType = "snapshot" 44 | TemporaryVolType LogVolType = "temp" 45 | ) 46 | 47 | var ( 48 | encodedTagPrefix = encodeTag(tagPrefix) 49 | ) 50 | 51 | func decodeTagKV(encoded string) (TagKey, string, bool, error) { 52 | if !strings.HasPrefix(encoded, encodedTagPrefix) { 53 | return "", "", false, nil 54 | } 55 | decoded, err := decodeTag(encoded[len(encodedTagPrefix):]) 56 | if err != nil { 57 | return "", "", false, err 58 | } 59 | i := strings.Index(decoded, "=") 60 | if i == -1 { 61 | return "", "", false, fmt.Errorf("invalid tag value %q", encoded) 62 | } 63 | return TagKey(decoded[0:i]), decoded[i+1:], true, nil 64 | } 65 | 66 | func encodeTagKV(key TagKey, value string) string { 67 | return encodeTag(fmt.Sprintf("%s%s=%s", tagPrefix, key, value)) 68 | } 69 | 70 | func encodeTagKeyPrefix(key TagKey) string { 71 | return encodeTag(fmt.Sprintf("%s%s=", tagPrefix, key)) 72 | } 73 | 74 | func encodeTags(tags map[TagKey]string) []string { 75 | // Sort keys to get a deterministic result. 76 | keys := maps.Keys(tags) 77 | sort.Slice(keys, func(t, u int) bool { 78 | return keys[t] < keys[u] 79 | }) 80 | r := make([]string, len(tags)) 81 | i := 0 82 | for _, key := range keys { 83 | r[i] = encodeTagKV(key, tags[key]) 84 | i++ 85 | } 86 | return r 87 | } 88 | 89 | func decodeTags(encodedTags []string) (map[TagKey]string, error) { 90 | r := make(map[TagKey]string) 91 | for _, v := range encodedTags { 92 | key, value, ok, err := decodeTagKV(v) 93 | if err != nil { 94 | return nil, err 95 | } 96 | if !ok { 97 | continue 98 | } 99 | if _, ok := r[key]; ok { 100 | return nil, fmt.Errorf("duplicate tag entry for key %q", key) 101 | } 102 | r[key] = value 103 | } 104 | return r, nil 105 | } 106 | -------------------------------------------------------------------------------- /pkg/driverd/volumeinfo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | pb "github.com/aleofreddi/csi-sanlock-lvm/pkg/proto" 19 | "google.golang.org/grpc/codes" 20 | "google.golang.org/grpc/status" 21 | ) 22 | 23 | // A type that holds details of a volume by wrapping a LogicalVolume. 24 | type VolumeInfo struct { 25 | VolumeRef 26 | *pb.LogicalVolume 27 | } 28 | 29 | func NewVolumeInfoFromLv(lv *pb.LogicalVolume) *VolumeInfo { 30 | return &VolumeInfo{ 31 | VolumeRef{lv.LvName, lv.VgName}, 32 | lv, 33 | } 34 | } 35 | 36 | func (v *VolumeInfo) OriginRef() *VolumeRef { 37 | return &VolumeRef{ 38 | LvName: v.Origin, 39 | VgName: v.LogicalVolume.VgName, 40 | } 41 | } 42 | 43 | func (v *VolumeInfo) Tags() (map[TagKey]string, error) { 44 | tags, err := decodeTags(v.LvTags) 45 | if err != nil { 46 | return nil, status.Errorf(codes.Internal /* is this the right code? this should be non retryable */, "failed to decode tags for volume %q: %v", v.ID(), err) 47 | } 48 | return tags, nil 49 | } 50 | 51 | func (v *VolumeInfo) String() string { 52 | return v.ID() 53 | } 54 | -------------------------------------------------------------------------------- /pkg/driverd/volumelock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "sync" 21 | 22 | diskrpc "github.com/aleofreddi/csi-sanlock-lvm/pkg/diskrpc" 23 | "github.com/aleofreddi/csi-sanlock-lvm/pkg/proto" // FIXME 24 | pb "github.com/aleofreddi/csi-sanlock-lvm/pkg/proto" 25 | "google.golang.org/grpc/codes" 26 | "google.golang.org/grpc/status" 27 | "k8s.io/klog" 28 | ) 29 | 30 | type VolumeLocker interface { 31 | // Lock the given volume for the given operation. 32 | // 33 | // Volume locks are reentrant per volume (but not per pairs), 34 | // that is one can lock the same volume with multiple ops. 35 | LockVolume(ctx context.Context, vol VolumeRef, op string) error 36 | 37 | // Unlock a volume for a given pair. 38 | UnlockVolume(ctx context.Context, vol VolumeRef, op string) error 39 | 40 | // Retrieve the owner mailbox and name for a volume, if any. 41 | GetOwner(ctx context.Context, ref VolumeRef) (diskrpc.MailBoxID, string, error) 42 | } 43 | 44 | type volumeLocker struct { 45 | baseServer 46 | nodeName string 47 | 48 | locks map[string]map[string]struct{} 49 | mutex *sync.Mutex 50 | } 51 | 52 | const ( 53 | defaultLockOp = "stage" 54 | ) 55 | 56 | func NewVolumeLocker(lvmctrld pb.LvmCtrldClient, nodeName string) (VolumeLocker, error) { 57 | bs, err := newBaseServer(lvmctrld) 58 | if err != nil { 59 | return nil, err 60 | } 61 | vl := &volumeLocker{ 62 | baseServer: *bs, 63 | nodeName: nodeName, 64 | locks: map[string]map[string]struct{}{}, 65 | mutex: &sync.Mutex{}, 66 | } 67 | ctx := context.Background() 68 | if err = vl.sync(ctx); err != nil { 69 | return nil, fmt.Errorf("failed to sync LVM state: %v", err) 70 | } 71 | return vl, nil 72 | } 73 | 74 | func (vl *volumeLocker) LockVolume(ctx context.Context, vol VolumeRef, op string) error { 75 | vl.mutex.Lock() 76 | defer vl.mutex.Unlock() 77 | 78 | key := vol.VgLv() 79 | m, ok := vl.locks[key] 80 | if !ok { 81 | m = make(map[string]struct{}) 82 | vl.locks[key] = m 83 | } 84 | if len(m) > 0 { 85 | m[op] = struct{}{} 86 | return nil 87 | } 88 | 89 | // Lock the volume. 90 | _, err := vl.lvmctrld.LvChange(ctx, &proto.LvChangeRequest{ 91 | Target: []string{vol.VgLv()}, 92 | Activate: proto.LvActivationMode_LV_ACTIVATION_MODE_ACTIVE_EXCLUSIVE, 93 | }) 94 | if err != nil { 95 | return status.Errorf(status.Code(err), "failed to lock volume %s: %v", vol, err) 96 | } 97 | 98 | // Update tags. 99 | err = vl.setOwner(ctx, vol, &vl.nodeID, &vl.nodeName) 100 | if err != nil { 101 | // Try to unlock the volume. 102 | _, err2 := vl.lvmctrld.LvChange(ctx, &proto.LvChangeRequest{ 103 | Target: []string{vol.VgLv()}, 104 | Activate: proto.LvActivationMode_LV_ACTIVATION_MODE_DEACTIVATE, 105 | }) 106 | if err2 != nil { 107 | klog.Errorf("Failed to unlock volume %s: %v (ignoring error)", vol, err2) 108 | } 109 | return status.Errorf(status.Code(err), "failed to update owner tags on volume %s: %v", vol, err) 110 | } 111 | 112 | m[op] = struct{}{} 113 | return nil 114 | } 115 | 116 | func (vl *volumeLocker) UnlockVolume(ctx context.Context, vol VolumeRef, op string) error { 117 | vl.mutex.Lock() 118 | defer vl.mutex.Unlock() 119 | 120 | // If volume is not locked, return. 121 | key := vol.VgLv() 122 | m, ok := vl.locks[key] 123 | if !ok { 124 | return nil 125 | } 126 | delete(m, op) 127 | if len(m) > 0 { 128 | return nil 129 | } 130 | 131 | // Remove owner tags. 132 | err := vl.setOwner(ctx, vol, nil, nil) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | // Unlock the volume. 138 | _, err = vl.lvmctrld.LvChange(ctx, &proto.LvChangeRequest{ 139 | Target: []string{vol.VgLv()}, 140 | Activate: proto.LvActivationMode_LV_ACTIVATION_MODE_DEACTIVATE, 141 | }) 142 | if err != nil { 143 | return status.Errorf(status.Code(err), "failed to unlock volume %s: %v", vol, err) 144 | } 145 | 146 | delete(vl.locks, key) 147 | return nil 148 | } 149 | 150 | // Retrieve the owner node for a given volume. 151 | func (vl *volumeLocker) GetOwner(ctx context.Context, ref VolumeRef) (diskrpc.MailBoxID, string, error) { 152 | // Fetch the volume and get its tags. 153 | vol, err := vl.fetch(ctx, &ref) 154 | if err != nil { 155 | return 0, "", status.Errorf(codes.Internal, "failed to list logical volume %s: %v", vol, err) 156 | } 157 | tags, err := vol.Tags() 158 | if err != nil { 159 | return 0, "", status.Errorf(codes.Internal, "failed to decode tags on volume %s: %v", vol, err) 160 | } 161 | 162 | var nodeID uint16 163 | if v, ok := tags[ownerIdTagKey]; ok { 164 | nodeID, err = nodeIDFromString(v) 165 | if err != nil { 166 | return 0, "", status.Errorf(codes.Internal, "failed to parse owner id tag on volume %s: %v", vol, err) 167 | } 168 | } 169 | nodeName, _ := tags[ownerNodeTagKey] 170 | return diskrpc.MailBoxID(nodeID), nodeName, nil 171 | } 172 | 173 | // Update the owner tag of a volume, removing any stale owner tag and replacing 174 | // them with a new ownerID entry (if present). 175 | // 176 | // Calling this function with a nil ownerID will cause it to remove all owner tags. 177 | func (vl *volumeLocker) setOwner(ctx context.Context, ref VolumeRef, ownerID *uint16, ownerNode *string) error { 178 | // Fetch the volume and get its tags 179 | vol, err := vl.fetch(ctx, &ref) 180 | if err != nil { 181 | return err 182 | } 183 | tags, err := vol.Tags() 184 | if err != nil { 185 | return err 186 | } 187 | 188 | // Decide which tags to add/remove 189 | var addTags, delTags []string 190 | pOwnerID, ok := tags[ownerIdTagKey] 191 | if ok && (ownerID == nil || pOwnerID != nodeIDToString(*ownerID)) { 192 | delTags = append(delTags, encodeTagKV(ownerIdTagKey, pOwnerID)) 193 | } 194 | pOwnerNode, ok := tags[ownerNodeTagKey] 195 | if ok && (ownerNode == nil || pOwnerNode != *ownerNode) { 196 | delTags = append(delTags, encodeTagKV(ownerNodeTagKey, pOwnerNode)) 197 | } 198 | if ownerID != nil { 199 | addTags = append(addTags, encodeTagKV(ownerIdTagKey, nodeIDToString(*ownerID))) 200 | } 201 | if ownerNode != nil { 202 | addTags = append(addTags, encodeTagKV(ownerNodeTagKey, *ownerNode)) 203 | } 204 | 205 | // Update volume tags 206 | if len(addTags) == 0 && len(delTags) == 0 { 207 | return nil 208 | } 209 | _, err = vl.lvmctrld.LvChange(ctx, &pb.LvChangeRequest{ 210 | Target: []string{vol.VgLv()}, 211 | AddTag: addTags, 212 | DelTag: delTags, 213 | }) 214 | return err 215 | } 216 | 217 | func (vl *volumeLocker) sync(ctx context.Context) error { 218 | vl.mutex.Lock() 219 | defer vl.mutex.Unlock() 220 | 221 | klog.Infof("Syncing LVM status") 222 | 223 | // Find all active volumes. 224 | lvs, err := vl.lvmctrld.Lvs(ctx, &pb.LvsRequest{ 225 | Select: fmt.Sprintf("lv_name=~^%s && lv_active=active", volumeLvPrefix), 226 | }) 227 | if err != nil { 228 | return fmt.Errorf("failed to list logical volumes: %v", err) 229 | } 230 | 231 | for _, lv := range lvs.Lvs { 232 | vol := NewVolumeRefFromLv(lv) 233 | // Try to deactivate volume if it is not opened 234 | if lv.LvDeviceOpen != pb.LvDeviceOpen_LV_DEVICE_OPEN_OPEN { 235 | _, err = vl.lvmctrld.LvChange(ctx, &pb.LvChangeRequest{ 236 | Target: []string{vol.VgLv()}, 237 | Activate: pb.LvActivationMode_LV_ACTIVATION_MODE_DEACTIVATE, 238 | }) 239 | if err != nil { 240 | klog.Warningf("Failed to unlock volume %q: %v", *vol, err) 241 | } 242 | // Try to remove owner tag. We could race here and have another host lock the 243 | // volume. In such case, the update will fail with permission denied. 244 | err := vl.setOwner(ctx, *vol, nil, nil) 245 | if err != nil && status.Code(err) != codes.PermissionDenied { 246 | klog.Warningf("failed to update tags on volume %q: %v", *vol, err) 247 | } 248 | continue 249 | } 250 | // The volume is not locked, take note 251 | vl.locks[vol.VgLv()] = map[string]struct{}{defaultLockOp: {}} 252 | } 253 | return nil 254 | } 255 | -------------------------------------------------------------------------------- /pkg/driverd/volumelock_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd_test 16 | 17 | import ( 18 | "context" 19 | "testing" 20 | 21 | pkg "github.com/aleofreddi/csi-sanlock-lvm/pkg/driverd" 22 | "github.com/aleofreddi/csi-sanlock-lvm/pkg/mock" 23 | "github.com/aleofreddi/csi-sanlock-lvm/pkg/proto" 24 | "github.com/golang/mock/gomock" 25 | "google.golang.org/grpc/codes" 26 | "google.golang.org/grpc/status" 27 | "google.golang.org/protobuf/testing/protocmp" 28 | ) 29 | 30 | func Test_volumeLocker_LockVolume(t *testing.T) { 31 | type fields struct { 32 | lvmctrld proto.LvmCtrldClient 33 | nodeName string 34 | } 35 | type args struct { 36 | ctx context.Context 37 | vol pkg.VolumeRef 38 | op string 39 | } 40 | tests := []struct { 41 | name string 42 | fields func(controller *gomock.Controller) *fields 43 | args args 44 | wantErr bool 45 | wantErrCode codes.Code 46 | }{ 47 | { 48 | "Should fail when volume activation fails", 49 | func(controller *gomock.Controller) *fields { 50 | client := mock.NewMockLvmCtrldClient(controller) 51 | gomock.InOrder( 52 | expectGetStatus(t, client), 53 | expectLockerSyncLvs(t, client), 54 | 55 | client.EXPECT(). 56 | LvChange(gomock.Any(), 57 | CmpMatcher(t, &proto.LvChangeRequest{ 58 | Target: []string{"vg00/volume1"}, 59 | Activate: proto.LvActivationMode_LV_ACTIVATION_MODE_ACTIVE_EXCLUSIVE, 60 | }, protocmp.Transform()), 61 | gomock.Any(), 62 | ). 63 | Return( 64 | nil, 65 | status.Error(codes.Internal, "internal error"), 66 | ), 67 | ) 68 | return &fields{ 69 | lvmctrld: client, 70 | nodeName: "node1", 71 | } 72 | }, 73 | args{ 74 | context.Background(), 75 | *MustVolumeRefFromID("volume1@vg00"), 76 | "", 77 | }, 78 | true, 79 | codes.Internal, 80 | }, 81 | { 82 | "Should fail when update tag fails", 83 | func(controller *gomock.Controller) *fields { 84 | client := mock.NewMockLvmCtrldClient(controller) 85 | gomock.InOrder( 86 | expectGetStatus(t, client), 87 | expectLockerSyncLvs(t, client), 88 | 89 | client.EXPECT(). 90 | LvChange(gomock.Any(), 91 | CmpMatcher(t, &proto.LvChangeRequest{Target: []string{"vg00/volume1"}, Activate: proto.LvActivationMode_LV_ACTIVATION_MODE_ACTIVE_EXCLUSIVE}, protocmp.Transform()), 92 | gomock.Any(), 93 | ). 94 | Return( 95 | &proto.LvChangeResponse{}, 96 | nil, 97 | ), 98 | client.EXPECT(). 99 | Lvs( 100 | gomock.Any(), 101 | CmpMatcher(t, &proto.LvsRequest{Target: []string{"vg00/volume1"}}, protocmp.Transform()), 102 | gomock.Any(), 103 | ). 104 | Return( 105 | &proto.LvsResponse{Lvs: []*proto.LogicalVolume{{ 106 | VgName: "vg00", 107 | LvName: "volume1", 108 | LvTags: []string{ 109 | "ignore_invalid_tags_&", 110 | "csi-sanlock-lvm.vleo.net/ownerId=4321", 111 | "csi-sanlock-lvm.vleo.net/ownerNode=node2", 112 | }, 113 | }}}, 114 | nil, 115 | ), 116 | client.EXPECT(). 117 | LvChange(gomock.Any(), 118 | CmpMatcher(t, &proto.LvChangeRequest{ 119 | Target: []string{"vg00/volume1"}, 120 | AddTag: []string{"csi-sanlock-lvm.vleo.net/ownerId=1234", "csi-sanlock-lvm.vleo.net/ownerNode=node1"}, 121 | DelTag: []string{"csi-sanlock-lvm.vleo.net/ownerId=4321", "csi-sanlock-lvm.vleo.net/ownerNode=node2"}, 122 | }, protocmp.Transform()), 123 | gomock.Any(), 124 | ). 125 | Return( 126 | nil, 127 | status.Error(codes.Internal, "internal error"), 128 | ), 129 | client.EXPECT(). 130 | LvChange(gomock.Any(), 131 | CmpMatcher(t, &proto.LvChangeRequest{Target: []string{"vg00/volume1"}, Activate: proto.LvActivationMode_LV_ACTIVATION_MODE_DEACTIVATE}, protocmp.Transform()), 132 | gomock.Any(), 133 | ). 134 | Return( 135 | &proto.LvChangeResponse{}, 136 | nil, 137 | ), 138 | ) 139 | return &fields{ 140 | lvmctrld: client, 141 | nodeName: "node1", 142 | } 143 | }, 144 | args{ 145 | context.Background(), 146 | *MustVolumeRefFromID("volume1@vg00"), 147 | "", 148 | }, 149 | true, 150 | codes.Internal, 151 | }, 152 | { 153 | "Should activate the volume and set owner tag", 154 | func(controller *gomock.Controller) *fields { 155 | client := mock.NewMockLvmCtrldClient(controller) 156 | gomock.InOrder( 157 | expectGetStatus(t, client), 158 | expectLockerSyncLvs(t, client), 159 | 160 | client.EXPECT(). 161 | LvChange(gomock.Any(), 162 | CmpMatcher(t, &proto.LvChangeRequest{Target: []string{"vg00/volume1"}, Activate: proto.LvActivationMode_LV_ACTIVATION_MODE_ACTIVE_EXCLUSIVE}, protocmp.Transform()), 163 | gomock.Any(), 164 | ). 165 | Return( 166 | &proto.LvChangeResponse{}, 167 | nil, 168 | ), 169 | client.EXPECT(). 170 | Lvs( 171 | gomock.Any(), 172 | CmpMatcher(t, &proto.LvsRequest{Target: []string{"vg00/volume1"}}, protocmp.Transform()), 173 | gomock.Any(), 174 | ). 175 | Return( 176 | &proto.LvsResponse{Lvs: []*proto.LogicalVolume{{ 177 | VgName: "vg00", 178 | LvName: "volume1", 179 | LvTags: []string{ 180 | "ignore_invalid_tags_&", 181 | "csi-sanlock-lvm.vleo.net/ownerId=4321", 182 | "csi-sanlock-lvm.vleo.net/ownerNode=node2", 183 | }}}, 184 | }, 185 | nil, 186 | ), 187 | client.EXPECT(). 188 | LvChange(gomock.Any(), 189 | CmpMatcher(t, &proto.LvChangeRequest{ 190 | Target: []string{"vg00/volume1"}, 191 | AddTag: []string{"csi-sanlock-lvm.vleo.net/ownerId=1234", "csi-sanlock-lvm.vleo.net/ownerNode=node1"}, 192 | DelTag: []string{"csi-sanlock-lvm.vleo.net/ownerId=4321", "csi-sanlock-lvm.vleo.net/ownerNode=node2"}, 193 | }, protocmp.Transform()), 194 | gomock.Any(), 195 | ). 196 | Return( 197 | &proto.LvChangeResponse{}, 198 | nil, 199 | ), 200 | ) 201 | return &fields{ 202 | lvmctrld: client, 203 | nodeName: "node1", 204 | } 205 | }, 206 | args{ 207 | context.Background(), 208 | *MustVolumeRefFromID("volume1@vg00"), 209 | "", 210 | }, 211 | false, 212 | codes.OK, 213 | }, 214 | } 215 | for _, tt := range tests { 216 | t.Run(tt.name, func(t *testing.T) { 217 | mockCtrl := gomock.NewController(t) 218 | defer mockCtrl.Finish() 219 | fields := tt.fields(mockCtrl) 220 | vl, _ := pkg.NewVolumeLocker( 221 | fields.lvmctrld, 222 | fields.nodeName, 223 | ) 224 | err := vl.LockVolume( 225 | tt.args.ctx, 226 | tt.args.vol, 227 | tt.args.op, 228 | ) 229 | if (err != nil) != tt.wantErr { 230 | t.Errorf("LockVolume() error = %v, wantErr %v", err, tt.wantErr) 231 | return 232 | } 233 | if (err != nil) && status.Code(err) != tt.wantErrCode { 234 | t.Errorf("LockVolume() error code = %v, wantErrCode %v", status.Code(err), tt.wantErrCode) 235 | return 236 | } 237 | }) 238 | } 239 | } 240 | 241 | func expectLockerSyncLvs(t *testing.T, client *mock.MockLvmCtrldClient) *gomock.Call { 242 | return client.EXPECT(). 243 | Lvs(gomock.Any(), CmpMatcher(t, &proto.LvsRequest{ 244 | Select: "lv_name=~^csl-v- && lv_active=active", 245 | Sort: nil, 246 | Target: nil, 247 | }, protocmp.Transform())). 248 | Return(&proto.LvsResponse{}, nil) 249 | } 250 | -------------------------------------------------------------------------------- /pkg/driverd/volumeref.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package driverd 16 | 17 | import ( 18 | "crypto/sha256" 19 | "encoding/base64" 20 | "fmt" 21 | "strings" 22 | 23 | pb "github.com/aleofreddi/csi-sanlock-lvm/pkg/proto" 24 | ) 25 | 26 | const ( 27 | // Prefix to be used for LVM volumes. 28 | volumeLvPrefix = "csl-v-" 29 | 30 | // Prefix to be used for LVM snapshots. 31 | snapshotLvPrefix = "csl-s-" 32 | 33 | // Prefix to be used for LVM temporary volumes or snapshots. 34 | tempLvPrefix = "csl-t-" 35 | ) 36 | 37 | var volTypeToPrefix = map[LogVolType]string{ 38 | VolumeVolType: volumeLvPrefix, 39 | SnapshotVolType: snapshotLvPrefix, 40 | TemporaryVolType: tempLvPrefix, 41 | } 42 | 43 | // A type that holds a volume reference. Only the VgName and LvName fields are expected to be set. 44 | type VolumeRef struct { 45 | LvName string 46 | VgName string 47 | } 48 | 49 | func NewVolumeRefFromVgTypeName(vg string, t LogVolType, name string) *VolumeRef { 50 | h := sha256.New() 51 | h.Write([]byte(name)) 52 | b64 := base64.StdEncoding.WithPadding(base64.NoPadding).EncodeToString(h.Sum(nil)) 53 | prefix, ok := volTypeToPrefix[t] 54 | if !ok { 55 | panic(fmt.Errorf("unexpected volume type %s", t)) 56 | } 57 | // LVM doesn't like the '/' character, replace with '_' 58 | lv := prefix + strings.ReplaceAll(b64, "/", "_") 59 | return &VolumeRef{ 60 | LvName: lv, 61 | VgName: vg, 62 | } 63 | } 64 | 65 | func NewVolumeRefFromID(volumeID string) (*VolumeRef, error) { 66 | tokens := strings.Split(volumeID, "@") 67 | if len(tokens) != 2 { 68 | return nil, fmt.Errorf("invalid volume id %q", volumeID) 69 | } 70 | return &VolumeRef{ 71 | LvName: tokens[0], 72 | VgName: tokens[1], 73 | }, nil 74 | } 75 | 76 | func NewVolumeRefFromLv(lv *pb.LogicalVolume) *VolumeRef { 77 | return &VolumeRef{ 78 | lv.LvName, 79 | lv.VgName, 80 | } 81 | } 82 | 83 | func (v *VolumeRef) ID() string { 84 | return fmt.Sprintf("%s@%s", v.LvName, v.VgName) 85 | } 86 | 87 | func (v *VolumeRef) Vg() string { 88 | return v.VgName 89 | } 90 | 91 | func (v *VolumeRef) Lv() string { 92 | return v.LvName 93 | } 94 | 95 | func (v *VolumeRef) VgLv() string { 96 | return fmt.Sprintf("%s/%s", v.VgName, v.LvName) 97 | } 98 | 99 | func (v *VolumeRef) DevPath() string { 100 | return fmt.Sprintf("/dev/%s/%s", v.VgName, v.LvName) 101 | } 102 | 103 | func (v *VolumeRef) String() string { 104 | return v.ID() 105 | } 106 | -------------------------------------------------------------------------------- /pkg/grpclogger/grpclogger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package grpclogger 16 | 17 | import ( 18 | "context" 19 | "sync/atomic" 20 | 21 | "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" 22 | "google.golang.org/grpc" 23 | "k8s.io/klog" 24 | ) 25 | 26 | var logUid uint32 = 0 27 | 28 | func GrpcLogger(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 29 | id := atomic.AddUint32(&logUid, 1) 30 | klog.V(6).Infof("gRPC[%d]: calling %s(%+v)...", id, info.FullMethod, protosanitizer.StripSecrets(req)) 31 | resp, err := handler(ctx, req) 32 | if err != nil { 33 | klog.Errorf("gRPC[%d]: call %s(%+v) returned error %v", id, info.FullMethod, protosanitizer.StripSecrets(req), err) 34 | } else { 35 | klog.V(5).Infof("gRPC[%d]: call %s(%+v) returned %+v", id, info.FullMethod, protosanitizer.StripSecrets(req), protosanitizer.StripSecrets(resp)) 36 | } 37 | return resp, err 38 | } 39 | -------------------------------------------------------------------------------- /pkg/lvmctrld/fakerunner_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lvmctrld 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | type fakeCommand struct { 23 | exe string 24 | args []string 25 | rc int 26 | stdout string 27 | stderr string 28 | err error 29 | } 30 | 31 | type fakeRunner struct { 32 | t *testing.T 33 | 34 | executions []fakeCommand 35 | current int 36 | } 37 | 38 | func (c *fakeRunner) Exec(exe string, args ...string) (code int, stdout, stderr []byte, err error) { 39 | if c.current == len(c.executions) { 40 | c.t.Errorf("unexpected exec invocation (%s, %v)", exe, args) 41 | return 255, []byte{}, []byte{}, fmt.Errorf("unexpected invocation") 42 | } 43 | expected := c.executions[c.current] 44 | if expected.exe != exe || len(expected.args) != len(args) { 45 | c.t.Errorf("exec: got invocation (%s, %v), expected (%s, %v)", exe, args, expected.exe, expected.args) 46 | return 255, []byte{}, []byte{}, fmt.Errorf("unexpected invocation") 47 | } 48 | for i, v := range expected.args { 49 | if v != args[i] { 50 | c.t.Errorf("exec: got invocation (%s, %v), expected (%s, %v)", exe, args, expected.exe, expected.args) 51 | return 255, []byte{}, []byte{}, fmt.Errorf("unexpected invocation") 52 | } 53 | } 54 | c.current++ 55 | return expected.rc, []byte(expected.stdout), []byte(expected.stderr), expected.err 56 | } 57 | -------------------------------------------------------------------------------- /pkg/lvmctrld/listener.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lvmctrld 16 | 17 | import ( 18 | "fmt" 19 | "net" 20 | "net/url" 21 | "os" 22 | 23 | "github.com/aleofreddi/csi-sanlock-lvm/pkg/grpclogger" 24 | "github.com/aleofreddi/csi-sanlock-lvm/pkg/proto" 25 | "google.golang.org/grpc" 26 | "k8s.io/klog" 27 | ) 28 | 29 | type Listener struct { 30 | addr string 31 | 32 | ls *lvmctrldServer 33 | } 34 | 35 | func NewListener(addr string, id uint16) (*Listener, error) { 36 | return &Listener{ 37 | addr: addr, 38 | 39 | ls: NewLvmctrldServer(id), 40 | }, nil 41 | } 42 | 43 | func (l *Listener) Init() error { 44 | return nil 45 | } 46 | 47 | func (l *Listener) Run() error { 48 | // Start gRPC server 49 | lsProto, lsAddr, err := parseAddress(l.addr) 50 | if err != nil { 51 | return fmt.Errorf("invalid listen address: %s", err.Error()) 52 | } 53 | if lsProto == "unix" { 54 | if err := os.Remove(lsAddr); err != nil && !os.IsNotExist(err) { 55 | return fmt.Errorf("failed to remove %s: %s", lsAddr, err.Error()) 56 | } 57 | } 58 | klog.Infof("Binding proto %s, address %s", lsProto, lsAddr) 59 | listener, err := net.Listen(lsProto, lsAddr) 60 | if err != nil { 61 | return fmt.Errorf("failed to listen %s://%s: %s", lsProto, lsAddr, err.Error()) 62 | } 63 | opts := []grpc.ServerOption{ 64 | grpc.UnaryInterceptor(grpclogger.GrpcLogger), 65 | } 66 | grpcServer := grpc.NewServer(opts...) 67 | proto.RegisterLvmCtrldServer(grpcServer, l.ls) 68 | klog.Infof("Starting gRPC server") 69 | if err := grpcServer.Serve(listener); err != nil { 70 | return fmt.Errorf("failed to start server: %s", err.Error()) 71 | } 72 | return nil 73 | } 74 | 75 | func parseAddress(addr string) (string, string, error) { 76 | u, err := url.Parse(addr) 77 | if err != nil || u.Host != "" && u.Path != "" { 78 | return "", "", fmt.Errorf("failed to parse listen address: %s", err.Error()) 79 | } 80 | if u.Host != "" { 81 | return u.Scheme, u.Host, nil 82 | } 83 | return u.Scheme, u.Path, nil 84 | } 85 | -------------------------------------------------------------------------------- /pkg/lvmctrld/lock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lvmctrld 16 | 17 | import ( 18 | "fmt" 19 | "io" 20 | "os" 21 | "os/exec" 22 | "time" 23 | 24 | "k8s.io/klog" 25 | ) 26 | 27 | func StartLock(id uint16, volumeGroups []string) error { 28 | if err := daemonize("wdmd", os.Stdout, os.Stderr, "-D"); err != nil { 29 | return err 30 | } 31 | if err := daemonize("sanlock", nil, nil, "daemon", "-D"); err != nil { 32 | return err 33 | } 34 | if err := daemonize("lvmlockd", os.Stdout, os.Stderr, "--host-id", fmt.Sprintf("%d", id), "-f"); err != nil { 35 | return err 36 | } 37 | time.Sleep(1 * time.Second) 38 | klog.Infof("Starting global lock (can take up to 3 minutes)") 39 | vgchange := exec.Command("vgchange", "--lockstart", "--verbose") 40 | vgchange.Stdout = os.Stdout 41 | vgchange.Stderr = os.Stderr 42 | if err := vgchange.Run(); err != nil { 43 | // On Ubuntu 20 LTS, with LVM 2.03.07(2) (2019-11-30), I've encountered a bug where 44 | // the first lockstart fails with error code 5, and a second one succeeds. 45 | // 46 | // So we wait 1 second and retry. 47 | time.Sleep(1 * time.Second) 48 | vgchange = exec.Command("vgchange", "--lockstart", "--verbose") 49 | vgchange.Stdout = os.Stdout 50 | vgchange.Stderr = os.Stderr 51 | if err = vgchange.Run(); err != nil { 52 | return fmt.Errorf("failed to start global lock: %v", err) 53 | } 54 | } 55 | klog.Info("Global lock started") 56 | return nil 57 | } 58 | 59 | func daemonize(executable string, stdout io.Writer, stderr io.Writer, args ...string) error { 60 | klog.Infof("Running %s with args %v", executable, args) 61 | cmd := exec.Command(executable, args...) 62 | cmd.Stdout = stdout 63 | cmd.Stderr = stderr 64 | err := cmd.Start() 65 | if err != nil { 66 | return err 67 | } 68 | go func() { 69 | err = cmd.Wait() 70 | klog.Fatalf("Process %s terminated with error: %v", executable, err.Error()) 71 | }() 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /pkg/lvmctrld/runner.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lvmctrld 16 | 17 | import ( 18 | "bytes" 19 | "os/exec" 20 | ) 21 | 22 | type runner interface { 23 | Exec(exe string, args ...string) (code int, stdout, stderr []byte, err error) 24 | } 25 | 26 | type osRunner struct{} 27 | 28 | func (osRunner) Exec(exe string, args ...string) (code int, stdout, stderr []byte, err error) { 29 | proc := exec.Command(exe, args...) 30 | stdoutBuf, stderrBuf := new(bytes.Buffer), new(bytes.Buffer) 31 | proc.Stdout = stdoutBuf 32 | proc.Stderr = stderrBuf 33 | err = proc.Run() 34 | return proc.ProcessState.ExitCode(), stdoutBuf.Bytes(), stderrBuf.Bytes(), err 35 | } 36 | 37 | func NewCommander() runner { 38 | return osRunner{} 39 | } 40 | -------------------------------------------------------------------------------- /pkg/mock/diskrpc.mock: -------------------------------------------------------------------------------- 1 | github.com/aleofreddi/csi-sanlock-lvm/pkg/diskrpc DiskRpc 2 | -------------------------------------------------------------------------------- /pkg/mock/filesystem.mock: -------------------------------------------------------------------------------- 1 | github.com/aleofreddi/csi-sanlock-lvm/pkg/driverd FileSystem 2 | -------------------------------------------------------------------------------- /pkg/mock/filesystemregistry.mock: -------------------------------------------------------------------------------- 1 | github.com/aleofreddi/csi-sanlock-lvm/pkg/driverd FileSystemRegistry 2 | -------------------------------------------------------------------------------- /pkg/mock/lvmctrldclient.mock: -------------------------------------------------------------------------------- 1 | github.com/aleofreddi/csi-sanlock-lvm/pkg/proto LvmCtrldClient 2 | -------------------------------------------------------------------------------- /pkg/mock/mount.mock: -------------------------------------------------------------------------------- 1 | k8s.io/utils/mount Interface 2 | -------------------------------------------------------------------------------- /pkg/mock/volumelocker.mock: -------------------------------------------------------------------------------- 1 | github.com/aleofreddi/csi-sanlock-lvm/pkg/driverd VolumeLocker 2 | -------------------------------------------------------------------------------- /pkg/proto/diskrpc.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | import "google/protobuf/timestamp.proto"; 18 | 19 | option go_package = "github.com/aleofreddi/csi-sanlock-lvm/proto"; 20 | 21 | message MailBoxMessage { 22 | uint32 next = 1; 23 | uint32 sender = 2; 24 | uint32 length = 3; 25 | bytes payload = 4; 26 | } 27 | 28 | enum DiskRpcType { 29 | DISK_RPC_TYPE_UNKNOWN = 0; 30 | DISK_RPC_TYPE_REQUEST = 1; 31 | DISK_RPC_TYPE_RESPONSE = 2; 32 | } 33 | 34 | message DiskRpcMessage { 35 | google.protobuf.Timestamp time = 1; 36 | DiskRpcType type = 2; 37 | uint32 channel = 3; 38 | bytes uuid = 4; 39 | string method = 5; 40 | bytes request = 6; 41 | bytes response = 7; 42 | string error_msg = 8; 43 | uint32 error_code = 9; 44 | } 45 | 46 | -------------------------------------------------------------------------------- /pkg/proto/lvmctrld.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | import "google/protobuf/timestamp.proto"; 18 | 19 | option go_package = "github.com/aleofreddi/csi-sanlock-lvm/proto"; 20 | 21 | message GetStatusRequest { 22 | } 23 | 24 | message GetStatusResponse { 25 | uint32 node_id = 1; 26 | } 27 | 28 | enum LvActivationMode { 29 | LV_ACTIVATION_MODE_NONE = 0; 30 | LV_ACTIVATION_MODE_ACTIVE_EXCLUSIVE = 1; 31 | LV_ACTIVATION_MODE_ACTIVE_SHARED = 2; 32 | LV_ACTIVATION_MODE_DEACTIVATE = 3; 33 | } 34 | 35 | message VolumeGroup { 36 | string vg_name = 1; 37 | uint32 pv_count = 2; 38 | uint32 lv_count = 3; 39 | uint32 snap_count = 4; 40 | string vg_attr = 5; 41 | uint64 vg_size = 6; 42 | uint64 vg_free = 7; 43 | repeated string vg_tags = 8; 44 | } 45 | 46 | message VgsRequest { 47 | string select = 2; 48 | repeated string target = 3; 49 | } 50 | 51 | message VgsResponse { 52 | repeated VolumeGroup vgs = 1; 53 | } 54 | 55 | enum LvDeviceOpen { 56 | LV_DEVICE_OPEN_UNKNOWN = 0; 57 | LV_DEVICE_OPEN_OPEN = 1; 58 | } 59 | 60 | message LogicalVolume { 61 | string lv_name = 1; 62 | string vg_name = 2; 63 | string lv_attr = 3; 64 | uint64 lv_size = 4; 65 | string pool_lv = 5; 66 | string origin = 6; 67 | string data_percent = 7; 68 | string metadata_percent = 8; 69 | string move_pv = 9; 70 | string mirror_log = 10; 71 | string copy_percent = 11; 72 | string convert_lv = 12; 73 | repeated string lv_tags = 13; 74 | repeated string lv_role = 14; 75 | google.protobuf.Timestamp lv_time = 15; 76 | LvDeviceOpen lv_device_open = 16; 77 | } 78 | 79 | message LvCreateRequest { 80 | string vg_name = 1; 81 | string lv_name = 2; 82 | uint64 size = 3; 83 | repeated string lv_tags = 4; 84 | string origin = 5; 85 | LvActivationMode activate = 6; 86 | } 87 | 88 | message LvCreateResponse { 89 | } 90 | 91 | message LvRemoveRequest { 92 | string select = 3; 93 | repeated string target = 4; 94 | } 95 | 96 | message LvRemoveResponse { 97 | } 98 | 99 | message LvsRequest { 100 | string select = 2; 101 | repeated string sort = 3; 102 | repeated string target = 4; 103 | } 104 | 105 | message LvsResponse { 106 | repeated LogicalVolume lvs = 1; 107 | } 108 | 109 | message LvChangeRequest { 110 | LvActivationMode activate = 2; 111 | repeated string add_tag = 3; 112 | repeated string del_tag = 4; 113 | string select = 5; 114 | repeated string target = 6; 115 | } 116 | 117 | message LvChangeResponse { 118 | } 119 | 120 | message LvResizeRequest { 121 | string vg_name = 1; 122 | string lv_name = 2; 123 | uint64 size = 3; 124 | } 125 | 126 | message LvResizeResponse { 127 | } 128 | 129 | service LvmCtrld { 130 | rpc GetStatus (GetStatusRequest) returns (GetStatusResponse) { 131 | } 132 | rpc Vgs (VgsRequest) returns (VgsResponse) { 133 | } 134 | rpc LvCreate (LvCreateRequest) returns (LvCreateResponse) { 135 | } 136 | rpc LvRemove (LvRemoveRequest) returns (LvRemoveResponse) { 137 | } 138 | rpc Lvs (LvsRequest) returns (LvsResponse) { 139 | } 140 | rpc LvChange (LvChangeRequest) returns (LvChangeResponse) { 141 | } 142 | rpc LvResize (LvResizeRequest) returns (LvResizeResponse) { 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /pkg/proto/prototest/helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototest 16 | 17 | import ( 18 | "github.com/golang/protobuf/proto" 19 | "reflect" 20 | ) 21 | 22 | // Without returns a clone of the given a protobuf message with all the given fields cleared. 23 | func Without[T proto.Message](msg T, fields ...string) T { 24 | msg = proto.Clone(msg).(T) 25 | vs := reflect.Indirect(reflect.ValueOf(msg)) 26 | for _, field := range fields { 27 | f := vs.FieldByName(field) 28 | f.Set(reflect.Zero(f.Type())) 29 | } 30 | return msg 31 | } 32 | 33 | // Merge returns the result of merging two messages. 34 | func Merge[T proto.Message](dst, src T) T { 35 | dst = proto.Clone(dst).(T) 36 | proto.Merge(dst, src) 37 | return dst 38 | } 39 | --------------------------------------------------------------------------------