├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature-request.md ├── pull_request_template.md └── workflows │ ├── build.yml │ ├── fossa.yml │ ├── pull_request.yml │ └── release.yml ├── .gitignore ├── .golangci.yml ├── .muse └── config.toml ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── GOVERNANCE.md ├── LICENSE ├── MAINTAINERS.md ├── Makefile ├── Makefile.buildx.mk ├── README.md ├── RELEASE.md ├── changelogs └── released │ ├── v1.10.0-RC1 │ ├── 72-mynktl │ ├── 76-mynktl │ └── 79-mynktl │ ├── v1.10.0-RC2 │ └── 85-mynktl │ ├── v1.11.0-RC2 │ └── 93-sonasingh46 │ ├── v2.1.0 │ ├── 102-pawanpraka1 │ ├── 108-mittachaitu │ ├── 110-pawanpraka1 │ └── 111-pawanpraka1 │ ├── v2.2.0-RC1 │ ├── 115-mynktl │ ├── 116-pawanpraka1 │ ├── 117-pawanpraka1 │ ├── 118-pawanpraka1 │ └── 121-pawanpraka1 │ ├── v2.2.0-RC2 │ ├── 124-pawanpraka1 │ └── 128-pawanpraka1 │ ├── v2.3.0-RC1 │ ├── 132-shubham14bajpai │ ├── 133-prateek │ └── 99-mynktl │ ├── v2.4.0-RC1 │ ├── 131-zlymeda │ ├── 139-pawanpraka1 │ └── 140-mynktl │ ├── v2.5.0-RC1 │ └── 144-mynktl │ ├── v2.7.0-RC1 │ ├── 147-pawanpraka1 │ └── 149-shubham14bajpai │ └── v2.9.0-RC1 │ ├── 154-mynktl │ └── 161-prateekpandey14 ├── code-standard.md ├── developer-setup.md ├── example ├── 05-backupstoragelocation.yaml ├── 06-local-volumesnapshotlocation.yaml ├── 06-volumesnapshotlocation.yaml └── 10-deployment.yaml ├── go.mod ├── go.sum ├── pkg ├── clouduploader │ ├── conn.go │ ├── operation.go │ ├── server.go │ └── server_utils.go ├── cstor │ ├── api_service.go │ ├── cstor.go │ ├── cvr_operation.go │ ├── pv_operation.go │ ├── pvc_operation.go │ └── status.go ├── snapshot │ └── snap.go ├── velero │ ├── restore.go │ ├── sort.go │ └── velero.go └── zfs │ ├── plugin │ ├── backup.go │ ├── restore.go │ └── zfs.go │ ├── snapshot │ └── snap.go │ └── utils │ └── utils.go ├── plugin.Dockerfile ├── script ├── buildxpush.sh ├── install-openebs.sh ├── install-velero.sh ├── minio-credentials ├── minio.yaml └── volumesnapshotlocation.yaml ├── tests ├── README.md ├── app │ ├── application_setup.go │ └── application_yaml.go ├── config │ └── config.go ├── k8s │ ├── exec.go │ ├── log.go │ ├── status.go │ └── storage_install.go ├── openebs │ ├── logs.go │ ├── storage_install.go │ ├── storage_status.go │ └── storage_yaml.go ├── sanity │ └── backup_test.go └── velero │ ├── backup.go │ ├── logs.go │ ├── restore.go │ ├── schedule.go │ └── status.go └── velero-blockstore-openebs └── main.go /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Default code owners 2 | * @openebs/localpv-zfs-approvers 3 | 4 | /.github/CODEOWNERS @openebs/org-maintainers 5 | 6 | # Policy docs owners 7 | /CODE_OF_CONDUCT.md @openebs/org-maintainers 8 | /CONTRIBUTING.md @openebs/org-maintainers 9 | /GOVERNANCE.md @openebs/org-maintainers 10 | /LICENSE @openebs/org-maintainers 11 | /MAINTAINERS.md @openebs/org-maintainers 12 | /SECURITY.md @openebs/org-maintainers 13 | /SECURITY_CONTACTS.md @openebs/org-maintainers 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Tell us about a problem you are experiencing 4 | labels: Bug 5 | 6 | --- 7 | 8 | **What steps did you take and what happened:** 9 | [A clear and concise description of what the bug is, and what commands you ran.] 10 | 11 | 12 | **What did you expect to happen:** 13 | 14 | 15 | **The output of the following commands will help us better understand what's going on**: 16 | (Pasting long output into a [GitHub gist](https://gist.github.com) or other [Pastebin](https://pastebin.com/) is fine.) 17 | 18 | * `kubectl logs deployment/velero -n velero` 19 | * `kubectl logs deployment/maya-apiserver -n openebs` 20 | * `velero backup describe ` or `kubectl get backup/ -n velero -o yaml` 21 | * `velero backup logs ` 22 | * `velero restore describe ` or `kubectl get restore/ -n velero -o yaml` 23 | * `velero restore logs ` 24 | 25 | 26 | **Anything else you would like to add:** 27 | [Miscellaneous information that will assist in solving the issue.] 28 | 29 | 30 | **Environment:** 31 | - Velero version (use `velero version`): 32 | - Velero features (use `velero client config get features`): 33 | - Velero-plugin version 34 | - OpenEBS version 35 | - Kubernetes version (use `kubectl version`): 36 | - Kubernetes installer & version: 37 | - Cloud provider or hardware configuration: 38 | - OS (e.g. from `/etc/os-release`): 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | labels: Enhancement 5 | 6 | --- 7 | 8 | **Describe the problem/challenge you have** 9 | [A description of the current limitation/problem/challenge that you are experiencing.] 10 | 11 | 12 | **Describe the solution you'd like** 13 | [A clear and concise description of what you want to happen.] 14 | 15 | 16 | **Anything else you would like to add:** 17 | [Miscellaneous information that will assist in solving the issue.] 18 | 19 | 20 | **Environment:** 21 | - Velero version (use `velero version`): 22 | - Velero features (use `velero client config get features`): 23 | - Velero-plugin version 24 | - OpenEBS version 25 | - Kubernetes version (use `kubectl version`): 26 | - Kubernetes installer & version: 27 | - Cloud provider or hardware configuration: 28 | - OS (e.g. from `/etc/os-release`): 29 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Pull Request template 2 | 3 | Please, go through these steps before you submit a PR. ## Remove this line 4 | 5 | **Why is this PR required? What issue does it fix?**: 6 | 7 | 8 | **What this PR does?**: 9 | 10 | 11 | **Does this PR require any upgrade changes?**: 12 | 13 | 14 | **If the changes in this PR are manually verified, list down the scenarios covered and commands you used for testing with logs:** 15 | 16 | 17 | **Any additional information for your reviewer?**: 18 | _Mention if this PR is part of any design or a continuation of previous PRs_ 19 | 20 | 21 | **Checklist:** 22 | - [ ] Fixes # 23 | - [ ] PR Title follows the convention of `(): ` 24 | - [ ] Has the change log section been updated? 25 | - [ ] Commit has unit tests 26 | - [ ] Commit has integration tests 27 | - [ ] (Optional) Are upgrade changes included in this PR? If not, mention the issue/PR to track: 28 | - [ ] (Optional) If documentation changes are required, which issue on https://github.com/openebs/openebs-docs is used to track them: 29 | 30 | 31 | **PLEASE REMOVE THIS TEMPLATE BEFORE SUBMITTING** 32 | 33 | PR title/commit message must follow convention: `(): `. 34 | 35 | Most common types are: 36 | * `feat` - for new features, not a new feature for build script 37 | * `fix` - for bug fixes or improvements, not a fix for build script 38 | * `chore` - changes not related to production code 39 | * `docs` - changes related to documentation 40 | * `style` - formatting, missing semi colons, linting fix etc; no significant production code changes 41 | * `test` - adding missing tests, refactoring tests; no production code change 42 | * `refactor` - refactoring production code, eg. renaming a variable or function name, there should not be any significant production code changes 43 | 44 | IMPORTANT: Please review the [CONTRIBUTING.md](../CONTRIBUTING.md) file for detailed contributing guidelines. 45 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018-2020 The OpenEBS Authors. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | name: build 15 | 16 | on: 17 | create: 18 | push: 19 | branches: 20 | - 'develop' 21 | - 'v*' 22 | paths-ignore: 23 | - '*.md' 24 | - 'changelogs/**' 25 | - 'example/**' 26 | - 'LICENSE' 27 | - 'MAINTAINERS' 28 | 29 | jobs: 30 | lint: 31 | # to ignore builds on release 32 | if: ${{ (github.event.ref_type != 'tag') }} 33 | runs-on: ubuntu-latest 34 | steps: 35 | - name: Checkout 36 | uses: actions/checkout@v4 37 | 38 | - name: Shellcheck 39 | uses: reviewdog/action-shellcheck@v1 40 | with: 41 | github_token: ${{ secrets.github_token }} 42 | reporter: github-pr-review 43 | path: '.' 44 | pattern: '*.sh' 45 | exclude: './vendor/*' 46 | test: 47 | runs-on: ubuntu-latest 48 | needs: ['lint'] 49 | steps: 50 | - name: Checkout 51 | uses: actions/checkout@v4 52 | 53 | - name: Set up Go 1.14 54 | uses: actions/setup-go@v5 55 | with: 56 | go-version: 1.14.7 57 | 58 | - name: Setup Minikube-Kubernetes 59 | uses: manusa/actions-setup-minikube@v2.3.0 60 | with: 61 | minikube version: v1.16.0 62 | kubernetes version: v1.20.1 63 | github token: ${{ secrets.GITHUB_TOKEN }} 64 | 65 | - name: Build images locally 66 | run: make build && make container || exit 1; 67 | 68 | - name: Setting environment variables 69 | run: | 70 | kubectl cluster-info 71 | echo "KUBECONFIG=$HOME/.kube/config" >> $GITHUB_ENV 72 | echo "VELERO_RELEASE=v1.6.0" >> $GITHUB_ENV 73 | echo "OPENEBS_RELEASE=master" >> $GITHUB_ENV 74 | 75 | - name: Installation 76 | run: | 77 | ./script/install-openebs.sh 78 | ./script/install-velero.sh 79 | 80 | - name: Running tests 81 | run: make test 82 | 83 | plugin: 84 | runs-on: ubuntu-latest 85 | needs: ['test'] 86 | steps: 87 | - name: Checkout 88 | uses: actions/checkout@v4 89 | 90 | - name: Set Image Org 91 | # sets the default IMAGE_ORG to openebs 92 | run: | 93 | [ -z "${{ secrets.IMAGE_ORG }}" ] && IMAGE_ORG=openebs || IMAGE_ORG=${{ secrets.IMAGE_ORG }} 94 | echo "IMAGE_ORG=${IMAGE_ORG}" >> $GITHUB_ENV 95 | 96 | - name: Set Build Date 97 | id: date 98 | run: | 99 | echo "DATE=$(date -u +'%Y-%m-%dT%H:%M:%S%Z')" >> $GITHUB_OUTPUT 100 | 101 | - name: Set Tag 102 | run: | 103 | BRANCH="${GITHUB_REF##*/}" 104 | CI_TAG=${BRANCH#v}-ci 105 | if [ ${BRANCH} = "develop" ]; then 106 | CI_TAG="ci" 107 | fi 108 | echo "TAG=${CI_TAG}" >> $GITHUB_ENV 109 | echo "BRANCH=${BRANCH}" >> $GITHUB_ENV 110 | 111 | - name: Docker meta 112 | id: docker_meta 113 | uses: docker/metadata-action@v5 114 | with: 115 | # add each registry to which the image needs to be pushed here 116 | images: | 117 | ${{ env.IMAGE_ORG }}/velero-plugin 118 | quay.io/${{ env.IMAGE_ORG }}/velero-plugin 119 | ghcr.io/${{ env.IMAGE_ORG }}/velero-plugin 120 | tags: | 121 | type=raw,value=latest,enable=false 122 | type=raw,value=${{ env.TAG }} 123 | 124 | - name: Print Tag info 125 | run: | 126 | echo "BRANCH: ${BRANCH}" 127 | echo "${{ steps.docker_meta.outputs.tags }}" 128 | 129 | - name: Set up QEMU 130 | uses: docker/setup-qemu-action@v3 131 | with: 132 | platforms: all 133 | 134 | - name: Set up Docker Buildx 135 | id: buildx 136 | uses: docker/setup-buildx-action@v3 137 | with: 138 | version: v0.5.1 139 | 140 | - name: Login to Docker Hub 141 | uses: docker/login-action@v3 142 | with: 143 | username: ${{ secrets.DOCKERHUB_USERNAME }} 144 | password: ${{ secrets.DOCKERHUB_TOKEN }} 145 | 146 | - name: Login to Quay 147 | uses: docker/login-action@v3 148 | with: 149 | registry: quay.io 150 | username: ${{ secrets.QUAY_USERNAME }} 151 | password: ${{ secrets.QUAY_TOKEN }} 152 | 153 | - name: Login to GHCR 154 | uses: docker/login-action@v3 155 | with: 156 | registry: ghcr.io 157 | username: ${{ github.actor }} 158 | password: ${{ secrets.GITHUB_TOKEN }} 159 | 160 | - name: Build & Push Image 161 | uses: docker/build-push-action@v6 162 | with: 163 | context: . 164 | file: ./plugin.Dockerfile 165 | push: true 166 | platforms: linux/amd64, linux/arm64 167 | tags: | 168 | ${{ steps.docker_meta.outputs.tags }} 169 | build-args: | 170 | DBUILD_DATE=${{ steps.date.outputs.DATE }} 171 | DBUILD_REPO_URL=https://github.com/openebs/velero-plugin 172 | DBUILD_SITE_URL=https://openebs.io 173 | BRANCH=${{ env.BRANCH }} 174 | 175 | -------------------------------------------------------------------------------- /.github/workflows/fossa.yml: -------------------------------------------------------------------------------- 1 | name: Fossa CLI 2 | on: 3 | push: 4 | branches: 5 | - 'develop' 6 | - 'release/**' 7 | 8 | jobs: 9 | fossa-scan: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | with: 14 | submodules: recursive 15 | - uses: fossas/fossa-action@v1.4.0 16 | with: 17 | api-key: ${{ secrets.FOSSA_API_KEY }} 18 | -------------------------------------------------------------------------------- /.github/workflows/pull_request.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018-2020 The OpenEBS Authors. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: ci 16 | 17 | on: 18 | pull_request: 19 | branches: 20 | # on pull requests to develop and release branches 21 | - 'develop' 22 | - 'v*' 23 | paths-ignore: 24 | - '*.md' 25 | - 'changelogs/**' 26 | - 'example/**' 27 | - 'LICENSE' 28 | - 'MAINTAINERS.md' 29 | 30 | jobs: 31 | lint: 32 | runs-on: ubuntu-22.04 33 | steps: 34 | - name: Checkout 35 | uses: actions/checkout@v4 36 | 37 | - name: Shellcheck 38 | uses: reviewdog/action-shellcheck@v1 39 | with: 40 | github_token: ${{ secrets.github_token }} 41 | reporter: github-pr-review 42 | path: '.' 43 | pattern: '*.sh' 44 | exclude: './vendor/*' 45 | 46 | test: 47 | runs-on: ubuntu- 48 | needs: ['lint'] 49 | steps: 50 | - name: Checkout 51 | uses: actions/checkout@v4 52 | 53 | - name: Set up Go 1.14 54 | uses: actions/setup-go@v5 55 | with: 56 | go-version: 1.14.7 57 | 58 | - name: Setup Minikube-Kubernetes 59 | uses: medyagh/setup-minikube@latest 60 | with: 61 | cache: false 62 | minikube-version: 1.31.1 63 | driver: none 64 | kubernetes-version: v1.29.13 65 | cni: calico 66 | start-args: "--install-addons=false" 67 | 68 | - name: Build images locally 69 | run: make build && make container || exit 1; 70 | 71 | - name: Setting environment variables 72 | run: | 73 | kubectl cluster-info 74 | echo "KUBECONFIG=$HOME/.kube/config" >> $GITHUB_ENV 75 | echo "VELERO_RELEASE=v1.6.0" >> $GITHUB_ENV 76 | echo "OPENEBS_RELEASE=master" >> $GITHUB_ENV 77 | 78 | - name: Installation 79 | run: | 80 | ./script/install-openebs.sh 81 | ./script/install-velero.sh 82 | 83 | - name: Running tests 84 | run: make test 85 | 86 | plugin: 87 | runs-on: ubuntu-latest 88 | needs: ['test'] 89 | steps: 90 | - name: Checkout 91 | uses: actions/checkout@v4 92 | 93 | - name: Set up QEMU 94 | uses: docker/setup-qemu-action@v3 95 | with: 96 | platforms: all 97 | 98 | - name: Set up Docker Buildx 99 | id: buildx 100 | uses: docker/setup-buildx-action@v3 101 | with: 102 | version: v0.5.1 103 | 104 | - name: Build Image 105 | uses: docker/build-push-action@v6 106 | with: 107 | context: . 108 | file: ./plugin.Dockerfile 109 | push: false 110 | platforms: linux/amd64, linux/arm64 111 | tags: | 112 | openebs/velero-plugin:ci 113 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018-2020 The OpenEBS Authors. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | name: release 15 | 16 | on: 17 | release: 18 | types: 19 | - 'created' 20 | tags: 21 | - 'v*' 22 | 23 | jobs: 24 | plugin: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - name: Checkout 28 | uses: actions/checkout@v4 29 | 30 | - name: Set Image Org 31 | # sets the default IMAGE_ORG to openebs 32 | run: | 33 | [ -z "${{ secrets.IMAGE_ORG }}" ] && IMAGE_ORG=openebs || IMAGE_ORG=${{ secrets.IMAGE_ORG }} 34 | echo "IMAGE_ORG=${IMAGE_ORG}" >> $GITHUB_ENV 35 | 36 | - name: Set Build Date 37 | id: date 38 | run: | 39 | echo "DATE=$(date -u +'%Y-%m-%dT%H:%M:%S%Z')" >> $GITHUB_OUTPUT 40 | 41 | - name: Set Tag 42 | run: | 43 | TAG="${GITHUB_REF#refs/*/v}" 44 | echo "TAG=${TAG}" >> $GITHUB_ENV 45 | echo "RELEASE_TAG=${TAG}" >> $GITHUB_ENV 46 | 47 | - name: Docker meta 48 | id: docker_meta 49 | uses: docker/metadata-action@v5 50 | with: 51 | # add each registry to which the image needs to be pushed here 52 | images: | 53 | ${{ env.IMAGE_ORG }}/velero-plugin 54 | quay.io/${{ env.IMAGE_ORG }}/velero-plugin 55 | ghcr.io/${{ env.IMAGE_ORG }}/velero-plugin 56 | tags: | 57 | type=semver,pattern={{version}} 58 | 59 | - name: Print Tag info 60 | run: | 61 | echo "${{ steps.docker_meta.outputs.tags }}" 62 | echo "RELEASE TAG: ${RELEASE_TAG}" 63 | 64 | - name: Set up QEMU 65 | uses: docker/setup-qemu-action@v3 66 | with: 67 | platforms: all 68 | 69 | - name: Set up Docker Buildx 70 | id: buildx 71 | uses: docker/setup-buildx-action@v3 72 | with: 73 | version: v0.5.1 74 | 75 | - name: Login to Docker Hub 76 | uses: docker/login-action@v3 77 | with: 78 | username: ${{ secrets.DOCKERHUB_USERNAME }} 79 | password: ${{ secrets.DOCKERHUB_TOKEN }} 80 | 81 | - name: Login to Quay 82 | uses: docker/login-action@v3 83 | with: 84 | registry: quay.io 85 | username: ${{ secrets.QUAY_USERNAME }} 86 | password: ${{ secrets.QUAY_TOKEN }} 87 | 88 | - name: Login to GHCR 89 | uses: docker/login-action@v3 90 | with: 91 | registry: ghcr.io 92 | username: ${{ github.actor }} 93 | password: ${{ secrets.GITHUB_TOKEN }} 94 | 95 | - name: Build & Push Image 96 | uses: docker/build-push-action@v6 97 | with: 98 | context: . 99 | file: ./plugin.Dockerfile 100 | push: true 101 | platforms: linux/amd64, linux/arm64 102 | tags: | 103 | ${{ steps.docker_meta.outputs.tags }} 104 | build-args: | 105 | DBUILD_DATE=${{ steps.date.outputs.DATE }} 106 | DBUILD_REPO_URL=https://github.com/openebs/velero-plugin 107 | DBUILD_SITE_URL=https://openebs.io 108 | RELEASE_TAG=${{ env.RELEASE_TAG }} 109 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.out 2 | _output 3 | push 4 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | golint: 3 | min-confidence: 0 4 | govet: 5 | check-shadowing: true 6 | misspell: 7 | locale: US 8 | nakedret: 9 | max-func-lines: 20 10 | gocritic: 11 | enabled-tags: 12 | - diagnostic 13 | - experimental 14 | - opinionated 15 | - performance 16 | - style 17 | disabled-checks: 18 | - dupImport # https://github.com/go-critic/go-critic/issues/845 19 | - ifElseChain 20 | - octalLiteral 21 | - whyNoLint 22 | - wrapperFunc 23 | - rangeValCopy 24 | - unnamedResult 25 | - hugeParam 26 | - sloppyReassign 27 | linters: 28 | disable-all: true 29 | 30 | run: 31 | timeout: 10m 32 | 33 | issues-exit-code: 10 34 | 35 | skip-dirs: 36 | - vendor 37 | -------------------------------------------------------------------------------- /.muse/config.toml: -------------------------------------------------------------------------------- 1 | ignoreRules = [ "G101", 2 | "ST1005" 3 | ] 4 | 5 | ignoreFiles = """ 6 | vendor/** 7 | """ 8 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 |
3 | 4 | ## Umbrella Project 5 | OpenEBS is an "umbrella project". Every project, repository and file in the OpenEBS organization adopts and follows the policies found in the Community repo umbrella project files. 6 |
7 | 8 | This project follows the [OpenEBS Code of Conduct](https://github.com/openebs/community/blob/HEAD/CODE_OF_CONDUCT.md) 9 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 |
3 | 4 | ## Umbrella Project 5 | OpenEBS is an "umbrella project". Every project, repository and file in the OpenEBS organization adopts and follows the policies found in the Community repo umbrella project files. 6 |
7 | 8 | This project follows the [OpenEBS Contributor Guidelines](https://github.com/openebs/community/blob/HEAD/CONTRIBUTING.md) 9 | 10 | # Contributing to velero-plugin 11 | 12 | velero-plugin is an open-source project. 13 | 14 | velero-plugin uses the standard GitHub pull requests process to review and accept contributions. 15 | 16 | You can contribute to velero-plugin by filling an issue at [openebs/velero-plugin](https://github.com/openebs/velero-plugin/issues) or submitting a pull request to this repository. 17 | 18 | * If you want to file an issue for bug or feature request, please see [Filing an issue](#filing-an-issue) 19 | * If you are a first-time contributor, please see [Steps to Contribute](#steps-to-contribute) and code standard(code-standard.md). 20 | * If you would like to work on something more involved, please connect with the OpenEBS Contributors. See [OpenEBS Community](https://github.com/openebs/openebs/tree/HEAD/community) 21 | 22 | ## Filing an issue 23 | 24 | ### Before filing an issue 25 | 26 | If you are unsure whether you have found a bug, please consider asking in the [Slack](https://kubernetes.slack.com/messages/openebs) first. If 27 | the behavior you are seeing is confirmed as a bug or issue, it can easily be re-raised in the [issue tracker](https://github.com/openebs/velero-plugin/issues). 28 | 29 | ### Filing issue 30 | 31 | When filing an issue, make sure to answer these seven questions: 32 | 33 | 1. What version of Velero are you using (`velero version`)? 34 | 2. What version of OpenEBS are you using? 35 | 3. What version of velero-plugin are you using? 36 | 4. What steps did you follow to create a backup or restore? 37 | 5. What did you expect to see? 38 | 6. What did you see instead? 39 | 7. Logs of velero/velero pod and openebs/maya-apiserver pod. 40 | 41 | #### For maintainers 42 | 43 | * We are using labeling for the issue to track it more effectively. The following are valid labels for the issue. 44 | - **Bug** - If the issue is a **bug to existing feature** 45 | - **Enhancement** - If the issue is a **feature request** 46 | - **Maintenance** - If the issue is not related to production code. **build, document or test related issues fall into this category** 47 | - **Question** - If the issue is about **querying information about how the product or build works, or internal of product**. 48 | - **Documentation** - If the issue is about **tracking the documentation work for the feature**. This label should not be applied to the issue of a bug in documentations. 49 | - **Good First Issue** - If the issue is easy to get started with. Please make sure that the issue should be ideal for beginners to dive into the codebase. 50 | - **Design** - If the issue **needs a design decision prior to code implementation** 51 | - **Duplicate** - If the issue is **duplicate of another issue** 52 | 53 | * We are using following labels for issue work-flow: 54 | - **Backlog** - If the issue has **not been planned for current release cycle** 55 | - **Release blocker** - If the issue is **blocking the release** 56 | - **Priority: high** - issue with this label **should be resolved as quickly as possible** 57 | - **Priority: low** - issue with this label **won’t have the immediate focus of the core team** 58 | 59 | **If you want to introduce a new label then you need to raise a PR to update this document with the new label details.** 60 | 61 | ## Steps to Contribute 62 | 63 | velero-plugin is an Apache 2.0 Licensed project and all your commits should be signed with Developer Certificate of Origin. See [Sign your work](#sign-your-work). 64 | 65 | For setting up a development environment on your local machine, see the detailed instructions [here](developer-setup.md). 66 | 67 | * Find an issue to work on or create a new issue. The issues are maintained at [openebs/velero-plugin](https://github.com/openebs/velero-plugin/issues). You can pick up from a list of [good-first-issues](https://github.com/openebs/velero-plugin/labels/good%20first%20issue). 68 | * Claim your issue by commenting your intent to work on it to avoid duplication of efforts. 69 | * Fork the repository on GitHub. 70 | * Create a branch from where you want to base your work (usually develop). 71 | * Commit your changes by making sure the commit messages convey the need and notes about the commit. 72 | * Please make sure than your code is aligned with the standard mentioned at [code-standard](code-standard.md). 73 | * Verify that your changes pass `make lint` or `make lint-docker` (docker version of `make lint`) 74 | * Push your changes to the branch in your fork of the repository. 75 | * Submit a pull request to the original repository. See [Pull Request checklist](#pull-request-checklist) 76 | 77 | ## Pull Request Checklist 78 | 79 | * Rebase to the current develop branch before submitting your pull request. 80 | * Commits should be as small as possible. Each commit should follow the checklist below: 81 | - For code changes, add tests relevant to the fixed bug or new feature. 82 | - Commit header (first line) should convey what changed 83 | - Commit body should include details such as why the changes are required and how the proposed changes help 84 | - DCO Signed, please refer [signing commit](code-standard.md#sign-your-commits) 85 | * If your PR is about fixing an issue or new feature, make sure you add a change-log. Refer [Adding a Change log](code-standard.md#adding-a-changelog) 86 | * PR title must follow convention: `(): `. 87 | 88 | For example: 89 | 90 | ``` 91 | feat(backup): support for backup to aws 92 | ^--^ ^-----^ ^-----------------------^ 93 | | | | 94 | | | +-> PR subject, summary of the changes 95 | | | 96 | | +-> scope of the PR, i.e. component of the project this PR is intend to update 97 | | 98 | +-> type of the PR. 99 | ``` 100 | 101 | Most common types are: 102 | * `feat` - for new features, not a new feature for the build script 103 | * `fix` - for bug fixes or improvements, not a fix for the build script 104 | * `chore` - changes not related to production code 105 | * `docs` - changes related to documentation 106 | * `style` - formatting, missing semicolons, linting fix, etc; no significant production code changes 107 | * `test` - adding missing tests, refactoring tests; no production code change 108 | * `refactor` - refactoring production code, eg. renaming a variable or function name, there should not be any significant production code changes 109 | * `cherry-pick` - if PR is merged in the develop branch and raised to release branch(like v1.9.x) 110 | --- 111 | 112 | ### Sign your work 113 | 114 | We use the Developer Certificate of Origin (DCO) as an additional safeguard for the OpenEBS project. This is a well established and widely used mechanism to assure contributors have confirmed their right to license their contribution under the project's license. Please read [developer-certificate-of-origin](./contribute/developer-certificate-of-origin). 115 | 116 | Please certify it by just adding a line to every git commit message. Any PR with Commits which does not have DCO Signoff will not be accepted: 117 | 118 | ``` 119 | Signed-off-by: Random J Developer 120 | ``` 121 | 122 | or use the command `git commit -s -m "commit message comes here"` to sign-off on your commits. 123 | 124 | Use your real name (sorry, no pseudonyms or anonymous contributions). If you set your `user.name` and `user.email` git configs, you can sign your commit automatically with `git commit -s`. You can also use git [aliases](https://git-scm.com/book/en/v2/Git-Basics-Git-Aliases) like `git config --global alias.ci 'commit -s'`. Now you can commit with `git ci` and the commit will be signed. 125 | 126 | --- 127 | 128 | ## Code Reviews 129 | 130 | All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult [GitHub Help](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests) for more information on using pull requests. 131 | 132 | * If your PR is not getting reviewed or you need a specific person to review it, please reach out to the OpenEBS Contributors. See [OpenEBS Community](https://github.com/openebs/openebs/tree/HEAD/community) 133 | 134 | * If PR is fixing any issues from [github-issues](github.com/openebs/velero-plugin/issues) then you need to mention the issue number with a link in PR description. like: _fixes _ 135 | 136 | * If PR is for bug-fix and release branch(like v1.9.x) is created then cherry-pick for the same PR needs to be created against the release branch. Maintainer of the Project needs to make sure that all the bug fixes after RC release are cherry-picked to release branch and their changelog files are created under `changelogs/v1.9.x` instead of `changelogs/unreleased`, if release branch is `v1.10.x` then this folder will be `changelogs/v1.10.x` 137 | 138 | ## Design document 139 | 140 | Detailed design document for velero-plugin is available at [Google Doc](https://docs.google.com/document/d/1-4WsM0AjLORb3lTCUUGyYOY_LNdTOATFesi7kTAr7SA). 141 | 142 | ### For maintainers 143 | 144 | * We are using labeling for PR to track it more effectively. The following are valid labels for the PR. 145 | - **Bug** - if PR is a **bug to existing feature** 146 | - **Enhancement** - if PR is a **feature request** 147 | - **Maintenance** - if PR is not related to production code. **build, document or test related PR falls into this category** 148 | - **Documentation** - if PR is about **tracking the documentation work for the feature**. This label should not be applied to the PR fixing bug in documentations. 149 | 150 | * We are using the following label for PR work-flow: 151 | - **DO NOT MERGE** - if PR is about critical changes and no scope of testing before release branch creation 152 | - **On Hold** - if PR doesn't have sufficient changes, all the scenarios are not covered or changes are requested from contributor 153 | - **Release blocker** - if PR is created for the issue having label **Release blocker** 154 | - **Priority: high** - if PR is created for the issue having label **Priority: high** 155 | - **Priority: low** - if PR is created for the issue having label **Priority: low** 156 | 157 | * Maintainer needs to make sure that appropriate milestone and project tracker is assigned to the PR. 158 | 159 | **If you want to introduce a new label then you need to raise a PR to update this document with the new label details.** 160 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The OpenEBS Authors. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This Dockerfile builds velero-plugin 16 | 17 | FROM alpine:3.11.5 18 | RUN mkdir /plugins 19 | ADD velero-* /plugins/ 20 | USER nobody:nobody 21 | 22 | ARG ARCH 23 | ARG BUILD_DATE 24 | ARG DBUILD_DATE 25 | ARG DBUILD_REPO_URL 26 | ARG DBUILD_SITE_URL 27 | 28 | LABEL org.label-schema.schema-version="1.0" 29 | LABEL org.label-schema.name="velero-plugin" 30 | LABEL org.label-schema.description="OpenEBS velero-plugin" 31 | LABEL org.label-schema.build-date=$DBUILD_DATE 32 | LABEL org.label-schema.vcs-url=$DBUILD_REPO_URL 33 | LABEL org.label-schema.url=$DBUILD_SITE_URL 34 | 35 | ENTRYPOINT ["/bin/ash", "-c", "cp /plugins/* /target/."] 36 | -------------------------------------------------------------------------------- /GOVERNANCE.md: -------------------------------------------------------------------------------- 1 | # Governance 2 |
3 | 4 | ## Umbrella Project 5 | OpenEBS is an "umbrella project". Every project, repository and file in the OpenEBS organization adopts and follows the policies found in the Community repo umbrella project files. 6 |
7 | 8 | This project follows the [OpenEBS Governance](https://github.com/openebs/community/blob/HEAD/GOVERNANCE.md) 9 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # Maintainers 2 | 3 | ## Umbrella Project 4 | 5 | OpenEBS is an `Umbrella Project` whose governance and policies are defined in the [community](https://github.com/openebs/community/) repository. 6 | These policies are applicable to every sub-project, repository and file existing within the [OpenEBS GitHub organization](https://github.com/openebs/). 7 | 8 | Please refer to the [OpenEBS Maintainers](https://github.com/openebs/community/blob/HEAD/MAINTAINERS.md). -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The OpenEBS Authors 2 | # Copyright 2017 the Heptio Ark contributors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | BIN = $(wildcard velero-*) 17 | 18 | # list only velero-plugin source code directories 19 | PACKAGES = $(shell go list ./... | grep -v 'vendor') 20 | 21 | ARCH ?= $(shell go env GOARCH) 22 | 23 | 24 | # The images can be pushed to any docker/image registeries 25 | # like docker hub, quay. The registries are specified in 26 | # the script https://raw.githubusercontent.com/openebs/charts/gh-pages/scripts/release/buildscripts/push. 27 | # 28 | # The images of a project or company can then be grouped 29 | # or hosted under a unique organization key like `openebs` 30 | # 31 | # Each component (container) will be pushed to a unique 32 | # repository under an organization. 33 | # Putting all this together, an unique uri for a given 34 | # image comprises of: 35 | # //: 36 | # 37 | # IMAGE_ORG can be used to customize the organization 38 | # under which images should be pushed. 39 | # By default the organization name is `openebs`. 40 | 41 | ifeq (${IMAGE_ORG}, ) 42 | IMAGE_ORG="openebs" 43 | export IMAGE_ORG 44 | endif 45 | 46 | # Specify the date of build 47 | DBUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') 48 | 49 | # Specify the docker arg for repository url 50 | ifeq (${DBUILD_REPO_URL}, ) 51 | DBUILD_REPO_URL="https://github.com/openebs/velero-plugin" 52 | export DBUILD_REPO_URL 53 | endif 54 | 55 | # Specify the docker arg for website url 56 | ifeq (${DBUILD_SITE_URL}, ) 57 | DBUILD_SITE_URL="https://openebs.io" 58 | export DBUILD_SITE_URL 59 | endif 60 | 61 | export DBUILD_ARGS=--build-arg DBUILD_DATE=${DBUILD_DATE} --build-arg DBUILD_REPO_URL=${DBUILD_REPO_URL} --build-arg DBUILD_SITE_URL=${DBUILD_SITE_URL} 62 | 63 | IMAGE = ${IMAGE_ORG}/velero-plugin-amd64 64 | 65 | # if the architecture is arm64, image name will have arm64 suffix 66 | ifeq (${ARCH}, arm64) 67 | IMAGE = ${IMAGE_ORG}/velero-plugin-arm64 68 | endif 69 | 70 | ifeq (${IMAGE_TAG}, ) 71 | IMAGE_TAG = ci 72 | export IMAGE_TAG 73 | endif 74 | 75 | # Specify the date of build 76 | BUILD_DATE = $(shell date +'%Y%m%d%H%M%S') 77 | 78 | #List of linters used by docker lint and local lint 79 | LINTERS ?= "goconst,gofmt,goimports,gosec,unparam" 80 | 81 | all: build 82 | 83 | container: all 84 | @echo ">> building container" 85 | @cp Dockerfile _output/Dockerfile 86 | @sudo docker build -t $(IMAGE):$(IMAGE_TAG) ${DBUILD_ARGS} -f _output/Dockerfile _output 87 | 88 | build: 89 | @echo ">> building binary" 90 | @mkdir -p _output 91 | CGO_ENABLED=0 go build -v -o _output/$(BIN) ./$(BIN) 92 | 93 | gomod: ## Ensures fresh go.mod and go.sum. 94 | @echo ">> verifying go modules" 95 | @go mod tidy 96 | @go mod verify 97 | @git diff --exit-code -- go.sum go.mod 98 | 99 | # Run linter using docker image 100 | lint-docker: gomod 101 | @echo ">> running golangci-lint" 102 | @sudo docker run -i \ 103 | --rm -v $$(pwd):/app -w /app \ 104 | golangci/golangci-lint:v1.24.0 \ 105 | bash -c "GOGC=75 golangci-lint run -E $(LINTERS)" 106 | 107 | # Run linter using local binary 108 | lint: gomod 109 | @echo ">> running golangci-lint" 110 | @golangci-lint run -E $(LINTERS) 111 | 112 | test: 113 | @CGO_ENABLED=0 go test -v ${PACKAGES} -timeout 20m 114 | 115 | deploy-image: 116 | @curl --fail --show-error -s https://raw.githubusercontent.com/openebs/charts/gh-pages/scripts/release/buildscripts/push > ./push 117 | @chmod +x ./push 118 | @DIMAGE=${IMAGE} ./push 119 | 120 | clean: 121 | rm -rf .go _output 122 | 123 | 124 | .PHONY: check-license 125 | check-license: 126 | @echo ">> checking license header" 127 | @licRes=$$(for file in $$(find . -type f -iname '*.go' ! -path './vendor/*' ! -path './pkg/debug/*' ) ; do \ 128 | awk 'NR<=3' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \ 129 | done); \ 130 | if [ -n "$${licRes}" ]; then \ 131 | echo "license header checking failed:"; echo "$${licRes}"; \ 132 | exit 1; \ 133 | fi 134 | 135 | include Makefile.buildx.mk 136 | -------------------------------------------------------------------------------- /Makefile.buildx.mk: -------------------------------------------------------------------------------- 1 | # Copyright 2018-2020 The OpenEBS Authors. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # ============================================================================== 16 | # Build Options 17 | 18 | ifeq (${TAG}, ) 19 | export TAG=ci 20 | endif 21 | 22 | # Build velero-plugin docker image with buildx 23 | # Experimental docker feature to build cross platform multi-architecture docker images 24 | # https://docs.docker.com/buildx/working-with-buildx/ 25 | 26 | # default list of platforms for which multiarch image is built 27 | ifeq (${PLATFORMS}, ) 28 | export PLATFORMS="linux/amd64,linux/arm64" 29 | endif 30 | 31 | # if IMG_RESULT is unspecified, by default the image will be pushed to registry 32 | ifeq (${IMG_RESULT}, load) 33 | export PUSH_ARG="--load" 34 | # if load is specified, image will be built only for the build machine architecture. 35 | export PLATFORMS="local" 36 | else ifeq (${IMG_RESULT}, cache) 37 | # if cache is specified, image will only be available in the build cache, it won't be pushed or loaded 38 | # therefore no PUSH_ARG will be specified 39 | else 40 | export PUSH_ARG="--push" 41 | endif 42 | 43 | # Name of the multiarch image for velero-plugin 44 | DOCKERX_IMAGE_PLUGIN:=${IMAGE_ORG}/velero-plugin:${TAG} 45 | 46 | .PHONY: docker.buildx.plugin 47 | docker.buildx.plugin: 48 | export DOCKER_CLI_EXPERIMENTAL=enabled 49 | @if ! docker buildx ls | grep -q container-builder; then\ 50 | docker buildx create --platform ${PLATFORMS} --name container-builder --use;\ 51 | fi 52 | @docker buildx build --platform ${PLATFORMS} \ 53 | -t "$(DOCKERX_IMAGE_PLUGIN)" ${DBUILD_ARGS} -f $(PWD)/plugin.Dockerfile \ 54 | . ${PUSH_ARG} 55 | @echo "--> Build docker image: $(DOCKERX_IMAGE_PLUGIN)" 56 | @echo 57 | 58 | .PHONY: buildx.push.plugin 59 | buildx.push.plugin: 60 | BUILDX=true DIMAGE=${IMAGE_ORG}/velero-plugin ./script/buildxpush.sh 61 | 62 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Process 2 | velero-plugin follows a monthly release cadence. The scope of the release is determined by contributor availability. The scope is published in the [Release Tracker Projects](https://github.com/orgs/openebs/projects). 3 | 4 | ## Release Candidate Verification Checklist 5 | 6 | Every release has release candidate builds that are created starting from the third week into the release. These release candidate builds help to freeze the scope and maintain the quality of the release. The release candidate builds will go through: 7 | - Platform Verification 8 | - Regression and Feature Verification Automated tests. 9 | - Exploratory testing by QA engineers 10 | - Strict security scanners on the container images 11 | - Upgrade from previous releases 12 | - Beta testing by users on issues that they are interested in. 13 | - Dogfooding on OpenEBS workload and e2e infrastructure clusters. 14 | 15 | If any issues are found during the above stages, they are fixed and a new release candidate builds are generated. 16 | 17 | Once all the above tests are completed, a main release tagged image is published. 18 | 19 | ## Release Tagging 20 | 21 | velero-plugin is released as a container image with a versioned tag. 22 | 23 | Before creating a release, the repo owner needs to create a separate branch from the active branch, which is `develop`. Name of the branch should follow the naming convention of `v.1.9.x` if the release is for v1.9.0. 24 | 25 | Once the release branch is created, changelog from folder `changelogs/unreleased` needs to be moved to release specific folder `changelogs/v1.9.x`, if release branch is `v1.10.x` then folder will be `changelogs/v1.10.x`. 26 | 27 | The format of the release tag is either "Release-Name-RC1" or "Release-Name" depending on whether the tag is a release candidate or a release. (Example: v1.9.0-RC1 is a GitHub release tag for the velero-plugin release build. v1.9.0 is the release tag that is created after the release criteria are satisfied by the velero-plugin builds.) 28 | 29 | Once the release is triggered, github actions release workflow process has to be monitored. Once github actions release workflow is passed images are pushed to docker hub and quay.io. Images can be verified by going through docker hub and quay.io. Also the images shouldn't have any high-level vulnerabilities. 30 | 31 | Multiarch (amd64/arm64) images are published at the following location: 32 | ``` 33 | https://hub.docker.com/r/openebs/velero-plugin/tags 34 | ``` 35 | 36 | 37 | Once a release is created, update the release description with the changelog mentioned in folder `changelog/v1.9.x`. Once the changelogs are updated in the release, the repo owner needs to create a PR to `develop` with the following details: 38 | 1. update the changelog from folder `changelog/v1.9.x` to file `velero-plugin/CHANGELOG-v1.9.md` 39 | 2. If a release is an RC tag then PR should include the changes to remove the changelog from folder`changelog/v1.9.x` which are already mentioned in `velero-plugin/CHANGELOG-v1.9.md` as part of step number 1. 40 | 3. If a release is not an RC tag then 41 | - PR should include the changes to remove files from `changelog/v1.9.x` folder. 42 | - PR should update the root [CHANGELOG file](https://github.com/openebs/velero-plugin/blob/HEAD/CHANGELOG.md) with contents of file `velero-plugin/CHANGELOG-v1.9.md` 43 | 44 | Format of the `velero-plugin/CHANGELOG-v1.9.md` file must be as below: 45 | ``` 46 | 1.9.0 / 2020-04-14 47 | ======================== 48 | * ARM build for velero-plugin, ARM image is published under openebs/velero-plugin-arm64 ([#61](https://github.com/openebs/velero-plugin/pull/61),[@akhilerm](https://github.com/akhilerm)) 49 | * Updating alpine image version for velero-plugin to 3.10.4 ([#64](https://github.com/openebs/velero-plugin/pull/64),[@mynktl](https://github.com/mynktl)) 50 | * support for local snapshot and restore(in different namespace) ([#53](https://github.com/openebs/velero-plugin/pull/53),[@mynktl](https://github.com/mynktl)) 51 | * added support for multiPartChunkSize for S3 based remote storage ([#55](https://github.com/openebs/velero-plugin/pull/55),[@mynktl](https://github.com/mynktl)) 52 | * added auto clean-up of CStor volume snapshot generated for remote backup ([#57](https://github.com/openebs/velero-plugin/pull/57),[@mynktl](https://github.com/mynktl)) 53 | 54 | 55 | 1.9.0-RC2 / 2020-04-12 56 | ======================== 57 | * ARM build for velero-plugin, ARM image is published under openebs/velero-plugin-arm64 ([#61](https://github.com/openebs/velero-plugin/pull/61),[@akhilerm](https://github.com/akhilerm)) 58 | * Updating alpine image version for velero-plugin to 3.10.4 ([#64](https://github.com/openebs/velero-plugin/pull/64),[@mynktl](https://github.com/mynktl)) 59 | 60 | 61 | 1.9.0-RC1 / 2020-04-08 62 | ======================== 63 | * support for local snapshot and restore(in different namespace) ([#53](https://github.com/openebs/velero-plugin/pull/53),[@mynktl](https://github.com/mynktl)) 64 | * added support for multiPartChunkSize for S3 based remote storage ([#55](https://github.com/openebs/velero-plugin/pull/55),[@mynktl](https://github.com/mynktl)) 65 | * added auto clean-up of CStor volume snapshot generated for remote backup ([#57](https://github.com/openebs/velero-plugin/pull/57),[@mynktl](https://github.com/mynktl)) 66 | ``` 67 | -------------------------------------------------------------------------------- /changelogs/released/v1.10.0-RC1/72-mynktl: -------------------------------------------------------------------------------- 1 | Adding support to restore remote backup in different namespace 2 | -------------------------------------------------------------------------------- /changelogs/released/v1.10.0-RC1/76-mynktl: -------------------------------------------------------------------------------- 1 | Adding support for multiple s3 profile to backup cstor volumes to different s3 location 2 | -------------------------------------------------------------------------------- /changelogs/released/v1.10.0-RC1/79-mynktl: -------------------------------------------------------------------------------- 1 | Fixed panic, created because of empty snapshotID, while deleting failed backup 2 | -------------------------------------------------------------------------------- /changelogs/released/v1.10.0-RC2/85-mynktl: -------------------------------------------------------------------------------- 1 | Fixing failure restic backup of cstor volumes, restored by velero-plugin [issue:84](https://github.com/openebs/velero-plugin/issues/84) -------------------------------------------------------------------------------- /changelogs/released/v1.11.0-RC2/93-sonasingh46: -------------------------------------------------------------------------------- 1 | add restore support for cStor CSI based volumes 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.1.0/102-pawanpraka1: -------------------------------------------------------------------------------- 1 | adding velero plugin for ZFS-LocalPV 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.1.0/108-mittachaitu: -------------------------------------------------------------------------------- 1 | Add support for local restore of cStor CSI volume 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.1.0/110-pawanpraka1: -------------------------------------------------------------------------------- 1 | making directory and binary name openebs specific 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.1.0/111-pawanpraka1: -------------------------------------------------------------------------------- 1 | adding support for parallel backup and restore 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.2.0-RC1/115-mynktl: -------------------------------------------------------------------------------- 1 | Added support to use custom certificate and option to skip certificate verification for s3 object storage 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.2.0-RC1/116-pawanpraka1: -------------------------------------------------------------------------------- 1 | making log level available for velero plugin 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.2.0-RC1/117-pawanpraka1: -------------------------------------------------------------------------------- 1 | wait for plugin server to be ready before doing backup/restore 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.2.0-RC1/118-pawanpraka1: -------------------------------------------------------------------------------- 1 | adding support to restore on different setup/nodes 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.2.0-RC1/121-pawanpraka1: -------------------------------------------------------------------------------- 1 | adding support to do incremental backup/restore for ZFS-LocalPV 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.2.0-RC2/124-pawanpraka1: -------------------------------------------------------------------------------- 1 | use schedule name to identify the scheduled backup for ZFS-LocalPV 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.2.0-RC2/128-pawanpraka1: -------------------------------------------------------------------------------- 1 | fixing the backup deletion for ZFS-LocalPV 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.3.0-RC1/132-shubham14bajpai: -------------------------------------------------------------------------------- 1 | Adding github action workflows to build multiarch images using docker buildx. 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.3.0-RC1/133-prateek: -------------------------------------------------------------------------------- 1 | Multi-arch container image support for velero plugin. Migrate the multi-arch builds to github-action to support amd64 and arm64 architectures. 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.3.0-RC1/99-mynktl: -------------------------------------------------------------------------------- 1 | Adding new config parameter "restoreAllIncrementalSnapshots" to restore all the scheduled backups, from base backup to the given backup, using single restore 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.4.0-RC1/131-zlymeda: -------------------------------------------------------------------------------- 1 | Marking CVRs after successful restore with openebs.io/restore-completed if autoSetTargetIP=true or restoreAllIncrementalSnapshots=true 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.4.0-RC1/139-pawanpraka1: -------------------------------------------------------------------------------- 1 | updating the label selector for restoring the ZFS-LocalPV volumes on different node 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.4.0-RC1/140-mynktl: -------------------------------------------------------------------------------- 1 | Adding support to create destination namespace, for restore, if it doesn't exist 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.5.0-RC1/144-mynktl: -------------------------------------------------------------------------------- 1 | Removing wait on setting target-ip for CVR. This fixes the restore for application pod having target-affinity set. -------------------------------------------------------------------------------- /changelogs/released/v2.7.0-RC1/147-pawanpraka1: -------------------------------------------------------------------------------- 1 | adding support to restore in an encrypted pool for ZFS-LocalPV 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.7.0-RC1/149-shubham14bajpai: -------------------------------------------------------------------------------- 1 | moved travis tests to github action and removed travis.yml 2 | -------------------------------------------------------------------------------- /changelogs/released/v2.9.0-RC1/154-mynktl: -------------------------------------------------------------------------------- 1 | Adding a new VolumeSnapshotLocation config parameter restApiTimeout to configure timeout for HTTP REST request between plugin and cstor services 2 | 3 | -------------------------------------------------------------------------------- /changelogs/released/v2.9.0-RC1/161-prateekpandey14: -------------------------------------------------------------------------------- 1 | refact(deps): bump zfslocalpv, maya, api and k8s client-go dependencies 2 | -------------------------------------------------------------------------------- /code-standard.md: -------------------------------------------------------------------------------- 1 | # Code Standards 2 | 3 | ## Sign your commits 4 | 5 | We use the Developer Certificate of Origin (DCO) as an additional safeguard for the OpenEBS projects. This is a well established and widely used mechanism to assure that contributors have confirmed their right to license their contribution under the project's license. Please read [dcofile](https://github.com/openebs/openebs/blob/HEAD/contribute/developer-certificate-of-origin). If you can certify it, then just add a line to every git commit message: 6 | 7 | ```` 8 | Signed-off-by: Random J Developer 9 | ```` 10 | 11 | Use your real name (sorry, no pseudonyms or anonymous contributions). The email id should match the email id provided in your GitHub profile. 12 | If you set your `user.name` and `user.email` in git config, you can sign your commit automatically with `git commit -s`. 13 | 14 | ## Verifying code style 15 | 16 | We are using [golangci-lint](https://github.com/golangci/golangci-lint) to verify the linting errors. Make sure that your changes pass linting check by executing `make lint` or `make lint-docker` (docker version of `make lint`) 17 | 18 | ## Adding a changelog 19 | If PR is about adding a new feature or bug fix then the Author of the PR is expected to add a changelog file with their pull request. This changelog file should be a new file created under the `changelogs/unreleased` folder. Name of this file must be in `pr_number-username` format and contents of the file should be the one-liner text which explains the feature or bug fix. 20 | 21 | ```sh 22 | velero-plugin/changelogs/unreleased <- folder 23 | 12-github_user_name <- file 24 | ``` 25 | 26 | For example, if `xyz` user has raised a PR `16` to add support for s3 base storage for remote backup then 27 | channgelog file name will be 28 | ```sh 29 | velero-plugin/changelogs/unreleased/15-xyz 30 | ``` 31 | and content of the file will be 32 | ```sh 33 | adding support for s3 base storage for remote backup 34 | ``` 35 | -------------------------------------------------------------------------------- /developer-setup.md: -------------------------------------------------------------------------------- 1 | 2 | # Development Workflow 3 | 4 | ## Prerequisites 5 | 6 | * You have Go 1.14 installed on your local development machine. 7 | * You have Docker installed on your local development machine. Docker is required for building velero-plugin container images and to push them into a Kubernetes cluster for testing. 8 | * You have `kubectl` installed. For running integration tests, you will require an existing single node cluster with [openebs](https://blog.openebs.io/how-to-install-openebs-with-kubernetes-using-minikube-2ed488dff1c2) and [velero](https://velero.io/docs/main/basic-install/) installed. Don't worry if you don't have access to the Kubernetes cluster, raising a PR with the velero-plugin repository will run integration tests for your changes against a Minikube cluster. 9 | 10 | ## Initial Setup 11 | 12 | ### Fork in the cloud 13 | 14 | 1. Visit 15 | 2. Click the `Fork` button (top right) to establish a cloud-based fork. 16 | 17 | ### Clone fork to the local machine 18 | 19 | Place openebs/velero-plugin's code on your `GOPATH` using the following cloning procedure. 20 | Create your clone: 21 | 22 | ```sh 23 | 24 | mkdir -p $GOPATH/src/github.com/openebs 25 | cd $GOPATH/src/github.com/openebs 26 | 27 | # Note: Here user= your github profile name 28 | git clone https://github.com/$user/velero-plugin.git 29 | 30 | # Configure remote upstream 31 | cd $GOPATH/src/github.com/openebs/velero-plugin 32 | git remote add upstream https://github.com/openebs/velero-plugin.git 33 | 34 | # Never push to upstream develop 35 | git remote set-url --push upstream no_push 36 | 37 | # Confirm that your remotes make sense: 38 | git remote -v 39 | ``` 40 | 41 | > **Note:** If your `GOPATH` has more than one (`:` separated) paths in it, then you should use *one of your go path* instead of `$GOPATH` in the commands mentioned here. This statement holds throughout this document. 42 | 43 | ### Building and Testing your changes 44 | 45 | * To build the velero-plugin binary 46 | 47 | ```sh 48 | 49 | make 50 | ``` 51 | 52 | * To build the docker image 53 | 54 | ```sh 55 | 56 | make container REPO= 57 | ``` 58 | 59 | * Test your changes 60 | Integration tests are written in ginkgo and run against a minikube cluster. The Minikube cluster should be running to execute the tests. To install the Minikube follow the doc [here](https://kubernetes.io/docs/tasks/tools/install-minikube/). 61 | To run the integration tests on the minikube cluster. 62 | 63 | ```sh 64 | make tet 65 | ``` 66 | 67 | ## Git Development Workflow 68 | 69 | ### Always sync your local repository 70 | 71 | Open a terminal on your local machine. Change directory to the velero-plugin fork root. 72 | 73 | ```sh 74 | cd $GOPATH/src/github.com/openebs/velero-plugin 75 | ``` 76 | 77 | Check out the develop branch. 78 | 79 | ```sh 80 | $ git checkout develop 81 | Switched to branch 'develop' 82 | Your branch is up-to-date with 'origin/develop'. 83 | ``` 84 | 85 | Recall that origin/develop is a branch on your remote GitHub repository. 86 | Make sure you have the upstream remote openebs/velero-plugin by listing them. 87 | 88 | ```sh 89 | $ git remote -v 90 | origin https://github.com/$user/velero-plugin.git (fetch) 91 | origin https://github.com/$user/velero-plugin.git (push) 92 | upstream https://github.com/openebs/velero-plugin.git (fetch) 93 | upstream https://github.com/openebs/velero-plugin.git (no_push) 94 | ``` 95 | 96 | If the upstream is missing, add it by using the below command. 97 | 98 | ```sh 99 | git remote add upstream https://github.com/openebs/velero-plugin.git 100 | ``` 101 | 102 | Fetch all the changes from the upstream develop branch. 103 | 104 | ```sh 105 | $ git fetch upstream develop 106 | remote: Counting objects: 141, done. 107 | remote: Compressing objects: 100% (29/29), done. 108 | remote: Total 141 (delta 52), reused 46 (delta 46), pack-reused 66 109 | Receiving objects: 100% (141/141), 112.43 KiB | 0 bytes/s, done. 110 | Resolving deltas: 100% (79/79), done. 111 | From github.com:openebs/velero-plugin 112 | * branch develop -> FETCH_HEAD 113 | ``` 114 | 115 | Rebase your local develop with the upstream/develop. 116 | 117 | ```sh 118 | $ git rebase upstream/develop 119 | First, rewinding head to replay your work on top of it... 120 | Fast-forwarded develop to upstream/develop. 121 | ``` 122 | 123 | This command applies all the commits from the upstream develop to your local develop. 124 | 125 | Check the status of your local branch. 126 | 127 | ```sh 128 | $ git status 129 | On branch develop 130 | Your branch is ahead of 'origin/develop' by 12 commits. 131 | (use "git push" to publish your local commits) 132 | nothing to commit, working directory clean 133 | ``` 134 | 135 | Your local repository now has all the changes from the upstream remote. You need to push the changes to your remote fork which is origin develop. 136 | 137 | Push the rebased develop to origin develop. 138 | 139 | ```sh 140 | $ git push origin develop 141 | Username for 'https://github.com': $user 142 | Password for 'https://$user@github.com': 143 | Counting objects: 223, done. 144 | Compressing objects: 100% (38/38), done. 145 | Writing objects: 100% (69/69), 8.76 KiB | 0 bytes/s, done. 146 | Total 69 (delta 53), reused 47 (delta 31) 147 | To https://github.com/$user/velero-plugin.git 148 | 8e107a9..5035fa1 develop -> develop 149 | ``` 150 | 151 | ### Contributing to a feature or bugfix 152 | 153 | Always start with creating a new branch from develop to work on a new feature or bugfix. Your branch name should have the format XX-descriptive where XX is the issue number you are working on followed by some descriptive text. For example: 154 | 155 | ```sh 156 | $ git checkout develop 157 | # Make sure the develop is rebased with the latest changes as described in the previous step. 158 | $ git checkout -b 1234-fix-developer-docs 159 | Switched to a new branch '1234-fix-developer-docs' 160 | ``` 161 | 162 | Happy Hacking! 163 | 164 | ### Keep your branch in sync 165 | 166 | [Rebasing](https://git-scm.com/docs/git-rebase) is very important to keep your branch in sync with the changes being made by others and to avoid huge merge conflicts while raising your Pull Requests. You will always have to rebase before raising the PR. 167 | 168 | ```sh 169 | # While on your myfeature branch (see above) 170 | git fetch upstream 171 | git rebase upstream/develop 172 | ``` 173 | 174 | While you rebase your changes, you must resolve any conflicts that might arise and build and test your changes using the above steps. 175 | 176 | ## Submission 177 | 178 | ### Create a pull request 179 | 180 | Before you raise the Pull Requests, ensure you have reviewed the checklist in the [CONTRIBUTING GUIDE](CONTRIBUTING.md): 181 | 182 | * Ensure that you have re-based your changes with the upstream using the steps above. 183 | * Ensure that you have added the required unit tests for the bug fix or a new feature that you have introduced. 184 | * Ensure your commit history is clean with proper header and descriptions. 185 | 186 | Go to the [openebs/velero-plugin github](https://github.com/openebs/velero-plugin) and follow the Open Pull Request link to raise your PR from your development branch. 187 | -------------------------------------------------------------------------------- /example/05-backupstoragelocation.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 the Heptio Ark contributors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | apiVersion: velero.io/v1 17 | kind: BackupStorageLocation 18 | metadata: 19 | name: default 20 | namespace: velero 21 | spec: 22 | provider: gcp 23 | objectStorage: 24 | bucket: 25 | -------------------------------------------------------------------------------- /example/06-local-volumesnapshotlocation.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 the Heptio Ark contributors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | apiVersion: velero.io/v1 17 | kind: VolumeSnapshotLocation 18 | metadata: 19 | # name -- volumeSnapshotLocation Name (local-default...) 20 | name: 21 | namespace: velero 22 | spec: 23 | provider: openebs.io/cstor-blockstore 24 | config: 25 | # namespace -- namespace in which openebs is installed (default: openebs) 26 | namespace: 27 | 28 | local: "true" 29 | 30 | # restApiTimeout -- http timeout for rest call between velero-plugin and openebs services 31 | # if not set, default timeout will be 60s. 32 | # example value: 60s, 2m.. 33 | restApiTimeout: 1m 34 | -------------------------------------------------------------------------------- /example/06-volumesnapshotlocation.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 the Heptio Ark contributors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | apiVersion: velero.io/v1 17 | kind: VolumeSnapshotLocation 18 | metadata: 19 | # name -- volumeSnapshotLocation Name (gcp-default, aws-default...) 20 | name: 21 | namespace: velero 22 | spec: 23 | provider: openebs.io/cstor-blockstore 24 | config: 25 | # bucket -- bucket Name (velero, dev-cluster...) 26 | bucket: 27 | 28 | # prefix -- prefix for backed up file (default: empty), name of the backed up cstor files will be in prefix-volume_name-backup_name format) 29 | prefix: 30 | 31 | # backupPathPrefix -- backupPathPrefix for remote backup path (default: empty), cstor backed up file be stored under "bucket/backupPathPrefix/backups/backup_name/") 32 | backupPathPrefix: 33 | 34 | # provider -- cloud provider name (default: empty, value can be gcp, aws) 35 | provider: 36 | 37 | # region -- cloud provider region 38 | region: 39 | 40 | # namespace -- namespace in which openebs is installed (default: openebs) 41 | namespace: 42 | 43 | # restoreAllIncrementalSnapshots -- set it to "true", to restore all the backups from base backup to the given backup 44 | restoreAllIncrementalSnapshots: "false" 45 | 46 | # autoSetTargetIP -- set it to "true", to automatically set target ip on CVR after successful restore 47 | autoSetTargetIP: "true" 48 | 49 | # restApiTimeout -- http timeout for rest call between velero-plugin and openebs services 50 | # if not set, default timeout will be 60s. 51 | # example value: 60s, 2m.. 52 | restApiTimeout: 1m 53 | 54 | ### Sample VolumeSnapshotLocation YAML for various cloud-providers 55 | # # For GCP 56 | #--- 57 | # apiVersion: velero.io/v1 58 | # kind: VolumeSnapshotLocation 59 | # metadata: 60 | # name: gcp-bucket 61 | # namespace: velero 62 | # spec: 63 | # provider: openebs.io/cstor-blockstore 64 | # config: 65 | # bucket: openebs-velero-example 66 | # prefix: cstor 67 | # provider: gcp 68 | # restApiTimeout: 1m 69 | 70 | # 71 | # # For MinIO 72 | # --- 73 | # apiVersion: velero.io/v1 74 | # kind: VolumeSnapshotLocation 75 | # metadata: 76 | # name: minio 77 | # namespace: velero 78 | # spec: 79 | # provider: openebs.io/cstor-blockstore 80 | # config: 81 | # bucket: openebs-velero-example 82 | # 83 | # prefix: cstor 84 | # 85 | # provider: aws 86 | # 87 | # # The region where the server is located. 88 | # region: minio 89 | # 90 | # # profile for credential, if not mentioned then plugin will use profile=default 91 | # profile: user1 92 | # 93 | # # Whether to use path-style addressing instead of virtual hosted bucket addressing. 94 | # # Set to "true" 95 | # s3ForcePathStyle: "true" 96 | # 97 | # # S3 URL, By default it will be generated from "region" and "bucket" 98 | # s3Url: http://minio.velero.svc:9000 99 | # 100 | # # You can specify the multipart_chunksize here for explicitness. 101 | # # multiPartChunkSize can be from 5Mi(5*1024*1024 Bytes) to 5Gi 102 | # # For more information: https://docs.min.io/docs/minio-server-limits-per-tenant.html 103 | # # If not set then it will be calculated from the file size 104 | # multiPartChunkSize: 64Mi 105 | # 106 | # # If MinIO is configured with custom certificate then certificate can be passed to plugin through caCert 107 | # # Value of caCert must be base64 encoded 108 | # # To encode, execute command: cat ca.crt |base64 -w 0 109 | # caCert: LS0tLS1CRU...tRU5EIENFUlRJRklDQVRFLS0tLS0K 110 | # 111 | # # If you want to disable certificate verification then set insecureSkipTLSVerify to "true" 112 | # # By default insecureSkipTLSVerify is set to "false" 113 | # insecureSkipTLSVerify: "false" 114 | # 115 | # restApiTimeout: 1m 116 | 117 | # 118 | # # For AWS 119 | # --- 120 | # apiVersion: velero.io/v1 121 | # kind: VolumeSnapshotLocation 122 | # metadata: 123 | # name: aws 124 | # namespace: velero 125 | # spec: 126 | # provider: openebs.io/cstor-blockstore 127 | # config: 128 | # bucket: openebs-velero-example 129 | # 130 | # prefix: cstor 131 | # 132 | # provider: aws 133 | # 134 | # # The AWS region where the bucket is located. 135 | # region: ap-south-1 136 | # 137 | # # profile for credential, if not mentioned then plugin will use profile=default 138 | # profile: user1 139 | # 140 | # # You can specify the multipart_chunksize here for explicitness. 141 | # # multiPartChunkSize can be from 5Mi(5*1024*1024 Bytes) to 5Gi 142 | # # For more information: https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html 143 | # # If not set then it will be calculated from the file size 144 | # multiPartChunkSize: 64Mi 145 | # 146 | # # if s3 endpoint is configured with custom certificate then certificate can be passed to plugin through caCert. 147 | # # Value of caCert must be base64 encoded 148 | # # To encode, execute command: cat ca.crt |base64 -w 0 149 | # caCert: LS0tLS1CRU...tRU5EIENFUlRJRklDQVRFLS0tLS0K 150 | # 151 | # # If you want to disable certificate verification then set insecureSkipTLSVerify to "true" 152 | # # By default insecureSkipTLSVerify is set to "false" 153 | # insecureSkipTLSVerify: "false" 154 | # 155 | # restApiTimeout: 1m -------------------------------------------------------------------------------- /example/10-deployment.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 the Heptio Ark contributors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | apiVersion: apps/v1beta1 17 | kind: Deployment 18 | metadata: 19 | namespace: velero 20 | name: velero 21 | spec: 22 | replicas: 1 23 | template: 24 | metadata: 25 | labels: 26 | component: velero 27 | annotations: 28 | prometheus.io/scrape: "true" 29 | prometheus.io/port: "8085" 30 | prometheus.io/path: "/metrics" 31 | spec: 32 | restartPolicy: Always 33 | serviceAccountName: velero 34 | containers: 35 | - name: velero 36 | image: gcr.io/heptio-images/velero:latest 37 | ports: 38 | - name: metrics 39 | containerPort: 8085 40 | command: 41 | - /velero 42 | args: 43 | - server 44 | ## uncomment following line and specify values if needed for multiple provider snapshot locations 45 | # - --default-volume-snapshot-locations= 46 | volumeMounts: 47 | - name: cloud-credentials 48 | mountPath: /credentials 49 | - name: plugins 50 | mountPath: /plugins 51 | - name: scratch 52 | mountPath: /scratch 53 | env: 54 | - name: GOOGLE_APPLICATION_CREDENTIALS 55 | value: /credentials/cloud 56 | - name: VELERO_SCRATCH_DIR 57 | value: /scratch 58 | volumes: 59 | - name: cloud-credentials 60 | secret: 61 | secretName: cloud-credentials 62 | - name: plugins 63 | emptyDir: {} 64 | - name: scratch 65 | emptyDir: {} 66 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/openebs/velero-plugin 2 | 3 | go 1.13 4 | 5 | require ( 6 | cloud.google.com/go v0.58.0 // indirect 7 | cloud.google.com/go/storage v1.9.0 // indirect 8 | github.com/Azure/azure-pipeline-go v0.2.2 // indirect 9 | github.com/Azure/azure-storage-blob-go v0.8.0 // indirect 10 | github.com/aws/aws-sdk-go v1.35.24 11 | github.com/ghodss/yaml v1.0.0 12 | github.com/gofrs/uuid v3.2.0+incompatible 13 | github.com/google/wire v0.4.0 // indirect 14 | github.com/hashicorp/go-plugin v1.0.1-0.20190610192547-a1bc61569a26 // indirect 15 | github.com/mattn/go-ieproxy v0.0.1 // indirect 16 | github.com/onsi/ginkgo v1.15.2 17 | github.com/onsi/gomega v1.10.2 18 | github.com/openebs/api/v2 v2.3.0 19 | github.com/openebs/maya v1.12.1-0.20210416090832-ad9c32f086d5 20 | github.com/openebs/zfs-localpv v1.6.1-0.20210504173514-62b3a0b7fe5d 21 | github.com/pkg/errors v0.9.1 22 | github.com/sirupsen/logrus v1.6.0 23 | github.com/spf13/pflag v1.0.5 24 | github.com/vmware-tanzu/velero v1.5.0 25 | gocloud.dev v0.15.0 26 | google.golang.org/api v0.26.0 27 | k8s.io/api v0.20.2 28 | k8s.io/apimachinery v0.20.2 29 | k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible 30 | ) 31 | 32 | replace ( 33 | k8s.io/api => k8s.io/api v0.20.2 34 | k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.2 35 | k8s.io/apimachinery => k8s.io/apimachinery v0.20.2 36 | k8s.io/apiserver => k8s.io/apiserver v0.20.2 37 | k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.2 38 | k8s.io/client-go => k8s.io/client-go v0.20.2 39 | k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.2 40 | k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.2 41 | k8s.io/code-generator => k8s.io/code-generator v0.20.2 42 | k8s.io/component-base => k8s.io/component-base v0.20.2 43 | k8s.io/cri-api => k8s.io/cri-api v0.20.2 44 | k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.2 45 | k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.2 46 | k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.2 47 | k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.2 48 | k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.2 49 | k8s.io/kubectl => k8s.io/kubectl v0.20.2 50 | k8s.io/kubelet => k8s.io/kubelet v0.20.2 51 | k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.2 52 | k8s.io/metrics => k8s.io/metrics v0.20.2 53 | k8s.io/node-api => k8s.io/node-api v0.20.2 54 | k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.2 55 | k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.20.2 56 | k8s.io/sample-controller => k8s.io/sample-controller v0.20.2 57 | ) 58 | -------------------------------------------------------------------------------- /pkg/clouduploader/operation.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package clouduploader 18 | 19 | import ( 20 | "io" 21 | "strings" 22 | 23 | "github.com/pkg/errors" 24 | 25 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 26 | "gocloud.dev/blob" 27 | ) 28 | 29 | const ( 30 | // backupDir is remote storage-bucket directory 31 | backupDir = "backups" 32 | ) 33 | 34 | const ( 35 | // Type of Key, used while listing keys 36 | 37 | // KeyFile - if key is a file 38 | ListKeyFile int = 1 << iota 39 | 40 | // KeyDirectory - if key is a directory 41 | ListKeyDir 42 | 43 | // KeyBoth - if key is a file or directory 44 | ListKeyBoth 45 | ) 46 | 47 | // Upload will perform upload operation for given file. 48 | // It will create a TCP server through which client can 49 | // connect and upload data to cloud blob storage file 50 | func (c *Conn) Upload(file string, fileSize int64, port int) bool { 51 | c.Log.Infof("Uploading snapshot to '%s' with provider{%s} to bucket{%s}", file, c.provider, c.bucketname) 52 | 53 | c.file = file 54 | if c.partSize == 0 { 55 | // MaxUploadParts is limited to 10k 56 | // 100 is arbitrary value considering snapshot metadata 57 | partSize := (fileSize / s3manager.MaxUploadParts) + 100 58 | if partSize < s3manager.MinUploadPartSize { 59 | partSize = s3manager.MinUploadPartSize 60 | } 61 | c.partSize = partSize 62 | } 63 | 64 | s := &Server{ 65 | Log: c.Log, 66 | cl: c, 67 | } 68 | err := s.Run(OpBackup, port) 69 | if err != nil { 70 | c.Log.Errorf("Failed to upload snapshot to bucket: %s", err.Error()) 71 | if c.bucket.Delete(c.ctx, file) != nil { 72 | c.Log.Errorf("Failed to delete uncompleted snapshot{%s} from cloud", file) 73 | } 74 | return false 75 | } 76 | 77 | c.Log.Infof("successfully uploaded object{%s} to {%s}", file, c.provider) 78 | return true 79 | } 80 | 81 | // Delete will delete file from cloud blob storage 82 | func (c *Conn) Delete(file string) bool { 83 | c.Log.Infof("Removing snapshot:'%s' from bucket{%s} provider{%s}", file, c.bucketname, c.provider) 84 | 85 | if c.bucket.Delete(c.ctx, file) != nil { 86 | c.Log.Errorf("Failed to remove snapshot{%s} from cloud", file) 87 | return false 88 | } 89 | return true 90 | } 91 | 92 | // Download will perform restore operation for given file. 93 | // It will create a TCP server through which client can 94 | // connect and download data from cloud blob storage file 95 | func (c *Conn) Download(file string, port int) bool { 96 | c.file = file 97 | s := &Server{ 98 | Log: c.Log, 99 | cl: c, 100 | } 101 | err := s.Run(OpRestore, port) 102 | if err != nil { 103 | c.Log.Errorf("Failed to receive snapshot from bucket: %s", err.Error()) 104 | return false 105 | } 106 | c.Log.Infof("successfully restored object{%s} from {%s}", file, c.provider) 107 | return true 108 | } 109 | 110 | // Write will write data to cloud blob storage file 111 | func (c *Conn) Write(data []byte, file string) bool { 112 | c.Log.Infof("Writing to {%s} with provider{%v} to bucket{%v}", file, c.provider, c.bucketname) 113 | 114 | w, err := c.bucket.NewWriter(c.ctx, file, nil) 115 | if err != nil { 116 | c.Log.Errorf("Failed to obtain writer: %s", err.Error()) 117 | return false 118 | } 119 | _, err = w.Write(data) 120 | if err != nil { 121 | c.Log.Errorf("Failed to write data to file{%s} : %s", file, err.Error()) 122 | if err = c.bucket.Delete(c.ctx, file); err != nil { 123 | c.Log.Warnf("Failed to delete file {%v} : %s", file, err.Error()) 124 | } 125 | return false 126 | } 127 | 128 | if err = w.Close(); err != nil { 129 | c.Log.Errorf("Failed to close cloud conn : %s", err.Error()) 130 | return false 131 | } 132 | c.Log.Infof("Successfully written object{%s} to {%s}", file, c.provider) 133 | return true 134 | } 135 | 136 | // Read will return content of file from cloud blob storage 137 | func (c *Conn) Read(file string) ([]byte, bool) { 138 | c.Log.Infof("Reading from {%s} with provider{%s} to bucket{%s}", file, c.provider, c.bucketname) 139 | 140 | data, err := c.bucket.ReadAll(c.ctx, file) 141 | if err != nil { 142 | c.Log.Errorf("Failed to read data from file{%s} : %s", file, err.Error()) 143 | return nil, false 144 | } 145 | 146 | c.Log.Infof("successfully read object{%s} to {%s}", file, c.provider) 147 | return data, true 148 | } 149 | 150 | // GenerateRemoteFilename will create a file-name specific for given backup 151 | func (c *Conn) GenerateRemoteFilename(file, backup string) string { 152 | if c.backupPathPrefix == "" { 153 | return backupDir + "/" + backup + "/" + c.prefix + "-" + file + "-" + backup 154 | } 155 | return c.backupPathPrefix + "/" + backupDir + "/" + backup + "/" + c.prefix + "-" + file + "-" + backup 156 | } 157 | 158 | // GenerateRemoteFileWithSchd will create a file-name specific for given backup and schedule name 159 | func (c *Conn) GenerateRemoteFileWithSchd(file, schdname, backup string) string { 160 | filePath := backupDir + "/" + backup + "/" + c.prefix 161 | if len(schdname) > 0 { 162 | filePath += "-" + schdname 163 | } 164 | 165 | if c.backupPathPrefix == "" { 166 | return filePath + "-" + file + "-" + backup 167 | } 168 | 169 | return c.backupPathPrefix + "/" + filePath + "-" + file + "-" + backup 170 | } 171 | 172 | func (c *Conn) GetFileNameWithSchd(file, schdname string) string { 173 | return schdname + "-" + file 174 | } 175 | 176 | // ConnStateReset resets the channel and exit server flag 177 | func (c *Conn) ConnStateReset() { 178 | ch := make(chan bool, 1) 179 | c.ConnReady = &ch 180 | c.ExitServer = false 181 | } 182 | 183 | // ConnReadyWait will return when connection is ready to accept the connection 184 | func (c *Conn) ConnReadyWait() bool { 185 | ok := <-*c.ConnReady 186 | return ok 187 | } 188 | 189 | // listKeys return list of Keys -- files/directories 190 | // Note: 191 | // - list may contain incomplete list of keys, check for error before using list 192 | // - listKeys uses '/' as delimiter. 193 | func (c *Conn) listKeys(prefix string, keyType int) ([]string, error) { 194 | keys := []string{} 195 | 196 | lister := c.bucket.List(&blob.ListOptions{ 197 | Delimiter: "/", 198 | Prefix: prefix, 199 | }) 200 | for { 201 | obj, err := lister.Next(c.ctx) 202 | if err == io.EOF { 203 | break 204 | } 205 | 206 | if err != nil { 207 | c.Log.Errorf("Failed to get next blob err=%v", err) 208 | return keys, err 209 | } 210 | 211 | switch keyType { 212 | case ListKeyBoth: 213 | case ListKeyFile: 214 | if obj.IsDir { 215 | continue 216 | } 217 | case ListKeyDir: 218 | if !obj.IsDir { 219 | continue 220 | } 221 | default: 222 | c.Log.Warningf("Invalid keyType=%d, Ignored", keyType) 223 | continue 224 | } 225 | 226 | keys = append(keys, obj.Key) 227 | } 228 | return keys, nil 229 | } 230 | 231 | // bkpPathPrefix return 'prefix path' for the given 'backup name prefix' 232 | func (c *Conn) bkpPathPrefix(backupPrefix string) string { 233 | if c.backupPathPrefix == "" { 234 | return backupDir + "/" + backupPrefix 235 | } 236 | return c.backupPathPrefix + "/" + backupDir + "/" + backupPrefix 237 | } 238 | 239 | // filePathPrefix generate prefix for the given file name prefix using 'configured file prefix' 240 | func (c *Conn) filePathPrefix(filePrefix string) string { 241 | return c.prefix + "-" + filePrefix 242 | } 243 | 244 | // GetSnapListFromCloud gets the list of a snapshot for the given backup name 245 | // the argument should be same as that of GenerateRemoteFilename(file, backup) call 246 | // used while doing the backup of the volume 247 | func (c *Conn) GetSnapListFromCloud(file, backup string) ([]string, error) { 248 | var snapList []string 249 | 250 | // list directory having schedule/backup name as prefix 251 | dirs, err := c.listKeys(c.bkpPathPrefix(backup), ListKeyDir) 252 | if err != nil { 253 | return snapList, errors.Wrapf(err, "failed to get list of directory") 254 | } 255 | 256 | for _, dir := range dirs { 257 | // list files for dir having volume name as prefix 258 | files, err := c.listKeys(dir+c.filePathPrefix(file), ListKeyFile) 259 | if err != nil { 260 | return snapList, errors.Wrapf(err, "failed to get list of snapshot file at path=%v", dir) 261 | } 262 | 263 | if len(files) != 0 { 264 | // snapshot exist in the backup directory 265 | 266 | // add backup name from dir path to snapList 267 | s := strings.Split(dir, "/") 268 | 269 | // dir will contain path with trailing '/', example: 'backups/b-0/' 270 | snapList = append(snapList, s[len(s)-2]) 271 | } 272 | } 273 | return snapList, nil 274 | } 275 | 276 | // FileExists check if the given file exists or not in the given backup 277 | // the argument should be the same as that of GenerateRemoteFilename(file, backup) call 278 | // used while doing the backup of the volume 279 | func (c *Conn) FileExists(file, backup string) (bool, error) { 280 | c.Log.Debugf("Checking if file=%s exist", c.GenerateRemoteFilename(file, backup)) 281 | 282 | return c.bucket.Exists(c.ctx, c.GenerateRemoteFilename(file, backup)) 283 | } 284 | -------------------------------------------------------------------------------- /pkg/clouduploader/server_utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package clouduploader 18 | 19 | import ( 20 | "syscall" 21 | "time" 22 | "unsafe" 23 | 24 | "github.com/pkg/errors" 25 | "gocloud.dev/blob" 26 | ) 27 | 28 | // ReadWriter is used for read/write operation on cloud blob storage file 29 | type ReadWriter unsafe.Pointer 30 | 31 | // updateClientStatus updates the status of client for current operation 32 | func (s *Server) updateClientStatus(c *Client, status TransferStatus) { 33 | c.status = status 34 | } 35 | 36 | // getClientStatus returns the status of client for current operation 37 | func (s *Server) getClientStatus(c *Client) TransferStatus { 38 | return c.status 39 | } 40 | 41 | // appendToClientList add given to client to client's link-list 42 | func (s *Server) appendToClientList(c *Client) { 43 | if s.FirstClient == nil { 44 | s.FirstClient = c 45 | s.LastClient = c 46 | } else { 47 | lastEntry := s.LastClient 48 | lastEntry.next = c 49 | s.LastClient = c 50 | } 51 | s.state.runningCount++ 52 | } 53 | 54 | // removeFromClientList removes given client from client's link-list 55 | func (s *Server) removeFromClientList(c *Client) { 56 | var prevClient *Client 57 | 58 | if s.FirstClient == nil || s.state.runningCount == 0 { 59 | // epoll may have returned multiple event for the same fd 60 | s.Log.Warningf("ClientList is empty") 61 | return 62 | } else if s.FirstClient == c { 63 | s.FirstClient = c.next 64 | } else { 65 | curClient := s.FirstClient 66 | for curClient != c { 67 | if curClient.next == nil { 68 | // epoll may have returned multiple event for the same fd 69 | s.Log.Warningf("entry{%v} not found in ClientList", c.fd) 70 | return 71 | } 72 | 73 | prevClient = curClient 74 | curClient = curClient.next 75 | } 76 | prevClient.next = curClient.next 77 | } 78 | s.state.runningCount-- 79 | } 80 | 81 | // getClientFromEvent returns client for given event 82 | func (s *Server) getClientFromEvent(event syscall.EpollEvent) *Client { 83 | var off *uint64 84 | var v **Client 85 | 86 | p := unsafe.Pointer(&event) 87 | off = (*uint64)(unsafe.Pointer(uintptr(p) + unsafe.Offsetof(event.Fd))) 88 | v = (**Client)(unsafe.Pointer(off)) 89 | return *v 90 | } 91 | 92 | // addClientToEvent add client to given event 93 | func (s *Server) addClientToEvent(c *Client, event *syscall.EpollEvent) { 94 | var off *uint64 95 | var v **Client 96 | 97 | p := unsafe.Pointer(event) 98 | off = (*uint64)(unsafe.Pointer(uintptr(p) + unsafe.Offsetof(event.Fd))) 99 | v = (**Client)(unsafe.Pointer(off)) 100 | *v = c 101 | } 102 | 103 | // SendData send data(stored in client's buffer) to given client 104 | func (s *Server) SendData(c *Client, dataLen int) error { 105 | var index int 106 | 107 | if dataLen == 0 { 108 | return nil 109 | } 110 | 111 | for { 112 | nbytes, e := syscall.Write(c.fd, c.buffer[index:dataLen]) 113 | if nbytes > 0 { 114 | index += nbytes 115 | if index == dataLen { 116 | return nil 117 | } 118 | continue 119 | } else { 120 | if e == syscall.EAGAIN { 121 | time.Sleep(time.Millisecond) 122 | continue 123 | } else { 124 | return errors.Errorf("Write returned error for fd{%v} : %s", c.fd, e.Error()) 125 | } 126 | } 127 | } 128 | } 129 | 130 | // RecvData receives data from client and store into client's buffer 131 | func (s *Server) RecvData(c *Client) (int, error) { 132 | nbytes, e := syscall.Read(c.fd, c.buffer) 133 | if nbytes > 0 { 134 | return nbytes, nil 135 | } else if nbytes < 0 { 136 | if e == syscall.EAGAIN { 137 | return 0, nil 138 | } 139 | return (-1), errors.Errorf("read returned error for fd{%v} : %s", c.fd, e.Error()) 140 | } else { 141 | /* got EOF from Client.. */ 142 | s.updateClientStatus(c, TransferStatusDone) 143 | return (-1), errors.Errorf("connection closed for fd{%v}", c.fd) 144 | } 145 | } 146 | 147 | // GetReadWriter will return interface for cloud blob storage file operation 148 | func (s *Server) GetReadWriter(bwriter *blob.Writer, breader *blob.Reader, opType ServerOperation) (ReadWriter, error) { 149 | if opType != OpBackup && opType != OpRestore { 150 | return nil, errors.Errorf("Invalid server operation {%v}", opType) 151 | } 152 | if opType == OpBackup { 153 | return ReadWriter(bwriter), nil 154 | } 155 | return ReadWriter(breader), nil 156 | } 157 | 158 | // handleClientError performs error handling for given event/client 159 | func (s *Server) handleClientError(err error, event syscall.EpollEvent, efd int) { 160 | var c = s.getClientFromEvent(event) 161 | if s.getClientStatus(c) == TransferStatusDone || 162 | event.Events&syscall.EPOLLHUP != 0 || 163 | event.Events&syscall.EPOLLERR != 0 || err == nil { 164 | s.state.successCount++ 165 | } else { 166 | s.state.failedCount++ 167 | } 168 | 169 | if err := syscall.EpollCtl(efd, syscall.EPOLL_CTL_DEL, c.fd, nil); err != nil { 170 | s.Log.Warnf("Failed to delete {%v} from EPOLL: %s", c.fd, err.Error()) 171 | } 172 | 173 | if err := syscall.Close(c.fd); err != nil { 174 | s.Log.Warnf("Failed to close {%v}: %s", c.fd, err.Error()) 175 | } 176 | 177 | s.cl.Destroy(c.file, s.OpType) 178 | s.Log.Infof("Client{%v} operation completed.. completed count{%v}", c.fd, s.state.successCount) 179 | s.removeFromClientList(c) 180 | } 181 | 182 | // disconnectAllClient disconnects all client connected to server 183 | func (s *Server) disconnectAllClient(efd int) { 184 | var nextClient *Client 185 | 186 | if s.FirstClient == nil || s.state.runningCount == 0 { 187 | return 188 | } 189 | 190 | curClient := s.FirstClient 191 | for curClient.next != nil { 192 | if s.getClientStatus(curClient) == TransferStatusDone { 193 | s.state.successCount++ 194 | } else { 195 | s.state.failedCount++ 196 | } 197 | 198 | if err := syscall.EpollCtl(efd, syscall.EPOLL_CTL_DEL, curClient.fd, nil); err != nil { 199 | s.Log.Warnf("Failed to delete {%v} from EPOLL: %s", curClient.fd, err.Error()) 200 | } 201 | 202 | if err := syscall.Close(curClient.fd); err != nil { 203 | s.Log.Warnf("Failed to close {%v}: %s", curClient.fd, err.Error()) 204 | } 205 | 206 | s.cl.Destroy(curClient.file, s.OpType) 207 | s.Log.Infof("Disconnecting Client{%v}", curClient.fd) 208 | 209 | nextClient = curClient.next 210 | s.removeFromClientList(curClient) 211 | curClient = nextClient 212 | } 213 | } 214 | 215 | // isEINTR check if given error is generated because of EINTR 216 | func isEINTR(err error) bool { 217 | if err == nil { 218 | return false 219 | } 220 | 221 | errno, ok := err.(syscall.Errno) 222 | if ok && errno == syscall.EINTR { 223 | return true 224 | } 225 | return false 226 | } 227 | -------------------------------------------------------------------------------- /pkg/cstor/api_service.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cstor 18 | 19 | import ( 20 | "bytes" 21 | "context" 22 | "encoding/json" 23 | "io/ioutil" 24 | "net/http" 25 | "strconv" 26 | 27 | v1alpha1 "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" 28 | "github.com/pkg/errors" 29 | k8serrors "k8s.io/apimachinery/pkg/api/errors" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | ) 32 | 33 | // httpRestCall execute REST API over HTTP 34 | func (p *Plugin) httpRestCall(url, reqtype string, data []byte) ([]byte, error) { 35 | req, err := http.NewRequest(reqtype, url, bytes.NewBuffer(data)) 36 | if err != nil { 37 | return nil, err 38 | } 39 | req.Header.Add("Content-Type", "application/json") 40 | 41 | c := &http.Client{ 42 | Timeout: p.restTimeout, 43 | } 44 | 45 | resp, err := c.Do(req) 46 | if err != nil { 47 | return nil, errors.Errorf("Error when connecting to maya-apiserver : %s", err.Error()) 48 | } 49 | 50 | defer func() { 51 | if err = resp.Body.Close(); err != nil { 52 | p.Log.Warnf("Failed to close response : %s", err.Error()) 53 | } 54 | }() 55 | 56 | respdata, err := ioutil.ReadAll(resp.Body) 57 | if err != nil { 58 | return nil, errors.Errorf( 59 | "Unable to read response from maya-apiserver, err=%s data=%s", 60 | err.Error(), string(respdata)) 61 | } 62 | 63 | code := resp.StatusCode 64 | if code != http.StatusOK { 65 | return nil, errors.Errorf("Status error{%v}, response=%s", http.StatusText(code), string(respdata)) 66 | } 67 | return respdata, nil 68 | } 69 | 70 | // getMapiAddr return maya API server's ip address 71 | func (p *Plugin) getMapiAddr() (string, error) { 72 | var openebsNs string 73 | 74 | // check if user has provided openebs namespace 75 | if p.namespace != "" { 76 | openebsNs = p.namespace 77 | } else { 78 | openebsNs = metav1.NamespaceAll 79 | } 80 | 81 | svclist, err := p.K8sClient. 82 | CoreV1(). 83 | Services(openebsNs). 84 | List( 85 | context.TODO(), 86 | metav1.ListOptions{ 87 | LabelSelector: mayaAPIServiceLabel, 88 | }, 89 | ) 90 | 91 | if err != nil { 92 | if k8serrors.IsNotFound(err) { 93 | return "", nil 94 | } 95 | p.Log.Errorf("Error getting maya-apiservice : %v", err.Error()) 96 | return "", err 97 | } 98 | 99 | if len(svclist.Items) != 0 { 100 | goto fetchip 101 | } 102 | 103 | // There are no any services having MayaApiService Label 104 | // Let's try to find by name only.. 105 | svclist, err = p.K8sClient. 106 | CoreV1(). 107 | Services(openebsNs). 108 | List( 109 | context.TODO(), 110 | metav1.ListOptions{ 111 | FieldSelector: "metadata.name=" + mayaAPIServiceName, 112 | }) 113 | if err != nil { 114 | if k8serrors.IsNotFound(err) { 115 | return "", nil 116 | } 117 | p.Log.Errorf("Error getting IP Address for service{%s} : %v", mayaAPIServiceName, err.Error()) 118 | return "", err 119 | } 120 | 121 | fetchip: 122 | for _, s := range svclist.Items { 123 | if s.Spec.ClusterIP != "" { 124 | // update the namespace 125 | p.namespace = s.Namespace 126 | return "http://" + s.Spec.ClusterIP + ":" + strconv.FormatInt(int64(s.Spec.Ports[0].Port), 10), nil 127 | } 128 | } 129 | 130 | return "", nil 131 | } 132 | 133 | // getCVCAddr return CVC server's ip address 134 | func (p *Plugin) getCVCAddr() (string, error) { 135 | var openebsNs string 136 | 137 | // check if user has provided openebs namespace 138 | if p.namespace != "" { 139 | openebsNs = p.namespace 140 | } else { 141 | openebsNs = metav1.NamespaceAll 142 | } 143 | 144 | svclist, err := p.K8sClient. 145 | CoreV1(). 146 | Services(openebsNs). 147 | List( 148 | context.TODO(), 149 | metav1.ListOptions{ 150 | LabelSelector: cvcAPIServiceLabel, 151 | }, 152 | ) 153 | 154 | if err != nil { 155 | if k8serrors.IsNotFound(err) { 156 | return "", nil 157 | } 158 | p.Log.Errorf("Error getting cvc service: %v", err.Error()) 159 | return "", err 160 | } 161 | 162 | if len(svclist.Items) == 0 { 163 | return "", nil 164 | } 165 | 166 | for _, s := range svclist.Items { 167 | if s.Spec.ClusterIP != "" { 168 | // update the namespace 169 | p.namespace = s.Namespace 170 | return "http://" + s.Spec.ClusterIP + ":" + strconv.FormatInt(int64(s.Spec.Ports[0].Port), 10), nil 171 | } 172 | } 173 | 174 | return "", nil 175 | } 176 | 177 | func (p *Plugin) sendBackupRequest(vol *Volume) (*v1alpha1.CStorBackup, error) { 178 | var url string 179 | 180 | scheduleName := p.getScheduleName(vol.backupName) // This will be backup/schedule name 181 | 182 | serverAddr := p.cstorServerAddr + ":" + strconv.Itoa(CstorBackupPort) 183 | 184 | bkpSpec := &v1alpha1.CStorBackupSpec{ 185 | BackupName: scheduleName, 186 | VolumeName: vol.volname, 187 | SnapName: vol.backupName, 188 | BackupDest: serverAddr, 189 | LocalSnap: p.local, 190 | } 191 | 192 | bkp := &v1alpha1.CStorBackup{ 193 | ObjectMeta: metav1.ObjectMeta{ 194 | Namespace: vol.namespace, 195 | }, 196 | Spec: *bkpSpec, 197 | } 198 | 199 | if vol.isCSIVolume { 200 | url = p.cvcAddr + backupEndpoint 201 | } else { 202 | url = p.mayaAddr + backupEndpoint 203 | } 204 | 205 | bkpData, err := json.Marshal(bkp) 206 | if err != nil { 207 | return nil, errors.Wrapf(err, "Error parsing json") 208 | } 209 | 210 | _, err = p.httpRestCall(url, "POST", bkpData) 211 | if err != nil { 212 | return nil, errors.Wrapf(err, "Error calling REST api") 213 | } 214 | 215 | return bkp, nil 216 | } 217 | 218 | func (p *Plugin) sendRestoreRequest(vol *Volume) (*v1alpha1.CStorRestore, error) { 219 | var url string 220 | 221 | restoreSrc := p.cstorServerAddr + ":" + strconv.Itoa(CstorRestorePort) 222 | 223 | if p.local { 224 | restoreSrc = vol.srcVolname 225 | } 226 | 227 | restore := &v1alpha1.CStorRestore{ 228 | ObjectMeta: metav1.ObjectMeta{ 229 | Namespace: p.namespace, 230 | }, 231 | Spec: v1alpha1.CStorRestoreSpec{ 232 | RestoreName: vol.backupName, 233 | VolumeName: vol.volname, 234 | RestoreSrc: restoreSrc, 235 | StorageClass: vol.storageClass, 236 | Size: vol.size, 237 | Local: p.local, 238 | }, 239 | } 240 | 241 | if vol.isCSIVolume { 242 | url = p.cvcAddr + restorePath 243 | } else { 244 | url = p.mayaAddr + restorePath 245 | } 246 | 247 | restoreData, err := json.Marshal(restore) 248 | if err != nil { 249 | return nil, err 250 | } 251 | 252 | data, err := p.httpRestCall(url, "POST", restoreData) 253 | if err != nil { 254 | return nil, errors.Wrapf(err, "Error executing REST api for restore") 255 | } 256 | 257 | // if apiserver is having version <=1.8 then it will return empty response 258 | ok, err := isEmptyRestResponse(data) 259 | if !ok && err == nil { 260 | err = p.updateVolCASInfo(data, vol.volname) 261 | if err != nil { 262 | err = errors.Wrapf(err, "Error parsing restore API response") 263 | } 264 | } 265 | 266 | return restore, err 267 | } 268 | 269 | func isEmptyRestResponse(data []byte) (bool, error) { 270 | var obj interface{} 271 | 272 | dec := json.NewDecoder(bytes.NewReader(data)) 273 | err := dec.Decode(&obj) 274 | if err != nil { 275 | return false, err 276 | } 277 | 278 | res, ok := obj.(string) 279 | if ok && res == "" { 280 | return true, nil 281 | } 282 | 283 | return false, nil 284 | } 285 | 286 | func (p *Plugin) sendDeleteRequest(backup, volume, namespace, schedule string, isCSIVolume bool) error { 287 | var url string 288 | 289 | if isCSIVolume { 290 | url = p.cvcAddr + backupEndpoint + backup 291 | } else { 292 | url = p.mayaAddr + backupEndpoint + backup 293 | } 294 | 295 | req, err := http.NewRequest("DELETE", url, nil) 296 | if err != nil { 297 | return errors.Wrapf(err, "failed to create HTTP request") 298 | } 299 | 300 | q := req.URL.Query() 301 | q.Add("volume", volume) 302 | q.Add("namespace", namespace) 303 | q.Add("schedule", schedule) 304 | 305 | req.URL.RawQuery = q.Encode() 306 | 307 | c := &http.Client{ 308 | Timeout: p.restTimeout, 309 | } 310 | 311 | resp, err := c.Do(req) 312 | if err != nil { 313 | return errors.Wrapf(err, "failed to connect maya-apiserver") 314 | } 315 | 316 | defer func() { 317 | if err = resp.Body.Close(); err != nil { 318 | p.Log.Warnf("Failed to close response err=%s", err) 319 | } 320 | }() 321 | 322 | respdata, err := ioutil.ReadAll(resp.Body) 323 | if err != nil { 324 | return errors.Wrapf(err, "failed to read response from maya-apiserver, response=%s", string(respdata)) 325 | } 326 | 327 | code := resp.StatusCode 328 | if code != http.StatusOK { 329 | return errors.Errorf("HTTP Status error{%v} from maya-apiserver, response=%s", code, string(respdata)) 330 | } 331 | 332 | return nil 333 | } 334 | -------------------------------------------------------------------------------- /pkg/cstor/cvr_operation.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cstor 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strings" 23 | "time" 24 | 25 | cstorv1 "github.com/openebs/api/v2/pkg/apis/cstor/v1" 26 | "github.com/pkg/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | ) 29 | 30 | const ( 31 | cVRPVLabel = "openebs.io/persistent-volume" 32 | restoreCompletedAnnotation = "openebs.io/restore-completed" 33 | ) 34 | 35 | var validCvrStatuses = []string{ 36 | string(cstorv1.CVRStatusOnline), 37 | string(cstorv1.CVRStatusError), 38 | string(cstorv1.CVRStatusDegraded), 39 | } 40 | 41 | // CVRWaitCount control time limit for waitForAllCVR 42 | var CVRWaitCount = 100 43 | 44 | // CVRCheckInterval defines amount of delay for CVR check 45 | var CVRCheckInterval = 5 * time.Second 46 | 47 | // waitForAllCVRs will ensure that all CVR related to 48 | // the given volume is created 49 | func (p *Plugin) waitForAllCVRs(vol *Volume) error { 50 | return p.waitForAllCVRsToBeInValidStatus(vol, validCvrStatuses) 51 | } 52 | 53 | func (p *Plugin) waitForAllCVRsToBeInValidStatus(vol *Volume, statuses []string) error { 54 | replicaCount := p.getCVRCount(vol.volname, vol.isCSIVolume) 55 | if replicaCount == -1 { 56 | return errors.Errorf("Failed to fetch replicaCount for volume{%s}", vol.volname) 57 | } 58 | 59 | if vol.isCSIVolume { 60 | return p.waitForCSIBasedCVRs(vol, replicaCount, statuses) 61 | } 62 | return p.waitFoNonCSIBasedCVRs(vol, replicaCount, statuses) 63 | } 64 | 65 | // waitFoNonCSIBasedCVRs will ensure that all CVRs related to 66 | // given non CSI based volume is created 67 | func (p *Plugin) waitFoNonCSIBasedCVRs(vol *Volume, replicaCount int, statuses []string) error { 68 | for cnt := 0; cnt < CVRWaitCount; cnt++ { 69 | cvrList, err := p.OpenEBSClient. 70 | OpenebsV1alpha1(). 71 | CStorVolumeReplicas(p.namespace). 72 | List(context.TODO(), metav1.ListOptions{ 73 | LabelSelector: cVRPVLabel + "=" + vol.volname, 74 | }) 75 | if err != nil { 76 | return errors.Errorf("Failed to fetch CVR for volume=%s %s", vol.volname, err) 77 | } 78 | if len(cvrList.Items) != replicaCount { 79 | time.Sleep(CVRCheckInterval) 80 | continue 81 | } 82 | cvrCount := 0 83 | for _, cvr := range cvrList.Items { 84 | if contains(statuses, string(cvr.Status.Phase)) { 85 | cvrCount++ 86 | } 87 | } 88 | if cvrCount == replicaCount { 89 | return nil 90 | } 91 | time.Sleep(CVRCheckInterval) 92 | } 93 | return errors.Errorf("CVR for volume{%s} are not ready!", vol.volname) 94 | } 95 | 96 | // waitForCSIBasedCVRs will ensure that all CVRs related to 97 | // the given CSI volume is created. 98 | func (p *Plugin) waitForCSIBasedCVRs(vol *Volume, replicaCount int, statuses []string) error { 99 | for cnt := 0; cnt < CVRWaitCount; cnt++ { 100 | cvrList, err := p.OpenEBSAPIsClient. 101 | CstorV1(). 102 | CStorVolumeReplicas(p.namespace). 103 | List(context.TODO(), metav1.ListOptions{ 104 | LabelSelector: cVRPVLabel + "=" + vol.volname, 105 | }) 106 | if err != nil { 107 | return errors.Errorf("Failed to fetch CVR for volume=%s %s", vol.volname, err) 108 | } 109 | 110 | if len(cvrList.Items) != replicaCount { 111 | time.Sleep(CVRCheckInterval) 112 | continue 113 | } 114 | 115 | cvrCount := 0 116 | for _, cvr := range cvrList.Items { 117 | if contains(statuses, string(cvr.Status.Phase)) { 118 | cvrCount++ 119 | } 120 | } 121 | if cvrCount == replicaCount { 122 | return nil 123 | } 124 | time.Sleep(CVRCheckInterval) 125 | } 126 | return errors.Errorf("CVR for volume{%s} are not ready!", vol.volname) 127 | } 128 | 129 | // getCVRCount returns the number of CVR for a given volume 130 | func (p *Plugin) getCVRCount(volname string, isCSIVolume bool) int { 131 | // For CSI based volume, CVR of v1 is used. 132 | if isCSIVolume { 133 | // If the volume is CSI based, then CVR V1 is used. 134 | obj, err := p.OpenEBSAPIsClient. 135 | CstorV1(). 136 | CStorVolumes(p.namespace). 137 | Get(context.TODO(), volname, metav1.GetOptions{}) 138 | if err != nil { 139 | p.Log.Errorf("Failed to fetch cstorVolume.. %s", err) 140 | return -1 141 | } 142 | 143 | return obj.Spec.ReplicationFactor 144 | } 145 | // For non CSI based volume, CVR of v1alpha1 is used. 146 | obj, err := p.OpenEBSClient. 147 | OpenebsV1alpha1(). 148 | CStorVolumes(p.namespace). 149 | Get(context.TODO(), volname, metav1.GetOptions{}) 150 | if err != nil { 151 | p.Log.Errorf("Failed to fetch cstorVolume.. %s", err) 152 | return -1 153 | } 154 | return obj.Spec.ReplicationFactor 155 | } 156 | 157 | // markCVRsAsRestoreCompleted annotate relevant CVR with restoreCompletedAnnotation 158 | // Note: It will not wait for CVR to become healthy. This is mainly to avoid the scenarios 159 | // where target-affinity is used. 160 | func (p *Plugin) markCVRsAsRestoreCompleted(vol *Volume) error { 161 | p.Log.Infof("Waiting for all CVRs to be ready") 162 | if err := p.waitForAllCVRs(vol); err != nil { 163 | return err 164 | } 165 | 166 | p.Log.Infof("Marking restore as completed on CVRs") 167 | if err := p.markRestoreAsCompleted(vol); err != nil { 168 | p.Log.Errorf("Failed to mark restore as completed : %s", err) 169 | return err 170 | } 171 | 172 | p.Log.Infof("Successfully annotated all CVRs") 173 | return nil 174 | } 175 | 176 | // markRestoreAsCompleted will mark CVRs that the restore was completed 177 | func (p *Plugin) markRestoreAsCompleted(vol *Volume) error { 178 | if vol.isCSIVolume { 179 | return p.markRestoreAsCompletedForCSIBasedCVRs(vol) 180 | } 181 | return p.markRestoreAsCompletedForNonCSIBasedCVRs(vol) 182 | } 183 | 184 | func (p *Plugin) markRestoreAsCompletedForCSIBasedCVRs(vol *Volume) error { 185 | replicas := p.OpenEBSAPIsClient.CstorV1(). 186 | CStorVolumeReplicas(p.namespace) 187 | 188 | cvrList, err := replicas. 189 | List(context.TODO(), metav1.ListOptions{ 190 | LabelSelector: cVRPVLabel + "=" + vol.volname, 191 | }) 192 | 193 | if err != nil { 194 | return errors.Errorf("Failed to fetch CVR for volume=%s %s", vol.volname, err) 195 | } 196 | 197 | var errs []string 198 | for i := range cvrList.Items { 199 | cvr := cvrList.Items[i] 200 | p.Log.Infof("Updating CVRs %s", cvr.Name) 201 | 202 | cvr.Annotations[restoreCompletedAnnotation] = trueStr 203 | _, err := replicas.Update(context.TODO(), &cvr, metav1.UpdateOptions{}) 204 | 205 | if err != nil { 206 | p.Log.Warnf("could not update CVR %s", cvr.Name) 207 | errs = append(errs, err.Error()) 208 | } 209 | } 210 | 211 | if len(errs) > 0 { 212 | return fmt.Errorf(strings.Join(errs, "; ")) 213 | } 214 | 215 | return nil 216 | } 217 | 218 | func (p *Plugin) markRestoreAsCompletedForNonCSIBasedCVRs(vol *Volume) error { 219 | replicas := p.OpenEBSClient.OpenebsV1alpha1(). 220 | CStorVolumeReplicas(p.namespace) 221 | 222 | cvrList, err := replicas. 223 | List(context.TODO(), metav1.ListOptions{ 224 | LabelSelector: cVRPVLabel + "=" + vol.volname, 225 | }) 226 | 227 | if err != nil { 228 | return errors.Errorf("Failed to fetch CVR for volume=%s %s", vol.volname, err) 229 | } 230 | 231 | var errs []string 232 | for i := range cvrList.Items { 233 | cvr := cvrList.Items[i] 234 | p.Log.Infof("Updating CVRs %s", cvr.Name) 235 | 236 | cvr.Annotations[restoreCompletedAnnotation] = trueStr 237 | _, err := replicas.Update(context.TODO(), &cvr, metav1.UpdateOptions{}) 238 | 239 | if err != nil { 240 | p.Log.Warnf("could not update CVR %s", cvr.Name) 241 | errs = append(errs, err.Error()) 242 | } 243 | } 244 | 245 | if len(errs) > 0 { 246 | return fmt.Errorf(strings.Join(errs, "; ")) 247 | } 248 | 249 | return nil 250 | } 251 | -------------------------------------------------------------------------------- /pkg/cstor/pv_operation.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cstor 18 | 19 | import ( 20 | "context" 21 | "encoding/json" 22 | "sort" 23 | 24 | uuid "github.com/gofrs/uuid" 25 | v1alpha1 "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" 26 | "github.com/pkg/errors" 27 | v1 "k8s.io/api/core/v1" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | ) 30 | 31 | const ( 32 | // PvClonePrefix prefix for clone volume in case restore from local backup 33 | PvClonePrefix = "cstor-clone-" 34 | ) 35 | 36 | func (p *Plugin) updateVolCASInfo(data []byte, volumeID string) error { 37 | var cas v1alpha1.CASVolume 38 | 39 | vol := p.volumes[volumeID] 40 | if vol == nil { 41 | return errors.Errorf("Volume{%s} not found in volume list", volumeID) 42 | } 43 | 44 | if !vol.isCSIVolume { 45 | err := json.Unmarshal(data, &cas) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | vol.iscsi = v1.ISCSIPersistentVolumeSource{ 51 | TargetPortal: cas.Spec.TargetPortal, 52 | IQN: cas.Spec.Iqn, 53 | Lun: cas.Spec.Lun, 54 | FSType: cas.Spec.FSType, 55 | ReadOnly: false, 56 | } 57 | return nil 58 | } 59 | //NOTE: As of now no need to handle restore response for cStor CSI volumes 60 | return nil 61 | } 62 | 63 | // restoreVolumeFromCloud restore remote snapshot for the given volume 64 | // Note: cstor snapshots are incremental in nature, so restore will be executed 65 | // from base snapshot to incremental snapshot 'vol.backupName' if p.restoreAllSnapshots is set 66 | // else restore will be performed for the given backup only. 67 | func (p *Plugin) restoreVolumeFromCloud(vol *Volume, targetBackupName string) error { 68 | var ( 69 | snapshotList []string 70 | err error 71 | ) 72 | 73 | if p.restoreAllSnapshots { 74 | // We are restoring from base backup to targeted Backup 75 | snapshotList, err = p.cl.GetSnapListFromCloud(vol.snapshotTag, p.getScheduleName(targetBackupName)) 76 | if err != nil { 77 | return err 78 | } 79 | } else { 80 | // We are restoring only given backup 81 | snapshotList = []string{targetBackupName} 82 | } 83 | 84 | if !contains(snapshotList, targetBackupName) { 85 | return errors.Errorf("Targeted backup=%s not found in snapshot list", targetBackupName) 86 | } 87 | 88 | // snapshots are created using timestamp, we need to sort it in ascending order 89 | sort.Strings(snapshotList) 90 | 91 | for _, snap := range snapshotList { 92 | // Check if snapshot file exists or not. 93 | // There is a possibility where only PVC file exists, 94 | // in case of failed/partially-failed backup, but not snapshot file. 95 | exists, err := p.cl.FileExists(vol.snapshotTag, snap) 96 | if err != nil { 97 | p.Log.Errorf("Failed to check remote snapshot=%s, skipping restore of this snapshot, err=%s", snap, err) 98 | continue 99 | } 100 | 101 | // If the snapshot doesn't exist, skip the restore for that snapshot. 102 | // Since the snapshots are incremental, we need to continue to restore for the next snapshot. 103 | if !exists { 104 | p.Log.Warningf("Remote snapshot=%s doesn't exist, skipping restore of this snapshot", snap) 105 | continue 106 | } 107 | 108 | p.Log.Infof("Restoring snapshot=%s", snap) 109 | 110 | vol.backupName = snap 111 | 112 | err = p.restoreSnapshotFromCloud(vol) 113 | if err != nil { 114 | return errors.Wrapf(err, "failed to restor snapshot=%s", snap) 115 | } 116 | p.Log.Infof("Restore of snapshot=%s completed", snap) 117 | 118 | if snap == targetBackupName { 119 | // we restored till the targetBackupName, no need to restore next snapshot 120 | break 121 | } 122 | } 123 | return nil 124 | } 125 | 126 | // restoreSnapshotFromCloud restore snapshot 'vol.backupName` to volume 'vol.volname' 127 | func (p *Plugin) restoreSnapshotFromCloud(vol *Volume) error { 128 | p.cl.ExitServer = false 129 | 130 | restore, err := p.sendRestoreRequest(vol) 131 | if err != nil { 132 | return errors.Wrapf(err, "Restore request to apiServer failed") 133 | } 134 | 135 | filename := p.cl.GenerateRemoteFilename(vol.snapshotTag, vol.backupName) 136 | if filename == "" { 137 | return errors.Errorf("Error creating remote file name for restore") 138 | } 139 | 140 | go p.checkRestoreStatus(restore, vol) 141 | 142 | ret := p.cl.Download(filename, CstorRestorePort) 143 | if !ret { 144 | return errors.New("failed to restore snapshot") 145 | } 146 | 147 | if vol.restoreStatus != v1alpha1.RSTCStorStatusDone { 148 | return errors.Errorf("failed to restore.. status {%s}", vol.restoreStatus) 149 | } 150 | 151 | return nil 152 | } 153 | 154 | func (p *Plugin) getPV(volumeID string) (*v1.PersistentVolume, error) { 155 | return p.K8sClient. 156 | CoreV1(). 157 | PersistentVolumes(). 158 | Get(context.TODO(), volumeID, metav1.GetOptions{}) 159 | } 160 | 161 | func (p *Plugin) restoreVolumeFromLocal(vol *Volume) error { 162 | _, err := p.sendRestoreRequest(vol) 163 | if err != nil { 164 | return errors.Wrapf(err, "Restore request to apiServer failed") 165 | } 166 | vol.restoreStatus = v1alpha1.RSTCStorStatusDone 167 | return nil 168 | } 169 | 170 | // getVolumeForLocalRestore return volume information to restore locally for the given volumeID and snapName 171 | // volumeID : pv name from backup 172 | // snapName : snapshot name from where new volume will be created 173 | func (p *Plugin) getVolumeForLocalRestore(volumeID, snapName string) (*Volume, error) { 174 | pv, err := p.getPV(volumeID) 175 | if err != nil { 176 | return nil, errors.Wrapf(err, "error fetching PV=%s", volumeID) 177 | } 178 | 179 | clonePvName, err := generateClonePVName() 180 | if err != nil { 181 | return nil, err 182 | } 183 | p.Log.Infof("Renaming PV %s to %s", pv.Name, clonePvName) 184 | 185 | isCSIVolume := isCSIPv(*pv) 186 | 187 | vol := &Volume{ 188 | volname: clonePvName, 189 | srcVolname: pv.Name, 190 | backupName: snapName, 191 | storageClass: pv.Spec.StorageClassName, 192 | size: pv.Spec.Capacity[v1.ResourceStorage], 193 | isCSIVolume: isCSIVolume, 194 | } 195 | p.volumes[vol.volname] = vol 196 | return vol, nil 197 | } 198 | 199 | // getVolumeForRemoteRestore return volume information to restore from remote backup for the given volumeID and snapName 200 | // volumeID : pv name from backup 201 | // snapName : snapshot name from where new volume will be created 202 | func (p *Plugin) getVolumeForRemoteRestore(volumeID, snapName string) (*Volume, error) { 203 | vol, err := p.createPVC(volumeID, snapName) 204 | if err != nil { 205 | p.Log.Errorf("CreatePVC returned error=%s", err) 206 | return nil, err 207 | } 208 | 209 | p.Log.Infof("Generated PV name is %s", vol.volname) 210 | 211 | return vol, nil 212 | } 213 | 214 | // generateClonePVName return new name for clone pv for the given pv 215 | func generateClonePVName() (string, error) { 216 | nuuid, err := uuid.NewV4() 217 | if err != nil { 218 | return "", errors.Wrapf(err, "Error generating uuid for PV rename") 219 | } 220 | 221 | return PvClonePrefix + nuuid.String(), nil 222 | } 223 | 224 | // isCSIPv returns true if given PV is created by cstor CSI driver 225 | func isCSIPv(pv v1.PersistentVolume) bool { 226 | if pv.Spec.CSI != nil && 227 | pv.Spec.CSI.Driver == openebsCSIName { 228 | return true 229 | } 230 | return false 231 | } 232 | 233 | // contains return true if given target string exists in slice s 234 | func contains(s []string, target string) bool { 235 | for _, v := range s { 236 | if v == target { 237 | return true 238 | } 239 | } 240 | 241 | return false 242 | } 243 | -------------------------------------------------------------------------------- /pkg/cstor/status.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cstor 18 | 19 | import ( 20 | "encoding/json" 21 | "time" 22 | 23 | "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" 24 | ) 25 | 26 | // checkBackupStatus queries MayaAPI server for given backup status 27 | // and wait until backup completes 28 | func (p *Plugin) checkBackupStatus(bkp *v1alpha1.CStorBackup, isCSIVolume bool) { 29 | var ( 30 | bkpDone bool 31 | url string 32 | ) 33 | 34 | if isCSIVolume { 35 | url = p.cvcAddr + backupEndpoint 36 | } else { 37 | url = p.mayaAddr + backupEndpoint 38 | } 39 | 40 | bkpvolume, exists := p.volumes[bkp.Spec.VolumeName] 41 | if !exists { 42 | p.Log.Errorf("Failed to fetch volume info for {%s}", bkp.Spec.VolumeName) 43 | p.cl.ExitServer = true 44 | bkpvolume.backupStatus = v1alpha1.BKPCStorStatusInvalid 45 | return 46 | } 47 | 48 | bkpData, err := json.Marshal(bkp) 49 | if err != nil { 50 | p.Log.Errorf("JSON marshal failed : %s", err.Error()) 51 | p.cl.ExitServer = true 52 | bkpvolume.backupStatus = v1alpha1.BKPCStorStatusInvalid 53 | return 54 | } 55 | 56 | for !bkpDone { 57 | var bs v1alpha1.CStorBackup 58 | 59 | time.Sleep(backupStatusInterval * time.Second) 60 | resp, err := p.httpRestCall(url, "GET", bkpData) 61 | if err != nil { 62 | p.Log.Warnf("Failed to fetch backup status : %s", err.Error()) 63 | continue 64 | } 65 | 66 | err = json.Unmarshal(resp, &bs) 67 | if err != nil { 68 | p.Log.Warnf("Unmarshal failed : %s", err.Error()) 69 | continue 70 | } 71 | 72 | bkpvolume.backupStatus = bs.Status 73 | 74 | switch bs.Status { 75 | case v1alpha1.BKPCStorStatusDone, v1alpha1.BKPCStorStatusFailed, v1alpha1.BKPCStorStatusInvalid: 76 | bkpDone = true 77 | p.cl.ExitServer = true 78 | if err = p.cleanupCompletedBackup(bs, isCSIVolume); err != nil { 79 | p.Log.Warningf("failed to execute clean-up request for backup=%s err=%s", bs.Name, err) 80 | } 81 | } 82 | } 83 | } 84 | 85 | // checkRestoreStatus queries MayaAPI server for given restore status 86 | // and wait until restore completes 87 | func (p *Plugin) checkRestoreStatus(rst *v1alpha1.CStorRestore, vol *Volume) { 88 | var ( 89 | rstDone bool 90 | url string 91 | ) 92 | 93 | if vol.isCSIVolume { 94 | url = p.cvcAddr + restorePath 95 | } else { 96 | url = p.mayaAddr + restorePath 97 | } 98 | 99 | rstData, err := json.Marshal(rst) 100 | if err != nil { 101 | p.Log.Errorf("JSON marshal failed : %s", err.Error()) 102 | vol.restoreStatus = v1alpha1.RSTCStorStatusInvalid 103 | p.cl.ExitServer = true 104 | } 105 | 106 | for !rstDone { 107 | var rs v1alpha1.CStorRestore 108 | 109 | time.Sleep(restoreStatusInterval * time.Second) 110 | resp, err := p.httpRestCall(url, "GET", rstData) 111 | if err != nil { 112 | p.Log.Warnf("Failed to fetch backup status : %s", err.Error()) 113 | continue 114 | } 115 | 116 | err = json.Unmarshal(resp, &rs.Status) 117 | if err != nil { 118 | p.Log.Warnf("Unmarshal failed : %s", err.Error()) 119 | continue 120 | } 121 | 122 | vol.restoreStatus = rs.Status 123 | 124 | switch rs.Status { 125 | case v1alpha1.RSTCStorStatusDone, v1alpha1.RSTCStorStatusFailed, v1alpha1.RSTCStorStatusInvalid: 126 | rstDone = true 127 | p.cl.ExitServer = true 128 | } 129 | } 130 | } 131 | 132 | // cleanupCompletedBackup send the delete request to apiserver 133 | // to cleanup backup resources 134 | // If it is normal backup then it will delete the current backup, it can be failed or succeeded backup 135 | // If it is scheduled backup then 136 | // - if current backup is base backup, not incremental one, then it will not perform any clean-up 137 | // - if current backup is incremental backup and failed one then it will delete that(current) backup 138 | // - if current backup is incremental backup and completed successfully then 139 | // it will delete the last completed or previous backup 140 | func (p *Plugin) cleanupCompletedBackup(bkp v1alpha1.CStorBackup, isCSIVolume bool) error { 141 | targetedSnapName := bkp.Spec.SnapName 142 | 143 | // In case of scheduled backup we are using the last completed backup to send 144 | // differential snapshot. So We don't need to delete the last completed backup. 145 | if isScheduledBackup(bkp) && isBackupSucceeded(bkp) { 146 | // For incremental backup We are using PrevSnapName to send the differential snapshot 147 | // Since Given backup is completed successfully We can delete the 2nd last completed backup 148 | if bkp.Spec.PrevSnapName == "" { 149 | // PrevSnapName will be empty if the given backup is base backup 150 | // clean-up is not required for base backup 151 | return nil 152 | } 153 | targetedSnapName = bkp.Spec.PrevSnapName 154 | } 155 | 156 | p.Log.Infof("executing clean-up request.. snapshot=%s volume=%s ns=%s backup=%s", 157 | targetedSnapName, 158 | bkp.Spec.VolumeName, 159 | bkp.Namespace, 160 | bkp.Spec.BackupName, 161 | ) 162 | 163 | return p.sendDeleteRequest(targetedSnapName, 164 | bkp.Spec.VolumeName, 165 | bkp.Namespace, 166 | bkp.Spec.BackupName, 167 | isCSIVolume) 168 | } 169 | 170 | // return true if given backup is part of schedule 171 | func isScheduledBackup(bkp v1alpha1.CStorBackup) bool { 172 | // if backup is scheduled backup then snapshot name and backup name are different 173 | return bkp.Spec.SnapName != bkp.Spec.BackupName 174 | } 175 | 176 | // isBackupSucceeded returns true if backup completed successfully 177 | func isBackupSucceeded(bkp v1alpha1.CStorBackup) bool { 178 | return bkp.Status == v1alpha1.BKPCStorStatusDone 179 | } 180 | -------------------------------------------------------------------------------- /pkg/snapshot/snap.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package snapshot 18 | 19 | import ( 20 | "github.com/openebs/velero-plugin/pkg/cstor" 21 | "github.com/sirupsen/logrus" 22 | "github.com/vmware-tanzu/velero/pkg/plugin/velero" 23 | "k8s.io/apimachinery/pkg/runtime" 24 | ) 25 | 26 | // BlockStore : Plugin for containing state for the blockstore plugin 27 | type BlockStore struct { 28 | Log logrus.FieldLogger 29 | plugin velero.VolumeSnapshotter 30 | } 31 | 32 | var _ velero.VolumeSnapshotter = (*BlockStore)(nil) 33 | 34 | // Init the plugin 35 | func (p *BlockStore) Init(config map[string]string) error { 36 | p.Log.Infof("Initializing velero plugin for CStor") 37 | 38 | p.plugin = &cstor.Plugin{Log: p.Log} 39 | return p.plugin.Init(config) 40 | } 41 | 42 | // CreateVolumeFromSnapshot Create a volume from given snapshot 43 | func (p *BlockStore) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) { 44 | return p.plugin.CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ, iops) 45 | } 46 | 47 | // GetVolumeInfo Get information about the volume 48 | func (p *BlockStore) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { 49 | return p.plugin.GetVolumeInfo(volumeID, volumeAZ) 50 | } 51 | 52 | // IsVolumeReady Check if the volume is ready. 53 | func (p *BlockStore) IsVolumeReady(volumeID, volumeAZ string) (ready bool, err error) { 54 | return true, nil 55 | } 56 | 57 | // CreateSnapshot Create a snapshot 58 | func (p *BlockStore) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { 59 | return p.plugin.CreateSnapshot(volumeID, volumeAZ, tags) 60 | } 61 | 62 | // DeleteSnapshot Delete a snapshot 63 | func (p *BlockStore) DeleteSnapshot(snapshotID string) error { 64 | return p.plugin.DeleteSnapshot(snapshotID) 65 | } 66 | 67 | // GetVolumeID Get the volume ID from the spec 68 | func (p *BlockStore) GetVolumeID(unstructuredPV runtime.Unstructured) (string, error) { 69 | return p.plugin.GetVolumeID(unstructuredPV) 70 | } 71 | 72 | // SetVolumeID Set the volume ID in the spec 73 | func (p *BlockStore) SetVolumeID(unstructuredPV runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { 74 | return p.plugin.SetVolumeID(unstructuredPV, volumeID) 75 | } 76 | -------------------------------------------------------------------------------- /pkg/velero/restore.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package velero 18 | 19 | import ( 20 | "context" 21 | "sort" 22 | 23 | "github.com/pkg/errors" 24 | "github.com/sirupsen/logrus" 25 | velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/client-go/kubernetes" 28 | ) 29 | 30 | // GetRestoreNamespace return the namespace mapping for the given namespace 31 | // if namespace mapping not found then it will return the same namespace in which backup was created 32 | // if namespace mapping found then it will return the mapping/target namespace 33 | // 34 | // velero doesn't pass the restore name to plugin, so we are following the below 35 | // approach to fetch the namespace mapping: 36 | // 37 | // plugin find the relevant restore from the sorted list(creationTimestamp in decreasing order) of 38 | // restore resource using following criteria: 39 | // - retore is in in-progress state AND 40 | // backup for that restore matches with the backup name from snapshotID 41 | // Above approach works because velero support sequential restore 42 | func GetRestoreNamespace(ns, bkpName string, log logrus.FieldLogger) (string, error) { 43 | listOpts := metav1.ListOptions{} 44 | list, err := clientSet.VeleroV1().Restores(veleroNs).List(context.TODO(), listOpts) 45 | if err != nil { 46 | return "", errors.Wrapf(err, "failed to get list of restore") 47 | } 48 | 49 | sort.Sort(sort.Reverse(RestoreByCreationTimestamp(list.Items))) 50 | 51 | for _, r := range list.Items { 52 | if r.Status.Phase == velerov1api.RestorePhaseInProgress && r.Spec.BackupName == bkpName { 53 | targetedNs, ok := r.Spec.NamespaceMapping[ns] 54 | if ok { 55 | return targetedNs, nil 56 | } 57 | return ns, nil 58 | } 59 | } 60 | return "", errors.Errorf("restore not found for backup %s", bkpName) 61 | } 62 | 63 | // GetTargetNode return the node mapping for the given node 64 | // if node mapping not found then it will return the same nodename in which backup was created 65 | // if node mapping found then it will return the mapping/target nodename 66 | func GetTargetNode(k8s *kubernetes.Clientset, node string) (string, error) { 67 | opts := metav1.ListOptions{ 68 | LabelSelector: "velero.io/plugin-config,velero.io/change-pvc-node-selector=RestoreItemAction", 69 | } 70 | 71 | list, err := k8s.CoreV1().ConfigMaps(veleroNs).List(context.TODO(), opts) 72 | if err != nil { 73 | return "", errors.Wrapf(err, "failed to get list of node mapping configmap") 74 | } 75 | 76 | if len(list.Items) == 0 { 77 | return node, nil 78 | } 79 | 80 | if len(list.Items) > 1 { 81 | var items []string 82 | for _, item := range list.Items { 83 | items = append(items, item.Name) 84 | } 85 | return "", errors.Errorf("found more than one ConfigMap matching label selector %q: %v", opts.LabelSelector, items) 86 | } 87 | 88 | config := list.Items[0] 89 | 90 | tnode, ok := config.Data[node] 91 | if !ok { 92 | return node, nil 93 | } 94 | 95 | return tnode, nil 96 | } 97 | -------------------------------------------------------------------------------- /pkg/velero/sort.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package velero 18 | 19 | import velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" 20 | 21 | // RestoreByCreationTimestamp sorts a list of Restore by creation timestamp, using their names as a tie breaker. 22 | type RestoreByCreationTimestamp []velerov1api.Restore 23 | 24 | func (o RestoreByCreationTimestamp) Len() int { return len(o) } 25 | func (o RestoreByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } 26 | func (o RestoreByCreationTimestamp) Less(i, j int) bool { 27 | if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { 28 | return o[i].Name < o[j].Name 29 | } 30 | return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/velero/velero.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package velero 18 | 19 | import ( 20 | "os" 21 | 22 | veleroclient "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" 23 | "k8s.io/client-go/rest" 24 | ) 25 | 26 | var ( 27 | // clientSet will be used to fetch velero customo resources 28 | clientSet veleroclient.Interface 29 | 30 | // veleroNs velero installation namespace 31 | veleroNs string 32 | ) 33 | 34 | func init() { 35 | veleroNs = os.Getenv("VELERO_NAMESPACE") 36 | } 37 | 38 | // InitializeClientSet initialize velero clientset 39 | func InitializeClientSet(config *rest.Config) error { 40 | var err error 41 | 42 | clientSet, err = veleroclient.NewForConfig(config) 43 | 44 | return err 45 | } 46 | -------------------------------------------------------------------------------- /pkg/zfs/plugin/backup.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package plugin 18 | 19 | import ( 20 | "context" 21 | "encoding/json" 22 | "sort" 23 | "strconv" 24 | "sync" 25 | "time" 26 | 27 | "github.com/openebs/velero-plugin/pkg/zfs/utils" 28 | apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/zfs/v1" 29 | "github.com/openebs/zfs-localpv/pkg/builder/bkpbuilder" 30 | "github.com/openebs/zfs-localpv/pkg/builder/volbuilder" 31 | "github.com/pkg/errors" 32 | v1 "k8s.io/api/core/v1" 33 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 34 | ) 35 | 36 | const ( 37 | VeleroBkpKey = "velero.io/backup" 38 | VeleroSchdKey = "velero.io/schedule-name" 39 | VeleroVolKey = "velero.io/volname" 40 | VeleroNsKey = "velero.io/namespace" 41 | ) 42 | 43 | func (p *Plugin) getPV(volumeID string) (*v1.PersistentVolume, error) { 44 | return p.K8sClient. 45 | CoreV1(). 46 | PersistentVolumes(). 47 | Get(context.TODO(), volumeID, metav1.GetOptions{}) 48 | } 49 | 50 | func (p *Plugin) uploadZFSVolume(vol *apis.ZFSVolume, filename string) error { 51 | data, err := json.MarshalIndent(vol, "", "\t") 52 | if err != nil { 53 | return errors.New("zfs: error doing json parsing") 54 | } 55 | 56 | if ok := p.cl.Write(data, filename+".zfsvol"); !ok { 57 | return errors.New("zfs: failed to upload ZFSVolume") 58 | } 59 | 60 | return nil 61 | } 62 | 63 | // deleteBackup deletes the backup resource 64 | func (p *Plugin) deleteBackup(snapshotID string) error { 65 | pvname, _, snapname, err := utils.GetInfoFromSnapshotID(snapshotID) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | bkpname := utils.GenerateResourceName(pvname, snapname) 71 | err = bkpbuilder.NewKubeclient().WithNamespace(p.namespace).Delete(bkpname) 72 | if err != nil { 73 | p.Log.Errorf("zfs: Failed to delete the backup %s", snapshotID) 74 | } 75 | 76 | return err 77 | } 78 | 79 | func (p *Plugin) getPrevSnap(volname, schdname string) (string, error) { 80 | if p.incremental < 1 || len(schdname) == 0 { 81 | // not an incremental backup, take the full backup 82 | return "", nil 83 | } 84 | 85 | listOptions := metav1.ListOptions{ 86 | LabelSelector: VeleroSchdKey + "=" + schdname + "," + VeleroVolKey + "=" + volname, 87 | } 88 | 89 | bkpList, err := bkpbuilder.NewKubeclient(). 90 | WithNamespace(p.namespace).List(listOptions) 91 | 92 | if err != nil { 93 | return "", err 94 | } 95 | 96 | var backups []string 97 | 98 | size := len(bkpList.Items) 99 | count := p.incremental + 1 100 | 101 | if uint64(size)%count == 0 { 102 | // have to start the next snapshot incremental group, take the full backup 103 | return "", nil 104 | } 105 | 106 | /* 107 | * Backup names are in the form of - 108 | * to get the last snapshot, sort the list of successful backups, 109 | * the previous snapshot will be the last element in the sorted list 110 | */ 111 | if len(bkpList.Items) > 0 { 112 | for _, bkp := range bkpList.Items { 113 | if bkp.Status == apis.BKPZFSStatusDone { 114 | backups = append(backups, bkp.Spec.SnapName) 115 | } 116 | } 117 | size := len(backups) 118 | sort.Strings(backups) 119 | return backups[size-1], nil 120 | } 121 | 122 | return "", nil 123 | } 124 | 125 | func (p *Plugin) createBackup(vol *apis.ZFSVolume, schdname, snapname string, port int) (string, error) { 126 | bkpname := utils.GenerateResourceName(vol.Name, snapname) 127 | 128 | p.Log.Debugf("zfs: creating ZFSBackup vol = %s bkp = %s schd = %s", vol.Name, bkpname, schdname) 129 | 130 | var err error 131 | labels := map[string]string{} 132 | prevSnap := "" 133 | 134 | if len(schdname) > 0 { 135 | // add schdeule name as label 136 | labels[VeleroSchdKey] = schdname 137 | labels[VeleroVolKey] = vol.Name 138 | prevSnap, err = p.getPrevSnap(vol.Name, schdname) 139 | if err != nil { 140 | p.Log.Errorf("zfs: Failed to get prev snapshot bkp %s err: {%v}", snapname, err) 141 | return "", err 142 | } 143 | } 144 | 145 | p.Log.Debugf("zfs: backup incr(%d) schd=%s snap=%s prevsnap=%s vol=%s", p.incremental, schdname, snapname, prevSnap, vol.Name) 146 | 147 | serverAddr := p.remoteAddr + ":" + strconv.Itoa(port) 148 | 149 | bkp, err := bkpbuilder.NewBuilder(). 150 | WithName(bkpname). 151 | WithLabels(labels). 152 | WithVolume(vol.Name). 153 | WithPrevSnap(prevSnap). 154 | WithSnap(snapname). 155 | WithNode(vol.Spec.OwnerNodeID). 156 | WithStatus(apis.BKPZFSStatusInit). 157 | WithRemote(serverAddr). 158 | Build() 159 | 160 | if err != nil { 161 | return "", err 162 | } 163 | _, err = bkpbuilder.NewKubeclient().WithNamespace(p.namespace).Create(bkp) 164 | if err != nil { 165 | return "", err 166 | } 167 | 168 | return bkpname, nil 169 | } 170 | 171 | func (p *Plugin) checkBackupStatus(bkpname string) error { 172 | for { 173 | getOptions := metav1.GetOptions{} 174 | bkp, err := bkpbuilder.NewKubeclient(). 175 | WithNamespace(p.namespace).Get(bkpname, getOptions) 176 | 177 | if err != nil { 178 | p.Log.Errorf("zfs: Failed to fetch backup info {%s}", bkpname) 179 | return errors.Errorf("zfs: error in getting bkp status err %v", err) 180 | } 181 | 182 | switch bkp.Status { 183 | case apis.BKPZFSStatusDone: 184 | return nil 185 | case apis.BKPZFSStatusFailed, apis.BKPZFSStatusInvalid: 186 | return errors.Errorf("zfs: error in uploading snapshot, status:{%v}", bkp.Status) 187 | } 188 | 189 | time.Sleep(backupStatusInterval * time.Second) 190 | } 191 | } 192 | 193 | func (p *Plugin) doUpload(wg *sync.WaitGroup, filename string, size int64, port int) { 194 | defer wg.Done() 195 | 196 | ok := p.cl.Upload(filename, size, port) 197 | if !ok { 198 | p.Log.Errorf("zfs: Failed to upload file %s", filename) 199 | *p.cl.ConnReady <- false 200 | } 201 | // done with the channel, close it 202 | close(*p.cl.ConnReady) 203 | } 204 | 205 | func (p *Plugin) doBackup(volumeID string, snapname string, schdname string, port int) (string, error) { 206 | pv, err := p.getPV(volumeID) 207 | if err != nil { 208 | p.Log.Errorf("zfs: Failed to get pv %s snap %s schd %s err %v", volumeID, snapname, schdname, err) 209 | return "", err 210 | } 211 | 212 | if pv.Spec.PersistentVolumeSource.CSI == nil { 213 | return "", errors.New("zfs: err not a CSI pv") 214 | } 215 | 216 | volHandle := pv.Spec.PersistentVolumeSource.CSI.VolumeHandle 217 | 218 | getOptions := metav1.GetOptions{} 219 | vol, err := volbuilder.NewKubeclient(). 220 | WithNamespace(p.namespace).Get(volHandle, getOptions) 221 | if err != nil { 222 | return "", err 223 | } 224 | 225 | if pv.Spec.ClaimRef != nil { 226 | // add source namespace in the label to filter it at restore time 227 | if vol.Labels == nil { 228 | vol.Labels = map[string]string{} 229 | } 230 | vol.Labels[VeleroNsKey] = pv.Spec.ClaimRef.Namespace 231 | } else { 232 | return "", errors.Errorf("zfs: err pv is not claimed") 233 | } 234 | 235 | filename := p.cl.GenerateRemoteFileWithSchd(volumeID, schdname, snapname) 236 | if filename == "" { 237 | return "", errors.Errorf("zfs: error creating remote file name for backup") 238 | } 239 | 240 | err = p.uploadZFSVolume(vol, filename) 241 | if err != nil { 242 | return "", err 243 | } 244 | 245 | size, err := strconv.ParseInt(vol.Spec.Capacity, 10, 64) 246 | if err != nil { 247 | return "", errors.Errorf("zfs: error parsing the size %s", vol.Spec.Capacity) 248 | } 249 | 250 | p.Log.Debugf("zfs: uploading Snapshot %s file %s", snapname, filename) 251 | 252 | // reset the connection state 253 | p.cl.ConnStateReset() 254 | 255 | var wg sync.WaitGroup 256 | 257 | wg.Add(1) 258 | go p.doUpload(&wg, filename, size, port) 259 | 260 | // wait for the upload server to exit 261 | defer func() { 262 | p.cl.ExitServer = true 263 | wg.Wait() 264 | p.cl.ConnReady = nil 265 | }() 266 | 267 | // wait for the connection to be ready 268 | ok := p.cl.ConnReadyWait() 269 | if !ok { 270 | return "", errors.New("zfs: error in uploading snapshot") 271 | } 272 | 273 | bkpname, err := p.createBackup(vol, schdname, snapname, port) 274 | if err != nil { 275 | return "", err 276 | } 277 | 278 | err = p.checkBackupStatus(bkpname) 279 | if err != nil { 280 | p.deleteBackup(bkpname) 281 | p.Log.Errorf("zfs: backup failed vol %s snap %s bkpname %s err: %v", volumeID, snapname, bkpname, err) 282 | return "", err 283 | } 284 | 285 | // generate the snapID 286 | snapID := utils.GenerateSnapshotID(volumeID, schdname, snapname) 287 | 288 | p.Log.Debugf("zfs: backup done vol %s bkp %s snapID %s", volumeID, bkpname, snapID) 289 | 290 | return snapID, nil 291 | } 292 | -------------------------------------------------------------------------------- /pkg/zfs/plugin/zfs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package plugin 18 | 19 | import ( 20 | "strconv" 21 | 22 | cloud "github.com/openebs/velero-plugin/pkg/clouduploader" 23 | "github.com/openebs/velero-plugin/pkg/velero" 24 | "github.com/openebs/velero-plugin/pkg/zfs/utils" 25 | "github.com/openebs/zfs-localpv/pkg/builder/volbuilder" 26 | "github.com/openebs/zfs-localpv/pkg/zfs" 27 | "github.com/pkg/errors" 28 | "github.com/sirupsen/logrus" 29 | v1 "k8s.io/api/core/v1" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 32 | "k8s.io/apimachinery/pkg/runtime" 33 | "k8s.io/client-go/kubernetes" 34 | "k8s.io/client-go/rest" 35 | ) 36 | 37 | const ( 38 | // ZfsPvNamespace config key for OpenEBS namespace 39 | ZfsPvNamespace = "namespace" 40 | 41 | // ZfsPvIncr config key for providing count of incremental backups 42 | ZfsPvIncr = "incrBackupCount" 43 | 44 | // zfs csi driver name 45 | ZfsDriverName = "zfs.csi.openebs.io" 46 | 47 | backupStatusInterval = 5 48 | 49 | // port to connect for restoring the data 50 | ZFSRestorePort = 9010 51 | 52 | // port to connect for backup 53 | ZFSBackupPort = 9011 54 | ) 55 | 56 | // Plugin is a plugin for containing state for the blockstore 57 | type Plugin struct { 58 | config map[string]string 59 | Log logrus.FieldLogger 60 | 61 | // K8sClient is used for kubernetes operation 62 | K8sClient *kubernetes.Clientset 63 | 64 | // on this address cloud server will perform data operation(backup/restore) 65 | remoteAddr string 66 | 67 | // this is the namespace where all the ZFSPV CRs will be created, 68 | // this should be same as what is passed to ZFS-LocalPV driver 69 | // as env OPENEBS_NAMESPACE while deploying it. 70 | namespace string 71 | 72 | // This specifies how many incremental backup we have to keep 73 | incremental uint64 74 | 75 | // cl stores cloud connection information 76 | cl *cloud.Conn 77 | } 78 | 79 | // Init prepares the VolumeSnapshotter for usage using the provided map of 80 | // configuration key-value pairs. It returns an error if the VolumeSnapshotter 81 | // cannot be initialized from the provided config. Note that after v0.10.0, this will happen multiple times. 82 | func (p *Plugin) Init(config map[string]string) error { 83 | p.Log.Debugf("zfs: Init called %v", config) 84 | p.config = config 85 | 86 | p.remoteAddr, _ = utils.GetServerAddress() 87 | if p.remoteAddr == "" { 88 | return errors.New("zfs: error fetching Server address") 89 | } 90 | 91 | if ns, ok := config[ZfsPvNamespace]; ok { 92 | p.namespace = ns 93 | } else { 94 | return errors.New("zfs: namespace not provided for ZFS-LocalPV") 95 | } 96 | 97 | if count, ok := config[ZfsPvIncr]; ok { 98 | incr, err := strconv.ParseUint(count, 10, 64) 99 | if err != nil { 100 | return errors.Wrapf(err, "zfs: invalid incrBackupCount value=%s", count) 101 | } 102 | p.incremental = incr 103 | } 104 | 105 | conf, err := rest.InClusterConfig() 106 | if err != nil { 107 | p.Log.Errorf("Failed to get cluster config : %s", err.Error()) 108 | return errors.New("error fetching cluster config") 109 | } 110 | 111 | clientset, err := kubernetes.NewForConfig(conf) 112 | if err != nil { 113 | p.Log.Errorf("Error creating clientset : %s", err.Error()) 114 | return errors.New("error creating k8s client") 115 | } 116 | 117 | if err := velero.InitializeClientSet(conf); err != nil { 118 | return errors.Wrapf(err, "failed to initialize velero clientSet") 119 | } 120 | 121 | p.K8sClient = clientset 122 | 123 | p.cl = &cloud.Conn{Log: p.Log} 124 | return p.cl.Init(config) 125 | } 126 | 127 | // CreateVolumeFromSnapshot creates a new volume from the specified snapshot 128 | func (p *Plugin) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) { 129 | p.Log.Debugf("zfs: CreateVolumeFromSnapshot called snap %s", snapshotID) 130 | 131 | volumeID, err := p.doRestore(snapshotID, ZFSRestorePort) 132 | if err != nil { 133 | p.Log.Errorf("zfs: error CreateVolumeFromSnapshot returning snap %s err %v", snapshotID, err) 134 | return "", err 135 | } 136 | 137 | p.Log.Infof("zfs: CreateVolumeFromSnapshot returning snap %s vol %s", snapshotID, volumeID) 138 | return volumeID, nil 139 | } 140 | 141 | // GetVolumeInfo returns the type and IOPS (if using provisioned IOPS) for 142 | // the specified volume in the given availability zone. 143 | func (p *Plugin) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { 144 | p.Log.Debugf("zfs: GetVolumeInfo called", volumeID, volumeAZ) 145 | return "zfs-localpv", nil, nil 146 | } 147 | 148 | // IsVolumeReady Check if the volume is ready. 149 | func (p *Plugin) IsVolumeReady(volumeID, volumeAZ string) (ready bool, err error) { 150 | p.Log.Debugf("zfs: IsVolumeReady called", volumeID, volumeAZ) 151 | 152 | return p.isVolumeReady(volumeID) 153 | } 154 | 155 | // CreateSnapshot creates a snapshot of the specified volume, and applies any provided 156 | // set of tags to the snapshot. 157 | func (p *Plugin) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { 158 | p.Log.Debugf("zfs: CreateSnapshot called", volumeID, volumeAZ, tags) 159 | 160 | bkpname, ok := tags[VeleroBkpKey] 161 | if !ok { 162 | return "", errors.New("zfs: error get backup name") 163 | } 164 | 165 | schdname := tags[VeleroSchdKey] 166 | 167 | snapshotID, err := p.doBackup(volumeID, bkpname, schdname, ZFSBackupPort) 168 | 169 | if err != nil { 170 | p.Log.Errorf("zfs: error createBackup %s@%s failed %v", volumeID, bkpname, err) 171 | return "", err 172 | } 173 | 174 | p.Log.Infof("zfs: CreateSnapshot returning %s", snapshotID) 175 | return snapshotID, nil 176 | } 177 | 178 | // DeleteSnapshot deletes the specified volume snapshot. 179 | func (p *Plugin) DeleteSnapshot(snapshotID string) error { 180 | p.Log.Debugf("zfs: DeleteSnapshot called %s", snapshotID) 181 | if snapshotID == "" { 182 | p.Log.Warning("zfs: Empty snapshotID") 183 | return nil 184 | } 185 | 186 | return p.deleteBackup(snapshotID) 187 | } 188 | 189 | // GetVolumeID returns the specific identifier for the PersistentVolume. 190 | func (p *Plugin) GetVolumeID(unstructuredPV runtime.Unstructured) (string, error) { 191 | p.Log.Debugf("zfs: GetVolumeID called %v", unstructuredPV) 192 | 193 | pv := new(v1.PersistentVolume) 194 | if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredPV.UnstructuredContent(), pv); err != nil { 195 | return "", errors.WithStack(err) 196 | } 197 | 198 | // If PV doesn't have sufficient info to consider as ZFS-LocalPV Volume 199 | // then we will return empty volumeId and error as nil. 200 | if pv.Name == "" || 201 | pv.Spec.StorageClassName == "" || 202 | (pv.Spec.ClaimRef != nil && pv.Spec.ClaimRef.Namespace == "") { 203 | return "", nil 204 | } 205 | 206 | // check if PV is created by ZFS driver 207 | 208 | if pv.Spec.CSI == nil || 209 | pv.Spec.CSI.Driver != ZfsDriverName { 210 | return "", nil 211 | } 212 | 213 | if pv.Status.Phase == v1.VolumeReleased || 214 | pv.Status.Phase == v1.VolumeFailed { 215 | return "", errors.New("pv is in released state") 216 | } 217 | 218 | return pv.Name, nil 219 | } 220 | 221 | // SetVolumeID sets the specific identifier for the PersistentVolume. 222 | func (p *Plugin) SetVolumeID(unstructuredPV runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { 223 | p.Log.Debugf("zfs: SetVolumeID called %v %s", unstructuredPV, volumeID) 224 | 225 | pv := new(v1.PersistentVolume) 226 | if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredPV.UnstructuredContent(), pv); err != nil { 227 | return nil, errors.WithStack(err) 228 | } 229 | 230 | // Set the PV Name and VolumeHandle 231 | pv.Name = volumeID 232 | pv.Spec.PersistentVolumeSource.CSI.VolumeHandle = volumeID 233 | 234 | // set the node affinity 235 | if pv.Spec.NodeAffinity != nil && pv.Spec.NodeAffinity.Required != nil { 236 | vol, err := volbuilder.NewKubeclient(). 237 | WithNamespace(p.namespace).Get(volumeID, metav1.GetOptions{}) 238 | if err != nil { 239 | p.Log.Errorf("zfs: Failed to fetch volume {%s}", volumeID) 240 | return nil, err 241 | } 242 | 243 | var expr []v1.NodeSelectorRequirement 244 | expr = append(expr, v1.NodeSelectorRequirement{ 245 | Key: zfs.ZFSTopologyKey, 246 | Operator: v1.NodeSelectorOpIn, 247 | Values: []string{vol.Spec.OwnerNodeID}, 248 | }) 249 | 250 | var terms []v1.NodeSelectorTerm 251 | terms = append(terms, v1.NodeSelectorTerm{ 252 | MatchExpressions: expr, 253 | }) 254 | 255 | pv.Spec.NodeAffinity = &v1.VolumeNodeAffinity{ 256 | Required: &v1.NodeSelector{ 257 | NodeSelectorTerms: terms, 258 | }, 259 | } 260 | } 261 | 262 | res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pv) 263 | if err != nil { 264 | return nil, errors.WithStack(err) 265 | } 266 | 267 | return &unstructured.Unstructured{Object: res}, nil 268 | } 269 | -------------------------------------------------------------------------------- /pkg/zfs/snapshot/snap.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package snapshot 18 | 19 | import ( 20 | zfs "github.com/openebs/velero-plugin/pkg/zfs/plugin" 21 | "github.com/sirupsen/logrus" 22 | "github.com/vmware-tanzu/velero/pkg/plugin/velero" 23 | "k8s.io/apimachinery/pkg/runtime" 24 | ) 25 | 26 | // BlockStore : Plugin for containing state for the blockstore plugin 27 | type BlockStore struct { 28 | Log logrus.FieldLogger 29 | plugin velero.VolumeSnapshotter 30 | } 31 | 32 | var _ velero.VolumeSnapshotter = (*BlockStore)(nil) 33 | 34 | // Init the plugin 35 | func (p *BlockStore) Init(config map[string]string) error { 36 | p.Log.Infof("zfs: Initializing velero plugin for ZFS-LocalPV") 37 | 38 | p.plugin = &zfs.Plugin{Log: p.Log} 39 | return p.plugin.Init(config) 40 | } 41 | 42 | // CreateVolumeFromSnapshot Create a volume form given snapshot 43 | func (p *BlockStore) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) { 44 | return p.plugin.CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ, iops) 45 | } 46 | 47 | // GetVolumeInfo Get information about the volume 48 | func (p *BlockStore) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { 49 | return p.plugin.GetVolumeInfo(volumeID, volumeAZ) 50 | } 51 | 52 | // IsVolumeReady Check if the volume is ready. 53 | func (p *BlockStore) IsVolumeReady(volumeID, volumeAZ string) (ready bool, err error) { 54 | return true, nil 55 | } 56 | 57 | // CreateSnapshot Create a snapshot 58 | func (p *BlockStore) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { 59 | return p.plugin.CreateSnapshot(volumeID, volumeAZ, tags) 60 | } 61 | 62 | // DeleteSnapshot Delete a snapshot 63 | func (p *BlockStore) DeleteSnapshot(snapshotID string) error { 64 | return p.plugin.DeleteSnapshot(snapshotID) 65 | } 66 | 67 | // GetVolumeID Get the volume ID from the spec 68 | func (p *BlockStore) GetVolumeID(unstructuredPV runtime.Unstructured) (string, error) { 69 | return p.plugin.GetVolumeID(unstructuredPV) 70 | } 71 | 72 | // SetVolumeID Set the volume ID in the spec 73 | func (p *BlockStore) SetVolumeID(unstructuredPV runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { 74 | return p.plugin.SetVolumeID(unstructuredPV, volumeID) 75 | } 76 | -------------------------------------------------------------------------------- /pkg/zfs/utils/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package utils 18 | 19 | import ( 20 | "net" 21 | "strings" 22 | "time" 23 | 24 | "github.com/gofrs/uuid" 25 | "github.com/pkg/errors" 26 | ) 27 | 28 | const ( 29 | // IdentifierKey is a word to generate snapshotID from volume name and backup name 30 | IdentifierKey = "." 31 | // restored PV prefix name 32 | RestorePrefix = "restored-" 33 | ) 34 | 35 | func GetServerAddress() (string, error) { 36 | netInterfaceAddresses, err := net.InterfaceAddrs() 37 | 38 | if err != nil { 39 | return "", err 40 | } 41 | 42 | for _, netInterfaceAddress := range netInterfaceAddresses { 43 | networkIP, ok := netInterfaceAddress.(*net.IPNet) 44 | if ok && !networkIP.IP.IsLoopback() && networkIP.IP.To4() != nil { 45 | ip := networkIP.IP.String() 46 | return ip, nil 47 | } 48 | } 49 | return "", errors.New("error: fetching the interface") 50 | } 51 | 52 | func GenerateResourceName(volumeID, backupName string) string { 53 | return volumeID + IdentifierKey + backupName 54 | } 55 | 56 | func GenerateSnapshotID(volumeID, schdname, backupName string) string { 57 | return volumeID + IdentifierKey + schdname + IdentifierKey + backupName 58 | } 59 | 60 | // GetInfoFromSnapshotID return backup, schdname and volume id from the given snapshotID 61 | func GetInfoFromSnapshotID(snapshotID string) (volumeID, schdname, backupName string, err error) { 62 | s := strings.Split(snapshotID, IdentifierKey) 63 | 64 | if len(s) == 2 { 65 | // backward compatibility, old backups 66 | volumeID = s[0] 67 | backupName = s[1] 68 | // for old backups fetch the schdeule from the bkpname 69 | schdname = GetScheduleName(backupName) 70 | } else if len(s) == 3 { 71 | volumeID = s[0] 72 | schdname = s[1] 73 | backupName = s[2] 74 | } else { 75 | err = errors.New("invalid snapshot id") 76 | return 77 | } 78 | 79 | if volumeID == "" || backupName == "" { 80 | err = errors.Errorf("invalid volumeID=%s backupName=%s", volumeID, backupName) 81 | } 82 | return 83 | } 84 | 85 | // GetRestorePVName return new name for clone pv for the given pv 86 | func GetRestorePVName() (string, error) { 87 | nuuid, err := uuid.NewV4() 88 | if err != nil { 89 | return "", errors.Wrapf(err, "zfs: error generating uuid for PV rename") 90 | } 91 | 92 | return RestorePrefix + nuuid.String(), nil 93 | } 94 | 95 | // GetScheduleName return the schedule name for the given backup 96 | // It will check if backup name have 'bkp-20060102150405' format 97 | func GetScheduleName(backupName string) string { 98 | // for non-scheduled backup, we are considering backup name as schedule name only 99 | schdName := "" 100 | 101 | // If it is scheduled backup then we need to get the schedule name 102 | splitName := strings.Split(backupName, "-") 103 | if len(splitName) >= 2 { 104 | _, err := time.Parse("20060102150405", splitName[len(splitName)-1]) 105 | if err != nil { 106 | // last substring is not timestamp, so it is not generated from schedule 107 | return schdName 108 | } 109 | schdName = strings.Join(splitName[0:len(splitName)-1], "-") 110 | } 111 | return schdName 112 | } 113 | -------------------------------------------------------------------------------- /plugin.Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The OpenEBS Authors. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This Dockerfile builds velero-plugin 16 | 17 | FROM golang:1.14.7 as build 18 | 19 | ARG TARGETOS 20 | ARG TARGETARCH 21 | ARG TARGETVARIANT="" 22 | 23 | ENV GO111MODULE=on \ 24 | GOOS=${TARGETOS} \ 25 | GOARCH=${TARGETARCH} \ 26 | GOARM=${TARGETVARIANT} \ 27 | DEBIAN_FRONTEND=noninteractive \ 28 | PATH="/root/go/bin:${PATH}" 29 | 30 | WORKDIR /go/src/github.com/openebs/velero-plugin/ 31 | 32 | RUN apt-get update && apt-get install -y make git 33 | 34 | COPY go.mod go.sum ./ 35 | # Get dependancies - will also be cached if we won't change mod/sum 36 | RUN go mod download 37 | 38 | COPY . . 39 | 40 | RUN make build 41 | 42 | FROM alpine:3.11.5 43 | 44 | ARG BUILD_DATE 45 | ARG DBUILD_DATE 46 | ARG DBUILD_REPO_URL 47 | ARG DBUILD_SITE_URL 48 | 49 | LABEL org.label-schema.schema-version="1.0" 50 | LABEL org.label-schema.name="velero-plugin" 51 | LABEL org.label-schema.description="OpenEBS velero-plugin" 52 | LABEL org.label-schema.build-date=$DBUILD_DATE 53 | LABEL org.label-schema.vcs-url=$DBUILD_REPO_URL 54 | LABEL org.label-schema.url=$DBUILD_SITE_URL 55 | 56 | RUN mkdir /plugins 57 | COPY --from=build /go/src/github.com/openebs/velero-plugin/_output/velero-* /plugins/ 58 | USER nobody:nobody 59 | 60 | ENTRYPOINT ["/bin/ash", "-c", "cp /plugins/* /target/."] 61 | -------------------------------------------------------------------------------- /script/buildxpush.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019-2020 The OpenEBS Authors. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | if [ -z ${DIMAGE} ]; 20 | then 21 | echo "Error: DIMAGE is not specified"; 22 | exit 1 23 | fi 24 | 25 | function pushBuildx() { 26 | BUILD_TAG="latest" 27 | TARGET_IMG=${DIMAGE} 28 | 29 | # TODO Currently ci builds with commit tag will not be generated, 30 | # since buildx does not support multiple repo 31 | # if not a release build set the tag and ci image 32 | if [ -z "${RELEASE_TAG}" ]; then 33 | return 34 | # BUILD_ID=$(git describe --tags --always) 35 | # BUILD_TAG="${BRANCH}-${BUILD_ID}" 36 | # TARGET_IMG="${DIMAGE}-ci" 37 | fi 38 | 39 | echo "Tagging and pushing ${DIMAGE}:${TAG} as ${TARGET_IMG}:${BUILD_TAG}" 40 | docker buildx imagetools create "${DIMAGE}:${TAG}" -t "${TARGET_IMG}:${BUILD_TAG}" 41 | } 42 | 43 | # if the push is for a buildx build 44 | if [[ ${BUILDX} ]]; then 45 | pushBuildx 46 | exit 0 47 | fi 48 | -------------------------------------------------------------------------------- /script/install-openebs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 The OpenEBS Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if [ -z $VELERO_RELEASE ] || [ -z $OPENEBS_RELEASE ]; then 18 | exit 19 | fi 20 | 21 | echo "Installing iscsi packages" 22 | sudo apt-get install --yes -qq open-iscsi 23 | sudo service iscsid start 24 | sudo systemctl status iscsid --no-pager 25 | echo "Installation complete" 26 | 27 | #TODO add openebs release 28 | kubectl apply -f https://raw.githubusercontent.com/openebs/openebs/master/k8s/openebs-operator.yaml 29 | 30 | function waitForDeployment() { 31 | DEPLOY=$1 32 | NS=$2 33 | CREATE=true 34 | 35 | if [ $# -eq 3 ] && ! $3 ; then 36 | CREATE=false 37 | fi 38 | 39 | for i in $(seq 1 50) ; do 40 | kubectl get deployment -n ${NS} ${DEPLOY} 41 | kstat=$? 42 | if [ $kstat -ne 0 ] && ! $CREATE ; then 43 | return 44 | elif [ $kstat -eq 0 ] && ! $CREATE; then 45 | sleep 3 46 | continue 47 | fi 48 | 49 | replicas=$(kubectl get deployment -n ${NS} ${DEPLOY} -o json | jq ".status.readyReplicas") 50 | if [ "$replicas" == "1" ]; then 51 | break 52 | else 53 | echo "Waiting for ${DEPLOY} to be ready" 54 | if [ ${DEPLOY} != "maya-apiserver" ] && [ ${DEPLOY} != "openebs-provisioner" ]; then 55 | dumpMayaAPIServerLogs 10 56 | fi 57 | sleep 10 58 | fi 59 | done 60 | } 61 | 62 | function dumpMayaAPIServerLogs() { 63 | LC=$1 64 | MAPIPOD=$(kubectl get pods -o jsonpath='{.items[?(@.spec.containers[0].name=="maya-apiserver")].metadata.name}' -n openebs) 65 | kubectl logs --tail=${LC} $MAPIPOD -n openebs 66 | printf "\n\n" 67 | } 68 | 69 | waitForDeployment maya-apiserver openebs 70 | waitForDeployment openebs-provisioner openebs 71 | waitForDeployment openebs-ndm-operator openebs 72 | 73 | kubectl get pods --all-namespaces 74 | -------------------------------------------------------------------------------- /script/install-velero.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 The OpenEBS Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if [ -z $VELERO_RELEASE ] || [ -z $OPENEBS_RELEASE ]; then 18 | exit 19 | fi 20 | 21 | function waitForDeployment() { 22 | DEPLOY=$1 23 | NS=$2 24 | CREATE=true 25 | 26 | if [ $# -eq 3 ] && ! $3 ; then 27 | CREATE=false 28 | fi 29 | 30 | for i in $(seq 1 50) ; do 31 | kubectl get deployment -n ${NS} ${DEPLOY} 32 | kstat=$? 33 | if [ $kstat -ne 0 ] && ! $CREATE ; then 34 | return 35 | elif [ $kstat -eq 0 ] && ! $CREATE; then 36 | sleep 3 37 | continue 38 | fi 39 | 40 | replicas=$(kubectl get deployment -n ${NS} ${DEPLOY} -o json | jq ".status.readyReplicas") 41 | if [ "$replicas" == "1" ]; then 42 | break 43 | else 44 | echo "Waiting for ${DEPLOY} to be ready" 45 | if [ ${DEPLOY} != "maya-apiserver" ] && [ ${DEPLOY} != "openebs-provisioner" ]; then 46 | dumpMayaAPIServerLogs 10 47 | fi 48 | sleep 10 49 | fi 50 | done 51 | } 52 | 53 | MAPI_SVC_ADDR=`kubectl get service -n openebs maya-apiserver-service -o json | grep clusterIP | awk -F\" '{print $4}'` 54 | export MAPI_ADDR="http://${MAPI_SVC_ADDR}:5656" 55 | export KUBERNETES_SERVICE_HOST="127.0.0.1" 56 | export KUBECONFIG=$HOME/.kube/config 57 | 58 | wget -nv -O velero.tar.gz https://github.com/vmware-tanzu/velero/releases/download/${VELERO_RELEASE}/velero-${VELERO_RELEASE}-linux-amd64.tar.gz 59 | mkdir velero 60 | tar xf velero.tar.gz -C velero 61 | velero=$PWD/velero/velero-${VELERO_RELEASE}-linux-amd64/velero 62 | if [ ! -f ${velero} ]; then 63 | echo "${velero} file does not exist" 64 | exit 65 | fi 66 | 67 | if [ ! -f minio ]; then 68 | wget -nv https://dl.min.io/server/minio/release/linux-amd64/minio 69 | fi 70 | 71 | if [ ! -f mc ]; then 72 | wget -nv https://dl.min.io/client/mc/release/linux-amd64/mc 73 | fi 74 | chmod +x minio 75 | chmod +x mc 76 | 77 | BACKUP_DIR=$PWD/data 78 | 79 | export MINIO_ACCESS_KEY=minio 80 | export MINIO_SECRET_KEY=minio123 81 | 82 | ip addr show docker0 >> /dev/null 83 | if [ $? -ne 0 ]; then 84 | exit 1 85 | fi 86 | 87 | MINIO_SERVER_IP=`ip addr show docker0 |grep docker0 |grep inet |awk -F ' ' '{print $2}' |awk -F '/' '{print $1}'` 88 | if [ $? -ne 0 ] || [ -z ${MINIO_SERVER_IP} ]; then 89 | exit 1 90 | fi 91 | 92 | ./minio server --address ${MINIO_SERVER_IP}:9000 ${BACKUP_DIR} & 93 | MINIO_PID=$! 94 | sleep 5 95 | 96 | BUCKET=velero 97 | REGION=minio 98 | ./mc config host add velero http://${MINIO_SERVER_IP}:9000 minio minio123 99 | ./mc mb -p velero/velero 100 | 101 | ${velero} install \ 102 | --provider aws \ 103 | --bucket $BUCKET \ 104 | --secret-file ./script/minio-credentials \ 105 | --backup-location-config region=${REGION},s3ForcePathStyle="true",s3Url=http://${MINIO_SERVER_IP}:9000 \ 106 | --plugins velero/velero-plugin-for-aws-amd64:v1.2.0 \ 107 | --wait 108 | 109 | sed "s/MINIO_ENDPOINT/http:\/\/$MINIO_SERVER_IP\:9000/" script/volumesnapshotlocation.yaml > /tmp/s.yaml 110 | kubectl apply -f /tmp/s.yaml 111 | ${velero} plugin add openebs/velero-plugin-amd64:ci 112 | -------------------------------------------------------------------------------- /script/minio-credentials: -------------------------------------------------------------------------------- 1 | [default] 2 | aws_access_key_id=minio 3 | aws_secret_access_key=minio123 4 | 5 | -------------------------------------------------------------------------------- /script/minio.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | namespace: velero 5 | name: minio 6 | labels: 7 | component: minio 8 | spec: 9 | selector: 10 | matchLabels: 11 | component: minio 12 | strategy: 13 | type: Recreate 14 | template: 15 | metadata: 16 | labels: 17 | component: minio 18 | spec: 19 | volumes: 20 | - name: storage 21 | emptyDir: {} 22 | - name: config 23 | emptyDir: {} 24 | containers: 25 | - name: minio 26 | image: minio/minio:latest 27 | imagePullPolicy: IfNotPresent 28 | args: 29 | - server 30 | - /storage 31 | - --config-dir=/config 32 | env: 33 | - name: MINIO_ACCESS_KEY 34 | value: "minio" 35 | - name: MINIO_SECRET_KEY 36 | value: "minio123" 37 | ports: 38 | - containerPort: 9000 39 | volumeMounts: 40 | - name: storage 41 | mountPath: "/storage" 42 | - name: config 43 | mountPath: "/config" 44 | 45 | --- 46 | apiVersion: v1 47 | kind: Service 48 | metadata: 49 | namespace: velero 50 | name: minio 51 | labels: 52 | component: minio 53 | spec: 54 | # ClusterIP is recommended for production environments. 55 | # Change to NodePort if needed per documentation, 56 | # but only if you run Minio in a test/trial environment, for example with Minikube. 57 | type: ClusterIP 58 | ports: 59 | - port: 9000 60 | targetPort: 9000 61 | protocol: TCP 62 | selector: 63 | component: minio 64 | 65 | --- 66 | apiVersion: v1 67 | kind: Secret 68 | metadata: 69 | namespace: velero 70 | name: cloud-credentials 71 | labels: 72 | component: minio 73 | stringData: 74 | cloud: | 75 | [default] 76 | aws_access_key_id = minio 77 | aws_secret_access_key = minio123 78 | 79 | --- 80 | apiVersion: batch/v1 81 | kind: Job 82 | metadata: 83 | namespace: velero 84 | name: minio-setup 85 | labels: 86 | component: minio 87 | spec: 88 | template: 89 | metadata: 90 | name: minio-setup 91 | spec: 92 | restartPolicy: OnFailure 93 | volumes: 94 | - name: config 95 | emptyDir: {} 96 | containers: 97 | - name: mc 98 | image: minio/mc:latest 99 | imagePullPolicy: IfNotPresent 100 | command: 101 | - /bin/sh 102 | - -c 103 | - "mc --debug --config-dir=/config config host add velero http://minio:9000 minio minio123 && mc --config-dir=/config mb -p velero/velero" 104 | volumeMounts: 105 | - name: config 106 | mountPath: "/config" 107 | -------------------------------------------------------------------------------- /script/volumesnapshotlocation.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 the Heptio Ark contributors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | apiVersion: velero.io/v1 17 | kind: VolumeSnapshotLocation 18 | metadata: 19 | name: default 20 | namespace: velero 21 | spec: 22 | provider: openebs.io/cstor-blockstore 23 | config: 24 | bucket: velero 25 | provider: aws 26 | region: minio 27 | s3Url: MINIO_ENDPOINT 28 | s3ForcePathStyle: "true" 29 | restoreAllIncrementalSnapshots: "true" 30 | autoSetTargetIP: "true" 31 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Executing the integration test cases 2 | ## Table of Contents 3 | - [Prerequisite for integration tests](#prerequisite-for-integration-tests) 4 | - [Configuring integration tests](#configuring-integration-tests) 5 | - [Application](#application) 6 | - [OpenEBS](#openebs) 7 | - [StorageClass](#storageclass) 8 | - [StoragePoolClaim](#storagepoolclaim) 9 | - [PersistentVolumeClaim](#persistentvolumeclaim) 10 | - [Velero](#velero) 11 | - [BackupStorageLocation](#backupstoragelocation) 12 | - [VolumeSnapshotLocation](#volumesnapshotlocation) 13 | - [Executing integration test](#executing-integration-test) 14 | 15 | ## Prerequisite for integration tests 16 | To execute the integration test cases under `velero-plugin/tests`, you need to have a working installation of the following components. 17 | 1. OpenEBS 18 | 2. Velero 19 | 3. openebs/velero-plugin. 20 | 21 | ## Configuring integration tests 22 | ### Application 23 | By default, `velero-plugin/tests/sanity` creates a namespace `test` to deploy the application. 24 | you can configure the application by updating the variable `velero-plugin/tests/app.BusyboxYaml`. 25 | To update the volume configuration, check PersistentVolumeClaim section. 26 | 27 | ### OpenEBS 28 | `velero-plugin/tests/sanity` assumes that OpenEBS is installed in a namespace `openebs`. If you have installed OpenEBS in different a namespace then you need to update the variable `velero-plugin/tests/openebs/OpenEBSNs` accordingly. 29 | 30 | #### StorageClass 31 | `velero-plugin/tests/sanity` creates storageClass `openebs-cstor-sparse-auto` having replicaCount as 1 for cStor Volume. You can configure the storageClassing by updating the variable `velero-plugin/tests/openebs/SCYaml`. 32 | 33 | #### StoragePoolClaim 34 | `velero-plugin/tests/sanity` creates StoragePoolClaim `sparse-claim-auto` for cStor Volume. You can configure the StoragePoolClaim by updating the variable `velero-plugin/tests/openebs/SPCYaml`. 35 | 36 | #### PersistentVolumeClaim 37 | `velero-plugin/tests/sanity` creates PersistentVolumeClaim `cstor-vol1-1r-claim` for cStor Volume. You can configure the PersistentVolumeClaim by updating the variable `velero-plugin/tests/openebs/PVCYaml`. 38 | 39 | ### Velero 40 | `velero-plugin/tests/sanity` assumes that Velero is installed in a namespace `velero`. If you have installed Velero in different a namespace then you need to update the variable `velero-plugin/tests/velero/VeleroNamespace` accordingly. 41 | 42 | #### BackupStorageLocation 43 | The default value of `BackupStorageLocation` in tests is `default`. If you have different `BackupStorageLocation` then you need to update the variable `velero-plugin/tests/sanity/BackupLocation`. 44 | 45 | #### VolumeSnapshotLocation 46 | The default value of `VolumeSnapshotLocation` in tests is `default`. If you have different `VolumeSnapshotLocation` then you need to update the variable `velero-plugin/tests/sanity/SnapshotLocation`. 47 | 48 | 49 | ## Executing integration test 50 | To execute the test under `velero-plugin/tests`, execute the following command: 51 | 52 | `make test` 53 | 54 | or 55 | 56 | `go test -v ./tests/sanity/...` 57 | -------------------------------------------------------------------------------- /tests/app/application_setup.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package app 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/ghodss/yaml" 23 | k8s "github.com/openebs/velero-plugin/tests/k8s" 24 | corev1 "k8s.io/api/core/v1" 25 | k8serrors "k8s.io/apimachinery/pkg/api/errors" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | ) 28 | 29 | var ( 30 | // PVCName PVC name used by this package 31 | PVCName string 32 | ) 33 | 34 | // CreateNamespace create namespace for application 35 | func CreateNamespace(ns string) error { 36 | _, err := k8s.Client.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) 37 | if err != nil { 38 | if k8serrors.IsNotFound(err) { 39 | o := &corev1.Namespace{ 40 | ObjectMeta: metav1.ObjectMeta{ 41 | Name: ns, 42 | }, 43 | } 44 | _, err = k8s.Client.CoreV1().Namespaces().Create(context.TODO(), o, metav1.CreateOptions{}) 45 | } 46 | } 47 | return err 48 | } 49 | 50 | // DestroyNamespace destory the given namespace 51 | func DestroyNamespace(ns string) error { 52 | err := k8s.Client.CoreV1().Namespaces().Delete(context.TODO(), ns, metav1.DeleteOptions{}) 53 | if err != nil { 54 | if k8serrors.IsNotFound(err) { 55 | return nil 56 | } 57 | return k8s.Client.WaitForNamespaceCleanup(ns) 58 | } 59 | return nil 60 | } 61 | 62 | // DeployApplication deploy application 63 | func DeployApplication(appYaml, ns string) error { 64 | var p corev1.Pod 65 | if err := yaml.Unmarshal([]byte(appYaml), &p); err != nil { 66 | return err 67 | } 68 | p.Namespace = ns 69 | _, err := k8s.Client.CoreV1().Pods(ns).Create(context.TODO(), &p, metav1.CreateOptions{}) 70 | if err != nil { 71 | if !k8serrors.IsAlreadyExists(err) { 72 | return err 73 | } 74 | } 75 | 76 | // update the PVC name used by this application 77 | PVCName = p.Spec.Volumes[0].PersistentVolumeClaim.ClaimName 78 | 79 | return k8s.Client.WaitForPod(p.Name, p.Namespace) 80 | } 81 | 82 | // DestroyApplication destroy the given application 83 | func DestroyApplication(appYaml, ns string) error { 84 | var p corev1.Pod 85 | if err := yaml.Unmarshal([]byte(appYaml), &p); err != nil { 86 | return err 87 | } 88 | err := k8s.Client. 89 | CoreV1(). 90 | Pods(ns). 91 | Delete(context.TODO(), p.Name, metav1.DeleteOptions{}) 92 | 93 | if !k8serrors.IsNotFound(err) { 94 | return err 95 | } 96 | 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /tests/app/application_yaml.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package app 18 | 19 | const ( 20 | // BusyboxYaml for busybox application 21 | BusyboxYaml = `apiVersion: v1 22 | kind: Pod 23 | metadata: 24 | name: busybox-cstor 25 | namespace: default 26 | spec: 27 | containers: 28 | - command: 29 | - sh 30 | - -c 31 | - 'date > /mnt/store1/date.txt; hostname >> /mnt/store1/hostname.txt; sync; sleep 5; sync; tail -f /dev/null;' 32 | image: busybox 33 | imagePullPolicy: Always 34 | name: busybox 35 | volumeMounts: 36 | - mountPath: /mnt/store1 37 | name: demo-vol1 38 | volumes: 39 | - name: demo-vol1 40 | persistentVolumeClaim: 41 | claimName: cstor-vol1-1r-claim 42 | ` 43 | ) 44 | -------------------------------------------------------------------------------- /tests/config/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package sanity 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | "k8s.io/client-go/rest" 24 | "k8s.io/client-go/tools/clientcmd" 25 | ) 26 | 27 | // GetClusterConfig return the config for k8s. 28 | func GetClusterConfig() (*rest.Config, error) { 29 | kubeConfigPath, err := getConfigPath() 30 | if err != nil { 31 | return nil, err 32 | } 33 | config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return config, err 38 | } 39 | 40 | // GetHomeDir gets the home directory for the system. 41 | // It is required to locate the .kube/config file 42 | func getHomeDir() (string, error) { 43 | if h := os.Getenv("HOME"); h != "" { 44 | return h, nil 45 | } 46 | 47 | return "", fmt.Errorf("not able to locate home directory") 48 | } 49 | 50 | // GetConfigPath returns the filepath of kubeconfig file 51 | func getConfigPath() (string, error) { 52 | home, err := getHomeDir() 53 | if err != nil { 54 | return "", err 55 | } 56 | kubeConfigPath := home + "/.kube/config" 57 | return kubeConfigPath, nil 58 | } 59 | -------------------------------------------------------------------------------- /tests/k8s/exec.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package k8s 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "strings" 23 | 24 | "github.com/pkg/errors" 25 | corev1 "k8s.io/api/core/v1" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | "k8s.io/client-go/tools/remotecommand" 28 | ) 29 | 30 | // Exec execute the given command in given ns/pod/container and return the output 31 | func (k *KubeClient) Exec(command, pod, container, ns string) (string, string, error) { 32 | var stderr, stdout bytes.Buffer 33 | 34 | req := k.CoreV1(). 35 | RESTClient(). 36 | Post(). 37 | Resource("pods"). 38 | Name(pod). 39 | Namespace(ns). 40 | SubResource("exec") 41 | scheme := runtime.NewScheme() 42 | if err := corev1.AddToScheme(scheme); err != nil { 43 | return "", "", errors.Errorf("error adding to scheme: %v", err) 44 | } 45 | 46 | paramCodec := runtime.NewParameterCodec(scheme) 47 | req.VersionedParams(&corev1.PodExecOptions{ 48 | Command: strings.Fields(command), 49 | Container: container, 50 | Stdout: true, 51 | Stderr: true, 52 | }, paramCodec) 53 | 54 | exec, err := remotecommand.NewSPDYExecutor(cfg, "POST", req.URL()) 55 | if err != nil { 56 | return "", "", fmt.Errorf("error while creating Executor: %v", err) 57 | } 58 | 59 | err = exec.Stream(remotecommand.StreamOptions{ 60 | Stdout: &stdout, 61 | Stderr: &stderr, 62 | }) 63 | if err != nil { 64 | return "", "", errors.Errorf("error in Stream: %v", err) 65 | } 66 | 67 | return stdout.String(), stderr.String(), nil 68 | } 69 | -------------------------------------------------------------------------------- /tests/k8s/log.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package k8s 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "io" 23 | "os" 24 | ) 25 | 26 | // DumpLogs will dump log for given ns/pod 27 | func (k *KubeClient) DumpLogs(ns, podName, container string) error { 28 | fmt.Printf("################################################\n") 29 | fmt.Printf("Logs of %s/%s:%s\n", ns, podName, container) 30 | fmt.Printf("################################################\n") 31 | req := k.CoreV1().RESTClient().Get(). 32 | Namespace(ns). 33 | Name(podName). 34 | Resource("pods"). 35 | SubResource("log"). 36 | Param("container", container). 37 | Param("timestamps", "true") 38 | 39 | readCloser, err := req.Stream(context.TODO()) 40 | if err != nil { 41 | fmt.Printf("DumpLogs: Error occurred for %s/%s:%s.. %s", ns, podName, container, err) 42 | return err 43 | } 44 | 45 | defer func() { 46 | _ = readCloser.Close() 47 | }() 48 | 49 | _, err = io.Copy(os.Stdout, readCloser) 50 | fmt.Println(err) 51 | return err 52 | } 53 | -------------------------------------------------------------------------------- /tests/k8s/status.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package k8s 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "time" 23 | 24 | "github.com/pkg/errors" 25 | corev1 "k8s.io/api/core/v1" 26 | k8serrors "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | ) 29 | 30 | // GetPVCPhase return given PVC's phase 31 | func (k *KubeClient) GetPVCPhase(pvc, ns string) (corev1.PersistentVolumeClaimPhase, error) { 32 | o, err := k.CoreV1(). 33 | PersistentVolumeClaims(ns). 34 | Get(context.TODO(), pvc, metav1.GetOptions{}) 35 | if err != nil { 36 | return "", err 37 | } 38 | 39 | return o.Status.Phase, nil 40 | } 41 | 42 | func (k *KubeClient) waitForPVCBound(pvc, ns string) (corev1.PersistentVolumeClaimPhase, error) { 43 | for { 44 | phase, err := k.GetPVCPhase(pvc, ns) 45 | if err != nil || phase == corev1.ClaimLost { 46 | return phase, errors.Errorf("PVC:%s/%s is in Lost state", ns, pvc) 47 | } 48 | if phase == corev1.ClaimBound { 49 | return phase, nil 50 | } 51 | time.Sleep(5 * time.Second) 52 | } 53 | } 54 | 55 | func (k *KubeClient) WaitForPVCCleanup(pvc, ns string) error { 56 | for { 57 | _, err := k.GetPVCPhase(pvc, ns) 58 | 59 | if k8serrors.IsNotFound(err) { 60 | return nil 61 | } 62 | 63 | time.Sleep(5 * time.Second) 64 | } 65 | } 66 | 67 | // WaitForDeployment wait for deployment having given labelSelector and namespace to be ready 68 | func (k *KubeClient) WaitForDeployment(labelSelector, ns string) error { 69 | var ready bool 70 | dumpLog := 0 71 | for { 72 | deploymentList, err := k.AppsV1(). 73 | Deployments(ns). 74 | List(context.TODO(), metav1.ListOptions{ 75 | LabelSelector: labelSelector, 76 | }) 77 | if err != nil { 78 | return err 79 | } else if len(deploymentList.Items) == 0 { 80 | fmt.Printf("Deployment for %s/%s is not availabel..\n", ns, labelSelector) 81 | time.Sleep(2 * time.Second) 82 | continue 83 | } 84 | 85 | for _, d := range deploymentList.Items { 86 | o, err := k.AppsV1(). 87 | Deployments(d.Namespace). 88 | Get(context.TODO(), d.Name, metav1.GetOptions{}) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | if *o.Spec.Replicas == o.Status.UpdatedReplicas { 94 | ready = true 95 | } else { 96 | ready = false 97 | break 98 | } 99 | } 100 | 101 | if ready { 102 | return nil 103 | } 104 | if dumpLog > 6 { 105 | fmt.Printf("Waiting for deployment for %s/%s to be ready..\n", ns, labelSelector) 106 | dumpLog = 0 107 | } 108 | dumpLog++ 109 | time.Sleep(5 * time.Second) 110 | } 111 | } 112 | 113 | // WaitForPod wait for given pod to become ready 114 | func (k *KubeClient) WaitForPod(podName, podNamespace string) error { 115 | dumpLog := 0 116 | for { 117 | o, err := k.CoreV1().Pods(podNamespace).Get(context.TODO(), podName, metav1.GetOptions{}) 118 | if err != nil { 119 | return err 120 | } 121 | if o.Status.Phase == corev1.PodRunning { 122 | return nil 123 | } 124 | time.Sleep(5 * time.Second) 125 | if dumpLog > 6 { 126 | fmt.Printf("checking for pod %s/%s\n", podNamespace, podName) 127 | dumpLog = 0 128 | } 129 | dumpLog++ 130 | } 131 | } 132 | 133 | // WaitForDeploymentCleanup wait for cleanup of deployment having given labelSelector and namespace 134 | func (k *KubeClient) WaitForDeploymentCleanup(labelSelector, ns string) error { 135 | dumpLog := 0 136 | for { 137 | deploymentList, err := k.AppsV1(). 138 | Deployments(ns). 139 | List(context.TODO(), metav1.ListOptions{ 140 | LabelSelector: labelSelector, 141 | }) 142 | 143 | if k8serrors.IsNotFound(err) { 144 | return nil 145 | } 146 | 147 | if err != nil { 148 | return err 149 | } else if len(deploymentList.Items) == 0 { 150 | return nil 151 | } 152 | if dumpLog > 6 { 153 | fmt.Printf("Waiting for cleanup of deployment %s/%s..\n", ns, labelSelector) 154 | dumpLog = 0 155 | } 156 | dumpLog++ 157 | time.Sleep(5 * time.Second) 158 | } 159 | } 160 | 161 | // WaitForNamespaceCleanup wait for cleanup of the given namespace 162 | func (k *KubeClient) WaitForNamespaceCleanup(ns string) error { 163 | dumpLog := 0 164 | for { 165 | _, err := k.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) 166 | 167 | if k8serrors.IsNotFound(err) { 168 | return nil 169 | } 170 | 171 | if err != nil { 172 | return err 173 | } 174 | 175 | if dumpLog > 6 { 176 | fmt.Printf("Waiting for cleanup of namespace %s\n", ns) 177 | dumpLog = 0 178 | } 179 | 180 | dumpLog++ 181 | time.Sleep(5 * time.Second) 182 | } 183 | } 184 | 185 | // GetPodList return list of pod for given label and namespace 186 | func (k *KubeClient) GetPodList(ns, label string) (*corev1.PodList, error) { 187 | return k.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{ 188 | LabelSelector: label, 189 | }) 190 | } 191 | -------------------------------------------------------------------------------- /tests/k8s/storage_install.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package k8s 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/ghodss/yaml" 23 | config "github.com/openebs/velero-plugin/tests/config" 24 | corev1 "k8s.io/api/core/v1" 25 | storagev1 "k8s.io/api/storage/v1" 26 | k8serrors "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/client-go/kubernetes" 29 | "k8s.io/client-go/rest" 30 | 31 | // for GCP 32 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 33 | ) 34 | 35 | // KubeClient interface for k8s API 36 | type KubeClient struct { 37 | kubernetes.Interface 38 | } 39 | 40 | // Client for KubeClient 41 | var Client *KubeClient 42 | 43 | var ( 44 | cfg *rest.Config 45 | ) 46 | 47 | func init() { 48 | var err error 49 | cfg, err = config.GetClusterConfig() 50 | if err != nil { 51 | panic(err) 52 | } 53 | client, err := kubernetes.NewForConfig(cfg) 54 | if err != nil { 55 | panic(err) 56 | } 57 | Client = &KubeClient{client} 58 | } 59 | 60 | // CreatePVC creates the PVC from given yaml 61 | func (k *KubeClient) CreatePVC(pvc corev1.PersistentVolumeClaim) error { 62 | _, err := k.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{}) 63 | if err != nil { 64 | if !k8serrors.IsAlreadyExists(err) { 65 | return err 66 | } 67 | } 68 | 69 | _, err = k.waitForPVCBound(pvc.Name, pvc.Namespace) 70 | return err 71 | } 72 | 73 | // DeletePVC creates the PVC from given yaml 74 | func (k *KubeClient) DeletePVC(pvc corev1.PersistentVolumeClaim) error { 75 | err := k.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}) 76 | if err != nil { 77 | if k8serrors.IsNotFound(err) { 78 | err = nil 79 | } 80 | } 81 | 82 | return err 83 | } 84 | 85 | // CreateStorageClass creates storageClass from given yaml 86 | func (k *KubeClient) CreateStorageClass(scYAML string) error { 87 | var sc storagev1.StorageClass 88 | if err := yaml.Unmarshal([]byte(scYAML), &sc); err != nil { 89 | return err 90 | } 91 | 92 | _, err := k.StorageV1().StorageClasses().Create(context.TODO(), &sc, metav1.CreateOptions{}) 93 | if err != nil { 94 | if !k8serrors.IsAlreadyExists(err) { 95 | return err 96 | } 97 | return nil 98 | } 99 | 100 | return nil 101 | } 102 | -------------------------------------------------------------------------------- /tests/openebs/logs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package openebs 18 | 19 | import ( 20 | "fmt" 21 | 22 | k8s "github.com/openebs/velero-plugin/tests/k8s" 23 | corev1 "k8s.io/api/core/v1" 24 | ) 25 | 26 | const ( 27 | mayaAPIPodLabel = "openebs.io/component-name=maya-apiserver" 28 | cstorPodLabel = "app=cstor-pool" 29 | pvcPodLabel = "openebs.io/target=cstor-target" 30 | ) 31 | 32 | // DumpLogs will dump openebs logs 33 | func (c *ClientSet) DumpLogs() { 34 | mayaPod := c.getMayaAPIServerPodName() 35 | spcPod := c.getSPCPodName() 36 | pvcPod := c.getPVCPodName() 37 | 38 | for _, v := range mayaPod { 39 | if err := k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]); err != nil { 40 | fmt.Printf("Failed to dump maya-apiserver logs err=%s\n", err) 41 | } 42 | } 43 | for _, v := range spcPod { 44 | if err := k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]); err != nil { 45 | fmt.Printf("Failed to dump cstor pod logs err=%s\n", err) 46 | } 47 | } 48 | for _, v := range pvcPod { 49 | if err := k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]); err != nil { 50 | fmt.Printf("Failed to dump target pod logs err=%s\n", err) 51 | } 52 | } 53 | } 54 | 55 | // getMayaAPIServerPodName return Maya-API server pod name and container 56 | // {{"pod1","container1"},{"pod2","container2"},} 57 | func (c *ClientSet) getMayaAPIServerPodName() [][]string { 58 | podList, err := k8s.Client.GetPodList(OpenEBSNs, 59 | mayaAPIPodLabel, 60 | ) 61 | if err != nil { 62 | return [][]string{} 63 | } 64 | return getPodContainerList(podList) 65 | } 66 | 67 | // getSPCPodName return SPC pod name and container 68 | // {{"pod1","container1"},{"pod2","container2"},} 69 | func (c *ClientSet) getSPCPodName() [][]string { 70 | podList, err := k8s.Client.GetPodList(OpenEBSNs, 71 | cstorPodLabel, 72 | ) 73 | if err != nil { 74 | return [][]string{} 75 | } 76 | return getPodContainerList(podList) 77 | } 78 | 79 | // getPVCPodName return PVC pod name and container 80 | // {{"pod1","container1"},{"pod2","container2"},} 81 | func (c *ClientSet) getPVCPodName() [][]string { 82 | podList, err := k8s.Client.GetPodList(OpenEBSNs, 83 | pvcPodLabel, 84 | ) 85 | if err != nil { 86 | return [][]string{} 87 | } 88 | return getPodContainerList(podList) 89 | } 90 | 91 | // returns {{"pod1","container1"},{"pod2","container2"},} 92 | func getPodContainerList(podList *corev1.PodList) [][]string { 93 | pod := make([][]string, 0) 94 | 95 | for _, p := range podList.Items { 96 | for _, c := range p.Spec.Containers { 97 | pod = append(pod, []string{p.Name, c.Name}) 98 | } 99 | } 100 | return pod 101 | } 102 | -------------------------------------------------------------------------------- /tests/openebs/storage_install.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package openebs 18 | 19 | import ( 20 | "context" 21 | "time" 22 | 23 | "github.com/ghodss/yaml" 24 | v1alpha1 "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" 25 | clientset "github.com/openebs/maya/pkg/client/generated/clientset/versioned" 26 | config "github.com/openebs/velero-plugin/tests/config" 27 | k8s "github.com/openebs/velero-plugin/tests/k8s" 28 | corev1 "k8s.io/api/core/v1" 29 | k8serrors "k8s.io/apimachinery/pkg/api/errors" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | ) 32 | 33 | // ClientSet interface for OpenEBS API 34 | type ClientSet struct { 35 | clientset.Interface 36 | } 37 | 38 | var ( 39 | // Client for openebs ClientSet 40 | Client *ClientSet 41 | 42 | // SPCName for SPC 43 | SPCName string 44 | 45 | // PVCName for PVC 46 | PVCName string 47 | 48 | // AppPVC created by openebs 49 | AppPVC *corev1.PersistentVolumeClaim 50 | ) 51 | 52 | const ( 53 | // OpenEBSNs openebs Namespace 54 | OpenEBSNs = "openebs" 55 | 56 | // PVDeploymentLabel for target pod Deployment 57 | PVDeploymentLabel = "openebs.io/persistent-volume" 58 | ) 59 | 60 | func init() { 61 | cfg, err := config.GetClusterConfig() 62 | if err != nil { 63 | panic(err) 64 | } 65 | client, err := clientset.NewForConfig(cfg) 66 | if err != nil { 67 | panic(err) 68 | } 69 | Client = &ClientSet{client} 70 | } 71 | 72 | // CreateSPC create SPC for given YAML 73 | func (c *ClientSet) CreateSPC(spcYAML string) error { 74 | var spc v1alpha1.StoragePoolClaim 75 | 76 | if err := yaml.Unmarshal([]byte(spcYAML), &spc); err != nil { 77 | return err 78 | } 79 | 80 | _, err := c.OpenebsV1alpha1().StoragePoolClaims().Create(context.TODO(), &spc, metav1.CreateOptions{}) 81 | if err != nil { 82 | if !k8serrors.IsAlreadyExists(err) { 83 | return err 84 | } 85 | SPCName = spc.Name 86 | return nil 87 | } 88 | 89 | SPCName = spc.Name 90 | 91 | return k8s.Client.WaitForDeployment( 92 | string(v1alpha1.StoragePoolClaimCPK)+"="+spc.Name, 93 | OpenEBSNs) 94 | } 95 | 96 | // CreateVolume create volume from given PVC yaml 97 | func (c *ClientSet) CreateVolume(pvcYAML, pvcNs string, wait bool) error { 98 | var pvc corev1.PersistentVolumeClaim 99 | var err error 100 | 101 | if err = yaml.Unmarshal([]byte(pvcYAML), &pvc); err != nil { 102 | return err 103 | } 104 | pvc.Namespace = pvcNs 105 | 106 | if err = k8s.Client.CreatePVC(pvc); err != nil { 107 | return err 108 | } 109 | 110 | time.Sleep(5 * time.Second) 111 | PVCName = pvc.Name 112 | if wait { 113 | err = c.WaitForHealthyCVR(&pvc) 114 | } 115 | if err == nil { 116 | AppPVC = &pvc 117 | } 118 | return err 119 | } 120 | 121 | // DeleteVolume delete volume for given PVC YAML 122 | func (c *ClientSet) DeleteVolume(pvcYAML, pvcNs string) error { 123 | var pvc corev1.PersistentVolumeClaim 124 | if err := yaml.Unmarshal([]byte(pvcYAML), &pvc); err != nil { 125 | return err 126 | } 127 | 128 | pv, err := c.getPVCVolumeName(pvc.Name, pvcNs) 129 | if err != nil { 130 | return err 131 | } 132 | 133 | pvc.Namespace = pvcNs 134 | if err := k8s.Client.DeletePVC(pvc); err != nil { 135 | return err 136 | } 137 | 138 | err = k8s.Client.WaitForDeploymentCleanup( 139 | PVDeploymentLabel+"="+pv, 140 | OpenEBSNs) 141 | 142 | if err != nil { 143 | return err 144 | } 145 | 146 | return k8s.Client.WaitForPVCCleanup(pvc.Name, pvc.Namespace) 147 | } 148 | -------------------------------------------------------------------------------- /tests/openebs/storage_status.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package openebs 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strings" 23 | "time" 24 | 25 | v1alpha1 "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" 26 | k8s "github.com/openebs/velero-plugin/tests/k8s" 27 | "github.com/pkg/errors" 28 | v1 "k8s.io/api/core/v1" 29 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 | ) 31 | 32 | const ( 33 | cVRPVLabel = "openebs.io/persistent-volume" 34 | cVRLabel = "cstorpool.openebs.io/uid" 35 | cstorID = "OPENEBS_IO_CSTOR_ID" 36 | cstorPoolContainer = "cstor-pool" 37 | // CVRMaxRetry count to check CVR updated state 38 | CVRMaxRetry = 5 39 | ) 40 | 41 | // WaitForHealthyCVR wait till CVR for given PVC becomes healthy 42 | func (c *ClientSet) WaitForHealthyCVR(pvc *v1.PersistentVolumeClaim) error { 43 | dumpLog := 0 44 | for { 45 | if healthy := c.CheckCVRStatus(pvc.Name, 46 | pvc.Namespace, 47 | v1alpha1.CVRStatusOnline); healthy { 48 | break 49 | } 50 | time.Sleep(5 * time.Second) 51 | if dumpLog > 6 { 52 | fmt.Printf("Waiting for %s/%s's CVR\n", pvc.Namespace, pvc.Name) 53 | dumpLog = 0 54 | } 55 | dumpLog++ 56 | } 57 | return nil 58 | } 59 | 60 | // CheckCVRStatus check CVR status for given PVC 61 | func (c *ClientSet) CheckCVRStatus(pvc, ns string, status v1alpha1.CStorVolumeReplicaPhase) bool { 62 | var match bool 63 | 64 | for i := 0; i < CVRMaxRetry; i++ { 65 | cvrlist, err := c.getPVCCVRList(pvc, ns) 66 | if err != nil { 67 | return match 68 | } 69 | 70 | match = true 71 | if len(cvrlist.Items) == 0 { 72 | match = false 73 | } 74 | 75 | for _, v := range cvrlist.Items { 76 | if v.Status.Phase != status { 77 | match = false 78 | } 79 | } 80 | 81 | if match { 82 | break 83 | } 84 | time.Sleep(5 * time.Second) 85 | } 86 | 87 | return match 88 | } 89 | 90 | func (c *ClientSet) getPVCCVRList(pvc, ns string) (*v1alpha1.CStorVolumeReplicaList, error) { 91 | vol, err := c.getPVCVolumeName(pvc, ns) 92 | if err != nil { 93 | return nil, err 94 | } 95 | 96 | return c.OpenebsV1alpha1(). 97 | CStorVolumeReplicas(OpenEBSNs). 98 | List(context.TODO(), metav1.ListOptions{ 99 | LabelSelector: cVRPVLabel + "=" + vol, 100 | }) 101 | } 102 | 103 | func (c *ClientSet) getPVCVolumeName(pvc, ns string) (string, error) { 104 | o, err := k8s.Client. 105 | CoreV1(). 106 | PersistentVolumeClaims(ns). 107 | Get(context.TODO(), pvc, metav1.GetOptions{}) 108 | if err != nil { 109 | return "", err 110 | } 111 | if o.Spec.VolumeName == "" { 112 | return "", errors.Errorf("Volume name is empty") 113 | } 114 | return o.Spec.VolumeName, nil 115 | } 116 | 117 | // CheckSnapshot check if given snapshot is created or not 118 | func (c *ClientSet) CheckSnapshot(pvc, pvcNs, snapshot string) (bool, error) { 119 | var availabel bool 120 | 121 | podList, err := k8s.Client.CoreV1().Pods(OpenEBSNs).List(context.TODO(), metav1.ListOptions{ 122 | LabelSelector: string(v1alpha1.StoragePoolClaimCPK) + "=" + SPCName, 123 | }) 124 | if err != nil { 125 | return availabel, err 126 | } 127 | 128 | if len(podList.Items) == 0 { 129 | return availabel, errors.Errorf("No cStor Pod for %s/%s", OpenEBSNs, SPCName) 130 | } 131 | cvrlist, err := c.getPVCCVRList(PVCName, pvcNs) 132 | if err != nil { 133 | return availabel, err 134 | } 135 | 136 | for _, k := range cvrlist.Items { 137 | for _, p := range podList.Items { 138 | v := getEnvValueFromName(p.Spec.Containers[0].Env, cstorID) 139 | if v == k.Labels[cVRLabel] { 140 | cmd := "zfs list -t all " + 141 | getPoolNameFromCVR(k) + 142 | "/" + 143 | getVolumeNameFromCVR(k) + 144 | "@" + 145 | snapshot 146 | _, e, err := k8s.Client.Exec(cmd, p.Name, cstorPoolContainer, p.Namespace) 147 | if err != nil || e != "" { 148 | return false, errors.Errorf("Error occurred for %v/%v@%v.. stderr:%v err:%v", 149 | getPoolNameFromCVR(k), getVolumeNameFromCVR(k), snapshot, e, err) 150 | } 151 | availabel = true 152 | continue 153 | } 154 | } 155 | } 156 | return availabel, nil 157 | } 158 | 159 | func getEnvValueFromName(env []v1.EnvVar, name string) string { 160 | for _, l := range env { 161 | if l.Name == name { 162 | return l.Value 163 | } 164 | } 165 | return "" 166 | } 167 | 168 | func getPoolNameFromCVR(k v1alpha1.CStorVolumeReplica) string { 169 | return "cstor-" + k.Labels[cVRLabel] 170 | } 171 | 172 | func getVolumeNameFromCVR(k v1alpha1.CStorVolumeReplica) string { 173 | return k.Labels[cVRPVLabel] 174 | } 175 | 176 | // GetCStorBackups returns cstorbackup list for the given backup 177 | func (c *ClientSet) GetCStorBackups(backup, ns string) (*v1alpha1.CStorBackupList, error) { 178 | return c.OpenebsV1alpha1(). 179 | CStorBackups(ns). 180 | List(context.TODO(), metav1.ListOptions{ 181 | LabelSelector: "openebs.io/backup=" + backup, 182 | }) 183 | } 184 | 185 | // GetCStorCompletedBackups returns cstorcompletedbackup list for the given backup 186 | func (c *ClientSet) GetCStorCompletedBackups(backup, ns string) (*v1alpha1.CStorCompletedBackupList, error) { 187 | return c.OpenebsV1alpha1(). 188 | CStorCompletedBackups(ns). 189 | List(context.TODO(), metav1.ListOptions{ 190 | LabelSelector: "openebs.io/backup=" + backup, 191 | }) 192 | } 193 | 194 | // IsBackupResourcesExist checks if backupResources, for the given backup, exist or not 195 | func (c *ClientSet) IsBackupResourcesExist(backup, pvc, ns string) (bool, error) { 196 | isSchedule := false 197 | isLastBackup := false 198 | 199 | scheduleName := backup 200 | splitName := strings.Split(backup, "-") 201 | if len(splitName) >= 2 { 202 | isSchedule = true 203 | scheduleName = strings.Join(splitName[0:len(splitName)-1], "-") 204 | } 205 | 206 | blist, err := c.GetCStorBackups(scheduleName, ns) 207 | if err != nil { 208 | return false, errors.Wrapf(err, "failed to fetch cstorbackup list for backup %s/%s", ns, backup) 209 | } 210 | 211 | cblist, err := c.GetCStorCompletedBackups(scheduleName, ns) 212 | if err != nil { 213 | return false, errors.Wrapf(err, "failed to fetch cstorcompletedbackup list for backup %s/%s", ns, backup) 214 | } 215 | 216 | if isSchedule && len(cblist.Items) == 0 { 217 | return true, errors.Errorf("for schedule cstorcompletedbackups should be present") 218 | } 219 | 220 | // for schedule cstorcompletedbackups is not deleted by apiserver to support incremental backup 221 | if isSchedule && len(cblist.Items) == 1 { 222 | cbkp := cblist.Items[0] 223 | // if given backup is the last backup then relevant cstorbackup will not be deleted 224 | for i, bkp := range blist.Items { 225 | if bkp.Spec.SnapName == cbkp.Spec.PrevSnapName { 226 | blist.Items = append(blist.Items[:i], blist.Items[i+1:]...) 227 | } 228 | if bkp.Spec.SnapName == backup { 229 | isLastBackup = true 230 | } 231 | } 232 | cblist.Items = cblist.Items[:0] 233 | } 234 | 235 | snapshotExist, err := c.CheckSnapshot(pvc, ns, backup) 236 | if err != nil { 237 | if !strings.Contains(err.Error(), "command terminated with exit code 1") { 238 | return false, errors.Wrapf(err, "failed to verify snapshot for backup %s/%s", ns, backup) 239 | } 240 | } 241 | 242 | if len(blist.Items) != 0 || len(cblist.Items) != 0 || (snapshotExist && !isLastBackup) { 243 | return true, errors.Errorf( 244 | "backup %s/%s backup:%d cbackup:%d snapshot:%v isLastBackup:%v", 245 | ns, backup, len(blist.Items), len(cblist.Items), snapshotExist, isLastBackup) 246 | } 247 | 248 | return false, nil 249 | } 250 | -------------------------------------------------------------------------------- /tests/openebs/storage_yaml.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package openebs 18 | 19 | const ( 20 | // SPCYaml yaml for SPC CR 21 | SPCYaml = `apiVersion: openebs.io/v1alpha1 22 | kind: StoragePoolClaim 23 | metadata: 24 | name: sparse-claim-auto 25 | spec: 26 | name: sparse-claim-auto 27 | type: sparse 28 | maxPools: 1 29 | minPools: 1 30 | poolSpec: 31 | poolType: striped 32 | cacheFile: /var/openebs/sparse/sparse-claim-auto.cache 33 | overProvisioning: false 34 | ` 35 | // SCYaml for SC CR 36 | SCYaml = `apiVersion: storage.k8s.io/v1 37 | kind: StorageClass 38 | metadata: 39 | name: openebs-cstor-sparse-auto 40 | annotations: 41 | openebs.io/cas-type: cstor 42 | cas.openebs.io/config: | 43 | - name: StoragePoolClaim 44 | value: "sparse-claim-auto" 45 | - name: ReplicaCount 46 | value: "1" 47 | provisioner: openebs.io/provisioner-iscsi 48 | ` 49 | // PVCYaml for PVC CR 50 | PVCYaml = `kind: PersistentVolumeClaim 51 | apiVersion: v1 52 | metadata: 53 | name: cstor-vol1-1r-claim 54 | spec: 55 | storageClassName: openebs-cstor-sparse-auto 56 | accessModes: 57 | - ReadWriteOnce 58 | resources: 59 | requests: 60 | storage: 4G 61 | ` 62 | ) 63 | -------------------------------------------------------------------------------- /tests/velero/backup.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package velero 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "math/rand" 23 | "time" 24 | 25 | config "github.com/openebs/velero-plugin/tests/config" 26 | "github.com/pkg/errors" 27 | v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" 28 | clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" 29 | k8serrors "k8s.io/apimachinery/pkg/api/errors" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | ) 32 | 33 | const ( 34 | charset = "abcdefghijklmnopqrstuvwxyz" 35 | 36 | // VeleroNamespace is velero namespace 37 | VeleroNamespace = "velero" 38 | ) 39 | 40 | // ClientSet interface for Velero API 41 | type ClientSet struct { 42 | clientset.Interface 43 | } 44 | 45 | var ( 46 | // Client client for Velero API interface 47 | Client *ClientSet 48 | 49 | // BackupLocation backup location 50 | BackupLocation string 51 | 52 | // SnapshotLocation snapshot location 53 | SnapshotLocation string 54 | ) 55 | 56 | // #nosec 57 | var seededRand *rand.Rand = rand.New(rand.NewSource(time.Now().UnixNano())) 58 | 59 | func init() { 60 | cfg, err := config.GetClusterConfig() 61 | if err != nil { 62 | panic(err) 63 | } 64 | client, err := clientset.NewForConfig(cfg) 65 | if err != nil { 66 | panic(err) 67 | } 68 | Client = &ClientSet{client} 69 | } 70 | 71 | func generateRandomString(length int) string { 72 | b := make([]byte, length) 73 | for i := range b { 74 | b[i] = charset[seededRand.Intn(len(charset))] 75 | } 76 | return string(b) 77 | } 78 | 79 | func (c *ClientSet) generateBackupName() (string, error) { 80 | for i := 0; i < 10; i++ { 81 | b := generateRandomString(8) 82 | if b == "" { 83 | continue 84 | } 85 | _, err := c.VeleroV1(). 86 | Backups(VeleroNamespace). 87 | Get(context.TODO(), b, metav1.GetOptions{}) 88 | if err != nil && k8serrors.IsNotFound(err) { 89 | return b, nil 90 | } 91 | } 92 | return "", errors.New("failed to generate unique backup name") 93 | } 94 | 95 | // CreateBackup creates the backup for given namespace 96 | func (c *ClientSet) CreateBackup(ns string) (string, v1.BackupPhase, error) { 97 | var status v1.BackupPhase 98 | snapVolume := true 99 | 100 | bname, err := c.generateBackupName() 101 | if err != nil { 102 | return "", status, err 103 | } 104 | bkp := &v1.Backup{ 105 | ObjectMeta: metav1.ObjectMeta{ 106 | Name: bname, 107 | Namespace: VeleroNamespace, 108 | }, 109 | Spec: v1.BackupSpec{ 110 | IncludedNamespaces: []string{ns}, 111 | SnapshotVolumes: &snapVolume, 112 | StorageLocation: BackupLocation, 113 | VolumeSnapshotLocations: []string{SnapshotLocation}, 114 | }, 115 | } 116 | o, err := c.VeleroV1(). 117 | Backups(VeleroNamespace). 118 | Create(context.TODO(), bkp, metav1.CreateOptions{}) 119 | if err != nil { 120 | return "", status, err 121 | } 122 | 123 | if status, err = c.waitForBackupCompletion(o.Name); err == nil { 124 | return o.Name, status, nil 125 | } 126 | 127 | return bname, status, err 128 | } 129 | 130 | func (c *ClientSet) waitForBackupCompletion(name string) (v1.BackupPhase, error) { 131 | dumpLog := 0 132 | for { 133 | bkp, err := c.getBackup(name) 134 | if err != nil { 135 | return "", err 136 | } 137 | if isBackupDone(bkp) { 138 | return bkp.Status.Phase, nil 139 | } 140 | if dumpLog > 6 { 141 | fmt.Printf("Waiting for backup %s completion..\n", name) 142 | dumpLog = 0 143 | } 144 | dumpLog++ 145 | time.Sleep(5 * time.Second) 146 | } 147 | } 148 | 149 | func (c *ClientSet) getBackup(name string) (*v1.Backup, error) { 150 | return c.VeleroV1(). 151 | Backups(VeleroNamespace). 152 | Get(context.TODO(), name, metav1.GetOptions{}) 153 | } 154 | -------------------------------------------------------------------------------- /tests/velero/logs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package velero 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "time" 23 | 24 | "github.com/openebs/velero-plugin/tests/k8s" 25 | v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" 26 | log "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" 27 | corev1 "k8s.io/api/core/v1" 28 | ) 29 | 30 | // DumpBackupLogs dump logs of given backup on stdout 31 | func (c *ClientSet) DumpBackupLogs(backupName string) error { 32 | return log.Stream(c.VeleroV1(), 33 | VeleroNamespace, 34 | backupName, 35 | v1.DownloadTargetKindBackupLog, 36 | os.Stdout, 37 | time.Minute, false, "") 38 | } 39 | 40 | // DumpLogs dump logs of velero pod on stdout 41 | func (c *ClientSet) DumpLogs() { 42 | veleroPod := c.getPodName() 43 | 44 | for _, v := range veleroPod { 45 | if err := k8s.Client.DumpLogs(VeleroNamespace, v[0], v[1]); err != nil { 46 | fmt.Printf("Failed to dump velero logs, err=%s\n", err) 47 | } 48 | } 49 | } 50 | 51 | // getPodName return velero pod name and container name 52 | // {{"pod_1","container_1"},{"pod_2","container_2"},} 53 | func (c *ClientSet) getPodName() [][]string { 54 | podList, err := k8s.Client.GetPodList(VeleroNamespace, 55 | "deploy=velero", 56 | ) 57 | if err != nil { 58 | return [][]string{} 59 | } 60 | return getPodContainerList(podList) 61 | } 62 | 63 | // returns {{"pod1","container1"},{"pod2","container2"},} 64 | func getPodContainerList(podList *corev1.PodList) [][]string { 65 | pod := make([][]string, 0) 66 | 67 | for _, p := range podList.Items { 68 | for _, c := range p.Spec.Containers { 69 | pod = append(pod, []string{p.Name, c.Name}) 70 | } 71 | } 72 | return pod 73 | } 74 | -------------------------------------------------------------------------------- /tests/velero/restore.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package velero 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "sort" 23 | "time" 24 | 25 | "github.com/pkg/errors" 26 | v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" 27 | k8serrors "k8s.io/apimachinery/pkg/api/errors" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | ) 30 | 31 | type byCreationTimeStamp []v1.Backup 32 | 33 | func (rc byCreationTimeStamp) Len() int { 34 | return len(rc) 35 | } 36 | 37 | func (rc byCreationTimeStamp) Swap(i, j int) { 38 | rc[i], rc[j] = rc[j], rc[i] 39 | } 40 | 41 | func (rc byCreationTimeStamp) Less(i, j int) bool { 42 | return rc[i].Name < rc[j].Name 43 | } 44 | 45 | func (c *ClientSet) generateRestoreName(backup string) (string, error) { 46 | for i := 0; i < 10; i++ { 47 | r := generateRandomString(8) + "-" + backup 48 | if r == "" { 49 | continue 50 | } 51 | _, err := c.VeleroV1(). 52 | Restores(VeleroNamespace). 53 | Get(context.TODO(), r, metav1.GetOptions{}) 54 | if err != nil && k8serrors.IsNotFound(err) { 55 | return r, nil 56 | } 57 | } 58 | return "", errors.New("failed to generate unique restore name") 59 | } 60 | 61 | // GetScheduledBackups list out the generated backup for given schedule 62 | func (c *ClientSet) GetScheduledBackups(schedule string) ([]string, error) { 63 | var bkplist []string 64 | 65 | olist, err := c.VeleroV1(). 66 | Backups(VeleroNamespace). 67 | List(context.TODO(), metav1.ListOptions{ 68 | LabelSelector: v1.ScheduleNameLabel + "=" + schedule, 69 | }) 70 | if err != nil && k8serrors.IsNotFound(err) { 71 | return nil, err 72 | } 73 | 74 | sort.Sort(byCreationTimeStamp(olist.Items)) 75 | 76 | for _, o := range olist.Items { 77 | bkplist = append(bkplist, o.Name) 78 | } 79 | return bkplist, nil 80 | } 81 | 82 | // CreateRestore create restore from given backup/schedule for ns Namespace to targetedNs 83 | // Arguments: 84 | // - ns : source namespace or namespace from backup 85 | // - targetedNs : targeted namespace for namespace 'ns' 86 | // - backup : name of the backup, from which restore will happen 87 | // - schedule : name of schedule, from which restore should happen. If mentioned, backup should be empty 88 | func (c *ClientSet) CreateRestore(ns, targetedNs, backup, schedule string) (v1.RestorePhase, error) { 89 | var ( 90 | status v1.RestorePhase 91 | restoreName string 92 | err error 93 | ) 94 | 95 | snapVolume := true 96 | nsMapping := make(map[string]string) 97 | 98 | if targetedNs != "" && ns != targetedNs { 99 | nsMapping[ns] = targetedNs 100 | } 101 | 102 | if backup != "" { 103 | restoreName, err = c.generateRestoreName(backup) 104 | } else { 105 | restoreName, err = c.generateRestoreName(schedule) 106 | } 107 | 108 | if err != nil { 109 | return status, err 110 | } 111 | 112 | rst := &v1.Restore{ 113 | ObjectMeta: metav1.ObjectMeta{ 114 | Name: restoreName, 115 | Namespace: VeleroNamespace, 116 | }, 117 | Spec: v1.RestoreSpec{ 118 | IncludedNamespaces: []string{ns}, 119 | RestorePVs: &snapVolume, 120 | BackupName: backup, 121 | ScheduleName: schedule, 122 | NamespaceMapping: nsMapping, 123 | }, 124 | } 125 | o, err := c.VeleroV1(). 126 | Restores(VeleroNamespace). 127 | Create(context.TODO(), rst, metav1.CreateOptions{}) 128 | if err != nil { 129 | return status, err 130 | } 131 | 132 | return c.waitForRestoreCompletion(o.Name) 133 | } 134 | 135 | func (c *ClientSet) waitForRestoreCompletion(rst string) (v1.RestorePhase, error) { 136 | dumpLog := 0 137 | for { 138 | rst, err := c.getRestore(rst) 139 | if err != nil { 140 | return "", err 141 | } 142 | if isRestoreDone(rst) { 143 | return rst.Status.Phase, nil 144 | } 145 | if dumpLog > 6 { 146 | fmt.Printf("Waiting for restore %s completion..\n", rst.Name) 147 | dumpLog = 0 148 | } 149 | dumpLog++ 150 | time.Sleep(5 * time.Second) 151 | } 152 | } 153 | 154 | func (c *ClientSet) getRestore(name string) (*v1.Restore, error) { 155 | return c.VeleroV1(). 156 | Restores(VeleroNamespace). 157 | Get(context.TODO(), name, metav1.GetOptions{}) 158 | } 159 | 160 | // GetRestoredSnapshotFromSchedule list out the snapshot restored from given schedule 161 | func (c *ClientSet) GetRestoredSnapshotFromSchedule(scheduleName string) (map[string]v1.RestorePhase, error) { 162 | snapshotList := make(map[string]v1.RestorePhase) 163 | 164 | bkplist, err := c.GetScheduledBackups(scheduleName) 165 | if err != nil { 166 | return snapshotList, err 167 | } 168 | 169 | restoreList, err := c.VeleroV1(). 170 | Restores(VeleroNamespace).List(context.TODO(), metav1.ListOptions{}) 171 | if err == nil { 172 | for _, r := range restoreList.Items { 173 | restore := r 174 | for _, b := range bkplist { 175 | if r.Spec.BackupName == b && isRestoreDone(&restore) { 176 | snapshotList[b] = r.Status.Phase 177 | } 178 | } 179 | } 180 | } 181 | 182 | return snapshotList, nil 183 | } 184 | -------------------------------------------------------------------------------- /tests/velero/schedule.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package velero 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "time" 23 | 24 | "github.com/pkg/errors" 25 | v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" 26 | k8serrors "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | ) 29 | 30 | func (c *ClientSet) generateScheduleName() (string, error) { 31 | for i := 0; i < 10; i++ { 32 | b := generateRandomString(8) 33 | if b == "" { 34 | continue 35 | } 36 | _, err := c.VeleroV1(). 37 | Schedules(VeleroNamespace). 38 | Get(context.TODO(), b, metav1.GetOptions{}) 39 | if err != nil && k8serrors.IsNotFound(err) { 40 | return b, nil 41 | } 42 | } 43 | return "", errors.New("failed to generate unique backup name") 44 | } 45 | 46 | // CreateSchedule create scheduled backup for given namespace and wait till 'count' backup completed 47 | func (c *ClientSet) CreateSchedule(ns, period string, count int) (string, v1.BackupPhase, error) { 48 | var status v1.BackupPhase 49 | snapVolume := true 50 | 51 | sname, err := c.generateScheduleName() 52 | if err != nil { 53 | return "", status, err 54 | } 55 | 56 | sched := &v1.Schedule{ 57 | ObjectMeta: metav1.ObjectMeta{ 58 | Name: sname, 59 | Namespace: VeleroNamespace, 60 | }, 61 | Spec: v1.ScheduleSpec{ 62 | Template: v1.BackupSpec{ 63 | IncludedNamespaces: []string{ns}, 64 | SnapshotVolumes: &snapVolume, 65 | StorageLocation: BackupLocation, 66 | VolumeSnapshotLocations: []string{SnapshotLocation}, 67 | }, 68 | Schedule: period, 69 | }, 70 | } 71 | o, err := c.VeleroV1(). 72 | Schedules(VeleroNamespace). 73 | Create(context.TODO(), sched, metav1.CreateOptions{}) 74 | if err != nil { 75 | return "", status, err 76 | } 77 | 78 | if count < 0 { 79 | return o.Name, status, nil 80 | } 81 | if status, err = c.waitForScheduleCompletion(o.Name, count); err == nil { 82 | return o.Name, status, nil 83 | } 84 | return o.Name, status, err 85 | } 86 | 87 | func (c *ClientSet) waitForScheduleCompletion(name string, count int) (v1.BackupPhase, error) { 88 | dumpLog := 0 89 | for bcount := 0; bcount < count; { 90 | olist, err := c.VeleroV1(). 91 | Backups(VeleroNamespace). 92 | List(context.TODO(), metav1.ListOptions{ 93 | LabelSelector: v1.ScheduleNameLabel + "=" + name, 94 | }) 95 | if err != nil { 96 | return "", err 97 | } 98 | bcount = 0 99 | for _, bkp := range olist.Items { 100 | bkp := bkp 101 | if isBackupDone(&bkp) { 102 | if bkp.Status.Phase != v1.BackupPhaseCompleted { 103 | _ = c.DumpBackupLogs(bkp.Name) 104 | return bkp.Status.Phase, 105 | errors.Errorf("Backup{%s} failed.. %s", bkp.Name, bkp.Status.Phase) 106 | } 107 | bcount++ 108 | } 109 | } 110 | if dumpLog > 6 { 111 | fmt.Printf("Waiting for schedule %s completion completed Backup:%d/%d\n", name, bcount, count) 112 | dumpLog = 0 113 | } 114 | dumpLog++ 115 | time.Sleep(5 * time.Second) 116 | } 117 | return v1.BackupPhaseCompleted, nil 118 | } 119 | 120 | // DeleteSchedule delete given schedule 121 | func (c *ClientSet) DeleteSchedule(schedule string) error { 122 | return c.VeleroV1(). 123 | Schedules(VeleroNamespace). 124 | Delete(context.TODO(), schedule, metav1.DeleteOptions{}) 125 | } 126 | -------------------------------------------------------------------------------- /tests/velero/status.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package velero 18 | 19 | import v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" 20 | 21 | func isBackupDone(bkp *v1.Backup) bool { 22 | var completed bool 23 | 24 | switch bkp.Status.Phase { 25 | case v1.BackupPhaseFailedValidation, v1.BackupPhaseCompleted, v1.BackupPhasePartiallyFailed, v1.BackupPhaseFailed: 26 | completed = true 27 | } 28 | return completed 29 | } 30 | 31 | func isRestoreDone(rst *v1.Restore) bool { 32 | var completed bool 33 | 34 | switch rst.Status.Phase { 35 | case v1.RestorePhaseCompleted, v1.RestorePhaseFailed, v1.RestorePhaseFailedValidation, v1.RestorePhasePartiallyFailed: 36 | completed = true 37 | } 38 | return completed 39 | } 40 | -------------------------------------------------------------------------------- /velero-blockstore-openebs/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The OpenEBS Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | snap "github.com/openebs/velero-plugin/pkg/snapshot" 21 | zfssnap "github.com/openebs/velero-plugin/pkg/zfs/snapshot" 22 | "github.com/sirupsen/logrus" 23 | "github.com/spf13/pflag" 24 | veleroplugin "github.com/vmware-tanzu/velero/pkg/plugin/framework" 25 | ) 26 | 27 | func main() { 28 | veleroplugin.NewServer(). 29 | BindFlags(pflag.CommandLine). 30 | RegisterVolumeSnapshotter("openebs.io/cstor-blockstore", openebsSnapPlugin). 31 | RegisterVolumeSnapshotter("openebs.io/zfspv-blockstore", zfsSnapPlugin). 32 | Serve() 33 | } 34 | 35 | func openebsSnapPlugin(logger logrus.FieldLogger) (interface{}, error) { 36 | return &snap.BlockStore{Log: logger}, nil 37 | } 38 | 39 | func zfsSnapPlugin(logger logrus.FieldLogger) (interface{}, error) { 40 | return &zfssnap.BlockStore{Log: logger}, nil 41 | } 42 | --------------------------------------------------------------------------------