├── .SRCINFO ├── .env ├── .github ├── dependabot.yaml └── workflows │ ├── release.yaml │ └── test.yaml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── Pipfile ├── Pipfile.lock ├── README.md ├── SECURITY.md ├── docs ├── README.md ├── api │ ├── administrative │ │ └── README.md │ ├── collections │ │ └── README.md │ ├── common-responses.md │ ├── permissions.md │ └── users │ │ ├── README.md │ │ └── guide.md ├── client.md ├── examples-filesystem │ └── backup-repository │ │ ├── backups.riotkit.org │ │ └── v1alpha1 │ │ │ ├── backupcollections │ │ │ └── iwa-ait.yaml │ │ │ └── backupusers │ │ │ ├── admin.yaml │ │ │ ├── some-user.yaml │ │ │ └── unprivileged.yaml │ │ └── v1 │ │ └── secrets │ │ ├── backup-repository-collection-secrets.yaml │ │ └── backup-repository-passwords.yaml ├── examples │ ├── collection.yaml │ ├── dynamic │ │ ├── access.yaml │ │ └── version.yaml │ ├── secret.yaml │ ├── user.second-actor.yaml │ ├── user.without-any-roles.yaml │ └── user.yaml └── installing.md ├── env.mk ├── go.mod ├── go.sum ├── helm ├── backup-repository-server │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── templates │ │ ├── NOTES.txt │ │ ├── _helpers.tpl │ │ ├── crd.yaml │ │ ├── deployment.yaml │ │ ├── hpa.yaml │ │ ├── ingress.yaml │ │ ├── role.yaml │ │ ├── route.yaml │ │ ├── sealedsecret.yaml │ │ ├── secret.yaml │ │ ├── service.yaml │ │ ├── serviceaccount.yaml │ │ └── tests │ │ │ └── test-connection.yaml │ └── values.yaml └── examples │ ├── backup-repository-ci.values.yaml │ ├── minio.values.yaml │ └── postgresql.values.yaml ├── main.go ├── pkg ├── collections │ ├── model.go │ ├── model_test.go │ ├── repository.go │ └── service.go ├── concurrency │ └── locking.go ├── config │ ├── README.md │ ├── action.go │ ├── action_test.go │ ├── cache.go │ ├── filesystem.go │ ├── filesystem_test.go │ ├── generic.go │ ├── interface.go │ └── kubernetes.go ├── core │ └── ctx.go ├── db │ └── main.go ├── health │ ├── README.md │ ├── backupwindow.go │ ├── backupwindow_test.go │ ├── config.go │ ├── db.go │ ├── interface.go │ ├── size.go │ ├── storage.go │ └── sumofversions.go ├── http │ ├── auth.go │ ├── collection.go │ ├── health.go │ ├── logging.go │ ├── main.go │ ├── responses.go │ └── utils.go ├── security │ ├── constants.go │ ├── crypto.go │ ├── crypto_test.go │ ├── decision.go │ ├── decision_test.go │ ├── model.go │ ├── model_test.go │ ├── repository.go │ └── service.go ├── storage │ ├── .test_data │ │ └── test.gpg │ ├── entity.go │ ├── io.go │ ├── io_test.go │ ├── repository.go │ ├── rotation.go │ ├── rotation_test.go │ ├── service.go │ ├── validation.go │ └── validation_test.go └── users │ ├── model.go │ ├── repository.go │ └── service.go ├── skaffold.yaml ├── test.mk └── tests ├── .helpers └── local-registry.yaml ├── __init__.py ├── test_collection.py ├── test_health.py └── test_login.py /.SRCINFO: -------------------------------------------------------------------------------- 1 | arch = x86_64 2 | pkgname = backup-repository 3 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | ENV_CLUSTER_NAME=rkt 2 | ENV_NS=backups 3 | ENV_APP_SVC=server-backup-repository-server 4 | ENV_PORT_FORWARD=8050:8080 5 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | 8 | - package-ecosystem: "gomod" 9 | directory: "/" 10 | schedule: 11 | interval: "weekly" 12 | 13 | - package-ecosystem: "pip" 14 | directory: "/" 15 | schedule: 16 | interval: "weekly" 17 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - '*' 6 | 7 | env: 8 | REGISTRY: ghcr.io 9 | IMAGE_NAME: "riotkit-org/backup-repository" 10 | 11 | permissions: write-all 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@v3 19 | with: 20 | fetch-depth: 0 21 | 22 | - name: Set up Go 23 | uses: actions/setup-go@v3 24 | with: 25 | go-version: 1.19 26 | 27 | - name: Build 28 | run: "make build" 29 | 30 | - name: Release binaries to GitHub releases 31 | uses: goreleaser/goreleaser-action@v2 32 | with: 33 | distribution: goreleaser 34 | version: latest 35 | args: release --rm-dist 36 | env: 37 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | - uses: actions/upload-artifact@v2 40 | with: 41 | name: binary 42 | path: .build/backup-repository 43 | 44 | release-docker: 45 | runs-on: ubuntu-latest 46 | needs: ["build"] 47 | steps: 48 | - name: Checkout 49 | uses: actions/checkout@v3 50 | with: 51 | fetch-depth: 0 52 | 53 | - uses: actions/download-artifact@v2 54 | with: 55 | name: binary 56 | path: .build/ 57 | 58 | - name: Log in to the Container registry 59 | uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 60 | with: 61 | registry: ${{ env.REGISTRY }} 62 | username: ${{ github.actor }} 63 | password: ${{ secrets.GITHUB_TOKEN }} 64 | 65 | - name: Extract metadata (tags, labels) for Docker 66 | id: meta 67 | uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 68 | with: 69 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 70 | 71 | - name: Build and release to container registry 72 | uses: docker/build-push-action@v2 73 | with: 74 | context: . 75 | push: ${{ startsWith(github.ref, 'refs/tags/') }} 76 | tags: ${{ steps.meta.outputs.tags }} 77 | labels: ${{ steps.meta.outputs.labels }} 78 | 79 | chart-release: 80 | runs-on: ubuntu-latest 81 | needs: ["build", "release-docker"] 82 | steps: 83 | - name: Checkout 84 | uses: actions/checkout@v3 85 | with: 86 | fetch-depth: 0 87 | 88 | - name: Configure Git 89 | run: | 90 | git config user.name "$GITHUB_ACTOR" 91 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 92 | 93 | - name: Publish Helm chart 94 | uses: stefanprodan/helm-gh-pages@master 95 | with: 96 | token: "${{ secrets.GH_RW_TOKEN }}" 97 | charts_dir: helm 98 | charts_url: https://riotkit-org.github.io/helm-of-revolution 99 | owner: riotkit-org 100 | repository: helm-of-revolution 101 | branch: gh-pages 102 | target_dir: ./ 103 | commit_username: "${{ env.GITHUB_ACTOR }}" 104 | commit_email: "${{ env.GITHUB_ACTOR }}@users.noreply.github.com" 105 | 106 | app_version: "${{github.ref_name}}" 107 | chart_version: "${{github.ref_name}}" 108 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | pull_request: 4 | push: 5 | 6 | env: 7 | REGISTRY: ghcr.io 8 | IMAGE_NAME: "riotkit-org/backup-repository" 9 | 10 | permissions: 11 | packages: write 12 | 13 | jobs: 14 | test-and-release-snapshot: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@v3 19 | with: 20 | fetch-depth: 0 21 | 22 | - name: Set up Go 23 | uses: actions/setup-go@v3 24 | with: 25 | go-version: 1.19 26 | 27 | - name: Build 28 | run: "make build" 29 | 30 | - name: Test 31 | run: "make test" 32 | if: "!contains(github.event.head_commit.message, '!test skip')" 33 | 34 | - name: Convert coverage to lcov format 35 | uses: jandelgado/gcov2lcov-action@v1.0.8 36 | if: "!contains(github.event.head_commit.message, '!test skip')" 37 | 38 | - name: Coveralls 39 | uses: coverallsapp/github-action@1.1.3 40 | if: "!contains(github.event.head_commit.message, '!test skip')" 41 | with: 42 | github-token: ${{ secrets.github_token }} 43 | path-to-lcov: coverage.lcov 44 | 45 | # ======= 46 | # Docker 47 | # ======= 48 | 49 | - name: Log in to the Container registry 50 | uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 51 | with: 52 | registry: ${{ env.REGISTRY }} 53 | username: ${{ github.actor }} 54 | password: ${{ secrets.GITHUB_TOKEN }} 55 | 56 | - name: Extract metadata (tags, labels) for Docker 57 | id: meta 58 | uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 59 | with: 60 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 61 | 62 | - name: Build and release to container registry 63 | uses: docker/build-push-action@v2 64 | with: 65 | context: . 66 | push: true 67 | tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:snapshot 68 | labels: ${{ steps.meta.outputs.labels }} 69 | 70 | test-on-kubernetes: 71 | uses: riotkit-org/.github/.github/workflows/python.release.yaml@main 72 | with: 73 | pythonVersion: 3.11 74 | testCmd: "make k3d skaffold-deploy integration-test" 75 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.build 2 | /coverage.out 3 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | * Promotion of direct democracy, respecting the voice of others, egalitarian lifestyle 17 | 18 | Examples of unacceptable behavior by participants include: 19 | 20 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 21 | * Trolling, insulting/derogatory comments, and personal or political attacks 22 | * Public or private harassment 23 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 24 | * Intrusive promotion of commercial products, services or companies 25 | * Racism, homophobia, xenophobia 26 | * Promotion of totalitarian methods, totalitarian regimes, elite lifestyle 27 | * Other conduct which could reasonably be considered inappropriate in a professional setting 28 | 29 | ## Our Responsibilities 30 | 31 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 32 | 33 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 34 | 35 | ## Scope 36 | 37 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 38 | 39 | ## Enforcement 40 | 41 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at riotkit[this-special-symbol]riseup.net. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 42 | 43 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other maintainers of the project. 44 | 45 | ## Attribution 46 | 47 | This Code of Conduct is based on the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version], modified by RiotKit according to the character of the organization 48 | 49 | [homepage]: http://contributor-covenant.org 50 | [version]: http://contributor-covenant.org/version/1/4/ 51 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | It would be fantastic if you would create an issue or better, create a PR, or do a code review. Contributions are always welcome! 5 | Before doing a contribution please take a look at our guideline. 6 | 7 | Architecture 8 | ------------ 9 | 10 | The architecture is module based, like in Golang projects. A little inspiration was taken from PHP and Python to use `Repository pattern`. 11 | 12 | **Most of the modules are split into:** 13 | - entity: Structs containing domain logic 14 | - repository: Interaction with data store via `ConfigurationProvider` or `GORM` (only private methods there) 15 | - service: Public methods performing actual actions on the model, repository. There are defined actions that aggregates logic as much as it is possible 16 | 17 | **'http' module** 18 | 19 | It is a special module that defines all routes, it's authentication and security logic for every endpoint. 20 | 21 | - auth: Endpoints related to authorization and users management 22 | - collection: Endpoints for operations on collections 23 | - responses: HTTP responses format standardization 24 | - utils: Various utils for validating user input, checking session etc. 25 | - main: Registers all the endpoints to the router 26 | 27 | Development environment & testing 28 | --------------------------------- 29 | 30 | 1. Application should be tested with all supported software listed in README.md if there is a risk that something could be broken 31 | 2. Unit tests coverage is required 32 | 3. [API tests in PyTest](./tests) should be written, especially when code is difficult to cover with unit tests 33 | 34 | ### Manual testing helpers 35 | 36 | Take a look at [test.mk](./test.mk) file, which contains Makefile tasks for manual testing, used at development time as handy shortcuts. 37 | 38 | ```bash 39 | make dev 40 | make -f test.mk import-examples 41 | ``` 42 | 43 | ```bash 44 | # at first login and export the retrieved token into the shell 45 | make -f test.mk test_login 46 | export TOKEN=... 47 | 48 | # then use prepared curl snippets to test functionalities 49 | make -f test.mk test_collection_health test_whoami # ... 50 | ``` 51 | 52 | ### Automated testing 53 | 54 | **Unit tests can be executed within:** 55 | 56 | ```bash 57 | make test 58 | ``` 59 | 60 | **There are also E2E tests on a real application image:** 61 | 62 | ```bash 63 | make integration-test 64 | ``` 65 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.15 as builder 2 | ADD .build/backup-repository /backup-repository 3 | RUN chmod +x /backup-repository && chmod 755 /backup-repository && chown 1001 /backup-repository 4 | 5 | 6 | FROM gcr.io/distroless/base 7 | ENV GIN_MODE=release 8 | ADD docs /usr/share/backup-repository 9 | COPY --from=builder /backup-repository /backup-repository 10 | USER 65532 11 | ENTRYPOINT ["/backup-repository"] 12 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include env.mk 2 | 3 | SUDO= 4 | 5 | .EXPORT_ALL_VARIABLES: 6 | PATH = $(shell pwd)/.build:$(shell echo $$PATH) 7 | 8 | all: build run 9 | 10 | build: 11 | CGO_ENABLED=0 GO111MODULE=on go build -tags=nomsgpack -o ./.build/backup-repository 12 | 13 | test: ## Unit tests 14 | go test -v ./... -covermode=count -coverprofile=coverage.out 15 | 16 | integration-test: prepare-tools _prepare-env _pytest ## End-To-End tests with Kubernetes 17 | _pytest: ## Shortcut for E2E tests without setting up the environment 18 | pipenv sync 19 | pipenv run pytest -s 20 | 21 | _prepare-env: 22 | kubectl apply -f "helm/backup-repository-server/templates/crd.yaml" 23 | kubectl apply -f "docs/examples/" -n backups 24 | 25 | run: 26 | export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE; \ 27 | export AWS_SECRET_ACCESS_KEY=wJaFuCKtnFEMI/CApItaliSM/bPxRfiCYEXAMPLEKEY; \ 28 | \ 29 | backup-repository \ 30 | --db-password=postgres \ 31 | --db-user=postgres \ 32 | --db-password=postgres \ 33 | --db-name=postgres \ 34 | --health-check-key=changeme \ 35 | --jwt-secret-key="secret key" \ 36 | --storage-io-timeout="5m" \ 37 | --listen=":${SERVER_PORT}" \ 38 | --provider=kubernetes \ 39 | --storage-url="s3://mybucket?endpoint=localhost:9000&disableSSL=true&s3ForcePathStyle=true®ion=eu-central-1" 40 | 41 | run_with_local_config_storage: 42 | export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE; \ 43 | export AWS_SECRET_ACCESS_KEY=wJaFuCKtnFEMI/CApItaliSM/bPxRfiCYEXAMPLEKEY; \ 44 | \ 45 | ./.build/backup-repository \ 46 | --db-password=postgres \ 47 | --db-user=postgres \ 48 | --db-password=postgres \ 49 | --db-name=postgres \ 50 | --health-check-key=changeme \ 51 | --jwt-secret-key="secret key" \ 52 | --storage-io-timeout="5m" \ 53 | --listen=":${SERVER_PORT}" \ 54 | --provider=filesystem \ 55 | --config-local-path=$$(pwd)/docs/examples-filesystem/\ 56 | --storage-url="s3://mybucket?endpoint=localhost:9000&disableSSL=true&s3ForcePathStyle=true®ion=eu-central-1" 57 | 58 | postgres: ## Runs local PostgreSQL for running project as local binary 59 | docker run -p 5432:5432 -d --rm --name postgres -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres postgres:15.3-alpine 60 | 61 | minio: ## Runs local Min.io for running project as local binary 62 | docker run -d \ 63 | --name br_minio \ 64 | -p 9000:9000 \ 65 | -p 9001:9001 \ 66 | -v $$(pwd)/.build/minio:/data \ 67 | -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \ 68 | -e "MINIO_ROOT_PASSWORD=wJaFuCKtnFEMI/CApItaliSM/bPxRfiCYEXAMPLEKEY" \ 69 | --entrypoint /bin/sh \ 70 | quay.io/minio/minio:RELEASE.2022-02-16T00-35-27Z -c 'mkdir -p /data/mybucket && minio server /data --console-address 0.0.0.0:9001' 71 | 72 | 73 | lint: 74 | export GO111MODULE=on; \ 75 | golangci-lint run \ 76 | --verbose \ 77 | --build-tags build 78 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | pytest = "*" 8 | requests = "*" 9 | 10 | [dev-packages] 11 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported | 6 | | ------- | ------------------ | 7 | | 4.x.x | :white_check_mark: | 8 | | 2.x.x | :x: | 9 | | 1.x.x | :x: | 10 | 11 | ## Reporting a Vulnerability 12 | 13 | To report a vulnerability please send a mail to riotkit[this-special-character]riseup.net 14 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | Documentation 2 | ============= 3 | 4 | Kubernetes resources 5 | -------------------- 6 | 7 | Backup Repository server is designed to be Kubernetes-first and Security-first. 8 | Configuration of basic entities such as **users** and **backup collections** are done using YAMLs in Kubernetes syntax. 9 | 10 | ```yaml 11 | --- 12 | apiVersion: backups.riotkit.org/v1alpha1 13 | kind: BackupCollection 14 | # (...) 15 | 16 | --- 17 | apiVersion: backups.riotkit.org/v1alpha1 18 | kind: BackupUser 19 | # (...) 20 | ``` 21 | 22 | **Check examples of Kubernetes YAMLs:** 23 | 24 | - [`kind: BackupUser`](examples/user.yaml) 25 | - [`kind: BackupCollection`](examples/collection.yaml) 26 | - [`kind: Secret` (secrets referenced in above examples for `kind: BackupUser` and `kind: BackupCollection`)](examples/secret.yaml) 27 | 28 | **Your server instance can be configured using those YAML's basically, the rest are highly dynamic changing data that is configured via API, it includes `Authentication keys` and `Uploaded backup versions that are rotating`** 29 | 30 | Installation 31 | ------------ 32 | 33 | Read more about [how to install Backup Repository](./installing.md). 34 | 35 | API 36 | --- 37 | 38 | Interactions with server are done using HTTP API that talks JSON in both ways, and identifies with JWT. 39 | 40 | ### [Users and Authentication](api/users/README.md) 41 | 42 | ### [Permissions](api/permissions.md) 43 | 44 | ### [Collections](api/collections/README.md) 45 | 46 | ### [Administrative](api/administrative/README.md) 47 | 48 | ### [Clients](./client.md) 49 | 50 | Development 51 | ----------- 52 | 53 | ### [Developer guide](../CONTRIBUTING.md) 54 | 55 | Security and support 56 | -------------------- 57 | 58 | There is no commercial support, this is a community project. 59 | 60 | ### [List of maintained versions](../SECURITY.md) 61 | -------------------------------------------------------------------------------- /docs/api/administrative/README.md: -------------------------------------------------------------------------------- 1 | Administrative API endpoints 2 | ============================ 3 | 4 | ## GET `/ready` 5 | 6 | **Parameters:** 7 | - `?code` or header `Authorization: ...` to pass secret passphrase defined in server's startup commandline switch `--health-check-key` 8 | 9 | **Example:** 10 | 11 | ```bash 12 | curl -s -X GET 'http://localhost:8080/ready' 13 | ``` 14 | 15 | **Example response (200):** 16 | 17 | ```json 18 | { 19 | "data": { 20 | "health": [ 21 | { 22 | "message": "OK", 23 | "name": "DbValidator", 24 | "status": true, 25 | "statusText": "DbValidator=true" 26 | }, 27 | { 28 | "message": "OK", 29 | "name": "StorageAvailabilityValidator", 30 | "status": true, 31 | "statusText": "StorageAvailabilityValidator=true" 32 | }, 33 | { 34 | "message": "OK", 35 | "name": "ConfigurationProviderValidator", 36 | "status": true, 37 | "statusText": "ConfigurationProviderValidator=true" 38 | } 39 | ] 40 | }, 41 | "status": true 42 | } 43 | ``` 44 | 45 | **Unauthorized response (403):** 46 | 47 | ```json 48 | { 49 | "data": {}, 50 | "error": "health code invalid. Should be provided withing 'Authorization' header or 'code' query string. Must match --health-check-code commandline switch value", 51 | "status": false 52 | } 53 | ``` 54 | 55 | **Example error response (500):** 56 | 57 | ```json 58 | { 59 | "data": { 60 | "health": [ 61 | { 62 | "message": "OK", 63 | "name": "DbValidator", 64 | "status": true, 65 | "statusText": "DbValidator=true" 66 | }, 67 | { 68 | "message": "storage not operable: blob (key \".health-1646488208126295105\") (code=Unknown): RequestError: send request failed\ncaused by: Put \"http://minio.backup-repository.svc.cluster.local:9000/backups/.health-1646488208126295105\": dial tcp 10.43.153.2:9000: i/o timeout", 69 | "name": "StorageAvailabilityValidator", 70 | "status": false, 71 | "statusText": "StorageAvailabilityValidator=false" 72 | }, 73 | { 74 | "message": "OK", 75 | "name": "ConfigurationProviderValidator", 76 | "status": true, 77 | "statusText": "ConfigurationProviderValidator=true" 78 | } 79 | ] 80 | }, 81 | "error": "one of checks failed", 82 | "status": false 83 | } 84 | ``` 85 | 86 | ## GET `/health` 87 | 88 | **Example response (200):** 89 | 90 | ```json 91 | { 92 | "data": { 93 | "msg": "The server is up and running. Dependent services are not shown there. Take a look at /ready endpoint" 94 | }, 95 | "status": true 96 | } 97 | ``` 98 | 99 | **500:** 100 | 101 | There is no error response, if the server is unhealthy then it will not respond, there will be a gateway timeout on reverse proxy or connection timeout on client side. 102 | -------------------------------------------------------------------------------- /docs/api/common-responses.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/riotkit-org/backup-repository/66c8b0bde05c55320fa9cbc846d89f06935a968e/docs/api/common-responses.md -------------------------------------------------------------------------------- /docs/api/permissions.md: -------------------------------------------------------------------------------- 1 | Permissions 2 | =========== 3 | 4 | Users can be granted system-level permissions in `kind: BackupUser`, such permissions will apply to all objects in system. 5 | Permissions can be overridden on object-level, for example in `kind: BackupCollection` 6 | 7 | 8 | | Name | Description | Scope | 9 | |-------------------|-----------------------------------------------------|--------------------| 10 | | userManager | Use user managemet endpoints, lookup any users | system | 11 | | collectionManager | Manage collection settings, upload, download | system, collection | 12 | | backupDownloader | Upload to a collection | system, collection | 13 | | backupUploader | Download from a collection | system, collection | 14 | | uploadsAnytime | Can upload to collection outside Backup Window time | system, collection | 15 | | systemAdmin | Unlimited access | system | 16 | 17 | 18 | BackupCollection example 19 | ------------------------ 20 | 21 | ```yaml 22 | --- 23 | apiVersion: backups.riotkit.org/v1alpha1 24 | kind: BackupCollection 25 | metadata: 26 | name: iwa-ait 27 | spec: 28 | # ... 29 | accessControl: 30 | - userName: admin 31 | roles: 32 | - collectionManager 33 | ``` 34 | 35 | User example 36 | ------------ 37 | 38 | ```yaml 39 | --- 40 | apiVersion: backups.riotkit.org/v1alpha1 41 | kind: BackupUser 42 | metadata: 43 | name: iwa-backup-submitter 44 | namespace: backups 45 | spec: 46 | # ... 47 | roles: 48 | - backupDownloader 49 | - backupUploader 50 | ``` 51 | -------------------------------------------------------------------------------- /docs/api/users/README.md: -------------------------------------------------------------------------------- 1 | Users and Authentication API 2 | ============================ 3 | 4 | ## Before you begin 5 | 6 | Every operation needs to be authenticated with a JWT (JSON Web Token) generated single-time. 7 | 8 | The token can be passed within `Authorization: Bearer token-here` header of each request. 9 | 10 | Tokens are generated by the `login` endpoint for a given pair of username and password. Username and password is defined by a `kind: BackupUser` in Kubernetes, see [Kubernetes examples](../../examples) 11 | 12 | ## [User creation and authentication guide - step by step](./guide.md) 13 | 14 | ## POST `/api/stable/auth/login` 15 | 16 | **Example:** 17 | 18 | ```bash 19 | curl -s -X POST -d '{"username":"admin","password":"admin"}' -H 'Content-Type: application/json' 'http://localhost:8080/api/stable/auth/login' 20 | ``` 21 | 22 | **Example response (200):** 23 | 24 | ```json 25 | { 26 | "data": { 27 | "expire": "2032-02-25T00:32:56+01:00", 28 | "msg": "Use this sessionId to revoke this token anytime", 29 | "sessionId": "2d0aa5db61c02ea9a9d7fe6d768021aca98fff1fe75fe214a65ccd6926bb8b77", 30 | "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE5NjEyNzgzNzYsImxvZ2luIjoiYWRtaW4iLCJvcmlnX2lhdCI6MTY0NTkxODM3Nn0.my0WXXMxKCetkomtzRDNIKLWUm4cJ2gxyUCkuAHT6M4" 31 | }, 32 | "status": true 33 | } 34 | ``` 35 | 36 | The `.data.token` from response is a session secret key, use it to authenticate next API requests using `Authorization: Bearer token-here` header. 37 | 38 | ### Feature: Generating a JWT token with limited permissions 39 | 40 | User can create a restricted session with `/api/stable/auth/login` endpoint, by using extra parameter `operationsScope`. 41 | 42 | `operationsScope` extra parameter is a working like an allowlist/whitelist - optional, when specified, then User permissions 43 | are restricted to roles specified there. 44 | 45 | _Notice: Roles specified in `operationsScope` cannot be higher than specified in User's profile or in collection ACL._ 46 | 47 | **Example:** 48 | 49 | ```bash 50 | curl -s -X POST -d '{"username":"some-user","password":"test", "operationsScope": {"elements": [{"type": "collection", "name": "iwa-ait", "roles": ["collectionManager"]}]}}' -H 'Content-Type: application/json' 'http://localhost:8080/api/stable/auth/login' 51 | ``` 52 | 53 | _Notice `"operationsScope": {"elements": [{"type": "collection", "name": "iwa-ait", "roles": ["collectionManager"]}]}` in the request body._ 54 | 55 | ### Feature: Access Keys 56 | 57 | Second way to restrict user session is by creating **Access Keys** that are separate passwords associated with same User account, but with additional restrictions. 58 | 59 | **Example configuration:** 60 | 61 | ```yaml 62 | --- 63 | apiVersion: backups.riotkit.org/v1alpha1 64 | kind: BackupUser 65 | metadata: 66 | name: some-user 67 | spec: 68 | # (...) 69 | passwordFromRef: 70 | name: backup-repository-passwords 71 | entry: admin 72 | accessKeys: 73 | # 74 | # This entry creates a "sub-user" some-user$uploader which is restricted to only upload to collection 'iwa-ait' 75 | # This "sub-user" uses a different password. You may create as many "sub-users" (access keys) as you wish - per collection, per function. 76 | # 77 | # login: some-user$iwa 78 | # password: test 79 | # 80 | - name: iwa 81 | # password: "" 82 | passwordFromRef: 83 | name: backup-repository-passwords 84 | entry: admin_access_key_1 85 | objects: 86 | - name: iwa-ait 87 | type: collection 88 | roles: ["backupUploader"] 89 | # (...) 90 | ``` 91 | 92 | _Notice: Roles specified in `accessKeys` cannot be higher than specified in User's profile or in collection ACL._ 93 | 94 | **Example JSON payload:** 95 | 96 | ```json 97 | {"username":"some-user$uploader","password":"test"} 98 | ``` 99 | 100 | When using login endpoint you need to specify **Access Key** name after `$` in username field, and use password specified for that Access Key - it's not a regular password associated with the account. 101 | 102 | **Example:** 103 | 104 | ```bash 105 | curl -s -X POST -d '{"username":"some-user$uploader","password":"test", "operationsScope": {"elements": []}}' -H 'Content-Type: application/json' 'http://localhost:8080/api/stable/auth/login' 106 | ``` 107 | 108 | ## GET `/api/stable/auth/user/some-user` 109 | 110 | Displays information about user **of given username**. 111 | 112 | **Example:** 113 | 114 | ```bash 115 | curl -s -X GET -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE5NjEyNjMzOTUsImxvZ2luIjoiYWRtaW4iLCJvcmlnX2lhdCI6MTY0NTkwMzM5NX0.kV0baqRJ5DI-0ZSmES2zQTlIlNsd9RZz9DZvQYD7jDc' \ 116 | -H 'Content-Type: application/json' \ 117 | 'http://localhost:8080/api/stable/auth/user/some-user' 118 | ``` 119 | 120 | **Example response (200):** 121 | 122 | ```json 123 | { 124 | "data": { 125 | "email": "user@riseup.net", 126 | "permissions": [ 127 | "collectionManager" 128 | ] 129 | }, 130 | "status": true 131 | } 132 | ``` 133 | 134 | **Other responses:** 135 | - [403](../common-responses.md) 136 | - [404](../common-responses.md) 137 | 138 | 139 | ## GET `/api/stable/auth/whoami` 140 | 141 | Displays information about current session ad current user. 142 | 143 | **Example:** 144 | 145 | ```bash 146 | curl -s -X GET -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE5NjEyNjMzOTUsImxvZ2luIjoiYWRtaW4iLCJvcmlnX2lhdCI6MTY0NTkwMzM5NX0.kV0baqRJ5DI-0ZSmES2zQTlIlNsd9RZz9DZvQYD7jDc' \ 147 | -H 'Content-Type: application/json' \ 148 | 'http://localhost:8080/api/stable/auth/whoami' 149 | ``` 150 | 151 | **Example response (200):** 152 | 153 | ```json 154 | { 155 | "data": { 156 | "email": "riotkit@riseup.net", 157 | "grantedAccess": { 158 | "CreatedAt": "2022-02-26T20:23:15.272127+01:00", 159 | "UpdatedAt": "2022-02-26T20:23:15.272127+01:00", 160 | "DeletedAt": null, 161 | "id": "4e1a57bebab9a87faa413934015d9e2ae83cff39cce3b55b02a55e68392964b3", 162 | "expiresAt": "2032-02-24T20:23:15.271405+01:00", 163 | "deactivated": false, 164 | "description": "Login", 165 | "requesterIP": "::1", 166 | "user": "admin" 167 | }, 168 | "permissions": [ 169 | "collectionManager", 170 | "usersManager", 171 | "systemAdmin" 172 | ], 173 | "sessionId": "4e1a57bebab9a87faa413934015d9e2ae83cff39cce3b55b02a55e68392964b3" 174 | }, 175 | "status": true 176 | } 177 | ``` 178 | 179 | **Other responses:** 180 | - [403](../common-responses.md) 181 | 182 | ## GET `/api/stable/auth/logout` 183 | 184 | Allows log out own session, or session of other user (if user has access rights to do it). 185 | 186 | **Required roles:** 187 | - `systemAdmin` (if wanting to log out other user's session) 188 | 189 | **Optional query string parameters:** 190 | - sessionId 191 | 192 | **Example:** 193 | 194 | ```bash 195 | curl -s -X DELETE -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE5NjEyNjMzOTUsImxvZ2luIjoiYWRtaW4iLCJvcmlnX2lhdCI6MTY0NTkwMzM5NX0.kV0baqRJ5DI-0ZSmES2zQTlIlNsd9RZz9DZvQYD7jDc' \ 196 | -H 'Content-Type: application/json' \ 197 | 'http://localhost:8080/api/stable/auth/logout' 198 | ``` 199 | 200 | **Example response:** 201 | 202 | ```json 203 | { 204 | "data": { 205 | "message": "JWT was revoked", 206 | "sessionId": "4e1a57bebab9a87faa413934015d9e2ae83cff39cce3b55b02a55e68392964b3" 207 | }, 208 | "status": true 209 | } 210 | ``` 211 | 212 | **Other responses:** 213 | - [403](../common-responses.md) 214 | - [404](../common-responses.md) 215 | - [500](../common-responses.md) 216 | 217 | 218 | ## GET `/api/stable/auth/token` 219 | 220 | Lists all granted JWTs to given account. 221 | 222 | **Required roles:** 223 | - `systemAdmin` (if wanting to act as other user) 224 | 225 | **Optional query string parameters:** 226 | - userName 227 | 228 | **Example:** 229 | 230 | ```bash 231 | curl -s -X GET -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE5NjEyNzkxNDcsImxvZ2luIjoiYWRtaW4iLCJvcmlnX2lhdCI6MTY0NTkxOTE0N30.f8ANZup-rifDwTVUNsm9dEFerOjLh2Wsqz8IBw0j3zk' \ 232 | -H 'Content-Type: application/json' \ 233 | 'http://localhost:8080/api/stable/auth/token' 234 | ``` 235 | 236 | **Example response (200):** 237 | 238 | ```json 239 | { 240 | "data": { 241 | "grantedAccesses": [ 242 | { 243 | "CreatedAt": "2022-02-27T00:45:47.256269+01:00", 244 | "UpdatedAt": "2022-02-27T00:45:47.256269+01:00", 245 | "DeletedAt": null, 246 | "id": "8c313de003f20bd34f78c6fd26c3f5943e6339b744e111569d9ed1d121796839", 247 | "expiresAt": "2032-02-25T00:45:47.255996+01:00", 248 | "deactivated": false, 249 | "description": "Login", 250 | "requesterIP": "1.2.3.4", 251 | "user": "admin" 252 | } 253 | ] 254 | }, 255 | "status": true 256 | } 257 | ``` 258 | -------------------------------------------------------------------------------- /docs/api/users/guide.md: -------------------------------------------------------------------------------- 1 | User creation guide 2 | =================== 3 | 4 | By using `kind: BackupUser` custom resource definition create a user in a GitOps-way. 5 | 6 | **backupuser.yaml** 7 | 8 | ```yaml 9 | --- 10 | apiVersion: backups.riotkit.org/v1alpha1 11 | kind: BackupUser 12 | metadata: 13 | name: admin 14 | spec: 15 | # best practice is to set this e-mail to same e-mail as GPG key owner e-mail (GPG key used on client side to encrypt files) 16 | email: example@example.org 17 | deactivated: false 18 | organization: "Riotkit" 19 | about: "System administrator" 20 | passwordFromRef: 21 | name: backup-repository-passwords 22 | entry: admin 23 | restrictByIP: 24 | - 1.2.3.4 25 | roles: 26 | - systemAdmin 27 | ``` 28 | 29 | > :information_source: Passwords are not stored in plain-text inside `kind: BackupUser` custom resource definition 30 | 31 | Generate a password and encode it. `backup-repository` CLI will encode the password with Argon2 and base64. 32 | 33 | ```bash 34 | PASSWORD=$(openssl rand -base64 $((1024*1024)) | sha512sum -) 35 | backup-repository --encode-password="${PASSWORD}" 36 | ``` 37 | 38 | Create a `kind: Secret` referenced in `kind: BackupUser`, it will store user password in hashed form. 39 | Password hashed and encoded by `backup-repository` can be inserted into `data` section (not in `stringData` as it is already base64 encoded). 40 | 41 | **secret.yaml** 42 | 43 | ```yaml 44 | --- 45 | apiVersion: v1 46 | kind: Secret 47 | metadata: 48 | name: backup-repository-passwords 49 | type: Opaque 50 | data: 51 | # admin: admin 52 | # to generate: `backup-repository --encode-password "admin" 53 | admin: "JGFyZ29uMmlkJHY9MTkkbT02NTUzNix0PTEscD00JHpuVy9IT2Y4Q3RkdStvNSttYlR2REE9PSRaZlVpRGl2QWV2T2RZNndKYWJBb0FQdmM1a1hsemxDNkg2OFY2dGVmNUY0PQ==" 54 | 55 | ``` 56 | 57 | Apply user and password to the cluster. 58 | 59 | ```bash 60 | # notice: The namespace should match Backup Repository namespace 61 | kubectl apply -f secret.yaml -n backups 62 | kubectl apply -f backupuser.yaml -n backups 63 | ``` 64 | 65 | Generate a token allowing to operate on Backup Repository API. 66 | 67 | ```bash 68 | PASSWORD="..." 69 | curl -X POST -d '{"username":"admin","password":"${PASSWORD}"}' -H 'Content-Type: application/json' 'http://localhost:8080/api/stable/auth/login' -k 70 | ``` 71 | 72 | Copy the token from the repsponse and keep it safe - it will allow to perform interactions with API as your user. 73 | 74 | ```json 75 | { 76 | "data": { 77 | "expire": "2032-05-25T05:56:45Z", 78 | "sessionId": "xxxxxxxxxxxxx-USE-THIS-SESSION-ID-TO-REVOKE-YOUR-TOKEN-xxxxxxxxxxxxxxxxxxxxxxx", 79 | "token": "xxxxxxxxxxxxxxxxxx-COPY-THIS-SECRET-TOKEN-TO-ACCESS-YOUR-API-xxxxxxxxxxxxxxxxxxxxxxxxxxxx" 80 | }, 81 | "status": true 82 | } 83 | ``` 84 | -------------------------------------------------------------------------------- /docs/client.md: -------------------------------------------------------------------------------- 1 | Backup upload & download 2 | ======================== 3 | 4 | The server is accepting uploaded files using an HTTP endpoint in standardized format, [check out the API docs for upload endpoint before proceeding](./api/collections/README.md). 5 | 6 | ~ Basic usage with cURL 7 | ----------------------- 8 | 9 | 1) **Receive authorization token to sign requests** 10 | 11 | ```bash 12 | curl -s -X POST -d '{"username":"admin","password":"admin"}' \ 13 | -H 'Content-Type: application/json' \ 14 | 'http://localhost/api/stable/auth/login' | jq '.data.token' -r 15 | ``` 16 | 17 | 2) **Copy generated token (or export to variable in script)** 18 | 19 | 3) **Upload a file or a piped stream** 20 | 21 | ```bash 22 | curl -vvv -X POST -H 'Authorization: Bearer {{put-token-here}}' -F "file=@./archive.tar.gz.asc" 'http://localhost/api/stable/repository/collection/iwa-ait/version' 23 | ``` 24 | 25 | 4) **Download file (restoring a backup)** 26 | 27 | ```bash 28 | curl -vvv -X GET -H 'Authorization: Bearer {{put-token-here}}' 'http://localhost/api/stable/repository/collection/iwa-ait/version/latest' > ./my-file.tar.gz.asc 29 | ``` 30 | 31 | ### FAQ: How do I encrypt a file? 32 | 33 | ```bash 34 | gpg --encrypt -r test@riotkit.org ./archive.tar.gz 35 | ``` 36 | 37 | ### FAQ: How do I create GPG keys? 38 | 39 | Setup a GPG keyring: https://linuxhint.com/gpg-command-ubuntu/ 40 | 41 | ~ [Backup Maker](https://github.com/riotkit-org/br-backup-maker) 42 | ---------------- 43 | 44 | This is an official client for Backup Repository, it **automates GPG operations** almost transparently to the user and performs all operations on buffers to be 45 | lightweight. 46 | 47 | ### Backup 48 | 49 | ```bash 50 | export BM_AUTH_TOKEN="some-token" # JWT token 51 | export BM_COLLECTION_ID="111-222-333-444" # collection name/id 52 | 53 | backup-maker make --url https://example.org \ 54 | -c "tar -zcvf - ./" \ # backup command which output Backup Maker will send 55 | --key build/test/backup.pub \ # public key (or private) required to encrypt the file 56 | --recipient test@riotkit.org \ # target key recipient (usually is the same as key owner) 57 | --log-level info 58 | ``` 59 | 60 | ### Restore 61 | 62 | ```bash 63 | export BM_AUTH_TOKEN="some-token" 64 | export BM_COLLECTION_ID="111-222-333-444" 65 | export BM_PASSPHRASE="riotkit" 66 | 67 | backup-maker restore --url https://example.org \ 68 | -c "cat - > /tmp/test" \ 69 | --private-key .build/test/backup.key \ # to decrypt we need a PRIVATE KEY 70 | --recipient test@riotkit.org \ 71 | --log-level debug 72 | ``` 73 | 74 | ### FAQ: How do I get public and private keys? 75 | 76 | 1. **Setup GPG keyring** 77 | 78 | https://linuxhint.com/gpg-command-ubuntu/ 79 | 80 | 2. **Export keys** 81 | 82 | ```bash 83 | # public key 84 | gpg --armor --export user@example.com > public_key.asc 85 | 86 | # private key 87 | gpg --list-secret-keys user@example.com # list keys to find a id 88 | gpg --export-secret-keys YOUR_ID_HERE > private.key 89 | ``` 90 | -------------------------------------------------------------------------------- /docs/examples-filesystem/backup-repository/backups.riotkit.org/v1alpha1/backupcollections/iwa-ait.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: backups.riotkit.org/v1alpha1 3 | kind: BackupCollection 4 | metadata: 5 | name: iwa-ait 6 | namespace: backup-repository 7 | spec: 8 | description: IWA-AIT website files 9 | filenameTemplate: iwa-ait-${version}.tar.gz 10 | maxBackupsCount: 5 11 | maxOneVersionSize: 1M 12 | maxCollectionSize: 10M 13 | 14 | # optional 15 | windows: 16 | - from: "*/30 * * * *" 17 | duration: 30m 18 | 19 | # fifo, fifo-plus-older 20 | strategyName: fifo 21 | strategySpec: 22 | keepLastOlderNotMoreThan: 5d 23 | maxOlderCopies: 2 24 | 25 | healthSecretRef: 26 | name: backup-repository-collection-secrets 27 | entry: iwa-ait 28 | 29 | accessControl: 30 | - name: admin 31 | type: user 32 | roles: 33 | - collectionManager 34 | -------------------------------------------------------------------------------- /docs/examples-filesystem/backup-repository/backups.riotkit.org/v1alpha1/backupusers/admin.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: backups.riotkit.org/v1alpha1 3 | kind: BackupUser 4 | metadata: 5 | name: admin 6 | spec: 7 | email: riotkit@riseup.net 8 | deactivated: false 9 | organization: "Riotkit" 10 | about: "System administrator" 11 | password: "hashed" 12 | passwordFromRef: # alternatively, fetch from `kind: Secret` 13 | name: backup-repository-passwords 14 | entry: admin 15 | restrictByIP: 16 | - 1.2.3.4 17 | roles: 18 | - collectionManager 19 | - usersManager 20 | - systemAdmin 21 | -------------------------------------------------------------------------------- /docs/examples-filesystem/backup-repository/backups.riotkit.org/v1alpha1/backupusers/some-user.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: backups.riotkit.org/v1alpha1 3 | kind: BackupUser 4 | metadata: 5 | name: some-user 6 | spec: 7 | email: user@riseup.net 8 | deactivated: false 9 | organization: "Riotkit" 10 | about: "Example user" 11 | password: "" 12 | passwordFromRef: 13 | name: backup-repository-passwords 14 | entry: admin 15 | accessKeys: 16 | # 17 | # login: some-user$uploader 18 | # password: test 19 | # 20 | - name: uploader 21 | # password: "" 22 | passwordFromRef: 23 | name: backup-repository-passwords 24 | entry: admin_access_key_1 25 | objects: 26 | - name: iwa-ait 27 | type: collection 28 | roles: ["backupUploader"] 29 | roles: 30 | - collectionManager 31 | -------------------------------------------------------------------------------- /docs/examples-filesystem/backup-repository/backups.riotkit.org/v1alpha1/backupusers/unprivileged.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: backups.riotkit.org/v1alpha1 3 | kind: BackupUser 4 | metadata: 5 | name: unprivileged 6 | spec: 7 | email: unprivileged@example.org 8 | deactivated: false 9 | organization: "Riotkit" 10 | about: "Example unprivileged user" 11 | password: "" 12 | passwordFromRef: 13 | name: backup-repository-passwords 14 | entry: admin 15 | roles: [] 16 | -------------------------------------------------------------------------------- /docs/examples-filesystem/backup-repository/v1/secrets/backup-repository-collection-secrets.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: backup-repository-collection-secrets 6 | type: Opaque 7 | data: 8 | # to generate: use echo -n "admin" | sha256sum 9 | iwa-ait: "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918" 10 | -------------------------------------------------------------------------------- /docs/examples-filesystem/backup-repository/v1/secrets/backup-repository-passwords.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: backup-repository-passwords 6 | type: Opaque 7 | data: 8 | # admin: admin 9 | # to generate: `backup-repository --encode-password "admin" 10 | admin: "JGFyZ29uMmlkJHY9MTkkbT02NTUzNix0PTEscD00JHpuVy9IT2Y4Q3RkdStvNSttYlR2REE9PSRaZlVpRGl2QWV2T2RZNndKYWJBb0FQdmM1a1hsemxDNkg2OFY2dGVmNUY0PQ==" 11 | # to generate: `backup-repository --encode-password "admin"` 12 | admin_access_key_1: "JGFyZ29uMmlkJHY9MTkkbT02NTUzNix0PTEscD00JERzUzlPTzFOc0JVREhvR1RmQ01wemc9PSRrcXh4bFliS3A4Um81MXZEb0FQaUdBeFhkNTgrY1ZzdERyZ3p3NG16bjFVPQ==" 13 | -------------------------------------------------------------------------------- /docs/examples/collection.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: backups.riotkit.org/v1alpha1 3 | kind: BackupCollection 4 | metadata: 5 | name: iwa-ait 6 | spec: 7 | description: IWA-AIT website files 8 | filenameTemplate: iwa-ait-${version}.tar.gz 9 | maxBackupsCount: 5 10 | maxOneVersionSize: 1M 11 | maxCollectionSize: 10M 12 | 13 | # optional 14 | windows: 15 | - from: "*/30 * * * *" 16 | duration: 30m 17 | 18 | # fifo, fifo-plus-older 19 | strategyName: fifo 20 | strategySpec: 21 | keepLastOlderNotMoreThan: 5d 22 | maxOlderCopies: 2 23 | 24 | healthSecretRef: 25 | name: backup-repository-collection-secrets 26 | entry: iwa-ait 27 | 28 | accessControl: 29 | - userName: admin 30 | roles: 31 | - collectionManager 32 | -------------------------------------------------------------------------------- /docs/examples/dynamic/access.yaml: -------------------------------------------------------------------------------- 1 | # hash of the JWT 2 | id: 39671096ba800ca9b238c7c01b053fa0d5d09ca3151e050d148ddfffaefa9466ceba75d47c2098a0f72110aea4deeb24f6cd1b31f27e27aa6fe7b82dad956049 3 | user: admin 4 | expiresAt: "2022-01-01 08:00" 5 | active: true 6 | description: "Temporary token for single file upload" 7 | requesterIP: "1.2.3.4" 8 | -------------------------------------------------------------------------------- /docs/examples/dynamic/version.yaml: -------------------------------------------------------------------------------- 1 | name: 7a9ecd01-8dec-4be2-8ece-e2a6047e4ffd 2 | collectionId: iwa-ait 3 | version: 1 4 | filename: iwa-ait-v1.tar.gz 5 | uploadDate: "2022-01-01 06:30" 6 | uploadedBy: admin 7 | -------------------------------------------------------------------------------- /docs/examples/secret.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: backup-repository-passwords 6 | type: Opaque 7 | data: 8 | # admin: admin 9 | # to generate: `backup-repository --encode-password "admin"` 10 | admin: "JGFyZ29uMmlkJHY9MTkkbT02NTUzNix0PTEscD00JHpuVy9IT2Y4Q3RkdStvNSttYlR2REE9PSRaZlVpRGl2QWV2T2RZNndKYWJBb0FQdmM1a1hsemxDNkg2OFY2dGVmNUY0PQ==" 11 | # to generate: `backup-repository --encode-password "admin"` 12 | admin_access_key_1: "JGFyZ29uMmlkJHY9MTkkbT02NTUzNix0PTEscD00JERzUzlPTzFOc0JVREhvR1RmQ01wemc9PSRrcXh4bFliS3A4Um81MXZEb0FQaUdBeFhkNTgrY1ZzdERyZ3p3NG16bjFVPQ==" 13 | 14 | --- 15 | apiVersion: v1 16 | kind: Secret 17 | metadata: 18 | name: backup-repository-collection-secrets 19 | type: Opaque 20 | data: 21 | # to generate: use echo -n "admin" | sha256sum 22 | iwa-ait: "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918" 23 | -------------------------------------------------------------------------------- /docs/examples/user.second-actor.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: backups.riotkit.org/v1alpha1 3 | kind: BackupUser 4 | metadata: 5 | name: some-user 6 | spec: 7 | email: user@riseup.net 8 | deactivated: false 9 | organization: "Riotkit" 10 | about: "Example user" 11 | password: "" 12 | passwordFromRef: 13 | name: backup-repository-passwords 14 | entry: admin 15 | accessKeys: 16 | # 17 | # login: some-user$uploader 18 | # password: test 19 | # 20 | - name: uploader 21 | # password: "" 22 | passwordFromRef: 23 | name: backup-repository-passwords 24 | entry: admin_access_key_1 25 | objects: 26 | - name: iwa-ait 27 | type: collection 28 | roles: ["backupUploader"] 29 | roles: 30 | - collectionManager 31 | -------------------------------------------------------------------------------- /docs/examples/user.without-any-roles.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: backups.riotkit.org/v1alpha1 3 | kind: BackupUser 4 | metadata: 5 | name: unprivileged 6 | spec: 7 | email: unprivileged@example.org 8 | deactivated: false 9 | organization: "Riotkit" 10 | about: "Example unprivileged user" 11 | password: "" 12 | passwordFromRef: 13 | name: backup-repository-passwords 14 | entry: admin 15 | roles: [] 16 | -------------------------------------------------------------------------------- /docs/examples/user.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: backups.riotkit.org/v1alpha1 3 | kind: BackupUser 4 | metadata: 5 | name: admin 6 | spec: 7 | email: riotkit@riseup.net 8 | deactivated: false 9 | organization: "Riotkit" 10 | about: "System administrator" 11 | password: "hashed" 12 | passwordFromRef: # alternatively, fetch from `kind: Secret` 13 | name: backup-repository-passwords 14 | entry: admin 15 | restrictByIP: 16 | - 1.2.3.4 17 | roles: 18 | - collectionManager 19 | - usersManager 20 | - systemAdmin 21 | -------------------------------------------------------------------------------- /docs/installing.md: -------------------------------------------------------------------------------- 1 | Installing 2 | ========== 3 | 4 | In Kubernetes 5 | ------------- 6 | 7 | Before you begin make sure you have PostgreSQL and object storage. 8 | 9 | We recommend those Helm Charts for PostgreSQL and object storage if you do not have one: 10 | - [PostgreSQL](https://artifacthub.io/packages/helm/bitnami/postgresql) 11 | - [Min.io](https://artifacthub.io/packages/helm/minio/minio) 12 | 13 | Use Helm to import and install Backup Repository Helm Chart. 14 | 15 | ```bash 16 | helm repo add riotkit-org https://riotkit-org.github.io/helm-of-revolution/ 17 | helm upgrade --install backups riotkit-org/backup-repository-server -n backup-repository --values values.yaml 18 | ``` 19 | 20 | Example `values.yaml` file: 21 | 22 | ```yaml 23 | secrets: 24 | BR_JWT_SECRET_KEY: "87GHq66A+uGkcn/AyxrnPYdd5F0XUmGlHsREbY3tcM4CpO6/dFL7z/057DHnp9nMdoYOpKxwYWrM9XyffjrBidm6/VCzfam9GwMlOac7TsidcTnSHG5IasPICb9bKE3h" # MANDATORY 25 | BR_DB_HOSTNAME: "postgres-postgresql.backup-repository.svc.cluster.local" 26 | BR_DB_PASSWORD: "putinchuj" 27 | BR_DB_USERNAME: "riotkit" 28 | BR_DB_NAME: "backup-repository" 29 | BR_DB_PORT: "5432" 30 | 31 | env: 32 | AWS_SECRET_KEY: "wJaFuCKtnFEMI/CApItaliSM/bPxRfiCYEXAMPLEKEY" 33 | AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" 34 | GIN_MODE: debug 35 | 36 | ingress: 37 | enabled: true 38 | className: "" 39 | annotations: {} 40 | hosts: 41 | - host: backup-repository.example.org 42 | paths: 43 | - path: / 44 | pathType: ImplementationSpecific 45 | ``` 46 | 47 | ### Using with Ingress-Nginx 48 | 49 | In order to use with Ingress-Nginx you may want to adjust a few configuration options, to allow large file uploads. 50 | 51 | ```yaml 52 | # `kind: Ingress` annotations (.ingress.annotations in Helm) 53 | nginx.ingress.kubernetes.io/proxy-request-buffering: "off" 54 | nginx.ingress.kubernetes.io/proxy-buffering: "off" 55 | nginx.ingress.kubernetes.io/proxy-body-size: "0" 56 | ``` 57 | 58 | Bare-metal, no Docker, no Kubernetes 59 | ------------------------------------ 60 | 61 | **Requirements:** 62 | - PostgreSQL 63 | - Min.io or cloud storage 64 | 65 | ### Setting up Min.io 66 | 67 | Check [Min.io quickstart](https://docs.min.io/minio/baremetal/#quickstart) for the instructions on how to prepare the storage instance. 68 | 69 | ### Setting up PostgreSQL 70 | 71 | 1. [Install PostgreSQL](https://www.postgresql.org/docs/14/install-binaries.html) 72 | 2. [Configure pg_hba.conf](https://www.postgresql.org/docs/14/auth-pg-hba-conf.html) to make sure you can login using IP address instead of UNIX socket (Backup Repository uses TCP/IP connection mode) 73 | 3. [Create database](https://www.postgresql.org/docs/14/manage-ag-createdb.html) 74 | 75 | **Alternatively if you would like to use Docker to set up just the PostgreSQL, its easier:** 76 | 77 | ```bash 78 | docker run -d \ 79 | --name br_postgres \ 80 | -e POSTGRES_PASSWORD=postgres \ 81 | -e POSTGRES_USER=postgres \ 82 | -e POSTGRES_DB=postgres \ 83 | -e PGDATA=/var/lib/postgresql/data/pgdata \ 84 | -v $$(pwd)/postgres-data:/var/lib/postgresql \ 85 | -p 5432:5432 \ 86 | postgres:14.1-alpine 87 | ``` 88 | 89 | ### Setting up Backup Repository 90 | 91 | 1. Download `backup-repository` binary from [Releases tab](https://github.com/riotkit-org/backup-repository/releases) for selected stable version. 92 | 93 | 2. Prepare configuration directory 94 | 95 | Your configuration directory needs to have a proper structure. Every file is expected to be at given path according to the following pattern: 96 | 97 | ```bash 98 | # for Backup Repository resources 99 | {.metadata.namespace}/{.apiGroup}/{.apiVersion}/{kind}/{.metadata.name}.yaml 100 | 101 | # for Secrets and ConfigMaps 102 | {.metadata.namespace}/{.apiVersion}/{kind}/{.metadata.name}.yaml 103 | ``` 104 | 105 | Example structure: 106 | 107 | ``` 108 | └── backup-repository 109 | ├── backups.riotkit.org 110 | │ └── v1alpha1 111 | │ ├── backupcollections 112 | │ │ └── iwa-ait.yaml 113 | │ └── backupusers 114 | │ ├── admin.yaml 115 | │ ├── some-user.yaml 116 | │ └── unprivileged.yaml 117 | └── v1 118 | └── secrets 119 | ├── backup-repository-collection-secrets.yaml 120 | └── backup-repository-passwords.yaml 121 | ``` 122 | 123 | 4. Run unpacked binary 124 | 125 | ```bash 126 | # min.io credentials 127 | export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE; 128 | export AWS_SECRET_ACCESS_KEY=wJaFuCKtnFEMI/CApItaliSM/bPxRfiCYEXAMPLEKEY; 129 | 130 | ./backup-repository \ 131 | --db-hostname=127.0.0.1 \ 132 | --db-port=5432 \ 133 | --db-password=postgres \ 134 | --db-user=postgres \ 135 | --db-password=postgres \ 136 | --db-name=postgres \ 137 | --health-check-key=changeme \ 138 | --jwt-secret-key="secret key" \ 139 | --storage-io-timeout="5m" \ 140 | --listen=":8080" \ 141 | --provider=filesystem \ 142 | --config-local-path=./my-config-directory \ 143 | --storage-url="s3://mybucket?endpoint=localhost:9000&disableSSL=true&s3ForcePathStyle=true®ion=eu-central-1" 144 | ``` 145 | -------------------------------------------------------------------------------- /env.mk: -------------------------------------------------------------------------------- 1 | SUDO= 2 | 3 | ifneq (,$(wildcard ./.env)) 4 | include .env 5 | export 6 | endif 7 | 8 | # default values 9 | ENV_CLUSTER_NAME ?= "rkt" 10 | ENV_NS ?= "default" 11 | ENV_APP_SVC ?= "service-name" 12 | ENV_PORT_FORWARD ?= "8050:8080" 13 | 14 | 15 | .EXPORT_ALL_VARIABLES: 16 | PATH = $(shell pwd)/.build:$(shell echo $$PATH) 17 | KUBECONFIG = $(shell echo "$$HOME/.k3d/kubeconfig-${ENV_CLUSTER_NAME}.yaml") 18 | 19 | k3d: prepare-tools 20 | (${SUDO} docker ps | grep k3d-${ENV_CLUSTER_NAME}-server-0 > /dev/null 2>&1) || ${SUDO} k3d cluster create ${ENV_CLUSTER_NAME} --registry-create ${ENV_CLUSTER_NAME}-registry:0.0.0.0:5000 --agents 0 21 | k3d kubeconfig merge ${ENV_CLUSTER_NAME} 22 | kubectl create ns ${ENV_NS} || true 23 | cat /etc/hosts | grep "${ENV_CLUSTER_NAME}-registry" > /dev/null || (sudo /bin/bash -c "echo '127.0.0.1 ${ENV_CLUSTER_NAME}-registry' >> /etc/hosts") 24 | 25 | prepare-tools: ## Installs required tools 26 | mkdir -p .build 27 | # skaffold 28 | @test -f ./.build/skaffold || (curl -sL https://storage.googleapis.com/skaffold/releases/v2.2.0/skaffold-linux-amd64 --output ./.build/skaffold && chmod +x ./.build/skaffold) 29 | # kubectl 30 | @test -f ./.build/kubectl || (curl -sL https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl --output ./.build/kubectl && chmod +x ./.build/kubectl) 31 | # k3d 32 | @test -f ./.build/k3d || (curl -sL https://github.com/k3d-io/k3d/releases/download/v5.4.6/k3d-linux-amd64 --output ./.build/k3d && chmod +x ./.build/k3d) 33 | # helm 34 | @test -f ./.build/helm || (curl -sL https://get.helm.sh/helm-v3.11.2-linux-amd64.tar.gz --output /tmp/helm.tar.gz && tar xf /tmp/helm.tar.gz -C /tmp && mv /tmp/linux-amd64/helm ./.build/helm && chmod +x ./.build/helm) 35 | # kubens 36 | @test -f ./.build/kubens || (curl -sL https://raw.githubusercontent.com/ahmetb/kubectx/master/kubens --output ./.build/kubens && chmod +x ./.build/kubens) 37 | 38 | skaffold-deploy: prepare-tools ## Deploys app with dependencies using Skaffold 39 | skaffold deploy -p deps 40 | skaffold build -p app --tag e2e --default-repo ${ENV_CLUSTER_NAME}-registry:5000 --push --insecure-registry ${ENV_CLUSTER_NAME}-registry:5000 --disable-multi-platform-build=true --detect-minikube=false --cache-artifacts=false 41 | skaffold deploy -p app --tag e2e --assume-yes=true --default-repo ${ENV_CLUSTER_NAME}-registry:5000 42 | 43 | export KUBECONFIG=~/.k3d/kubeconfig-${ENV_CLUSTER_NAME}.yaml 44 | killall kubectl || true 45 | kubectl port-forward svc/${ENV_APP_SVC} -n ${ENV_NS} ${ENV_PORT_FORWARD} & 46 | 47 | dev: ## Runs the development environment in Kubernetes 48 | skaffold deploy -p deps 49 | skaffold dev -p app --tag e2e --assume-yes=true --default-repo ${ENV_CLUSTER_NAME}-registry:5000 --force=true 50 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/riotkit-org/backup-repository 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/appleboy/gin-jwt/v2 v2.9.1 7 | github.com/fatih/structs v1.1.0 8 | github.com/gin-contrib/timeout v0.0.3 9 | github.com/gin-gonic/gin v1.9.1 10 | github.com/google/uuid v1.3.0 11 | github.com/jessevdk/go-flags v1.5.0 12 | github.com/julianshen/gin-limiter v0.0.0-20161123033831-fc39b5e90fe7 13 | github.com/labstack/gommon v0.3.1 14 | github.com/pkg/errors v0.9.1 15 | github.com/robfig/cron/v3 v3.0.1 16 | github.com/sirupsen/logrus v1.8.1 17 | github.com/stretchr/testify v1.8.4 18 | github.com/tidwall/gjson v1.14.3 19 | gocloud.dev v0.25.0 20 | golang.org/x/crypto v0.14.0 21 | golang.org/x/net v0.17.0 22 | gorm.io/driver/postgres v1.3.4 23 | gorm.io/gorm v1.23.5 24 | k8s.io/apimachinery v0.28.3 25 | k8s.io/client-go v0.28.3 26 | k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 27 | sigs.k8s.io/controller-runtime v0.12.0 28 | ) 29 | 30 | require ( 31 | cloud.google.com/go v0.100.2 // indirect 32 | cloud.google.com/go/compute v1.5.0 // indirect 33 | cloud.google.com/go/iam v0.3.0 // indirect 34 | cloud.google.com/go/storage v1.21.0 // indirect 35 | github.com/aws/aws-sdk-go v1.43.31 // indirect 36 | github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect 37 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1 // indirect 38 | github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect 39 | github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect 40 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect 41 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.3 // indirect 42 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect 43 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect 44 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect 45 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1 // indirect 46 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.3 // indirect 47 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect 48 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.3 // indirect 49 | github.com/aws/aws-sdk-go-v2/service/s3 v1.26.3 // indirect 50 | github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect 51 | github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect 52 | github.com/aws/smithy-go v1.11.2 // indirect 53 | github.com/bytedance/sonic v1.9.1 // indirect 54 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect 55 | github.com/davecgh/go-spew v1.1.1 // indirect 56 | github.com/gabriel-vasile/mimetype v1.4.2 // indirect 57 | github.com/gin-contrib/sse v0.1.0 // indirect 58 | github.com/go-logr/logr v1.2.4 // indirect 59 | github.com/go-playground/locales v0.14.1 // indirect 60 | github.com/go-playground/universal-translator v0.18.1 // indirect 61 | github.com/go-playground/validator/v10 v10.14.0 // indirect 62 | github.com/goccy/go-json v0.10.2 // indirect 63 | github.com/gogo/protobuf v1.3.2 // indirect 64 | github.com/golang-jwt/jwt/v4 v4.4.3 // indirect 65 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 66 | github.com/golang/protobuf v1.5.3 // indirect 67 | github.com/google/go-cmp v0.5.9 // indirect 68 | github.com/google/gofuzz v1.2.0 // indirect 69 | github.com/google/wire v0.5.0 // indirect 70 | github.com/googleapis/gax-go/v2 v2.2.0 // indirect 71 | github.com/imdario/mergo v0.3.12 // indirect 72 | github.com/jackc/chunkreader/v2 v2.0.1 // indirect 73 | github.com/jackc/pgconn v1.11.0 // indirect 74 | github.com/jackc/pgio v1.0.0 // indirect 75 | github.com/jackc/pgpassfile v1.0.0 // indirect 76 | github.com/jackc/pgproto3/v2 v2.2.0 // indirect 77 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect 78 | github.com/jackc/pgtype v1.10.0 // indirect 79 | github.com/jackc/pgx/v4 v4.15.0 // indirect 80 | github.com/jinzhu/inflection v1.0.0 // indirect 81 | github.com/jinzhu/now v1.1.4 // indirect 82 | github.com/jmespath/go-jmespath v0.4.0 // indirect 83 | github.com/json-iterator/go v1.1.12 // indirect 84 | github.com/juju/ratelimit v1.0.1 // indirect 85 | github.com/klauspost/cpuid/v2 v2.2.4 // indirect 86 | github.com/leodido/go-urn v1.2.4 // indirect 87 | github.com/mattn/go-isatty v0.0.19 // indirect 88 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 89 | github.com/modern-go/reflect2 v1.0.2 // indirect 90 | github.com/pelletier/go-toml/v2 v2.0.8 // indirect 91 | github.com/pmezard/go-difflib v1.0.0 // indirect 92 | github.com/spf13/pflag v1.0.5 // indirect 93 | github.com/tidwall/match v1.1.1 // indirect 94 | github.com/tidwall/pretty v1.2.0 // indirect 95 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 96 | github.com/ugorji/go/codec v1.2.11 // indirect 97 | go.opencensus.io v0.23.0 // indirect 98 | golang.org/x/arch v0.3.0 // indirect 99 | golang.org/x/oauth2 v0.8.0 // indirect 100 | golang.org/x/sys v0.13.0 // indirect 101 | golang.org/x/term v0.13.0 // indirect 102 | golang.org/x/text v0.13.0 // indirect 103 | golang.org/x/time v0.3.0 // indirect 104 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 105 | google.golang.org/api v0.74.0 // indirect 106 | google.golang.org/appengine v1.6.7 // indirect 107 | google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de // indirect 108 | google.golang.org/grpc v1.45.0 // indirect 109 | google.golang.org/protobuf v1.30.0 // indirect 110 | gopkg.in/inf.v0 v0.9.1 // indirect 111 | gopkg.in/yaml.v2 v2.4.0 // indirect 112 | gopkg.in/yaml.v3 v3.0.1 // indirect 113 | k8s.io/klog/v2 v2.100.1 // indirect 114 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 115 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 116 | sigs.k8s.io/yaml v1.3.0 // indirect 117 | ) 118 | -------------------------------------------------------------------------------- /helm/backup-repository-server/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /helm/backup-repository-server/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: backup-repository-server 3 | description: "Backup storage for E2E GPG-encrypted files, with multi-user, quotas, versioning, using a object storage (S3/Min.io/GCS etc.) and deployed on Kubernetes or standalone." 4 | type: application 5 | version: 0.1.1 6 | appVersion: "latest" # todo 7 | 8 | -------------------------------------------------------------------------------- /helm/backup-repository-server/README.md: -------------------------------------------------------------------------------- 1 | Backup Repository 2 | ================= 3 | 4 | [![Coverage Status](https://coveralls.io/repos/github/riotkit-org/backup-repository/badge.svg?branch=main)](https://coveralls.io/github/riotkit-org/backup-repository?branch=main) 5 | [![Test](https://github.com/riotkit-org/backup-repository/actions/workflows/test.yaml/badge.svg)](https://github.com/riotkit-org/backup-repository/actions/workflows/test.yaml) 6 | [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/riotkit-org)](https://artifacthub.io/packages/search?repo=riotkit-org) 7 | 8 | Cloud-native, zero-knowledge, multi-tenant, security-first backup storage with minimal footprint. 9 | 10 | _TLDR; Primitive backup storage for E2E GPG-encrypted files, with multi-user, quotas, versioning, using a object storage (S3/Min.io/GCS etc.) and deployed on Kubernetes or standalone. No fancy stuff included, lightweight and stable as much as possible is the project target._ 11 | 12 | ```bash 13 | helm repo add riotkit-org https://riotkit-org.github.io/helm-of-revolution/ 14 | helm install backups riotkit-org/backup-repository-server -n backup-repository 15 | ``` 16 | 17 | Documentation 18 | ------------- 19 | 20 | ### [For documentation please look at Github](https://github.com/riotkit-org/backup-repository/blob/main/docs/README.md) 21 | 22 | **NOTICE:** Please consider selecting a versioned tag from branch/tag selector. 23 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/NOTES.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/riotkit-org/backup-repository/66c8b0bde05c55320fa9cbc846d89f06935a968e/helm/backup-repository-server/templates/NOTES.txt -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "backup-repository-server.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "backup-repository-server.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "backup-repository-server.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "backup-repository-server.labels" -}} 37 | helm.sh/chart: {{ include "backup-repository-server.chart" . }} 38 | {{ include "backup-repository-server.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "backup-repository-server.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "backup-repository-server.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "backup-repository-server.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "backup-repository-server.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "backup-repository-server.fullname" . }} 5 | labels: 6 | {{- include "backup-repository-server.labels" . | nindent 8 }} 7 | {{- with .Values.deploymentLabels }} 8 | {{- toYaml . | nindent 8 }} 9 | {{- end }} 10 | spec: 11 | {{- if not .Values.autoscaling.enabled }} 12 | replicas: {{ .Values.replicaCount }} 13 | {{- end }} 14 | selector: 15 | matchLabels: 16 | {{- include "backup-repository-server.selectorLabels" . | nindent 10 }} 17 | template: 18 | metadata: 19 | {{- with .Values.podAnnotations }} 20 | annotations: 21 | {{- toYaml . | nindent 16 }} 22 | {{- end }} 23 | labels: 24 | {{- include "backup-repository-server.selectorLabels" . | nindent 16 }} 25 | {{- if eq .Values.image.tag "latest" | or (eq .Values.image.tag "snapshot") }} 26 | refreshImageTag: "{{- randAlphaNum 24 | nospace -}}" 27 | {{- end }} 28 | {{- with .Values.podLabels }} 29 | {{- toYaml . | nindent 16 }} 30 | {{- end }} 31 | spec: 32 | # todo: implement termination procedure, so the pending upload would block from termination 33 | # and terminating container will block from taking new uploads 34 | terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} 35 | {{- with .Values.imagePullSecrets }} 36 | imagePullSecrets: 37 | {{- toYaml . | nindent 16 }} 38 | {{- end }} 39 | serviceAccountName: {{ include "backup-repository-server.serviceAccountName" . }} 40 | 41 | {{- with .Values.podSecurityContext }} 42 | securityContext: 43 | {{- toYaml . | nindent 14 }} 44 | {{- end }} 45 | containers: 46 | - name: {{ .Chart.Name }} 47 | securityContext: 48 | {{- toYaml .Values.securityContext | nindent 20 }} 49 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 50 | imagePullPolicy: {{ .Values.image.pullPolicy }} 51 | 52 | args: 53 | - "--health-check-key={{ .Values.settings.healthCode }}" 54 | - "--provider=kubernetes" 55 | - "--namespace={{ .Release.Namespace }}" 56 | 57 | {{- with .Values.env }} 58 | env: 59 | {{- range $key, $value := . }} 60 | - name: {{ $key }} 61 | value: "{{ $value }}" 62 | {{- end }} 63 | {{- end }} 64 | 65 | envFrom: 66 | - secretRef: 67 | name: {{ .Values.secrets.name }} 68 | optional: false 69 | 70 | ports: 71 | - name: http 72 | containerPort: 8080 73 | protocol: TCP 74 | 75 | {{- if .Values.health.liveness.enabled }} 76 | livenessProbe: 77 | {{- with .Values.health.liveness.attributes}} 78 | {{- toYaml . | nindent 22 }} 79 | {{- end}} 80 | httpGet: 81 | path: /health 82 | port: http 83 | httpHeaders: 84 | - name: Authorization 85 | value: "{{ .Values.settings.healthCode }}" 86 | {{- end }} 87 | {{- if .Values.health.readiness.enabled }} 88 | readinessProbe: 89 | {{- with .Values.health.readiness.attributes }} 90 | {{- toYaml . | nindent 22 }} 91 | {{- end }} 92 | httpGet: 93 | path: /ready 94 | port: http 95 | httpHeaders: 96 | - name: Authorization 97 | value: "{{ .Values.settings.healthCode }}" 98 | {{- end }} 99 | resources: 100 | {{- toYaml .Values.resources | nindent 20 }} 101 | {{- with .Values.nodeSelector }} 102 | nodeSelector: 103 | {{- toYaml . | nindent 14 }} 104 | {{- end }} 105 | {{- with .Values.affinity }} 106 | affinity: 107 | {{- toYaml . | nindent 14 }} 108 | {{- end }} 109 | {{- with .Values.tolerations }} 110 | tolerations: 111 | {{- toYaml . | nindent 14 }} 112 | {{- end }} 113 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/hpa.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.autoscaling.enabled }} 2 | apiVersion: autoscaling/v2beta1 3 | kind: HorizontalPodAutoscaler 4 | metadata: 5 | name: {{ include "backup-repository-server.fullname" . }} 6 | labels: 7 | {{- include "backup-repository-server.labels" . | nindent 6 }} 8 | spec: 9 | scaleTargetRef: 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | name: {{ include "backup-repository-server.fullname" . }} 13 | minReplicas: {{ .Values.autoscaling.minReplicas }} 14 | maxReplicas: {{ .Values.autoscaling.maxReplicas }} 15 | metrics: 16 | {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} 17 | - type: Resource 18 | resource: 19 | name: cpu 20 | targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} 21 | {{- end }} 22 | {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} 23 | - type: Resource 24 | resource: 25 | name: memory 26 | targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} 27 | {{- end }} 28 | {{- end }} 29 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "backup-repository-server.fullname" . -}} 3 | {{- $svcPort := .Values.service.port -}} 4 | {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} 5 | {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} 6 | {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} 7 | {{- end }} 8 | {{- end }} 9 | {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} 10 | apiVersion: networking.k8s.io/v1 11 | {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 12 | apiVersion: networking.k8s.io/v1beta1 13 | {{- else -}} 14 | apiVersion: extensions/v1beta1 15 | {{- end }} 16 | kind: Ingress 17 | metadata: 18 | name: {{ $fullName }} 19 | labels: 20 | {{- include "backup-repository-server.labels" . | nindent 6 }} 21 | {{- with .Values.ingress.annotations }} 22 | annotations: 23 | {{- toYaml . | nindent 6 }} 24 | {{- end }} 25 | spec: 26 | {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} 27 | ingressClassName: {{ .Values.ingress.className }} 28 | {{- end }} 29 | {{- if .Values.ingress.tls }} 30 | tls: 31 | {{- range .Values.ingress.tls }} 32 | - hosts: 33 | {{- range .hosts }} 34 | - {{ . | quote }} 35 | {{- end }} 36 | secretName: {{ .secretName | default $fullName }} 37 | {{- end }} 38 | {{- end }} 39 | rules: 40 | {{- range .Values.ingress.hosts }} 41 | - host: {{ .host | quote }} 42 | http: 43 | paths: 44 | {{- range .paths }} 45 | - path: {{ .path }} 46 | {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} 47 | pathType: {{ .pathType }} 48 | {{- end }} 49 | backend: 50 | {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} 51 | service: 52 | name: {{ $fullName }} 53 | port: 54 | number: {{ $svcPort }} 55 | {{- else }} 56 | serviceName: {{ $fullName }} 57 | servicePort: {{ $svcPort }} 58 | {{- end }} 59 | {{- end }} 60 | {{- end }} 61 | {{- end }} 62 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: Role 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: {{ include "backup-repository-server.fullname" . }}-instance-admin 6 | rules: 7 | - apiGroups: 8 | - "backups.riotkit.org" 9 | resources: 10 | - backupcollections 11 | - backupusers 12 | verbs: ["*"] 13 | - apiGroups: ["*"] 14 | resources: 15 | - secrets 16 | verbs: 17 | - get 18 | - list 19 | 20 | --- 21 | kind: RoleBinding 22 | apiVersion: rbac.authorization.k8s.io/v1 23 | metadata: 24 | name: {{ include "backup-repository-server.fullname" . }}-instance-admin 25 | roleRef: 26 | apiGroup: rbac.authorization.k8s.io 27 | kind: Role 28 | name: {{ include "backup-repository-server.fullname" . }}-instance-admin 29 | subjects: 30 | - kind: ServiceAccount 31 | name: {{ include "backup-repository-server.serviceAccountName" . }} 32 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/route.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.route.enabled -}} 2 | {{- with .Values.route -}} 3 | --- 4 | apiVersion: route.openshift.io/v1 5 | kind: Route 6 | metadata: 7 | name: {{ include "backup-repository-server.fullname" . }} 8 | spec: 9 | host: {{ .host }} 10 | port: 11 | targetPort: http 12 | to: 13 | kind: Service 14 | name: {{ include "backup-repository-server.fullname" . }} 15 | weight: 100 16 | wildcardPolicy: None 17 | {{- with .yaml -}} 18 | {{- toYaml . | nindent 4 }} 19 | {{- end }} 20 | {{- end }} 21 | {{- end }} 22 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/sealedsecret.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.secrets.create }} 2 | {{ if eq .Values.secrets.type "sealedSecrets" }} 3 | --- 4 | apiVersion: bitnami.com/v1alpha1 5 | kind: SealedSecret 6 | metadata: 7 | name: {{ .Values.secrets.name }} 8 | spec: 9 | {{- with .Values.secrets.spec }} 10 | encryptedData: 11 | {{- range $key, $value := . }} 12 | {{ $key }}: "{{ $value }}" 13 | {{- end }} 14 | {{- end }} 15 | 16 | 17 | {{ end }} 18 | {{- end }} 19 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.secrets.create }} 2 | {{ if eq .Values.secrets.type "plain" }} 3 | --- 4 | apiVersion: v1 5 | kind: Secret 6 | metadata: 7 | name: {{ .Values.secrets.name }} 8 | {{- with .Values.secrets.spec }} 9 | data: 10 | {{- range $key, $value := . }} 11 | {{ $key }}: "{{ $value | b64enc }}" 12 | {{- end }} 13 | {{- end }} 14 | 15 | {{ end }} 16 | 17 | # {{ required "You need to specify .secrets.spec.BR_JWT_SECRET_KEY" .Values.secrets.spec.BR_JWT_SECRET_KEY }} 18 | {{- end }} 19 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/service.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ include "backup-repository-server.fullname" . }} 6 | labels: 7 | {{- include "backup-repository-server.labels" . | nindent 8 }} 8 | spec: 9 | type: {{ .Values.service.type }} 10 | ports: 11 | - port: {{ .Values.service.port }} 12 | targetPort: http 13 | protocol: TCP 14 | name: http 15 | selector: 16 | {{- include "backup-repository-server.selectorLabels" . | nindent 8 }} 17 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "backup-repository-server.serviceAccountName" . }} 6 | labels: 7 | {{- include "backup-repository-server.labels" . | nindent 8 }} 8 | {{- with .Values.serviceAccount.annotations }} 9 | annotations: 10 | {{- toYaml . | nindent 8 }} 11 | {{- end }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /helm/backup-repository-server/templates/tests/test-connection.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: "{{ include "backup-repository-server.fullname" . }}-test-connection" 5 | labels: 6 | {{- include "backup-repository-server.labels" . | nindent 4 }} 7 | annotations: 8 | "helm.sh/hook": test 9 | spec: 10 | containers: 11 | - name: wget 12 | image: busybox 13 | command: ['wget'] 14 | args: ['{{ include "backup-repository-server.fullname" . }}:{{ .Values.service.port }}'] 15 | restartPolicy: Never 16 | -------------------------------------------------------------------------------- /helm/backup-repository-server/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for backup-repository-server. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 1 6 | terminationGracePeriodSeconds: 300 7 | 8 | settings: 9 | healthCode: changeme 10 | 11 | # supported values: sealedSecrets, plain 12 | # when using sealedSecrets please paste already ENCRYPTED secrets into `.secrets` section 13 | secretsType: plain 14 | 15 | secrets: 16 | name: backup-repository-secret-env 17 | type: plain # or "sealedSecrets" 18 | create: true 19 | spec: 20 | BR_JWT_SECRET_KEY: "changeme-important!" # MANDATORY 21 | BR_DB_HOSTNAME: "postgresql.db.svc.cluster.local" 22 | BR_DB_PASSWORD: "postgres" 23 | BR_DB_USERNAME: "postgres" 24 | BR_DB_NAME: "backup-repository" 25 | BR_DB_PORT: "5432" 26 | 27 | env: 28 | BR_LOG_LEVEL: info 29 | # BR_USE_GOOGLE_CLOUD: true 30 | #BR_STORAGE_DRIVER_URL: "s3://backups?endpoint=minio.backup-repository.svc.cluster.local:9000&disableSSL=true&s3ForcePathStyle=true®ion=eu-central-1" 31 | 32 | image: 33 | repository: ghcr.io/riotkit-org/backup-repository 34 | pullPolicy: Always 35 | # Overrides the image tag whose default is the chart appVersion. 36 | tag: "" 37 | 38 | imagePullSecrets: [] 39 | nameOverride: "" 40 | fullnameOverride: "" 41 | 42 | serviceAccount: 43 | create: true 44 | # Annotations to add to the service account 45 | annotations: {} 46 | # The name of the service account to use. 47 | # If not set and create is true, a name is generated using the fullname template 48 | name: "" 49 | 50 | podAnnotations: {} 51 | podSecurityContext: {} 52 | 53 | securityContext: 54 | capabilities: 55 | drop: ["ALL"] 56 | readOnlyRootFilesystem: true 57 | runAsNonRoot: true 58 | # runAsUser: 1000 59 | 60 | service: 61 | type: ClusterIP 62 | port: 8080 63 | 64 | ingress: 65 | enabled: false 66 | className: "" 67 | annotations: {} 68 | hosts: 69 | - host: backups.example.org 70 | paths: 71 | - path: / 72 | pathType: ImplementationSpecific 73 | tls: [] 74 | 75 | route: 76 | enabled: false 77 | host: backups.example.org 78 | yaml: {} # extra settings on spec level e.g. put TLS settings there 79 | 80 | resources: 81 | limits: 82 | cpu: 2 83 | memory: 1Gi 84 | requests: 85 | cpu: 50m 86 | memory: 128Mi 87 | 88 | autoscaling: 89 | enabled: false 90 | minReplicas: 1 91 | maxReplicas: 100 92 | targetCPUUtilizationPercentage: 80 93 | # targetMemoryUtilizationPercentage: 80 94 | 95 | health: 96 | liveness: 97 | enabled: true 98 | attributes: 99 | failureThreshold: 1 100 | readiness: 101 | enabled: true 102 | attributes: {} 103 | 104 | nodeSelector: {} 105 | tolerations: [] 106 | affinity: {} 107 | 108 | deploymentLabels: {} 109 | podLabels: {} 110 | -------------------------------------------------------------------------------- /helm/examples/backup-repository-ci.values.yaml: -------------------------------------------------------------------------------- 1 | secrets: 2 | name: backup-repository-secret-env 3 | type: plain 4 | create: true 5 | spec: 6 | BR_JWT_SECRET_KEY: "87GHq66A+uGkcn/AyxrnPYdd5F0XUmGlHsREbY3tcM4CpO6/dFL7z/057DHnp9nMdoYOpKxwYWrM9XyffjrBidm6/VCzfam9GwMlOac7TsidcTnSHG5IasPICb9bKE3h" # MANDATORY 7 | BR_DB_HOSTNAME: "postgresql.db.svc.cluster.local" 8 | BR_DB_PASSWORD: "putinchuj" 9 | BR_DB_USERNAME: "riotkit" 10 | BR_DB_NAME: "backup-repository" 11 | BR_DB_PORT: "5432" 12 | AWS_SECRET_KEY: "wJaFuCKtnFEMI/CApItaliSM/bPxRfiCYEXAMPLEKEY" 13 | AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" 14 | 15 | env: 16 | GIN_MODE: debug 17 | BR_STORAGE_DRIVER_URL: "s3://backups?endpoint=minio.storage.svc.cluster.local:9000&disableSSL=true&s3ForcePathStyle=true®ion=eu-central-1" 18 | 19 | ingress: 20 | enabled: true 21 | className: "" 22 | annotations: {} 23 | hosts: 24 | - host: backup-repository.localhost 25 | paths: 26 | - path: / 27 | pathType: ImplementationSpecific 28 | 29 | image: 30 | tag: snapshot 31 | 32 | # CI/testing specific 33 | health: 34 | liveness: 35 | enabled: false # disable, so during the tests pod will not be restarted 36 | readiness: 37 | enabled: false # the same as for liveness probe 38 | -------------------------------------------------------------------------------- /helm/examples/minio.values.yaml: -------------------------------------------------------------------------------- 1 | # existingSecret: ... 2 | 3 | users: 4 | - accessKey: "AKIAIOSFODNN7EXAMPLE" 5 | secretKey: "wJaFuCKtnFEMI/CApItaliSM/bPxRfiCYEXAMPLEKEY" 6 | policy: consoleAdmin 7 | 8 | mode: standalone # recommended to use "distributed" for production 9 | 10 | persistence: 11 | enabled: true 12 | size: 10Gi 13 | # existingClaim: "" 14 | # storageClass: "" 15 | 16 | resources: 17 | requests: 18 | memory: 0.1 19 | 20 | buckets: 21 | - name: backups 22 | purge: false 23 | -------------------------------------------------------------------------------- /helm/examples/postgresql.values.yaml: -------------------------------------------------------------------------------- 1 | auth: 2 | username: riotkit 3 | password: putinchuj 4 | database: backup-repository 5 | 6 | architecture: standalone # for production better use "replication" 7 | 8 | 9 | # speed up testing 10 | primary: 11 | terminationGracePeriodSeconds: 5 12 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/jessevdk/go-flags" 5 | "github.com/riotkit-org/backup-repository/pkg/collections" 6 | "github.com/riotkit-org/backup-repository/pkg/concurrency" 7 | "github.com/riotkit-org/backup-repository/pkg/config" 8 | "github.com/riotkit-org/backup-repository/pkg/core" 9 | "github.com/riotkit-org/backup-repository/pkg/db" 10 | "github.com/riotkit-org/backup-repository/pkg/http" 11 | security2 "github.com/riotkit-org/backup-repository/pkg/security" 12 | "github.com/riotkit-org/backup-repository/pkg/storage" 13 | "github.com/riotkit-org/backup-repository/pkg/users" 14 | log "github.com/sirupsen/logrus" 15 | "gorm.io/gorm" 16 | "os" 17 | "time" 18 | ) 19 | 20 | type options struct { 21 | Help bool `short:"h" long:"help" description:"Shows this help message"` 22 | Namespace string `short:"n" long:"namespace" default:"backup-repository" description:"Kubernetes namespace to operate in"` 23 | EncodePasswordAction string `long:"encode-password" description:"Encode a password from CLI instead of running a server"` 24 | HashJWT string `long:"hash-jwt" description:"Generate a hash from JWT"` 25 | DbHostname string `long:"db-hostname" description:"Hostname for database connection" default:"localhost" env:"BR_DB_HOSTNAME"` 26 | DbUsername string `long:"db-user" description:"Username for database connection" env:"BR_DB_USERNAME"` 27 | DbPassword string `long:"db-password" description:"Password for database connection" env:"BR_DB_PASSWORD"` 28 | DbName string `long:"db-name" description:"Database name inside a database" env:"BR_DB_NAME"` 29 | DbPort int `long:"db-port" description:"Database name inside a database" default:"5432" env:"BR_DB_PORT"` 30 | JwtSecretKey string `long:"jwt-secret-key" short:"s" description:"Secret used for generating JSON Web Tokens for authentication" env:"BR_JWT_SECRET_KEY"` 31 | HealthCheckKey string `long:"health-check-key" short:"k" description:"Secret key to access health check endpoint" env:"BR_HEALTH_CHECK_KEY"` 32 | Level string `long:"log-level" description:"Log level" default:"debug" env:"BR_LOG_LEVEL"` 33 | StorageDriverUrl string `long:"storage-url" description:"Storage driver url compatible with GO Cloud (https://gocloud.dev/howto/blob/)" env:"BR_STORAGE_DRIVER_URL"` 34 | IsGCS bool `long:"use-google-cloud" description:"If using Google Cloud Storage, then in --storage-url just type bucket name" env:"BR_USE_GOOGLE_CLOUD"` 35 | 36 | // storage timeouts 37 | StorageHealthTimeout string `long:"storage-health-timeout" description:"Maximum allowed storage ping" default:"5s" env:"BR_STORAGE_HEALTH_TIMEOUT"` 38 | StorageIOTimeout string `long:"storage-io-timeout" description:"Maximum time the server can read/write to storage for a single file. WARNING! If you have very large files you need to consider to increase this value." default:"2h" env:"BR_STORAGE_IO_TIMEOUT"` 39 | 40 | // http timeouts 41 | UploadTimeout string `long:"http-upload-timeout" description:"HTTP upload endpoint timeout" default:"180m" env:"BR_HTTP_UPLOAD_TIMEOUT"` 42 | DownloadTimeout string `long:"http-download-timeout" description:"HTTP download endpoint timeout" default:"180m" env:"BR_HTTP_DOWNLOAD_TIMEOUT"` 43 | 44 | // request rate limit 45 | DefaultRPS int16 `long:"rate-default-limit" description:"Request rate limit for all endpoints (except those that have it's dedicated limit). Unit: requests per second" default:"5"` 46 | AuthRPM int16 `long:"rate-auth-limit" description:"Request rate limit for login/authentication endpoints. Unit: requests per minute" default:"10"` 47 | CollectionHealthRPM int16 `long:"rate-collection-health-limit" description:"Request rate limit for collection's /health endpoint. Unit: requests per minute" default:"10"` 48 | ServerHealthRPM int16 `long:"rate-server-health-limit" description:"Request rate limit for server's /health and /ready endpoints. Unit: requests per minute. WARNING: Be careful in Kubernetes as Kube API also can hit this rate limit and restart your service!" default:"160"` 49 | 50 | ListenAddr string `long:"listen" description:"Address to listen on with HTTP API (e.g. :8080)" default:":8080"` 51 | 52 | // Configuration provider 53 | Provider string `short:"p" long:"provider" description:"Configuration provider. Choice: 'kubernetes', 'filesystem'" default:"kubernetes" env:"BR_CONFIG_PROVIDER"` 54 | ConfigLocalPath string `long:"config-local-path" description:"Configuration path (if using --provider=filesystem)" default:"~/.backup-repository" env:"BR_CONFIG_LOCAL_PATH"` 55 | } 56 | 57 | func main() { 58 | var opts options 59 | p := flags.NewParser(&opts, flags.Default&^flags.HelpFlag) 60 | _, err := p.Parse() 61 | if err != nil { 62 | println(err) 63 | os.Exit(1) 64 | } 65 | logLevel, _ := log.ParseLevel(opts.Level) 66 | log.SetLevel(logLevel) 67 | if opts.Help { 68 | p.WriteHelp(os.Stdout) 69 | os.Exit(0) 70 | } 71 | // allows encoding passwords from CLI to make the configmap creation easier 72 | if opts.EncodePasswordAction != "" { 73 | hash, _ := security2.CreateHashFromPassword(opts.EncodePasswordAction) 74 | println(hash) 75 | os.Exit(0) 76 | } 77 | // allows to hash a JWT, to be later used in comparison in `kind: GrantedAccess` 78 | if opts.HashJWT != "" { 79 | println(security2.HashJWT(opts.HashJWT)) 80 | os.Exit(0) 81 | } 82 | 83 | // 84 | // Application services container is built here 85 | // 86 | 87 | configProvider, err := config.CreateConfigurationProvider(opts.Provider, opts.Namespace, opts.ConfigLocalPath) 88 | if err != nil { 89 | log.Errorln("Cannot initialize Configuration Provider") 90 | log.Fatal(err) 91 | } 92 | dbDriver, err := db.CreateDatabaseDriver(opts.DbHostname, opts.DbUsername, opts.DbPassword, opts.DbName, opts.DbPort, "") 93 | if err != nil { 94 | log.Errorln("Cannot initialize database connection") 95 | log.Fatal(err) 96 | } 97 | locksService := concurrency.NewService(dbDriver) 98 | db.InitializeDatabase(dbDriver) 99 | 100 | usersService := users.NewUsersService(configProvider) 101 | gaService := security2.NewService(dbDriver) 102 | collectionsService := collections.NewService(configProvider) 103 | 104 | ctx := core.ApplicationContainer{ 105 | Db: dbDriver, 106 | Config: &configProvider, 107 | Users: usersService, 108 | GrantedAccesses: &gaService, 109 | JwtSecretKey: opts.JwtSecretKey, 110 | HealthCheckKey: opts.HealthCheckKey, 111 | Collections: &collectionsService, 112 | Storage: createStorage(dbDriver, &opts), 113 | Locks: &locksService, 114 | 115 | // timeouts 116 | UploadTimeout: toDurationOrFatal(opts.UploadTimeout), 117 | DownloadTimeout: toDurationOrFatal(opts.DownloadTimeout), 118 | 119 | // request limit rates 120 | DefaultRPS: opts.DefaultRPS, 121 | AuthRPM: opts.AuthRPM, 122 | CollectionHealthRPM: opts.CollectionHealthRPM, 123 | ServerHealthRPM: opts.ServerHealthRPM, 124 | } 125 | 126 | if err := http.SpawnHttpApplication(&ctx, opts.ListenAddr); err != nil { 127 | log.Errorf("Cannot spawn HTTP server: %v", err) 128 | os.Exit(1) 129 | } 130 | } 131 | 132 | func createStorage(dbDriver *gorm.DB, opts *options) *storage.Service { 133 | healthTimeout, storageTimeoutErr := time.ParseDuration(opts.StorageHealthTimeout) 134 | if storageTimeoutErr != nil { 135 | log.Errorln("Cannot parse --storage-health-timeout duration") 136 | log.Fatal(storageTimeoutErr) 137 | } 138 | 139 | ioTimeout, ioTimeoutErr := time.ParseDuration(opts.StorageIOTimeout) 140 | if ioTimeoutErr != nil { 141 | log.Errorln("Cannot parse --storage-io-timeout duration") 142 | log.Fatal(ioTimeoutErr) 143 | } 144 | 145 | log.Debugf("Creating storage with ioTimeout=%v, healthTimeout=%v", ioTimeout, healthTimeout) 146 | 147 | storageService, storageError := storage.NewService(dbDriver, opts.StorageDriverUrl, opts.IsGCS, healthTimeout, ioTimeout) 148 | if storageError != nil { 149 | log.Errorln("Cannot initialize storage driver") 150 | log.Fatal(storageError) 151 | } 152 | 153 | return &storageService 154 | } 155 | 156 | func toDurationOrFatal(durationStr string) time.Duration { 157 | duration, err := time.ParseDuration(durationStr) 158 | if err != nil { 159 | log.Errorf("Cannot parse %s to duration", durationStr) 160 | log.Fatal(err) 161 | } 162 | return duration 163 | } 164 | -------------------------------------------------------------------------------- /pkg/collections/model.go: -------------------------------------------------------------------------------- 1 | package collections 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "github.com/labstack/gommon/bytes" 8 | "github.com/riotkit-org/backup-repository/pkg/config" 9 | "github.com/riotkit-org/backup-repository/pkg/security" 10 | "github.com/robfig/cron/v3" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | type StrategySpec struct { 16 | KeepLastOlderNotMoreThan string `json:"keepLastOlderNotMoreThan"` 17 | MaxOlderCopies int `json:"maxOlderCopies"` 18 | } 19 | 20 | type BackupWindow struct { 21 | From string `json:"from"` 22 | Duration string `json:"duration"` 23 | 24 | parsed cron.Schedule 25 | parsedDuration time.Duration 26 | } 27 | 28 | // UnmarshalJSON performs a validation when decoding a JSON 29 | func (b *BackupWindow) UnmarshalJSON(in []byte) error { 30 | v := struct { 31 | From string `json:"from"` 32 | Duration string `json:"duration"` 33 | }{} 34 | 35 | if unmarshalErr := json.Unmarshal(in, &v); unmarshalErr != nil { 36 | return unmarshalErr 37 | } 38 | 39 | return b.applyRawAttributes(v.From, v.Duration) 40 | } 41 | 42 | // applyRawAttributes sets construction attributes just like it was unmarshalled from JSON 43 | func (b *BackupWindow) applyRawAttributes(from string, duration string) error { 44 | b.From = from 45 | b.Duration = duration 46 | 47 | parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.DowOptional) 48 | err := errors.New("") 49 | b.parsed, err = parser.Parse(b.From) 50 | if err != nil { 51 | return errors.New(fmt.Sprintf("cannot parse Backup Window: %v. IsError: %v", b.From, err)) 52 | } 53 | 54 | b.parsedDuration, err = time.ParseDuration(b.Duration) 55 | if err != nil { 56 | return errors.New(fmt.Sprintf("cannot parse Backup Window - duation parsing, IsError: %v", err)) 57 | } 58 | 59 | return nil 60 | } 61 | 62 | // NewBackupWindow is a factory method 63 | func NewBackupWindow(from string, duration string) (BackupWindow, error) { 64 | w := BackupWindow{} 65 | err := w.applyRawAttributes(from, duration) 66 | return w, err 67 | } 68 | 69 | // IsInWindowNow checks if given time is between BackupWindow time slot (<-start--O--end->) 70 | func (b *BackupWindow) IsInWindowNow(current time.Time) (bool, error) { 71 | startDate, err := b.GetStartingDateOfPreviousScheduledRun(current) 72 | if err != nil { 73 | return false, err 74 | } 75 | 76 | endDate := startDate.Add(b.parsedDuration) 77 | 78 | // previous run -> previous run + duration 79 | return current.After(startDate) && current.Before(endDate), nil 80 | } 81 | 82 | func (b *BackupWindow) GetStartingDateOfPreviousScheduledRun(current time.Time) (time.Time, error) { 83 | nextRun := b.parsed.Next(current) 84 | 85 | // check if next run is NOW 86 | possibleNextRunNow := current.Add(time.Second * time.Duration(-1)) 87 | if b.parsed.Next(possibleNextRunNow).Format(time.RFC822) != nextRun.Format(time.RFC822) { 88 | nextRun = b.parsed.Next(possibleNextRunNow) 89 | } 90 | 91 | startDate := current 92 | retries := 0 93 | 94 | // First calculate startDate run to get "START DATE" and calculate "END DATE" 95 | // because the library does not provide a "Previous" method unfortunately 96 | for true { 97 | retries = retries + 1 98 | startDate = startDate.Add(time.Minute * time.Duration(-1)) 99 | possiblePrevRun := b.parsed.Next(startDate) 100 | 101 | if possiblePrevRun.Format(time.RFC822) != nextRun.Format(time.RFC822) { 102 | return possiblePrevRun, nil 103 | } 104 | 105 | // 12 months 106 | if retries > 60*24*30*12 { 107 | return time.Time{}, errors.New("cannot find a previous date in the backup window") 108 | } 109 | } 110 | 111 | return time.Time{}, errors.New("unknown error while attempting to find start date for backup window") 112 | } 113 | 114 | func (b *BackupWindow) IsInPreviousWindowTimeSlot(now time.Time, latestVersionCreation time.Time) (bool, error) { 115 | previousRun, err := b.GetStartingDateOfPreviousScheduledRun(now) 116 | if err != nil { 117 | return false, err 118 | } 119 | 120 | endDate := previousRun.Add(b.parsedDuration) 121 | return latestVersionCreation.After(previousRun) && latestVersionCreation.Before(endDate), nil 122 | } 123 | 124 | type BackupWindows []BackupWindow 125 | 126 | type Spec struct { 127 | Description string `json:"description"` 128 | FilenameTemplate string `json:"filenameTemplate"` 129 | MaxBackupsCount int `json:"maxBackupsCount"` 130 | MaxOneVersionSize string `json:"maxOneVersionSize"` 131 | MaxCollectionSize string `json:"maxCollectionSize"` 132 | Windows BackupWindows `json:"windows"` 133 | StrategyName string `json:"strategyName"` 134 | StrategySpec StrategySpec `json:"strategySpec"` 135 | AccessControl security.AccessControlList `json:"accessControl"` 136 | HealthSecretRef security.PasswordFromSecretRef `json:"healthSecretRef"` 137 | } 138 | 139 | type Collection struct { 140 | Metadata config.ObjectMetadata `json:"metadata"` 141 | Spec Spec `json:"spec"` 142 | SecretFromSecret string 143 | } 144 | 145 | func (c *Collection) GetTypeName() string { 146 | return "collection" 147 | } 148 | 149 | func (c *Collection) GetAccessControlList() *security.AccessControlList { 150 | return &c.Spec.AccessControl 151 | } 152 | 153 | func (c *Collection) IsHealthCheckSecretValid(secret string) bool { 154 | // secret is optional 155 | if c.SecretFromSecret == "" { 156 | return true 157 | } 158 | return security.CompareFastCryptoHash(secret, c.SecretFromSecret) 159 | } 160 | 161 | // CanUploadToMe answers if user can add new versions to the collection 162 | func (c *Collection) CanUploadToMe(user security.Actor) bool { 163 | return security.DecideCanDo(&security.DecisionRequest{ 164 | Actor: user, 165 | Subject: c, 166 | Action: security.ActionUpload, 167 | }) 168 | } 169 | 170 | // CanDownloadFromMe answers if user can download versions to this collection 171 | func (c *Collection) CanDownloadFromMe(user security.Actor) bool { 172 | return security.DecideCanDo(&security.DecisionRequest{ 173 | Actor: user, 174 | Subject: c, 175 | Action: security.ActionDownload, 176 | }) 177 | } 178 | 179 | // CanListMyVersions answers if user can list versions 180 | func (c *Collection) CanListMyVersions(user security.Actor) bool { 181 | return c.CanUploadToMe(user) || c.CanDownloadFromMe(user) 182 | } 183 | 184 | func (c *Collection) GenerateNextVersionFilename(version int) string { 185 | return strings.Replace(c.Spec.FilenameTemplate, "${version}", fmt.Sprintf("%v", version), 1) 186 | } 187 | 188 | // getEstimatedDiskSpaceForFullCollectionInBytes returns a calculation how many disk space would be required to store all versions (excluding extra disk space) 189 | // in ideal case it would be: MaxBackupsCount * MaxOneVersionSize 190 | func (c *Collection) getEstimatedDiskSpaceForFullCollectionInBytes() (int64, error) { 191 | maxVersionSizeInBytes, err := c.GetMaxOneVersionSizeInBytes() 192 | if err != nil { 193 | return 0, errors.New(fmt.Sprintf("cannot calculate estimated collection size: %v", err)) 194 | } 195 | return int64(c.Spec.MaxBackupsCount) * maxVersionSizeInBytes, nil 196 | } 197 | 198 | func (c *Collection) GetMaxOneVersionSizeInBytes() (int64, error) { 199 | return bytes.Parse(c.Spec.MaxOneVersionSize) 200 | } 201 | 202 | func (c *Collection) GetCollectionMaxSize() (int64, error) { 203 | return bytes.Parse(c.Spec.MaxCollectionSize) 204 | } 205 | 206 | // GetEstimatedCollectionExtraSpace returns total space that can be extra allocated in case, when a single version exceeds its limit. Returned value is estimated, does not include real state. 207 | func (c *Collection) GetEstimatedCollectionExtraSpace() (int64, error) { 208 | estimatedStandardCollectionSize, err := c.getEstimatedDiskSpaceForFullCollectionInBytes() 209 | if err != nil { 210 | return 0, errors.New(fmt.Sprintf("cannot calculate GetEstimatedCollectionExtraSpace(): %v", err)) 211 | } 212 | maxCollectionSizeInBytes, err := c.getMaxCollectionSizeInBytes() 213 | if err != nil { 214 | return 0, errors.New(fmt.Sprintf("cannot calculate GetEstimatedCollectionExtraSpace(): %v", err)) 215 | } 216 | return maxCollectionSizeInBytes - estimatedStandardCollectionSize, nil 217 | } 218 | 219 | func (c *Collection) getMaxCollectionSizeInBytes() (int64, error) { 220 | return bytes.Parse(c.Spec.MaxCollectionSize) 221 | } 222 | 223 | func (c *Collection) GetId() string { 224 | return c.Metadata.Name 225 | } 226 | 227 | func (c *Collection) GetGlobalIdentifier() string { 228 | return "collection:" + c.GetId() 229 | } 230 | -------------------------------------------------------------------------------- /pkg/collections/model_test.go: -------------------------------------------------------------------------------- 1 | package collections 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | // When e.g. we are 5 minutes after previous run, then returns the closest run that just begun 10 | func TestGetStartingDateOfPreviousScheduledRun_WhenTimeIsJustAfterPreviousRun(t *testing.T) { 11 | window := BackupWindow{} 12 | _ = window.UnmarshalJSON([]byte("{\"from\": \"*/30 * * * *\", \"duration\": \"1h\"}")) 13 | now, _ := time.Parse("2006-01-02 15:04:05", "2022-01-01 01:35:05") // 5 minutes after */30 14 | 15 | previousRunStartDate, _ := window.GetStartingDateOfPreviousScheduledRun(now) 16 | 17 | assert.Equal(t, "2022-01-01 01:30:00 +0000 UTC", previousRunStartDate.String()) 18 | } 19 | 20 | // When it is just that moment, when date equals start date, then return that start date 21 | func TestGetStartingDateOfPreviousScheduledRun_WhenTimeEqualsCurrentRun(t *testing.T) { 22 | window := BackupWindow{} 23 | _ = window.UnmarshalJSON([]byte("{\"from\": \"*/30 * * * *\", \"duration\": \"1h\"}")) 24 | now, _ := time.Parse("2006-01-02 15:04:05", "2022-01-01 01:30:00") 25 | 26 | previousRunStartDate, _ := window.GetStartingDateOfPreviousScheduledRun(now) 27 | 28 | assert.Equal(t, "2022-01-01 01:00:00 +0000 UTC", previousRunStartDate.String()) 29 | } 30 | 31 | func TestIsInWindowNow(t *testing.T) { 32 | window := BackupWindow{} 33 | _ = window.UnmarshalJSON([]byte("{\"from\": \"*/30 * * * *\", \"duration\": \"10m\"}")) 34 | 35 | assert.True(t, isInWindowNow(window, "2022-01-01 01:30:01")) 36 | assert.True(t, isInWindowNow(window, "2022-01-01 01:39:00")) 37 | assert.True(t, isInWindowNow(window, "2022-01-01 01:39:59")) 38 | 39 | assert.False(t, isInWindowNow(window, "2022-01-01 01:45:34")) 40 | } 41 | 42 | func TestGenerateNextVersionFilename(t *testing.T) { 43 | c := Collection{Spec: Spec{FilenameTemplate: "zsp-net-pl-${version}.tar.gz"}} 44 | assert.Equal(t, "zsp-net-pl-1.tar.gz", c.GenerateNextVersionFilename(1)) 45 | } 46 | 47 | func TestGenerateNextVersionFilenameReplacesOnlyOnce(t *testing.T) { 48 | c := Collection{Spec: Spec{FilenameTemplate: "zsp-net-pl-${version}.tar.gz${version}"}} 49 | assert.Equal(t, "zsp-net-pl-1.tar.gz${version}", c.GenerateNextVersionFilename(1)) 50 | } 51 | 52 | func TestGetEstimatedDiskSpaceForFullCollectionInBytes(t *testing.T) { 53 | c := Collection{Spec: Spec{MaxBackupsCount: 30, MaxOneVersionSize: "50G"}} 54 | 55 | calc, _ := c.getEstimatedDiskSpaceForFullCollectionInBytes() 56 | assert.Equal(t, int64(1500*1024*1024*1024), calc) // 50GB * 30 = 1500 GB * 1024mb * 1024kb * 1024b = 1500 GB in bytes 57 | } 58 | 59 | // helper 60 | func isInWindowNow(window BackupWindow, nowStr string) bool { 61 | now, _ := time.Parse("2006-01-02 15:04:05", nowStr) 62 | result, _ := window.IsInWindowNow(now) 63 | 64 | return result 65 | } 66 | -------------------------------------------------------------------------------- /pkg/collections/repository.go: -------------------------------------------------------------------------------- 1 | package collections 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "github.com/riotkit-org/backup-repository/pkg/config" 8 | "github.com/riotkit-org/backup-repository/pkg/security" 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | const KindCollection = "backupcollections" 13 | 14 | type collectionRepository struct { 15 | config config.ConfigurationProvider 16 | } 17 | 18 | // getById Returns a `kind: BackupCollection` object by it's `metadata.name` 19 | func (c *collectionRepository) getById(id string) (*Collection, error) { 20 | doc, retrieveErr := c.config.GetSingleDocument(KindCollection, id) 21 | result := Collection{} 22 | 23 | if retrieveErr != nil { 24 | return &result, errors.New(fmt.Sprintf("error retrieving result: %v", retrieveErr)) 25 | } 26 | 27 | if err := json.Unmarshal([]byte(doc), &result); err != nil { 28 | logrus.Debugln(doc) 29 | return &Collection{}, errors.New(fmt.Sprintf("cannot unmarshal response fron Kubernetes to get collection of id=%v, error: %v", id, err)) 30 | } 31 | 32 | passwordSetter := func(password string) { 33 | result.SecretFromSecret = password 34 | } 35 | if fillErr := security.FillPasswordFromKindSecret(c.config, &result.Spec.HealthSecretRef, passwordSetter); fillErr != nil { 36 | return &Collection{}, fillErr 37 | } 38 | 39 | return &result, nil 40 | } 41 | -------------------------------------------------------------------------------- /pkg/collections/service.go: -------------------------------------------------------------------------------- 1 | package collections 2 | 3 | import ( 4 | "fmt" 5 | "github.com/riotkit-org/backup-repository/pkg/config" 6 | "github.com/sirupsen/logrus" 7 | "time" 8 | ) 9 | 10 | type Service struct { 11 | repository collectionRepository 12 | } 13 | 14 | func (s *Service) GetCollectionById(id string) (*Collection, error) { 15 | return s.repository.getById(id) 16 | } 17 | 18 | func (s *Service) ValidateIsBackupWindowAllowingToUpload(collection *Collection, contextTime time.Time) bool { 19 | // no defined Backup Windows = no limits, ITS OPTIONAL 20 | if len(collection.Spec.Windows) == 0 { 21 | return true 22 | } 23 | 24 | for _, window := range collection.Spec.Windows { 25 | result, err := window.IsInWindowNow(contextTime) 26 | 27 | if err != nil { 28 | logrus.Error(fmt.Sprintf("Backup Window validation error (collection id=%v): %v", collection.Metadata.Name, err)) 29 | } 30 | 31 | if result { 32 | return true 33 | } 34 | } 35 | 36 | return false 37 | } 38 | 39 | func NewService(config config.ConfigurationProvider) Service { 40 | return Service{ 41 | repository: collectionRepository{ 42 | config: config, 43 | }, 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /pkg/concurrency/locking.go: -------------------------------------------------------------------------------- 1 | package concurrency 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "fmt" 7 | "github.com/sirupsen/logrus" 8 | "gorm.io/gorm" 9 | "math/rand" 10 | "time" 11 | ) 12 | 13 | type LocksService struct { 14 | db *gorm.DB 15 | } 16 | 17 | func (ls *LocksService) Lock(id string, howLong time.Duration) (Lock, error) { 18 | if ls.isLockedAlready(id) { 19 | return Lock{}, errors.New("already locked") 20 | } 21 | if err := ls.addLock(id, howLong); err != nil { 22 | return Lock{}, errors.New(fmt.Sprintf("cannot lock transaction, %v", err)) 23 | } 24 | return Lock{ 25 | Id: id, 26 | unlock: func() { 27 | ls.unlock(id) 28 | }, 29 | }, nil 30 | } 31 | 32 | func (ls *LocksService) addLock(id string, howLong time.Duration) error { 33 | expiration := time.Now().Add(howLong) 34 | logrus.Debugf("Locking '%s' until '%v'", id, expiration) 35 | return ls.db.Exec("INSERT INTO locks (id, expires) VALUES (@id, @expires);", sql.Named("id", id), sql.Named("expires", expiration)).Error 36 | } 37 | 38 | func (ls *LocksService) unlock(id string) { 39 | ls.db.Exec("DELETE FROM locks WHERE locks.id = @id", sql.Named("id", id)) 40 | } 41 | 42 | func (ls *LocksService) isLockedAlready(id string) bool { 43 | var result int 44 | ls.db.Raw("SELECT count(*) FROM locks WHERE locks.id = @id AND locks.expires > @now", sql.Named("id", id), sql.Named("now", time.Now())).Scan(&result) 45 | 46 | if ls.shouldPerformCleanUpNow() { 47 | ls.cleanUp() 48 | } 49 | 50 | return result > 0 51 | } 52 | 53 | func (ls *LocksService) cleanUp() { 54 | ls.db.Exec("DELETE FROM locks WHERE locks.expires < @now", sql.Named("now", time.Now())) 55 | } 56 | 57 | func (ls *LocksService) shouldPerformCleanUpNow() bool { 58 | s1 := rand.NewSource(time.Now().UnixNano()) 59 | r1 := rand.New(s1) 60 | 61 | return r1.Intn(5) == 2 // PN-VI 62 | } 63 | 64 | func InitializeModel(db *gorm.DB) error { 65 | return db.AutoMigrate(&Lock{}) 66 | } 67 | 68 | func NewService(db *gorm.DB) LocksService { 69 | return LocksService{db} 70 | } 71 | 72 | type Lock struct { 73 | Id string 74 | Expires time.Time 75 | unlock func() 76 | } 77 | 78 | func (l *Lock) Unlock() { 79 | logrus.Debugf("Releasing lock '%s'", l.Id) 80 | l.unlock() 81 | } 82 | -------------------------------------------------------------------------------- /pkg/config/README.md: -------------------------------------------------------------------------------- 1 | Configuration as a Code module 2 | ============================== 3 | 4 | Stores application configuration in Kubernetes or in local filesystem as YAML files in Kubernetes syntax. 5 | The state is synchronized in both ways. 6 | 7 | Cache 8 | ----- 9 | 10 | Cache layer should be implemented at adapter level and know the source object (fetched from configuration) and target object (saved back into configuration) to clear both from cache. 11 | On each save, deletion, or external modification the cached object should be deleted. 12 | 13 | Cache is a service independent of adapter, it is an implementation shared across adapters. 14 | 15 | 16 | Filesystem adapter 17 | ------------------ 18 | 19 | Works in a directory structure that bases on object types and names. 20 | 21 | **Pattern:** 22 | 23 | ``` 24 | strings.ReplaceAll(o.path+"/"+o.namespace+"/"+apiGroup+"/"+apiVersion+"/"+kind+"/"+id+".yaml", "//", "/") 25 | ``` 26 | 27 | **Examples:** 28 | 29 | ``` 30 | # Example: 31 | # apiVersion: backups.riotkit.org/v1alpha1 32 | # kind: BackupUser 33 | # metadata: 34 | # name: admin 35 | # namespace: default 36 | # 37 | 38 | ./default/backups.riotkit.org_v1alpha1/BackupUser/admin.yaml 39 | 40 | 41 | # Example: 42 | # apiVersion: v1 43 | # kind: Secret 44 | # metadata: 45 | # name: backup-repository-passwords 46 | 47 | ./default/v1/Secret/backup-repository-passwords.yaml 48 | ``` 49 | 50 | #### Immutability 51 | 52 | Like in Kubernetes `apiVersion`, `kind` and `metadata.name` are immutable, in this context - when any of them are changed, then object from application cache should be removed. 53 | 54 | #### Labels 55 | 56 | To be discussed how to implement. 57 | -------------------------------------------------------------------------------- /pkg/config/action.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | // CreateConfigurationProvider is a configuration factory 9 | func CreateConfigurationProvider(providerName string, namespace string, localPath string) (ConfigurationProvider, error) { 10 | if providerName == "kubernetes" { 11 | return CreateKubernetesConfigurationProvider( 12 | namespace, 13 | ), nil 14 | } else if providerName == "filesystem" { 15 | return NewConfigurationInLocalFilesystemProvider( 16 | localPath, 17 | namespace, 18 | ), nil 19 | } 20 | 21 | return nil, errors.New(fmt.Sprintf("Invalid configuration provider name '%v'", providerName)) 22 | } 23 | -------------------------------------------------------------------------------- /pkg/config/action_test.go: -------------------------------------------------------------------------------- 1 | package config_test 2 | 3 | import ( 4 | "github.com/riotkit-org/backup-repository/pkg/config" 5 | "github.com/stretchr/testify/assert" 6 | "os" 7 | "reflect" 8 | "testing" 9 | ) 10 | 11 | // basically checks if filesystem provider can be set up 12 | func TestCreateConfigurationProvider_Filesystem(t *testing.T) { 13 | wd, _ := os.Getwd() 14 | provider, err := config.CreateConfigurationProvider("filesystem", "backup-repository", wd+"/../docs/examples-filesystem") 15 | 16 | assert.Nil(t, err) 17 | assert.Equal(t, "*config.ConfigurationInLocalFilesystem", reflect.TypeOf(provider).String()) 18 | } 19 | 20 | // checks if error is returned, when provider is unknown 21 | func TestCreateConfigurationProvider_UnknownProvider(t *testing.T) { 22 | _, err := config.CreateConfigurationProvider("this-is-not-valid", "backup-repository", "") 23 | assert.NotNil(t, err) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/config/cache.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // todo 4 | -------------------------------------------------------------------------------- /pkg/config/filesystem.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/pkg/errors" 7 | "github.com/sirupsen/logrus" 8 | "io/ioutil" 9 | "k8s.io/apimachinery/pkg/util/yaml" 10 | "os" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | type ConfigurationInLocalFilesystem struct { 16 | path string 17 | namespace string 18 | apiGroup string 19 | apiVersion string 20 | } 21 | 22 | func (fs *ConfigurationInLocalFilesystem) GetHealth() error { 23 | fileName := fmt.Sprintf("%s/.health-%v", fs.path, time.Now().UnixNano()) 24 | defer func() { 25 | _ = os.Remove(fileName) 26 | }() 27 | 28 | if err := ioutil.WriteFile(fileName, []byte(fileName), 0700); err != nil { 29 | return errors.Wrap(err, "The filesystem is not writeable") 30 | } 31 | 32 | content, err := ioutil.ReadFile(fileName) 33 | if err != nil { 34 | return errors.Wrap(err, "Cannot read file that was written to the filesystem") 35 | } 36 | if string(content) != fileName { 37 | return errors.Wrap(err, "Filesystem consistency error, read data does not match wrote data") 38 | } 39 | 40 | return nil 41 | } 42 | 43 | func (fs *ConfigurationInLocalFilesystem) GetSingleDocumentAnyType(kind string, id string, apiGroup string, apiVersion string) (string, error) { 44 | filePath := fs.buildPath(kind, id, apiGroup, apiVersion) 45 | logrus.Debugf("Looking for file at path '%s'", filePath) 46 | 47 | if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) { 48 | return "", errors.Wrap(err, "Object not found") 49 | } 50 | 51 | content, err := ioutil.ReadFile(filePath) 52 | if err != nil { 53 | return "", errors.Wrapf(err, "Cannot read object from filesystem storage at path '%s'", filePath) 54 | } 55 | recode, err := fs.recodeFromYamlToJson(content) 56 | if err != nil { 57 | return "", errors.Wrapf(err, "Cannot parse object from filesystem storage at path '%s'", filePath) 58 | } 59 | 60 | return recode, nil 61 | } 62 | 63 | func (fs *ConfigurationInLocalFilesystem) GetSingleDocument(kind string, id string) (string, error) { 64 | return fs.GetSingleDocumentAnyType(kind, id, fs.apiGroup, fs.apiVersion) 65 | } 66 | 67 | func (fs *ConfigurationInLocalFilesystem) StoreDocument(kind string, document interface{}) error { 68 | return errors.New("not implemented") 69 | } 70 | 71 | func (fs *ConfigurationInLocalFilesystem) buildPath(kind string, id string, apiGroup string, apiVersion string) string { 72 | return strings.ReplaceAll(fs.path+"/"+fs.namespace+"/"+apiGroup+"/"+apiVersion+"/"+kind+"/"+id+".yaml", "//", "/") 73 | } 74 | 75 | func (fs *ConfigurationInLocalFilesystem) recodeFromYamlToJson(yamlDoc []byte) (string, error) { 76 | var raw interface{} 77 | if err := yaml.Unmarshal(yamlDoc, &raw); err != nil { 78 | return "", errors.Wrap(err, "Cannot recode from YAML to JSON") 79 | } 80 | jsonDoc, err := json.Marshal(raw) 81 | if err != nil { 82 | return "", errors.Wrap(err, "Cannot recode from YAML to JSON") 83 | } 84 | return string(jsonDoc), nil 85 | } 86 | 87 | func NewConfigurationInLocalFilesystemProvider(path string, namespace string) *ConfigurationInLocalFilesystem { 88 | return &ConfigurationInLocalFilesystem{ 89 | path: path, 90 | namespace: namespace, 91 | apiVersion: "v1alpha1", 92 | apiGroup: "backups.riotkit.org", 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pkg/config/filesystem_test.go: -------------------------------------------------------------------------------- 1 | package config_test 2 | 3 | import ( 4 | "github.com/riotkit-org/backup-repository/pkg/config" 5 | "github.com/stretchr/testify/assert" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestConfigurationInLocalFilesystem_GetHealth_FailsOnNonWritableDirectory(t *testing.T) { 11 | // we hope nobody runs those tests as sudo/root ;) 12 | nonWritableFailing := config.NewConfigurationInLocalFilesystemProvider("/usr/share/putin-chuj", "default") 13 | err := nonWritableFailing.GetHealth() 14 | 15 | assert.NotNil(t, err) 16 | assert.Contains(t, err.Error(), "The filesystem is not writeable") 17 | } 18 | 19 | func TestConfigurationInLocalFilesystem_GetHealth(t *testing.T) { 20 | // we hope nobody runs those tests as sudo/root ;) 21 | wd, _ := os.Getwd() 22 | valid := config.NewConfigurationInLocalFilesystemProvider(wd, "default") 23 | err := valid.GetHealth() 24 | assert.Nil(t, err) 25 | } 26 | 27 | func TestConfigurationInLocalFilesystem_GetSingleDocument_WithCRD(t *testing.T) { 28 | wd, _ := os.Getwd() 29 | provider := config.NewConfigurationInLocalFilesystemProvider(wd+"/../../docs/examples-filesystem", "backup-repository") 30 | 31 | admin, err := provider.GetSingleDocument("backupusers", "admin") 32 | 33 | assert.Nil(t, err) 34 | assert.Contains(t, admin, `"kind":"BackupUser"`) 35 | } 36 | 37 | func TestConfigurationInLocalFilesystem_GetSingleDocument_WithStandardResource(t *testing.T) { 38 | wd, _ := os.Getwd() 39 | provider := config.NewConfigurationInLocalFilesystemProvider(wd+"/../../docs/examples-filesystem", "backup-repository") 40 | 41 | admin, err := provider.GetSingleDocumentAnyType("secrets", "backup-repository-passwords", "", "v1") 42 | 43 | assert.Nil(t, err) 44 | assert.Contains(t, admin, `"kind":"Secret"`) 45 | } 46 | 47 | func TestConfigurationInLocalFilesystem_GetSingleDocument_NotFound(t *testing.T) { 48 | wd, _ := os.Getwd() 49 | provider := config.NewConfigurationInLocalFilesystemProvider(wd+"/../../docs/examples-filesystem", "backup-repository") 50 | 51 | _, err := provider.GetSingleDocument("backupusers", "some-non-existing-object") 52 | 53 | assert.NotNil(t, err) 54 | assert.Contains(t, err.Error(), "Object not found") 55 | } 56 | -------------------------------------------------------------------------------- /pkg/config/generic.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type ObjectMetadata struct { 4 | Name string `json:"name" structs:"name"` 5 | } 6 | -------------------------------------------------------------------------------- /pkg/config/interface.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type ConfigurationProvider interface { 4 | GetSingleDocument(kind string, id string) (string, error) 5 | GetSingleDocumentAnyType(kind string, id string, apiGroup string, apiVersion string) (string, error) 6 | 7 | StoreDocument(kind string, document interface{}) error 8 | GetHealth() error 9 | } 10 | -------------------------------------------------------------------------------- /pkg/config/kubernetes.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "context" 5 | "github.com/fatih/structs" 6 | "github.com/pkg/errors" 7 | "github.com/sirupsen/logrus" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 10 | "k8s.io/apimachinery/pkg/runtime/schema" 11 | "k8s.io/client-go/dynamic" 12 | "sigs.k8s.io/controller-runtime/pkg/client/config" 13 | ) 14 | 15 | type ConfigurationInKubernetes struct { 16 | api dynamic.Interface 17 | namespace string 18 | apiGroup string 19 | apiVersion string 20 | } 21 | 22 | func (o *ConfigurationInKubernetes) GetHealth() error { 23 | resources := []schema.GroupVersionResource{ 24 | {Group: o.apiGroup, Version: o.apiVersion, Resource: "backupusers"}, 25 | {Group: o.apiGroup, Version: o.apiVersion, Resource: "backupcollections"}, 26 | } 27 | 28 | for _, resource := range resources { 29 | if _, err := o.api.Resource(resource).Namespace(o.namespace).List(context.Background(), metav1.ListOptions{}); err != nil { 30 | return errors.Wrapf(err, "cannot access Kubrenetes resources: '%v'", resource.String()) 31 | } 32 | } 33 | 34 | return nil 35 | } 36 | 37 | func (o *ConfigurationInKubernetes) GetSingleDocumentAnyType(kind string, id string, apiGroup string, apiVersion string) (string, error) { 38 | resource := schema.GroupVersionResource{Group: apiGroup, Version: apiVersion, Resource: kind} 39 | object, err := o.api.Resource(resource).Namespace(o.namespace).Get(context.Background(), id, metav1.GetOptions{}) 40 | 41 | if err != nil { 42 | logrus.Warnf("Kubernetes API returned error: %v", err) 43 | return "", err 44 | } 45 | 46 | content, err := object.MarshalJSON() 47 | if err != nil { 48 | logrus.Errorf("Cannot return Kubernetes object of kind '%v' as JSON", kind) 49 | return "", err 50 | } 51 | 52 | logrus.Debugf("GetSingleDocument(%v, %v) OK", kind, id) 53 | return string(content), nil 54 | } 55 | 56 | func (o *ConfigurationInKubernetes) GetSingleDocument(kind string, id string) (string, error) { 57 | return o.GetSingleDocumentAnyType(kind, id, o.apiGroup, o.apiVersion) 58 | } 59 | 60 | func (o *ConfigurationInKubernetes) StoreDocument(kind string, document interface{}) error { 61 | resource := schema.GroupVersionResource{Group: o.apiGroup, Version: o.apiVersion, Resource: kind} 62 | object := unstructured.Unstructured{Object: structs.Map(document)} 63 | 64 | _, err := o.api.Resource(resource).Namespace(o.namespace).Create( 65 | context.Background(), 66 | &object, 67 | metav1.CreateOptions{}, 68 | ) 69 | 70 | // todo: if update fails specifically, then attempt to create object 71 | 72 | if err != nil { 73 | logrus.Errorf("Cannot stored document of `kind: %v`. error: %v", kind, err) 74 | return err 75 | } 76 | 77 | return nil 78 | } 79 | 80 | func CreateKubernetesConfigurationProvider(namespace string) *ConfigurationInKubernetes { 81 | api, _ := dynamic.NewForConfig(config.GetConfigOrDie()) 82 | // todo: IsError handling 83 | 84 | // todo: Implement caching by composition 85 | return &ConfigurationInKubernetes{ 86 | api: api, 87 | namespace: namespace, 88 | apiVersion: "v1alpha1", 89 | apiGroup: "backups.riotkit.org", 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /pkg/core/ctx.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "github.com/riotkit-org/backup-repository/pkg/collections" 5 | "github.com/riotkit-org/backup-repository/pkg/concurrency" 6 | "github.com/riotkit-org/backup-repository/pkg/config" 7 | "github.com/riotkit-org/backup-repository/pkg/security" 8 | "github.com/riotkit-org/backup-repository/pkg/storage" 9 | "github.com/riotkit-org/backup-repository/pkg/users" 10 | "gorm.io/gorm" 11 | "time" 12 | ) 13 | 14 | type ApplicationContainer struct { 15 | Db *gorm.DB 16 | Config *config.ConfigurationProvider 17 | Users *users.Service 18 | GrantedAccesses *security.Service 19 | Collections *collections.Service 20 | Storage *storage.Service 21 | JwtSecretKey string 22 | HealthCheckKey string 23 | Locks *concurrency.LocksService 24 | 25 | // global timeouts 26 | UploadTimeout time.Duration 27 | DownloadTimeout time.Duration 28 | 29 | // global request limit rate 30 | DefaultRPS int16 31 | AuthRPM int16 32 | CollectionHealthRPM int16 33 | ServerHealthRPM int16 34 | } 35 | -------------------------------------------------------------------------------- /pkg/db/main.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | "github.com/riotkit-org/backup-repository/pkg/concurrency" 6 | "github.com/riotkit-org/backup-repository/pkg/security" 7 | "github.com/riotkit-org/backup-repository/pkg/storage" 8 | "github.com/sirupsen/logrus" 9 | "gorm.io/driver/postgres" 10 | "gorm.io/gorm" 11 | ) 12 | 13 | func CreateDatabaseDriver(hostname string, user string, password string, dbname string, port int, additionalDSN string) (*gorm.DB, error) { 14 | dsn := fmt.Sprintf("host=%v user=%v password=%v dbname=%v port=%v %v", 15 | hostname, user, password, dbname, port, additionalDSN) 16 | 17 | return gorm.Open(postgres.Open(dsn), &gorm.Config{}) 18 | } 19 | 20 | func InitializeDatabase(db *gorm.DB) bool { 21 | if err := security.InitializeModel(db); err != nil { 22 | logrus.Errorf("Cannot initialize GrantedAccess model: %v", err) 23 | return false 24 | } 25 | if err := storage.InitializeModel(db); err != nil { 26 | logrus.Errorf("Cannot initialize UploadedVersion model: %v", err) 27 | return false 28 | } 29 | if err := concurrency.InitializeModel(db); err != nil { 30 | logrus.Errorf("Cannot initialize Locks model: %v", err) 31 | return false 32 | } 33 | 34 | return true 35 | } 36 | -------------------------------------------------------------------------------- /pkg/health/README.md: -------------------------------------------------------------------------------- 1 | health 2 | ------ 3 | 4 | Checks health status of a collection. Uses various modules to verify the actual state. 5 | -------------------------------------------------------------------------------- /pkg/health/backupwindow.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pkg/errors" 6 | "github.com/riotkit-org/backup-repository/pkg/collections" 7 | "github.com/riotkit-org/backup-repository/pkg/storage" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | type BackupWindowValidator struct { 13 | svc StorageInterface 14 | c *collections.Collection 15 | nowFactory func() time.Time 16 | } 17 | 18 | func (v BackupWindowValidator) Validate() error { 19 | // Backup Windows are optional 20 | if len(v.c.Spec.Windows) == 0 { 21 | return nil 22 | } 23 | 24 | latest, err := v.svc.FindLatestVersion(v.c.GetId()) 25 | if err != nil { 26 | return errors.Wrap(err, "cannot find any backup in the collection") 27 | } 28 | 29 | allowedSlots := "" 30 | now := v.nowFactory() 31 | 32 | for _, window := range v.c.Spec.Windows { 33 | matches, err := window.IsInPreviousWindowTimeSlot(now, latest.CreatedAt) 34 | 35 | if err != nil { 36 | return errors.New(fmt.Sprintf("failed to calculate previous run for window '%v' - %v", window, err)) 37 | } 38 | if matches { 39 | return nil 40 | } 41 | 42 | // check if we are now in a timeslot (which could mean that the backup is in-progress) 43 | if matches, _ := window.IsInPreviousWindowTimeSlot(now, now); matches { 44 | return nil 45 | } 46 | 47 | allowedSlots += fmt.Sprintf(", interval(%v) + %v", window.From, window.Duration) 48 | } 49 | 50 | return errors.Errorf("previous backup was not executed in expected time slots: %v. Latest backup created at: %s", strings.Trim(allowedSlots, ", "), latest.CreatedAt) 51 | } 52 | 53 | func NewBackupWindowValidator(svc *storage.Service, c *collections.Collection) BackupWindowValidator { 54 | return BackupWindowValidator{svc, c, func() time.Time { return time.Now() }} 55 | } 56 | -------------------------------------------------------------------------------- /pkg/health/backupwindow_test.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "errors" 5 | "github.com/riotkit-org/backup-repository/pkg/collections" 6 | "github.com/riotkit-org/backup-repository/pkg/config" 7 | "github.com/riotkit-org/backup-repository/pkg/storage" 8 | "github.com/stretchr/testify/assert" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func TestBackupWindowValidator_Validate(t *testing.T) { 14 | type test struct { 15 | windows collections.BackupWindows 16 | storage *storageMock 17 | now time.Time 18 | expectedError string 19 | } 20 | 21 | referenceTime := time.Date(2021, 05, 01, 16, 0, 0, 0, time.UTC) 22 | 23 | variants := []test{ 24 | // Every 30 minutes, +5 minutes for backup sending 25 | // Last copy sent: Current hour (hh) : 00 minutes 26 | // Fail: hh:00 is not between hh:30 - hh:35 range 27 | { 28 | collections.BackupWindows{ 29 | createWindow("*/30 * * * *", "0h05m0s"), 30 | }, 31 | &storageMock{ 32 | storage.UploadedVersion{CreatedAt: referenceTime.Add(time.Minute * 5)}, 33 | nil, 34 | }, 35 | referenceTime, 36 | "previous backup was not executed in expected time slots: interval(*/30 * * * *) + 0h05m0s. Latest backup created at: 2021-05-01 16:05:00 +0000 UTC", 37 | }, 38 | 39 | // No version found 40 | { 41 | collections.BackupWindows{ 42 | createWindow("*/30 * * * *", "0h05m0s"), 43 | }, 44 | &storageMock{ 45 | storage.UploadedVersion{}, 46 | errors.New("not found"), 47 | }, 48 | referenceTime, 49 | "cannot find any backup in the collection: not found", 50 | }, 51 | 52 | // Success case 53 | { 54 | collections.BackupWindows{ 55 | createWindow("30 * * * *", "0h05m0s"), 56 | }, 57 | &storageMock{ 58 | storage.UploadedVersion{CreatedAt: referenceTime.Add(time.Minute * -26)}, // 15:34 (because referenceTime = 16:00) 59 | nil, 60 | }, 61 | referenceTime, 62 | "", 63 | }, 64 | } 65 | 66 | for _, variant := range variants { 67 | v := BackupWindowValidator{ 68 | svc: variant.storage, 69 | c: &collections.Collection{ 70 | Metadata: config.ObjectMetadata{Name: "some-name"}, 71 | Spec: collections.Spec{ 72 | Windows: variant.windows, 73 | }, 74 | }, 75 | nowFactory: func() time.Time { 76 | return variant.now 77 | }, 78 | } 79 | 80 | err := v.Validate() 81 | 82 | if variant.expectedError != "" { 83 | assert.Equal(t, variant.expectedError, err.Error()) 84 | } else { 85 | assert.Nil(t, err) 86 | } 87 | } 88 | } 89 | 90 | type storageMock struct { 91 | latestVersion storage.UploadedVersion 92 | err error 93 | } 94 | 95 | func (m *storageMock) FindLatestVersion(collectionId string) (storage.UploadedVersion, error) { 96 | return m.latestVersion, m.err 97 | } 98 | 99 | func createWindow(from string, duration string) collections.BackupWindow { 100 | b, _ := collections.NewBackupWindow(from, duration) 101 | return b 102 | } 103 | -------------------------------------------------------------------------------- /pkg/health/config.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/riotkit-org/backup-repository/pkg/config" 6 | ) 7 | 8 | type ConfigurationProviderValidator struct { 9 | cfg config.ConfigurationProvider 10 | } 11 | 12 | func (v ConfigurationProviderValidator) Validate() error { 13 | if err := v.cfg.GetHealth(); err != nil { 14 | return errors.Wrapf(err, "configuration provider is not usable") 15 | } 16 | 17 | return nil 18 | } 19 | 20 | func NewConfigurationProviderValidator(cfg config.ConfigurationProvider) ConfigurationProviderValidator { 21 | return ConfigurationProviderValidator{cfg} 22 | } 23 | -------------------------------------------------------------------------------- /pkg/health/db.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pkg/errors" 6 | "gorm.io/gorm" 7 | "math/rand" 8 | "time" 9 | ) 10 | 11 | type DbValidator struct { 12 | db *gorm.DB 13 | } 14 | 15 | func (v DbValidator) Validate() error { 16 | rand.Seed(time.Now().UnixNano()) 17 | key := rand.Intn(8) 18 | var result int 19 | 20 | err := v.db.Raw(fmt.Sprintf("SELECT %v", key)).Scan(&result).Error 21 | if err != nil || result != key { 22 | return errors.Wrapf(err, "cannot connect to database") 23 | } 24 | 25 | return nil 26 | } 27 | 28 | func NewDbValidator(db *gorm.DB) DbValidator { 29 | return DbValidator{db} 30 | } 31 | -------------------------------------------------------------------------------- /pkg/health/interface.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/riotkit-org/backup-repository/pkg/storage" 7 | "reflect" 8 | ) 9 | 10 | type Validator interface { 11 | Validate() error 12 | } 13 | type Validators []Validator 14 | 15 | func (v Validators) Validate() StatusCollection { 16 | var status StatusCollection 17 | 18 | for _, validator := range v { 19 | if err := validator.Validate(); err != nil { 20 | status = append(status, Status{ 21 | Name: reflect.TypeOf(validator).Name(), 22 | StatusMsg: err.Error(), 23 | IsError: true, 24 | }) 25 | } else { 26 | status = append(status, Status{ 27 | Name: reflect.TypeOf(validator).Name(), 28 | StatusMsg: "OK", 29 | IsError: false, 30 | }) 31 | } 32 | } 33 | 34 | return status 35 | } 36 | 37 | type Status struct { 38 | Name string 39 | StatusMsg string 40 | IsError bool 41 | } 42 | 43 | func (s *Status) MarshalJSON() ([]byte, error) { 44 | curr := make(map[string]interface{}) 45 | curr["name"] = s.Name 46 | curr["statusText"] = fmt.Sprintf("%v=%v", s.Name, !s.IsError) 47 | curr["status"] = !s.IsError 48 | curr["message"] = s.StatusMsg 49 | 50 | return json.Marshal(curr) 51 | } 52 | 53 | type StatusCollection []Status 54 | 55 | func (sc StatusCollection) GetOverallStatus() bool { 56 | for _, status := range sc { 57 | if status.IsError { 58 | return false 59 | } 60 | } 61 | return true 62 | } 63 | 64 | type StorageInterface interface { 65 | FindLatestVersion(collectionId string) (storage.UploadedVersion, error) 66 | } 67 | -------------------------------------------------------------------------------- /pkg/health/size.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/riotkit-org/backup-repository/pkg/collections" 6 | "github.com/riotkit-org/backup-repository/pkg/storage" 7 | ) 8 | 9 | type VersionsSizeValidator struct { 10 | svc *storage.Service 11 | c *collections.Collection 12 | } 13 | 14 | func (v VersionsSizeValidator) Validate() error { 15 | versions, err := v.svc.FindAllActiveVersionsFor(v.c.GetId()) 16 | if err != nil { 17 | return errors.Wrapf(err, "Cannot list versions for collection id=%v", v.c.GetId()) 18 | } 19 | 20 | maxVersionSize, err := v.c.GetMaxOneVersionSizeInBytes() 21 | if err != nil { 22 | return errors.Wrapf(err, "Cannot list versions for collection id=%v", v.c.GetId()) 23 | } 24 | 25 | for _, v := range versions { 26 | if v.Filesize > maxVersionSize { 27 | return errors.Errorf("maximum filesize is bigger than collection soft limit per file. Failed file: %v (size=%vb)", v.Filename, v.Filesize) 28 | } 29 | } 30 | 31 | return nil 32 | } 33 | 34 | func NewVersionsSizeValidator(svc *storage.Service, c *collections.Collection) VersionsSizeValidator { 35 | return VersionsSizeValidator{svc, c} 36 | } 37 | -------------------------------------------------------------------------------- /pkg/health/storage.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "context" 5 | "github.com/pkg/errors" 6 | "github.com/riotkit-org/backup-repository/pkg/storage" 7 | "time" 8 | ) 9 | 10 | type StorageAvailabilityValidator struct { 11 | storage *storage.Service 12 | ctx context.Context 13 | timeout time.Duration 14 | } 15 | 16 | func (v StorageAvailabilityValidator) Validate() error { 17 | err := v.storage.TestReadWrite(v.ctx, v.timeout) 18 | if err != nil { 19 | return errors.Wrapf(err, "storage not operable") 20 | } 21 | 22 | return nil 23 | } 24 | 25 | func NewStorageValidator(storage *storage.Service, ctx context.Context, timeout time.Duration) StorageAvailabilityValidator { 26 | return StorageAvailabilityValidator{storage, ctx, timeout} 27 | } 28 | -------------------------------------------------------------------------------- /pkg/health/sumofversions.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/riotkit-org/backup-repository/pkg/collections" 6 | "github.com/riotkit-org/backup-repository/pkg/storage" 7 | ) 8 | 9 | type SumOfVersionsValidator struct { 10 | svc *storage.Service 11 | c *collections.Collection 12 | } 13 | 14 | func (v SumOfVersionsValidator) Validate() error { 15 | var totalSize int64 16 | allActive, _ := v.svc.FindAllActiveVersionsFor(v.c.GetId()) 17 | 18 | for _, version := range allActive { 19 | totalSize += version.Filesize 20 | } 21 | 22 | maxCollectionSize, _ := v.c.GetCollectionMaxSize() 23 | 24 | if totalSize > maxCollectionSize { 25 | return errors.Errorf("Summary of all files is %vb, while collection hard limit is %vb", totalSize, maxCollectionSize) 26 | } 27 | 28 | return nil 29 | } 30 | 31 | func NewSumOfVersionsValidator(svc *storage.Service, c *collections.Collection) SumOfVersionsValidator { 32 | return SumOfVersionsValidator{svc, c} 33 | } 34 | -------------------------------------------------------------------------------- /pkg/http/health.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "github.com/gin-gonic/gin" 6 | "github.com/pkg/errors" 7 | "github.com/riotkit-org/backup-repository/pkg/core" 8 | health "github.com/riotkit-org/backup-repository/pkg/health" 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | func addServerHealthEndpoints(r *gin.Engine, ctx *core.ApplicationContainer, rateLimiter gin.HandlerFunc) { 13 | // responds if service is running (there is enough ram memory, HTTP request can be processed etc.) 14 | r.GET("/health", rateLimiter, func(c *gin.Context) { 15 | OKResponse(c, gin.H{ 16 | "msg": "The server is up and running. Dependent services are not shown there. Take a look at /ready endpoint", 17 | }) 18 | }) 19 | 20 | // responds if service can handle requests actually 21 | r.GET("/ready", rateLimiter, func(c *gin.Context) { 22 | // Authorization 23 | healthCode := c.GetHeader("Authorization") 24 | if healthCode == "" { 25 | healthCode = c.Query("code") 26 | } 27 | if healthCode != ctx.HealthCheckKey { 28 | UnauthorizedResponse(c, errors.New("health code invalid. Should be provided withing 'Authorization' header or 'code' query string. Must match --health-check-code commandline switch value")) 29 | return 30 | } 31 | 32 | healthStatuses := health.Validators{ 33 | health.NewDbValidator(ctx.Db), 34 | health.NewStorageValidator(ctx.Storage, context.Background(), ctx.Storage.HealthTimeout), 35 | health.NewConfigurationProviderValidator(*ctx.Config), 36 | }.Validate() 37 | 38 | if !healthStatuses.GetOverallStatus() { 39 | logrus.Errorf("The server is unhealthy: %v", healthStatuses) 40 | 41 | ServerErrorResponseWithData(c, errors.New("one of checks failed"), gin.H{ 42 | "health": healthStatuses, 43 | }) 44 | return 45 | } 46 | 47 | OKResponse(c, gin.H{ 48 | "health": healthStatuses, 49 | }) 50 | }) 51 | } 52 | -------------------------------------------------------------------------------- /pkg/http/logging.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "github.com/gin-gonic/gin" 7 | "github.com/sirupsen/logrus" 8 | "strings" 9 | ) 10 | 11 | type errorLogWriter struct { 12 | gin.ResponseWriter 13 | body *bytes.Buffer 14 | } 15 | 16 | func (w errorLogWriter) Write(b []byte) (int, error) { 17 | if w.Status() > 299 && strings.Contains(w.Header().Get("Content-Type"), "application/json") { 18 | w.body.Write(b) 19 | } 20 | return w.ResponseWriter.Write(b) 21 | } 22 | 23 | func responseErrorLoggerMiddleware() gin.HandlerFunc { 24 | return func(c *gin.Context) { 25 | blw := &errorLogWriter{body: bytes.NewBufferString(""), ResponseWriter: c.Writer} 26 | c.Writer = blw 27 | c.Next() 28 | if len(blw.body.String()) > 0 { 29 | response := GenericResponseType{} 30 | unmarshalErr := json.Unmarshal(blw.body.Bytes(), &response) 31 | if unmarshalErr != nil { 32 | logrus.Warningln("Cannot parse error response. Probably not in valid format") 33 | return 34 | } 35 | logrus.Errorf("Returned '%s' error, message: '%s'", response.Error, response.Message) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pkg/http/main.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/gin-gonic/gin" 5 | limiter "github.com/julianshen/gin-limiter" 6 | "github.com/riotkit-org/backup-repository/pkg/core" 7 | "time" 8 | ) 9 | 10 | func SpawnHttpApplication(app *core.ApplicationContainer, listenAddr string) error { 11 | r := gin.Default() 12 | 13 | authMiddleware := createAuthenticationMiddleware(r, app) 14 | errorLoggerMiddleware := responseErrorLoggerMiddleware() 15 | 16 | // set a rate limit of 10 requests per minute for IP address to protect against DoS attacks on login and refresh_token endpoints 17 | // for two reasons: 18 | // 1) to protect against brute force 19 | // 2) to protect against memory overflow attacks (argon2di uses a lot of memory to calculate hash during login. But that's intended - the password is a lot more difficult to crack in case, when hash would leak) 20 | authRateLimitMiddleware := limiter.NewRateLimiter(time.Minute, int64(app.AuthRPM), func(ctx *gin.Context) (string, error) { 21 | return "auth:" + ctx.ClientIP(), nil 22 | }) 23 | 24 | // default rate limiter 25 | defaultRateLimitMiddleware := limiter.NewRateLimiter(time.Second, int64(app.DefaultRPS), func(ctx *gin.Context) (string, error) { 26 | return "default:" + ctx.ClientIP(), nil 27 | }).Middleware() 28 | 29 | router := r.Group("/") 30 | router.POST("/api/stable/auth/login", errorLoggerMiddleware, authRateLimitMiddleware.Middleware(), authMiddleware.LoginHandler) 31 | router.GET("/api/stable/auth/refresh_token", errorLoggerMiddleware, authRateLimitMiddleware.Middleware(), authMiddleware.RefreshHandler) 32 | router.Use(authMiddleware.MiddlewareFunc(), errorLoggerMiddleware) 33 | { 34 | addLookupUserRoute(router, app, defaultRateLimitMiddleware) 35 | addWhoamiRoute(router, app, defaultRateLimitMiddleware) 36 | addLogoutRoute(router, app, defaultRateLimitMiddleware) 37 | addGrantedAccessSearchRoute(router, app, defaultRateLimitMiddleware) 38 | addUploadRoute(router, app, app.UploadTimeout) 39 | addDownloadRoute(router, app, app.DownloadTimeout, defaultRateLimitMiddleware) 40 | addCollectionListingRoute(router, app, 30*time.Second, defaultRateLimitMiddleware) 41 | } 42 | 43 | // collection health 44 | addCollectionHealthRoute(r, app, limiter.NewRateLimiter(time.Minute, int64(app.CollectionHealthRPM), func(ctx *gin.Context) (string, error) { 45 | return "collectionHealth:" + ctx.ClientIP(), nil 46 | }).Middleware()) 47 | 48 | // server health 49 | addServerHealthEndpoints(r, app, limiter.NewRateLimiter(time.Minute, int64(app.ServerHealthRPM), func(ctx *gin.Context) (string, error) { 50 | return "health:" + ctx.ClientIP(), nil 51 | }).Middleware()) 52 | 53 | return r.Run(listenAddr) 54 | } 55 | -------------------------------------------------------------------------------- /pkg/http/responses.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/gin-gonic/gin" 5 | "net/http" 6 | ) 7 | 8 | type GenericResponseType struct { 9 | Error string `json:"error"` 10 | Message string `json:"message"` 11 | } 12 | 13 | func NotFoundResponse(c *gin.Context, err error) { 14 | c.IndentedJSON(404, gin.H{ 15 | "status": false, 16 | "error": err.Error(), 17 | "data": gin.H{}, 18 | }) 19 | } 20 | 21 | func OKResponse(c *gin.Context, data gin.H) { 22 | c.IndentedJSON(200, gin.H{ 23 | "status": true, 24 | "data": data, 25 | }) 26 | } 27 | 28 | func UnauthorizedResponse(c *gin.Context, err error) { 29 | c.IndentedJSON(403, gin.H{ 30 | "status": false, 31 | "error": err.Error(), 32 | "data": gin.H{}, 33 | }) 34 | } 35 | 36 | func ServerErrorResponse(c *gin.Context, err error) { 37 | c.IndentedJSON(500, gin.H{ 38 | "status": false, 39 | "error": err.Error(), 40 | "data": gin.H{}, 41 | }) 42 | } 43 | 44 | func ServerErrorResponseWithData(c *gin.Context, err error, data gin.H) { 45 | c.IndentedJSON(500, gin.H{ 46 | "status": false, 47 | "error": err.Error(), 48 | "data": data, 49 | }) 50 | } 51 | 52 | func RequestTimeoutResponse(c *gin.Context) { 53 | c.IndentedJSON(http.StatusRequestTimeout, gin.H{ 54 | "status": false, 55 | "error": "Request took too long", 56 | "data": gin.H{}, 57 | }) 58 | } 59 | -------------------------------------------------------------------------------- /pkg/http/utils.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/gin-gonic/gin" 7 | "github.com/riotkit-org/backup-repository/pkg/core" 8 | "github.com/riotkit-org/backup-repository/pkg/security" 9 | "github.com/riotkit-org/backup-repository/pkg/users" 10 | "strings" 11 | ) 12 | 13 | // GetContextUser returns a User{} that is authenticated in current request 14 | func GetContextUser(ctx *core.ApplicationContainer, c *gin.Context) (*users.SessionAwareUser, error) { 15 | username, accessKeyName := security.ExtractLoginFromJWT(c.GetHeader("Authorization")) 16 | scope, scopeErr := security.ExtractSessionLimitedOperationsScopeFromJWT(c.GetHeader("Authorization")) 17 | if scopeErr != nil { 18 | return nil, errors.New(fmt.Sprintf("cannot create context user: %s", scopeErr.Error())) 19 | } 20 | 21 | identity := security.NewUserIdentityFromString(username) 22 | identity.AccessKeyName = accessKeyName 23 | 24 | if accessKeyName != "" { 25 | return ctx.Users.LookupSessionUser(identity, scope) 26 | } 27 | return ctx.Users.LookupSessionUser(identity, scope) 28 | } 29 | 30 | func GetCurrentSessionId(c *gin.Context) string { 31 | return security.HashJWT(strings.Trim(c.GetHeader("Authorization"), " ")[7:]) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/security/constants.go: -------------------------------------------------------------------------------- 1 | package security 2 | 3 | const ( 4 | RoleUserManager = "userManager" 5 | RoleCollectionManager = "collectionManager" 6 | RoleBackupDownloader = "backupDownloader" 7 | RoleBackupUploader = "backupUploader" 8 | 9 | // RoleUploadsAnytime allows uploading versions regardless of Backup Windows 10 | RoleUploadsAnytime = "uploadsAnytime" 11 | RoleSysAdmin = "systemAdmin" 12 | ) 13 | 14 | const ( 15 | ActionDownload = "file.download" 16 | ActionUpload = "file.upload" 17 | ActionUploadAnytime = "file.upload-anytime" 18 | ActionViewProfile = "profile.view" 19 | ) 20 | 21 | var AllActions = []string{ 22 | ActionDownload, ActionUpload, ActionUploadAnytime, ActionViewProfile, 23 | } 24 | 25 | func GetRolesInheritance() map[string][]string { 26 | return map[string][]string{ 27 | RoleSysAdmin: {RoleUserManager, RoleCollectionManager}, 28 | RoleCollectionManager: {RoleBackupDownloader, RoleBackupDownloader, RoleUploadsAnytime}, 29 | } 30 | } 31 | 32 | func GetRolesActions() map[string][]string { 33 | return map[string][]string{ 34 | RoleBackupDownloader: {ActionDownload}, 35 | RoleBackupUploader: {ActionUpload}, 36 | RoleUploadsAnytime: {ActionUpload, ActionUploadAnytime}, 37 | RoleUserManager: {ActionViewProfile}, 38 | } 39 | } 40 | 41 | const ( 42 | IdentityKeyClaimIndex = "login" 43 | AccessKeyClaimIndex = "accessKeyName" 44 | ScopeClaimIndex = "operationsScope" 45 | ) 46 | -------------------------------------------------------------------------------- /pkg/security/crypto.go: -------------------------------------------------------------------------------- 1 | // 2 | // See: https://golangcode.com/argon2-password-hashing/ 3 | // Thanks to Edd Turtle 4 | // 5 | 6 | package security 7 | 8 | import ( 9 | "crypto/rand" 10 | "crypto/sha256" 11 | "crypto/subtle" 12 | "encoding/base64" 13 | "encoding/hex" 14 | "errors" 15 | "fmt" 16 | "github.com/sirupsen/logrus" 17 | "golang.org/x/crypto/argon2" 18 | "runtime" 19 | "strings" 20 | ) 21 | 22 | type Argon2Config struct { 23 | time uint32 24 | memory uint32 25 | threads uint8 26 | keyLen uint32 27 | } 28 | 29 | func CreateDefaultPasswordConfig() *Argon2Config { 30 | return &Argon2Config{ 31 | time: 1, 32 | memory: 64 * 1024, 33 | threads: 4, 34 | keyLen: 32, 35 | } 36 | } 37 | 38 | // CreateHashFromPassword is used to generate a new password hash for storing and 39 | // comparing at a later date. 40 | func CreateHashFromPassword(password string) (string, error) { 41 | // Generate a Salt 42 | salt := make([]byte, 16) 43 | if _, err := rand.Read(salt); err != nil { 44 | return "", err 45 | } 46 | 47 | c := CreateDefaultPasswordConfig() 48 | hash := argon2.IDKey([]byte(password), salt, c.time, c.memory, c.threads, c.keyLen) 49 | 50 | // Base64 encode the salt and hashed password. 51 | b64Salt := base64.StdEncoding.EncodeToString(salt) 52 | b64Hash := base64.StdEncoding.EncodeToString(hash) 53 | 54 | format := "$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s" 55 | full := fmt.Sprintf(format, argon2.Version, c.memory, c.time, c.threads, b64Salt, b64Hash) 56 | 57 | runtime.GC() 58 | 59 | return base64.StdEncoding.EncodeToString([]byte(full)), nil 60 | } 61 | 62 | // ComparePassword is used to compare a user-inputted password to a hash to see 63 | // if the password matches or not. 64 | func ComparePassword(password string, hash string) (bool, error) { 65 | hashByte, _ := base64.StdEncoding.DecodeString(hash) 66 | hash = string(hashByte) 67 | parts := strings.Split(hash, "$") 68 | 69 | if len(parts) < 3 { 70 | logrus.Warning("Password format is invalid. To properly encode a password use `backup-repository " + 71 | "--encode-password='your-password'` and put it in `kind: Secret` or in `kind: BackupUser`") 72 | 73 | return false, errors.New("invalid password hash format. Check `kind: Secret` or `kind: BackupUser`") 74 | } 75 | 76 | c := &Argon2Config{} 77 | _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &c.memory, &c.time, &c.threads) 78 | if err != nil { 79 | logrus.Errorf("Cannot unpack password hash for parameters recognition. Invalid format") 80 | return false, err 81 | } 82 | 83 | salt, err := base64.StdEncoding.DecodeString(parts[4]) 84 | if err != nil { 85 | logrus.Errorf("Cannot decode salt. Check if it is a valid base64 string" + 86 | " (salt is base64 encoded part inside base64 encoded secret - 4th position)") 87 | 88 | return false, err 89 | } 90 | 91 | decodedHash, err := base64.StdEncoding.DecodeString(parts[5]) 92 | if err != nil { 93 | logrus.Errorf("Cannot decode 5th part of password hash, which is a password token") 94 | return false, err 95 | } 96 | 97 | c.keyLen = uint32(len(decodedHash)) 98 | comparisonHash := argon2.IDKey([]byte(password), salt, c.time, c.memory, c.threads, c.keyLen) 99 | runtime.GC() 100 | 101 | logrus.Debugf("Comparing passwords...") 102 | return subtle.ConstantTimeCompare(decodedHash, comparisonHash) == 1, nil 103 | } 104 | 105 | func HashJWT(jwt string) string { 106 | return HashSha256(jwt) 107 | } 108 | 109 | func CompareFastCryptoHash(plain string, hash string) bool { 110 | return HashSha256(plain) == hash 111 | } 112 | 113 | func HashSha256(input string) string { 114 | asByte := sha256.Sum256([]byte(input)) 115 | 116 | return hex.EncodeToString(asByte[:]) 117 | } 118 | -------------------------------------------------------------------------------- /pkg/security/crypto_test.go: -------------------------------------------------------------------------------- 1 | package security 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestComparePassword_WorksForPasswordsComparedShortly(t *testing.T) { 10 | hash, _ := CreateHashFromPassword("riotkit") 11 | result, _ := ComparePassword("riotkit", hash) 12 | 13 | assert.True(t, result, "Expected that hashed 'riotkit' password would be possible to compare") 14 | } 15 | 16 | // TestComparePassword_WorksForPasswordsComparedIn10Seconds checks in at least 10s interval because 17 | // hashing algorithm has a time-based comparison 18 | func TestComparePassword_WorksForPasswordsComparedIn10Seconds(t *testing.T) { 19 | hash, _ := CreateHashFromPassword("riotkit") 20 | result, _ := ComparePassword("riotkit", hash) 21 | 22 | time.Sleep(time.Second * 10) 23 | 24 | assert.True(t, result, "Expected that hashed 'riotkit' password would be possible to compare") 25 | } 26 | -------------------------------------------------------------------------------- /pkg/security/decision.go: -------------------------------------------------------------------------------- 1 | package security 2 | 3 | import "k8s.io/utils/strings/slices" 4 | 5 | type DecisionRequest struct { 6 | Actor Actor 7 | Subject Subject 8 | Action string 9 | } 10 | 11 | // DecideCanDo is taking a decision if specific Action can be made on by Actor on a Subject 12 | // 13 | // Logic: 14 | // 15 | // 1. Subject defines who can access it and how 16 | // 17 | // 2. There are SYSTEM-WIDE roles defined on the user that allows globally do everything 18 | // 19 | // 3. User can generate a LIMITED SCOPE JWT token with /auth/login endpoint. This kind of token can 20 | // define that in context of given Subject the roles should be limited to specific ones 21 | // Important! Those roles cannot be higher than defined on the Subject or on the Actor in its profile 22 | // 23 | // Cases: 24 | // 25 | // Has limited token: User generates JWT with "backupDownloader" role in context of "iwa-ait" collection. 26 | // So even if that User is a "collectionManager" for this collection, with that specific JWT token 27 | // its possible to only download backups. 28 | func DecideCanDo(dr *DecisionRequest) bool { 29 | // CASE: Decision about global action, not in context of a collection 30 | // For example - to see a system health check endpoint 31 | if dr.Subject == nil { 32 | return CanThoseRolesPerformAction(dr.Actor.GetRoles(), dr.Action) 33 | } 34 | 35 | // CASE: If we are in a context of an Access Key, then it has its own limited scope 36 | if dr.Actor.IsInAccessKeyContext() { 37 | scopedRoles := dr.Actor.GetAccessKeyRolesInContextOf(dr.Subject) 38 | if !CanThoseRolesPerformAction(scopedRoles, dr.Action) { 39 | return false 40 | } 41 | } 42 | 43 | if hasCurrentTokenLimitations(dr.Actor) { 44 | limitations := dr.Actor.GetSessionLimitedOperationsScope() 45 | foundAllowing := false 46 | 47 | for _, object := range limitations.Elements { 48 | if object.Type == dr.Subject.GetTypeName() && object.Name == dr.Subject.GetId() { 49 | foundAllowing = CanThoseRolesPerformAction(object.Roles, dr.Action) 50 | } 51 | } 52 | 53 | // CASE: User has a limited token generated, and no any entry in `operationsScope` field is 54 | // matching Subject for given Action 55 | if !foundAllowing { 56 | return false 57 | } 58 | } 59 | 60 | // CASE: User is explicitly listed in object's ACL, that it owns this object 61 | objectSpecificDecision := dr.Subject.GetAccessControlList().IsPermitted(dr.Actor.GetName(), dr.Actor.GetTypeName(), dr.Action) 62 | 63 | // CASE: e.g. is a system administrator 64 | systemWideRoleDecision := CanThoseRolesPerformAction(dr.Actor.GetRoles(), dr.Action) 65 | 66 | return objectSpecificDecision || systemWideRoleDecision 67 | } 68 | 69 | // CanThoseRolesPerformAction checks if any listed role is allowing to perform action 70 | func CanThoseRolesPerformAction(roles []string, action string) bool { 71 | return slices.Contains(expandActions(expandRoles(roles)), action) 72 | } 73 | 74 | func expandRoles(roles []string) []string { 75 | inheritance := GetRolesInheritance() 76 | expanded := make([]string, 0) 77 | expanded = append(expanded, roles...) 78 | 79 | // level: 0 80 | for _, role := range roles { 81 | children, expandable := inheritance[role] 82 | 83 | if expandable { 84 | for _, element := range children { 85 | if !slices.Contains(roles, element) { 86 | expanded = append(expanded, element) 87 | } 88 | expanded = append(expanded, expandRoles([]string{element})...) 89 | } 90 | } 91 | } 92 | return expanded 93 | } 94 | 95 | func expandActions(roles []string) []string { 96 | mapping := GetRolesActions() 97 | actions := make([]string, 0) 98 | for _, role := range roles { 99 | if roleActions, exists := mapping[role]; exists { 100 | actions = append(actions, roleActions...) 101 | } 102 | } 103 | return actions 104 | } 105 | 106 | func hasCurrentTokenLimitations(a Actor) bool { 107 | if a.GetSessionLimitedOperationsScope() == nil || a.GetSessionLimitedOperationsScope().Elements == nil { 108 | return false 109 | } 110 | return len(a.GetSessionLimitedOperationsScope().Elements) > 0 111 | } 112 | 113 | type Actor interface { 114 | IsInAccessKeyContext() bool 115 | GetAccessKeyRolesInContextOf(Subject) Roles 116 | GetRoles() Roles 117 | GetEmail() string 118 | GetName() string 119 | GetTypeName() string 120 | GetSessionLimitedOperationsScope() *SessionLimitedOperationsScope 121 | } 122 | 123 | type Subject interface { 124 | GetId() string 125 | GetTypeName() string 126 | GetAccessControlList() *AccessControlList 127 | } 128 | -------------------------------------------------------------------------------- /pkg/security/decision_test.go: -------------------------------------------------------------------------------- 1 | package security_test 2 | 3 | import ( 4 | "github.com/riotkit-org/backup-repository/pkg/security" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestCanThoseRolesPerformAction(t *testing.T) { 10 | // Multiple levels - a role is expanded two times 11 | // RoleSysAdmin -> RoleCollectionManager -> RoleBackupDownloader -> [ActionDownload] 12 | assert.Equal(t, true, security.CanThoseRolesPerformAction([]string{ 13 | security.RoleSysAdmin, 14 | }, security.ActionDownload)) 15 | 16 | // Two levels - expanded one time 17 | assert.Equal(t, true, security.CanThoseRolesPerformAction([]string{ 18 | security.RoleCollectionManager, 19 | }, security.ActionDownload)) 20 | 21 | // Direct role 22 | assert.Equal(t, true, security.CanThoseRolesPerformAction([]string{ 23 | security.RoleBackupDownloader, 24 | }, security.ActionDownload)) 25 | 26 | // RoleBackupUploader ! -> ActionDownload 27 | assert.Equal(t, false, security.CanThoseRolesPerformAction([]string{ 28 | security.RoleBackupUploader, 29 | }, security.ActionDownload)) 30 | } 31 | 32 | type TestActor struct { 33 | name string 34 | isInAccessKeyContext bool 35 | roles security.Roles 36 | accessTokenContextualRoles security.Roles 37 | jwtScopeLimitations *security.SessionLimitedOperationsScope 38 | } 39 | 40 | func (a *TestActor) IsInAccessKeyContext() bool { 41 | return a.isInAccessKeyContext 42 | } 43 | 44 | func (a *TestActor) GetAccessKeyRolesInContextOf(subject security.Subject) security.Roles { 45 | return a.accessTokenContextualRoles 46 | } 47 | 48 | func (a *TestActor) GetRoles() security.Roles { 49 | return a.roles 50 | } 51 | 52 | func (a *TestActor) GetEmail() string { 53 | return "" 54 | } 55 | 56 | func (a *TestActor) GetName() string { 57 | return a.name 58 | } 59 | 60 | func (a *TestActor) GetTypeName() string { 61 | return "user" 62 | } 63 | 64 | func (a *TestActor) GetSessionLimitedOperationsScope() *security.SessionLimitedOperationsScope { 65 | return a.jwtScopeLimitations 66 | } 67 | 68 | type FakeSubject struct { 69 | id string 70 | typeName string 71 | acl *security.AccessControlList 72 | } 73 | 74 | func (fs *FakeSubject) GetId() string { 75 | return fs.id 76 | } 77 | 78 | func (fs *FakeSubject) GetTypeName() string { 79 | return fs.typeName 80 | } 81 | 82 | func (fs *FakeSubject) GetAccessControlList() *security.AccessControlList { 83 | return fs.acl 84 | } 85 | 86 | func TestDecideCanDo_AsSysAdminCanDoEverything(t *testing.T) { 87 | actor := TestActor{ 88 | name: "bakunin", 89 | isInAccessKeyContext: false, 90 | roles: security.Roles{security.RoleSysAdmin}, 91 | jwtScopeLimitations: &security.SessionLimitedOperationsScope{Elements: []security.ScopedElement{}}, 92 | } 93 | 94 | for _, action := range security.AllActions { 95 | assert.True(t, security.DecideCanDo(&security.DecisionRequest{ 96 | Actor: &actor, 97 | Subject: nil, 98 | Action: action, 99 | })) 100 | } 101 | } 102 | 103 | func TestDecideCanDo_AsSysAdminCantDoActionWhenJWTForbids(t *testing.T) { 104 | actor := &TestActor{ 105 | name: "bakunin", 106 | isInAccessKeyContext: false, 107 | 108 | // the user is ADMIN. Can do everything 109 | roles: security.Roles{security.RoleSysAdmin}, 110 | 111 | // no access token limitations are applied 112 | accessTokenContextualRoles: security.Roles{}, 113 | 114 | // JWT token is limited to only DOWNLOAD 115 | jwtScopeLimitations: &security.SessionLimitedOperationsScope{Elements: []security.ScopedElement{ 116 | {Type: "collection", Name: "iwa-ait", Roles: security.Roles{security.RoleBackupDownloader}}, 117 | }}, 118 | } 119 | subject := &FakeSubject{ 120 | id: "iwa-ait", 121 | typeName: "collection", 122 | acl: &security.AccessControlList{ 123 | security.AccessControlObject{ 124 | Name: "bakunin", 125 | Type: "user", 126 | 127 | // the collection allows user to UPLOAD & DOWNLOAD explicitly 128 | Roles: security.Roles{ 129 | security.RoleBackupDownloader, 130 | security.RoleBackupUploader, 131 | }, 132 | }, 133 | }, 134 | } 135 | 136 | // CAN'T do: /auth/login generated a JWT that allows only to DOWNLOAD 137 | assert.False(t, security.DecideCanDo(&security.DecisionRequest{ 138 | Actor: actor, 139 | Subject: subject, 140 | Action: security.ActionUpload, 141 | })) 142 | 143 | // CAN DO: download is allowed by JWT limitations 144 | assert.True(t, security.DecideCanDo(&security.DecisionRequest{ 145 | Actor: actor, 146 | Subject: subject, 147 | Action: security.ActionDownload, 148 | })) 149 | } 150 | 151 | func TestDecideCanDo_AsSysAdminCantDoActionWhenAccessTokenForbids(t *testing.T) { 152 | actor := &TestActor{ 153 | name: "bakunin", 154 | isInAccessKeyContext: true, 155 | 156 | // the user is ADMIN. Can do everything 157 | roles: security.Roles{security.RoleSysAdmin}, 158 | 159 | // ACCESS TOKEN is limiting roles 160 | accessTokenContextualRoles: security.Roles{ 161 | security.RoleBackupDownloader, 162 | }, 163 | 164 | // JWT token is NOT limiting anything 165 | jwtScopeLimitations: &security.SessionLimitedOperationsScope{Elements: []security.ScopedElement{}}, 166 | } 167 | subject := &FakeSubject{ 168 | id: "iwa-ait", 169 | typeName: "collection", 170 | acl: &security.AccessControlList{ 171 | security.AccessControlObject{ 172 | Name: "bakunin", 173 | Type: "user", 174 | 175 | // the collection allows user to UPLOAD & DOWNLOAD explicitly 176 | Roles: security.Roles{ 177 | security.RoleBackupDownloader, 178 | security.RoleBackupUploader, 179 | }, 180 | }, 181 | }, 182 | } 183 | 184 | // CAN'T do: user logged in with ACCESS TOKEN that limits action to DOWNLOAD only 185 | assert.False(t, security.DecideCanDo(&security.DecisionRequest{ 186 | Actor: actor, 187 | Subject: subject, 188 | Action: security.ActionUpload, 189 | })) 190 | 191 | // CAN DO: download is allowed 192 | assert.True(t, security.DecideCanDo(&security.DecisionRequest{ 193 | Actor: actor, 194 | Subject: subject, 195 | Action: security.ActionDownload, 196 | })) 197 | } 198 | 199 | func TestDecideCanDo_AsCollectionManagerICanManageCollection(t *testing.T) { 200 | actor := &TestActor{ 201 | name: "bakunin", 202 | isInAccessKeyContext: false, 203 | 204 | // No global roles at all 205 | roles: security.Roles{}, 206 | 207 | // No contextual limitations at all 208 | accessTokenContextualRoles: security.Roles{}, 209 | jwtScopeLimitations: &security.SessionLimitedOperationsScope{Elements: []security.ScopedElement{}}, 210 | } 211 | subject := &FakeSubject{ 212 | id: "iwa-ait", 213 | typeName: "collection", 214 | acl: &security.AccessControlList{ 215 | security.AccessControlObject{ 216 | Name: "bakunin", 217 | Type: "user", 218 | 219 | // User is a Collection Manager in this collection 220 | Roles: security.Roles{ 221 | security.RoleCollectionManager, 222 | }, 223 | }, 224 | }, 225 | } 226 | 227 | for _, role := range []string{security.ActionUpload, security.ActionUploadAnytime, security.ActionDownload} { 228 | assert.True(t, security.DecideCanDo(&security.DecisionRequest{ 229 | Actor: actor, 230 | Subject: subject, 231 | Action: role, 232 | })) 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /pkg/security/model.go: -------------------------------------------------------------------------------- 1 | package security 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | "gorm.io/gorm" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | type ScopedElement struct { 11 | Type string `form:"type" json:"type" binding:"required"` 12 | Name string `form:"name" json:"name" binding:"required"` 13 | Roles []string `form:"roles" json:"roles" binding:"required"` 14 | } 15 | 16 | // SessionLimitedOperationsScope allows to define additional limitations on the user's JWT token, so even if user has higher permissions we can limit those permissions per JWT token 17 | type SessionLimitedOperationsScope struct { 18 | Elements []ScopedElement `form:"elements" json:"elements"` 19 | } 20 | 21 | // 22 | // User permissions - roles 23 | // 24 | 25 | type Roles []string 26 | 27 | func (p Roles) IsEmpty() bool { 28 | return len(p) == 0 29 | } 30 | 31 | func (p Roles) HasRole(name string) bool { 32 | return p.has(name) || p.has(RoleSysAdmin) 33 | } 34 | 35 | func (p Roles) has(name string) bool { 36 | for _, cursor := range p { 37 | if cursor == name { 38 | return true 39 | } 40 | } 41 | 42 | return false 43 | } 44 | 45 | // 46 | // Roles for objects 47 | // 48 | 49 | type AccessControlObject struct { 50 | Name string `json:"name"` 51 | Type string `json:"type"` 52 | Roles Roles `json:"roles"` 53 | } 54 | 55 | type AccessControlList []AccessControlObject 56 | 57 | // IsPermitted checks if given user is granted a role in this list 58 | func (acl AccessControlList) IsPermitted(name string, objType string, action string) bool { 59 | for _, permitted := range acl { 60 | if permitted.Name == name && permitted.Type == objType && CanThoseRolesPerformAction(permitted.Roles, action) { 61 | return true 62 | } 63 | } 64 | return false 65 | } 66 | 67 | // PasswordFromSecretRef references passwords stored in ConfigMaps 68 | // 69 | // Name is the ConfigMap name 70 | // Entry is the key name in .data 71 | type PasswordFromSecretRef struct { 72 | Name string `json:"name"` 73 | Entry string `json:"entry"` 74 | } 75 | 76 | // GrantedAccess stores information about generated JWT tokens (successful logins to the system) 77 | type GrantedAccess struct { 78 | CreatedAt time.Time 79 | UpdatedAt time.Time 80 | DeletedAt gorm.DeletedAt `gorm:"index"` 81 | 82 | ID string `json:"id" structs:"id" sql:"type:string;primary_key;default:uuid_generate_v4()"` 83 | ExpiresAt time.Time `json:"expiresAt" structs:"expiresAt"` 84 | Deactivated bool `json:"deactivated" structs:"deactivated"` 85 | Description string `json:"description" structs:"description"` 86 | RequesterIP string `json:"requesterIP" structs:"requesterIP"` 87 | User string `json:"user" structs:"user"` 88 | AccessKeyName string `json:"accessKeyName" structs:"accessKeyName"` 89 | } 90 | 91 | func (ga GrantedAccess) IsNotExpired() bool { 92 | return time.Now().After(ga.ExpiresAt) 93 | } 94 | 95 | func (ga GrantedAccess) IsValid() bool { 96 | if ga.Deactivated { 97 | logrus.Warningf("IsValid(false): Account is deactivated [id=%v]", ga.ID) 98 | return false 99 | } 100 | 101 | if ga.DeletedAt.Valid { 102 | logrus.Warningf("IsValid(false): JWT deleted [id=%v]", ga.ID) 103 | return false 104 | } 105 | 106 | if ga.IsNotExpired() { 107 | logrus.Warningf("IsValid(false): JWT expired [id=%v]", ga.ID) 108 | return false 109 | } 110 | 111 | return true 112 | } 113 | 114 | func NewGrantedAccess(jwt string, expiresAt time.Time, deactivated bool, description string, requesterIP string, username string, accessKeyName string) GrantedAccess { 115 | return GrantedAccess{ 116 | ID: HashJWT(jwt), 117 | ExpiresAt: expiresAt, 118 | Deactivated: deactivated, 119 | Description: description, 120 | RequesterIP: requesterIP, 121 | User: username, 122 | AccessKeyName: accessKeyName, 123 | } 124 | } 125 | 126 | type UserIdentity struct { 127 | Username string 128 | AccessKeyName string 129 | } 130 | 131 | func NewUserIdentityFromString(login string) UserIdentity { 132 | if strings.Contains(login, "$") { 133 | lastIndex := strings.LastIndex(login, "$") 134 | return UserIdentity{ 135 | Username: login[:lastIndex], 136 | AccessKeyName: login[lastIndex+1:], 137 | } 138 | } 139 | return UserIdentity{ 140 | Username: login, 141 | AccessKeyName: "", 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /pkg/security/model_test.go: -------------------------------------------------------------------------------- 1 | package security_test 2 | 3 | import ( 4 | "github.com/riotkit-org/backup-repository/pkg/security" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestNewUserIdentityFromString_WithAccessKey(t *testing.T) { 10 | identity := security.NewUserIdentityFromString("hello$161") 11 | 12 | assert.Equal(t, "hello", identity.Username) 13 | assert.Equal(t, "161", identity.AccessKeyName) 14 | } 15 | 16 | func TestNewUserIdentityFromString(t *testing.T) { 17 | identity := security.NewUserIdentityFromString("my-name-is-borat") 18 | 19 | assert.Equal(t, "my-name-is-borat", identity.Username) 20 | assert.Equal(t, "", identity.AccessKeyName) 21 | } 22 | -------------------------------------------------------------------------------- /pkg/security/repository.go: -------------------------------------------------------------------------------- 1 | package security 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | "gorm.io/gorm" 6 | ) 7 | 8 | const KindGrantedAccess = "grantedaccesses" 9 | 10 | // 11 | // GrantedAccess 12 | // 13 | 14 | type GrantedAccessRepository struct { 15 | db *gorm.DB 16 | } 17 | 18 | func (g GrantedAccessRepository) create(access *GrantedAccess) error { 19 | return g.db.Model(&GrantedAccess{}).Create(&access).Error 20 | } 21 | 22 | func (g GrantedAccessRepository) getGrantedAccessByHashedToken(hashedToken interface{}) (GrantedAccess, error) { 23 | var gaModel GrantedAccess 24 | 25 | if result := g.db.Model(&GrantedAccess{}).First(&gaModel, "id = ?", hashedToken); result.Error != nil { 26 | logrus.Debugf("Cannot find GrantedAccess id=%v, error: %v", hashedToken, result.Error) 27 | 28 | return gaModel, result.Error 29 | } 30 | 31 | return gaModel, nil 32 | } 33 | 34 | func (g GrantedAccessRepository) checkSessionExistsById(id string) bool { 35 | var exists bool 36 | 37 | if err := g.db.Model(&GrantedAccess{}).Select("count(*) > 0").Where("id = ?", id).Find(&exists).Error; err != nil { 38 | logrus.Errorf("checkSessionExistsById(): %v", err) 39 | } 40 | 41 | return exists 42 | } 43 | 44 | func (g GrantedAccessRepository) revokeById(id string) error { 45 | return g.db.Model(&GrantedAccess{}).Where("id = ?", id).Update("deactivated", true).Error 46 | } 47 | 48 | func (g GrantedAccessRepository) findForUsername(name string) []GrantedAccess { 49 | var result []GrantedAccess 50 | g.db.Model(&GrantedAccess{}).Where("granted_accesses.user = ?", name).Order("created_at desc").Limit(100).Find(&result) 51 | 52 | return result 53 | } 54 | 55 | func (g GrantedAccessRepository) findOneBySessionId(id string) (GrantedAccess, error) { 56 | var result GrantedAccess 57 | return result, g.db.Model(&GrantedAccess{}).Where("granted_accesses.id = ?", id).Limit(1).Find(&result).Error 58 | } 59 | 60 | func InitializeModel(db *gorm.DB) error { 61 | return db.AutoMigrate(&GrantedAccess{}) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/security/service.go: -------------------------------------------------------------------------------- 1 | package security 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "github.com/riotkit-org/backup-repository/pkg/config" 9 | "github.com/sirupsen/logrus" 10 | "github.com/tidwall/gjson" 11 | "gorm.io/gorm" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | type Service struct { 17 | repository GrantedAccessRepository 18 | } 19 | 20 | func (s Service) StoreJWTAsGrantedAccess(token string, expire time.Time, ip string, description string, username string, accessKeyName string) string { 21 | ga := NewGrantedAccess(token, expire, false, description, ip, username, accessKeyName) 22 | if err := s.repository.create(&ga); err != nil { 23 | logrus.Errorf("Cannot store GrantedAccess. Possibly tried to store same JWT twice. IsError: %v", err) 24 | return "" 25 | } 26 | return ga.ID 27 | } 28 | 29 | func (s Service) IsTokenStillValid(jwt string) bool { 30 | ga, err := s.repository.getGrantedAccessByHashedToken(HashJWT(jwt)) 31 | if err != nil { 32 | logrus.Errorf("IsTokenValid(false): %v", err) 33 | return false 34 | } 35 | return ga.IsValid() 36 | } 37 | 38 | func (s Service) GetGrantedAccessInformation(jwt string) (GrantedAccess, error) { 39 | return s.repository.getGrantedAccessByHashedToken(HashJWT(jwt)) 40 | } 41 | 42 | func (s Service) RevokeSessionBySessionId(sessionId string) error { 43 | if !s.repository.checkSessionExistsById(sessionId) { 44 | return errors.New("specified session identifier is not valid, cannot find GrantedAccess of specified id") 45 | } 46 | 47 | return s.repository.revokeById(sessionId) 48 | } 49 | 50 | func (s Service) RevokeSessionByJWT(jwt string) error { 51 | return s.RevokeSessionBySessionId(HashJWT(jwt)) 52 | } 53 | 54 | func (s Service) GetAllGrantedAccessesForUserByUsername(name string) []GrantedAccess { 55 | return s.repository.findForUsername(name) 56 | } 57 | 58 | func (s Service) GetGrantedAccessInformationBySessionId(sessionId string) (GrantedAccess, error) { 59 | return s.repository.findOneBySessionId(sessionId) 60 | } 61 | 62 | func NewService(db *gorm.DB) Service { 63 | return Service{ 64 | repository: GrantedAccessRepository{ 65 | db: db, 66 | }, 67 | } 68 | } 69 | 70 | func extractJsonFromJWT(jwt string) (string, error) { 71 | // optionally extract token from Authorization header 72 | if strings.HasPrefix(jwt, "Bearer") { 73 | jwt = jwt[7:] 74 | } 75 | split := strings.SplitN(jwt, ".", 3) 76 | json, err := base64.RawStdEncoding.DecodeString(split[1]) 77 | if err != nil { 78 | return "", errors.New(fmt.Sprintf("cannot extract JSON from JWT: %s", err.Error())) 79 | } 80 | return string(json), nil 81 | } 82 | 83 | // ExtractLoginFromJWT returns username of a user that owns this token 84 | func ExtractLoginFromJWT(jwt string) (string, string) { 85 | json, err := extractJsonFromJWT(jwt) 86 | if err != nil { 87 | logrus.Warnf("invalid JWT format: %s", err.Error()) 88 | } 89 | 90 | username := gjson.Get(json, "login") 91 | accessKeyName := gjson.Get(json, "accessKeyName") 92 | 93 | return username.String(), accessKeyName.String() 94 | } 95 | 96 | func ExtractSessionLimitedOperationsScopeFromJWT(jwt string) (*SessionLimitedOperationsScope, error) { 97 | asJson, err := extractJsonFromJWT(jwt) 98 | if err != nil { 99 | logrus.Warnf("invalid JWT format: %s", err.Error()) 100 | } 101 | scopeJson := gjson.Get(asJson, ScopeClaimIndex) 102 | if scopeJson.Exists() { 103 | return ExtractScopeFromString(scopeJson.String()) 104 | } 105 | return &SessionLimitedOperationsScope{Elements: []ScopedElement{}}, nil 106 | } 107 | 108 | func ExtractScopeFromString(asJson string) (*SessionLimitedOperationsScope, error) { 109 | scope := &SessionLimitedOperationsScope{} 110 | scopeAsTxtJson, decodeErr := base64.StdEncoding.DecodeString(asJson) 111 | if decodeErr != nil { 112 | return nil, errors.New(fmt.Sprintf("cannot base64 decode operations scope from JWT: %s", decodeErr.Error())) 113 | } 114 | 115 | scopeErr := json.Unmarshal(scopeAsTxtJson, &scope) 116 | if scopeErr != nil { 117 | return nil, errors.New(fmt.Sprintf("cannot unpack operations scope from JWT: %s", scopeErr.Error())) 118 | } 119 | return scope, nil 120 | } 121 | 122 | // FillPasswordFromKindSecret is able to fill up object from a data retrieved from `kind: Secret` in Kubernetes 123 | func FillPasswordFromKindSecret(r config.ConfigurationProvider, ref *PasswordFromSecretRef, setterCallback func(secret string)) error { 124 | if ref.Name != "" { 125 | // todo: cache support 126 | secretDoc, secretErr := r.GetSingleDocumentAnyType("secrets", ref.Name, "", "v1") 127 | if secretErr != nil { 128 | logrus.Errorf("Cannot fetch user hashed password from `kind: Secret`. Maybe it does not exist? %v", secretErr) 129 | return secretErr 130 | } 131 | 132 | secret := gjson.Get(secretDoc, fmt.Sprintf("data.%v", ref.Entry)) 133 | if secret.String() == "" { 134 | logrus.Errorf( 135 | "Cannot retrieve password from `kind: Secret` of name '%v', field '%v'", 136 | ref.Name, 137 | ref.Entry, 138 | ) 139 | return errors.New("invalid field name in `kind: Secret`") 140 | } 141 | 142 | setterCallback(secret.String()) 143 | return nil 144 | } 145 | 146 | logrus.Warn("`kind: Secret` not used for entity") 147 | return nil 148 | } 149 | -------------------------------------------------------------------------------- /pkg/storage/.test_data/test.gpg: -------------------------------------------------------------------------------- 1 | -----BEGIN PGP MESSAGE----- 2 | YnVpbGQ6CglnbyBidWlsZCAtdGFncz1ub21zZ3BhY2sgLgoKdGVzdF9sb2dpbjoKCWN1cmwgLXMg 3 | LVggUE9TVCAtZCAneyJ1c2VybmFtZSI6ImFkbWluIiwicGFzc3dvcmQiOiJhZG1pbiJ9JyAtSCAn 4 | Q29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29uJyAnaHR0cDovL2xvY2FsaG9zdDo4MDgwL2Fw 5 | aS9zdGFibGUvYXV0aC9sb2dpbicKCnRlc3RfbG9naW5fc29tZV91c2VyOgoJY3VybCAtcyAtWCBQ 6 | T1NUIC1kICd7InVzZXJuYW1lIjoic29tZS11c2VyIiwicGFzc3dvcmQiOiJhZG1pbiJ9JyAtSCAn 7 | Q29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29uJyAnaHR0cDovL2xvY2FsaG9zdDo4MDgwL2Fw 8 | aS9zdGFibGUvYXV0aC9sb2dpbicKCnRlc3RfbG9va3VwOgoJY3VybCAtcyAtWCBHRVQgLUggJ0F1 9 | dGhvcml6YXRpb246IEJlYXJlciAke1RPS0VOfScgLUggJ0NvbnRlbnQtVHlwZTogYXBwbGljYXRp 10 | b24vanNvbicgJ2h0dHA6Ly9sb2NhbGhvc3Q6ODA4MC9hcGkvc3RhYmxlL2F1dGgvdXNlci9zb21l 11 | LXVzZXInCgp0ZXN0X3dob2FtaToKCWN1cmwgLXMgLVggR0VUIC1IICdBdXRob3JpemF0aW9uOiBC 12 | ZWFyZXIgJHtUT0tFTn0nIC1IICdDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24nICdodHRw 13 | Oi8vbG9jYWxob3N0OjgwODAvYXBpL3N0YWJsZS9hdXRoL3dob2FtaScKCnRlc3RfbG9nb3V0OgoJ 14 | Y3VybCAtcyAtWCBERUxFVEUgLUggJ0F1dGhvcml6YXRpb246IEJlYXJlciAke1RPS0VOfScgLUgg 15 | J0NvbnRlbnQtVHlwZTogYXBwbGljYXRpb24vanNvbicgJ2h0dHA6Ly9sb2NhbGhvc3Q6ODA4MC9h 16 | cGkvc3RhYmxlL2F1dGgvbG9nb3V0JwoKdGVzdF9sb2dvdXRfb3RoZXJfdXNlcjoKCWN1cmwgLXMg 17 | LVggREVMRVRFIC1IICdBdXRob3JpemF0aW9uOiBCZWFyZXIgJHtUT0tFTn0nIC1IICdDb250ZW50 18 | LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24nICdodHRwOi8vbG9jYWxob3N0OjgwODAvYXBpL3N0YWJs 19 | ZS9hdXRoL2xvZ291dD9zZXNzaW9uSWQ9JHtPVEhFUl9VU0VSX1NFU1NJT05fSUR9JwoKdGVzdF9s 20 | aXN0X2F1dGhzOgoJY3VybCAtcyAtWCBHRVQgLUggJ0F1dGhvcml6YXRpb246IEJlYXJlciAke1RP 21 | S0VOfScgLUggJ0NvbnRlbnQtVHlwZTogYXBwbGljYXRpb24vanNvbicgJ2h0dHA6Ly9sb2NhbGhv 22 | c3Q6ODA4MC9hcGkvc3RhYmxlL2F1dGgvdG9rZW4nCgp0ZXN0X2xpc3RfYXV0aHNfb3RoZXJfdXNl 23 | cjoKCWN1cmwgLXMgLVggR0VUIC1IICdBdXRob3JpemF0aW9uOiBCZWFyZXIgJHtUT0tFTn0nIC1I 24 | ICdDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24nICdodHRwOi8vbG9jYWxob3N0OjgwODAv 25 | YXBpL3N0YWJsZS9hdXRoL3Rva2VuP3VzZXJOYW1lPXNvbWUtdXNlcicKCnRlc3RfdXBsb2FkX2J5 26 | X2Zvcm06CgljdXJsIC1zIC1YIFBPU1QgLUggJ0F1dGhvcml6YXRpb246IEJlYXJlciAke1RPS0VO 27 | fScgLUYgImZpbGU9QE1ha2VmaWxlIiAnaHR0cDovL2xvY2FsaG9zdDo4MDgwL2FwaS9zdGFibGUv 28 | cmVwb3NpdG9yeS9jb2xsZWN0aW9uL2l3YS1haXQvdmVyc2lvbicKCnBvc3RncmVzOgoJZG9ja2Vy 29 | IHJ1biAtZCBcCiAgICAgICAgLS1uYW1lIGJyX3Bvc3RncmVzIFwKICAgICAgICAtZSBQT1NUR1JF 30 | U19QQVNTV09SRD1wb3N0Z3JlcyBcCiAgICAgICAgLWUgUE9TVEdSRVNfVVNFUj1wb3N0Z3JlcyBc 31 | CiAgICAgICAgLWUgUE9TVEdSRVNfREI9cG9zdGdyZXMgXAogICAgICAgIC1lIFBHREFUQT0vdmFy 32 | L2xpYi9wb3N0Z3Jlc3FsL2RhdGEvcGdkYXRhIFwKICAgICAgICAtdiAvdG1wL2JyX3Bvc3RncmVz 33 | Oi92YXIvbGliL3Bvc3RncmVzcWwvZGF0YSBcCiAgICAgICAgLXAgNTQzMjo1NDMyIFwKICAgICAg 34 | ICBwb3N0Z3JlczoxNC4xLWFscGluZQoKcG9zdGdyZXNfcmVmcmVzaDoKCWRvY2tlciBybSAtZiBi 35 | cl9wb3N0Z3JlcyB8fCB0cnVlCglzdWRvIHJtIC1yZiAvdG1wL2JyX3Bvc3RncmVzCgltYWtlIHBv 36 | c3RncmVzCgptaW5pbzoKCWRvY2tlciBydW4gLWQgXAoJCS0tbmFtZSBicl9taW5pbyBcCgkJLXAg 37 | OTAwMDo5MDAwIFwKCSAgICAtcCA5MDAxOjkwMDEgXAoJICAgIC12IC90bXAvYnJfbWluaW86L2Rh 38 | dGEgXAoJICAgIC1lICJNSU5JT19ST09UX1VTRVI9QUtJQUlPU0ZPRE5ON0VYQU1QTEUiIFwKCSAg 39 | ICAtZSAiTUlOSU9fUk9PVF9QQVNTV09SRD13SmFGdUNLdG5GRU1JL0NBcEl0YWxpU00vYlB4UmZp 40 | Q1lFWEFNUExFS0VZIiBcCgkJcXVheS5pby9taW5pby9taW5pbzpSRUxFQVNFLjIwMjItMDItMTZU 41 | -----END PGP MESSAGE----- 42 | -------------------------------------------------------------------------------- /pkg/storage/entity.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "gorm.io/gorm" 5 | "time" 6 | ) 7 | 8 | type UploadedVersion struct { 9 | Id string `json:"id" structs:"id" sql:"type:string;primary_key;default:uuid_generate_v4()` 10 | CollectionId string `json:"collectionId"` 11 | VersionNumber int `json:"versionNumber"` 12 | Filename string `json:"filename"` // full filename e.g. iwa-ait-v1-db.tar.gz 13 | Filesize int64 `json:"filesize"` // in bytes 14 | 15 | // auditing 16 | UploadedBySessionId string `json:"uploadedBySessionId"` 17 | Uploader string `json:"user" structs:"user"` 18 | CreatedAt time.Time 19 | UpdatedAt time.Time 20 | DeletedAt gorm.DeletedAt `gorm:"index"` 21 | } 22 | 23 | func (u *UploadedVersion) GetTargetPath() string { 24 | return u.CollectionId + "/" + u.Filename 25 | } 26 | -------------------------------------------------------------------------------- /pkg/storage/io.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "github.com/sirupsen/logrus" 8 | "gocloud.dev/blob" 9 | "io" 10 | ) 11 | 12 | func (s *Service) UploadFile(parentCtx context.Context, inputStream io.ReadCloser, version *UploadedVersion, middlewares *NestedStreamMiddlewares) (int64, error) { 13 | ctx, cancellation := context.WithTimeout(parentCtx, s.IOTimeout) 14 | defer cancellation() 15 | 16 | writeStream, err := s.storage.NewWriter(ctx, version.GetTargetPath(), &blob.WriterOptions{}) 17 | defer func() { _ = writeStream.Close() }() 18 | 19 | if err != nil { 20 | return 0, errors.New(fmt.Sprintf("cannot upload file, attempted to open a writable stream, error: %v", err)) 21 | } 22 | 23 | wroteLen, writeErr := s.CopyStream(ctx, inputStream, writeStream, 1024*1024, middlewares) 24 | if writeErr != nil { 25 | return wroteLen, errors.New(fmt.Sprintf("cannot upload file, cannot copy stream, error: %v", writeErr)) 26 | } 27 | 28 | // Check if file exists at the storage 29 | _ = writeStream.Close() 30 | if exists, err := s.storage.Exists(ctx, version.GetTargetPath()); !exists || err != nil { 31 | logrus.Error(fmt.Sprintf("file was uploaded but does not exists on the storage at path '%v'. IsError: %v", version.GetTargetPath(), err)) 32 | return wroteLen, errors.New("storage error") 33 | } 34 | 35 | // Check if filesize matches buffered stream size 36 | attributes, err := s.storage.Attributes(ctx, version.GetTargetPath()) 37 | if attributes.Size != wroteLen { 38 | logrus.Errorln(fmt.Sprintf("file written to the storage does not match uploaded file, the filesize is not matching %v != %v for file '%v'", wroteLen, attributes.Size, version.GetTargetPath())) 39 | return wroteLen, errors.New("storage error") 40 | } 41 | 42 | return wroteLen, nil 43 | } 44 | 45 | // CopyStream copies a readable stream to writable stream, while providing a possibility to use a validation callbacks on-the-fly 46 | func (s *Service) CopyStream(ctx context.Context, inputStream io.ReadCloser, writeStream io.Writer, bufferLen int, middlewares *NestedStreamMiddlewares) (int64, error) { 47 | buff := make([]byte, bufferLen) 48 | previousBuff := make([]byte, bufferLen) 49 | var totalLength int64 50 | chunkNum := 0 51 | 52 | for { 53 | select { 54 | case <-ctx.Done(): 55 | deadline, _ := ctx.Deadline() 56 | logrus.Errorf("Upload hit a context cancellation: %s, deadline: %s", ctx.Err(), deadline) 57 | return totalLength, errors.New("context deadline exceeded, probably hit a Storage I/O timeout or Request timeout") 58 | 59 | default: 60 | n, err := inputStream.Read(buff) 61 | chunkNum += 1 62 | 63 | if err != nil { 64 | if err == io.EOF { 65 | totalLength += int64(len(buff[:n])) 66 | 67 | // validation callbacks 68 | if err := middlewares.processChunk(buff[:n], totalLength, previousBuff, chunkNum); err != nil { 69 | return totalLength, err 70 | } 71 | 72 | // write to target stream (copy) 73 | if _, writeErr := writeStream.Write(buff[:n]); writeErr != nil { 74 | return totalLength, writeErr 75 | } 76 | 77 | return totalLength, nil 78 | } 79 | 80 | return totalLength, errors.New(fmt.Sprintf("cannot copy stream, error: %v", err)) 81 | } 82 | 83 | totalLength += int64(len(buff[:n])) 84 | previousBuff = buff[:n] 85 | 86 | // validation callbacks 87 | if err := middlewares.processChunk(buff[:n], totalLength, []byte(""), chunkNum); err != nil { 88 | return totalLength, err 89 | } 90 | 91 | // write to target stream (copy) 92 | _, writeErr := writeStream.Write(buff[:n]) 93 | if writeErr != nil { 94 | return totalLength, writeErr 95 | } 96 | if err := middlewares.checkFinalStatusAfterFilesWasUploaded(); err != nil { 97 | return totalLength, err 98 | } 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /pkg/storage/io_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "github.com/stretchr/testify/assert" 7 | "io" 8 | "strings" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | // Basic test for reading small portion of data 14 | func TestService_CopyStream(t *testing.T) { 15 | s := Service{} 16 | 17 | ctx, _ := context.WithTimeout(context.TODO(), time.Second*5) 18 | 19 | readStream := io.NopCloser(strings.NewReader("hello-world")) 20 | var writeStream bytes.Buffer 21 | buff := make([]byte, 11) 22 | 23 | _, _ = s.CopyStream(ctx, readStream, &writeStream, 1024, &NestedStreamMiddlewares{}) 24 | _, _ = writeStream.Read(buff) 25 | 26 | assert.Equal(t, "hello-world", string(buff)) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/storage/repository.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "gorm.io/gorm" 5 | ) 6 | 7 | // VersionsRepository persistence layer for Versions 8 | type VersionsRepository struct { 9 | db *gorm.DB 10 | } 11 | 12 | // findLastHighestVersionNumber Finds latest backup's version number 13 | func (vr VersionsRepository) findLastHighestVersionNumber(collectionId string) (int, error) { 14 | maxNum := 0 15 | err := vr.db.Model(&UploadedVersion{}).Select("uploaded_versions.version_number").Where("uploaded_versions.collection_id = ?", collectionId).Order("uploaded_versions.version_number DESC").Limit(1).Find(&maxNum).Error 16 | if err != nil { 17 | return 0, err 18 | } 19 | return maxNum, nil 20 | } 21 | 22 | // findAllVersionsForCollectionId Finds multiple versions for given collection by it's id 23 | func (vr VersionsRepository) findAllVersionsForCollectionId(collectionId string) ([]UploadedVersion, error) { 24 | var foundVersions []UploadedVersion 25 | 26 | err := vr.db.Model(&UploadedVersion{}).Where("uploaded_versions.collection_id = ?", collectionId).Order("uploaded_versions.version_number DESC").Find(&foundVersions).Error 27 | if err != nil { 28 | return []UploadedVersion{}, err 29 | } 30 | return foundVersions, nil 31 | } 32 | 33 | // delete Deletes entry from database 34 | func (vr VersionsRepository) delete(version *UploadedVersion) (error, bool) { 35 | var result bool 36 | return vr.db.Model(&UploadedVersion{}).Where("uploaded_versions.id = ?", version.Id).Delete(&result).Error, result 37 | } 38 | 39 | // create Creates an entry in database 40 | func (vr VersionsRepository) create(version *UploadedVersion) error { 41 | return vr.db.Create(version).Error 42 | } 43 | 44 | func (vr VersionsRepository) getByVersionNum(id string, versionNum string) (UploadedVersion, error) { 45 | var version UploadedVersion 46 | err := vr.db.Model(&UploadedVersion{}).Where("uploaded_versions.collection_id = ? AND uploaded_versions.version_number = ?", id, versionNum).Limit(1).Find(&version).Error 47 | return version, err 48 | } 49 | 50 | func (vr VersionsRepository) findAllActiveVersions(id string) ([]UploadedVersion, error) { 51 | var results []UploadedVersion 52 | err := vr.db.Model(&UploadedVersion{}).Where("uploaded_versions.collection_id = ?", id).Find(&results).Error 53 | return results, err 54 | } 55 | 56 | // InitializeModel connects model to migrations 57 | func InitializeModel(db *gorm.DB) error { 58 | return db.AutoMigrate(&UploadedVersion{}) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/storage/rotation.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "github.com/riotkit-org/backup-repository/pkg/collections" 5 | "sort" 6 | ) 7 | 8 | // RotationStrategy defines how old backups should be rotated 9 | type RotationStrategy interface { 10 | // CanUpload returns nil if YES, error if NO 11 | CanUpload(version UploadedVersion) error 12 | 13 | // GetVersionsThatShouldBeDeletedIfThisVersionUploaded lists all the versions that should be deleted if a new version would be submitted 14 | GetVersionsThatShouldBeDeletedIfThisVersionUploaded(version UploadedVersion) []UploadedVersion 15 | } 16 | 17 | // FifoRotationStrategy implements a simple queue, first is appended, oldest will be deleted 18 | type FifoRotationStrategy struct { 19 | collection *collections.Collection 20 | existingVersions []UploadedVersion 21 | } 22 | 23 | // CanUpload RotationStrategy can decide if we can still upload 24 | func (frs *FifoRotationStrategy) CanUpload(version UploadedVersion) error { 25 | return nil 26 | } 27 | 28 | // GetVersionsThatShouldBeDeletedIfThisVersionUploaded interface implementation that allows RotationStrategy to decide which versions should be deleted right now, when uploading a new version 29 | func (frs *FifoRotationStrategy) GetVersionsThatShouldBeDeletedIfThisVersionUploaded(version UploadedVersion) []UploadedVersion { 30 | existingVersions := frs.existingVersions 31 | 32 | // nothing to do, there is still enough slots 33 | if len(existingVersions) < frs.collection.Spec.MaxBackupsCount { 34 | return []UploadedVersion{} 35 | } 36 | 37 | // order by version number DESCENDING 38 | sort.SliceStable(existingVersions, func(i, j int) bool { 39 | return existingVersions[i].VersionNumber < existingVersions[j].VersionNumber 40 | }) 41 | 42 | toDelete := len(existingVersions) - frs.collection.Spec.MaxBackupsCount 43 | toDelete = toDelete + 1 // consider tha we are going to upload new version now 44 | 45 | // oldest element 46 | return existingVersions[0:toDelete] 47 | } 48 | 49 | // NewFifoRotationStrategy is a factorial method 50 | func NewFifoRotationStrategy(collection *collections.Collection, existingVersions []UploadedVersion) *FifoRotationStrategy { 51 | return &FifoRotationStrategy{ 52 | collection: collection, 53 | existingVersions: existingVersions, 54 | } 55 | } 56 | 57 | // todo: implement "fifo-plus-older" 58 | -------------------------------------------------------------------------------- /pkg/storage/rotation_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "github.com/riotkit-org/backup-repository/pkg/collections" 6 | "github.com/stretchr/testify/assert" 7 | "testing" 8 | ) 9 | 10 | func TestNewFifoRotationStrategy_MutationTest(t *testing.T) { 11 | collectionFirst := collections.Collection{} 12 | versionsFirst := []UploadedVersion{ 13 | {Filename: "1.1"}, 14 | } 15 | 16 | collectionSecond := collections.Collection{} 17 | versionsSecond := []UploadedVersion{ 18 | {Filename: "2.1"}, 19 | } 20 | 21 | // one object from factory will not impact second 22 | strategyFirst := NewFifoRotationStrategy(&collectionFirst, versionsFirst) 23 | strategySecond := NewFifoRotationStrategy(&collectionSecond, versionsSecond) 24 | 25 | strategyFirst.collection.Metadata.Name = "first" 26 | strategySecond.collection.Metadata.Name = "second" 27 | 28 | assert.NotEqual(t, collectionFirst.Metadata.Name, collectionSecond.Metadata.Name) 29 | assert.NotEqual(t, strategyFirst, strategySecond) 30 | assert.NotEqual(t, &strategyFirst, &strategySecond) 31 | } 32 | 33 | func TestGetVersionsThatShouldBeDeletedIfThisVersionUploaded(t *testing.T) { 34 | collection := collections.Collection{} 35 | 36 | // given we have a collection with 5 backups already 37 | versions := []UploadedVersion{ 38 | {VersionNumber: 1}, 39 | {VersionNumber: 2}, 40 | {VersionNumber: 3}, 41 | {VersionNumber: 4}, 42 | {VersionNumber: 5}, 43 | } 44 | 45 | strategyFirst := NewFifoRotationStrategy(&collection, versions) 46 | 47 | // multiple test cases 48 | matrix := make(map[int]string) 49 | // matrix[maxBackupsCount] = expectedVersionNumbersToBeDeleted 50 | matrix[1] = "1,2,3,4,5," 51 | matrix[2] = "1,2,3,4," 52 | matrix[3] = "1,2,3," 53 | matrix[4] = "1,2," 54 | matrix[5] = "1," 55 | 56 | for maxBackupsCount, expectedResult := range matrix { 57 | collection.Spec.MaxBackupsCount = maxBackupsCount 58 | toDelete := strategyFirst.GetVersionsThatShouldBeDeletedIfThisVersionUploaded(UploadedVersion{}) 59 | toDeleteAsStr := "" 60 | 61 | for _, version := range toDelete { 62 | toDeleteAsStr += fmt.Sprintf("%v", version.VersionNumber) + "," 63 | } 64 | 65 | assert.Equal(t, expectedResult, toDeleteAsStr) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /pkg/storage/validation.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | // 12 | // Stream middleware is a pair of callbacks that are invoked during buffering and after finished buffering 13 | // of streamed upload 14 | // 15 | 16 | type streamMiddleware struct { 17 | // []byte - current buffer value 18 | // int64 - total read size till this moment 19 | // []byte - if current buffer is an END OF STREAM, then this parameter will contain previous hunk, 20 | // so you can join previous+last to have full information in case, when last hunk would be too small 21 | // int - processed chunk number 22 | processor func([]byte, int64, []byte, int) error 23 | resultReporter func() error 24 | } 25 | 26 | // 27 | // Aggregation of middlewares 28 | // 29 | 30 | type NestedStreamMiddlewares []streamMiddleware 31 | 32 | func (nv NestedStreamMiddlewares) processChunk(chunk []byte, processedTotalBytes int64, previousHunkBeforeEof []byte, chunkNum int) error { 33 | for _, processor := range nv { 34 | if processingError := processor.processor(chunk, processedTotalBytes, previousHunkBeforeEof, chunkNum); processingError != nil { 35 | return processingError 36 | } 37 | } 38 | return nil 39 | } 40 | 41 | func (nv NestedStreamMiddlewares) checkFinalStatusAfterFilesWasUploaded() error { 42 | for _, processor := range nv { 43 | if processingError := processor.resultReporter(); processingError != nil { 44 | return processingError 45 | } 46 | } 47 | return nil 48 | } 49 | 50 | // 51 | // Validators 52 | // 53 | 54 | // createGPGStreamMiddleware Checks if stream is a valid GPG encrypted file by checking GPG header and footer 55 | func (s *Service) createGPGStreamMiddleware() streamMiddleware { 56 | validator := func(buff []byte, totalLength int64, previousHunkBeforeEof []byte, chunkNum int) error { 57 | if chunkNum == 1 && !bytes.Contains(buff, []byte("-----BEGIN PGP MESSAGE")) { 58 | return errors.New("first chunk of uploaded data does not contain a valid GPG header") 59 | } 60 | 61 | // if previous hunk is not empty, then we are at the end of the stream 62 | if len(previousHunkBeforeEof) > 0 { 63 | concatenated := [][]byte{previousHunkBeforeEof, buff} 64 | 65 | if !bytes.Contains(bytes.Join(concatenated, []byte("")), []byte("-----END PGP MESSAGE")) { 66 | return errors.New("end of stream does not contain a valid GPG footer, suspecting a data corruption") 67 | } 68 | } 69 | return nil 70 | } 71 | 72 | resultReporter := func() error { 73 | return nil 74 | } 75 | 76 | return streamMiddleware{processor: validator, resultReporter: resultReporter} 77 | } 78 | 79 | // createNonEmptyMiddleware Checks if anything was sent at all 80 | func (s *Service) createNonEmptyMiddleware() streamMiddleware { 81 | var recordedTotalLength int64 82 | 83 | validator := func(buff []byte, totalLength int64, previousHunkBeforeEof []byte, chunkNum int) error { 84 | recordedTotalLength = totalLength 85 | return nil 86 | } 87 | 88 | resultReporter := func() error { 89 | if recordedTotalLength == 0 { 90 | return errors.New("sent empty data") 91 | } 92 | 93 | return nil 94 | } 95 | 96 | return streamMiddleware{processor: validator, resultReporter: resultReporter} 97 | } 98 | 99 | // createQuotaMaxFileSizeMiddleware Takes care about the maximum allowed filesize limit 100 | func (s *Service) createQuotaMaxFileSizeMiddleware(maxFileSize int64) streamMiddleware { 101 | validator := func(buff []byte, totalLength int64, previousHunkBeforeEof []byte, chunkNum int) error { 102 | if totalLength > maxFileSize { 103 | return errors.New(fmt.Sprintf("filesize reached allowed limit. Uploaded %v bytes, allowed to upload only %v bytes", totalLength, maxFileSize)) 104 | } 105 | return nil 106 | } 107 | 108 | return streamMiddleware{processor: validator, resultReporter: func() error { 109 | return nil 110 | }} 111 | } 112 | 113 | // createRequestCancelledMiddleware handles the request cancellation 114 | func (s *Service) createRequestCancelledMiddleware(context context.Context) streamMiddleware { 115 | return streamMiddleware{ 116 | processor: func(i []byte, i2 int64, i3 []byte, i4 int) error { 117 | if context.Err() != nil { 118 | logrus.Warning(fmt.Sprintf("Upload was cancelled: %v", context.Err())) 119 | return errors.New("upload was cancelled") 120 | } 121 | return nil 122 | }, 123 | resultReporter: func() error { 124 | return nil 125 | }, 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /pkg/storage/validation_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "golang.org/x/net/context" 6 | "testing" 7 | ) 8 | 9 | func TestQuotaMaxFileSizeMiddleware(t *testing.T) { 10 | s := Service{} 11 | middleware := s.createQuotaMaxFileSizeMiddleware(int64(1024)) 12 | 13 | // below limit 14 | assert.Nil(t, middleware.processor([]byte("current-chunk"), int64(0), []byte(""), 1)) 15 | assert.Nil(t, middleware.processor([]byte("current-chunk"), int64(600), []byte(""), 1)) 16 | 17 | // above limit 18 | assert.Equal(t, "filesize reached allowed limit. Uploaded 1025 bytes, allowed to upload only 1024 bytes", middleware.processor([]byte("current-chunk"), int64(1025), []byte(""), 1).Error()) 19 | 20 | assert.Nil(t, middleware.resultReporter()) // this should do nothing 21 | } 22 | 23 | func TestGPGStreamMiddleware(t *testing.T) { 24 | s := Service{} 25 | middleware := s.createGPGStreamMiddleware() 26 | 27 | // beginning 28 | assert.Nil(t, middleware.processor([]byte("-----BEGIN PGP MESSAGE .......hello......"), int64(0), []byte(""), 1)) 29 | assert.NotNil(t, middleware.processor([]byte("hello"), int64(0), []byte(""), 1)) 30 | 31 | // ending 32 | assert.Nil(t, middleware.processor([]byte("-----END PGP MESSAGE"), int64(0), []byte("previous-hunk"), 161)) // previous-hunk is non-empty, when END OF STREAM is happening 33 | assert.NotNil(t, middleware.processor([]byte("-------broken-ending"), int64(0), []byte("previous-hunk"), 161)) 34 | 35 | assert.Nil(t, middleware.resultReporter()) // this should do nothing 36 | } 37 | 38 | func TestNonEmptyMiddleware(t *testing.T) { 39 | s := Service{} 40 | middleware := s.createNonEmptyMiddleware() 41 | 42 | // at the beginning it will raise error 43 | assert.NotNil(t, middleware.resultReporter()) 44 | 45 | // after processing an empty chunk it will still report error 46 | _ = middleware.processor([]byte(""), int64(0), []byte(""), 1) 47 | assert.NotNil(t, middleware.resultReporter()) 48 | 49 | // but after processing at least one non-empty chunk it will be fine 50 | _ = middleware.processor([]byte("hello"), int64(5), []byte(""), 1) 51 | assert.Nil(t, middleware.resultReporter()) 52 | } 53 | 54 | func TestRequestCancelledMiddleware(t *testing.T) { 55 | s := Service{} 56 | 57 | ctx, cancel := context.WithCancel(context.TODO()) 58 | cancel() 59 | 60 | // cancelled 61 | middlewareCanceled := s.createRequestCancelledMiddleware(ctx) 62 | assert.NotNil(t, middlewareCanceled.processor([]byte("hello"), int64(5), []byte(""), 1)) 63 | 64 | // not cancelled 65 | middlewareNotCancelled := s.createRequestCancelledMiddleware(context.TODO()) 66 | assert.Nil(t, middlewareNotCancelled.processor([]byte("hello"), int64(5), []byte(""), 1)) 67 | 68 | assert.Nil(t, middlewareNotCancelled.resultReporter()) // this should do nothing 69 | } 70 | 71 | func TestNestedStreamMiddlewares(t *testing.T) { 72 | s := Service{} 73 | 74 | middlewares := NestedStreamMiddlewares{ 75 | s.createNonEmptyMiddleware(), 76 | s.createQuotaMaxFileSizeMiddleware(int64(1024)), 77 | } 78 | 79 | _ = middlewares.processChunk([]byte(""), int64(0), []byte(""), 1) 80 | assert.NotNil(t, middlewares.checkFinalStatusAfterFilesWasUploaded()) 81 | assert.Nil(t, middlewares.processChunk([]byte("hello"), int64(5), []byte(""), 1)) 82 | assert.Nil(t, middlewares.checkFinalStatusAfterFilesWasUploaded()) 83 | } 84 | -------------------------------------------------------------------------------- /pkg/users/model.go: -------------------------------------------------------------------------------- 1 | package users 2 | 3 | import ( 4 | "github.com/riotkit-org/backup-repository/pkg/config" 5 | "github.com/riotkit-org/backup-repository/pkg/security" 6 | "github.com/sirupsen/logrus" 7 | ) 8 | 9 | type AccessKey struct { 10 | Name string `json:"name"` 11 | Password string `json:"password"` // master password to this account, allows access limited by 12 | PasswordFromRef security.PasswordFromSecretRef `json:"passwordFromRef"` 13 | Objects []security.AccessControlObject `json:"objects"` 14 | } 15 | 16 | type Spec struct { 17 | Id string `json:"id"` 18 | Email string `json:"email"` 19 | Roles security.Roles `json:"roles"` 20 | Password string `json:"password"` // master password to this account, allows access limited by 21 | PasswordFromRef security.PasswordFromSecretRef `json:"passwordFromRef"` 22 | AccessKeys []*AccessKey `json:"accessKeys"` 23 | } 24 | 25 | type User struct { 26 | Metadata config.ObjectMetadata `json:"metadata"` 27 | Spec Spec `json:"spec"` 28 | 29 | // Dynamic property: User's password hash 30 | passwordFromSecret string 31 | } 32 | 33 | func (u User) GetRoles() security.Roles { 34 | return u.Spec.Roles 35 | } 36 | 37 | // getPasswordHash is returning a hashed password (Argon2 hash for comparison) 38 | func (u User) getPasswordHash() string { 39 | if u.passwordFromSecret != "" { 40 | return u.passwordFromSecret 41 | } 42 | return u.Spec.Password 43 | } 44 | 45 | // IsPasswordValid is checking if User supplied password matches User's main password 46 | func (u User) isPasswordValid(password string) bool { 47 | result, err := security.ComparePassword(password, u.getPasswordHash()) 48 | if err != nil { 49 | logrus.Errorf("Cannot decode password: '%v'", err) 50 | } 51 | return result 52 | } 53 | 54 | // 55 | // Security / RBAC 56 | // 57 | 58 | // CanViewMyProfile RBAC method 59 | func (u User) CanViewMyProfile(actor security.Actor) bool { 60 | // rbac 61 | if actor.GetRoles().HasRole(security.RoleUserManager) { 62 | return true 63 | } 64 | 65 | // user can view self info 66 | return u.Spec.Email == actor.GetEmail() 67 | } 68 | 69 | func NewSessionAwareUser(u *User, scope *security.SessionLimitedOperationsScope) *SessionAwareUser { 70 | return &SessionAwareUser{ 71 | User: u, 72 | sessionScope: scope, 73 | } 74 | } 75 | 76 | type SessionAwareUser struct { 77 | *User 78 | 79 | // Dynamic property: Copy of .spec.CollectionAccessKeys with password field filled up 80 | accessKeysFromSecret []*AccessKey 81 | 82 | // Dynamic property: Access key used in current session 83 | currentAccessKey *AccessKey 84 | 85 | // Dynamic property: Read from JWT token - operations scope, limited per session/token 86 | sessionScope *security.SessionLimitedOperationsScope 87 | } 88 | 89 | func (sau *SessionAwareUser) GetSessionLimitedOperationsScope() *security.SessionLimitedOperationsScope { 90 | return sau.sessionScope 91 | } 92 | 93 | func (sau *SessionAwareUser) GetEmail() string { 94 | return sau.Spec.Email 95 | } 96 | 97 | func (sau *SessionAwareUser) GetTypeName() string { 98 | return "user" 99 | } 100 | 101 | func (sau *SessionAwareUser) IsInAccessKeyContext() bool { 102 | return sau.currentAccessKey != nil 103 | } 104 | 105 | func (sau *SessionAwareUser) GetAccessKeyRolesInContextOf(subject security.Subject) security.Roles { 106 | if sau.currentAccessKey != nil { 107 | for _, object := range sau.currentAccessKey.Objects { 108 | if object.Type == subject.GetTypeName() && object.Name == subject.GetId() { 109 | return object.Roles 110 | } 111 | } 112 | } 113 | return security.Roles{} 114 | } 115 | 116 | // IsPasswordValid is checking if User supplied password matches User's main password, 117 | // or AccessKey password - depending on accessKeyName parameter 118 | func (sau *SessionAwareUser) IsPasswordValid(password string, accessKeyName string) bool { 119 | if accessKeyName != "" { 120 | for _, accessKey := range sau.accessKeysFromSecret { 121 | if accessKey.Name == accessKeyName { 122 | result, err := security.ComparePassword(password, accessKey.Password) 123 | if err != nil { 124 | logrus.Errorf("Cannot decode access key: '%v'", err) 125 | } 126 | return result 127 | } 128 | } 129 | logrus.Warnf("Invalid access key '%s' requested for user '%s'", accessKeyName, sau.Metadata.Name) 130 | return false 131 | } 132 | 133 | return sau.isPasswordValid(password) 134 | } 135 | 136 | func (sau *SessionAwareUser) GetRoles() security.Roles { 137 | return sau.User.GetRoles() 138 | } 139 | 140 | func (sau *SessionAwareUser) GetName() string { 141 | return sau.User.Metadata.Name 142 | } 143 | -------------------------------------------------------------------------------- /pkg/users/repository.go: -------------------------------------------------------------------------------- 1 | package users 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/pkg/errors" 7 | "github.com/riotkit-org/backup-repository/pkg/config" 8 | "github.com/riotkit-org/backup-repository/pkg/security" 9 | ) 10 | 11 | const KindBackupUser = "backupusers" 12 | 13 | type userRepository struct { 14 | config.ConfigurationProvider 15 | } 16 | 17 | func (r userRepository) findUserByLogin(login string) (*User, error) { 18 | doc, retrieveErr := r.GetSingleDocument(KindBackupUser, login) 19 | user := User{} 20 | if retrieveErr != nil { 21 | return &user, errors.New(fmt.Sprintf("IsError retrieving user: %v", retrieveErr)) 22 | } 23 | 24 | if err := json.Unmarshal([]byte(doc), &user); err != nil { 25 | return &User{}, err 26 | } 27 | if hydrateErr := r.hydrate(&user); hydrateErr != nil { 28 | return &User{}, hydrateErr 29 | } 30 | 31 | return &user, nil 32 | } 33 | 34 | func (r userRepository) hydrate(user *User) error { 35 | // password 36 | passwordSetter := func(password string) { 37 | user.passwordFromSecret = password 38 | } 39 | if fillErr := security.FillPasswordFromKindSecret(r, &user.Spec.PasswordFromRef, passwordSetter); fillErr != nil { 40 | return errors.Wrap(fillErr, "cannot fetch password") 41 | } 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /pkg/users/service.go: -------------------------------------------------------------------------------- 1 | package users 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/riotkit-org/backup-repository/pkg/config" 6 | "github.com/riotkit-org/backup-repository/pkg/security" 7 | ) 8 | 9 | type Service struct { 10 | repository userRepository 11 | config config.ConfigurationProvider 12 | } 13 | 14 | // NewUsersService is a factory method 15 | func NewUsersService(provider config.ConfigurationProvider) *Service { 16 | return &Service{ 17 | repository: userRepository{provider}, 18 | config: provider, 19 | } 20 | } 21 | 22 | func (a *Service) LookupUser(identity security.UserIdentity) (*User, error) { 23 | return a.repository.findUserByLogin(identity.Username) 24 | } 25 | 26 | func (a *Service) LookupSessionUser(identity security.UserIdentity, scope *security.SessionLimitedOperationsScope) (*SessionAwareUser, error) { 27 | user, findErr := a.repository.findUserByLogin(identity.Username) 28 | if findErr != nil { 29 | return nil, errors.Wrap(findErr, "LookupSessionUser error, cannot find user") 30 | } 31 | 32 | saUser := NewSessionAwareUser(user, scope) 33 | if err := a.fillUpAccessToken(saUser, identity.AccessKeyName); err != nil { 34 | return nil, errors.Wrap(err, "LookupSessionUser error") 35 | } 36 | 37 | return saUser, nil 38 | } 39 | 40 | func (a *Service) fillUpAccessToken(saUser *SessionAwareUser, currentlyUsedAccessKeyName string) error { 41 | // access keys 42 | accessKeys := make([]*AccessKey, 0) 43 | saUser.currentAccessKey = nil 44 | for _, accessKey := range saUser.Spec.AccessKeys { 45 | ak := *accessKey 46 | if ak.Password == "" && ak.PasswordFromRef.Name != "" { 47 | hashSetter := func(password string) { 48 | ak.Password = password 49 | } 50 | if hashFillErr := security.FillPasswordFromKindSecret(a.config, &ak.PasswordFromRef, hashSetter); hashFillErr != nil { 51 | return errors.Wrap(hashFillErr, "cannot fetch access key") 52 | } 53 | } 54 | if accessKey.Name == currentlyUsedAccessKeyName { 55 | saUser.currentAccessKey = &ak 56 | } 57 | accessKeys = append(accessKeys, &ak) 58 | } 59 | saUser.accessKeysFromSecret = accessKeys 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /skaffold.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: skaffold/v3 3 | kind: Config 4 | profiles: 5 | - name: app 6 | build: 7 | local: 8 | push: true 9 | artifacts: 10 | - image: rkt-registry:5000/backup-repository 11 | ko: 12 | dependencies: 13 | paths: ["**/*.go", "go.mod", "go.sum"] 14 | ignore: ["**/*_test.go"] 15 | tagPolicy: 16 | gitCommit: {} 17 | insecureRegistries: 18 | - rkt-registry:5000 19 | deploy: 20 | statusCheck: true 21 | statusCheckDeadlineSeconds: 120 22 | helm: 23 | releases: 24 | - name: server 25 | chartPath: helm/backup-repository-server 26 | recreatePods: true 27 | namespace: backups 28 | createNamespace: true 29 | valuesFiles: 30 | - helm/examples/backup-repository-ci.values.yaml 31 | setValueTemplates: 32 | installCRD: "false" 33 | image.repository: 'rkt-registry:5000/backup-repository' 34 | image.tag: '{{.IMAGE_TAG}}' 35 | terminationGracePeriodSeconds: 1 36 | env.GIN_MODE: "debug" 37 | portForward: 38 | - resourceType: service 39 | resourceName: server-backup-repository-server 40 | namespace: backups 41 | port: 8080 42 | localPort: 8050 43 | 44 | 45 | - name: deps 46 | deploy: 47 | statusCheck: true 48 | statusCheckDeadlineSeconds: 120 49 | helm: 50 | releases: 51 | - name: postgresql 52 | repo: https://charts.bitnami.com/bitnami 53 | version: 12.1.2 54 | remoteChart: postgresql 55 | namespace: db 56 | createNamespace: true 57 | wait: true 58 | valuesFiles: 59 | - helm/examples/postgresql.values.yaml 60 | 61 | - name: minio 62 | repo: https://charts.min.io/ 63 | version: 5.0.1 64 | remoteChart: minio 65 | namespace: storage 66 | createNamespace: true 67 | wait: true 68 | valuesFiles: 69 | - helm/examples/minio.values.yaml 70 | -------------------------------------------------------------------------------- /test.mk: -------------------------------------------------------------------------------- 1 | # 2 | # Common tasks 3 | # 4 | 5 | .EXPORT_ALL_VARIABLES: 6 | PATH = $(shell pwd)/.build:$(shell echo $$PATH) 7 | KUBECONFIG = $(shell /bin/bash -c "[[ -f \"$$HOME/.k3d/kubeconfig-rkt.yaml\" ]] && rm -f $$HOME/.k3d/kubeconfig-rkt.yaml; k3d kubeconfig merge rkt") 8 | 9 | SERVER_PORT=8050 10 | SERVER_URL=http://127.0.0.1:${SERVER_PORT} 11 | 12 | import-examples: 13 | kubectl apply -f docs/examples/ -n backups 14 | 15 | test_health: 16 | curl -s -X GET '${SERVER_URL}/health' 17 | 18 | test_login: 19 | curl -s -X POST -d '{"username":"admin","password":"admin"}' -H 'Content-Type: application/json' '${SERVER_URL}/api/stable/auth/login' 20 | @echo "Now do export TOKEN=..." 21 | 22 | test_login_some_user: 23 | curl -s -X POST -d '{"username":"some-user","password":"admin"}' -H 'Content-Type: application/json' '${SERVER_URL}/api/stable/auth/login' 24 | 25 | test_lookup: 26 | curl -s -X GET -H 'Authorization: Bearer ${TOKEN}' -H 'Content-Type: application/json' '${SERVER_URL}/api/stable/auth/user/some-user' 27 | 28 | test_whoami: 29 | curl -s -X GET -H 'Authorization: Bearer ${TOKEN}' -H 'Content-Type: application/json' '${SERVER_URL}/api/stable/auth/whoami' 30 | 31 | test_logout: 32 | curl -s -X DELETE -H 'Authorization: Bearer ${TOKEN}' -H 'Content-Type: application/json' '${SERVER_URL}/api/stable/auth/logout' 33 | 34 | test_logout_other_user: 35 | curl -s -X DELETE -H 'Authorization: Bearer ${TOKEN}' -H 'Content-Type: application/json' '${SERVER_URL}/api/stable/auth/logout?sessionId=${OTHER_USER_SESSION_ID}' 36 | 37 | test_list_auths: 38 | curl -s -X GET -H 'Authorization: Bearer ${TOKEN}' -H 'Content-Type: application/json' '${SERVER_URL}/api/stable/auth/token' 39 | 40 | test_list_auths_other_user: 41 | curl -s -X GET -H 'Authorization: Bearer ${TOKEN}' -H 'Content-Type: application/json' '${SERVER_URL}/api/stable/auth/token?userName=some-user' 42 | 43 | test_upload_by_form: 44 | curl -s -X POST -H 'Authorization: Bearer ${TOKEN}' -F "file=@./storage/.test_data/test.gpg" '${SERVER_URL}/api/alpha/repository/collection/iwa-ait/version' 45 | 46 | test_upload_by_form_1mb: 47 | @echo "-----BEGIN PGP MESSAGE-----" > /tmp/1mb.gpg 48 | @openssl rand -base64 $$((735*1024*1)) >> /tmp/1mb.gpg 49 | @echo "-----END PGP MESSAGE-----" >> /tmp/1mb.gpg 50 | curl -vvv -X POST -H 'Authorization: Bearer ${TOKEN}' -F "file=@/tmp/1mb.gpg" '${SERVER_URL}/api/alpha/repository/collection/iwa-ait/version' --limit-rate 400K 51 | 52 | test_upload_by_form_5mb: 53 | @echo "-----BEGIN PGP MESSAGE-----" > /tmp/5mb.gpg 54 | @openssl rand -base64 $$((735*1024*5)) >> /tmp/5mb.gpg 55 | @echo "-----END PGP MESSAGE-----" >> /tmp/5mb.gpg 56 | curl -vvv -X POST -H 'Authorization: Bearer ${TOKEN}' -F "file=@/tmp/5mb.gpg" '${SERVER_URL}/api/alpha/repository/collection/iwa-ait/version' --limit-rate 1000K 57 | 58 | test_download: 59 | curl -vvv -X GET -H 'Authorization: Bearer ${TOKEN}' '${SERVER_URL}/api/alpha/repository/collection/iwa-ait/version/latest' > /tmp/downloaded --limit-rate 100K 60 | 61 | test_collection_health: 62 | curl -s -X GET -H 'Authorization: admin' '${SERVER_URL}/api/stable/repository/collection/iwa-ait/health' 63 | 64 | test_list_versions: 65 | curl -s -X GET -H 'Authorization: Bearer ${TOKEN}' '${SERVER_URL}/api/stable/repository/collection/iwa-ait/version' 66 | -------------------------------------------------------------------------------- /tests/.helpers/local-registry.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: docker-registry-lb 6 | namespace: default 7 | spec: 8 | type: LoadBalancer 9 | selector: 10 | app: docker-registry 11 | ports: 12 | - port: 5000 13 | targetPort: 5000 14 | nodePort: 30050 15 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import time 3 | import unittest 4 | import requests 5 | from json import dumps as to_json 6 | 7 | 8 | class BaseTestCase(unittest.TestCase): 9 | _base_url: str = 'http://127.0.0.1:8050' 10 | 11 | def get(self, url: str, auth: str = None, timeout: int = 15) -> requests.Response: 12 | headers = {} 13 | if auth: 14 | headers['Authorization'] = f'Bearer {auth}' 15 | 16 | return requests.get(f"{self._base_url}{url}", headers=headers, timeout=timeout) 17 | 18 | def post(self, url: str, data: any, additional_headers: dict = None, auth: str = None) -> requests.Response: 19 | headers = {} 20 | if auth: 21 | headers['Authorization'] = auth 22 | 23 | if additional_headers: 24 | headers = {**headers, **additional_headers} 25 | 26 | if isinstance(data, dict): 27 | headers['Content-Type'] = "application/json" 28 | data = to_json(data) 29 | 30 | return requests.post(f"{self._base_url}{url}", headers=headers, data=data) 31 | 32 | def login(self, username: str, password: str) -> str: 33 | time.sleep(0.5) 34 | response = self.post("/api/stable/auth/login", data={'username': username, 'password': password}) 35 | data = response.json() 36 | 37 | assert "token" in data['data'], response.content 38 | 39 | return data['data']['token'] 40 | 41 | @staticmethod 42 | def scale(kind: str, name: str, replicas: int, ns: str = "backups"): 43 | print(f'>> Scaling {kind} - {name} to {replicas} replicas') 44 | subprocess.check_call(["kubectl", "scale", "-n", ns, kind, name, f"--replicas={replicas}"]) 45 | 46 | @staticmethod 47 | def wait_for(label: str, ready: bool = True, ns: str = "backups"): 48 | print(f'>> Waiting for {label} to be ready={ready}') 49 | 50 | if ready: 51 | condition = "=True" 52 | else: 53 | condition = "=False" 54 | 55 | try: 56 | subprocess.check_call(['kubectl', 'wait', '--for=condition=ready' + condition, 'pod', '-l', label, '-n', ns, '--timeout=300s']) 57 | except subprocess.CalledProcessError: 58 | subprocess.check_call(['kubectl', 'get', 'events', '-A']) 59 | raise 60 | -------------------------------------------------------------------------------- /tests/test_collection.py: -------------------------------------------------------------------------------- 1 | from tests import BaseTestCase 2 | 3 | 4 | class CollectionTest(BaseTestCase): 5 | def test_cannot_upload_if_uploader_role_not_present(self): 6 | token = self.login('unprivileged', 'admin') 7 | response = self.post("/api/alpha/repository/collection/iwa-ait/version", "something", auth=f'Bearer {token}') 8 | 9 | assert "not authorized to upload versions to this collection" in str(response.content) 10 | 11 | def test_gpg_detection(self): 12 | token = self.login('admin', 'admin') 13 | response = self.post("/api/alpha/repository/collection/iwa-ait/version", "hello this is not a gpg data", auth=f'Bearer {token}') 14 | 15 | assert "cannot upload version. cannot upload file, cannot copy stream, error: first chunk of uploaded data does not contain a valid GPG header" in str(response.content) 16 | -------------------------------------------------------------------------------- /tests/test_health.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import time 3 | 4 | from tests import BaseTestCase 5 | 6 | 7 | class HealthTest(BaseTestCase): 8 | def test_health_endpoint(self): 9 | response = self.get("/health", auth=None) 10 | 11 | assert response.status_code == 200 12 | 13 | def test_readiness_when_postgresql_is_off(self): 14 | try: 15 | self.scale("sts", "postgresql", 0, ns="db") 16 | self.wait_for("app.kubernetes.io/instance=postgresql", ready=False, ns="db") 17 | response = self.get("/ready?code=changeme", auth=None) 18 | 19 | assert response.status_code >= 500, response.content 20 | finally: 21 | self.scale("sts", "postgresql", 1, ns="db") 22 | self.wait_for("app.kubernetes.io/instance=postgresql", ready=True, ns="db") 23 | 24 | def test_readiness_when_storage_is_off(self): 25 | try: 26 | self.scale("deployment", "minio", 0, ns="storage") 27 | self.wait_for("app=minio", ready=False, ns="storage") 28 | response = self.get("/ready?code=changeme", auth=None) 29 | 30 | assert response.status_code >= 500, response.content 31 | finally: 32 | self.scale("deployment", "minio", 1, ns="storage") 33 | self.wait_for("app=minio", ready=True, ns="storage") 34 | 35 | def test_kubernetes_connection_will_be_degraded_if_crds_not_present(self): 36 | try: 37 | subprocess.check_call(['kubectl', 'delete', '-f', 'helm/backup-repository-server/templates/crd.yaml']) 38 | time.sleep(5) 39 | 40 | response = self.get("/ready?code=changeme", auth=None) 41 | assert response.status_code >= 500, response.content 42 | assert "configuration provider is not usable" in str(response.content) 43 | 44 | finally: 45 | subprocess.check_call(['kubectl', 'apply', '-f', 'helm/backup-repository-server/templates/crd.yaml']) 46 | subprocess.check_call(['kubectl', 'apply', '-f', 47 | 'docs/examples/', '-n', 'backups']) # restore test data 48 | -------------------------------------------------------------------------------- /tests/test_login.py: -------------------------------------------------------------------------------- 1 | from tests import BaseTestCase 2 | 3 | 4 | class LoginTest(BaseTestCase): 5 | def test_jwt_gaining(self): 6 | response = self.post("/api/stable/auth/login", data={'username': 'somebody', 'password': 'invalid'}) 7 | 8 | assert response.status_code == 401 9 | assert "incorrect Username or Password" in str(response.content) 10 | 11 | def test_jwt_granted(self): 12 | response = self.post("/api/stable/auth/login", data={'username': 'admin', 'password': 'admin'}) 13 | data = response.json() 14 | 15 | assert "expire" in data['data'] 16 | assert "sessionId" in data['data'] 17 | assert "token" in data['data'] 18 | assert data['status'] is True 19 | 20 | def test_whoami(self): 21 | token = self.login('admin', 'admin') 22 | response = self.get("/api/stable/auth/whoami", auth=token).json() 23 | 24 | assert response['data']['email'] == 'riotkit@riseup.net', response 25 | self.assertNotEqual(response['data']['sessionId'], '') # not empty 26 | --------------------------------------------------------------------------------