├── .github ├── ISSUE_TEMPLATE │ ├── BUG.md │ ├── FEATURE.md │ └── config.yml ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ └── tests.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── AUTHORS ├── CHANGELOG.md ├── Dockerfile ├── Dockerfile.goreleaser ├── LICENSE ├── README.md ├── Release.md ├── VERSION ├── build.go ├── changelog ├── 0.10.0_2020-09-13 │ ├── issue-102 │ ├── issue-117 │ ├── issue-44 │ ├── issue-60 │ └── pull-64 ├── 0.11.0_2022-02-10 │ ├── issue-119 │ ├── issue-122 │ ├── issue-126 │ ├── issue-131 │ ├── issue-146 │ ├── issue-148 │ ├── pull-112 │ ├── pull-142 │ ├── pull-158 │ └── pull-160 ├── 0.12.0_2023-04-24 │ ├── issue-133 │ ├── issue-182 │ ├── issue-187 │ ├── issue-219 │ ├── pull-194 │ ├── pull-207 │ └── pull-208 ├── 0.12.1_2023-07-09 │ ├── issue-230 │ ├── issue-238 │ └── pull-217 ├── 0.13.0_2024-07-26 │ ├── pull-267 │ ├── pull-271 │ ├── pull-272 │ └── pull-273 ├── 0.14.0_2025-05-31 │ ├── issue-189 │ ├── issue-318 │ ├── issue-321 │ ├── pull-295 │ ├── pull-307 │ ├── pull-315 │ └── pull-322 ├── CHANGELOG-GitHub.tmpl ├── CHANGELOG.tmpl ├── TEMPLATE └── unreleased │ └── .gitkeep ├── cmd └── rest-server │ ├── listener_unix.go │ ├── listener_unix_test.go │ ├── listener_windows.go │ ├── main.go │ └── main_test.go ├── docker ├── create_user ├── delete_user └── entrypoint.sh ├── examples ├── bsd │ ├── freebsd │ └── openbsd ├── compose-with-grafana │ ├── README.md │ ├── dashboards │ │ └── rest-server.json │ ├── datasource.png │ ├── demo-passwd │ ├── docker-compose.yml │ ├── grafana.ini │ ├── prometheus │ │ └── prometheus.yml │ └── screenshot.png └── systemd │ ├── rest-server.service │ └── rest-server.socket ├── go.mod ├── go.sum ├── handlers.go ├── handlers_test.go ├── htpasswd.go ├── htpasswd_test.go ├── metrics.go ├── mux.go ├── mux_test.go ├── quota └── quota.go └── repo ├── repo.go ├── repo_unix.go └── repo_windows.go /.github/ISSUE_TEMPLATE/BUG.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Report a problem with rest-server to help us resolve it and improve 4 | --- 5 | 6 | 29 | 30 | 31 | Output of `rest-server --version` 32 | --------------------------------- 33 | 34 | 35 | 36 | Problem description / Steps to reproduce 37 | ---------------------------------------- 38 | 39 | 53 | 54 | 55 | Expected behavior 56 | ----------------- 57 | 58 | 61 | 62 | Actual behavior 63 | --------------- 64 | 65 | 69 | 70 | Do you have any idea what may have caused this? 71 | ----------------------------------------------- 72 | 73 | 76 | 77 | 78 | Did rest-server help you today? Did it make you happy in any way? 79 | ----------------------------------------------------------------- 80 | 81 | 86 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/FEATURE.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest a new feature or enhancement for rest-server 4 | --- 5 | 6 | 23 | 24 | 25 | Output of `rest-server --version` 26 | --------------------------------- 27 | 28 | 33 | 34 | What should rest-server do differently? Which functionality do you think we should add? 35 | --------------------------------------------------------------------------------------- 36 | 37 | 40 | 41 | 42 | What are you trying to do? What problem would this solve? 43 | --------------------------------------------------------- 44 | 45 | 49 | 50 | Did rest-server help you today? Did it make you happy in any way? 51 | ----------------------------------------------------------------- 52 | 53 | 58 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | contact_links: 2 | - name: restic forum 3 | url: https://forum.restic.net 4 | about: Please ask questions about using restic here, do not open an issue for questions. 5 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | What does this PR change? What problem does it solve? 8 | ----------------------------------------------------- 9 | 10 | 13 | 14 | Was the change previously discussed in an issue or on the forum? 15 | ---------------------------------------------------------------- 16 | 17 | 23 | 24 | Checklist 25 | --------- 26 | 27 | 37 | 38 | - [ ] I have added tests for all code changes. 39 | - [ ] I have added documentation for relevant changes (in the manual). 40 | - [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/rest-server/blob/master/changelog/TEMPLATE)). 41 | - [ ] I'm done! This pull request is ready for review. 42 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Dependencies listed in go.mod 4 | - package-ecosystem: "gomod" 5 | directory: "/" # Location of package manifests 6 | schedule: 7 | interval: "weekly" 8 | 9 | # Dependencies listed in .github/workflows/*.yml 10 | - package-ecosystem: "github-actions" 11 | directory: "/" 12 | schedule: 13 | interval: "weekly" 14 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | on: 3 | # run tests on push to master, but not when other branches are pushed to 4 | push: 5 | branches: 6 | - master 7 | 8 | # run tests for all pull requests 9 | pull_request: 10 | merge_group: 11 | 12 | permissions: 13 | contents: read 14 | 15 | env: 16 | latest_go: "1.24.x" 17 | GO111MODULE: on 18 | 19 | jobs: 20 | test: 21 | strategy: 22 | matrix: 23 | include: 24 | - job_name: Linux 25 | go: 1.24.x 26 | os: ubuntu-latest 27 | check_changelog: true 28 | 29 | - job_name: Linux (race) 30 | go: 1.24.x 31 | os: ubuntu-latest 32 | test_opts: "-race" 33 | 34 | - job_name: Linux 35 | go: 1.23.x 36 | os: ubuntu-latest 37 | 38 | name: ${{ matrix.job_name }} Go ${{ matrix.go }} 39 | runs-on: ${{ matrix.os }} 40 | 41 | env: 42 | GOPROXY: https://proxy.golang.org 43 | 44 | steps: 45 | - name: Check out code 46 | uses: actions/checkout@v4 47 | 48 | - name: Set up Go ${{ matrix.go }} 49 | uses: actions/setup-go@v5 50 | with: 51 | go-version: ${{ matrix.go }} 52 | 53 | - name: Build with build.go 54 | run: | 55 | go run build.go --goos linux 56 | go run build.go --goos windows 57 | go run build.go --goos darwin 58 | 59 | - name: Run local Tests 60 | run: | 61 | go test -cover ${{matrix.test_opts}} ./... 62 | 63 | - name: Check changelog files with calens 64 | run: | 65 | echo "install calens" 66 | go install github.com/restic/calens@latest 67 | 68 | echo "check changelog files" 69 | calens 70 | if: matrix.check_changelog 71 | 72 | lint: 73 | name: lint 74 | runs-on: ubuntu-latest 75 | permissions: 76 | contents: read 77 | # allow annotating code in the PR 78 | checks: write 79 | steps: 80 | - name: Check out code 81 | uses: actions/checkout@v4 82 | 83 | - name: Set up Go ${{ env.latest_go }} 84 | uses: actions/setup-go@v5 85 | with: 86 | go-version: ${{ env.latest_go }} 87 | 88 | - name: golangci-lint 89 | uses: golangci/golangci-lint-action@v6 90 | with: 91 | # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. 92 | version: v1.64.8 93 | args: --verbose --timeout 5m 94 | 95 | # only run golangci-lint for pull requests, otherwise ALL hints get 96 | # reported. We need to slowly address all issues until we can enable 97 | # linting the master branch :) 98 | if: github.event_name == 'pull_request' 99 | 100 | - name: Check go.mod/go.sum 101 | run: | 102 | echo "check if go.mod and go.sum are up to date" 103 | go mod tidy 104 | git diff --exit-code go.mod go.sum 105 | 106 | analyze: 107 | name: Analyze results 108 | needs: [test, lint] 109 | if: always() 110 | 111 | permissions: # no need to access code 112 | contents: none 113 | 114 | runs-on: ubuntu-latest 115 | steps: 116 | - name: Decide whether the needed jobs succeeded or failed 117 | uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe 118 | with: 119 | jobs: ${{ toJSON(needs) }} 120 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /rest-server 2 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # This is the configuration for golangci-lint for the restic project. 2 | # 3 | # A sample config with all settings is here: 4 | # https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml 5 | 6 | linters: 7 | # only enable the linters listed below 8 | disable-all: true 9 | enable: 10 | # make sure all errors returned by functions are handled 11 | - errcheck 12 | 13 | # show how code can be simplified 14 | - gosimple 15 | 16 | # make sure code is formatted 17 | - gofmt 18 | 19 | # examine code and report suspicious constructs, such as Printf calls whose 20 | # arguments do not align with the format string 21 | - govet 22 | 23 | # make sure names and comments are used according to the conventions 24 | - revive 25 | 26 | # detect when assignments to existing variables are not used 27 | - ineffassign 28 | 29 | # run static analysis and find errors 30 | - staticcheck 31 | 32 | # find unused variables, functions, structs, types, etc. 33 | - unused 34 | 35 | # parse and typecheck code 36 | - typecheck 37 | 38 | # ensure that http response bodies are closed 39 | - bodyclose 40 | 41 | - importas 42 | 43 | issues: 44 | # don't use the default exclude rules, this hides (among others) ignored 45 | # errors from Close() calls 46 | exclude-use-default: false 47 | 48 | # list of things to not warn about 49 | exclude: 50 | # revive: do not warn about missing comments for exported stuff 51 | - exported (function|method|var|type|const) .* should have comment or be unexported 52 | # revive: ignore constants in all caps 53 | - don't use ALL_CAPS in Go names; use CamelCase 54 | # revive: lots of packages don't have such a comment 55 | - "package-comments: should have a package comment" 56 | - "redefines-builtin-id:" 57 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | 4 | before: 5 | # Run a few commands to check the state of things. When anything is changed 6 | # in files commited to the repo, goreleaser will abort before building 7 | # anything because the git checkout is dirty. 8 | hooks: 9 | # make sure all modules are available 10 | - go mod download 11 | # make sure all generated code is up to date 12 | - go generate ./... 13 | # check that $VERSION is set 14 | - test -n "{{ .Env.VERSION }}" 15 | # make sure the file VERSION contains the latest version (used for build.go) 16 | - bash -c 'echo "{{ .Env.VERSION }}" > VERSION' 17 | # make sure that main.go contains the latest version 18 | - echo sed -i 's/var version = "[^"]*"/var version = "{{ .Env.VERSION }}"/' cmd/rest-server/main.go 19 | # make sure the file CHANGELOG.md is up to date 20 | - calens --output CHANGELOG.md 21 | 22 | # build a single binary 23 | builds: 24 | - id: default 25 | # make sure everything is statically linked by disabling cgo altogether 26 | env: &build_env 27 | - CGO_ENABLED=0 28 | 29 | # set the package for the main binary 30 | main: ./cmd/rest-server 31 | 32 | flags: 33 | &build_flags # don't include any paths to source files in the resulting binary 34 | - -trimpath 35 | 36 | mod_timestamp: "{{ .CommitTimestamp }}" 37 | 38 | ldflags: &build_ldflags # set the version variable in the main package 39 | - "-s -w -X main.version={{ .Version }}" 40 | 41 | # list all operating systems and architectures we build binaries for 42 | goos: 43 | - linux 44 | - darwin 45 | - freebsd 46 | - netbsd 47 | - openbsd 48 | - dragonfly 49 | - solaris 50 | 51 | goarch: 52 | - amd64 53 | - "386" 54 | - arm 55 | - arm64 56 | - mips 57 | - mips64 58 | - mips64le 59 | - ppc64 60 | - ppc64le 61 | goarm: 62 | - "6" 63 | - "7" 64 | 65 | - id: windows-only 66 | env: *build_env 67 | main: ./cmd/rest-server 68 | flags: *build_flags 69 | mod_timestamp: "{{ .CommitTimestamp }}" 70 | ldflags: *build_ldflags 71 | goos: 72 | - windows 73 | goarch: 74 | - amd64 75 | - "386" 76 | - arm 77 | - arm64 78 | 79 | # configure the resulting archives to create 80 | archives: 81 | - id: default 82 | builds: [default, windows-only] 83 | format: tar.gz 84 | # package a directory which contains the source file 85 | wrap_in_directory: true 86 | 87 | builds_info: &archive_file_info 88 | owner: root 89 | group: root 90 | mtime: "{{ .CommitDate }}" 91 | mode: 0644 92 | 93 | # add these files to all archives 94 | files: &archive_files 95 | - src: LICENSE 96 | dst: LICENSE 97 | info: *archive_file_info 98 | - src: README.md 99 | dst: README.md 100 | info: *archive_file_info 101 | - src: CHANGELOG.md 102 | dst: CHANGELOG.md 103 | info: *archive_file_info 104 | 105 | - id: windows-only 106 | builds: [windows-only] 107 | formats: [zip] 108 | wrap_in_directory: true 109 | builds_info: *archive_file_info 110 | files: *archive_files 111 | 112 | # also build an archive of the source code 113 | source: 114 | enabled: true 115 | 116 | # build a file containing the SHA256 hashes 117 | checksum: 118 | name_template: "SHA256SUMS" 119 | 120 | # sign the checksum file 121 | signs: 122 | - artifacts: checksum 123 | signature: "${artifact}.asc" 124 | args: 125 | - "--armor" 126 | - "--output" 127 | - "${signature}" 128 | - "--detach-sign" 129 | - "${artifact}" 130 | 131 | # configure building the rest-server docker image 132 | dockers: 133 | - image_templates: 134 | - restic/rest-server:{{ .Version }}-amd64 135 | build_flag_templates: 136 | - "--platform=linux/amd64" 137 | - "--pull" 138 | - "--label=org.opencontainers.image.created={{.Date}}" 139 | - "--label=org.opencontainers.image.title={{.ProjectName}}" 140 | - "--label=org.opencontainers.image.source=https://github.com/restic/{{ .ProjectName }}" 141 | - "--label=org.opencontainers.image.revision={{.FullCommit}}" 142 | - "--label=org.opencontainers.image.version={{.Version}}" 143 | - "--label=org.opencontainers.image.licenses=BSD-2-Clause" 144 | use: buildx 145 | dockerfile: "Dockerfile.goreleaser" 146 | extra_files: &extra_files 147 | - docker/create_user 148 | - docker/delete_user 149 | - docker/entrypoint.sh 150 | - image_templates: 151 | - restic/rest-server:{{ .Version }}-i386 152 | goarch: "386" 153 | build_flag_templates: 154 | - "--platform=linux/386" 155 | - "--pull" 156 | - "--label=org.opencontainers.image.created={{.Date}}" 157 | - "--label=org.opencontainers.image.title={{.ProjectName}}" 158 | - "--label=org.opencontainers.image.source=https://github.com/restic/{{ .ProjectName }}" 159 | - "--label=org.opencontainers.image.revision={{.FullCommit}}" 160 | - "--label=org.opencontainers.image.version={{.Version}}" 161 | - "--label=org.opencontainers.image.licenses=BSD-2-Clause" 162 | use: buildx 163 | dockerfile: "Dockerfile.goreleaser" 164 | extra_files: *extra_files 165 | - image_templates: 166 | - restic/rest-server:{{ .Version }}-arm32v6 167 | goarch: arm 168 | goarm: 6 169 | build_flag_templates: 170 | - "--platform=linux/arm/v6" 171 | - "--pull" 172 | - "--label=org.opencontainers.image.created={{.Date}}" 173 | - "--label=org.opencontainers.image.title={{.ProjectName}}" 174 | - "--label=org.opencontainers.image.source=https://github.com/restic/{{ .ProjectName }}" 175 | - "--label=org.opencontainers.image.revision={{.FullCommit}}" 176 | - "--label=org.opencontainers.image.version={{.Version}}" 177 | - "--label=org.opencontainers.image.licenses=BSD-2-Clause" 178 | use: buildx 179 | dockerfile: "Dockerfile.goreleaser" 180 | extra_files: *extra_files 181 | - image_templates: 182 | - restic/rest-server:{{ .Version }}-arm32v7 183 | goarch: arm 184 | goarm: 7 185 | build_flag_templates: 186 | - "--platform=linux/arm/v7" 187 | - "--pull" 188 | - "--label=org.opencontainers.image.created={{.Date}}" 189 | - "--label=org.opencontainers.image.title={{.ProjectName}}" 190 | - "--label=org.opencontainers.image.source=https://github.com/restic/{{ .ProjectName }}" 191 | - "--label=org.opencontainers.image.revision={{.FullCommit}}" 192 | - "--label=org.opencontainers.image.version={{.Version}}" 193 | - "--label=org.opencontainers.image.licenses=BSD-2-Clause" 194 | use: buildx 195 | dockerfile: "Dockerfile.goreleaser" 196 | extra_files: *extra_files 197 | - image_templates: 198 | - restic/rest-server:{{ .Version }}-arm64v8 199 | goarch: arm64 200 | build_flag_templates: 201 | - "--platform=linux/arm64/v8" 202 | - "--pull" 203 | - "--label=org.opencontainers.image.created={{.Date}}" 204 | - "--label=org.opencontainers.image.title={{.ProjectName}}" 205 | - "--label=org.opencontainers.image.source=https://github.com/restic/{{ .ProjectName }}" 206 | - "--label=org.opencontainers.image.revision={{.FullCommit}}" 207 | - "--label=org.opencontainers.image.version={{.Version}}" 208 | - "--label=org.opencontainers.image.licenses=BSD-2-Clause" 209 | use: buildx 210 | dockerfile: "Dockerfile.goreleaser" 211 | extra_files: *extra_files 212 | - image_templates: 213 | - restic/rest-server:{{ .Version }}-ppc64le 214 | goarch: ppc64le 215 | build_flag_templates: 216 | - "--platform=linux/ppc64le" 217 | - "--pull" 218 | - "--label=org.opencontainers.image.created={{.Date}}" 219 | - "--label=org.opencontainers.image.title={{.ProjectName}}" 220 | - "--label=org.opencontainers.image.source=https://github.com/restic/{{ .ProjectName }}" 221 | - "--label=org.opencontainers.image.revision={{.FullCommit}}" 222 | - "--label=org.opencontainers.image.version={{.Version}}" 223 | - "--label=org.opencontainers.image.licenses=BSD-2-Clause" 224 | use: buildx 225 | dockerfile: "Dockerfile.goreleaser" 226 | extra_files: *extra_files 227 | 228 | docker_manifests: 229 | - name_template: "restic/rest-server:{{ .Version }}" 230 | image_templates: 231 | - "restic/rest-server:{{ .Version }}-amd64" 232 | - "restic/rest-server:{{ .Version }}-i386" 233 | - "restic/rest-server:{{ .Version }}-arm32v6" 234 | - "restic/rest-server:{{ .Version }}-arm32v7" 235 | - "restic/rest-server:{{ .Version }}-arm64v8" 236 | - "restic/rest-server:{{ .Version }}-ppc64le" 237 | - name_template: "restic/rest-server:latest" 238 | image_templates: 239 | - "restic/rest-server:{{ .Version }}-amd64" 240 | - "restic/rest-server:{{ .Version }}-i386" 241 | - "restic/rest-server:{{ .Version }}-arm32v6" 242 | - "restic/rest-server:{{ .Version }}-arm32v7" 243 | - "restic/rest-server:{{ .Version }}-arm64v8" 244 | - "restic/rest-server:{{ .Version }}-ppc64le" 245 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of Rest Server authors for copyright purposes. 2 | 3 | Aaron Bieber 4 | Alexander Neumann 5 | Bertil Chapuis 6 | Brice Waegeneire 7 | Bruno Clermont 8 | Chapuis Bertil 9 | Kenny Keslar 10 | Konrad Wojas 11 | Matthew Holt 12 | Mebus 13 | Wayne Scott 14 | Zlatko Čalušić 15 | cgonzalez 16 | n0npax 17 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog for rest-server 0.14.0 (2025-05-31) 2 | ============================================ 3 | 4 | The following sections list the changes in rest-server 0.14.0 relevant 5 | to users. The changes are ordered by importance. 6 | 7 | Summary 8 | ------- 9 | 10 | * Sec #318: Fix world-readable permissions on new `.htpasswd` files 11 | * Chg #322: Update dependencies and require Go 1.23 or newer 12 | * Enh #174: Support proxy-based authentication 13 | * Enh #189: Support group accessible repositories 14 | * Enh #295: Output status of append-only mode on startup 15 | * Enh #315: Hardened tls settings 16 | * Enh #321: Add zip archive format for Windows releases 17 | 18 | Details 19 | ------- 20 | 21 | * Security #318: Fix world-readable permissions on new `.htpasswd` files 22 | 23 | On startup the rest-server Docker container creates an empty `.htpasswd` file if 24 | none exists yet. This file was world-readable by default, which can be a 25 | security risk, even though the file only contains hashed passwords. 26 | 27 | This has been fixed such that new `.htpasswd` files are no longer 28 | world-readabble. 29 | 30 | The permissions of existing `.htpasswd` files must be manually changed if 31 | relevant in your setup. 32 | 33 | https://github.com/restic/rest-server/issues/318 34 | https://github.com/restic/rest-server/pull/340 35 | 36 | * Change #322: Update dependencies and require Go 1.23 or newer 37 | 38 | All dependencies have been updated. Rest-server now requires Go 1.23 or newer to 39 | build. 40 | 41 | This also disables support for TLS versions older than TLS 1.2. On Windows, 42 | rest-server now requires at least Windows 10 or Windows Server 2016. On macOS, 43 | rest-server now requires at least macOS 11 Big Sur. 44 | 45 | https://github.com/restic/rest-server/pull/322 46 | https://github.com/restic/rest-server/pull/338 47 | 48 | * Enhancement #174: Support proxy-based authentication 49 | 50 | Rest-server now supports authentication via HTTP proxy headers. This feature can 51 | be enabled by specifying the username header using the `--proxy-auth-username` 52 | option (e.g., `--proxy-auth-username=X-Forwarded-User`). 53 | 54 | When enabled, the server authenticates users based on the specified header and 55 | disables Basic Auth. Note that proxy authentication is disabled when `--no-auth` 56 | is set. 57 | 58 | https://github.com/restic/rest-server/issues/174 59 | https://github.com/restic/rest-server/pull/307 60 | 61 | * Enhancement #189: Support group accessible repositories 62 | 63 | Rest-server now supports making repositories accessible to the filesystem group 64 | by setting the `--group-accessible-repos` option. Note that permissions of 65 | existing files are not modified. To allow the group to read and write file, use 66 | a umask of `007`. To only grant read access use `027`. To make an existing 67 | repository group-accessible, use `chmod -R g+rwX /path/to/repo`. 68 | 69 | https://github.com/restic/rest-server/issues/189 70 | https://github.com/restic/rest-server/pull/308 71 | 72 | * Enhancement #295: Output status of append-only mode on startup 73 | 74 | Rest-server now displays the status of append-only mode during startup. 75 | 76 | https://github.com/restic/rest-server/pull/295 77 | 78 | * Enhancement #315: Hardened tls settings 79 | 80 | Rest-server now uses a secure TLS cipher suite set by default. The minimum TLS 81 | version is now TLS 1.2 and can be further increased using the new 82 | `--tls-min-ver` option, allowing users to enforce stricter security 83 | requirements. 84 | 85 | https://github.com/restic/rest-server/pull/315 86 | 87 | * Enhancement #321: Add zip archive format for Windows releases 88 | 89 | Windows users can now download rest-server binaries in zip archive format (.zip) 90 | in addition to the existing tar.gz archives. 91 | 92 | https://github.com/restic/rest-server/issues/321 93 | https://github.com/restic/rest-server/pull/346 94 | 95 | 96 | Changelog for rest-server 0.13.0 (2024-07-26) 97 | ============================================ 98 | 99 | The following sections list the changes in rest-server 0.13.0 relevant 100 | to users. The changes are ordered by importance. 101 | 102 | Summary 103 | ------- 104 | 105 | * Chg #267: Update dependencies and require Go 1.18 or newer 106 | * Chg #273: Shut down cleanly on TERM and INT signals 107 | * Enh #271: Print listening address after start-up 108 | * Enh #272: Support listening on a unix socket 109 | 110 | Details 111 | ------- 112 | 113 | * Change #267: Update dependencies and require Go 1.18 or newer 114 | 115 | Most dependencies have been updated. Since some libraries require newer language 116 | features, support for Go 1.17 has been dropped, which means that rest-server now 117 | requires at least Go 1.18 to build. 118 | 119 | https://github.com/restic/rest-server/pull/267 120 | 121 | * Change #273: Shut down cleanly on TERM and INT signals 122 | 123 | Rest-server now listens for TERM and INT signals and cleanly closes down the 124 | http.Server and listener when receiving either of them. 125 | 126 | This is particularly useful when listening on a unix socket, as the server will 127 | now remove the socket file when it shuts down. 128 | 129 | https://github.com/restic/rest-server/pull/273 130 | 131 | * Enhancement #271: Print listening address after start-up 132 | 133 | When started with `--listen :0`, rest-server would print `start server on :0` 134 | 135 | The message now also includes the actual address listened on, for example `start 136 | server on 0.0.0.0:37333`. This is useful when starting a server with an 137 | auto-allocated free port number (port 0). 138 | 139 | https://github.com/restic/rest-server/pull/271 140 | 141 | * Enhancement #272: Support listening on a unix socket 142 | 143 | It is now possible to make rest-server listen on a unix socket by prefixing the 144 | socket filename with `unix:` and passing it to the `--listen` option, for 145 | example `--listen unix:/tmp/foo`. 146 | 147 | This is useful in combination with remote port forwarding to enable a remote 148 | server to backup locally, e.g.: 149 | 150 | ``` 151 | rest-server --listen unix:/tmp/foo & 152 | ssh -R /tmp/foo:/tmp/foo user@host restic -r rest:http+unix:///tmp/foo:/repo backup 153 | ``` 154 | 155 | https://github.com/restic/rest-server/pull/272 156 | 157 | 158 | Changelog for rest-server 0.12.1 (2023-07-09) 159 | ============================================ 160 | 161 | The following sections list the changes in rest-server 0.12.1 relevant 162 | to users. The changes are ordered by importance. 163 | 164 | Summary 165 | ------- 166 | 167 | * Fix #230: Fix erroneous warnings about unsupported fsync 168 | * Fix #238: API: Return empty array when listing empty folders 169 | * Enh #217: Log to stdout using the `--log -` option 170 | 171 | Details 172 | ------- 173 | 174 | * Bugfix #230: Fix erroneous warnings about unsupported fsync 175 | 176 | Due to a regression in rest-server 0.12.0, it continuously printed `WARNING: 177 | fsync is not supported by the data storage. This can lead to data loss, if the 178 | system crashes or the storage is unexpectedly disconnected.` for systems that 179 | support fsync. We have fixed the warning. 180 | 181 | https://github.com/restic/rest-server/issues/230 182 | https://github.com/restic/rest-server/pull/231 183 | 184 | * Bugfix #238: API: Return empty array when listing empty folders 185 | 186 | Rest-server returned `null` when listing an empty folder. This has been changed 187 | to returning an empty array in accordance with the REST protocol specification. 188 | This change has no impact on restic users. 189 | 190 | https://github.com/restic/rest-server/issues/238 191 | https://github.com/restic/rest-server/pull/239 192 | 193 | * Enhancement #217: Log to stdout using the `--log -` option 194 | 195 | Logging to stdout was possible using `--log /dev/stdout`. However, when the rest 196 | server is run as a different user, for example, using 197 | 198 | `sudo -u restic rest-server [...] --log /dev/stdout` 199 | 200 | This did not work due to permission issues. 201 | 202 | For logging to stdout, the `--log` option now supports the special filename `-` 203 | which also works in these cases. 204 | 205 | https://github.com/restic/rest-server/pull/217 206 | 207 | 208 | Changelog for rest-server 0.12.0 (2023-04-24) 209 | ============================================ 210 | 211 | The following sections list the changes in rest-server 0.12.0 relevant 212 | to users. The changes are ordered by importance. 213 | 214 | Summary 215 | ------- 216 | 217 | * Fix #183: Allow usernames containing underscore and more 218 | * Fix #219: Ignore unexpected files in the data/ folder 219 | * Fix #1871: Return 500 "Internal server error" if files cannot be read 220 | * Chg #207: Return error if command-line arguments are specified 221 | * Chg #208: Update dependencies and require Go 1.17 or newer 222 | * Enh #133: Cache basic authentication credentials 223 | * Enh #187: Allow configurable location for `.htpasswd` file 224 | 225 | Details 226 | ------- 227 | 228 | * Bugfix #183: Allow usernames containing underscore and more 229 | 230 | The security fix in rest-server 0.11.0 (#131) disallowed usernames containing 231 | and underscore "_". The list of allowed characters has now been changed to 232 | include Unicode characters, numbers, "_", "-", "." and "@". 233 | 234 | https://github.com/restic/rest-server/issues/183 235 | https://github.com/restic/rest-server/pull/184 236 | 237 | * Bugfix #219: Ignore unexpected files in the data/ folder 238 | 239 | If the data folder of a repository contained files, this would prevent restic 240 | from retrieving a list of file data files. This has been fixed. As a workaround 241 | remove the files that are directly contained in the data folder (e.g., 242 | `.DS_Store` files). 243 | 244 | https://github.com/restic/rest-server/issues/219 245 | https://github.com/restic/rest-server/pull/221 246 | 247 | * Bugfix #1871: Return 500 "Internal server error" if files cannot be read 248 | 249 | When files in a repository cannot be read by rest-server, for example after 250 | running `restic prune` directly on the server hosting the repositories in a way 251 | that causes filesystem permissions to be wrong, rest-server previously returned 252 | 404 "Not Found" as status code. This was causing confusing for users. 253 | 254 | The error handling has now been fixed to only return 404 "Not Found" if the file 255 | actually does not exist. Otherwise a 500 "Internal server error" is reported to 256 | the client and the underlying error is logged at the server side. 257 | 258 | https://github.com/restic/rest-server/issues/1871 259 | https://github.com/restic/rest-server/pull/195 260 | 261 | * Change #207: Return error if command-line arguments are specified 262 | 263 | Command line arguments are ignored by rest-server, but there was previously no 264 | indication of this when they were supplied anyway. 265 | 266 | To prevent usage errors an error is now printed when command line arguments are 267 | supplied, instead of them being silently ignored. 268 | 269 | https://github.com/restic/rest-server/pull/207 270 | 271 | * Change #208: Update dependencies and require Go 1.17 or newer 272 | 273 | Most dependencies have been updated. Since some libraries require newer language 274 | features, support for Go 1.15-1.16 has been dropped, which means that 275 | rest-server now requires at least Go 1.17 to build. 276 | 277 | https://github.com/restic/rest-server/pull/208 278 | 279 | * Enhancement #133: Cache basic authentication credentials 280 | 281 | To speed up the verification of basic auth credentials, rest-server now caches 282 | passwords for a minute in memory. That way the expensive verification of basic 283 | auth credentials can be skipped for most requests issued by a single restic run. 284 | The password is kept in memory in a hashed form and not as plaintext. 285 | 286 | https://github.com/restic/rest-server/issues/133 287 | https://github.com/restic/rest-server/pull/138 288 | 289 | * Enhancement #187: Allow configurable location for `.htpasswd` file 290 | 291 | It is now possible to specify the location of the `.htpasswd` file using the 292 | `--htpasswd-file` option. 293 | 294 | https://github.com/restic/rest-server/issues/187 295 | https://github.com/restic/rest-server/pull/188 296 | 297 | 298 | Changelog for rest-server 0.11.0 (2022-02-10) 299 | ============================================ 300 | 301 | The following sections list the changes in rest-server 0.11.0 relevant 302 | to users. The changes are ordered by importance. 303 | 304 | Summary 305 | ------- 306 | 307 | * Sec #131: Prevent loading of usernames containing a slash 308 | * Fix #119: Fix Docker configuration for `DISABLE_AUTHENTICATION` 309 | * Fix #142: Fix possible data loss due to interrupted network connections 310 | * Fix #155: Reply "insufficient storage" on disk full or over-quota 311 | * Fix #157: Use platform-specific temporary directory as default data directory 312 | * Chg #112: Add subrepo support and refactor server code 313 | * Chg #146: Build rest-server at docker container build time 314 | * Enh #122: Verify uploaded files 315 | * Enh #126: Allow running rest-server via systemd socket activation 316 | * Enh #148: Expand use of security features in example systemd unit file 317 | 318 | Details 319 | ------- 320 | 321 | * Security #131: Prevent loading of usernames containing a slash 322 | 323 | "/" is valid char in HTTP authorization headers, but is also used in rest-server 324 | to map usernames to private repos. 325 | 326 | This commit prevents loading maliciously composed usernames like "/foo/config" 327 | by restricting the allowed characters to the unicode character class, numbers, 328 | "-", "." and "@". 329 | 330 | This prevents requests to other users files like: 331 | 332 | Curl -v -X DELETE -u foo/config:attack http://localhost:8000/foo/config 333 | 334 | https://github.com/restic/rest-server/issues/131 335 | https://github.com/restic/rest-server/pull/132 336 | https://github.com/restic/rest-server/pull/137 337 | 338 | * Bugfix #119: Fix Docker configuration for `DISABLE_AUTHENTICATION` 339 | 340 | Rest-server 0.10.0 introduced a regression which caused the 341 | `DISABLE_AUTHENTICATION` environment variable to stop working for the Docker 342 | container. This has been fixed by automatically setting the option `--no-auth` 343 | to disable authentication. 344 | 345 | https://github.com/restic/rest-server/issues/119 346 | https://github.com/restic/rest-server/pull/124 347 | 348 | * Bugfix #142: Fix possible data loss due to interrupted network connections 349 | 350 | When rest-server was run without `--append-only` it was possible to lose 351 | uploaded files in a specific scenario in which a network connection was 352 | interrupted. 353 | 354 | For the data loss to occur a file upload by restic would have to be interrupted 355 | such that restic notices the interrupted network connection before the 356 | rest-server. Then restic would have to retry the file upload and finish it 357 | before the rest-server notices that the initial upload has failed. Then the 358 | uploaded file would be accidentally removed by rest-server when trying to 359 | cleanup the failed upload. 360 | 361 | This has been fixed by always uploading to a temporary file first which is moved 362 | in position only once it was uploaded completely. 363 | 364 | https://github.com/restic/rest-server/pull/142 365 | 366 | * Bugfix #155: Reply "insufficient storage" on disk full or over-quota 367 | 368 | When there was no space left on disk, or any other write-related error occurred, 369 | rest-server replied with HTTP status code 400 (Bad request). This is misleading 370 | (restic client will dump the status code to the user). 371 | 372 | Rest-server now replies with two different status codes in these situations: * 373 | HTTP 507 "Insufficient storage" is the status on disk full or repository 374 | over-quota * HTTP 500 "Internal server error" is used for other disk-related 375 | errors 376 | 377 | https://github.com/restic/rest-server/issues/155 378 | https://github.com/restic/rest-server/pull/160 379 | 380 | * Bugfix #157: Use platform-specific temporary directory as default data directory 381 | 382 | If no data directory is specificed, then rest-server now uses the Go standard 383 | library functions to retrieve the standard temporary directory path for the 384 | current platform. 385 | 386 | https://github.com/restic/rest-server/issues/157 387 | https://github.com/restic/rest-server/pull/158 388 | 389 | * Change #112: Add subrepo support and refactor server code 390 | 391 | Support for multi-level repositories has been added, so now each user can have 392 | its own subrepositories. This feature is always enabled. 393 | 394 | Authentication for the Prometheus /metrics endpoint can now be disabled with the 395 | new `--prometheus-no-auth` flag. 396 | 397 | We have split out all HTTP handling to a separate `repo` subpackage to cleanly 398 | separate the server code from the code that handles a single repository. The new 399 | RepoHandler also makes it easier to reuse rest-server as a Go component in any 400 | other HTTP server. 401 | 402 | The refactoring makes the code significantly easier to follow and understand, 403 | which in turn makes it easier to add new features, audit for security and debug 404 | issues. 405 | 406 | https://github.com/restic/rest-server/issues/109 407 | https://github.com/restic/rest-server/issues/107 408 | https://github.com/restic/rest-server/pull/112 409 | 410 | * Change #146: Build rest-server at docker container build time 411 | 412 | The Dockerfile now includes a build stage such that the latest rest-server is 413 | always built and packaged. This is done in a standard golang container to ensure 414 | a clean build environment and only the final binary is shipped rather than the 415 | whole build environment. 416 | 417 | https://github.com/restic/rest-server/issues/146 418 | https://github.com/restic/rest-server/pull/145 419 | 420 | * Enhancement #122: Verify uploaded files 421 | 422 | The rest-server now by default verifies that the hash of content of uploaded 423 | files matches their filename. This ensures that transmission errors are detected 424 | and forces restic to retry the upload. On low-power devices it can make sense to 425 | disable this check by passing the `--no-verify-upload` flag. 426 | 427 | https://github.com/restic/rest-server/issues/122 428 | https://github.com/restic/rest-server/pull/130 429 | 430 | * Enhancement #126: Allow running rest-server via systemd socket activation 431 | 432 | We've added the option to have systemd create the listening socket and start the 433 | rest-server on demand. 434 | 435 | https://github.com/restic/rest-server/issues/126 436 | https://github.com/restic/rest-server/pull/151 437 | https://github.com/restic/rest-server/pull/127 438 | 439 | * Enhancement #148: Expand use of security features in example systemd unit file 440 | 441 | The example systemd unit file now enables additional systemd features to 442 | mitigate potential security vulnerabilities in rest-server and the various 443 | packages and operating system components which it relies upon. 444 | 445 | https://github.com/restic/rest-server/issues/148 446 | https://github.com/restic/rest-server/pull/149 447 | 448 | 449 | Changelog for rest-server 0.10.0 (2020-09-13) 450 | ============================================ 451 | 452 | The following sections list the changes in rest-server 0.10.0 relevant 453 | to users. The changes are ordered by importance. 454 | 455 | Summary 456 | ------- 457 | 458 | * Sec #60: Require auth by default, add --no-auth flag 459 | * Sec #64: Refuse overwriting config file in append-only mode 460 | * Sec #117: Stricter path sanitization 461 | * Chg #102: Remove vendored dependencies 462 | * Enh #44: Add changelog file 463 | 464 | Details 465 | ------- 466 | 467 | * Security #60: Require auth by default, add --no-auth flag 468 | 469 | In order to prevent users from accidentally exposing rest-server without 470 | authentication, rest-server now defaults to requiring a .htpasswd. If you want 471 | to disable authentication, you need to explicitly pass the new --no-auth flag. 472 | 473 | https://github.com/restic/rest-server/issues/60 474 | https://github.com/restic/rest-server/pull/61 475 | 476 | * Security #64: Refuse overwriting config file in append-only mode 477 | 478 | While working on the `rclone serve restic` command we noticed that is currently 479 | possible to overwrite the config file in a repo even if `--append-only` is 480 | specified. The first commit adds proper tests, and the second commit fixes the 481 | issue. 482 | 483 | https://github.com/restic/rest-server/pull/64 484 | 485 | * Security #117: Stricter path sanitization 486 | 487 | The framework we're using in rest-server to decode paths to repositories allowed 488 | specifying URL-encoded characters in paths, including sensitive characters such 489 | as `/` (encoded as `%2F`). 490 | 491 | We've changed this unintended behavior, such that rest-server now rejects such 492 | paths. In particular, it is no longer possible to specify sub-repositories for 493 | users by encoding the path with `%2F`, such as 494 | `http://localhost:8000/foo%2Fbar`, which means that this will unfortunately be a 495 | breaking change in that case. 496 | 497 | If using sub-repositories for users is important to you, please let us know in 498 | the forum, so we can learn about your use case and implement this properly. As 499 | it currently stands, the ability to use sub-repositories was an unintentional 500 | feature made possible by the URL decoding framework used, and hence never meant 501 | to be supported in the first place. If we wish to have this feature in 502 | rest-server, we'd like to have it implemented properly and intentionally. 503 | 504 | https://github.com/restic/rest-server/issues/117 505 | 506 | * Change #102: Remove vendored dependencies 507 | 508 | We've removed the vendored dependencies (in the subdir `vendor/`) similar to 509 | what we did for `restic` itself. When building restic, the Go compiler 510 | automatically fetches the dependencies. It will also cryptographically verify 511 | that the correct code has been fetched by using the hashes in `go.sum` (see the 512 | link to the documentation below). 513 | 514 | Building the rest-server now requires Go 1.11 or newer, since we're using Go 515 | Modules for dependency management. Older Go versions are not supported any more. 516 | 517 | https://github.com/restic/rest-server/issues/102 518 | https://golang.org/cmd/go/#hdr-Module_downloading_and_verification 519 | 520 | * Enhancement #44: Add changelog file 521 | 522 | https://github.com/restic/rest-server/issues/44 523 | https://github.com/restic/rest-server/pull/62 524 | 525 | 526 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine AS builder 2 | 3 | ENV CGO_ENABLED 0 4 | 5 | COPY . /build 6 | WORKDIR /build 7 | RUN go build -o rest-server ./cmd/rest-server 8 | 9 | 10 | 11 | 12 | FROM alpine 13 | 14 | ENV DATA_DIRECTORY /data 15 | ENV PASSWORD_FILE /data/.htpasswd 16 | 17 | RUN apk add --no-cache --update apache2-utils 18 | 19 | COPY docker/create_user /usr/bin/ 20 | COPY docker/delete_user /usr/bin/ 21 | COPY docker/entrypoint.sh /entrypoint.sh 22 | COPY --from=builder /build/rest-server /usr/bin 23 | 24 | VOLUME /data 25 | EXPOSE 8000 26 | 27 | CMD [ "/entrypoint.sh" ] 28 | -------------------------------------------------------------------------------- /Dockerfile.goreleaser: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | ENV DATA_DIRECTORY /data 4 | ENV PASSWORD_FILE /data/.htpasswd 5 | 6 | RUN apk add --no-cache --update apache2-utils 7 | 8 | COPY docker/create_user /usr/bin/ 9 | COPY docker/delete_user /usr/bin/ 10 | COPY docker/entrypoint.sh /entrypoint.sh 11 | COPY rest-server /usr/bin 12 | 13 | VOLUME /data 14 | EXPOSE 8000 15 | 16 | CMD [ "/entrypoint.sh" ] 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The BSD 2-Clause License 2 | 3 | Copyright © 2015, Bertil Chapuis 4 | Copyright © 2016, Zlatko Čalušić, Alexander Neumann 5 | Copyright © 2017, The Rest Server Authors 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are met: 10 | 11 | * Redistributions of source code must retain the above copyright notice, this 12 | list of conditions and the following disclaimer. 13 | 14 | * Redistributions in binary form must reproduce the above copyright notice, 15 | this list of conditions and the following disclaimer in the documentation 16 | and/or other materials provided with the distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rest Server 2 | 3 | 4 | [![Status badge for CI tests](https://github.com/restic/rest-server/workflows/test/badge.svg)](https://github.com/restic/rest-server/actions?query=workflow%3Atest) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/restic/rest-server)](https://goreportcard.com/report/github.com/restic/rest-server) 6 | [![GoDoc](https://godoc.org/github.com/restic/rest-server?status.svg)](https://godoc.org/github.com/restic/rest-server) 7 | [![License](https://img.shields.io/badge/license-BSD%20%282--Clause%29-003262.svg?maxAge=2592000)](https://github.com/restic/rest-server/blob/master/LICENSE) 8 | [![Powered by](https://img.shields.io/badge/powered_by-Go-5272b4.svg?maxAge=2592000)](https://golang.org/) 9 | 10 | Rest Server is a high performance HTTP server that implements restic's [REST backend API](https://restic.readthedocs.io/en/latest/100_references.html#rest-backend). It provides secure and efficient way to backup data remotely, using [restic](https://github.com/restic/restic) backup client via the [rest: URL](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server). 11 | 12 | ## Requirements 13 | 14 | Rest Server requires Go 1.23 or higher to build. The only tested compiler is the official Go compiler. 15 | 16 | The required version of restic backup client to use with `rest-server` is [v0.7.1](https://github.com/restic/restic/releases/tag/v0.7.1) or higher. 17 | 18 | ## Build 19 | 20 | For building the `rest-server` binary run `CGO_ENABLED=0 go build -o rest-server ./cmd/rest-server` 21 | 22 | ## Usage 23 | 24 | To learn how to use restic backup client with REST backend, please consult [restic manual](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server). 25 | 26 | ```console 27 | $ rest-server --help 28 | 29 | Run a REST server for use with restic 30 | 31 | Usage: 32 | rest-server [flags] 33 | 34 | Flags: 35 | --append-only enable append only mode 36 | --cpu-profile string write CPU profile to file 37 | --debug output debug messages 38 | --group-accessible-repos let filesystem group be able to access repo files 39 | -h, --help help for rest-server 40 | --htpasswd-file string location of .htpasswd file (default: "/.htpasswd)" 41 | --listen string listen address (default ":8000") 42 | --log filename write HTTP requests in the combined log format to the specified filename (use "-" for logging to stdout) 43 | --max-size int the maximum size of the repository in bytes 44 | --no-auth disable .htpasswd authentication 45 | --no-verify-upload do not verify the integrity of uploaded data. DO NOT enable unless the rest-server runs on a very low-power device 46 | --path string data directory (default "/tmp/restic") 47 | --private-repos users can only access their private repo 48 | --prometheus enable Prometheus metrics 49 | --prometheus-no-auth disable auth for Prometheus /metrics endpoint 50 | --proxy-auth-username string specifies the HTTP header containing the username for proxy-based authentication 51 | --tls turn on TLS support 52 | --tls-cert string TLS certificate path 53 | --tls-key string TLS key path 54 | --tls-min-ver string TLS min version, one of (1.2|1.3) (default "1.2") 55 | -v, --version version for rest-server 56 | ``` 57 | 58 | By default the server persists backup data in the OS temporary directory (`/tmp/restic` on Linux/BSD and others, in `%TEMP%\\restic` in Windows, etc). **If `rest-server` is launched using the default path, all backups will be lost**. To start the server with a custom persistence directory and with authentication disabled: 59 | 60 | ```sh 61 | rest-server --path /user/home/backup --no-auth 62 | ``` 63 | 64 | To authenticate users (for access to the rest-server), the server supports using a `.htpasswd` file to specify users. By default, the server looks for this file at the root of the persistence directory, but this can be changed using the `--htpasswd-file` option. You can create such a file by executing the following command (note that you need the `htpasswd` program from Apache's http-tools). In order to append new user to the file, just omit the `-c` argument. Only bcrypt and SHA encryption methods are supported, so use -B (very secure) or -s (insecure by today's standards) when adding/changing passwords. 65 | 66 | ```sh 67 | htpasswd -B -c .htpasswd username 68 | ``` 69 | 70 | If you want to disable authentication, you must add the `--no-auth` flag. If this flag is not specified and the `.htpasswd` cannot be opened, rest-server will refuse to start. 71 | 72 | NOTE: In older versions of rest-server (up to 0.9.7), this flag does not exist and the server disables authentication if `.htpasswd` is missing or cannot be opened. 73 | 74 | By default the server uses HTTP protocol. This is not very secure since with Basic Authentication, user name and passwords will be sent in clear text in every request. In order to enable TLS support just add the `--tls` argument and add a private and public key at the root of your persistence directory. You may also specify private and public keys by `--tls-cert` and `--tls-key` and set the minimum TLS version to 1.3 using `--tls-min-ver 1.3`. 75 | 76 | Signed certificate is normally required by the restic backend, but if you just want to test the feature you can generate password-less unsigned keys with the following command: 77 | 78 | ```sh 79 | openssl req -newkey rsa:2048 -nodes -x509 -keyout private_key -out public_key -days 365 -addext "subjectAltName = IP:127.0.0.1,DNS:yourdomain.com" 80 | ``` 81 | 82 | Omit the `IP:127.0.0.1` if you don't need your server be accessed via SSH Tunnels. No need to change default values in the openssl dialog, hitting enter every time is sufficient. To access this server via restic use `--cacert public_key`, meaning with a self-signed certificate you have to distribute your `public_key` file to every restic client. 83 | 84 | The `--append-only` mode allows creation of new backups but prevents deletion and modification of existing backups. This can be useful when backing up systems that have a potential of being hacked. 85 | 86 | To prevent your users from accessing each others' repositories, you may use the `--private-repos` flag which grants access only when a subdirectory with the same name as the user is specified in the repository URL. For example, user "foo" using the repository URLs `rest:https://foo:pass@host:8000/foo` or `rest:https://foo:pass@host:8000/foo/` would be granted access, but the same user using repository URLs `rest:https://foo:pass@host:8000/` or `rest:https://foo:pass@host:8000/foobar/` would be denied access. Users can also create their own subrepositories, like `/foo/bar/`. 87 | 88 | Rest Server uses exactly the same directory structure as local backend, so you should be able to access it both locally and via HTTP, even simultaneously. 89 | 90 | ### Systemd 91 | 92 | There's an example [systemd service file](https://github.com/restic/rest-server/blob/master/examples/systemd/rest-server.service) included with the source, so you can get Rest Server up & running as a proper Systemd service in no time. Before installing, adapt paths and options to your environment. 93 | 94 | ### Docker 95 | 96 | Rest Server works well inside a container, images are [published to Docker Hub](https://hub.docker.com/r/restic/rest-server). 97 | 98 | #### Start server 99 | 100 | You can run the server with any container runtime, like Docker: 101 | 102 | ```sh 103 | docker pull restic/rest-server:latest 104 | docker run -p 8000:8000 -v /my/data:/data --name rest_server restic/rest-server 105 | ``` 106 | 107 | Note that: 108 | 109 | - **contrary to the defaults** of `rest-server`, the persistent data volume is located to `/data`. 110 | - By default, the image uses authentication. To turn it off, set environment variable `DISABLE_AUTHENTICATION` to any value. 111 | - By default, the image loads the `.htpasswd` file from the persistent data volume (i.e. from `/data/.htpasswd`). To change the location of this file, set the environment variable `PASSWORD_FILE` to the path of the `.htpasswd` file. Please note that this path must be accessible from inside the container and should be persisted. This is normally done by bind-mounting a path into the container or with another docker volume. 112 | - It's suggested to set a container name to more easily manage users (`--name` parameter to `docker run`). 113 | - You can set environment variable `OPTIONS` to any extra flags you'd like to pass to rest-server. 114 | 115 | #### Customize the image 116 | 117 | The [published image](https://hub.docker.com/r/restic/rest-server) is built from the `Dockerfile` available on this repository, which you may use as a basis for building your own customized images. 118 | 119 | ```sh 120 | git clone https://github.com/restic/rest-server.git 121 | cd rest-server 122 | docker build -t restic/rest-server:latest . 123 | ``` 124 | 125 | #### Manage users 126 | 127 | ##### Add user 128 | 129 | ```sh 130 | docker exec -it rest_server create_user myuser 131 | ``` 132 | 133 | or 134 | 135 | ```sh 136 | docker exec -it rest_server create_user myuser mypassword 137 | ``` 138 | 139 | ##### Delete user 140 | 141 | ```sh 142 | docker exec -it rest_server delete_user myuser 143 | ``` 144 | 145 | ## Proxy Authentication 146 | 147 | See above for no authentication (`--no-auth`) and basic authentication. 148 | 149 | To delegate authentication to a proxy, use the `--proxy-auth-username` flag. The specified header name, for example `X-Forwarded-User`, 150 | must be present in the request headers and specifies the username. Basic authentication is disabled when this flag is set. 151 | 152 | Warning: rest-server trusts the username in the header. It is the responsibility of the proxy 153 | to ensure that the username is correct and cannot be forged by an attacker. 154 | 155 | 156 | ## Prometheus support and Grafana dashboard 157 | 158 | The server can be started with `--prometheus` to expose [Prometheus](https://prometheus.io/) metrics at `/metrics`. If authentication is enabled, this endpoint requires authentication for the 'metrics' user, but this can be overridden with the `--prometheus-no-auth` flag. 159 | 160 | This repository contains an example full stack Docker Compose setup with a Grafana dashboard in [examples/compose-with-grafana/](examples/compose-with-grafana/). 161 | 162 | 163 | ## Group-accessible Repositories 164 | 165 | Rest-server supports making repositories accessible to the filesystem group by setting the `--group-accessible-repos` option. Note that permissions of existing files are not modified. To allow the group to read and write file, use a umask of `007`. To only grant read access use `027`. To make an existing repository group-accessible, use `chmod -R g+rwX /path/to/repo`. 166 | 167 | ## Why use Rest Server? 168 | 169 | Compared to the SFTP backend, the REST backend has better performance, especially so if you can skip additional crypto overhead by using plain HTTP transport (restic already properly encrypts all data it sends, so using HTTPS is mostly about authentication). 170 | 171 | But, even if you use HTTPS transport, the REST protocol should be faster and more scalable, due to some inefficiencies of the SFTP protocol (everything needs to be transferred in chunks of 32 KiB at most, each packet needs to be acknowledged by the server). 172 | 173 | One important safety feature that Rest Server adds is the optional ability to run in append-only mode. This prevents an attacker from wiping your server backups when access is gained to the server being backed up. 174 | 175 | Finally, the Rest Server implementation is really simple and as such could be used on the low-end devices, no problem. Also, in some cases, for example behind corporate firewalls, HTTP/S might be the only protocol allowed. Here too REST backend might be the perfect option for your backup needs. 176 | 177 | ## Contributors 178 | 179 | Contributors are welcome, just open a new issue / pull request. 180 | -------------------------------------------------------------------------------- /Release.md: -------------------------------------------------------------------------------- 1 | 1. Export `$VERSION`: 2 | 3 | export VERSION=0.10.0 4 | 5 | 2. Add new version to file `VERSION` and `main.go` and commit the result: 6 | 7 | echo "${VERSION}" | tee VERSION 8 | sed -i "s/var version = \"[^\"]*\"/var version = \"${VERSION}\"/" cmd/rest-server/main.go 9 | git commit -m "Update VERSION files for ${VERSION}" VERSION cmd/rest-server/main.go 10 | 11 | 3. Move changelog files for `calens`: 12 | 13 | mv changelog/unreleased "changelog/${VERSION}_$(date +%Y-%m-%d)" 14 | rm -f "changelog/${VERSION}_$(date +%Y-%m-%d)/.gitkeep" 15 | git add "changelog/${VERSION}"* 16 | git rm -r changelog/unreleased 17 | mkdir changelog/unreleased 18 | touch changelog/unreleased/.gitkeep 19 | git add changelog/unreleased/.gitkeep 20 | git commit -m "Move changelog files for ${VERSION}" changelog/{unreleased,"${VERSION}"*} 21 | 22 | 4. Generate changelog: 23 | 24 | calens > CHANGELOG.md 25 | git add CHANGELOG.md 26 | git commit -m "Generate CHANGELOG.md for ${VERSION}" CHANGELOG.md 27 | 28 | 5. Tag new version and push the tag: 29 | 30 | git tag -a -s -m "v${VERSION}" "v${VERSION}" 31 | git push --tags 32 | 33 | 6. Build the project (use `--snapshot` for testing, or pass `--config` to 34 | use another config file): 35 | 36 | goreleaser \ 37 | release --parallelism 4 \ 38 | --release-notes <(calens --template changelog/CHANGELOG-GitHub.tmpl --version "${VERSION}") 39 | 40 | 7. Set a new version in `main.go` and commit the result: 41 | 42 | sed -i "s/var version = \"[^\"]*\"/var version = \"${VERSION}-dev\"/" cmd/rest-server/main.go 43 | git commit -m "Update version for development" cmd/rest-server/main.go 44 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.14.0 2 | -------------------------------------------------------------------------------- /build.go: -------------------------------------------------------------------------------- 1 | // Description 2 | // 3 | // This program aims to make building Go programs for end users easier by just 4 | // calling it with `go run`, without having to setup a GOPATH. 5 | // 6 | // This program needs Go >= 1.12. It'll use Go modules for compilation. It 7 | // builds the package configured as Main in the Config struct. 8 | 9 | // BSD 2-Clause License 10 | // 11 | // Copyright (c) 2016-2018, Alexander Neumann 12 | // All rights reserved. 13 | // 14 | // This file has been derived from the repository at: 15 | // https://github.com/fd0/build-go 16 | // 17 | // Redistribution and use in source and binary forms, with or without 18 | // modification, are permitted provided that the following conditions are met: 19 | // 20 | // * Redistributions of source code must retain the above copyright notice, this 21 | // list of conditions and the following disclaimer. 22 | // 23 | // * Redistributions in binary form must reproduce the above copyright notice, 24 | // this list of conditions and the following disclaimer in the documentation 25 | // and/or other materials provided with the distribution. 26 | // 27 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 28 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 30 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 31 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 33 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 34 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 35 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | 38 | //go:build ignore_build_go 39 | // +build ignore_build_go 40 | 41 | package main 42 | 43 | import ( 44 | "fmt" 45 | "io" 46 | "io/ioutil" 47 | "os" 48 | "os/exec" 49 | "path/filepath" 50 | "runtime" 51 | "strconv" 52 | "strings" 53 | ) 54 | 55 | // config contains the configuration for the program to build. 56 | var config = Config{ 57 | Name: "rest-server", // name of the program executable and directory 58 | Namespace: "github.com/restic/rest-server", // subdir of GOPATH, e.g. "github.com/foo/bar" 59 | Main: "github.com/restic/rest-server/cmd/rest-server", // package name for the main package 60 | Tests: []string{"./..."}, // tests to run 61 | MinVersion: GoVersion{Major: 1, Minor: 23, Patch: 0}, // minimum Go version supported 62 | } 63 | 64 | // Config configures the build. 65 | type Config struct { 66 | Name string 67 | Namespace string 68 | Main string 69 | DefaultBuildTags []string 70 | Tests []string 71 | MinVersion GoVersion 72 | } 73 | 74 | var ( 75 | verbose bool 76 | runTests bool 77 | enableCGO bool 78 | enablePIE bool 79 | goVersion = ParseGoVersion(runtime.Version()) 80 | ) 81 | 82 | // die prints the message with fmt.Fprintf() to stderr and exits with an error 83 | // code. 84 | func die(message string, args ...interface{}) { 85 | fmt.Fprintf(os.Stderr, message, args...) 86 | os.Exit(1) 87 | } 88 | 89 | func showUsage(output io.Writer) { 90 | fmt.Fprintf(output, "USAGE: go run build.go OPTIONS\n") 91 | fmt.Fprintf(output, "\n") 92 | fmt.Fprintf(output, "OPTIONS:\n") 93 | fmt.Fprintf(output, " -v --verbose output more messages\n") 94 | fmt.Fprintf(output, " -t --tags specify additional build tags\n") 95 | fmt.Fprintf(output, " -T --test run tests\n") 96 | fmt.Fprintf(output, " -o --output set output file name\n") 97 | fmt.Fprintf(output, " --enable-cgo use CGO to link against libc\n") 98 | fmt.Fprintf(output, " --enable-pie use PIE buildmode\n") 99 | fmt.Fprintf(output, " --goos value set GOOS for cross-compilation\n") 100 | fmt.Fprintf(output, " --goarch value set GOARCH for cross-compilation\n") 101 | fmt.Fprintf(output, " --goarm value set GOARM for cross-compilation\n") 102 | } 103 | 104 | func verbosePrintf(message string, args ...interface{}) { 105 | if !verbose { 106 | return 107 | } 108 | 109 | fmt.Printf("build: "+message, args...) 110 | } 111 | 112 | // printEnv prints Go-relevant environment variables in a nice way using verbosePrintf. 113 | func printEnv(env []string) { 114 | verbosePrintf("environment (GO*):\n") 115 | for _, v := range env { 116 | // ignore environment variables which do not start with GO*. 117 | if !strings.HasPrefix(v, "GO") { 118 | continue 119 | } 120 | verbosePrintf(" %s\n", v) 121 | } 122 | } 123 | 124 | // build runs "go build args..." with GOPATH set to gopath. 125 | func build(cwd string, env map[string]string, args ...string) error { 126 | // -trimpath removes all absolute paths from the binary. 127 | a := []string{"build", "-trimpath"} 128 | 129 | if enablePIE { 130 | a = append(a, "-buildmode=pie") 131 | } 132 | 133 | a = append(a, args...) 134 | cmd := exec.Command("go", a...) 135 | cmd.Env = os.Environ() 136 | for k, v := range env { 137 | cmd.Env = append(cmd.Env, k+"="+v) 138 | } 139 | if !enableCGO { 140 | cmd.Env = append(cmd.Env, "CGO_ENABLED=0") 141 | } 142 | 143 | printEnv(cmd.Env) 144 | 145 | cmd.Dir = cwd 146 | cmd.Stdout = os.Stdout 147 | cmd.Stderr = os.Stderr 148 | 149 | verbosePrintf("chdir %q\n", cwd) 150 | verbosePrintf("go %q\n", a) 151 | 152 | return cmd.Run() 153 | } 154 | 155 | // test runs "go test args..." with GOPATH set to gopath. 156 | func test(cwd string, env map[string]string, args ...string) error { 157 | args = append([]string{"test", "-count", "1"}, args...) 158 | cmd := exec.Command("go", args...) 159 | cmd.Env = os.Environ() 160 | for k, v := range env { 161 | cmd.Env = append(cmd.Env, k+"="+v) 162 | } 163 | if !enableCGO { 164 | cmd.Env = append(cmd.Env, "CGO_ENABLED=0") 165 | } 166 | cmd.Dir = cwd 167 | cmd.Stdout = os.Stdout 168 | cmd.Stderr = os.Stderr 169 | 170 | printEnv(cmd.Env) 171 | 172 | verbosePrintf("chdir %q\n", cwd) 173 | verbosePrintf("go %q\n", args) 174 | 175 | return cmd.Run() 176 | } 177 | 178 | // getVersion returns the version string from the file VERSION in the current 179 | // directory. 180 | func getVersionFromFile() string { 181 | buf, err := ioutil.ReadFile("VERSION") 182 | if err != nil { 183 | verbosePrintf("error reading file VERSION: %v\n", err) 184 | return "" 185 | } 186 | 187 | return strings.TrimSpace(string(buf)) 188 | } 189 | 190 | // getVersion returns a version string which is a combination of the contents 191 | // of the file VERSION in the current directory and the version from git (if 192 | // available). 193 | func getVersion() string { 194 | versionFile := getVersionFromFile() 195 | versionGit := getVersionFromGit() 196 | 197 | verbosePrintf("version from file 'VERSION' is %q, version from git %q\n", 198 | versionFile, versionGit) 199 | 200 | switch { 201 | case versionFile == "": 202 | return versionGit 203 | case versionGit == "": 204 | return versionFile 205 | } 206 | 207 | return fmt.Sprintf("%s (%s)", versionFile, versionGit) 208 | } 209 | 210 | // getVersionFromGit returns a version string that identifies the currently 211 | // checked out git commit. 212 | func getVersionFromGit() string { 213 | cmd := exec.Command("git", "describe", 214 | "--long", "--tags", "--dirty", "--always") 215 | out, err := cmd.Output() 216 | if err != nil { 217 | verbosePrintf("git describe returned error: %v\n", err) 218 | return "" 219 | } 220 | 221 | version := strings.TrimSpace(string(out)) 222 | verbosePrintf("git version is %s\n", version) 223 | return version 224 | } 225 | 226 | // Constants represents a set of constants that are set in the final binary to 227 | // the given value via compiler flags. 228 | type Constants map[string]string 229 | 230 | // LDFlags returns the string that can be passed to go build's `-ldflags`. 231 | func (cs Constants) LDFlags() string { 232 | l := make([]string, 0, len(cs)) 233 | 234 | for k, v := range cs { 235 | l = append(l, fmt.Sprintf(`-X "%s=%s"`, k, v)) 236 | } 237 | 238 | return strings.Join(l, " ") 239 | } 240 | 241 | // GoVersion is the version of Go used to compile the project. 242 | type GoVersion struct { 243 | Major int 244 | Minor int 245 | Patch int 246 | } 247 | 248 | // ParseGoVersion parses the Go version s. If s cannot be parsed, the returned GoVersion is null. 249 | func ParseGoVersion(s string) (v GoVersion) { 250 | if !strings.HasPrefix(s, "go") { 251 | return 252 | } 253 | 254 | s = s[2:] 255 | data := strings.Split(s, ".") 256 | if len(data) < 2 || len(data) > 3 { 257 | // invalid version 258 | return GoVersion{} 259 | } 260 | 261 | var err error 262 | 263 | v.Major, err = strconv.Atoi(data[0]) 264 | if err != nil { 265 | return GoVersion{} 266 | } 267 | 268 | // try to parse the minor version while removing an eventual suffix (like 269 | // "rc2" or so) 270 | for s := data[1]; s != ""; s = s[:len(s)-1] { 271 | v.Minor, err = strconv.Atoi(s) 272 | if err == nil { 273 | break 274 | } 275 | } 276 | 277 | if v.Minor == 0 { 278 | // no minor version found 279 | return GoVersion{} 280 | } 281 | 282 | if len(data) >= 3 { 283 | v.Patch, err = strconv.Atoi(data[2]) 284 | if err != nil { 285 | return GoVersion{} 286 | } 287 | } 288 | 289 | return 290 | } 291 | 292 | // AtLeast returns true if v is at least as new as other. If v is empty, true is returned. 293 | func (v GoVersion) AtLeast(other GoVersion) bool { 294 | var empty GoVersion 295 | 296 | // the empty version satisfies all versions 297 | if v == empty { 298 | return true 299 | } 300 | 301 | if v.Major < other.Major { 302 | return false 303 | } 304 | 305 | if v.Minor < other.Minor { 306 | return false 307 | } 308 | 309 | if v.Patch < other.Patch { 310 | return false 311 | } 312 | 313 | return true 314 | } 315 | 316 | func (v GoVersion) String() string { 317 | return fmt.Sprintf("Go %d.%d.%d", v.Major, v.Minor, v.Patch) 318 | } 319 | 320 | func main() { 321 | if !goVersion.AtLeast(GoVersion{1, 12, 0}) { 322 | die("Go version (%v) is too old, restic requires Go >= 1.12\n", goVersion) 323 | } 324 | 325 | if !goVersion.AtLeast(config.MinVersion) { 326 | fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", goVersion, config.MinVersion) 327 | os.Exit(1) 328 | } 329 | 330 | buildTags := config.DefaultBuildTags 331 | 332 | skipNext := false 333 | params := os.Args[1:] 334 | 335 | env := map[string]string{ 336 | "GO111MODULE": "on", // make sure we build in Module mode 337 | "GOOS": runtime.GOOS, 338 | "GOARCH": runtime.GOARCH, 339 | "GOARM": "", 340 | } 341 | 342 | var outputFilename string 343 | 344 | for i, arg := range params { 345 | if skipNext { 346 | skipNext = false 347 | continue 348 | } 349 | 350 | switch arg { 351 | case "-v", "--verbose": 352 | verbose = true 353 | case "-t", "-tags", "--tags": 354 | if i+1 >= len(params) { 355 | die("-t given but no tag specified") 356 | } 357 | skipNext = true 358 | buildTags = append(buildTags, strings.Split(params[i+1], " ")...) 359 | case "-o", "--output": 360 | skipNext = true 361 | outputFilename = params[i+1] 362 | case "-T", "--test": 363 | runTests = true 364 | case "--enable-cgo": 365 | enableCGO = true 366 | case "--enable-pie": 367 | enablePIE = true 368 | case "--goos": 369 | skipNext = true 370 | env["GOOS"] = params[i+1] 371 | case "--goarch": 372 | skipNext = true 373 | env["GOARCH"] = params[i+1] 374 | case "--goarm": 375 | skipNext = true 376 | env["GOARM"] = params[i+1] 377 | case "-h": 378 | showUsage(os.Stdout) 379 | return 380 | default: 381 | fmt.Fprintf(os.Stderr, "Error: unknown option %q\n\n", arg) 382 | showUsage(os.Stderr) 383 | os.Exit(1) 384 | } 385 | } 386 | 387 | verbosePrintf("detected Go version %v\n", goVersion) 388 | 389 | preserveSymbols := false 390 | for i := range buildTags { 391 | buildTags[i] = strings.TrimSpace(buildTags[i]) 392 | if buildTags[i] == "debug" || buildTags[i] == "profile" { 393 | preserveSymbols = true 394 | } 395 | } 396 | 397 | verbosePrintf("build tags: %s\n", buildTags) 398 | 399 | root, err := os.Getwd() 400 | if err != nil { 401 | die("Getwd(): %v\n", err) 402 | } 403 | 404 | if outputFilename == "" { 405 | outputFilename = config.Name 406 | if env["GOOS"] == "windows" { 407 | outputFilename += ".exe" 408 | } 409 | } 410 | 411 | output := outputFilename 412 | if !filepath.IsAbs(output) { 413 | output = filepath.Join(root, output) 414 | } 415 | 416 | version := getVersion() 417 | constants := Constants{} 418 | if version != "" { 419 | constants["main.version"] = version 420 | } 421 | ldflags := constants.LDFlags() 422 | if !preserveSymbols { 423 | // Strip debug symbols. 424 | ldflags = "-s -w " + ldflags 425 | } 426 | verbosePrintf("ldflags: %s\n", ldflags) 427 | 428 | var ( 429 | buildArgs []string 430 | testArgs []string 431 | ) 432 | 433 | mainPackage := config.Main 434 | if strings.HasPrefix(mainPackage, config.Namespace) { 435 | mainPackage = strings.Replace(mainPackage, config.Namespace, "./", 1) 436 | } 437 | 438 | buildTarget := filepath.FromSlash(mainPackage) 439 | buildCWD, err := os.Getwd() 440 | if err != nil { 441 | die("unable to determine current working directory: %v\n", err) 442 | } 443 | 444 | buildArgs = append(buildArgs, 445 | "-tags", strings.Join(buildTags, " "), 446 | "-ldflags", ldflags, 447 | "-o", output, buildTarget, 448 | ) 449 | 450 | err = build(buildCWD, env, buildArgs...) 451 | if err != nil { 452 | die("build failed: %v\n", err) 453 | } 454 | 455 | if runTests { 456 | verbosePrintf("running tests\n") 457 | 458 | testArgs = append(testArgs, config.Tests...) 459 | 460 | err = test(buildCWD, env, testArgs...) 461 | if err != nil { 462 | die("running tests failed: %v\n", err) 463 | } 464 | } 465 | } 466 | -------------------------------------------------------------------------------- /changelog/0.10.0_2020-09-13/issue-102: -------------------------------------------------------------------------------- 1 | Change: Remove vendored dependencies 2 | 3 | We've removed the vendored dependencies (in the subdir `vendor/`) similar to 4 | what we did for `restic` itself. When building restic, the Go compiler 5 | automatically fetches the dependencies. It will also cryptographically verify 6 | that the correct code has been fetched by using the hashes in `go.sum` (see the 7 | link to the documentation below). 8 | 9 | Building the rest-server now requires Go 1.11 or newer, since we're using Go 10 | Modules for dependency management. Older Go versions are not supported any more. 11 | 12 | https://github.com/restic/rest-server/issues/102 13 | https://golang.org/cmd/go/#hdr-Module_downloading_and_verification 14 | 15 | -------------------------------------------------------------------------------- /changelog/0.10.0_2020-09-13/issue-117: -------------------------------------------------------------------------------- 1 | Security: Stricter path sanitization 2 | 3 | The framework we're using in rest-server to decode paths to repositories 4 | allowed specifying URL-encoded characters in paths, including sensitive 5 | characters such as `/` (encoded as `%2F`). 6 | 7 | We've changed this unintended behavior, such that rest-server now rejects 8 | such paths. In particular, it is no longer possible to specify sub-repositories 9 | for users by encoding the path with `%2F`, such as `http://localhost:8000/foo%2Fbar`, 10 | which means that this will unfortunately be a breaking change in that case. 11 | 12 | If using sub-repositories for users is important to you, please let us know in 13 | the forum, so we can learn about your use case and implement this properly. As 14 | it currently stands, the ability to use sub-repositories was an unintentional 15 | feature made possible by the URL decoding framework used, and hence never meant 16 | to be supported in the first place. If we wish to have this feature in 17 | rest-server, we'd like to have it implemented properly and intentionally. 18 | 19 | https://github.com/restic/rest-server/issues/117 20 | -------------------------------------------------------------------------------- /changelog/0.10.0_2020-09-13/issue-44: -------------------------------------------------------------------------------- 1 | Enhancement: Add changelog file 2 | 3 | https://github.com/restic/rest-server/issues/44 4 | https://github.com/restic/rest-server/pull/62 5 | -------------------------------------------------------------------------------- /changelog/0.10.0_2020-09-13/issue-60: -------------------------------------------------------------------------------- 1 | Security: Require auth by default, add --no-auth flag 2 | 3 | In order to prevent users from accidentally exposing rest-server without 4 | authentication, rest-server now defaults to requiring a .htpasswd. If you want 5 | to disable authentication, you need to explicitly pass the new --no-auth flag. 6 | 7 | https://github.com/restic/rest-server/issues/60 8 | https://github.com/restic/rest-server/pull/61 9 | -------------------------------------------------------------------------------- /changelog/0.10.0_2020-09-13/pull-64: -------------------------------------------------------------------------------- 1 | Security: Refuse overwriting config file in append-only mode 2 | 3 | While working on the `rclone serve restic` command we noticed that is currently 4 | possible to overwrite the config file in a repo even if `--append-only` is 5 | specified. The first commit adds proper tests, and the second commit fixes the 6 | issue. 7 | 8 | https://github.com/restic/rest-server/pull/64 9 | -------------------------------------------------------------------------------- /changelog/0.11.0_2022-02-10/issue-119: -------------------------------------------------------------------------------- 1 | Bugfix: Fix Docker configuration for `DISABLE_AUTHENTICATION` 2 | 3 | rest-server 0.10.0 introduced a regression which caused the 4 | `DISABLE_AUTHENTICATION` environment variable to stop working for the Docker 5 | container. This has been fixed by automatically setting the option `--no-auth` 6 | to disable authentication. 7 | 8 | https://github.com/restic/rest-server/issues/119 9 | https://github.com/restic/rest-server/pull/124 10 | -------------------------------------------------------------------------------- /changelog/0.11.0_2022-02-10/issue-122: -------------------------------------------------------------------------------- 1 | Enhancement: Verify uploaded files 2 | 3 | The rest-server now by default verifies that the hash of content of uploaded 4 | files matches their filename. This ensures that transmission errors are 5 | detected and forces restic to retry the upload. On low-power devices it can 6 | make sense to disable this check by passing the `--no-verify-upload` flag. 7 | 8 | https://github.com/restic/rest-server/issues/122 9 | https://github.com/restic/rest-server/pull/130 10 | -------------------------------------------------------------------------------- /changelog/0.11.0_2022-02-10/issue-126: -------------------------------------------------------------------------------- 1 | Enhancement: Allow running rest-server via systemd socket activation 2 | 3 | We've added the option to have systemd create the listening socket and start the rest-server on demand. 4 | 5 | https://github.com/restic/rest-server/issues/126 6 | https://github.com/restic/rest-server/pull/151 7 | https://github.com/restic/rest-server/pull/127 8 | -------------------------------------------------------------------------------- /changelog/0.11.0_2022-02-10/issue-131: -------------------------------------------------------------------------------- 1 | Security: Prevent loading of usernames containing a slash 2 | 3 | "/" is valid char in HTTP authorization headers, but is also used in 4 | rest-server to map usernames to private repos. 5 | 6 | This commit prevents loading maliciously composed usernames like 7 | "/foo/config" by restricting the allowed characters to the unicode 8 | character class, numbers, "-", "." and "@". 9 | 10 | This prevents requests to other users files like: 11 | 12 | curl -v -X DELETE -u foo/config:attack http://localhost:8000/foo/config 13 | 14 | https://github.com/restic/rest-server/issues/131 15 | https://github.com/restic/rest-server/pull/132 16 | https://github.com/restic/rest-server/pull/137 17 | -------------------------------------------------------------------------------- /changelog/0.11.0_2022-02-10/issue-146: -------------------------------------------------------------------------------- 1 | Change: Build rest-server at docker container build time 2 | 3 | The Dockerfile now includes a build stage such that the latest rest-server is 4 | always built and packaged. This is done in a standard golang container to 5 | ensure a clean build environment and only the final binary is shipped rather 6 | than the whole build environment. 7 | 8 | https://github.com/restic/rest-server/issues/146 9 | https://github.com/restic/rest-server/pull/145 10 | -------------------------------------------------------------------------------- /changelog/0.11.0_2022-02-10/issue-148: -------------------------------------------------------------------------------- 1 | Enhancement: Expand use of security features in example systemd unit file 2 | 3 | The example systemd unit file now enables additional systemd features to 4 | mitigate potential security vulnerabilities in rest-server and the various 5 | packages and operating system components which it relies upon. 6 | 7 | https://github.com/restic/rest-server/issues/148 8 | https://github.com/restic/rest-server/pull/149 9 | -------------------------------------------------------------------------------- /changelog/0.11.0_2022-02-10/pull-112: -------------------------------------------------------------------------------- 1 | Change: Add subrepo support and refactor server code 2 | 3 | Support for multi-level repositories has been added, so now each user can have 4 | its own subrepositories. This feature is always enabled. 5 | 6 | Authentication for the Prometheus /metrics endpoint can now be disabled with the 7 | new `--prometheus-no-auth` flag. 8 | 9 | We have split out all HTTP handling to a separate `repo` subpackage to cleanly 10 | separate the server code from the code that handles a single repository. The new 11 | RepoHandler also makes it easier to reuse rest-server as a Go component in 12 | any other HTTP server. 13 | 14 | The refactoring makes the code significantly easier to follow and understand, 15 | which in turn makes it easier to add new features, audit for security and debug 16 | issues. 17 | 18 | https://github.com/restic/restic/pull/112 19 | https://github.com/restic/restic/issues/109 20 | https://github.com/restic/restic/issues/107 21 | -------------------------------------------------------------------------------- /changelog/0.11.0_2022-02-10/pull-142: -------------------------------------------------------------------------------- 1 | Bugfix: Fix possible data loss due to interrupted network connections 2 | 3 | When rest-server was run without `--append-only` it was possible to lose uploaded 4 | files in a specific scenario in which a network connection was interrupted. 5 | 6 | For the data loss to occur a file upload by restic would have to be interrupted 7 | such that restic notices the interrupted network connection before the 8 | rest-server. Then restic would have to retry the file upload and finish it 9 | before the rest-server notices that the initial upload has failed. Then the 10 | uploaded file would be accidentally removed by rest-server when trying to 11 | cleanup the failed upload. 12 | 13 | This has been fixed by always uploading to a temporary file first which is moved 14 | in position only once it was uploaded completely. 15 | 16 | https://github.com/restic/rest-server/pull/142 17 | -------------------------------------------------------------------------------- /changelog/0.11.0_2022-02-10/pull-158: -------------------------------------------------------------------------------- 1 | Bugfix: Use platform-specific temporary directory as default data directory 2 | 3 | If no data directory is specificed, then rest-server now uses the Go standard 4 | library functions to retrieve the standard temporary directory path for the 5 | current platform. 6 | 7 | https://github.com/restic/rest-server/issues/157 8 | https://github.com/restic/rest-server/pull/158 9 | -------------------------------------------------------------------------------- /changelog/0.11.0_2022-02-10/pull-160: -------------------------------------------------------------------------------- 1 | Bugfix: Reply "insufficient storage" on disk full or over-quota 2 | 3 | When there was no space left on disk, or any other write-related error 4 | occurred, rest-server replied with HTTP status code 400 (Bad request). 5 | This is misleading (restic client will dump the status code to the user). 6 | 7 | rest-server now replies with two different status codes in these situations: 8 | * HTTP 507 "Insufficient storage" is the status on disk full or repository 9 | over-quota 10 | * HTTP 500 "Internal server error" is used for other disk-related errors 11 | 12 | https://github.com/restic/rest-server/issues/155 13 | https://github.com/restic/rest-server/pull/160 14 | -------------------------------------------------------------------------------- /changelog/0.12.0_2023-04-24/issue-133: -------------------------------------------------------------------------------- 1 | Enhancement: Cache basic authentication credentials 2 | 3 | To speed up the verification of basic auth credentials, rest-server now caches 4 | passwords for a minute in memory. That way the expensive verification of basic 5 | auth credentials can be skipped for most requests issued by a single restic 6 | run. The password is kept in memory in a hashed form and not as plaintext. 7 | 8 | https://github.com/restic/rest-server/issues/133 9 | https://github.com/restic/rest-server/pull/138 10 | -------------------------------------------------------------------------------- /changelog/0.12.0_2023-04-24/issue-182: -------------------------------------------------------------------------------- 1 | Bugfix: Allow usernames containing underscore and more 2 | 3 | The security fix in rest-server 0.11.0 (#131) disallowed usernames containing 4 | and underscore "_". The list of allowed characters has now been changed to 5 | include Unicode characters, numbers, "_", "-", "." and "@". 6 | 7 | https://github.com/restic/restic/issues/183 8 | https://github.com/restic/restic/pull/184 9 | -------------------------------------------------------------------------------- /changelog/0.12.0_2023-04-24/issue-187: -------------------------------------------------------------------------------- 1 | Enhancement: Allow configurable location for `.htpasswd` file 2 | 3 | It is now possible to specify the location of the `.htpasswd` 4 | file using the `--htpasswd-file` option. 5 | 6 | https://github.com/restic/restic/issues/187 7 | https://github.com/restic/restic/pull/188 8 | -------------------------------------------------------------------------------- /changelog/0.12.0_2023-04-24/issue-219: -------------------------------------------------------------------------------- 1 | Bugfix: Ignore unexpected files in the data/ folder 2 | 3 | If the data folder of a repository contained files, this would prevent restic 4 | from retrieving a list of file data files. This has been fixed. As a workaround 5 | remove the files that are directly contained in the data folder (e.g., 6 | `.DS_Store` files). 7 | 8 | https://github.com/restic/rest-server/issues/219 9 | https://github.com/restic/rest-server/pull/221 10 | -------------------------------------------------------------------------------- /changelog/0.12.0_2023-04-24/pull-194: -------------------------------------------------------------------------------- 1 | Bugfix: Return 500 "Internal server error" if files cannot be read 2 | 3 | When files in a repository cannot be read by rest-server, for example after 4 | running `restic prune` directly on the server hosting the repositories in a 5 | way that causes filesystem permissions to be wrong, rest-server previously 6 | returned 404 "Not Found" as status code. This was causing confusing for users. 7 | 8 | The error handling has now been fixed to only return 404 "Not Found" if the 9 | file actually does not exist. Otherwise a 500 "Internal server error" is 10 | reported to the client and the underlying error is logged at the server side. 11 | 12 | https://github.com/restic/restic/issues/1871 13 | https://github.com/restic/rest-server/pull/195 14 | -------------------------------------------------------------------------------- /changelog/0.12.0_2023-04-24/pull-207: -------------------------------------------------------------------------------- 1 | Change: Return error if command-line arguments are specified 2 | 3 | Command line arguments are ignored by rest-server, but there was previously 4 | no indication of this when they were supplied anyway. 5 | 6 | To prevent usage errors an error is now printed when command line arguments 7 | are supplied, instead of them being silently ignored. 8 | 9 | https://github.com/restic/rest-server/pull/207 10 | -------------------------------------------------------------------------------- /changelog/0.12.0_2023-04-24/pull-208: -------------------------------------------------------------------------------- 1 | Change: Update dependencies and require Go 1.17 or newer 2 | 3 | Most dependencies have been updated. Since some libraries require newer language 4 | features, support for Go 1.15-1.16 has been dropped, which means that rest-server 5 | now requires at least Go 1.17 to build. 6 | 7 | https://github.com/restic/rest-server/pull/208 8 | -------------------------------------------------------------------------------- /changelog/0.12.1_2023-07-09/issue-230: -------------------------------------------------------------------------------- 1 | Bugfix: Fix erroneous warnings about unsupported fsync 2 | 3 | Due to a regression in rest-server 0.12.0, it continuously printed 4 | `WARNING: fsync is not supported by the data storage. This can lead to data loss, 5 | if the system crashes or the storage is unexpectedly disconnected.` for systems 6 | that support fsync. We have fixed the warning. 7 | 8 | https://github.com/restic/rest-server/issues/230 9 | https://github.com/restic/rest-server/pull/231 10 | -------------------------------------------------------------------------------- /changelog/0.12.1_2023-07-09/issue-238: -------------------------------------------------------------------------------- 1 | Bugfix: API: Return empty array when listing empty folders 2 | 3 | Rest-server returned `null` when listing an empty folder. This has been changed 4 | to returning an empty array in accordance with the REST protocol specification. 5 | This change has no impact on restic users. 6 | 7 | https://github.com/restic/rest-server/issues/238 8 | https://github.com/restic/rest-server/pull/239 9 | -------------------------------------------------------------------------------- /changelog/0.12.1_2023-07-09/pull-217: -------------------------------------------------------------------------------- 1 | Enhancement: Log to stdout using the `--log -` option 2 | 3 | Logging to stdout was possible using `--log /dev/stdout`. However, 4 | when the rest server is run as a different user, for example, using 5 | 6 | `sudo -u restic rest-server [...] --log /dev/stdout` 7 | 8 | this did not work due to permission issues. 9 | 10 | For logging to stdout, the `--log` option now supports the special 11 | filename `-` which also works in these cases. 12 | 13 | https://github.com/restic/rest-server/pull/217 14 | -------------------------------------------------------------------------------- /changelog/0.13.0_2024-07-26/pull-267: -------------------------------------------------------------------------------- 1 | Change: Update dependencies and require Go 1.18 or newer 2 | 3 | Most dependencies have been updated. Since some libraries require newer language 4 | features, support for Go 1.17 has been dropped, which means that rest-server 5 | now requires at least Go 1.18 to build. 6 | 7 | https://github.com/restic/rest-server/pull/267 8 | -------------------------------------------------------------------------------- /changelog/0.13.0_2024-07-26/pull-271: -------------------------------------------------------------------------------- 1 | Enhancement: Print listening address after start-up 2 | 3 | When started with `--listen :0`, rest-server would print `start server on :0` 4 | 5 | The message now also includes the actual address listened on, for example 6 | `start server on 0.0.0.0:37333`. This is useful when starting a server with 7 | an auto-allocated free port number (port 0). 8 | 9 | https://github.com/restic/rest-server/pull/271 10 | -------------------------------------------------------------------------------- /changelog/0.13.0_2024-07-26/pull-272: -------------------------------------------------------------------------------- 1 | Enhancement: Support listening on a unix socket 2 | 3 | It is now possible to make rest-server listen on a unix socket by prefixing 4 | the socket filename with `unix:` and passing it to the `--listen` option, 5 | for example `--listen unix:/tmp/foo`. 6 | 7 | This is useful in combination with remote port forwarding to enable a remote 8 | server to backup locally, e.g.: 9 | 10 | ``` 11 | rest-server --listen unix:/tmp/foo & 12 | ssh -R /tmp/foo:/tmp/foo user@host restic -r rest:http+unix:///tmp/foo:/repo backup 13 | ``` 14 | 15 | https://github.com/restic/rest-server/pull/272 16 | -------------------------------------------------------------------------------- /changelog/0.13.0_2024-07-26/pull-273: -------------------------------------------------------------------------------- 1 | Change: Shut down cleanly on TERM and INT signals 2 | 3 | Rest-server now listens for TERM and INT signals and cleanly closes down the 4 | http.Server and listener when receiving either of them. 5 | 6 | This is particularly useful when listening on a unix socket, as the server 7 | will now remove the socket file when it shuts down. 8 | 9 | https://github.com/restic/rest-server/pull/273 10 | -------------------------------------------------------------------------------- /changelog/0.14.0_2025-05-31/issue-189: -------------------------------------------------------------------------------- 1 | Enhancement: Support group accessible repositories 2 | 3 | Rest-server now supports making repositories accessible to the filesystem group 4 | by setting the `--group-accessible-repos` option. Note that permissions of 5 | existing files are not modified. To allow the group to read and write file, 6 | use a umask of `007`. To only grant read access use `027`. To make an existing 7 | repository group-accessible, use `chmod -R g+rwX /path/to/repo`. 8 | 9 | https://github.com/restic/rest-server/issues/189 10 | https://github.com/restic/rest-server/pull/308 11 | -------------------------------------------------------------------------------- /changelog/0.14.0_2025-05-31/issue-318: -------------------------------------------------------------------------------- 1 | Security: Fix world-readable permissions on new `.htpasswd` files 2 | 3 | On startup the rest-server Docker container creates an empty `.htpasswd` file 4 | if none exists yet. This file was world-readable by default, which can be 5 | a security risk, even though the file only contains hashed passwords. 6 | 7 | This has been fixed such that new `.htpasswd` files are no longer world-readabble. 8 | 9 | The permissions of existing `.htpasswd` files must be manually changed if 10 | relevant in your setup. 11 | 12 | https://github.com/restic/rest-server/issues/318 13 | https://github.com/restic/rest-server/pull/340 14 | -------------------------------------------------------------------------------- /changelog/0.14.0_2025-05-31/issue-321: -------------------------------------------------------------------------------- 1 | Enhancement: Add zip archive format for Windows releases 2 | 3 | Windows users can now download rest-server binaries in zip archive format (.zip) 4 | in addition to the existing tar.gz archives. 5 | 6 | https://github.com/restic/rest-server/issues/321 7 | https://github.com/restic/rest-server/pull/346 8 | -------------------------------------------------------------------------------- /changelog/0.14.0_2025-05-31/pull-295: -------------------------------------------------------------------------------- 1 | Enhancement: Output status of append-only mode on startup 2 | 3 | Rest-server now displays the status of append-only mode during startup. 4 | 5 | https://github.com/restic/rest-server/pull/295 6 | -------------------------------------------------------------------------------- /changelog/0.14.0_2025-05-31/pull-307: -------------------------------------------------------------------------------- 1 | Enhancement: Support proxy-based authentication 2 | 3 | Rest-server now supports authentication via HTTP proxy headers. This feature can 4 | be enabled by specifying the username header using the `--proxy-auth-username` 5 | option (e.g., `--proxy-auth-username=X-Forwarded-User`). 6 | 7 | When enabled, the server authenticates users based on the specified header and 8 | disables Basic Auth. Note that proxy authentication is disabled when `--no-auth` 9 | is set. 10 | 11 | https://github.com/restic/rest-server/issues/174 12 | https://github.com/restic/rest-server/pull/307 13 | -------------------------------------------------------------------------------- /changelog/0.14.0_2025-05-31/pull-315: -------------------------------------------------------------------------------- 1 | Enhancement: Hardened tls settings 2 | 3 | Rest-server now uses a secure TLS cipher suite set by default. The minimum TLS 4 | version is now TLS 1.2 and can be further increased using the new `--tls-min-ver` 5 | option, allowing users to enforce stricter security requirements. 6 | 7 | https://github.com/restic/rest-server/pull/315 8 | -------------------------------------------------------------------------------- /changelog/0.14.0_2025-05-31/pull-322: -------------------------------------------------------------------------------- 1 | Change: Update dependencies and require Go 1.23 or newer 2 | 3 | All dependencies have been updated. Rest-server now requires Go 1.23 or newer 4 | to build. 5 | 6 | This also disables support for TLS versions older than TLS 1.2. On Windows, 7 | rest-server now requires at least Windows 10 or Windows Server 2016. On macOS, 8 | rest-server now requires at least macOS 11 Big Sur. 9 | 10 | https://github.com/restic/rest-server/pull/322 11 | https://github.com/restic/rest-server/pull/338 12 | -------------------------------------------------------------------------------- /changelog/CHANGELOG-GitHub.tmpl: -------------------------------------------------------------------------------- 1 | {{- range $changes := . }}{{ with $changes -}} 2 | Changelog for rest-server {{ .Version }} ({{ .Date }}) 3 | ========================================= 4 | 5 | The following sections list the changes in rest-server {{ .Version }} relevant to users. The changes are ordered by importance. 6 | 7 | Summary 8 | ------- 9 | {{ range $entry := .Entries }}{{ with $entry }} 10 | * {{ .TypeShort }} [#{{ .PrimaryID }}]({{ .PrimaryURL }}): {{ .Title }} 11 | {{- end }}{{ end }} 12 | 13 | Details 14 | ------- 15 | {{ range $entry := .Entries }}{{ with $entry }} 16 | * {{ .Type }} #{{ .PrimaryID }}: {{ .Title }} 17 | {{ range $par := .Paragraphs }} 18 | {{ $par }} 19 | {{ end }} 20 | {{ range $id := .Issues -}} 21 | {{ ` ` }}[#{{ $id }}](https://github.com/restic/rest-server/issues/{{ $id -}}) 22 | {{- end -}} 23 | {{ range $id := .PRs -}} 24 | {{ ` ` }}[#{{ $id }}](https://github.com/restic/rest-server/pull/{{ $id -}}) 25 | {{- end -}} 26 | {{ ` ` }}{{ range $url := .OtherURLs -}} 27 | {{ $url -}} 28 | {{- end }} 29 | {{ end }}{{ end }} 30 | 31 | {{ end }}{{ end -}} 32 | -------------------------------------------------------------------------------- /changelog/CHANGELOG.tmpl: -------------------------------------------------------------------------------- 1 | {{- range $changes := . }}{{ with $changes -}} 2 | Changelog for rest-server {{ .Version }} ({{ .Date }}) 3 | ============================================ 4 | 5 | The following sections list the changes in rest-server {{ .Version }} relevant 6 | to users. The changes are ordered by importance. 7 | 8 | Summary 9 | ------- 10 | {{ range $entry := .Entries }}{{ with $entry }} 11 | * {{ .TypeShort }} #{{ .PrimaryID }}: {{ .Title }} 12 | {{- end }}{{ end }} 13 | 14 | Details 15 | ------- 16 | {{ range $entry := .Entries }}{{ with $entry }} 17 | * {{ .Type }} #{{ .PrimaryID }}: {{ .Title }} 18 | {{ range $par := .Paragraphs }} 19 | {{ wrapIndent $par 80 3 }} 20 | {{ end -}} 21 | {{ range $id := .Issues }} 22 | https://github.com/restic/rest-server/issues/{{ $id -}} 23 | {{ end -}} 24 | {{ range $id := .PRs }} 25 | https://github.com/restic/rest-server/pull/{{ $id -}} 26 | {{ end -}} 27 | {{ range $url := .OtherURLs }} 28 | {{ $url -}} 29 | {{ end }} 30 | {{ end }}{{ end }} 31 | 32 | {{ end }}{{ end -}} 33 | -------------------------------------------------------------------------------- /changelog/TEMPLATE: -------------------------------------------------------------------------------- 1 | # The first line must start with Bugfix:, Enhancement: or Change:, 2 | # including the colon. Use present tense. Remove lines starting with '#' 3 | # from this template. 4 | Bugfix: Fix behavior for foobar (in present tense) 5 | 6 | # Describe the problem in the past tense, the new behavior in the present 7 | # tense. Mention the affected commands, backends, operating systems, etc. 8 | # Focus on user-facing behavior, not the implementation. 9 | 10 | We've fixed the behavior for foobar, a long-standing annoyance for rest-server 11 | users. 12 | 13 | # The last section is a list of issue, PR and forum URLs. 14 | # The first issue ID determines the filename for the changelog entry: 15 | # changelog/unreleased/issue-1234. If there are no relevant issue links, 16 | # use the PR ID and call the file pull-55555. 17 | 18 | https://github.com/restic/rest-server/issues/1234 19 | https://github.com/restic/rest-server/pull/55555 20 | https://forum.restic.net/foo/bar/baz 21 | -------------------------------------------------------------------------------- /changelog/unreleased/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/restic/rest-server/2f31e10cebf335003f75fee5b4646fb73edcda45/changelog/unreleased/.gitkeep -------------------------------------------------------------------------------- /cmd/rest-server/listener_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | // +build !windows 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "log" 9 | "net" 10 | "strings" 11 | 12 | "github.com/coreos/go-systemd/v22/activation" 13 | ) 14 | 15 | // findListener tries to find a listener via systemd socket activation. If that 16 | // fails, it tries to create a listener on addr. 17 | func findListener(addr string) (listener net.Listener, err error) { 18 | // try systemd socket activation 19 | listeners, err := activation.Listeners() 20 | if err != nil { 21 | panic(err) 22 | } 23 | 24 | switch len(listeners) { 25 | case 0: 26 | // no listeners found, listen manually 27 | if strings.HasPrefix(addr, "unix:") { // if we want to listen on a unix socket 28 | unixAddr, err := net.ResolveUnixAddr("unix", strings.TrimPrefix(addr, "unix:")) 29 | if err != nil { 30 | return nil, fmt.Errorf("unable to understand unix address %s: %w", addr, err) 31 | } 32 | listener, err = net.ListenUnix("unix", unixAddr) 33 | if err != nil { 34 | return nil, fmt.Errorf("listen on %v failed: %w", addr, err) 35 | } 36 | } else { // assume tcp 37 | listener, err = net.Listen("tcp", addr) 38 | if err != nil { 39 | return nil, fmt.Errorf("listen on %v failed: %w", addr, err) 40 | } 41 | } 42 | 43 | log.Printf("start server on %v", listener.Addr()) 44 | return listener, nil 45 | 46 | case 1: 47 | // one listener supplied by systemd, use that one 48 | // 49 | // for testing, run rest-server with systemd-socket-activate as follows: 50 | // 51 | // systemd-socket-activate -l 8080 ./rest-server 52 | log.Printf("systemd socket activation mode") 53 | return listeners[0], nil 54 | 55 | default: 56 | return nil, fmt.Errorf("got %d listeners from systemd, expected one", len(listeners)) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /cmd/rest-server/listener_unix_test.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | // +build !windows 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "net" 10 | "net/http" 11 | "os" 12 | "path/filepath" 13 | "testing" 14 | "time" 15 | ) 16 | 17 | func TestUnixSocket(t *testing.T) { 18 | td := t.TempDir() 19 | 20 | // this is the socket we'll listen on and connect to 21 | tempSocket := filepath.Join(td, "sock") 22 | 23 | // create some content and parent dirs 24 | if err := os.MkdirAll(filepath.Join(td, "data", "repo1"), 0700); err != nil { 25 | t.Fatal(err) 26 | } 27 | if err := os.WriteFile(filepath.Join(td, "data", "repo1", "config"), []byte("foo"), 0700); err != nil { 28 | t.Fatal(err) 29 | } 30 | 31 | // run the following twice, to test that the server will 32 | // cleanup its socket file when quitting, which won't happen 33 | // if it doesn't exit gracefully 34 | for i := 0; i < 2; i++ { 35 | err := testServerWithArgs([]string{ 36 | "--no-auth", 37 | "--path", filepath.Join(td, "data"), 38 | "--listen", fmt.Sprintf("unix:%s", tempSocket), 39 | }, time.Second, func(ctx context.Context, _ *restServerApp) error { 40 | // custom client that will talk HTTP to unix socket 41 | client := http.Client{ 42 | Transport: &http.Transport{ 43 | DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { 44 | return net.Dial("unix", tempSocket) 45 | }, 46 | }, 47 | } 48 | for _, test := range []struct { 49 | Path string 50 | StatusCode int 51 | }{ 52 | {"/repo1/", http.StatusMethodNotAllowed}, 53 | {"/repo1/config", http.StatusOK}, 54 | {"/repo2/config", http.StatusNotFound}, 55 | } { 56 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://ignored"+test.Path, nil) 57 | if err != nil { 58 | return err 59 | } 60 | resp, err := client.Do(req) 61 | if err != nil { 62 | return err 63 | } 64 | err = resp.Body.Close() 65 | if err != nil { 66 | return err 67 | } 68 | if resp.StatusCode != test.StatusCode { 69 | return fmt.Errorf("expected %d from server, instead got %d (path %s)", test.StatusCode, resp.StatusCode, test.Path) 70 | } 71 | } 72 | return nil 73 | }) 74 | if err != nil { 75 | t.Fatal(err) 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /cmd/rest-server/listener_windows.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | ) 8 | 9 | // findListener creates a listener. 10 | func findListener(addr string) (listener net.Listener, err error) { 11 | // listen manually 12 | listener, err = net.Listen("tcp", addr) 13 | if err != nil { 14 | return nil, fmt.Errorf("listen on %v failed: %w", addr, err) 15 | } 16 | 17 | log.Printf("start server on %v", listener.Addr()) 18 | return listener, nil 19 | } 20 | -------------------------------------------------------------------------------- /cmd/rest-server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "errors" 7 | "fmt" 8 | "log" 9 | "net" 10 | "net/http" 11 | "os" 12 | "os/signal" 13 | "path/filepath" 14 | "runtime" 15 | "runtime/pprof" 16 | "sync" 17 | "syscall" 18 | 19 | restserver "github.com/restic/rest-server" 20 | "github.com/spf13/cobra" 21 | ) 22 | 23 | type restServerApp struct { 24 | CmdRoot *cobra.Command 25 | Server restserver.Server 26 | CPUProfile string 27 | 28 | listenerAddressMu sync.Mutex 29 | listenerAddress net.Addr // set after startup 30 | } 31 | 32 | // cmdRoot is the base command when no other command has been specified. 33 | func newRestServerApp() *restServerApp { 34 | rv := &restServerApp{ 35 | CmdRoot: &cobra.Command{ 36 | Use: "rest-server", 37 | Short: "Run a REST server for use with restic", 38 | SilenceErrors: true, 39 | SilenceUsage: true, 40 | Args: func(_ *cobra.Command, args []string) error { 41 | if len(args) != 0 { 42 | return fmt.Errorf("rest-server expects no arguments - unknown argument: %s", args[0]) 43 | } 44 | return nil 45 | }, 46 | Version: fmt.Sprintf("rest-server %s compiled with %v on %v/%v\n", version, runtime.Version(), runtime.GOOS, runtime.GOARCH), 47 | }, 48 | Server: restserver.Server{ 49 | Path: filepath.Join(os.TempDir(), "restic"), 50 | Listen: ":8000", 51 | TLSMinVer: "1.2", 52 | }, 53 | } 54 | rv.CmdRoot.RunE = rv.runRoot 55 | flags := rv.CmdRoot.Flags() 56 | 57 | flags.StringVar(&rv.CPUProfile, "cpu-profile", rv.CPUProfile, "write CPU profile to file") 58 | flags.BoolVar(&rv.Server.Debug, "debug", rv.Server.Debug, "output debug messages") 59 | flags.StringVar(&rv.Server.Listen, "listen", rv.Server.Listen, "listen address") 60 | flags.StringVar(&rv.Server.Log, "log", rv.Server.Log, "write HTTP requests in the combined log format to the specified `filename` (use \"-\" for logging to stdout)") 61 | flags.Int64Var(&rv.Server.MaxRepoSize, "max-size", rv.Server.MaxRepoSize, "the maximum size of the repository in bytes") 62 | flags.StringVar(&rv.Server.Path, "path", rv.Server.Path, "data directory") 63 | flags.BoolVar(&rv.Server.TLS, "tls", rv.Server.TLS, "turn on TLS support") 64 | flags.StringVar(&rv.Server.TLSCert, "tls-cert", rv.Server.TLSCert, "TLS certificate path") 65 | flags.StringVar(&rv.Server.TLSKey, "tls-key", rv.Server.TLSKey, "TLS key path") 66 | flags.StringVar(&rv.Server.TLSMinVer, "tls-min-ver", rv.Server.TLSMinVer, "TLS min version, one of (1.2|1.3)") 67 | flags.BoolVar(&rv.Server.NoAuth, "no-auth", rv.Server.NoAuth, "disable authentication") 68 | flags.StringVar(&rv.Server.HtpasswdPath, "htpasswd-file", rv.Server.HtpasswdPath, "location of .htpasswd file (default: \"/.htpasswd)\"") 69 | flags.StringVar(&rv.Server.ProxyAuthUsername, "proxy-auth-username", rv.Server.ProxyAuthUsername, "specifies the HTTP header containing the username for proxy-based authentication") 70 | flags.BoolVar(&rv.Server.NoVerifyUpload, "no-verify-upload", rv.Server.NoVerifyUpload, 71 | "do not verify the integrity of uploaded data. DO NOT enable unless the rest-server runs on a very low-power device") 72 | flags.BoolVar(&rv.Server.AppendOnly, "append-only", rv.Server.AppendOnly, "enable append only mode") 73 | flags.BoolVar(&rv.Server.PrivateRepos, "private-repos", rv.Server.PrivateRepos, "users can only access their private repo") 74 | flags.BoolVar(&rv.Server.Prometheus, "prometheus", rv.Server.Prometheus, "enable Prometheus metrics") 75 | flags.BoolVar(&rv.Server.PrometheusNoAuth, "prometheus-no-auth", rv.Server.PrometheusNoAuth, "disable auth for Prometheus /metrics endpoint") 76 | flags.BoolVar(&rv.Server.GroupAccessibleRepos, "group-accessible-repos", rv.Server.GroupAccessibleRepos, "let filesystem group be able to access repo files") 77 | 78 | return rv 79 | } 80 | 81 | var version = "0.14.0-dev" 82 | 83 | func (app *restServerApp) tlsSettings() (bool, string, string, error) { 84 | var key, cert string 85 | if !app.Server.TLS && (app.Server.TLSKey != "" || app.Server.TLSCert != "") { 86 | return false, "", "", errors.New("requires enabled TLS") 87 | } else if !app.Server.TLS { 88 | return false, "", "", nil 89 | } 90 | if app.Server.TLSKey != "" { 91 | key = app.Server.TLSKey 92 | } else { 93 | key = filepath.Join(app.Server.Path, "private_key") 94 | } 95 | if app.Server.TLSCert != "" { 96 | cert = app.Server.TLSCert 97 | } else { 98 | cert = filepath.Join(app.Server.Path, "public_key") 99 | } 100 | return app.Server.TLS, key, cert, nil 101 | } 102 | 103 | // returns the address that the app is listening on. 104 | // returns nil if the application hasn't finished starting yet 105 | func (app *restServerApp) ListenerAddress() net.Addr { 106 | app.listenerAddressMu.Lock() 107 | defer app.listenerAddressMu.Unlock() 108 | return app.listenerAddress 109 | } 110 | 111 | func (app *restServerApp) runRoot(_ *cobra.Command, _ []string) error { 112 | log.SetFlags(0) 113 | 114 | log.Printf("Data directory: %s", app.Server.Path) 115 | 116 | if app.CPUProfile != "" { 117 | f, err := os.Create(app.CPUProfile) 118 | if err != nil { 119 | return err 120 | } 121 | defer func() { 122 | _ = f.Close() 123 | }() 124 | 125 | if err := pprof.StartCPUProfile(f); err != nil { 126 | return err 127 | } 128 | defer pprof.StopCPUProfile() 129 | 130 | log.Println("CPU profiling enabled") 131 | defer log.Println("Stopped CPU profiling") 132 | } 133 | 134 | if app.Server.NoAuth { 135 | log.Println("Authentication disabled") 136 | } else { 137 | if app.Server.ProxyAuthUsername == "" { 138 | log.Println("Authentication enabled") 139 | } else { 140 | log.Println("Proxy Authentication enabled.") 141 | } 142 | } 143 | 144 | handler, err := restserver.NewHandler(&app.Server) 145 | if err != nil { 146 | log.Fatalf("error: %v", err) 147 | } 148 | 149 | if app.Server.AppendOnly { 150 | log.Println("Append only mode enabled") 151 | } else { 152 | log.Println("Append only mode disabled") 153 | } 154 | 155 | if app.Server.PrivateRepos { 156 | log.Println("Private repositories enabled") 157 | } else { 158 | log.Println("Private repositories disabled") 159 | } 160 | 161 | if app.Server.GroupAccessibleRepos { 162 | log.Println("Group accessible repos enabled") 163 | } else { 164 | log.Println("Group accessible repos disabled") 165 | } 166 | 167 | enabledTLS, privateKey, publicKey, err := app.tlsSettings() 168 | if err != nil { 169 | return err 170 | } 171 | 172 | listener, err := findListener(app.Server.Listen) 173 | if err != nil { 174 | return fmt.Errorf("unable to listen: %w", err) 175 | } 176 | 177 | // set listener address, this is useful for tests 178 | app.listenerAddressMu.Lock() 179 | app.listenerAddress = listener.Addr() 180 | app.listenerAddressMu.Unlock() 181 | 182 | tlscfg := &tls.Config{ 183 | MinVersion: tls.VersionTLS12, 184 | CipherSuites: []uint16{ 185 | tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 186 | tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 187 | tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, 188 | tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, 189 | tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, 190 | tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, 191 | }, 192 | } 193 | switch app.Server.TLSMinVer { 194 | case "1.2": 195 | tlscfg.MinVersion = tls.VersionTLS12 196 | case "1.3": 197 | tlscfg.MinVersion = tls.VersionTLS13 198 | default: 199 | return fmt.Errorf("Unsupported TLS min version: %s. Allowed versions are 1.2 or 1.3", app.Server.TLSMinVer) 200 | } 201 | 202 | srv := &http.Server{ 203 | Handler: handler, 204 | TLSConfig: tlscfg, 205 | } 206 | 207 | // run server in background 208 | go func() { 209 | if !enabledTLS { 210 | err = srv.Serve(listener) 211 | } else { 212 | log.Printf("TLS enabled, private key %s, pubkey %v", privateKey, publicKey) 213 | err = srv.ServeTLS(listener, publicKey, privateKey) 214 | } 215 | if err != nil && !errors.Is(err, http.ErrServerClosed) { 216 | log.Fatalf("listen and serve returned err: %v", err) 217 | } 218 | }() 219 | 220 | // wait until done 221 | <-app.CmdRoot.Context().Done() 222 | 223 | // gracefully shutdown server 224 | if err := srv.Shutdown(context.Background()); err != nil { 225 | return fmt.Errorf("server shutdown returned an err: %w", err) 226 | } 227 | 228 | log.Println("shutdown cleanly") 229 | return nil 230 | } 231 | 232 | func main() { 233 | // create context to be notified on interrupt or term signal so that we can shutdown cleanly 234 | ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) 235 | defer stop() 236 | 237 | if err := newRestServerApp().CmdRoot.ExecuteContext(ctx); err != nil { 238 | log.Fatalf("error: %v", err) 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /cmd/rest-server/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "sync" 13 | "testing" 14 | "time" 15 | 16 | restserver "github.com/restic/rest-server" 17 | ) 18 | 19 | func TestTLSSettings(t *testing.T) { 20 | type expected struct { 21 | TLSKey string 22 | TLSCert string 23 | Error bool 24 | } 25 | type passed struct { 26 | Path string 27 | TLS bool 28 | TLSKey string 29 | TLSCert string 30 | } 31 | 32 | var tests = []struct { 33 | passed passed 34 | expected expected 35 | }{ 36 | {passed{TLS: false}, expected{"", "", false}}, 37 | {passed{TLS: true}, expected{ 38 | filepath.Join(os.TempDir(), "restic/private_key"), 39 | filepath.Join(os.TempDir(), "restic/public_key"), 40 | false, 41 | }}, 42 | {passed{ 43 | Path: os.TempDir(), 44 | TLS: true, 45 | }, expected{ 46 | filepath.Join(os.TempDir(), "private_key"), 47 | filepath.Join(os.TempDir(), "public_key"), 48 | false, 49 | }}, 50 | {passed{Path: os.TempDir(), TLS: true, TLSKey: "/etc/restic/key", TLSCert: "/etc/restic/cert"}, expected{"/etc/restic/key", "/etc/restic/cert", false}}, 51 | {passed{Path: os.TempDir(), TLS: false, TLSKey: "/etc/restic/key", TLSCert: "/etc/restic/cert"}, expected{"", "", true}}, 52 | {passed{Path: os.TempDir(), TLS: false, TLSKey: "/etc/restic/key"}, expected{"", "", true}}, 53 | {passed{Path: os.TempDir(), TLS: false, TLSCert: "/etc/restic/cert"}, expected{"", "", true}}, 54 | } 55 | 56 | for _, test := range tests { 57 | app := newRestServerApp() 58 | t.Run("", func(t *testing.T) { 59 | // defer func() { restserver.Server = defaultConfig }() 60 | if test.passed.Path != "" { 61 | app.Server.Path = test.passed.Path 62 | } 63 | app.Server.TLS = test.passed.TLS 64 | app.Server.TLSKey = test.passed.TLSKey 65 | app.Server.TLSCert = test.passed.TLSCert 66 | 67 | gotTLS, gotKey, gotCert, err := app.tlsSettings() 68 | if err != nil && !test.expected.Error { 69 | t.Fatalf("tls_settings returned err (%v)", err) 70 | } 71 | if test.expected.Error { 72 | if err == nil { 73 | t.Fatalf("Error not returned properly (%v)", test) 74 | } else { 75 | return 76 | } 77 | } 78 | if gotTLS != test.passed.TLS { 79 | t.Errorf("TLS enabled, want (%v), got (%v)", test.passed.TLS, gotTLS) 80 | } 81 | wantKey := test.expected.TLSKey 82 | if gotKey != wantKey { 83 | t.Errorf("wrong TLSPrivPath path, want (%v), got (%v)", wantKey, gotKey) 84 | } 85 | 86 | wantCert := test.expected.TLSCert 87 | if gotCert != wantCert { 88 | t.Errorf("wrong TLSCertPath path, want (%v), got (%v)", wantCert, gotCert) 89 | } 90 | 91 | }) 92 | } 93 | } 94 | 95 | func TestGetHandler(t *testing.T) { 96 | dir, err := os.MkdirTemp("", "rest-server-test") 97 | if err != nil { 98 | t.Fatal(err) 99 | } 100 | defer func() { 101 | err := os.Remove(dir) 102 | if err != nil { 103 | t.Fatal(err) 104 | } 105 | }() 106 | 107 | getHandler := restserver.NewHandler 108 | 109 | // With NoAuth = false and no .htpasswd 110 | _, err = getHandler(&restserver.Server{Path: dir}) 111 | if err == nil { 112 | t.Errorf("NoAuth=false: expected error, got nil") 113 | } 114 | 115 | // With NoAuth = true and no .htpasswd 116 | _, err = getHandler(&restserver.Server{NoAuth: true, Path: dir}) 117 | if err != nil { 118 | t.Errorf("NoAuth=true: expected no error, got %v", err) 119 | } 120 | 121 | // With NoAuth = false, no .htpasswd and ProxyAuth = X-Remote-User 122 | _, err = getHandler(&restserver.Server{Path: dir, ProxyAuthUsername: "X-Remote-User"}) 123 | if err != nil { 124 | t.Errorf("NoAuth=false, ProxyAuthUsername = X-Remote-User: expected no error, got %v", err) 125 | } 126 | 127 | // With NoAuth = false and custom .htpasswd 128 | htpFile, err := os.CreateTemp(dir, "custom") 129 | if err != nil { 130 | t.Fatal(err) 131 | } 132 | defer func() { 133 | err := os.Remove(htpFile.Name()) 134 | if err != nil { 135 | t.Fatal(err) 136 | } 137 | }() 138 | _, err = getHandler(&restserver.Server{HtpasswdPath: htpFile.Name()}) 139 | if err != nil { 140 | t.Errorf("NoAuth=false with custom htpasswd: expected no error, got %v", err) 141 | } 142 | 143 | // Create .htpasswd 144 | htpasswd := filepath.Join(dir, ".htpasswd") 145 | err = os.WriteFile(htpasswd, []byte(""), 0644) 146 | if err != nil { 147 | t.Fatal(err) 148 | } 149 | defer func() { 150 | err := os.Remove(htpasswd) 151 | if err != nil { 152 | t.Fatal(err) 153 | } 154 | }() 155 | 156 | // With NoAuth = false and with .htpasswd 157 | _, err = getHandler(&restserver.Server{Path: dir}) 158 | if err != nil { 159 | t.Errorf("NoAuth=false with .htpasswd: expected no error, got %v", err) 160 | } 161 | } 162 | 163 | // helper method to test the app. Starts app with passed arguments, 164 | // then will call the callback function which can make requests against 165 | // the application. If the callback function fails due to errors returned 166 | // by http.Do() (i.e. *url.Error), then it will be retried until successful, 167 | // or the passed timeout passes. 168 | func testServerWithArgs(args []string, timeout time.Duration, cb func(context.Context, *restServerApp) error) error { 169 | // create the app with passed args 170 | app := newRestServerApp() 171 | app.CmdRoot.SetArgs(args) 172 | 173 | // create context that will timeout 174 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 175 | defer cancel() 176 | 177 | // wait group for our client and server tasks 178 | jobs := &sync.WaitGroup{} 179 | jobs.Add(2) 180 | 181 | // run the server, saving the error 182 | var serverErr error 183 | go func() { 184 | defer jobs.Done() 185 | defer cancel() // if the server is stopped, no point keep the client alive 186 | serverErr = app.CmdRoot.ExecuteContext(ctx) 187 | }() 188 | 189 | // run the client, saving the error 190 | var clientErr error 191 | go func() { 192 | defer jobs.Done() 193 | defer cancel() // once the client is done, stop the server 194 | 195 | var urlError *url.Error 196 | 197 | // execute in loop, as we will retry for network errors 198 | // (such as the server hasn't started yet) 199 | for { 200 | clientErr = cb(ctx, app) 201 | switch { 202 | case clientErr == nil: 203 | return // success, we're done 204 | case errors.As(clientErr, &urlError): 205 | // if a network error (url.Error), then wait and retry 206 | // as server may not be ready yet 207 | select { 208 | case <-time.After(time.Millisecond * 100): 209 | continue 210 | case <-ctx.Done(): // unless we run out of time first 211 | clientErr = context.Canceled 212 | return 213 | } 214 | default: 215 | return // other error type, we're done 216 | } 217 | } 218 | }() 219 | 220 | // wait for both to complete 221 | jobs.Wait() 222 | 223 | // report back if either failed 224 | if clientErr != nil || serverErr != nil { 225 | return fmt.Errorf("client or server error, client: %v, server: %v", clientErr, serverErr) 226 | } 227 | 228 | return nil 229 | } 230 | 231 | func TestHttpListen(t *testing.T) { 232 | td := t.TempDir() 233 | 234 | // create some content and parent dirs 235 | if err := os.MkdirAll(filepath.Join(td, "data", "repo1"), 0700); err != nil { 236 | t.Fatal(err) 237 | } 238 | if err := os.WriteFile(filepath.Join(td, "data", "repo1", "config"), []byte("foo"), 0700); err != nil { 239 | t.Fatal(err) 240 | } 241 | 242 | for _, args := range [][]string{ 243 | {"--no-auth", "--path", filepath.Join(td, "data"), "--listen", "127.0.0.1:0"}, // test emphemeral port 244 | {"--no-auth", "--path", filepath.Join(td, "data"), "--listen", "127.0.0.1:9000"}, // test "normal" port 245 | {"--no-auth", "--path", filepath.Join(td, "data"), "--listen", "127.0.0.1:9000"}, // test that server was shutdown cleanly and that we can re-use that port 246 | } { 247 | err := testServerWithArgs(args, time.Second*10, func(ctx context.Context, app *restServerApp) error { 248 | for _, test := range []struct { 249 | Path string 250 | StatusCode int 251 | }{ 252 | {"/repo1/", http.StatusMethodNotAllowed}, 253 | {"/repo1/config", http.StatusOK}, 254 | {"/repo2/config", http.StatusNotFound}, 255 | } { 256 | listenAddr := app.ListenerAddress() 257 | if listenAddr == nil { 258 | return &url.Error{} // return this type of err, as we know this will retry 259 | } 260 | port := strings.Split(listenAddr.String(), ":")[1] 261 | 262 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:%s%s", port, test.Path), nil) 263 | if err != nil { 264 | return err 265 | } 266 | resp, err := http.DefaultClient.Do(req) 267 | if err != nil { 268 | return err 269 | } 270 | err = resp.Body.Close() 271 | if err != nil { 272 | return err 273 | } 274 | if resp.StatusCode != test.StatusCode { 275 | return fmt.Errorf("expected %d from server, instead got %d (path %s)", test.StatusCode, resp.StatusCode, test.Path) 276 | } 277 | } 278 | return nil 279 | }) 280 | if err != nil { 281 | t.Fatal(err) 282 | } 283 | } 284 | } 285 | -------------------------------------------------------------------------------- /docker/create_user: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -z "$1" ]; then 4 | echo "create_user [username]" 5 | echo "or" 6 | echo "create_user [username] [password]" 7 | exit 1 8 | fi 9 | 10 | if [ -z "$2" ]; then 11 | # password from prompt 12 | htpasswd -B "$PASSWORD_FILE" "$1" 13 | else 14 | # read password from command line 15 | htpasswd -B -b "$PASSWORD_FILE" "$1" "$2" 16 | fi 17 | -------------------------------------------------------------------------------- /docker/delete_user: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -z "$1" ]; then 4 | echo "delete_user [username]" 5 | exit 1 6 | fi 7 | 8 | htpasswd -D "$PASSWORD_FILE" "$1" 9 | -------------------------------------------------------------------------------- /docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | if [ -n "$DISABLE_AUTHENTICATION" ]; then 6 | OPTIONS="--no-auth $OPTIONS" 7 | else 8 | if [ ! -f "$PASSWORD_FILE" ]; then 9 | ( umask 027 && touch "$PASSWORD_FILE" ) 10 | fi 11 | 12 | if [ ! -s "$PASSWORD_FILE" ]; then 13 | echo 14 | echo "**WARNING** No user exists, please 'docker exec -it \$CONTAINER_ID create_user'" 15 | echo 16 | fi 17 | fi 18 | 19 | exec rest-server --path "$DATA_DIRECTORY" --htpasswd-file "$PASSWORD_FILE" $OPTIONS 20 | -------------------------------------------------------------------------------- /examples/bsd/freebsd: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . /etc/rc.subr 4 | 5 | name=restserver 6 | rcvar=restserver_enable 7 | 8 | start_cmd="${name}_start" 9 | stop_cmd=":" 10 | 11 | load_rc_config $name 12 | : ${restserver_enable:=no} 13 | : ${restserver_msg="Nothing started."} 14 | 15 | datadir="/backups" 16 | 17 | restserver_start() 18 | { 19 | rest-server --path $datadir \ 20 | --private-repos \ 21 | --tls \ 22 | --tls-cert "/etc/ssl/rest-server.crt" \ 23 | --tls-key "/etc/ssl/private/rest-server.key" & 24 | } 25 | 26 | run_rc_command "$1" 27 | -------------------------------------------------------------------------------- /examples/bsd/openbsd: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | # 3 | # $OpenBSD: $ 4 | 5 | daemon="/usr/local/bin/rest-server" 6 | daemon_flags="--path /var/restic" 7 | daemon_user="_restic" 8 | 9 | . /etc/rc.d/rc.subr 10 | 11 | rc_bg=YES 12 | rc_reload=NO 13 | 14 | rc_cmd $1 15 | -------------------------------------------------------------------------------- /examples/compose-with-grafana/README.md: -------------------------------------------------------------------------------- 1 | # Rest Server Grafana Dashboard 2 | 3 | This is a demo [Docker Compose](https://docs.docker.com/compose/) setup for [Rest Server](https://github.com/restic/rest-server) with [Prometheus](https://prometheus.io/) and [Grafana](https://grafana.com/). 4 | 5 | ![Grafana dashboard screenshot](screenshot.png) 6 | 7 | ## Quickstart 8 | 9 | Build `rest-server` in Docker: 10 | 11 | cd ../.. 12 | make docker_build 13 | cd - 14 | 15 | Bring up the Docker Compose stack: 16 | 17 | docker-compose build 18 | docker-compose up -d 19 | 20 | Check if everything is up and running: 21 | 22 | docker-compose ps 23 | 24 | Grafana will be running on [http://localhost:8030/](http://localhost:8030/) with username "admin" and password "admin". The first time you access it you will be asked to setup a data source. Configure it like this (make sure you name it "prometheus", as this is hardcoded in the example dashboard): 25 | 26 | ![Add data source](datasource.png) 27 | 28 | The Rest Server dashboard can be accessed on [http://localhost:8030/dashboard/file/rest-server.json](http://localhost:8030/dashboard/file/rest-server.json). 29 | 30 | Prometheus can be accessed on [http://localhost:8020/](http://localhost:8020/). 31 | 32 | If you do a backup like this, some graphs should show up: 33 | 34 | restic -r rest:http://127.0.0.1:8010/demo1 -p ./demo-passwd init 35 | restic -r rest:http://127.0.0.1:8010/demo1 -p ./demo-passwd backup . 36 | -------------------------------------------------------------------------------- /examples/compose-with-grafana/dashboards/rest-server.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_PROMETHEUS-INFRA", 5 | "label": "prometheus-infra", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "prometheus", 9 | "pluginName": "Prometheus" 10 | } 11 | ], 12 | "__requires": [ 13 | { 14 | "type": "grafana", 15 | "id": "grafana", 16 | "name": "Grafana", 17 | "version": "4.6.0" 18 | }, 19 | { 20 | "type": "panel", 21 | "id": "graph", 22 | "name": "Graph", 23 | "version": "" 24 | }, 25 | { 26 | "type": "datasource", 27 | "id": "prometheus", 28 | "name": "Prometheus", 29 | "version": "1.0.0" 30 | } 31 | ], 32 | "annotations": { 33 | "list": [ 34 | { 35 | "builtIn": 1, 36 | "datasource": "-- Grafana --", 37 | "enable": true, 38 | "hide": true, 39 | "iconColor": "rgba(0, 211, 255, 1)", 40 | "name": "Annotations & Alerts", 41 | "type": "dashboard" 42 | } 43 | ] 44 | }, 45 | "editable": true, 46 | "gnetId": null, 47 | "graphTooltip": 0, 48 | "hideControls": false, 49 | "id": null, 50 | "links": [], 51 | "refresh": "10s", 52 | "rows": [ 53 | { 54 | "collapse": false, 55 | "height": 244, 56 | "panels": [ 57 | { 58 | "aliasColors": {}, 59 | "bars": false, 60 | "dashLength": 10, 61 | "dashes": false, 62 | "datasource": "${DS_PROMETHEUS-INFRA}", 63 | "fill": 1, 64 | "id": 1, 65 | "legend": { 66 | "avg": false, 67 | "current": false, 68 | "max": false, 69 | "min": false, 70 | "show": true, 71 | "total": false, 72 | "values": false 73 | }, 74 | "lines": true, 75 | "linewidth": 1, 76 | "links": [], 77 | "nullPointMode": "null", 78 | "percentage": false, 79 | "pointradius": 5, 80 | "points": false, 81 | "renderer": "flot", 82 | "seriesOverrides": [], 83 | "spaceLength": 10, 84 | "span": 6, 85 | "stack": false, 86 | "steppedLine": false, 87 | "targets": [ 88 | { 89 | "expr": "sum(rate(rest_server_blob_write_bytes_total{instance=\"$instance\"}[15s])) by ($group)", 90 | "format": "time_series", 91 | "interval": "", 92 | "intervalFactor": 1, 93 | "legendFormat": "{{$group}}", 94 | "refId": "A" 95 | } 96 | ], 97 | "thresholds": [], 98 | "timeFrom": null, 99 | "timeShift": null, 100 | "title": "Blob Write Throughput by $group", 101 | "tooltip": { 102 | "shared": true, 103 | "sort": 0, 104 | "value_type": "individual" 105 | }, 106 | "type": "graph", 107 | "xaxis": { 108 | "buckets": null, 109 | "mode": "time", 110 | "name": null, 111 | "show": true, 112 | "values": [] 113 | }, 114 | "yaxes": [ 115 | { 116 | "format": "Bps", 117 | "label": null, 118 | "logBase": 1, 119 | "max": null, 120 | "min": "0", 121 | "show": true 122 | }, 123 | { 124 | "format": "short", 125 | "label": null, 126 | "logBase": 1, 127 | "max": null, 128 | "min": null, 129 | "show": true 130 | } 131 | ] 132 | }, 133 | { 134 | "aliasColors": {}, 135 | "bars": false, 136 | "dashLength": 10, 137 | "dashes": false, 138 | "datasource": "${DS_PROMETHEUS-INFRA}", 139 | "fill": 1, 140 | "id": 4, 141 | "legend": { 142 | "avg": false, 143 | "current": false, 144 | "max": false, 145 | "min": false, 146 | "show": true, 147 | "total": false, 148 | "values": false 149 | }, 150 | "lines": true, 151 | "linewidth": 1, 152 | "links": [], 153 | "nullPointMode": "null", 154 | "percentage": false, 155 | "pointradius": 5, 156 | "points": false, 157 | "renderer": "flot", 158 | "seriesOverrides": [], 159 | "spaceLength": 10, 160 | "span": 6, 161 | "stack": false, 162 | "steppedLine": false, 163 | "targets": [ 164 | { 165 | "expr": "sum(rate(rest_server_blob_write_total{instance=\"$instance\"}[15s])) by ($group)", 166 | "format": "time_series", 167 | "interval": "", 168 | "intervalFactor": 1, 169 | "legendFormat": "{{$group}}", 170 | "refId": "A" 171 | } 172 | ], 173 | "thresholds": [], 174 | "timeFrom": null, 175 | "timeShift": null, 176 | "title": "Blob Write Operations by $group", 177 | "tooltip": { 178 | "shared": true, 179 | "sort": 0, 180 | "value_type": "individual" 181 | }, 182 | "type": "graph", 183 | "xaxis": { 184 | "buckets": null, 185 | "mode": "time", 186 | "name": null, 187 | "show": true, 188 | "values": [] 189 | }, 190 | "yaxes": [ 191 | { 192 | "format": "ops", 193 | "label": null, 194 | "logBase": 1, 195 | "max": null, 196 | "min": "0", 197 | "show": true 198 | }, 199 | { 200 | "format": "short", 201 | "label": null, 202 | "logBase": 1, 203 | "max": null, 204 | "min": null, 205 | "show": true 206 | } 207 | ] 208 | } 209 | ], 210 | "repeat": null, 211 | "repeatIteration": null, 212 | "repeatRowId": null, 213 | "showTitle": false, 214 | "title": "Dashboard Row", 215 | "titleSize": "h6" 216 | }, 217 | { 218 | "collapse": false, 219 | "height": 258, 220 | "panels": [ 221 | { 222 | "aliasColors": {}, 223 | "bars": false, 224 | "dashLength": 10, 225 | "dashes": false, 226 | "datasource": "${DS_PROMETHEUS-INFRA}", 227 | "fill": 1, 228 | "id": 2, 229 | "legend": { 230 | "avg": false, 231 | "current": false, 232 | "max": false, 233 | "min": false, 234 | "show": true, 235 | "total": false, 236 | "values": false 237 | }, 238 | "lines": true, 239 | "linewidth": 1, 240 | "links": [], 241 | "nullPointMode": "null", 242 | "percentage": false, 243 | "pointradius": 5, 244 | "points": false, 245 | "renderer": "flot", 246 | "seriesOverrides": [], 247 | "spaceLength": 10, 248 | "span": 6, 249 | "stack": false, 250 | "steppedLine": false, 251 | "targets": [ 252 | { 253 | "expr": "sum(rate(rest_server_blob_read_bytes_total{instance=\"$instance\"}[15s])) by ($group)", 254 | "format": "time_series", 255 | "interval": "", 256 | "intervalFactor": 1, 257 | "legendFormat": "{{$group}}", 258 | "refId": "A" 259 | } 260 | ], 261 | "thresholds": [], 262 | "timeFrom": null, 263 | "timeShift": null, 264 | "title": "Blob Read Throughput by $group", 265 | "tooltip": { 266 | "shared": true, 267 | "sort": 0, 268 | "value_type": "individual" 269 | }, 270 | "type": "graph", 271 | "xaxis": { 272 | "buckets": null, 273 | "mode": "time", 274 | "name": null, 275 | "show": true, 276 | "values": [] 277 | }, 278 | "yaxes": [ 279 | { 280 | "format": "Bps", 281 | "label": null, 282 | "logBase": 1, 283 | "max": null, 284 | "min": "0", 285 | "show": true 286 | }, 287 | { 288 | "format": "short", 289 | "label": null, 290 | "logBase": 1, 291 | "max": null, 292 | "min": null, 293 | "show": true 294 | } 295 | ] 296 | }, 297 | { 298 | "aliasColors": {}, 299 | "bars": false, 300 | "dashLength": 10, 301 | "dashes": false, 302 | "datasource": "${DS_PROMETHEUS-INFRA}", 303 | "fill": 1, 304 | "id": 5, 305 | "legend": { 306 | "avg": false, 307 | "current": false, 308 | "max": false, 309 | "min": false, 310 | "show": true, 311 | "total": false, 312 | "values": false 313 | }, 314 | "lines": true, 315 | "linewidth": 1, 316 | "links": [], 317 | "nullPointMode": "null", 318 | "percentage": false, 319 | "pointradius": 5, 320 | "points": false, 321 | "renderer": "flot", 322 | "seriesOverrides": [], 323 | "spaceLength": 10, 324 | "span": 6, 325 | "stack": false, 326 | "steppedLine": false, 327 | "targets": [ 328 | { 329 | "expr": "sum(rate(rest_server_blob_read_total{instance=\"$instance\"}[15s])) by ($group)", 330 | "format": "time_series", 331 | "interval": "", 332 | "intervalFactor": 1, 333 | "legendFormat": "{{$group}}", 334 | "refId": "A" 335 | } 336 | ], 337 | "thresholds": [], 338 | "timeFrom": null, 339 | "timeShift": null, 340 | "title": "Blob Read Operations by $group", 341 | "tooltip": { 342 | "shared": true, 343 | "sort": 0, 344 | "value_type": "individual" 345 | }, 346 | "type": "graph", 347 | "xaxis": { 348 | "buckets": null, 349 | "mode": "time", 350 | "name": null, 351 | "show": true, 352 | "values": [] 353 | }, 354 | "yaxes": [ 355 | { 356 | "format": "ops", 357 | "label": null, 358 | "logBase": 1, 359 | "max": null, 360 | "min": "0", 361 | "show": true 362 | }, 363 | { 364 | "format": "short", 365 | "label": null, 366 | "logBase": 1, 367 | "max": null, 368 | "min": null, 369 | "show": true 370 | } 371 | ] 372 | } 373 | ], 374 | "repeat": null, 375 | "repeatIteration": null, 376 | "repeatRowId": null, 377 | "showTitle": false, 378 | "title": "Dashboard Row", 379 | "titleSize": "h6" 380 | }, 381 | { 382 | "collapse": false, 383 | "height": 250, 384 | "panels": [ 385 | { 386 | "aliasColors": {}, 387 | "bars": false, 388 | "dashLength": 10, 389 | "dashes": false, 390 | "datasource": "${DS_PROMETHEUS-INFRA}", 391 | "fill": 1, 392 | "id": 3, 393 | "legend": { 394 | "avg": false, 395 | "current": false, 396 | "max": false, 397 | "min": false, 398 | "show": true, 399 | "total": false, 400 | "values": false 401 | }, 402 | "lines": true, 403 | "linewidth": 1, 404 | "links": [], 405 | "nullPointMode": "null", 406 | "percentage": false, 407 | "pointradius": 5, 408 | "points": false, 409 | "renderer": "flot", 410 | "seriesOverrides": [], 411 | "spaceLength": 10, 412 | "span": 6, 413 | "stack": false, 414 | "steppedLine": false, 415 | "targets": [ 416 | { 417 | "expr": "sum(rate(rest_server_blob_delete_bytes_total{instance=\"$instance\"}[15s])) by ($group)", 418 | "format": "time_series", 419 | "interval": "", 420 | "intervalFactor": 1, 421 | "legendFormat": "{{$group}}", 422 | "refId": "A" 423 | } 424 | ], 425 | "thresholds": [], 426 | "timeFrom": null, 427 | "timeShift": null, 428 | "title": "Blob Delete Throughput by $group", 429 | "tooltip": { 430 | "shared": true, 431 | "sort": 0, 432 | "value_type": "individual" 433 | }, 434 | "type": "graph", 435 | "xaxis": { 436 | "buckets": null, 437 | "mode": "time", 438 | "name": null, 439 | "show": true, 440 | "values": [] 441 | }, 442 | "yaxes": [ 443 | { 444 | "format": "Bps", 445 | "label": null, 446 | "logBase": 1, 447 | "max": null, 448 | "min": "0", 449 | "show": true 450 | }, 451 | { 452 | "format": "short", 453 | "label": null, 454 | "logBase": 1, 455 | "max": null, 456 | "min": null, 457 | "show": true 458 | } 459 | ] 460 | }, 461 | { 462 | "aliasColors": {}, 463 | "bars": false, 464 | "dashLength": 10, 465 | "dashes": false, 466 | "datasource": "${DS_PROMETHEUS-INFRA}", 467 | "fill": 1, 468 | "id": 6, 469 | "legend": { 470 | "avg": false, 471 | "current": false, 472 | "max": false, 473 | "min": false, 474 | "show": true, 475 | "total": false, 476 | "values": false 477 | }, 478 | "lines": true, 479 | "linewidth": 1, 480 | "links": [], 481 | "nullPointMode": "null", 482 | "percentage": false, 483 | "pointradius": 5, 484 | "points": false, 485 | "renderer": "flot", 486 | "seriesOverrides": [], 487 | "spaceLength": 10, 488 | "span": 6, 489 | "stack": false, 490 | "steppedLine": false, 491 | "targets": [ 492 | { 493 | "expr": "sum(rate(rest_server_blob_delete_total{instance=\"$instance\"}[15s])) by ($group)", 494 | "format": "time_series", 495 | "interval": "", 496 | "intervalFactor": 1, 497 | "legendFormat": "{{$group}}", 498 | "refId": "A" 499 | } 500 | ], 501 | "thresholds": [], 502 | "timeFrom": null, 503 | "timeShift": null, 504 | "title": "Blob Delete Operations by $group", 505 | "tooltip": { 506 | "shared": true, 507 | "sort": 0, 508 | "value_type": "individual" 509 | }, 510 | "type": "graph", 511 | "xaxis": { 512 | "buckets": null, 513 | "mode": "time", 514 | "name": null, 515 | "show": true, 516 | "values": [] 517 | }, 518 | "yaxes": [ 519 | { 520 | "format": "ops", 521 | "label": null, 522 | "logBase": 1, 523 | "max": null, 524 | "min": "0", 525 | "show": true 526 | }, 527 | { 528 | "format": "short", 529 | "label": null, 530 | "logBase": 1, 531 | "max": null, 532 | "min": null, 533 | "show": true 534 | } 535 | ] 536 | } 537 | ], 538 | "repeat": null, 539 | "repeatIteration": null, 540 | "repeatRowId": null, 541 | "showTitle": false, 542 | "title": "Dashboard Row", 543 | "titleSize": "h6" 544 | } 545 | ], 546 | "schemaVersion": 14, 547 | "style": "dark", 548 | "tags": [], 549 | "templating": { 550 | "list": [ 551 | { 552 | "allValue": null, 553 | "current": {}, 554 | "datasource": "${DS_PROMETHEUS-INFRA}", 555 | "hide": 0, 556 | "includeAll": false, 557 | "label": "Instance", 558 | "multi": false, 559 | "name": "instance", 560 | "options": [], 561 | "query": "label_values(process_start_time_seconds{job=\"rest_server\"}, instance)", 562 | "refresh": 2, 563 | "regex": "", 564 | "sort": 1, 565 | "tagValuesQuery": "", 566 | "tags": [], 567 | "tagsQuery": "", 568 | "type": "query", 569 | "useTags": false 570 | }, 571 | { 572 | "allValue": null, 573 | "current": { 574 | "tags": [], 575 | "text": "type", 576 | "value": "type" 577 | }, 578 | "hide": 0, 579 | "includeAll": false, 580 | "label": "Group By", 581 | "multi": false, 582 | "name": "group", 583 | "options": [ 584 | { 585 | "selected": true, 586 | "text": "type", 587 | "value": "type" 588 | }, 589 | { 590 | "selected": false, 591 | "text": "repo", 592 | "value": "repo" 593 | }, 594 | { 595 | "selected": false, 596 | "text": "user", 597 | "value": "user" 598 | } 599 | ], 600 | "query": "type,repo,user", 601 | "type": "custom" 602 | } 603 | ] 604 | }, 605 | "time": { 606 | "from": "now-5m", 607 | "to": "now" 608 | }, 609 | "timepicker": { 610 | "refresh_intervals": [ 611 | "5s", 612 | "10s", 613 | "30s", 614 | "1m", 615 | "5m", 616 | "15m", 617 | "30m", 618 | "1h", 619 | "2h", 620 | "1d" 621 | ], 622 | "time_options": [ 623 | "5m", 624 | "15m", 625 | "1h", 626 | "6h", 627 | "12h", 628 | "24h", 629 | "2d", 630 | "7d", 631 | "30d" 632 | ] 633 | }, 634 | "timezone": "", 635 | "title": "Restic Rest Server", 636 | "version": 8 637 | } 638 | -------------------------------------------------------------------------------- /examples/compose-with-grafana/datasource.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/restic/rest-server/2f31e10cebf335003f75fee5b4646fb73edcda45/examples/compose-with-grafana/datasource.png -------------------------------------------------------------------------------- /examples/compose-with-grafana/demo-passwd: -------------------------------------------------------------------------------- 1 | demo-passwd 2 | -------------------------------------------------------------------------------- /examples/compose-with-grafana/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Demo of rest-server with prometheus and grafana 2 | version: '2' 3 | 4 | services: 5 | restserver: 6 | # NOTE: You must run `make docker_build` in the repo root first 7 | # If you want to run this in production, you want auth and tls! 8 | build: 9 | context: ../.. 10 | dockerfile: Dockerfile 11 | volumes: 12 | - data:/data 13 | environment: 14 | DISABLE_AUTHENTICATION: 1 15 | OPTIONS: "--prometheus" 16 | ports: 17 | - "127.0.0.1:8010:8000" 18 | networks: 19 | - net 20 | 21 | prometheus: 22 | image: prom/prometheus 23 | ports: 24 | - "127.0.0.1:8020:9090" 25 | volumes: 26 | - prometheusdata:/prometheus 27 | - ./prometheus:/etc/prometheus:ro 28 | depends_on: 29 | - restserver 30 | networks: 31 | - net 32 | 33 | grafana: 34 | image: grafana/grafana 35 | volumes: 36 | - grafanadata:/var/lib/grafana 37 | - ./dashboards:/dashboards 38 | - ./grafana.ini:/etc/grafana/grafana.ini 39 | ports: 40 | - "127.0.0.1:8030:3000" 41 | environment: 42 | GF_USERS_DEFAULT_THEME: light 43 | # GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource 44 | depends_on: 45 | - prometheus 46 | networks: 47 | - net 48 | 49 | networks: 50 | net: 51 | 52 | volumes: 53 | data: 54 | driver: local 55 | prometheusdata: 56 | driver: local 57 | grafanadata: 58 | driver: local 59 | 60 | -------------------------------------------------------------------------------- /examples/compose-with-grafana/grafana.ini: -------------------------------------------------------------------------------- 1 | ##################### Grafana Configuration Example ##################### 2 | # 3 | # Everything has defaults so you only need to uncomment things you want to 4 | # change 5 | 6 | # possible values : production, development 7 | ; app_mode = production 8 | 9 | # instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty 10 | ; instance_name = ${HOSTNAME} 11 | 12 | #################################### Paths #################################### 13 | [paths] 14 | # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) 15 | # 16 | ;data = /var/lib/grafana 17 | # 18 | # Directory where grafana can store logs 19 | # 20 | ;logs = /var/log/grafana 21 | # 22 | # Directory where grafana will automatically scan and look for plugins 23 | # 24 | ;plugins = /var/lib/grafana/plugins 25 | 26 | # 27 | #################################### Server #################################### 28 | [server] 29 | # Protocol (http or https) 30 | ;protocol = http 31 | 32 | # The ip address to bind to, empty will bind to all interfaces 33 | ;http_addr = 34 | 35 | # The http port to use 36 | ;http_port = 3000 37 | 38 | # The public facing domain name used to access grafana from a browser 39 | ;domain = localhost 40 | 41 | # Redirect to correct domain if host header does not match domain 42 | # Prevents DNS rebinding attacks 43 | ;enforce_domain = false 44 | 45 | # The full public facing url 46 | ;root_url = %(protocol)s://%(domain)s:%(http_port)s/ 47 | 48 | # Log web requests 49 | ;router_logging = false 50 | 51 | # the path relative working path 52 | ;static_root_path = public 53 | 54 | # enable gzip 55 | ;enable_gzip = false 56 | 57 | # https certs & key file 58 | ;cert_file = 59 | ;cert_key = 60 | 61 | #################################### Database #################################### 62 | [database] 63 | # Either "mysql", "postgres" or "sqlite3", it's your choice 64 | ;type = sqlite3 65 | ;host = 127.0.0.1:3306 66 | ;name = grafana 67 | ;user = root 68 | ;password = 69 | 70 | # For "postgres" only, either "disable", "require" or "verify-full" 71 | ;ssl_mode = disable 72 | 73 | # For "sqlite3" only, path relative to data_path setting 74 | ;path = grafana.db 75 | 76 | #################################### Session #################################### 77 | [session] 78 | # Either "memory", "file", "redis", "mysql", "postgres", default is "file" 79 | ;provider = file 80 | 81 | # Provider config options 82 | # memory: not have any config yet 83 | # file: session dir path, is relative to grafana data_path 84 | # redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana` 85 | # mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` 86 | # postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable 87 | ;provider_config = sessions 88 | 89 | # Session cookie name 90 | ;cookie_name = grafana_sess 91 | 92 | # If you use session in https only, default is false 93 | ;cookie_secure = false 94 | 95 | # Session life time, default is 86400 96 | ;session_life_time = 86400 97 | 98 | #################################### Analytics #################################### 99 | [analytics] 100 | # Server reporting, sends usage counters to stats.grafana.org every 24 hours. 101 | # No ip addresses are being tracked, only simple counters to track 102 | # running instances, dashboard and error counts. It is very helpful to us. 103 | # Change this option to false to disable reporting. 104 | ;reporting_enabled = true 105 | 106 | # Set to false to disable all checks to https://grafana.net 107 | # for new vesions (grafana itself and plugins), check is used 108 | # in some UI views to notify that grafana or plugin update exists 109 | # This option does not cause any auto updates, nor send any information 110 | # only a GET request to http://grafana.net to get latest versions 111 | check_for_updates = true 112 | 113 | # Google Analytics universal tracking code, only enabled if you specify an id here 114 | ;google_analytics_ua_id = 115 | 116 | #################################### Security #################################### 117 | [security] 118 | # default admin user, created on startup 119 | ;admin_user = admin 120 | 121 | # default admin password, can be changed before first start of grafana, or in profile settings 122 | ;admin_password = admin 123 | 124 | # used for signing 125 | ;secret_key = SW2YcwTIb9zpOOhoPsMm 126 | 127 | # Auto-login remember days 128 | ;login_remember_days = 7 129 | ;cookie_username = grafana_user 130 | ;cookie_remember_name = grafana_remember 131 | 132 | # disable gravatar profile images 133 | ;disable_gravatar = false 134 | 135 | # data source proxy whitelist (ip_or_domain:port separated by spaces) 136 | ;data_source_proxy_whitelist = 137 | 138 | [snapshots] 139 | # snapshot sharing options 140 | ;external_enabled = true 141 | ;external_snapshot_url = https://snapshots-origin.raintank.io 142 | ;external_snapshot_name = Publish to snapshot.raintank.io 143 | 144 | #################################### Users #################################### 145 | [users] 146 | # disable user signup / registration 147 | ;allow_sign_up = true 148 | 149 | # Allow non admin users to create organizations 150 | ;allow_org_create = true 151 | 152 | # Set to true to automatically assign new users to the default organization (id 1) 153 | ;auto_assign_org = true 154 | 155 | # Default role new users will be automatically assigned (if disabled above is set to true) 156 | ;auto_assign_org_role = Viewer 157 | 158 | # Background text for the user field on the login page 159 | ;login_hint = email or username 160 | 161 | # Default UI theme ("dark" or "light") 162 | default_theme = dark 163 | 164 | #################################### Anonymous Auth ########################## 165 | [auth.anonymous] 166 | # enable anonymous access 167 | ;enabled = false 168 | 169 | # specify organization name that should be used for unauthenticated users 170 | ;org_name = Main Org. 171 | 172 | # specify role for unauthenticated users 173 | ;org_role = Viewer 174 | 175 | #################################### Github Auth ########################## 176 | [auth.github] 177 | ;enabled = false 178 | ;allow_sign_up = false 179 | ;client_id = some_id 180 | ;client_secret = some_secret 181 | ;scopes = user:email,read:org 182 | ;auth_url = https://github.com/login/oauth/authorize 183 | ;token_url = https://github.com/login/oauth/access_token 184 | ;api_url = https://api.github.com/user 185 | ;team_ids = 186 | ;allowed_organizations = 187 | 188 | #################################### Google Auth ########################## 189 | [auth.google] 190 | ;enabled = false 191 | ;allow_sign_up = false 192 | ;client_id = some_client_id 193 | ;client_secret = some_client_secret 194 | ;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email 195 | ;auth_url = https://accounts.google.com/o/oauth2/auth 196 | ;token_url = https://accounts.google.com/o/oauth2/token 197 | ;api_url = https://www.googleapis.com/oauth2/v1/userinfo 198 | ;allowed_domains = 199 | 200 | #################################### Auth Proxy ########################## 201 | [auth.proxy] 202 | ;enabled = false 203 | ;header_name = X-WEBAUTH-USER 204 | ;header_property = username 205 | ;auto_sign_up = true 206 | 207 | #################################### Basic Auth ########################## 208 | [auth.basic] 209 | ;enabled = true 210 | 211 | #################################### Auth LDAP ########################## 212 | [auth.ldap] 213 | ;enabled = false 214 | ;config_file = /etc/grafana/ldap.toml 215 | 216 | #################################### SMTP / Emailing ########################## 217 | [smtp] 218 | ;enabled = false 219 | ;host = localhost:25 220 | ;user = 221 | ;password = 222 | ;cert_file = 223 | ;key_file = 224 | ;skip_verify = false 225 | ;from_address = admin@grafana.localhost 226 | 227 | [emails] 228 | ;welcome_email_on_sign_up = false 229 | 230 | #################################### Logging ########################## 231 | [log] 232 | # Either "console", "file", "syslog". Default is console and file 233 | # Use space to separate multiple modes, e.g. "console file" 234 | ;mode = console, file 235 | 236 | # Either "trace", "debug", "info", "warn", "error", "critical", default is "info" 237 | ;level = info 238 | 239 | # For "console" mode only 240 | [log.console] 241 | ;level = 242 | 243 | # log line format, valid options are text, console and json 244 | ;format = console 245 | 246 | # For "file" mode only 247 | [log.file] 248 | ;level = 249 | 250 | # log line format, valid options are text, console and json 251 | ;format = text 252 | 253 | # This enables automated log rotate(switch of following options), default is true 254 | ;log_rotate = true 255 | 256 | # Max line number of single file, default is 1000000 257 | ;max_lines = 1000000 258 | 259 | # Max size shift of single file, default is 28 means 1 << 28, 256MB 260 | ;max_size_shift = 28 261 | 262 | # Segment log daily, default is true 263 | ;daily_rotate = true 264 | 265 | # Expired days of log file(delete after max days), default is 7 266 | ;max_days = 7 267 | 268 | [log.syslog] 269 | ;level = 270 | 271 | # log line format, valid options are text, console and json 272 | ;format = text 273 | 274 | # Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used. 275 | ;network = 276 | ;address = 277 | 278 | # Syslog facility. user, daemon and local0 through local7 are valid. 279 | ;facility = 280 | 281 | # Syslog tag. By default, the process' argv[0] is used. 282 | ;tag = 283 | 284 | 285 | #################################### AMQP Event Publisher ########################## 286 | [event_publisher] 287 | ;enabled = false 288 | ;rabbitmq_url = amqp://localhost/ 289 | ;exchange = grafana_events 290 | 291 | ;#################################### Dashboard JSON files ########################## 292 | [dashboards.json] 293 | enabled = true 294 | path = /dashboards 295 | 296 | #################################### Internal Grafana Metrics ########################## 297 | # Metrics available at HTTP API Url /api/metrics 298 | [metrics] 299 | # Disable / Enable internal metrics 300 | ;enabled = true 301 | 302 | # Publish interval 303 | ;interval_seconds = 10 304 | 305 | # Send internal metrics to Graphite 306 | ; [metrics.graphite] 307 | ; address = localhost:2003 308 | ; prefix = prod.grafana.%(instance_name)s. 309 | 310 | #################################### Internal Grafana Metrics ########################## 311 | # Url used to to import dashboards directly from Grafana.net 312 | [grafana_net] 313 | url = https://grafana.net 314 | -------------------------------------------------------------------------------- /examples/compose-with-grafana/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s # By default, scrape targets every 15 seconds. 3 | 4 | # Attach these labels to any time series or alerts when communicating with 5 | # external systems (federation, remote storage, Alertmanager). 6 | external_labels: 7 | monitor: 'restic-rest-server-demo' 8 | 9 | scrape_configs: 10 | - job_name: 'prometheus' # monitor self 11 | scrape_interval: 5s 12 | static_configs: 13 | - targets: ['localhost:9090'] 14 | 15 | - job_name: 'rest_server' 16 | scrape_interval: 5s 17 | # Uncomment these if you use auth and/or https 18 | #basic_auth: 19 | # username: test 20 | # password: test 21 | #scheme: https 22 | static_configs: 23 | - targets: ['restserver:8000'] 24 | -------------------------------------------------------------------------------- /examples/compose-with-grafana/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/restic/rest-server/2f31e10cebf335003f75fee5b4646fb73edcda45/examples/compose-with-grafana/screenshot.png -------------------------------------------------------------------------------- /examples/systemd/rest-server.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Rest Server 3 | After=syslog.target 4 | After=network.target 5 | Requires=rest-server.socket 6 | After=rest-server.socket 7 | 8 | [Service] 9 | Type=simple 10 | # You may prefer to use a different user or group on your system. 11 | User=www-data 12 | Group=www-data 13 | ExecStart=/usr/local/bin/rest-server --path /path/to/backups 14 | Restart=always 15 | RestartSec=5 16 | 17 | # The following options are available (in systemd v247) to restrict the 18 | # actions of the rest-server. 19 | 20 | # As a whole, the purpose of these are to provide an additional layer of 21 | # security by mitigating any unknown security vulnerabilities which may exist 22 | # in rest-server or in the libraries, tools and operating system components 23 | # which it relies upon. 24 | 25 | # IMPORTANT! 26 | # The following line must be customised to your individual requirements. 27 | ReadWritePaths=/path/to/backups 28 | 29 | # Files in the data repository are only user accessible by default. Default to 30 | # `UMask=077` for consistency. To make created files group-readable, set to 31 | # `UMask=007` and pass `--group-accessible-repos` to rest-server via `ExecStart`. 32 | UMask=077 33 | 34 | # If your system doesn't support all of the features below (e.g. because of 35 | # the use of an older version of systemd), you may wish to comment-out 36 | # some of the lines below as appropriate. 37 | CapabilityBoundingSet= 38 | LockPersonality=true 39 | MemoryDenyWriteExecute=true 40 | NoNewPrivileges=yes 41 | 42 | # As the listen socket is created by systemd via the rest-server.socket unit, it is 43 | # no longer necessary for rest-server to have access to the host network namespace. 44 | PrivateNetwork=yes 45 | 46 | PrivateTmp=yes 47 | PrivateDevices=true 48 | PrivateUsers=true 49 | ProtectSystem=strict 50 | ProtectHome=yes 51 | ProtectClock=true 52 | ProtectControlGroups=true 53 | ProtectKernelLogs=true 54 | ProtectKernelModules=true 55 | ProtectKernelTunables=true 56 | ProtectProc=invisible 57 | ProtectHostname=true 58 | RemoveIPC=true 59 | RestrictNamespaces=true 60 | RestrictAddressFamilies=none 61 | RestrictSUIDSGID=true 62 | RestrictRealtime=true 63 | # if your service crashes with "code=killed, status=31/SYS", you probably tried to run linux_i386 (32bit) binary on a amd64 host 64 | SystemCallArchitectures=native 65 | SystemCallFilter=@system-service 66 | 67 | # Additionally, you may wish to use some of the systemd options documented in 68 | # systemd.resource-control(5) to limit the CPU, memory, file-system I/O and 69 | # network I/O that the rest-server is permitted to consume according to the 70 | # individual requirements of your installation. 71 | #CPUQuota=25% 72 | #MemoryHigh=bytes 73 | #MemoryMax=bytes 74 | #MemorySwapMax=bytes 75 | #TasksMax=N 76 | #IOReadBandwidthMax=device bytes 77 | #IOWriteBandwidthMax=device bytes 78 | #IOReadIOPSMax=device IOPS, IOWriteIOPSMax=device IOPS 79 | #IPAccounting=true 80 | #IPAddressAllow= 81 | 82 | [Install] 83 | WantedBy=multi-user.target 84 | -------------------------------------------------------------------------------- /examples/systemd/rest-server.socket: -------------------------------------------------------------------------------- 1 | [Socket] 2 | ListenStream = 8000 3 | 4 | [Install] 5 | WantedBy = sockets.target 6 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/restic/rest-server 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | github.com/coreos/go-systemd/v22 v22.5.0 7 | github.com/gorilla/handlers v1.5.2 8 | github.com/minio/sha256-simd v1.0.1 9 | github.com/miolini/datacounter v1.0.3 10 | github.com/prometheus/client_golang v1.22.0 11 | github.com/spf13/cobra v1.9.1 12 | golang.org/x/crypto v0.38.0 13 | ) 14 | 15 | require ( 16 | github.com/beorn7/perks v1.0.1 // indirect 17 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 18 | github.com/felixge/httpsnoop v1.0.4 // indirect 19 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 20 | github.com/klauspost/cpuid/v2 v2.2.9 // indirect 21 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 22 | github.com/prometheus/client_model v0.6.1 // indirect 23 | github.com/prometheus/common v0.62.0 // indirect 24 | github.com/prometheus/procfs v0.15.1 // indirect 25 | github.com/spf13/pflag v1.0.6 // indirect 26 | golang.org/x/sys v0.33.0 // indirect 27 | google.golang.org/protobuf v1.36.5 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 2 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 3 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 4 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 5 | github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= 6 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 7 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 11 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 12 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 13 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 14 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 15 | github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= 16 | github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= 17 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 18 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 19 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 20 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 21 | github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= 22 | github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= 23 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 24 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 25 | github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= 26 | github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= 27 | github.com/miolini/datacounter v1.0.3 h1:tanOZPVblGXQl7/bSZWoEM8l4KK83q24qwQLMrO/HOA= 28 | github.com/miolini/datacounter v1.0.3/go.mod h1:C45dc2hBumHjDpEU64IqPwR6TDyPVpzOqqRTN7zmBUA= 29 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 30 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 31 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 32 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 33 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 34 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 35 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 36 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 37 | github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= 38 | github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= 39 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 40 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 41 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 42 | github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= 43 | github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= 44 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 45 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 46 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 47 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 48 | golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= 49 | golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= 50 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 51 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 52 | google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= 53 | google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 54 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 55 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 56 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 57 | -------------------------------------------------------------------------------- /handlers.go: -------------------------------------------------------------------------------- 1 | package restserver 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "net/http" 7 | "path" 8 | "path/filepath" 9 | "strings" 10 | "sync" 11 | 12 | "github.com/restic/rest-server/quota" 13 | "github.com/restic/rest-server/repo" 14 | ) 15 | 16 | // Server encapsulates the rest-server's settings and repo management logic 17 | type Server struct { 18 | Path string 19 | HtpasswdPath string 20 | Listen string 21 | Log string 22 | CPUProfile string 23 | TLSKey string 24 | TLSCert string 25 | TLSMinVer string 26 | TLS bool 27 | NoAuth bool 28 | ProxyAuthUsername string 29 | AppendOnly bool 30 | PrivateRepos bool 31 | Prometheus bool 32 | PrometheusNoAuth bool 33 | Debug bool 34 | MaxRepoSize int64 35 | PanicOnError bool 36 | NoVerifyUpload bool 37 | GroupAccessibleRepos bool 38 | 39 | htpasswdFile *HtpasswdFile 40 | quotaManager *quota.Manager 41 | fsyncWarning sync.Once 42 | } 43 | 44 | // MaxFolderDepth is the maxDepth param passed to splitURLPath. 45 | // A max depth of 2 mean that we accept folders like: '/', '/foo' and '/foo/bar' 46 | // TODO: Move to a Server option 47 | const MaxFolderDepth = 2 48 | 49 | // httpDefaultError write a HTTP error with the default description 50 | func httpDefaultError(w http.ResponseWriter, code int) { 51 | http.Error(w, http.StatusText(code), code) 52 | } 53 | 54 | // ServeHTTP makes this server an http.Handler. It handlers the administrative 55 | // part of the request (figuring out the filesystem location, performing 56 | // authentication, etc) and then passes it on to repo.Handler for actual 57 | // REST API processing. 58 | func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { 59 | // First of all, check auth (will always pass if NoAuth is set) 60 | username, ok := s.checkAuth(r) 61 | if !ok { 62 | httpDefaultError(w, http.StatusUnauthorized) 63 | return 64 | } 65 | 66 | // Perform the path parsing to determine the repo folder and remainder for the 67 | // repo handler. 68 | folderPath, remainder := splitURLPath(r.URL.Path, MaxFolderDepth) 69 | if !folderPathValid(folderPath) { 70 | log.Printf("Invalid request path: %s", r.URL.Path) 71 | httpDefaultError(w, http.StatusNotFound) 72 | return 73 | } 74 | 75 | // Check if the current user is allowed to access this path 76 | if !s.NoAuth && s.PrivateRepos { 77 | if len(folderPath) == 0 || folderPath[0] != username { 78 | httpDefaultError(w, http.StatusUnauthorized) 79 | return 80 | } 81 | } 82 | 83 | // Determine filesystem path for this repo 84 | fsPath, err := join(s.Path, folderPath...) 85 | if err != nil { 86 | // We did not expect an error at this stage, because we just checked the path 87 | log.Printf("Unexpected join error for path %q", r.URL.Path) 88 | httpDefaultError(w, http.StatusNotFound) 89 | return 90 | } 91 | 92 | // Pass the request to the repo.Handler 93 | opt := repo.Options{ 94 | AppendOnly: s.AppendOnly, 95 | Debug: s.Debug, 96 | QuotaManager: s.quotaManager, // may be nil 97 | PanicOnError: s.PanicOnError, 98 | NoVerifyUpload: s.NoVerifyUpload, 99 | FsyncWarning: &s.fsyncWarning, 100 | GroupAccessible: s.GroupAccessibleRepos, 101 | } 102 | if s.Prometheus { 103 | opt.BlobMetricFunc = makeBlobMetricFunc(username, folderPath) 104 | } 105 | repoHandler, err := repo.New(fsPath, opt) 106 | if err != nil { 107 | log.Printf("repo.New error: %v", err) 108 | httpDefaultError(w, http.StatusInternalServerError) 109 | return 110 | } 111 | r.URL.Path = remainder // strip folderPath for next handler 112 | repoHandler.ServeHTTP(w, r) 113 | } 114 | 115 | func valid(name string) bool { 116 | // taken from net/http.Dir 117 | if strings.Contains(name, "\x00") { 118 | return false 119 | } 120 | 121 | if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) { 122 | return false 123 | } 124 | 125 | return true 126 | } 127 | 128 | func isValidType(name string) bool { 129 | for _, tpe := range repo.ObjectTypes { 130 | if name == tpe { 131 | return true 132 | } 133 | } 134 | for _, tpe := range repo.FileTypes { 135 | if name == tpe { 136 | return true 137 | } 138 | } 139 | return false 140 | } 141 | 142 | // join takes a number of path names, sanitizes them, and returns them joined 143 | // with base for the current operating system to use (dirs separated by 144 | // filepath.Separator). The returned path is always either equal to base or a 145 | // subdir of base. 146 | func join(base string, names ...string) (string, error) { 147 | clean := make([]string, 0, len(names)+1) 148 | clean = append(clean, base) 149 | 150 | // taken from net/http.Dir 151 | for _, name := range names { 152 | if !valid(name) { 153 | return "", errors.New("invalid character in path") 154 | } 155 | 156 | clean = append(clean, filepath.FromSlash(path.Clean("/"+name))) 157 | } 158 | 159 | return filepath.Join(clean...), nil 160 | } 161 | 162 | // splitURLPath splits the URL path into a folderPath of the subrepo, and 163 | // a remainder that can be passed to repo.Handler. 164 | // Example: /foo/bar/locks/0123... will be split into: 165 | // 166 | // ["foo", "bar"] and "/locks/0123..." 167 | func splitURLPath(urlPath string, maxDepth int) (folderPath []string, remainder string) { 168 | if !strings.HasPrefix(urlPath, "/") { 169 | // Really should start with "/" 170 | return nil, urlPath 171 | } 172 | p := strings.SplitN(urlPath, "/", maxDepth+2) 173 | // Skip the empty first one and the remainder in the last one 174 | for _, name := range p[1 : len(p)-1] { 175 | if isValidType(name) { 176 | // We found a part that is a special repo file or dir 177 | break 178 | } 179 | folderPath = append(folderPath, name) 180 | } 181 | // If the folder path is empty, the whole path is the remainder (do not strip '/') 182 | if len(folderPath) == 0 { 183 | return nil, urlPath 184 | } 185 | // Check that the urlPath starts with the reconstructed path, which should 186 | // always be the case. 187 | fullFolderPath := "/" + strings.Join(folderPath, "/") 188 | if !strings.HasPrefix(urlPath, fullFolderPath) { 189 | return nil, urlPath 190 | } 191 | return folderPath, urlPath[len(fullFolderPath):] 192 | } 193 | 194 | // folderPathValid checks if a folderPath returned by splitURLPath is valid and 195 | // safe. 196 | func folderPathValid(folderPath []string) bool { 197 | for _, name := range folderPath { 198 | if name == "" || name == ".." || name == "." || !valid(name) { 199 | return false 200 | } 201 | } 202 | return true 203 | } 204 | -------------------------------------------------------------------------------- /handlers_test.go: -------------------------------------------------------------------------------- 1 | package restserver 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "encoding/hex" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "net/http/httptest" 11 | "os" 12 | "path" 13 | "path/filepath" 14 | "reflect" 15 | "strings" 16 | "sync" 17 | "testing" 18 | 19 | "github.com/minio/sha256-simd" 20 | ) 21 | 22 | func TestJoin(t *testing.T) { 23 | var tests = []struct { 24 | base string 25 | names []string 26 | result string 27 | }{ 28 | {"/", []string{"foo", "bar"}, "/foo/bar"}, 29 | {"/srv/server", []string{"foo", "bar"}, "/srv/server/foo/bar"}, 30 | {"/srv/server", []string{"foo", "..", "bar"}, "/srv/server/foo/bar"}, 31 | {"/srv/server", []string{"..", "bar"}, "/srv/server/bar"}, 32 | {"/srv/server", []string{".."}, "/srv/server"}, 33 | {"/srv/server", []string{"..", ".."}, "/srv/server"}, 34 | {"/srv/server", []string{"repo", "data"}, "/srv/server/repo/data"}, 35 | {"/srv/server", []string{"repo", "data", "..", ".."}, "/srv/server/repo/data"}, 36 | {"/srv/server", []string{"repo", "data", "..", "data", "..", "..", ".."}, "/srv/server/repo/data/data"}, 37 | } 38 | 39 | for _, test := range tests { 40 | t.Run("", func(t *testing.T) { 41 | got, err := join(filepath.FromSlash(test.base), test.names...) 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | 46 | want := filepath.FromSlash(test.result) 47 | if got != want { 48 | t.Fatalf("wrong result returned, want %v, got %v", want, got) 49 | } 50 | }) 51 | } 52 | } 53 | 54 | // declare a few helper functions 55 | 56 | // wantFunc tests the HTTP response in res and calls t.Error() if something is incorrect. 57 | type wantFunc func(t testing.TB, res *httptest.ResponseRecorder) 58 | 59 | // newRequest returns a new HTTP request with the given params. On error, t.Fatal is called. 60 | func newRequest(t testing.TB, method, path string, body io.Reader) *http.Request { 61 | req, err := http.NewRequest(method, path, body) 62 | if err != nil { 63 | t.Fatal(err) 64 | } 65 | return req 66 | } 67 | 68 | // wantCode returns a function which checks that the response has the correct HTTP status code. 69 | func wantCode(code int) wantFunc { 70 | return func(t testing.TB, res *httptest.ResponseRecorder) { 71 | t.Helper() 72 | if res.Code != code { 73 | t.Errorf("wrong response code, want %v, got %v", code, res.Code) 74 | } 75 | } 76 | } 77 | 78 | // wantBody returns a function which checks that the response has the data in the body. 79 | func wantBody(body string) wantFunc { 80 | return func(t testing.TB, res *httptest.ResponseRecorder) { 81 | t.Helper() 82 | if res.Body == nil { 83 | t.Errorf("body is nil, want %q", body) 84 | return 85 | } 86 | 87 | if !bytes.Equal(res.Body.Bytes(), []byte(body)) { 88 | t.Errorf("wrong response body, want:\n %q\ngot:\n %q", body, res.Body.Bytes()) 89 | } 90 | } 91 | } 92 | 93 | // checkRequest uses f to process the request and runs the checker functions on the result. 94 | func checkRequest(t testing.TB, f http.HandlerFunc, req *http.Request, want []wantFunc) { 95 | t.Helper() 96 | rr := httptest.NewRecorder() 97 | f(rr, req) 98 | 99 | for _, fn := range want { 100 | fn(t, rr) 101 | } 102 | } 103 | 104 | // TestRequest is a sequence of HTTP requests with (optional) tests for the response. 105 | type TestRequest struct { 106 | req *http.Request 107 | want []wantFunc 108 | } 109 | 110 | // createOverwriteDeleteSeq returns a sequence which will create a new file at 111 | // path, and then try to overwrite and delete it. 112 | func createOverwriteDeleteSeq(t testing.TB, path string, data string) []TestRequest { 113 | // add a file, try to overwrite and delete it 114 | req := []TestRequest{ 115 | { 116 | req: newRequest(t, "GET", path, nil), 117 | want: []wantFunc{wantCode(http.StatusNotFound)}, 118 | }, 119 | } 120 | 121 | if !strings.HasSuffix(path, "/config") { 122 | req = append(req, TestRequest{ 123 | // broken upload must fail 124 | req: newRequest(t, "POST", path, strings.NewReader(data+"broken")), 125 | want: []wantFunc{wantCode(http.StatusBadRequest)}, 126 | }) 127 | } 128 | 129 | req = append(req, 130 | TestRequest{ 131 | req: newRequest(t, "POST", path, strings.NewReader(data)), 132 | want: []wantFunc{wantCode(http.StatusOK)}, 133 | }, 134 | TestRequest{ 135 | req: newRequest(t, "GET", path, nil), 136 | want: []wantFunc{ 137 | wantCode(http.StatusOK), 138 | wantBody(data), 139 | }, 140 | }, 141 | TestRequest{ 142 | req: newRequest(t, "POST", path, strings.NewReader(data+"other stuff")), 143 | want: []wantFunc{wantCode(http.StatusForbidden)}, 144 | }, 145 | TestRequest{ 146 | req: newRequest(t, "GET", path, nil), 147 | want: []wantFunc{ 148 | wantCode(http.StatusOK), 149 | wantBody(data), 150 | }, 151 | }, 152 | TestRequest{ 153 | req: newRequest(t, "DELETE", path, nil), 154 | want: []wantFunc{wantCode(http.StatusForbidden)}, 155 | }, 156 | TestRequest{ 157 | req: newRequest(t, "GET", path, nil), 158 | want: []wantFunc{ 159 | wantCode(http.StatusOK), 160 | wantBody(data), 161 | }, 162 | }, 163 | ) 164 | return req 165 | } 166 | 167 | func createTestHandler(t *testing.T, conf *Server) (http.Handler, string, string, string, func()) { 168 | buf := make([]byte, 32) 169 | _, err := io.ReadFull(rand.Reader, buf) 170 | if err != nil { 171 | t.Fatal(err) 172 | } 173 | data := "random data file " + hex.EncodeToString(buf) 174 | dataHash := sha256.Sum256([]byte(data)) 175 | fileID := hex.EncodeToString(dataHash[:]) 176 | 177 | // setup the server with a local backend in a temporary directory 178 | tempdir, err := os.MkdirTemp("", "rest-server-test-") 179 | if err != nil { 180 | t.Fatal(err) 181 | } 182 | 183 | // make sure the tempdir is properly removed 184 | cleanup := func() { 185 | err := os.RemoveAll(tempdir) 186 | if err != nil { 187 | t.Fatal(err) 188 | } 189 | } 190 | 191 | conf.Path = tempdir 192 | mux, err := NewHandler(conf) 193 | if err != nil { 194 | t.Fatalf("error from NewHandler: %v", err) 195 | } 196 | return mux, data, fileID, tempdir, cleanup 197 | } 198 | 199 | // TestResticAppendOnlyHandler runs tests on the restic handler code, especially in append-only mode. 200 | func TestResticAppendOnlyHandler(t *testing.T) { 201 | mux, data, fileID, _, cleanup := createTestHandler(t, &Server{ 202 | AppendOnly: true, 203 | NoAuth: true, 204 | Debug: true, 205 | PanicOnError: true, 206 | }) 207 | defer cleanup() 208 | 209 | var tests = []struct { 210 | seq []TestRequest 211 | }{ 212 | {createOverwriteDeleteSeq(t, "/config", data)}, 213 | {createOverwriteDeleteSeq(t, "/data/"+fileID, data)}, 214 | { 215 | // ensure we can add and remove lock files 216 | []TestRequest{ 217 | { 218 | req: newRequest(t, "GET", "/locks/"+fileID, nil), 219 | want: []wantFunc{wantCode(http.StatusNotFound)}, 220 | }, 221 | { 222 | req: newRequest(t, "POST", "/locks/"+fileID, strings.NewReader(data+"broken")), 223 | want: []wantFunc{wantCode(http.StatusBadRequest)}, 224 | }, 225 | { 226 | req: newRequest(t, "POST", "/locks/"+fileID, strings.NewReader(data)), 227 | want: []wantFunc{wantCode(http.StatusOK)}, 228 | }, 229 | { 230 | req: newRequest(t, "GET", "/locks/"+fileID, nil), 231 | want: []wantFunc{ 232 | wantCode(http.StatusOK), 233 | wantBody(data), 234 | }, 235 | }, 236 | { 237 | req: newRequest(t, "POST", "/locks/"+fileID, strings.NewReader(data+"other data")), 238 | want: []wantFunc{wantCode(http.StatusForbidden)}, 239 | }, 240 | { 241 | req: newRequest(t, "DELETE", "/locks/"+fileID, nil), 242 | want: []wantFunc{wantCode(http.StatusOK)}, 243 | }, 244 | { 245 | req: newRequest(t, "GET", "/locks/"+fileID, nil), 246 | want: []wantFunc{wantCode(http.StatusNotFound)}, 247 | }, 248 | }, 249 | }, 250 | 251 | // Test subrepos 252 | {createOverwriteDeleteSeq(t, "/parent1/sub1/config", "foobar")}, 253 | {createOverwriteDeleteSeq(t, "/parent1/sub1/data/"+fileID, data)}, 254 | {createOverwriteDeleteSeq(t, "/parent1/config", "foobar")}, 255 | {createOverwriteDeleteSeq(t, "/parent1/data/"+fileID, data)}, 256 | {createOverwriteDeleteSeq(t, "/parent2/config", "foobar")}, 257 | {createOverwriteDeleteSeq(t, "/parent2/data/"+fileID, data)}, 258 | } 259 | 260 | // create the repos 261 | for _, path := range []string{"/", "/parent1/sub1/", "/parent1/", "/parent2/"} { 262 | checkRequest(t, mux.ServeHTTP, 263 | newRequest(t, "POST", path+"?create=true", nil), 264 | []wantFunc{wantCode(http.StatusOK)}) 265 | } 266 | 267 | for _, test := range tests { 268 | t.Run("", func(t *testing.T) { 269 | for i, seq := range test.seq { 270 | t.Logf("request %v: %v %v", i, seq.req.Method, seq.req.URL.Path) 271 | checkRequest(t, mux.ServeHTTP, seq.req, seq.want) 272 | } 273 | }) 274 | } 275 | } 276 | 277 | // createOverwriteDeleteSeq returns a sequence which will create a new file at 278 | // path, and then deletes it twice. 279 | func createIdempotentDeleteSeq(t testing.TB, path string, data string) []TestRequest { 280 | return []TestRequest{ 281 | { 282 | req: newRequest(t, "POST", path, strings.NewReader(data)), 283 | want: []wantFunc{wantCode(http.StatusOK)}, 284 | }, 285 | { 286 | req: newRequest(t, "DELETE", path, nil), 287 | want: []wantFunc{wantCode(http.StatusOK)}, 288 | }, 289 | { 290 | req: newRequest(t, "GET", path, nil), 291 | want: []wantFunc{wantCode(http.StatusNotFound)}, 292 | }, 293 | { 294 | req: newRequest(t, "DELETE", path, nil), 295 | want: []wantFunc{wantCode(http.StatusOK)}, 296 | }, 297 | } 298 | } 299 | 300 | // TestResticHandler runs tests on the restic handler code, especially in append-only mode. 301 | func TestResticHandler(t *testing.T) { 302 | mux, data, fileID, _, cleanup := createTestHandler(t, &Server{ 303 | NoAuth: true, 304 | Debug: true, 305 | PanicOnError: true, 306 | }) 307 | defer cleanup() 308 | 309 | var tests = []struct { 310 | seq []TestRequest 311 | }{ 312 | {createIdempotentDeleteSeq(t, "/config", data)}, 313 | {createIdempotentDeleteSeq(t, "/data/"+fileID, data)}, 314 | } 315 | 316 | // create the repo 317 | checkRequest(t, mux.ServeHTTP, 318 | newRequest(t, "POST", "/?create=true", nil), 319 | []wantFunc{wantCode(http.StatusOK)}) 320 | 321 | for _, test := range tests { 322 | t.Run("", func(t *testing.T) { 323 | for i, seq := range test.seq { 324 | t.Logf("request %v: %v %v", i, seq.req.Method, seq.req.URL.Path) 325 | checkRequest(t, mux.ServeHTTP, seq.req, seq.want) 326 | } 327 | }) 328 | } 329 | } 330 | 331 | // TestResticErrorHandler runs tests on the restic handler error handling. 332 | func TestResticErrorHandler(t *testing.T) { 333 | mux, _, _, tempdir, cleanup := createTestHandler(t, &Server{ 334 | AppendOnly: true, 335 | NoAuth: true, 336 | Debug: true, 337 | }) 338 | defer cleanup() 339 | 340 | var tests = []struct { 341 | seq []TestRequest 342 | }{ 343 | // Test inaccessible file 344 | { 345 | []TestRequest{{ 346 | req: newRequest(t, "GET", "/config", nil), 347 | want: []wantFunc{wantCode(http.StatusInternalServerError)}, 348 | }}, 349 | }, 350 | { 351 | []TestRequest{{ 352 | req: newRequest(t, "GET", "/parent4/config", nil), 353 | want: []wantFunc{wantCode(http.StatusNotFound)}, 354 | }}, 355 | }, 356 | } 357 | 358 | // create the repo 359 | checkRequest(t, mux.ServeHTTP, 360 | newRequest(t, "POST", "/?create=true", nil), 361 | []wantFunc{wantCode(http.StatusOK)}) 362 | // create inaccessible config 363 | checkRequest(t, mux.ServeHTTP, 364 | newRequest(t, "POST", "/config", strings.NewReader("example")), 365 | []wantFunc{wantCode(http.StatusOK)}) 366 | err := os.Chmod(path.Join(tempdir, "config"), 0o000) 367 | if err != nil { 368 | t.Fatal(err) 369 | } 370 | 371 | for _, test := range tests { 372 | t.Run("", func(t *testing.T) { 373 | for i, seq := range test.seq { 374 | t.Logf("request %v: %v %v", i, seq.req.Method, seq.req.URL.Path) 375 | checkRequest(t, mux.ServeHTTP, seq.req, seq.want) 376 | } 377 | }) 378 | } 379 | } 380 | 381 | func TestEmptyList(t *testing.T) { 382 | mux, _, _, _, cleanup := createTestHandler(t, &Server{ 383 | AppendOnly: true, 384 | NoAuth: true, 385 | Debug: true, 386 | }) 387 | defer cleanup() 388 | 389 | // create the repo 390 | checkRequest(t, mux.ServeHTTP, 391 | newRequest(t, "POST", "/?create=true", nil), 392 | []wantFunc{wantCode(http.StatusOK)}) 393 | 394 | for i := 1; i <= 2; i++ { 395 | req := newRequest(t, "GET", "/data/", nil) 396 | if i == 2 { 397 | req.Header.Set("Accept", "application/vnd.x.restic.rest.v2") 398 | } 399 | 400 | checkRequest(t, mux.ServeHTTP, req, 401 | []wantFunc{wantCode(http.StatusOK), wantBody("[]")}) 402 | } 403 | } 404 | 405 | func TestListWithUnexpectedFiles(t *testing.T) { 406 | mux, _, _, tempdir, cleanup := createTestHandler(t, &Server{ 407 | AppendOnly: true, 408 | NoAuth: true, 409 | Debug: true, 410 | }) 411 | defer cleanup() 412 | 413 | // create the repo 414 | checkRequest(t, mux.ServeHTTP, 415 | newRequest(t, "POST", "/?create=true", nil), 416 | []wantFunc{wantCode(http.StatusOK)}) 417 | err := os.WriteFile(path.Join(tempdir, "data", "temp"), []byte{}, 0o666) 418 | if err != nil { 419 | t.Fatalf("creating unexpected file failed: %v", err) 420 | } 421 | 422 | for i := 1; i <= 2; i++ { 423 | req := newRequest(t, "GET", "/data/", nil) 424 | if i == 2 { 425 | req.Header.Set("Accept", "application/vnd.x.restic.rest.v2") 426 | } 427 | 428 | checkRequest(t, mux.ServeHTTP, req, 429 | []wantFunc{wantCode(http.StatusOK)}) 430 | } 431 | } 432 | 433 | func TestSplitURLPath(t *testing.T) { 434 | var tests = []struct { 435 | // Params 436 | urlPath string 437 | maxDepth int 438 | // Expected result 439 | folderPath []string 440 | remainder string 441 | }{ 442 | {"/", 0, nil, "/"}, 443 | {"/", 2, nil, "/"}, 444 | {"/foo/bar/locks/0123", 0, nil, "/foo/bar/locks/0123"}, 445 | {"/foo/bar/locks/0123", 1, []string{"foo"}, "/bar/locks/0123"}, 446 | {"/foo/bar/locks/0123", 2, []string{"foo", "bar"}, "/locks/0123"}, 447 | {"/foo/bar/locks/0123", 3, []string{"foo", "bar"}, "/locks/0123"}, 448 | {"/foo/bar/zzz/locks/0123", 2, []string{"foo", "bar"}, "/zzz/locks/0123"}, 449 | {"/foo/bar/zzz/locks/0123", 3, []string{"foo", "bar", "zzz"}, "/locks/0123"}, 450 | {"/foo/bar/locks/", 2, []string{"foo", "bar"}, "/locks/"}, 451 | {"/foo/locks/", 2, []string{"foo"}, "/locks/"}, 452 | {"/foo/data/", 2, []string{"foo"}, "/data/"}, 453 | {"/foo/index/", 2, []string{"foo"}, "/index/"}, 454 | {"/foo/keys/", 2, []string{"foo"}, "/keys/"}, 455 | {"/foo/snapshots/", 2, []string{"foo"}, "/snapshots/"}, 456 | {"/foo/config", 2, []string{"foo"}, "/config"}, 457 | {"/foo/", 2, []string{"foo"}, "/"}, 458 | {"/foo/bar/", 2, []string{"foo", "bar"}, "/"}, 459 | {"/foo/bar", 2, []string{"foo"}, "/bar"}, 460 | {"/locks/", 2, nil, "/locks/"}, 461 | // This function only splits, it does not check the path components! 462 | {"/././locks/", 2, []string{".", "."}, "/locks/"}, 463 | {"/../../locks/", 2, []string{"..", ".."}, "/locks/"}, 464 | {"///locks/", 2, []string{"", ""}, "/locks/"}, 465 | {"////locks/", 2, []string{"", ""}, "//locks/"}, 466 | // Robustness against broken input 467 | {"/", -42, nil, "/"}, 468 | {"foo", 2, nil, "foo"}, 469 | {"foo/bar", 2, nil, "foo/bar"}, 470 | {"", 2, nil, ""}, 471 | } 472 | 473 | for i, test := range tests { 474 | t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { 475 | folderPath, remainder := splitURLPath(test.urlPath, test.maxDepth) 476 | 477 | var fpEqual bool 478 | if len(test.folderPath) == 0 && len(folderPath) == 0 { 479 | fpEqual = true // this check allows for nil vs empty slice 480 | } else { 481 | fpEqual = reflect.DeepEqual(test.folderPath, folderPath) 482 | } 483 | if !fpEqual { 484 | t.Errorf("wrong folderPath: want %v, got %v", test.folderPath, folderPath) 485 | } 486 | 487 | if test.remainder != remainder { 488 | t.Errorf("wrong remainder: want %v, got %v", test.remainder, remainder) 489 | } 490 | }) 491 | } 492 | } 493 | 494 | // delayErrorReader blocks until Continue is closed, closes the channel FirstRead and then returns Err. 495 | type delayErrorReader struct { 496 | FirstRead chan struct{} 497 | firstReadOnce sync.Once 498 | 499 | Err error 500 | 501 | Continue chan struct{} 502 | } 503 | 504 | func newDelayedErrorReader(err error) *delayErrorReader { 505 | return &delayErrorReader{ 506 | Err: err, 507 | Continue: make(chan struct{}), 508 | FirstRead: make(chan struct{}), 509 | } 510 | } 511 | 512 | func (d *delayErrorReader) Read(_ []byte) (int, error) { 513 | d.firstReadOnce.Do(func() { 514 | // close the channel to signal that the first read has happened 515 | close(d.FirstRead) 516 | }) 517 | <-d.Continue 518 | return 0, d.Err 519 | } 520 | 521 | // TestAbortedRequest runs tests with concurrent upload requests for the same file. 522 | func TestAbortedRequest(t *testing.T) { 523 | // the race condition doesn't happen for append-only repositories 524 | mux, _, _, _, cleanup := createTestHandler(t, &Server{ 525 | NoAuth: true, 526 | Debug: true, 527 | PanicOnError: true, 528 | }) 529 | defer cleanup() 530 | 531 | // create the repo 532 | checkRequest(t, mux.ServeHTTP, 533 | newRequest(t, "POST", "/?create=true", nil), 534 | []wantFunc{wantCode(http.StatusOK)}) 535 | 536 | var ( 537 | id = "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" 538 | wg sync.WaitGroup 539 | ) 540 | 541 | // the first request is an upload to a file which blocks while reading the 542 | // body and then after some data returns an error 543 | rd := newDelayedErrorReader(io.ErrUnexpectedEOF) 544 | 545 | wg.Add(1) 546 | go func() { 547 | defer wg.Done() 548 | 549 | // first, read some string, then read from rd (which blocks and then 550 | // returns an error) 551 | dataReader := io.MultiReader(strings.NewReader("invalid data from aborted request\n"), rd) 552 | 553 | t.Logf("start first upload") 554 | req := newRequest(t, "POST", "/data/"+id, dataReader) 555 | rr := httptest.NewRecorder() 556 | mux.ServeHTTP(rr, req) 557 | t.Logf("first upload done, response %v (%v)", rr.Code, rr.Result().Status) 558 | }() 559 | 560 | // wait until the first request starts reading from the body 561 | <-rd.FirstRead 562 | 563 | // then while the first request is blocked we send a second request to 564 | // delete the file and a third request to upload to the file again, only 565 | // then the first request is unblocked. 566 | 567 | t.Logf("delete file") 568 | checkRequest(t, mux.ServeHTTP, 569 | newRequest(t, "DELETE", "/data/"+id, nil), 570 | nil) // don't check anything, restic also ignores errors here 571 | 572 | t.Logf("upload again") 573 | checkRequest(t, mux.ServeHTTP, 574 | newRequest(t, "POST", "/data/"+id, strings.NewReader("foo\n")), 575 | []wantFunc{wantCode(http.StatusOK)}) 576 | 577 | // unblock the reader for the first request now so it can continue 578 | close(rd.Continue) 579 | 580 | // wait for the first request to continue 581 | wg.Wait() 582 | 583 | // request the file again, it must exist and contain the string from the 584 | // second request 585 | checkRequest(t, mux.ServeHTTP, 586 | newRequest(t, "GET", "/data/"+id, nil), 587 | []wantFunc{ 588 | wantCode(http.StatusOK), 589 | wantBody("foo\n"), 590 | }, 591 | ) 592 | } 593 | -------------------------------------------------------------------------------- /htpasswd.go: -------------------------------------------------------------------------------- 1 | package restserver 2 | 3 | /* 4 | Original version copied from: github.com/bitly/oauth2_proxy 5 | 6 | MIT License 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in 16 | all copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24 | THE SOFTWARE. 25 | */ 26 | 27 | import ( 28 | "crypto/sha1" 29 | "crypto/sha256" 30 | "crypto/subtle" 31 | "encoding/base64" 32 | "encoding/csv" 33 | "log" 34 | "os" 35 | "os/signal" 36 | "regexp" 37 | "sync" 38 | "syscall" 39 | "time" 40 | 41 | "golang.org/x/crypto/bcrypt" 42 | ) 43 | 44 | // CheckInterval represents how often we check for changes in htpasswd file. 45 | const CheckInterval = 30 * time.Second 46 | 47 | // PasswordCacheDuration represents how long authentication credentials are 48 | // cached in memory after they were successfully verified. This allows avoiding 49 | // repeatedly verifying the same authentication credentials. 50 | const PasswordCacheDuration = time.Minute 51 | 52 | // Lookup passwords in a htpasswd file. The entries must have been created with -s for SHA encryption. 53 | 54 | type cacheEntry struct { 55 | expiry time.Time 56 | verifier []byte 57 | } 58 | 59 | // HtpasswdFile is a map for usernames to passwords. 60 | type HtpasswdFile struct { 61 | mutex sync.Mutex 62 | path string 63 | stat os.FileInfo 64 | throttle chan struct{} 65 | users map[string]string 66 | cache map[string]cacheEntry 67 | } 68 | 69 | // NewHtpasswdFromFile reads the users and passwords from a htpasswd file and returns them. If an error is encountered, 70 | // it is returned, together with a nil-Pointer for the HtpasswdFile. 71 | func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) { 72 | c := make(chan os.Signal, 1) 73 | signal.Notify(c, syscall.SIGHUP) 74 | stat, err := os.Stat(path) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | h := &HtpasswdFile{ 80 | mutex: sync.Mutex{}, 81 | path: path, 82 | stat: stat, 83 | throttle: make(chan struct{}), 84 | cache: make(map[string]cacheEntry), 85 | } 86 | 87 | if err := h.Reload(); err != nil { 88 | return nil, err 89 | } 90 | 91 | // Start a goroutine that limits reload checks to once per CheckInterval 92 | go h.throttleTimer() 93 | go h.expiryTimer() 94 | 95 | go func() { 96 | for range c { 97 | err := h.Reload() 98 | if err == nil { 99 | log.Printf("Reloaded htpasswd file") 100 | } else { 101 | log.Printf("Could not reload htpasswd file: %v", err) 102 | } 103 | } 104 | }() 105 | 106 | return h, nil 107 | } 108 | 109 | // throttleTimer sends at most one message per CheckInterval to throttle file change checks. 110 | func (h *HtpasswdFile) throttleTimer() { 111 | var check struct{} 112 | for { 113 | time.Sleep(CheckInterval) 114 | h.throttle <- check 115 | } 116 | } 117 | 118 | func (h *HtpasswdFile) expiryTimer() { 119 | for { 120 | time.Sleep(5 * time.Second) 121 | now := time.Now() 122 | h.mutex.Lock() 123 | var zeros [sha256.Size]byte 124 | // try to wipe expired cache entries 125 | for user, entry := range h.cache { 126 | if entry.expiry.After(now) { 127 | copy(entry.verifier, zeros[:]) 128 | delete(h.cache, user) 129 | } 130 | } 131 | h.mutex.Unlock() 132 | } 133 | } 134 | 135 | var validUsernameRegexp = regexp.MustCompile(`^[\p{L}\d@._-]+$`) 136 | 137 | // Reload reloads the htpasswd file. If the reload fails, the Users map is not changed and the error is returned. 138 | func (h *HtpasswdFile) Reload() error { 139 | r, err := os.Open(h.path) 140 | if err != nil { 141 | return err 142 | } 143 | 144 | cr := csv.NewReader(r) 145 | cr.Comma = ':' 146 | cr.Comment = '#' 147 | cr.TrimLeadingSpace = true 148 | 149 | records, err := cr.ReadAll() 150 | if err != nil { 151 | _ = r.Close() 152 | return err 153 | } 154 | users := make(map[string]string) 155 | for _, record := range records { 156 | if !validUsernameRegexp.MatchString(record[0]) { 157 | log.Printf("Ignoring invalid username %q in htpasswd, consists of characters other than letters, numbers, '_', '-', '.' and '@'", record[0]) 158 | continue 159 | } 160 | users[record[0]] = record[1] 161 | } 162 | 163 | // Replace the Users map 164 | h.mutex.Lock() 165 | var zeros [sha256.Size]byte 166 | // try to wipe the old cache entries 167 | for _, entry := range h.cache { 168 | copy(entry.verifier, zeros[:]) 169 | } 170 | h.cache = make(map[string]cacheEntry) 171 | 172 | h.users = users 173 | h.mutex.Unlock() 174 | 175 | _ = r.Close() 176 | return nil 177 | } 178 | 179 | // ReloadCheck checks at most once per CheckInterval if the file changed and will reload the file if it did. 180 | // It logs errors and successful reloads, and returns an error if any was encountered. 181 | func (h *HtpasswdFile) ReloadCheck() error { 182 | select { 183 | case <-h.throttle: 184 | stat, err := os.Stat(h.path) 185 | if err != nil { 186 | log.Printf("Could not stat htpasswd file: %v", err) 187 | return err 188 | } 189 | 190 | reload := false 191 | 192 | h.mutex.Lock() 193 | if stat.ModTime() != h.stat.ModTime() || stat.Size() != h.stat.Size() { 194 | reload = true 195 | h.stat = stat 196 | } 197 | h.mutex.Unlock() 198 | 199 | if reload { 200 | err := h.Reload() 201 | if err == nil { 202 | log.Printf("Reloaded htpasswd file") 203 | } else { 204 | log.Printf("Could not reload htpasswd file: %v", err) 205 | return err 206 | } 207 | } 208 | default: 209 | // No need to check 210 | } 211 | return nil 212 | } 213 | 214 | var shaRe = regexp.MustCompile(`^{SHA}`) 215 | var bcrRe = regexp.MustCompile(`^\$2b\$|^\$2a\$|^\$2y\$`) 216 | 217 | // Validate returns true if password matches the stored password for user. If no password for user is stored, or the 218 | // password is wrong, false is returned. 219 | func (h *HtpasswdFile) Validate(user string, password string) bool { 220 | _ = h.ReloadCheck() 221 | 222 | hash := sha256.New() 223 | // hash.Write can never fail 224 | _, _ = hash.Write([]byte(user)) 225 | _, _ = hash.Write([]byte(":")) 226 | _, _ = hash.Write([]byte(password)) 227 | 228 | h.mutex.Lock() 229 | // avoid race conditions with cache replacements 230 | cache := h.cache 231 | hashedPassword, exists := h.users[user] 232 | entry, cacheExists := h.cache[user] 233 | h.mutex.Unlock() 234 | 235 | if !exists { 236 | return false 237 | } 238 | 239 | if cacheExists && subtle.ConstantTimeCompare(entry.verifier, hash.Sum(nil)) == 1 { 240 | h.mutex.Lock() 241 | // repurpose mutex to prevent concurrent cache updates 242 | // extend cache entry 243 | cache[user] = cacheEntry{ 244 | verifier: entry.verifier, 245 | expiry: time.Now().Add(PasswordCacheDuration), 246 | } 247 | h.mutex.Unlock() 248 | return true 249 | } 250 | 251 | isValid := isMatchingHashAndPassword(hashedPassword, password) 252 | 253 | if !isValid { 254 | log.Printf("Invalid htpasswd entry for %s.", user) 255 | return false 256 | } 257 | 258 | h.mutex.Lock() 259 | // repurpose mutex to prevent concurrent cache updates 260 | cache[user] = cacheEntry{ 261 | verifier: hash.Sum(nil), 262 | expiry: time.Now().Add(PasswordCacheDuration), 263 | } 264 | h.mutex.Unlock() 265 | 266 | return true 267 | } 268 | 269 | func isMatchingHashAndPassword(hashedPassword string, password string) bool { 270 | switch { 271 | case shaRe.MatchString(hashedPassword): 272 | d := sha1.New() 273 | _, _ = d.Write([]byte(password)) 274 | if subtle.ConstantTimeCompare([]byte(hashedPassword[5:]), []byte(base64.StdEncoding.EncodeToString(d.Sum(nil)))) == 1 { 275 | return true 276 | } 277 | case bcrRe.MatchString(hashedPassword): 278 | err := bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte(password)) 279 | if err == nil { 280 | return true 281 | } 282 | } 283 | return false 284 | } 285 | -------------------------------------------------------------------------------- /htpasswd_test.go: -------------------------------------------------------------------------------- 1 | package restserver 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestValidate(t *testing.T) { 9 | user := "restic" 10 | pwd := "$2y$05$z/OEmNQamd6m6LSegUErh.r/Owk9Xwmc5lxDheIuHY2Z7XiS6FtJm" 11 | rawPwd := "test" 12 | wrongPwd := "wrong" 13 | 14 | tmpfile, err := os.CreateTemp("", "rest-validate-") 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | if _, err = tmpfile.Write([]byte(user + ":" + pwd + "\n")); err != nil { 19 | t.Fatal(err) 20 | } 21 | if err = tmpfile.Close(); err != nil { 22 | t.Fatal(err) 23 | } 24 | 25 | htpass, err := NewHtpasswdFromFile(tmpfile.Name()) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | 30 | for i := 0; i < 10; i++ { 31 | isValid := htpass.Validate(user, rawPwd) 32 | if !isValid { 33 | t.Fatal("correct password not accepted") 34 | } 35 | 36 | isValid = htpass.Validate(user, wrongPwd) 37 | if isValid { 38 | t.Fatal("wrong password accepted") 39 | } 40 | } 41 | 42 | if err = os.Remove(tmpfile.Name()); err != nil { 43 | t.Fatal(err) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /metrics.go: -------------------------------------------------------------------------------- 1 | package restserver 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/restic/rest-server/repo" 8 | ) 9 | 10 | var metricLabelList = []string{"user", "repo", "type"} 11 | 12 | var metricBlobWriteTotal = prometheus.NewCounterVec( 13 | prometheus.CounterOpts{ 14 | Name: "rest_server_blob_write_total", 15 | Help: "Total number of blobs written", 16 | }, 17 | metricLabelList, 18 | ) 19 | 20 | var metricBlobWriteBytesTotal = prometheus.NewCounterVec( 21 | prometheus.CounterOpts{ 22 | Name: "rest_server_blob_write_bytes_total", 23 | Help: "Total number of bytes written to blobs", 24 | }, 25 | metricLabelList, 26 | ) 27 | 28 | var metricBlobReadTotal = prometheus.NewCounterVec( 29 | prometheus.CounterOpts{ 30 | Name: "rest_server_blob_read_total", 31 | Help: "Total number of blobs read", 32 | }, 33 | metricLabelList, 34 | ) 35 | 36 | var metricBlobReadBytesTotal = prometheus.NewCounterVec( 37 | prometheus.CounterOpts{ 38 | Name: "rest_server_blob_read_bytes_total", 39 | Help: "Total number of bytes read from blobs", 40 | }, 41 | metricLabelList, 42 | ) 43 | 44 | var metricBlobDeleteTotal = prometheus.NewCounterVec( 45 | prometheus.CounterOpts{ 46 | Name: "rest_server_blob_delete_total", 47 | Help: "Total number of blobs deleted", 48 | }, 49 | metricLabelList, 50 | ) 51 | 52 | var metricBlobDeleteBytesTotal = prometheus.NewCounterVec( 53 | prometheus.CounterOpts{ 54 | Name: "rest_server_blob_delete_bytes_total", 55 | Help: "Total number of bytes of blobs deleted", 56 | }, 57 | metricLabelList, 58 | ) 59 | 60 | // makeBlobMetricFunc creates a metrics callback function that increments the 61 | // Prometheus metrics. 62 | func makeBlobMetricFunc(username string, folderPath []string) repo.BlobMetricFunc { 63 | var f repo.BlobMetricFunc = func(objectType string, operation repo.BlobOperation, nBytes uint64) { 64 | labels := prometheus.Labels{ 65 | "user": username, 66 | "repo": strings.Join(folderPath, "/"), 67 | "type": objectType, 68 | } 69 | switch operation { 70 | case repo.BlobRead: 71 | metricBlobReadTotal.With(labels).Inc() 72 | metricBlobReadBytesTotal.With(labels).Add(float64(nBytes)) 73 | case repo.BlobWrite: 74 | metricBlobWriteTotal.With(labels).Inc() 75 | metricBlobWriteBytesTotal.With(labels).Add(float64(nBytes)) 76 | case repo.BlobDelete: 77 | metricBlobDeleteTotal.With(labels).Inc() 78 | metricBlobDeleteBytesTotal.With(labels).Add(float64(nBytes)) 79 | } 80 | } 81 | return f 82 | } 83 | 84 | func init() { 85 | // These are always initialized, but only updated if Config.Prometheus is set 86 | prometheus.MustRegister(metricBlobWriteTotal) 87 | prometheus.MustRegister(metricBlobWriteBytesTotal) 88 | prometheus.MustRegister(metricBlobReadTotal) 89 | prometheus.MustRegister(metricBlobReadBytesTotal) 90 | prometheus.MustRegister(metricBlobDeleteTotal) 91 | prometheus.MustRegister(metricBlobDeleteBytesTotal) 92 | } 93 | -------------------------------------------------------------------------------- /mux.go: -------------------------------------------------------------------------------- 1 | package restserver 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/gorilla/handlers" 12 | "github.com/prometheus/client_golang/prometheus/promhttp" 13 | "github.com/restic/rest-server/quota" 14 | ) 15 | 16 | func (s *Server) debugHandler(next http.Handler) http.Handler { 17 | return http.HandlerFunc( 18 | func(w http.ResponseWriter, r *http.Request) { 19 | log.Printf("%s %s", r.Method, r.URL) 20 | next.ServeHTTP(w, r) 21 | }) 22 | } 23 | 24 | func (s *Server) logHandler(next http.Handler) http.Handler { 25 | var accessLog io.Writer 26 | 27 | if s.Log == "-" { 28 | accessLog = os.Stdout 29 | } else { 30 | var err error 31 | accessLog, err = os.OpenFile(s.Log, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) 32 | if err != nil { 33 | log.Fatalf("error: %v", err) 34 | } 35 | } 36 | 37 | return handlers.CombinedLoggingHandler(accessLog, next) 38 | } 39 | 40 | func (s *Server) checkAuth(r *http.Request) (username string, ok bool) { 41 | if s.NoAuth { 42 | return username, true 43 | } 44 | if s.ProxyAuthUsername != "" { 45 | username = r.Header.Get(s.ProxyAuthUsername) 46 | if username == "" { 47 | return "", false 48 | } 49 | } else { 50 | var password string 51 | username, password, ok = r.BasicAuth() 52 | if !ok || !s.htpasswdFile.Validate(username, password) { 53 | return "", false 54 | } 55 | } 56 | return username, true 57 | } 58 | 59 | func (s *Server) wrapMetricsAuth(f http.HandlerFunc) http.HandlerFunc { 60 | return func(w http.ResponseWriter, r *http.Request) { 61 | username, ok := s.checkAuth(r) 62 | if !ok { 63 | httpDefaultError(w, http.StatusUnauthorized) 64 | return 65 | } 66 | if s.PrivateRepos && username != "metrics" { 67 | httpDefaultError(w, http.StatusUnauthorized) 68 | return 69 | } 70 | f(w, r) 71 | } 72 | } 73 | 74 | // NewHandler returns the master HTTP multiplexer/router. 75 | func NewHandler(server *Server) (http.Handler, error) { 76 | if !server.NoAuth && server.ProxyAuthUsername == "" { 77 | var err error 78 | if server.HtpasswdPath == "" { 79 | server.HtpasswdPath = filepath.Join(server.Path, ".htpasswd") 80 | } 81 | server.htpasswdFile, err = NewHtpasswdFromFile(server.HtpasswdPath) 82 | if err != nil { 83 | return nil, fmt.Errorf("cannot load %s (use --no-auth to disable): %v", server.HtpasswdPath, err) 84 | } 85 | log.Printf("Loaded htpasswd file %s", server.HtpasswdPath) 86 | } 87 | 88 | const GiB = 1024 * 1024 * 1024 89 | 90 | if server.MaxRepoSize > 0 { 91 | log.Printf("Initializing quota (can take a while)...") 92 | qm, err := quota.New(server.Path, server.MaxRepoSize) 93 | if err != nil { 94 | return nil, err 95 | } 96 | server.quotaManager = qm 97 | log.Printf("Quota initialized, currently using %.2f GiB", float64(qm.SpaceUsed())/GiB) 98 | } 99 | 100 | mux := http.NewServeMux() 101 | if server.Prometheus { 102 | if server.PrometheusNoAuth { 103 | mux.Handle("/metrics", promhttp.Handler()) 104 | } else { 105 | mux.HandleFunc("/metrics", server.wrapMetricsAuth(promhttp.Handler().ServeHTTP)) 106 | } 107 | } 108 | mux.Handle("/", server) 109 | 110 | var handler http.Handler = mux 111 | if server.Debug { 112 | handler = server.debugHandler(handler) 113 | } 114 | if server.Log != "" { 115 | handler = server.logHandler(handler) 116 | } 117 | return handler, nil 118 | } 119 | -------------------------------------------------------------------------------- /mux_test.go: -------------------------------------------------------------------------------- 1 | package restserver 2 | 3 | import ( 4 | "net/http/httptest" 5 | "testing" 6 | ) 7 | 8 | func TestCheckAuth(t *testing.T) { 9 | tests := []struct { 10 | name string 11 | server *Server 12 | requestHeaders map[string]string 13 | basicAuth bool 14 | basicUser string 15 | basicPassword string 16 | expectedUser string 17 | expectedOk bool 18 | }{ 19 | { 20 | name: "NoAuth enabled", 21 | server: &Server{ 22 | NoAuth: true, 23 | }, 24 | expectedOk: true, 25 | }, 26 | { 27 | name: "Proxy Auth successful", 28 | server: &Server{ 29 | ProxyAuthUsername: "X-Remote-User", 30 | }, 31 | requestHeaders: map[string]string{ 32 | "X-Remote-User": "restic", 33 | }, 34 | expectedUser: "restic", 35 | expectedOk: true, 36 | }, 37 | { 38 | name: "Proxy Auth empty header", 39 | server: &Server{ 40 | ProxyAuthUsername: "X-Remote-User", 41 | }, 42 | requestHeaders: map[string]string{ 43 | "X-Remote-User": "", 44 | }, 45 | expectedOk: false, 46 | }, 47 | { 48 | name: "Proxy Auth missing header", 49 | server: &Server{ 50 | ProxyAuthUsername: "X-Remote-User", 51 | }, 52 | expectedOk: false, 53 | }, 54 | { 55 | name: "Proxy Auth send but not enabled", 56 | server: &Server{}, 57 | requestHeaders: map[string]string{ 58 | "X-Remote-User": "restic", 59 | }, 60 | expectedOk: false, 61 | }, 62 | } 63 | 64 | for _, tt := range tests { 65 | t.Run(tt.name, func(t *testing.T) { 66 | req := httptest.NewRequest("GET", "/", nil) 67 | for header, value := range tt.requestHeaders { 68 | req.Header.Set(header, value) 69 | } 70 | if tt.basicAuth { 71 | req.SetBasicAuth(tt.basicUser, tt.basicPassword) 72 | } 73 | 74 | username, ok := tt.server.checkAuth(req) 75 | if username != tt.expectedUser || ok != tt.expectedOk { 76 | t.Errorf("expected (%v, %v), got (%v, %v)", tt.expectedUser, tt.expectedOk, username, ok) 77 | } 78 | }) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /quota/quota.go: -------------------------------------------------------------------------------- 1 | package quota 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "path/filepath" 9 | "strconv" 10 | "sync/atomic" 11 | ) 12 | 13 | // New creates a new quota Manager for given path. 14 | // It will tally the current disk usage before returning. 15 | func New(path string, maxSize int64) (*Manager, error) { 16 | m := &Manager{ 17 | path: path, 18 | maxRepoSize: maxSize, 19 | } 20 | if err := m.updateSize(); err != nil { 21 | return nil, err 22 | } 23 | return m, nil 24 | } 25 | 26 | // Manager manages the repo quota for given filesystem root path, including subrepos 27 | type Manager struct { 28 | path string 29 | maxRepoSize int64 30 | repoSize int64 // must be accessed using sync/atomic 31 | } 32 | 33 | // WrapWriter limits the number of bytes written 34 | // to the space that is currently available as given by 35 | // the server's MaxRepoSize. This type is safe for use 36 | // by multiple goroutines sharing the same *Server. 37 | type maxSizeWriter struct { 38 | io.Writer 39 | m *Manager 40 | } 41 | 42 | func (w maxSizeWriter) Write(p []byte) (n int, err error) { 43 | if int64(len(p)) > w.m.SpaceRemaining() { 44 | return 0, fmt.Errorf("repository has reached maximum size (%d bytes)", w.m.maxRepoSize) 45 | } 46 | n, err = w.Writer.Write(p) 47 | w.m.IncUsage(int64(n)) 48 | return n, err 49 | } 50 | 51 | func (m *Manager) updateSize() error { 52 | // if we haven't yet computed the size of the repo, do so now 53 | initialSize, err := tallySize(m.path) 54 | if err != nil { 55 | return err 56 | } 57 | atomic.StoreInt64(&m.repoSize, initialSize) 58 | return nil 59 | } 60 | 61 | // WrapWriter wraps w in a writer that enforces s.MaxRepoSize. 62 | // If there is an error, a status code and the error are returned. 63 | func (m *Manager) WrapWriter(req *http.Request, w io.Writer) (io.Writer, int, error) { 64 | currentSize := atomic.LoadInt64(&m.repoSize) 65 | 66 | // if content-length is set and is trustworthy, we can save some time 67 | // and issue a polite error if it declares a size that's too big; since 68 | // we expect the vast majority of clients will be honest, so this check 69 | // can only help save time 70 | if contentLenStr := req.Header.Get("Content-Length"); contentLenStr != "" { 71 | contentLen, err := strconv.ParseInt(contentLenStr, 10, 64) 72 | if err != nil { 73 | return nil, http.StatusLengthRequired, err 74 | } 75 | if currentSize+contentLen > m.maxRepoSize { 76 | err := fmt.Errorf("incoming blob (%d bytes) would exceed maximum size of repository (%d bytes)", 77 | contentLen, m.maxRepoSize) 78 | return nil, http.StatusInsufficientStorage, err 79 | } 80 | } 81 | 82 | // since we can't always trust content-length, we will wrap the writer 83 | // in a custom writer that enforces the size limit during writes 84 | return maxSizeWriter{Writer: w, m: m}, 0, nil 85 | } 86 | 87 | // SpaceRemaining returns how much space is available in the repo 88 | // according to s.MaxRepoSize. s.repoSize must already be set. 89 | // If there is no limit, -1 is returned. 90 | func (m *Manager) SpaceRemaining() int64 { 91 | if m.maxRepoSize == 0 { 92 | return -1 93 | } 94 | maxSize := m.maxRepoSize 95 | currentSize := atomic.LoadInt64(&m.repoSize) 96 | return maxSize - currentSize 97 | } 98 | 99 | // SpaceUsed returns how much space is used in the repo. 100 | func (m *Manager) SpaceUsed() int64 { 101 | return atomic.LoadInt64(&m.repoSize) 102 | } 103 | 104 | // IncUsage increments the current repo size (which 105 | // must already be initialized). 106 | func (m *Manager) IncUsage(by int64) { 107 | atomic.AddInt64(&m.repoSize, by) 108 | } 109 | 110 | // tallySize counts the size of the contents of path. 111 | func tallySize(path string) (int64, error) { 112 | if path == "" { 113 | path = "." 114 | } 115 | var size int64 116 | err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { 117 | if err != nil { 118 | return err 119 | } 120 | size += info.Size() 121 | return nil 122 | }) 123 | return size, err 124 | } 125 | -------------------------------------------------------------------------------- /repo/repo_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | // +build !windows 3 | 4 | package repo 5 | 6 | import ( 7 | "errors" 8 | "runtime" 9 | "syscall" 10 | ) 11 | 12 | // The ExFAT driver on some versions of macOS can return ENOTTY, 13 | // "inappropriate ioctl for device", for fsync. 14 | // 15 | // https://github.com/restic/restic/issues/4016 16 | // https://github.com/realm/realm-core/issues/5789 17 | func isMacENOTTY(err error) bool { 18 | return runtime.GOOS == "darwin" && errors.Is(err, syscall.ENOTTY) 19 | } 20 | -------------------------------------------------------------------------------- /repo/repo_windows.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | // Windows is not macOS. 4 | func isMacENOTTY(err error) bool { return false } 5 | --------------------------------------------------------------------------------