├── .github ├── CODEOWNERS └── workflows │ ├── build.yaml │ └── docker.yaml ├── .gitignore ├── CHANGES.md ├── CONTRIBUTING.rst ├── Dockerfile ├── LICENSE ├── README.md ├── RELEASING.md ├── changelog.d ├── .gitignore └── 38.bugfix ├── docker-bake.hcl ├── docs ├── api.md ├── blocked_rageshake.md ├── generic_webhook.md └── submitted_reports.md ├── errors.go ├── go.mod ├── go.sum ├── hooks ├── install.sh └── pre-commit ├── logserver.go ├── main.go ├── main_test.go ├── rageshake.sample.yaml ├── scripts ├── cleanup.py └── lint.sh ├── slack.go ├── submit.go ├── submit_test.go ├── templates ├── README.md ├── email_body.tmpl └── issue_body.tmpl └── towncrier.toml /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @richvdh 2 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: Linting, build, test and release 2 | 3 | # Runs on each PR to lint and test 4 | 5 | # Runs on each release (via github UI) to lint and test, then upload binary to the release 6 | 7 | on: 8 | pull_request: 9 | push: 10 | branches: [master] 11 | release: 12 | types: [published] 13 | 14 | permissions: 15 | contents: write 16 | 17 | jobs: 18 | changelog: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v3 22 | with: 23 | fetch-depth: 0 24 | # do a full clone; we need to also get the branch master, required to allow towncrier to diff properly. 25 | - uses: actions/setup-python@v3 26 | - name: Install towncrier 27 | run: pip install 'towncrier>19.2' 28 | - name: Run towncrier 29 | run: python -m towncrier.check 30 | lint: 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v3 34 | - uses: actions/setup-go@v3 35 | with: 36 | go-version: 1.16 37 | - name: Install lint deps 38 | run: | 39 | go get golang.org/x/lint/golint 40 | go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow 41 | go get github.com/fzipp/gocyclo/cmd/gocyclo 42 | - name: lint 43 | run: ./scripts/lint.sh 44 | test: 45 | runs-on: ubuntu-latest 46 | strategy: 47 | matrix: 48 | golang: ["1.17", "1.16"] 49 | steps: 50 | - uses: actions/checkout@v3 51 | - uses: actions/setup-go@v3 52 | with: 53 | go-version: "${{ matrix.golang }}" 54 | - name: Build 55 | run: go build 56 | - name: Test 57 | run: go test 58 | - name: Create tarball for release 59 | if: github.event.release 60 | run: tar -czf rageshake.tar.gz rageshake 61 | - name: Upload tarball to matching release 62 | if: github.event.release 63 | uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 64 | with: 65 | files: rageshake.tar.gz 66 | env: 67 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 68 | 69 | -------------------------------------------------------------------------------- /.github/workflows/docker.yaml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | on: 4 | push: 5 | pull_request: 6 | branches: [ master ] 7 | 8 | jobs: 9 | build: 10 | name: Build and push Docker image 11 | runs-on: ubuntu-latest 12 | env: 13 | IMAGE: ghcr.io/${{ github.repository }} 14 | 15 | permissions: 16 | packages: write 17 | contents: read 18 | 19 | steps: 20 | - name: Checkout the code 21 | uses: actions/checkout@v2 22 | 23 | - name: Docker meta 24 | id: meta 25 | uses: docker/metadata-action@v3 26 | with: 27 | images: "${{ env.IMAGE }}" 28 | bake-target: docker-metadata-action 29 | tags: | 30 | type=ref,event=branch 31 | type=semver,pattern={{version}} 32 | type=semver,pattern={{major}}.{{minor}} 33 | type=semver,pattern={{major}} 34 | type=sha 35 | flavor: | 36 | latest=auto 37 | 38 | - name: Docker meta (debug variant) 39 | id: meta-debug 40 | uses: docker/metadata-action@v3 41 | with: 42 | images: "${{ env.IMAGE }}" 43 | bake-target: docker-metadata-action-debug 44 | tags: | 45 | type=ref,event=branch 46 | type=semver,pattern={{version}} 47 | type=semver,pattern={{major}}.{{minor}} 48 | type=semver,pattern={{major}} 49 | type=sha 50 | flavor: | 51 | latest=false 52 | suffix=-debug 53 | 54 | - name: Docker meta (scripts variant) 55 | id: meta-scripts 56 | uses: docker/metadata-action@v3 57 | with: 58 | images: "${{ env.IMAGE }}/scripts" 59 | bake-target: docker-metadata-action-scripts 60 | tags: | 61 | type=ref,event=branch 62 | type=semver,pattern={{version}} 63 | type=semver,pattern={{major}}.{{minor}} 64 | type=semver,pattern={{major}} 65 | type=sha 66 | flavor: | 67 | latest=auto 68 | 69 | - name: Set up Docker Buildx 70 | uses: docker/setup-buildx-action@v1 71 | with: 72 | config-inline: | 73 | [registry."docker.io"] 74 | mirrors = ["mirror.gcr.io"] 75 | 76 | - name: Login to GitHub Container Registry 77 | if: github.event_name != 'pull_request' 78 | uses: docker/login-action@v1 79 | with: 80 | registry: ghcr.io 81 | username: ${{ github.repository_owner }} 82 | password: ${{ secrets.GITHUB_TOKEN }} 83 | 84 | # For pull-requests, only read from the cache, do not try to push to the 85 | # cache or the image itself 86 | - name: Build 87 | uses: docker/bake-action@v2 88 | if: github.event_name == 'pull_request' 89 | with: 90 | files: | 91 | docker-bake.hcl 92 | ${{ steps.meta.outputs.bake-file }} 93 | ${{ steps.meta-debug.outputs.bake-file }} 94 | ${{ steps.meta-scripts.outputs.bake-file }} 95 | set: | 96 | base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache 97 | 98 | - name: Build and push 99 | uses: docker/bake-action@v2 100 | if: github.event_name != 'pull_request' 101 | with: 102 | files: | 103 | docker-bake.hcl 104 | ${{ steps.meta.outputs.bake-file }} 105 | ${{ steps.meta-debug.outputs.bake-file }} 106 | ${{ steps.meta-scripts.outputs.bake-file }} 107 | set: | 108 | base.output=type=image,push=true 109 | base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache 110 | base.cache-to=type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max 111 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /bin 3 | /bugs 4 | /pkg 5 | /rageshake.yaml 6 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | 1.16.2 (2025-03-19) 2 | =================== 3 | 4 | Bugfixes 5 | -------- 6 | 7 | - `cleanup.py`: Make resilient to malformed UTF-8 in the details file. ([\#95](https://github.com/matrix-org/rageshake/issues/95)) 8 | 9 | 10 | 1.16.1 (2025-03-17) 11 | =================== 12 | 13 | Features 14 | -------- 15 | 16 | - Add `create_time` field to webhook payload and `details.json` ([\#93](https://github.com/matrix-org/rageshake/issues/93)) 17 | 18 | 19 | 1.16.0 (2025-03-17) 20 | =================== 21 | 22 | Features 23 | -------- 24 | 25 | - Write a file called `details.json` for each submission. ([\#92](https://github.com/matrix-org/rageshake/issues/92)) 26 | 27 | 28 | 1.15.0 (2025-03-10) 29 | =================== 30 | 31 | Features 32 | -------- 33 | 34 | - The `/api/submit` endpoint responds with JSON when it encounters an error. 35 | Please read the documentation in [docs/api.md](https://github.com/matrix-org/rageshake/blob/main/docs/api.md) to learn more. ([\#90](https://github.com/matrix-org/rageshake/issues/90)) 36 | 37 | 38 | 1.14.0 (2025-02-11) 39 | =================== 40 | 41 | Features 42 | -------- 43 | 44 | - Parse User-Agent into a human readable format and attach to the report alongside the raw UA string. ([\#86](https://github.com/matrix-org/rageshake/issues/86)) 45 | - Reject user text (problem description) matching a regex and send the reason why to the client-side. ([\#88](https://github.com/matrix-org/rageshake/issues/88)) 46 | 47 | 48 | 1.13.0 (2024-05-10) 49 | =================== 50 | 51 | Features 52 | -------- 53 | 54 | - Add support for blocking specific app/version/label combinations. ([\#85](https://github.com/matrix-org/rageshake/issues/85)) 55 | 56 | 57 | 1.12.0 (2024-03-18) 58 | =================== 59 | 60 | Features 61 | -------- 62 | 63 | - Allow configuration of the body of created Github/Gitlab issues via a template in the configuration file. ([\#84](https://github.com/matrix-org/rageshake/issues/84)) 64 | 65 | 66 | 1.11.0 (2023-08-11) 67 | =================== 68 | 69 | Features 70 | -------- 71 | 72 | - Add a link to the archive containing all the logs in the issue body. ([\#81](https://github.com/matrix-org/rageshake/issues/81)) 73 | 74 | 75 | 1.10.1 (2023-05-04) 76 | =================== 77 | 78 | Bugfixes 79 | -------- 80 | 81 | - cleanup.py: Handle --repeat-delay-hours not being passed correctly, introduced in 1.10.0 ([\#78](https://github.com/matrix-org/rageshake/issues/78)) 82 | 83 | 84 | 1.10.0 (2023-05-02) 85 | =================== 86 | 87 | Features 88 | -------- 89 | 90 | - Add --repeat-delay-hours option to cleanup script to run persistently outside of a cronjob. ([\#72](https://github.com/matrix-org/rageshake/issues/72)) 91 | - Allow gzipped json & txt files to be uploaded as attachments to rageshakes. ([\#75](https://github.com/matrix-org/rageshake/issues/75)) 92 | 93 | 94 | Internal Changes 95 | ---------------- 96 | 97 | - Creates a new `rageshake/scripts` image with cleanup script, ensure `latest` tag is correctly applied. ([\#71](https://github.com/matrix-org/rageshake/issues/71)) 98 | - Update README.md to include json as a valid extension for file uploads. ([\#74](https://github.com/matrix-org/rageshake/issues/74)) 99 | 100 | 101 | 1.9.0 (2023-03-22) 102 | ================== 103 | 104 | VERSIONING NOTE: From this release onwards rageshake will be versioned in `x.y.z` format, not `x.y`. 105 | 106 | Features 107 | -------- 108 | 109 | - Add a zero-dependency python script to cleanup old rageshakes. ([\#61](https://github.com/matrix-org/rageshake/issues/61)) 110 | 111 | 112 | Internal Changes 113 | ---------------- 114 | 115 | - Update deployment process to automatically build docker containers and binaries. ([\#70](https://github.com/matrix-org/rageshake/issues/70)) 116 | 117 | 118 | 1.8 (2023-01-13) 119 | ================ 120 | 121 | Features 122 | -------- 123 | 124 | - Add config option to block unknown appplication names. ([\#67](https://github.com/matrix-org/rageshake/issues/67)) 125 | 126 | 127 | Internal Changes 128 | ---------------- 129 | 130 | - Reimplement buildkite linting and changelog in GHA. ([\#64](https://github.com/matrix-org/rageshake/issues/64)) 131 | 132 | 133 | 1.7 (2022-04-14) 134 | ================ 135 | 136 | Features 137 | -------- 138 | 139 | - Pass the prefix as a unique ID for the rageshake to the generic webhook mechanism. ([\#54](https://github.com/matrix-org/rageshake/issues/54)) 140 | 141 | 142 | 1.6 (2022-02-22) 143 | ================ 144 | 145 | Features 146 | -------- 147 | 148 | - Provide ?format=tar.gz option on directory listings to download tarball. ([\#53](https://github.com/matrix-org/rageshake/issues/53)) 149 | 150 | 151 | 1.5 (2022-02-08) 152 | ================ 153 | 154 | Features 155 | -------- 156 | 157 | - Allow upload of Files with a .json postfix. ([\#52](https://github.com/matrix-org/rageshake/issues/52)) 158 | 159 | 160 | 1.4 (2022-02-01) 161 | ================ 162 | 163 | Features 164 | -------- 165 | 166 | - Allow forwarding of a request to a webhook endpoint. ([\#50](https://github.com/matrix-org/rageshake/issues/50)) 167 | 168 | 169 | 1.3 (2022-01-25) 170 | ================ 171 | 172 | Features 173 | -------- 174 | 175 | - Add support for creating GitLab issues. Contributed by @tulir. ([\#37](https://github.com/matrix-org/rageshake/issues/37)) 176 | - Support element-android submitting logs with .gz suffix. ([\#40](https://github.com/matrix-org/rageshake/issues/40)) 177 | 178 | 179 | Bugfixes 180 | -------- 181 | 182 | - Prevent timestamp collisions when reports are submitted within 1 second of each other. ([\#39](https://github.com/matrix-org/rageshake/issues/39)) 183 | 184 | 185 | Internal Changes 186 | ---------------- 187 | 188 | - Update minimum Go version to 1.16. ([\#37](https://github.com/matrix-org/rageshake/issues/37), [\#42](https://github.com/matrix-org/rageshake/issues/42)) 189 | - Add documentation on the types and formats of files submitted to the rageshake server. ([\#44](https://github.com/matrix-org/rageshake/issues/44)) 190 | - Build and push a multi-arch Docker image on the GitHub Container Registry. ([\#47](https://github.com/matrix-org/rageshake/issues/47)) 191 | - Add a /health endpoint that always replies with a 200 OK. ([\#48](https://github.com/matrix-org/rageshake/issues/48)) 192 | 193 | 194 | 1.2 (2020-09-16) 195 | ================ 196 | 197 | Features 198 | -------- 199 | 200 | - Add email support. ([\#35](https://github.com/matrix-org/rageshake/issues/35)) 201 | 202 | 203 | 1.1 (2020-06-04) 204 | ================ 205 | 206 | Features 207 | -------- 208 | 209 | - Add support for Slack notifications. Contributed by @awesome-manuel. ([\#28](https://github.com/matrix-org/rageshake/issues/28)) 210 | 211 | 212 | Internal Changes 213 | ---------------- 214 | 215 | - Update minimum go version to 1.11. ([\#29](https://github.com/matrix-org/rageshake/issues/29), [\#30](https://github.com/matrix-org/rageshake/issues/30)) 216 | - Replace vendored libraries with `go mod`. ([\#31](https://github.com/matrix-org/rageshake/issues/31)) 217 | - Add Dockerfile. Contributed by @awesome-manuel. ([\#32](https://github.com/matrix-org/rageshake/issues/32)) 218 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Contributing code to rageshake 2 | ============================== 3 | 4 | Everyone is welcome to contribute code to rageshake, provided that they are 5 | willing to license their contributions under the same license as the project 6 | itself. We follow a simple 'inbound=outbound' model for contributions: the act 7 | of submitting an 'inbound' contribution means that the contributor agrees to 8 | license the code under the same terms as the project's overall 'outbound' 9 | license - in this case, Apache Software License v2 (see ``_). 10 | 11 | How to contribute 12 | ~~~~~~~~~~~~~~~~~ 13 | 14 | The preferred and easiest way to contribute changes to the project is to fork 15 | it on github, and then create a pull request to ask us to pull your changes 16 | into our repo (https://help.github.com/articles/using-pull-requests/). 17 | 18 | The workflow is that contributors should fork the master branch to make a 19 | 'feature' branch for a particular contribution, and then make a pull request to 20 | merge this back into the matrix.org 'official' master branch. We use github's 21 | pull request workflow to review the contribution, and either ask you to make 22 | any refinements needed or merge it and make them ourselves. 23 | 24 | We use Buildkite for continuous integration, and all pull requests get 25 | automatically tested: if your change breaks the build, then the PR will show 26 | that there are failed checks, so please check back after a few minutes. 27 | 28 | Code style 29 | ~~~~~~~~~~ 30 | 31 | Please ensure your changes match the cosmetic style of the existing project, 32 | and **never** mix cosmetic and functional changes in the same commit, as it 33 | makes it horribly hard to review otherwise. 34 | 35 | Attribution 36 | ~~~~~~~~~~~ 37 | 38 | Everyone who contributes anything to Matrix is welcome to be listed in the 39 | AUTHORS.rst file for the project in question. Please feel free to include a 40 | change to AUTHORS.rst in your pull request to list yourself and a short 41 | description of the area(s) you've worked on. Also, we sometimes have swag to 42 | give away to contributors - if you feel that Matrix-branded apparel is missing 43 | from your life, please mail us your shipping address to matrix at matrix.org 44 | and we'll try to fix it :) 45 | 46 | Sign off 47 | ~~~~~~~~ 48 | 49 | In order to have a concrete record that your contribution is intentional 50 | and you agree to license it under the same terms as the project's license, we've adopted the 51 | same lightweight approach that the Linux Kernel 52 | (https://www.kernel.org/doc/Documentation/SubmittingPatches), Docker 53 | (https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other 54 | projects use: the DCO (Developer Certificate of Origin: 55 | http://developercertificate.org/). This is a simple declaration that you wrote 56 | the contribution or otherwise have the right to contribute it to Matrix:: 57 | 58 | Developer Certificate of Origin 59 | Version 1.1 60 | 61 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 62 | 660 York Street, Suite 102, 63 | San Francisco, CA 94110 USA 64 | 65 | Everyone is permitted to copy and distribute verbatim copies of this 66 | license document, but changing it is not allowed. 67 | 68 | Developer's Certificate of Origin 1.1 69 | 70 | By making a contribution to this project, I certify that: 71 | 72 | (a) The contribution was created in whole or in part by me and I 73 | have the right to submit it under the open source license 74 | indicated in the file; or 75 | 76 | (b) The contribution is based upon previous work that, to the best 77 | of my knowledge, is covered under an appropriate open source 78 | license and I have the right under that license to submit that 79 | work with modifications, whether created in whole or in part 80 | by me, under the same open source license (unless I am 81 | permitted to submit under a different license), as indicated 82 | in the file; or 83 | 84 | (c) The contribution was provided directly to me by some other 85 | person who certified (a), (b) or (c) and I have not modified 86 | it. 87 | 88 | (d) I understand and agree that this project and the contribution 89 | are public and that a record of the contribution (including all 90 | personal information I submit with it, including my sign-off) is 91 | maintained indefinitely and may be redistributed consistent with 92 | this project or the open source license(s) involved. 93 | 94 | If you agree to this for your contribution, then all that's needed is to 95 | include the line in your commit or pull request comment:: 96 | 97 | Signed-off-by: Your Name 98 | 99 | ...using your real name; unfortunately pseudonyms and anonymous contributions 100 | can't be accepted. Git makes this trivial - just use the -s flag when you do 101 | ``git commit``, having first set ``user.name`` and ``user.email`` git configs 102 | (which you should have done anyway :) 103 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG GO_VERSION=1.17 2 | ARG DEBIAN_VERSION=11 3 | ARG DEBIAN_VERSION_NAME=bullseye 4 | 5 | ## Build stage ## 6 | FROM --platform=${BUILDPLATFORM} docker.io/library/golang:${GO_VERSION}-${DEBIAN_VERSION_NAME} AS builder 7 | 8 | WORKDIR /build 9 | COPY go.mod go.sum ./ 10 | RUN go mod download 11 | 12 | COPY . . 13 | RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o rageshake 14 | 15 | ## Runtime stage, python scripts ## 16 | FROM python:3-slim AS scripts 17 | COPY scripts/cleanup.py /cleanup.py 18 | WORKDIR / 19 | 20 | ## Runtime stage, debug variant ## 21 | FROM --platform=${TARGETPLATFORM} gcr.io/distroless/static-debian${DEBIAN_VERSION}:debug-nonroot AS debug 22 | COPY --from=builder /build/rageshake /rageshake 23 | WORKDIR / 24 | EXPOSE 9110 25 | ENTRYPOINT ["/rageshake"] 26 | 27 | ## Runtime stage ## 28 | FROM --platform=${TARGETPLATFORM} gcr.io/distroless/static-debian${DEBIAN_VERSION}:nonroot 29 | COPY --from=builder /build/rageshake /rageshake 30 | WORKDIR / 31 | EXPOSE 9110 32 | ENTRYPOINT ["/rageshake"] 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rageshake ![Build status](https://github.com/matrix-org/rageshake/actions/workflows/build.yaml/badge.svg) 2 | 3 | Web service which collects and serves bug reports. 4 | 5 | rageshake requires Go version 1.16 or later. 6 | 7 | To run it, do: 8 | 9 | ``` 10 | go build 11 | ./bin/rageshake 12 | ``` 13 | 14 | Optional parameters: 15 | 16 | * `-config `: The path to a YAML config file; see 17 | [rageshake.sample.yaml](rageshake.sample.yaml) for more information. 18 | * `-listen
`: TCP network address to listen for HTTP requests 19 | on. Example: `:9110`. 20 | 21 | ## Issue template 22 | 23 | It is possible to override the templates used to construct emails, and Github and Gitlab issues. 24 | See [templates/README.md](templates/README.md) for more information. 25 | 26 | ## API Documentation 27 | 28 | See [docs/api.md](docs/api.md) for more information. 29 | 30 | ## Data stored on disk 31 | 32 | Each request to `POST /api/submit` results in data being written to the local disk. 33 | A new directory is created within `./bugs` (relative to the working directory of the `rageshake` server) for 34 | each submission; within that directory is created: 35 | * Any log files attached to the submission, named as chosen by the client (provided the name is moderately sensible), 36 | and gzipped. 37 | * `details.log.gz`: a gzipped text file giving metadata about the submission, in an undocumented format. Now 38 | deprecated, but retained for backwards compatibility with existing tooling. 39 | * `details.json`: Metadata about the submission, in the same format as submitted to the 40 | [generic webhooks](./docs/generic_webhook.md). 41 | 42 | ## Notifications 43 | 44 | You can get notifications when a new rageshake arrives on the server. 45 | 46 | Currently this tool supports pushing notifications as GitHub issues in a repo, 47 | through a Slack webhook or by email, cf sample config file for how to 48 | configure them. 49 | 50 | ### Generic Webhook Notifications 51 | 52 | You can receive a webhook notifications when a new rageshake arrives on the server. 53 | 54 | These requests contain all the parsed metadata, and links to the uploaded files, and any github/gitlab 55 | issues created. 56 | 57 | Details on the request and expected response are [available](docs/generic\_webhook.md). 58 | 59 | 60 | ## Cleanup script 61 | 62 | A python script is provided in scripts/cleanup.py and in a 63 | [docker container](https://github.com/orgs/matrix-org/packages/container/package/rageshake%2Fscripts). 64 | It can be configured using the commandline options available via `cleaup.py --help`. 65 | 66 | It can either be run via a cronjob at appropriate intervals (typically daily), or 67 | be set to run in a continual mode with something like `--repeat-delay-hours 24` 68 | to repeat running after approximately 24 hours. 69 | 70 | Note that this script will scan all logs older than the smallest configured retention period, 71 | up to the limit specified by `--max-days` or each of the days in `--days-to-check`. 72 | This can be an IO and CPU intensive process if a large number of files are scanned. 73 | 74 | -------------------------------------------------------------------------------- /RELEASING.md: -------------------------------------------------------------------------------- 1 | 1. Set a variable to the version number for convenience: 2 | ```sh 3 | ver=x.y.z 4 | ``` 5 | 1. Update the changelog: 6 | ```sh 7 | pip3 install --pre 'towncrier~=21.9' 8 | towncrier build --version=$ver 9 | ``` 10 | 1. Push your changes: 11 | ```sh 12 | git add -u && git commit -m $ver && git push 13 | ``` 14 | 1. Sanity-check the 15 | [changelog](https://github.com/matrix-org/rageshake/blob/master/CHANGES.md) 16 | and update if need be. 17 | 1. Create release on GH project page: 18 | ```sh 19 | xdg-open https://github.com/matrix-org/rageshake/releases/new?tag=v$ver&title=v$ver 20 | ``` 21 | Describe the release based on the changelog. 22 | 23 | This will trigger a docker image to be built as well as a binary to be uploaded to the release 24 | 1. Check that the docker image has been created and tagged (a few mins) 25 | ``` 26 | xdg-open https://github.com/matrix-org/rageshake/pkgs/container/rageshake/versions?filters%5Bversion_type%5D=tagged 27 | ``` 28 | 1. Check that the rageshake binary has been built and added to the release (a few mins) 29 | ``` 30 | xdg-open https://github.com/matrix-org/rageshake/releases 31 | ``` 32 | -------------------------------------------------------------------------------- /changelog.d/.gitignore: -------------------------------------------------------------------------------- 1 | !.gitignore 2 | -------------------------------------------------------------------------------- /changelog.d/38.bugfix: -------------------------------------------------------------------------------- 1 | Fix email support to support authenticated mail with a custom port. 2 | -------------------------------------------------------------------------------- /docker-bake.hcl: -------------------------------------------------------------------------------- 1 | // This is what is baked by GitHub Actions 2 | group "default" { targets = ["regular", "debug", "scripts"] } 3 | 4 | // Targets filled by GitHub Actions for each tag 5 | target "docker-metadata-action" {} 6 | target "docker-metadata-action-debug" {} 7 | target "docker-metadata-action-scripts" {} 8 | 9 | // This sets the platforms and is further extended by GitHub Actions to set the 10 | // output and the cache locations 11 | target "base" { 12 | platforms = [ 13 | "linux/amd64", 14 | "linux/arm64", 15 | "linux/arm", 16 | ] 17 | } 18 | 19 | target "regular" { 20 | inherits = ["base", "docker-metadata-action"] 21 | } 22 | 23 | target "debug" { 24 | inherits = ["base", "docker-metadata-action-debug"] 25 | target = "debug" 26 | } 27 | 28 | target "scripts" { 29 | inherits = ["base", "docker-metadata-action-scripts"] 30 | target = "scripts" 31 | } 32 | -------------------------------------------------------------------------------- /docs/api.md: -------------------------------------------------------------------------------- 1 | ## HTTP endpoints 2 | 3 | The following HTTP endpoints are exposed: 4 | 5 | ### GET `/api/listing/` 6 | 7 | Serves submitted bug reports. Protected by basic HTTP auth using the 8 | username/password provided in the environment. A browsable list, collated by 9 | report submission date and time. 10 | 11 | A whole directory can be downloaded as a tarball by appending the parameter `?format=tar.gz` to the end of the URL path 12 | 13 | ### POST `/api/submit` 14 | 15 | Submission endpoint: this is where applications should send their reports. 16 | 17 | The body of the request should be a multipart form-data submission, with the 18 | following form field names. (For backwards compatibility, it can also be a JSON 19 | object, but multipart is preferred as it allows more efficient transfer of the 20 | logs.) 21 | 22 | * `text`: A textual description of the problem. Included in the 23 | `details.log.gz` file. 24 | 25 | * `user_agent`: Application user-agent. Included in the `details.log.gz` file. 26 | 27 | * `app`: Identifier for the application (eg 'riot-web'). Should correspond to a 28 | mapping configured in the configuration file for github issue reporting to 29 | work. 30 | 31 | * `version`: Application version. Included in the `details.log.gz` file. 32 | 33 | * `label`: Label to attach to the github issue, and include in the details file. 34 | 35 | If using the JSON upload encoding, this should be encoded as a `labels` field, 36 | whose value should be a list of strings. 37 | 38 | * `log`: a log file, with lines separated by newline characters. Multiple log 39 | files can be included by including several `log` parts. 40 | 41 | If the log is uploaded with a filename `name.ext`, where `name` contains only 42 | alphanumerics, `.`, `-` or `_`, and `ext` is one of `log` or `txt`, then the 43 | file saved to disk is based on that. Otherwise, a suitable name is 44 | constructed. 45 | 46 | If using the JSON upload encoding, the request object should instead include 47 | a single `logs` field, which is an array of objects with the following 48 | fields: 49 | 50 | * `id`: textual identifier for the logs. Used as the filename, as above. 51 | * `lines`: log data. Newlines should be encoded as `\n`, as normal in JSON). 52 | 53 | A summary of the current log file formats that are uploaded for `log` and 54 | `compressed-log` is [available](docs/submitted_reports.md). 55 | 56 | * `compressed-log`: a gzipped logfile. Decompressed and then treated the same as 57 | `log`. 58 | 59 | Compressed logs are not supported for the JSON upload encoding. 60 | 61 | A summary of the current log file formats that are uploaded for `log` and 62 | `compressed-log` is [available](docs/submitted_reports.md). 63 | 64 | * `file`: an arbitrary file to attach to the report. Saved as-is to disk, and 65 | a link is added to the github issue. The filename must be in the format 66 | `name.ext`, where `name` contains only alphanumerics, `-` or `_`, and `ext` 67 | is one of `jpg`, `png`, `txt`, `json`, `txt.gz` or `json.gz`. 68 | 69 | Not supported for the JSON upload encoding. 70 | 71 | * Any other form field names are interpreted as arbitrary name/value strings to 72 | include in the `details.log.gz` file. 73 | 74 | If using the JSON upload encoding, this additional metadata should insted be 75 | encoded as a `data` field, whose value should be a JSON map. (Note that the 76 | values must be strings; numbers, objects and arrays will be rejected.) 77 | 78 | The response (if successful) will be a JSON object with the following fields: 79 | 80 | * `report_url`: A URL where the user can track their bug report. Omitted if 81 | issue submission was disabled. 82 | 83 | ## Error responses 84 | 85 | The rageshake server will respond with a specific JSON payload when encountering an error. 86 | 87 | ```json 88 | { 89 | "error": "A human readable error string.", 90 | "errcode": "UNKNOWN", 91 | "policy_url": "https://github.com/matrix-org/rageshake/blob/master/docs/blocked_rageshake.md" 92 | } 93 | ``` 94 | 95 | Where the fields are as follows: 96 | 97 | - `error` is an error string to explain the error, in English. 98 | - `errcode` is a machine readable error code which can be used by clients to give a localized error. 99 | - `policy_url` is an optional URL that links to a reference document, which may be presented to users. 100 | 101 | ### Error codes 102 | 103 | - `UNKNOWN` is a catch-all error when the appliation does not have a specific error. 104 | - `METHOD_NOT_ALLOWED` is reported when you have used the wrong method for an endpoint. E.g. GET instead of POST. 105 | - `DISALLOWED_APP` is reported when a report was rejected due to the report being sent from an unsupported 106 | app (see the `allowed_app_names` config option). 107 | - `BAD_HEADER` is reported when a header was not able to be parsed, such as `Content-Length`. 108 | - `CONTENT_TOO_LARGE` is reported when the reported content size is too large. 109 | - `BAD_CONTENT` is reported when the reports content could not be parsed. 110 | - `REJECTED` is reported when the submission could be understood but was rejected by `rejection_conditions`. 111 | This is the default value, see below for more information. 112 | 113 | In addition to these error codes, the configuration allows application developers to specify specific error codes 114 | for report rejection under the `REJECTED_*` namespace. (see the `rejection_conditions` config option). Consult the 115 | administrator of your rageshake server in order to determine what error codes may be presented. 116 | -------------------------------------------------------------------------------- /docs/blocked_rageshake.md: -------------------------------------------------------------------------------- 1 | # Rageshake server not accepting rageshakes 2 | 3 | This page contains information useful to someone who has had their rageshake rejected by a rageshake server. 4 | 5 | We include it within the error messages to provide a place with context for users reading the error message and wanting 6 | to know more. 7 | 8 | ## For matrix client users 9 | 10 | Thank you for attempting to report a bug with your matrix client; unfortunately your client application is likely incorrectly configured. 11 | 12 | The rageshake server you attempted to upload a report to is not accepting rageshakes from your client at this time. 13 | 14 | Generally, the developers who run a rageshake server will only be able to handle reports for applications they are developing, 15 | and your application is not listed as one of those applications. 16 | 17 | The rageshake server could also be rejecting the text you wrote in your bug report because its content matches a rejection rule. This usually happens to prevent you from disclosing private information in the bug report itself. 18 | 19 | Please contact the distributor of your application or the administrator of the web site you visit to report this as a problem. 20 | 21 | ## For developers of matrix clients 22 | 23 | Your application is likely based on one of the matrix SDKs or element applications, if it is submitting rageshakes to a rageshake server. 24 | 25 | A change has been made to pre-filter reports that the developers using this rageshake server for applications they do not have control over. 26 | Typically reports from unknown applications would have to be manually triaged and discarded; there is now automatic filtering in place, which reduces overall effort. 27 | 28 | There is generally a configuration file in your application that you can alter to change where these reports are sent, which may require rebuilding and releasing the client. 29 | 30 | The easiest solution to this error is to stop sending rageshakes entirely, which may require a code or configuration change in your client. 31 | 32 | However, if you wish to accept bug reports from your users applications, you will need to run your own copy of this rageshake server and update the URL appropriately. 33 | 34 | ## Application specific config locations: 35 | * element-web: `bug_report_endpoint_url` in the [sample configuration for element web](https://github.com/vector-im/element-web/blob/develop/config.sample.json). 36 | * element-ios: `bugReportEndpointUrlString` in the [BuildSettings.swift](https://github.com/vector-im/element-ios/blob/develop/Config/BuildSettings.swift) 37 | * element-android: `bug_report_url` in the [config.xml file for the build](https://github.com/vector-im/element-android/blob/develop/vector-config/src/main/res/values/config.xml) 38 | -------------------------------------------------------------------------------- /docs/generic_webhook.md: -------------------------------------------------------------------------------- 1 | ## Generic webhook request 2 | 3 | If the configuration option `generic_webhook_urls` is set, then an asynchronous request to 4 | each endpoint listed will be sent in parallel, after the incoming request is parsed and the 5 | files are uploaded. 6 | 7 | The webhook is designed for notification or other tracking services, and does not contain 8 | the original log files uploaded. 9 | 10 | (If you want the original log files, we suggest to implement the rageshake interface itself). 11 | 12 | A sample JSON body is as follows: 13 | 14 | ```json5 15 | { 16 | // Unique ID for this rageshake. 17 | "id": "2022-01-25/154742-OOXBVGIX", 18 | // The time this rageshake was submitted, in milliseconds past the epoch 19 | "create_time": 1643125662456, 20 | "user_text": "test\r\n\r\nIssue: No issue link given", 21 | "app": "element-web", 22 | "data": { 23 | "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0", 24 | "Version": "0f15ba34cdf5-react-0f15ba34cdf5-js-0f15ba34cdf5", 25 | // ... 26 | "user_id": "@michaelgoesforawalk:matrix.org"}, 27 | "labels": null, 28 | "logs": [ 29 | "logs-0000.log.gz", 30 | "logs-0001.log.gz", 31 | "logs-0002.log.gz", 32 | ], 33 | "logErrors": null, 34 | "files": [ 35 | "screenshot.png" 36 | ], 37 | "fileErrors": null, 38 | "report_url": "https://github.com/your-org/your-repo/issues/1251", 39 | "listing_url": "http://your-rageshake-server/api/listing/2022-01-25/154742-OOXBVGIX" 40 | } 41 | ``` 42 | 43 | The log and other files can be individually downloaded by concatenating the `listing_url` and the `logs` or `files` name. 44 | You may need to provide a HTTP basic auth user/pass if configured on your rageshake server. 45 | -------------------------------------------------------------------------------- /docs/submitted_reports.md: -------------------------------------------------------------------------------- 1 | # Common report styles 2 | 3 | Rageshakes can come from a number of applications, and we provide some practical notes on the generated format. 4 | 5 | At present these should not be considered absolute nor a structure to follow; but an attempt to document the currently visible formats as of January 2022. 6 | 7 | ## Feedback 8 | 9 | Log files are not transmitted; the main feedback is entirely within the user message body. 10 | 11 | This occurs from all platforms. 12 | 13 | ## Element Web / Element Desktop 14 | 15 | Log files are transmitted in reverse order (0000 is the youngest) 16 | 17 | Log line format: 18 | ``` 19 | 2022-01-17T14:57:20.806Z I Using WebAssembly Olm 20 | < ---- TIMESTAMP ------> L <-- Message ---- 21 | 22 | L = log level, (W=Warn, I=Info, etc) 23 | ``` 24 | 25 | New log files are started each restart of the app, but some log files may not contain all data from the start of the session. 26 | 27 | ## Element iOS 28 | 29 | Crash Log is special and is sent only once (and deleted on the device afterwards) 30 | 31 | `crash.log` 32 | 33 | Following logs are available, going back in time with ascending number. 34 | console.log with no number is the current log file. 35 | ``` 36 | console.log (newest) 37 | console-1.log 38 | ... 39 | console-49.log (oldest) 40 | 41 | console-nse.log (newest) 42 | console-nse-1.log 43 | ... 44 | console-nse-49.log (oldest) 45 | 46 | console-share.log (newest) 47 | console-share-1.log 48 | console-share-49.log (oldest) 49 | ``` 50 | 51 | ## Element Android 52 | 53 | There is a historical issue with the naming of files, documented in [issue #40](https://github.com/matrix-org/rageshake/issues/40). 54 | 55 | Log file 0000 is odd, it contains the logcat data if sent. 56 | 57 | Log line format: 58 | ``` 59 | 01-17 14:59:30.657 14303 14303 W Activity: Slow Operation: 60 | <-- TIMESTAMP ---> <-P-> <-T-> L <-- Message -- 61 | 62 | L = Log Level (W=Warn, I=Info etc) 63 | P = Process ID 64 | T = Thread ID 65 | ``` 66 | Remaining log files are transmitted according to their position in the round-robin logging to file - there will be (up to) 7 files written to in a continious loop; one of the seven will be the oldest, the rest will be in order. 67 | 68 | Log line format: 69 | ``` 70 | 2022-01-17T13:06:36*838GMT+00:00Z 12226 D/ /Tag: Migration: Importing legacy session 71 | < ---- TIMESTAMP ---------------> <-P-> L <-- Message ---- 72 | 73 | L = log level, (W=Warn, I=Info, etc) 74 | P = Process ID 75 | ``` 76 | 77 | Once the fix to #40 is in place, we will see the following files: 78 | 79 | ``` 80 | logcatError.log 81 | logcat.log 82 | crash.log 83 | keyrequests.log 84 | log-[1-7].log 85 | ``` 86 | 87 | Log 1-7 are logs from a round-robin buffer and are ordered but the start point is undefined 88 | 89 | ## Third Room 90 | 91 | Third Room submits logs as a single `logs.json` file. 92 | 93 | `logs.json` file format: 94 | ``` 95 | { 96 | "formatVersion": 1, 97 | "appVersion": "0.0.0", 98 | "platform": "", 99 | "items": [LogItem, LogItem, ...] 100 | } 101 | ``` 102 | 103 | Each `LogItem` is an object with the following properties: 104 | | Name | Type | Description | 105 | |------|-----------|---------------------------------------------------------------------------------------------------------------| 106 | | `s` | `number` | **Required**: Start timestamp (in milliseconds since the unix epoch) on client when this log item is created. | 107 | | `d` | `number` | Log item active duration (in milliseconds). | 108 | | `v` | `object` | **Required**: Value of log item. | 109 | | `l` | `number` | **Required**: Log level assigned to by the client. | 110 | | `f` | `boolean` | Force flag. | 111 | | `c` | `array` | An array containing child log items. | 112 | | `e` | `object` | Error object with **required** `name`, `message` and *optional* `stack` properties of `string` type. | 113 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package main 2 | const ( 3 | // ErrCodeBadContent is reported when the reports content could not be parsed. 4 | ErrCodeBadContent = "BAD_CONTENT"; 5 | // ErrCodeBadHeader is reported when a header was not able to be parsed. 6 | ErrCodeBadHeader = "BAD_HEADER"; 7 | // ErrCodeContentTooLarge is reported when the reported content size is too large. 8 | ErrCodeContentTooLarge = "CONTENT_TOO_LARGE"; 9 | // ErrCodeDisallowedApp is reported when a report was rejected due to the report being sent from an unsupported 10 | ErrCodeDisallowedApp = "DISALLOWED_APP"; 11 | // ErrCodeMethodNotAllowed is reported when you have used the wrong method for an endpoint. 12 | ErrCodeMethodNotAllowed = "METHOD_NOT_ALLOWED"; 13 | // ErrCodeRejected is reported when the submission could be understood but was rejected by RejectionConditions. 14 | ErrCodeRejected = "REJECTED"; 15 | // ErrCodeUnknown is a catch-all error when the appliation does not have a specific error. 16 | ErrCodeUnknown = "UNKNOWN"; 17 | ) -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/matrix-org/rageshake 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/google/go-github v0.0.0-20170401000335-12363ffc1001 7 | github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible 8 | github.com/ua-parser/uap-go v0.0.0-20241012191800-bbb40edc15aa // indirect 9 | github.com/xanzy/go-gitlab v0.50.2 10 | golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 11 | gopkg.in/yaml.v2 v2.2.8 12 | ) 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 5 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 6 | github.com/google/go-github v0.0.0-20170401000335-12363ffc1001 h1:OK4gfzCBCtPg14E4sYsczwFhjVu1jQJZI+OEOpiTigw= 7 | github.com/google/go-github v0.0.0-20170401000335-12363ffc1001/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= 8 | github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= 9 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 10 | github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= 11 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 12 | github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= 13 | github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 14 | github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= 15 | github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= 16 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 17 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 18 | github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible h1:d60x4RsAHk/UX/0OT8Gc6D7scVvhBbEANpTAWrDhA/I= 19 | github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A= 20 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 21 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 22 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 23 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 24 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 25 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 26 | github.com/ua-parser/uap-go v0.0.0-20241012191800-bbb40edc15aa h1:VzPR4xFM7HARqNocjdHg75ZL9SAgFtaF3P57ZdDcG6I= 27 | github.com/ua-parser/uap-go v0.0.0-20241012191800-bbb40edc15aa/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= 28 | github.com/xanzy/go-gitlab v0.50.2 h1:Qm/um2Jryuqusc6VmN7iZYVTQVzNynzSiuMJDnCU1wE= 29 | github.com/xanzy/go-gitlab v0.50.2/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= 30 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 31 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 32 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 33 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 34 | golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= 35 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 36 | golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 h1:JIqe8uIcRBHXDQVvZtHwp80ai3Lw3IJAeJEs55Dc1W0= 37 | golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 38 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= 39 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 40 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 41 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 42 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 43 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 44 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 45 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= 46 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 47 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 48 | google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk= 49 | google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 50 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 51 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 52 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 53 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 54 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 55 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 56 | -------------------------------------------------------------------------------- /hooks/install.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | DOT_GIT="$(dirname $0)/../.git" 4 | 5 | ln -s "../../hooks/pre-commit" "$DOT_GIT/hooks/pre-commit" -------------------------------------------------------------------------------- /hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | # make git_dir and GIT_INDEX_FILE absolute, before we change dir 6 | # 7 | # (don't actually set GIT_DIR, because it messes up `go get`, and several of 8 | # the go commands run `go get` indirectly) 9 | # 10 | git_dir=$(readlink -f `git rev-parse --git-dir`) 11 | if [ -n "${GIT_INDEX_FILE:+x}" ]; then 12 | export GIT_INDEX_FILE=$(readlink -f "$GIT_INDEX_FILE") 13 | fi 14 | 15 | wd=`pwd` 16 | 17 | # create a temp dir. The `trap` incantation will ensure that it is removed 18 | # again when this script completes. 19 | tmpdir=`mktemp -d` 20 | trap 'rm -rf "$tmpdir"' EXIT 21 | cd "$tmpdir" 22 | 23 | # get a clean copy of the index (ie, what has been `git add`ed), so that we can 24 | # run the checks against what we are about to commit, rather than what is in 25 | # the working copy. 26 | git --git-dir="${git_dir}" checkout-index -a 27 | 28 | # run our checks 29 | go fmt 30 | ./scripts/lint.sh 31 | go test 32 | 33 | # we're done with go so can set GIT_DIR 34 | export GIT_DIR="$git_dir" 35 | 36 | # if there are no changes from the index, we are done 37 | git diff --quiet && exit 0 38 | 39 | # we now need to apply any changes made to both the index and the working copy. 40 | # so, first get a patch 41 | git diff > "$GIT_DIR/pre-commit.patch" 42 | 43 | # add the changes to the index 44 | git add -u 45 | 46 | # attempt to apply the changes to the wc (but don't fail the commit for it if 47 | # there are conflicts). 48 | cd "$wd" 49 | git apply "$GIT_DIR/pre-commit.patch" 2>/dev/null && 50 | rm "$GIT_DIR/pre-commit.patch" || 51 | echo "warning: unable to apply changes from commit hook to working copy; patch is in $GIT_DIR/pre-commit.patch" >&2 52 | -------------------------------------------------------------------------------- /logserver.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Vector Creations Ltd 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "archive/tar" 21 | "compress/gzip" 22 | "io" 23 | "log" 24 | "net/http" 25 | "os" 26 | "path" 27 | "path/filepath" 28 | "strconv" 29 | "strings" 30 | ) 31 | 32 | // logServer is an http.handler which will serve up bugreports 33 | type logServer struct { 34 | root string 35 | } 36 | 37 | func (f *logServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { 38 | upath := r.URL.Path 39 | 40 | if !strings.HasPrefix(upath, "/") { 41 | upath = "/" + upath 42 | r.URL.Path = upath 43 | } 44 | 45 | log.Println("Serving", upath) 46 | 47 | // eliminate ., .., //, etc 48 | upath = path.Clean(upath) 49 | 50 | // reject some dodgy paths. This is based on the code for http.Dir.Open (see https://golang.org/src/net/http/fs.go#L37). 51 | // 52 | // the check for '..' is a sanity-check because my understanding of `path.Clean` is that it should never return 53 | // a value including '..' for input starting with '/'. It's taken from the code for http.ServeFile 54 | // (https://golang.org/src/net/http/fs.go#L637). 55 | if containsDotDot(upath) || strings.Contains(upath, "\x00") || (filepath.Separator != '/' && strings.IndexRune(upath, filepath.Separator) >= 0) { 56 | http.Error(w, "invalid URL path", http.StatusBadRequest) 57 | return 58 | } 59 | 60 | // convert to abs path 61 | upath, err := filepath.Abs(filepath.Join(f.root, filepath.FromSlash(upath))) 62 | 63 | if err != nil { 64 | msg, code := toHTTPError(err) 65 | http.Error(w, msg, code) 66 | return 67 | } 68 | 69 | serveFile(w, r, upath) 70 | } 71 | 72 | func serveFile(w http.ResponseWriter, r *http.Request, path string) { 73 | d, err := os.Stat(path) 74 | if err != nil { 75 | msg, code := toHTTPError(err) 76 | http.Error(w, msg, code) 77 | return 78 | } 79 | 80 | // for anti-XSS belt-and-braces, set a very restrictive CSP 81 | w.Header().Set("Content-Security-Policy", "default-src: none") 82 | 83 | // if it's a directory, serve a listing or a tarball 84 | if d.IsDir() { 85 | serveDirectory(w, r, path) 86 | return 87 | } 88 | 89 | // if it's a gzipped log file, serve it as text 90 | if strings.HasSuffix(path, ".gz") { 91 | serveGzippedFile(w, r, path, d.Size()) 92 | return 93 | } 94 | 95 | // otherwise, limit ourselves to a number of known-safe content-types, to 96 | // guard against XSS vulnerabilities. 97 | // http.serveFile preserves the content-type header if one is already set. 98 | w.Header().Set("Content-Type", extensionToMimeType(path)) 99 | 100 | http.ServeFile(w, r, path) 101 | } 102 | 103 | // extensionToMimeType returns a suitable mime type for the given filename 104 | // 105 | // Unlike mime.TypeByExtension, the results are limited to a set of types which 106 | // should be safe to serve to a browser without introducing XSS vulnerabilities. 107 | // 108 | // We handle all of the extensions we allow on files uploaded as attachments to a rageshake, 109 | // plus 'log' which we do not allow as an attachment, but is used as the extension when serving 110 | // the logs submitted as `logs` or `compressed-log`. 111 | func extensionToMimeType(path string) string { 112 | if strings.HasSuffix(path, ".txt") || strings.HasSuffix(path, ".log") { 113 | // anyone uploading text in anything other than utf-8 needs to be 114 | // re-educated. 115 | return "text/plain; charset=utf-8" 116 | } 117 | 118 | if strings.HasSuffix(path, ".png") { 119 | return "image/png" 120 | } 121 | 122 | if strings.HasSuffix(path, ".jpg") { 123 | return "image/jpeg" 124 | } 125 | 126 | if strings.HasSuffix(path, ".json") { 127 | return "application/json" 128 | } 129 | return "application/octet-stream" 130 | } 131 | 132 | // Chooses to serve either a directory listing or tarball based on the 'format' parameter. 133 | func serveDirectory(w http.ResponseWriter, r *http.Request, path string) { 134 | format, _ := r.URL.Query()["format"] 135 | if len(format) == 1 && format[0] == "tar.gz" { 136 | log.Println("Serving tarball of", path) 137 | err := serveTarball(w, r, path) 138 | if err != nil { 139 | msg, code := toHTTPError(err) 140 | http.Error(w, msg, code) 141 | log.Println("Error", err) 142 | } 143 | return 144 | } 145 | log.Println("Serving directory listing of", path) 146 | http.ServeFile(w, r, path) 147 | } 148 | 149 | // Streams a dynamically created tar.gz file with the contents of the given directory 150 | // Will serve a partial, corrupted response if there is a error partway through the 151 | // operation as we stream the response. 152 | // 153 | // The resultant tarball will contain a single directory containing all the files 154 | // so it can unpack cleanly without overwriting other files. 155 | // 156 | // Errors are only returned if generated before the tarball has started being 157 | // written to the ResponseWriter 158 | func serveTarball(w http.ResponseWriter, r *http.Request, dir string) error { 159 | directory, err := os.Open(dir) 160 | if err != nil { 161 | return err 162 | } 163 | 164 | // Creates a "disposition filename" 165 | // Take a URL.path like `/2022-01-10/184843-BZZXEGYH/` 166 | // and removes leading and trailing `/` and replaces internal `/` with `_` 167 | // to form a suitable filename for use in the content-disposition header 168 | // dfilename would turn into `2022-01-10_184843-BZZXEGYH` 169 | dfilename := strings.Trim(r.URL.Path, "/") 170 | dfilename = strings.Replace(dfilename, "/", "_", -1) 171 | 172 | // There is no application/tgz or similar; return a gzip file as best option. 173 | // This tends to trigger archive type tools, which will then use the filename to 174 | // identify the contents correctly. 175 | w.Header().Set("Content-Type", "application/gzip") 176 | w.Header().Set("Content-Disposition", "attachment; filename="+dfilename+".tar.gz") 177 | 178 | files, err := directory.Readdir(-1) 179 | if err != nil { 180 | return err 181 | } 182 | 183 | gzip := gzip.NewWriter(w) 184 | defer gzip.Close() 185 | targz := tar.NewWriter(gzip) 186 | defer targz.Close() 187 | 188 | for _, file := range files { 189 | if file.IsDir() { 190 | // We avoid including nested directories 191 | // This will result in requests for directories with only directories in 192 | // to return an empty tarball instead of recursively including directories. 193 | // This helps the server remain performant as a download of 'everything' would be slow 194 | continue 195 | } 196 | path := dir + "/" + file.Name() 197 | // We use the existing disposition filename to create a base directory structure for the files 198 | // so when they are unpacked, they are grouped in a unique folder on disk 199 | err := addToArchive(targz, dfilename, path) 200 | if err != nil { 201 | // From this point we assume that data may have been sent to the client already. 202 | // We therefore do not http.Error() after this point, instead closing the stream and 203 | // allowing the client to deal with a partial file as if there was a network issue. 204 | log.Println("Error streaming tarball", err) 205 | return nil 206 | } 207 | } 208 | return nil 209 | } 210 | 211 | // Add a single file into the archive. 212 | func addToArchive(targz *tar.Writer, dfilename string, filename string) error { 213 | file, err := os.Open(filename) 214 | if err != nil { 215 | return err 216 | } 217 | defer file.Close() 218 | 219 | info, err := file.Stat() 220 | if err != nil { 221 | return err 222 | } 223 | 224 | header, err := tar.FileInfoHeader(info, info.Name()) 225 | if err != nil { 226 | return err 227 | } 228 | header.Name = dfilename + "/" + info.Name() 229 | 230 | err = targz.WriteHeader(header) 231 | if err != nil { 232 | return err 233 | } 234 | 235 | _, err = io.Copy(targz, file) 236 | if err != nil { 237 | return err 238 | } 239 | return nil 240 | } 241 | 242 | func serveGzippedFile(w http.ResponseWriter, r *http.Request, path string, size int64) { 243 | cType := "text/plain; charset=utf-8" 244 | if strings.HasSuffix(path, ".gz") { 245 | // Guess the mime type from the extension as we do in serveFile, but without 246 | // the .gz header (in practice, either plain text or application/json). 247 | cType = extensionToMimeType(path[:len(path)-len(".gz")]) 248 | } 249 | w.Header().Set("Content-Type", cType) 250 | 251 | acceptsGzip := false 252 | splitRune := func(s rune) bool { return s == ' ' || s == '\t' || s == '\n' || s == ',' } 253 | for _, hdr := range r.Header["Accept-Encoding"] { 254 | for _, enc := range strings.FieldsFunc(hdr, splitRune) { 255 | if enc == "gzip" { 256 | acceptsGzip = true 257 | break 258 | } 259 | } 260 | } 261 | 262 | if acceptsGzip { 263 | serveGzip(w, r, path, size) 264 | } else { 265 | serveUngzipped(w, r, path) 266 | } 267 | } 268 | 269 | // serveGzip serves a gzipped file with gzip content-encoding 270 | func serveGzip(w http.ResponseWriter, r *http.Request, path string, size int64) { 271 | f, err := os.Open(path) 272 | if err != nil { 273 | msg, code := toHTTPError(err) 274 | http.Error(w, msg, code) 275 | return 276 | } 277 | defer f.Close() 278 | 279 | w.Header().Set("Content-Encoding", "gzip") 280 | w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) 281 | 282 | w.WriteHeader(http.StatusOK) 283 | io.Copy(w, f) 284 | } 285 | 286 | // serveUngzipped ungzips a gzipped file and serves it 287 | func serveUngzipped(w http.ResponseWriter, r *http.Request, path string) { 288 | f, err := os.Open(path) 289 | if err != nil { 290 | msg, code := toHTTPError(err) 291 | http.Error(w, msg, code) 292 | return 293 | } 294 | defer f.Close() 295 | 296 | gz, err := gzip.NewReader(f) 297 | if err != nil { 298 | msg, code := toHTTPError(err) 299 | http.Error(w, msg, code) 300 | return 301 | } 302 | defer gz.Close() 303 | 304 | w.WriteHeader(http.StatusOK) 305 | io.Copy(w, gz) 306 | } 307 | 308 | func toHTTPError(err error) (msg string, httpStatus int) { 309 | if os.IsNotExist(err) { 310 | return "404 page not found", http.StatusNotFound 311 | } 312 | if os.IsPermission(err) { 313 | return "403 Forbidden", http.StatusForbidden 314 | } 315 | // Default: 316 | return "500 Internal Server Error", http.StatusInternalServerError 317 | } 318 | 319 | func containsDotDot(v string) bool { 320 | if !strings.Contains(v, "..") { 321 | return false 322 | } 323 | for _, ent := range strings.FieldsFunc(v, isSlashRune) { 324 | if ent == ".." { 325 | return true 326 | } 327 | } 328 | return false 329 | } 330 | func isSlashRune(r rune) bool { return r == '/' || r == '\\' } 331 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Vector Creations Ltd 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "crypto/subtle" 22 | "flag" 23 | "fmt" 24 | "io/ioutil" 25 | "log" 26 | "math/rand" 27 | "net" 28 | "net/http" 29 | "os" 30 | "regexp" 31 | "strings" 32 | "text/template" 33 | "time" 34 | 35 | "github.com/google/go-github/github" 36 | "github.com/xanzy/go-gitlab" 37 | "golang.org/x/oauth2" 38 | 39 | "gopkg.in/yaml.v2" 40 | 41 | _ "embed" 42 | ) 43 | 44 | // DefaultIssueBodyTemplate is the default template used for `issue_body_template_file` in the config. 45 | // 46 | //go:embed templates/issue_body.tmpl 47 | var DefaultIssueBodyTemplate string 48 | 49 | // DefaultEmailBodyTemplate is the default template used for `email_body_template_file` in the config. 50 | // 51 | //go:embed templates/email_body.tmpl 52 | var DefaultEmailBodyTemplate string 53 | 54 | var configPath = flag.String("config", "rageshake.yaml", "The path to the config file. For more information, see the config file in this repository.") 55 | var bindAddr = flag.String("listen", ":9110", "The port to listen on.") 56 | 57 | // defaultErrorReason is the default reason string when not present for a rejection condition 58 | const defaultErrorReason string = "app or user text rejected" 59 | 60 | type config struct { 61 | // Username and password required to access the bug report listings 62 | BugsUser string `yaml:"listings_auth_user"` 63 | BugsPass string `yaml:"listings_auth_pass"` 64 | 65 | // External URI to /api 66 | APIPrefix string `yaml:"api_prefix"` 67 | 68 | // Allowed rageshake app names 69 | AllowedAppNames []string `yaml:"allowed_app_names"` 70 | 71 | // List of rejection conditions 72 | RejectionConditions []RejectionCondition `yaml:"rejection_conditions"` 73 | 74 | // A GitHub personal access token, to create a GitHub issue for each report. 75 | GithubToken string `yaml:"github_token"` 76 | 77 | GithubProjectMappings map[string]string `yaml:"github_project_mappings"` 78 | 79 | GitlabURL string `yaml:"gitlab_url"` 80 | GitlabToken string `yaml:"gitlab_token"` 81 | 82 | GitlabProjectMappings map[string]int `yaml:"gitlab_project_mappings"` 83 | GitlabProjectLabels map[string][]string `yaml:"gitlab_project_labels"` 84 | GitlabIssueConfidential bool `yaml:"gitlab_issue_confidential"` 85 | 86 | IssueBodyTemplateFile string `yaml:"issue_body_template_file"` 87 | EmailBodyTemplateFile string `yaml:"email_body_template_file"` 88 | 89 | SlackWebhookURL string `yaml:"slack_webhook_url"` 90 | 91 | EmailAddresses []string `yaml:"email_addresses"` 92 | 93 | EmailFrom string `yaml:"email_from"` 94 | 95 | SMTPServer string `yaml:"smtp_server"` 96 | 97 | SMTPUsername string `yaml:"smtp_username"` 98 | 99 | SMTPPassword string `yaml:"smtp_password"` 100 | 101 | GenericWebhookURLs []string `yaml:"generic_webhook_urls"` 102 | } 103 | 104 | // RejectionCondition contains the fields that can match a bug report for it to be rejected. 105 | // All the (optional) fields must match for the rejection condition to apply 106 | type RejectionCondition struct { 107 | // App name, applies only if not empty 108 | App string `yaml:"app"` 109 | // Version, applies only if not empty 110 | Version string `yaml:"version"` 111 | // Label, applies only if not empty 112 | Label string `yaml:"label"` 113 | // Message sent by the user, applies only if not empty 114 | UserTextMatch string `yaml:"usertext"` 115 | // Send this text to the client-side to inform the user why the server rejects the rageshake. Uses a default generic value if empty. 116 | Reason string `yaml:"reason"` 117 | // Send this text to the client-side to inform the user why the server rejects the rageshake. Uses a default error code REJECTED if empty. 118 | ErrorCode string `yaml:"errorcode"` 119 | } 120 | 121 | func (c RejectionCondition) matchesApp(p *payload) bool { 122 | // Empty `RejectionCondition.App` is a wildcard which matches anything 123 | return c.App == "" || c.App == p.AppName 124 | } 125 | 126 | func (c RejectionCondition) matchesVersion(p *payload) bool { 127 | version := "" 128 | if p.Data != nil { 129 | version = p.Data["Version"] 130 | } 131 | // Empty `RejectionCondition.Version` is a wildcard which matches anything 132 | return c.Version == "" || c.Version == version 133 | } 134 | 135 | func (c RejectionCondition) matchesLabel(p *payload) bool { 136 | // Empty `RejectionCondition.Label` is a wildcard which matches anything 137 | if c.Label == "" { 138 | return true 139 | } 140 | // Otherwise return true only if there is a label that matches 141 | labelMatch := false 142 | for _, l := range p.Labels { 143 | if l == c.Label { 144 | labelMatch = true 145 | break 146 | } 147 | } 148 | return labelMatch 149 | } 150 | 151 | func (c RejectionCondition) matchesUserText(p *payload) bool { 152 | // Empty `RejectionCondition.UserTextMatch` is a wildcard which matches anything 153 | return c.UserTextMatch == "" || regexp.MustCompile(c.UserTextMatch).MatchString(p.UserText) 154 | } 155 | 156 | // Returns a rejection reason and error code if the payload should be rejected by this condition, condition; otherwise returns `nil` for both results. 157 | func (c RejectionCondition) shouldReject(p *payload) (*string, *string) { 158 | if c.matchesApp(p) && c.matchesVersion(p) && c.matchesLabel(p) && c.matchesUserText(p) { 159 | // RejectionCondition matches all of the conditions: we should reject this submission/ 160 | var reason = defaultErrorReason 161 | if c.Reason != "" { 162 | reason = c.Reason 163 | } 164 | var code = ErrCodeRejected 165 | if c.ErrorCode != "" { 166 | code = c.ErrorCode 167 | } 168 | return &reason, &code 169 | } 170 | return nil, nil 171 | } 172 | 173 | // Returns a rejection reason and error code if the payload should be rejected by any condition, condition; otherwise returns `nil` for both results. 174 | func (c *config) matchesRejectionCondition(p *payload) (*string, *string) { 175 | for _, rc := range c.RejectionConditions { 176 | reject, code := rc.shouldReject(p) 177 | if reject != nil { 178 | return reject, code 179 | } 180 | } 181 | return nil, nil 182 | } 183 | 184 | func basicAuth(handler http.Handler, username, password, realm string) http.Handler { 185 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 186 | user, pass, ok := r.BasicAuth() // pull creds from the request 187 | 188 | // check user and pass securely 189 | if !ok || subtle.ConstantTimeCompare([]byte(user), []byte(username)) != 1 || subtle.ConstantTimeCompare([]byte(pass), []byte(password)) != 1 { 190 | w.Header().Set("WWW-Authenticate", `Basic realm="`+realm+`"`) 191 | w.WriteHeader(401) 192 | w.Write([]byte("Unauthorised.\n")) 193 | return 194 | } 195 | 196 | handler.ServeHTTP(w, r) 197 | }) 198 | } 199 | 200 | func main() { 201 | flag.Parse() 202 | 203 | cfg, err := loadConfig(*configPath) 204 | if err != nil { 205 | log.Fatalf("Invalid config file: %s", err) 206 | } 207 | 208 | var ghClient *github.Client 209 | 210 | if cfg.GithubToken == "" { 211 | fmt.Println("No github_token configured. Reporting bugs to github is disabled.") 212 | } else { 213 | ctx := context.Background() 214 | ts := oauth2.StaticTokenSource( 215 | &oauth2.Token{AccessToken: cfg.GithubToken}, 216 | ) 217 | tc := oauth2.NewClient(ctx, ts) 218 | tc.Timeout = time.Duration(5) * time.Minute 219 | ghClient = github.NewClient(tc) 220 | } 221 | 222 | var glClient *gitlab.Client 223 | if cfg.GitlabToken == "" { 224 | fmt.Println("No gitlab_token configured. Reporting bugs to gitlab is disaled.") 225 | } else { 226 | glClient, err = gitlab.NewClient(cfg.GitlabToken, gitlab.WithBaseURL(cfg.GitlabURL)) 227 | if err != nil { 228 | // This probably only happens if the base URL is invalid 229 | log.Fatalln("Failed to create GitLab client:", err) 230 | } 231 | } 232 | 233 | var slack *slackClient 234 | 235 | if cfg.SlackWebhookURL == "" { 236 | fmt.Println("No slack_webhook_url configured. Reporting bugs to slack is disabled.") 237 | } else { 238 | slack = newSlackClient(cfg.SlackWebhookURL) 239 | } 240 | 241 | if len(cfg.EmailAddresses) > 0 && cfg.SMTPServer == "" { 242 | log.Fatal("Email address(es) specified but no smtp_server configured. Wrong configuration, aborting...") 243 | } 244 | 245 | genericWebhookClient := configureGenericWebhookClient(cfg) 246 | 247 | apiPrefix := cfg.APIPrefix 248 | if apiPrefix == "" { 249 | _, port, err := net.SplitHostPort(*bindAddr) 250 | if err != nil { 251 | log.Fatal(err) 252 | } 253 | apiPrefix = fmt.Sprintf("http://localhost:%s/api", port) 254 | } else { 255 | // remove trailing / 256 | apiPrefix = strings.TrimRight(apiPrefix, "/") 257 | } 258 | 259 | appNameMap := configureAppNameMap(cfg) 260 | 261 | log.Printf("Using %s/listing as public URI", apiPrefix) 262 | 263 | rand.Seed(time.Now().UnixNano()) 264 | http.Handle("/api/submit", &submitServer{ 265 | issueTemplate: parseTemplate(DefaultIssueBodyTemplate, cfg.IssueBodyTemplateFile, "issue"), 266 | emailTemplate: parseTemplate(DefaultEmailBodyTemplate, cfg.EmailBodyTemplateFile, "email"), 267 | ghClient: ghClient, 268 | glClient: glClient, 269 | apiPrefix: apiPrefix, 270 | slack: slack, 271 | genericWebhookClient: genericWebhookClient, 272 | allowedAppNameMap: appNameMap, 273 | cfg: cfg, 274 | }) 275 | 276 | // Make sure bugs directory exists 277 | _ = os.Mkdir("bugs", os.ModePerm) 278 | 279 | // serve files under "bugs" 280 | ls := &logServer{"bugs"} 281 | fs := http.StripPrefix("/api/listing/", ls) 282 | 283 | // set auth if env vars exist 284 | usr := cfg.BugsUser 285 | pass := cfg.BugsPass 286 | if usr == "" || pass == "" { 287 | fmt.Println("No listings_auth_user/pass configured. No authentication is running for /api/listing") 288 | } else { 289 | fs = basicAuth(fs, usr, pass, "Riot bug reports") 290 | } 291 | http.Handle("/api/listing/", fs) 292 | 293 | http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { 294 | fmt.Fprint(w, "ok") 295 | }) 296 | 297 | log.Println("Listening on", *bindAddr) 298 | 299 | log.Fatal(http.ListenAndServe(*bindAddr, nil)) 300 | } 301 | 302 | // parseTemplate parses a template file, with fallback to default. 303 | // 304 | // If `templateFilePath` is non-empty, it is used as the name of a file to read. Otherwise, `defaultTemplate` is 305 | // used. 306 | // 307 | // The template text is then parsed into a template named `templateName`. 308 | func parseTemplate(defaultTemplate string, templateFilePath string, templateName string) *template.Template { 309 | templateText := defaultTemplate 310 | if templateFilePath != "" { 311 | issueTemplateBytes, err := os.ReadFile(templateFilePath) 312 | if err != nil { 313 | log.Fatalf("Unable to read template file `%s`: %s", templateFilePath, err) 314 | } 315 | templateText = string(issueTemplateBytes) 316 | } 317 | parsedTemplate, err := template.New(templateName).Parse(templateText) 318 | if err != nil { 319 | log.Fatalf("Invalid template file %s in config file: %s", templateFilePath, err) 320 | } 321 | return parsedTemplate 322 | } 323 | 324 | func configureAppNameMap(cfg *config) map[string]bool { 325 | if len(cfg.AllowedAppNames) == 0 { 326 | fmt.Println("Warning: allowed_app_names is empty. Accepting requests from all app names") 327 | } 328 | var allowedAppNameMap = make(map[string]bool) 329 | for _, app := range cfg.AllowedAppNames { 330 | allowedAppNameMap[app] = true 331 | } 332 | return allowedAppNameMap 333 | } 334 | 335 | func configureGenericWebhookClient(cfg *config) *http.Client { 336 | if len(cfg.GenericWebhookURLs) == 0 { 337 | fmt.Println("No generic_webhook_urls configured.") 338 | return nil 339 | } 340 | fmt.Println("Will forward metadata of all requests to ", cfg.GenericWebhookURLs) 341 | return &http.Client{ 342 | Timeout: time.Second * 300, 343 | } 344 | } 345 | 346 | func loadConfig(configPath string) (*config, error) { 347 | contents, err := ioutil.ReadFile(configPath) 348 | if err != nil { 349 | return nil, err 350 | } 351 | var cfg config 352 | if err = yaml.Unmarshal(contents, &cfg); err != nil { 353 | return nil, err 354 | } 355 | 356 | for idx, condition := range cfg.RejectionConditions { 357 | if condition.ErrorCode != "" && !strings.HasPrefix(condition.ErrorCode, "REJECTED_") { 358 | return nil, fmt.Errorf("Rejected condition %d was invalid. `errorcode` must be use the namespace REJECTED_", idx); 359 | } 360 | } 361 | return &cfg, nil 362 | } 363 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func TestConfigRejectionCondition(t *testing.T) { 6 | cfg := config{ 7 | RejectionConditions: []RejectionCondition{ 8 | { 9 | App: "my-app", 10 | Version: "0.1.0", 11 | }, 12 | { 13 | App: "my-app", 14 | Label: "0.1.1", 15 | }, 16 | { 17 | App: "my-app", 18 | Version: "0.1.2", 19 | Label: "nightly", 20 | Reason: "no nightlies", 21 | ErrorCode: "BAD_VERSION", 22 | }, 23 | { 24 | App: "block-my-app", 25 | }, 26 | { 27 | UserTextMatch: "(\\w{4}\\s){11}\\w{4}", 28 | Reason: "it matches a recovery key and recovery keys are private", 29 | ErrorCode: "EXPOSED_RECOVERY_KEY", 30 | }, 31 | }, 32 | } 33 | rejectPayloads := []payload{ 34 | { 35 | AppName: "my-app", 36 | Data: map[string]string{ 37 | "Version": "0.1.0", 38 | // Hack add how we expect the rageshake to be rejected to the test 39 | // The actual data in a rageshake has no ExpectedRejectReason field 40 | "ExpectedRejectReason": "app or user text rejected", 41 | "ExpectedErrorCode": ErrCodeRejected, 42 | }, 43 | }, 44 | { 45 | AppName: "my-app", 46 | Data: map[string]string{ 47 | "ExpectedRejectReason": "app or user text rejected", 48 | "ExpectedErrorCode": ErrCodeRejected, 49 | }, 50 | Labels: []string{"0.1.1"}, 51 | }, 52 | { 53 | AppName: "my-app", 54 | Labels: []string{"foo", "nightly"}, 55 | Data: map[string]string{ 56 | "Version": "0.1.2", 57 | "ExpectedRejectReason": "no nightlies", 58 | "ExpectedErrorCode": "BAD_VERSION", 59 | }, 60 | }, 61 | { 62 | AppName: "block-my-app", 63 | Data: map[string]string{ 64 | "ExpectedRejectReason": "app or user text rejected", 65 | "ExpectedErrorCode": ErrCodeRejected, 66 | }, 67 | }, 68 | { 69 | AppName: "block-my-app", 70 | Labels: []string{"foo"}, 71 | Data: map[string]string{ 72 | "ExpectedRejectReason": "app or user text rejected", 73 | "ExpectedErrorCode": ErrCodeRejected, 74 | }, 75 | }, 76 | { 77 | AppName: "block-my-app", 78 | Data: map[string]string{ 79 | "Version": "42", 80 | "ExpectedRejectReason": "app or user text rejected", 81 | "ExpectedErrorCode": ErrCodeRejected, 82 | }, 83 | }, 84 | { 85 | AppName: "block-my-app", 86 | Labels: []string{"foo"}, 87 | Data: map[string]string{ 88 | "Version": "42", 89 | "ExpectedRejectReason": "app or user text rejected", 90 | "ExpectedErrorCode": ErrCodeRejected, 91 | }, 92 | }, 93 | { 94 | AppName: "my-app", 95 | UserText: "Looks like a recover key abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd", 96 | Data: map[string]string{ 97 | "ExpectedRejectReason": "it matches a recovery key and recovery keys are private", 98 | "ExpectedErrorCode": "EXPOSED_RECOVERY_KEY", 99 | }, 100 | }, 101 | } 102 | for _, p := range rejectPayloads { 103 | reject, code := cfg.matchesRejectionCondition(&p) 104 | if reject == nil || code == nil { 105 | t.Errorf("payload was accepted when it should be rejected:\n payload=%+v\nconfig=%+v", p, cfg) 106 | } 107 | if reject != nil { 108 | if *reject != p.Data["ExpectedRejectReason"] { 109 | t.Errorf("payload was rejected with the wrong reason:\n payload=%+v\nconfig=%+v", p, cfg) 110 | } 111 | } 112 | if code != nil { 113 | if *code != p.Data["ExpectedErrorCode"] { 114 | t.Errorf("payload was rejected with the wrong code:\n payload=%+v\nconfig=%+v\ncode=%s", p, cfg, *code) 115 | } 116 | } 117 | } 118 | acceptPayloads := []payload{ 119 | { 120 | AppName: "different-app", 121 | Data: map[string]string{ 122 | "Version": "0.1.0", 123 | }, 124 | }, 125 | { 126 | AppName: "different-app", 127 | Data: map[string]string{}, 128 | Labels: []string{"0.1.1"}, 129 | }, 130 | { 131 | AppName: "different-app", 132 | Labels: []string{"foo", "nightly"}, 133 | Data: map[string]string{ 134 | "Version": "0.1.2", 135 | }, 136 | }, 137 | { 138 | AppName: "my-app", 139 | Data: map[string]string{ 140 | "Version": "0.1.0-suffix", 141 | }, 142 | }, 143 | { 144 | AppName: "my-app", 145 | Data: map[string]string{}, 146 | Labels: []string{"0.1.1-suffix"}, 147 | }, 148 | { 149 | AppName: "my-app", 150 | Labels: []string{"foo", "nightly-suffix"}, 151 | Data: map[string]string{ 152 | "Version": "0.1.2", 153 | }, 154 | }, 155 | { // version matches but label does not (it's Label AND Version not OR) 156 | AppName: "my-app", 157 | Labels: []string{"foo"}, 158 | Data: map[string]string{ 159 | "Version": "0.1.2", 160 | }, 161 | }, 162 | { 163 | AppName: "my-app", 164 | UserText: "Some description", 165 | }, 166 | } 167 | for _, p := range acceptPayloads { 168 | reject, code := cfg.matchesRejectionCondition(&p) 169 | if reject != nil || code != nil { 170 | t.Errorf("payload was rejected when it should be accepted:\n payload=%+v\nconfig=%+v", p, cfg) 171 | } 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /rageshake.sample.yaml: -------------------------------------------------------------------------------- 1 | # username/password pair which will be required to access the bug report 2 | # listings at `/api/listing`, via HTTP basic auth. If omitted, there will be 3 | # *no* authentication on this access! 4 | 5 | # the external URL at which /api is accessible; it is used to add a link to the 6 | # report to the GitHub issue. If unspecified, based on the listen address. 7 | # api_prefix: https://riot.im/bugreports 8 | 9 | # List of approved AppNames we accept. Names not in the list or missing an application name will be rejected. 10 | # An empty or missing list will retain legacy behaviour and permit reports from any application name. 11 | allowed_app_names: [] 12 | 13 | # If any submission matches one of these rejection conditions, the submission is rejected. 14 | # A condition is made by an union of optional fields: app, version, labels, user text. They all need to match for rejecting the rageshake 15 | # It can also contain an optional reason to explain why this server is rejecting a user's submission. 16 | # An errorcode can be provided to give a precise machine-readable error description under the `REJECTED_` namespace. 17 | # Otherwise, this defaults to REJECTED. 18 | rejection_conditions: 19 | - app: my-app 20 | version: "0.4.9" # if the submission has a Version which is exactly this value, reject the submission. 21 | - app: my-app 22 | label: "0.4.9" # if any label matches this value, the submission is rejected. 23 | - app: my-app 24 | version: "0.4.9" 25 | label: "nightly" # both label and Version must match for this condition to be true 26 | reason: "this server does not accept rageshakes from nightlies" 27 | errorcode: "REJECTED_BAD_VERSION" 28 | - usertext: "(\\w{4}\\s){11}\\w{4}" # reject text containing possible recovery keys 29 | reason: "it matches a recovery key and recovery keys are private" 30 | errorcode: "REJECTED_UNEXPECTED_RECOVERY_KEY" 31 | 32 | # a GitHub personal access token (https://github.com/settings/tokens), which 33 | # will be used to create a GitHub issue for each report. It requires 34 | # `public_repo` scope. If omitted, no issues will be created. 35 | github_token: secrettoken 36 | 37 | # mappings from app name (as submitted in the API) to github repo for issue reporting. 38 | github_project_mappings: 39 | my-app: octocat/HelloWorld 40 | 41 | # a GitLab personal access token (https://gitlab.com/-/profile/personal_access_tokens), which 42 | # will be used to create a GitLab issue for each report. It requires 43 | # `api` scope. If omitted, no issues will be created. 44 | gitlab_token: secrettoken 45 | # the base URL of the GitLab instance to use 46 | gitlab_url: https://gitlab.com 47 | 48 | # mappings from app name (as submitted in the API) to the GitLab Project ID (not name!) for issue reporting. 49 | gitlab_project_mappings: 50 | my-app: 12345 51 | # mappings from app name to a list of GitLab label names for issue reporting. 52 | gitlab_project_labels: 53 | my-app: 54 | - client::my-app 55 | # whether GitLab issues should be created as confidential issues. Defaults to false. 56 | gitlab_issue_confidential: true 57 | 58 | # a Slack personal webhook URL (https://api.slack.com/incoming-webhooks), which 59 | # will be used to post a notification on Slack for each report. 60 | slack_webhook_url: https://hooks.slack.com/services/TTTTTTT/XXXXXXXXXX/YYYYYYYYYYY 61 | 62 | # notification can also be pushed by email. 63 | # this param controls the target emails 64 | email_addresses: 65 | - support@matrix.org 66 | 67 | # this is the from field that will be used in the email notifications 68 | email_from: Rageshake 69 | 70 | # SMTP server configuration 71 | smtp_server: localhost:25 72 | smtp_username: myemailuser 73 | smtp_password: myemailpass 74 | 75 | # a list of webhook URLs, (see docs/generic_webhook.md) 76 | generic_webhook_urls: 77 | - https://server.example.com/your-server/api 78 | - http://another-server.com/api 79 | 80 | # The paths of template files for the body of Github and Gitlab issues, and emails. 81 | # See `templates/README.md` for more information. 82 | issue_body_template_file: path/to/issue_body.tmpl 83 | email_body_template_file: path/to/email_body.tmpl 84 | -------------------------------------------------------------------------------- /scripts/cleanup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import glob 4 | import gzip 5 | import os 6 | import sys 7 | import time 8 | from datetime import datetime, timedelta 9 | from typing import Dict, Iterable, List, Set 10 | 11 | # Cleanup for rageshake server output files 12 | # 13 | # Example usage: 14 | # 15 | # ./cleanup.py --dry-run --path /home/rageshakes/store --max-days 100 element-auto-uisi:90 16 | # 17 | # No dependencies required beyond a modern python3. 18 | 19 | 20 | class Cleanup: 21 | """ 22 | Cleanup a rageshake bug repository. 23 | 24 | Once created, call cleanup() to begin the actual operation. Statistics are available after cleanup completes. 25 | """ 26 | 27 | def __init__( 28 | self, 29 | limits: Dict[str, int], 30 | days_to_check: Iterable[int], 31 | dry_run: bool, 32 | root_path: str, 33 | mxids_to_exclude: List[str], 34 | ): 35 | """ 36 | Set options for a cleanup run of a rageshake repository. 37 | 38 | @param limits: Map of app name to integer number of days that application's rageshakes should be retained 39 | @param days_to_check: List of ints each representing "days ago" that should be checked for rageshakes to delete 40 | @param dry_run: If set, perform all actions but do not complete deletion of files 41 | @param root_path: Base path to rageshake bug repository 42 | @param mxids_to_exclude: Rageshakes sent by this list of mxids should always be preserved. 43 | """ 44 | self._limits = limits 45 | self._days_to_check = days_to_check 46 | self._dry_run = dry_run 47 | self._root_path = root_path 48 | self._mxids_to_exclude = mxids_to_exclude 49 | # Count of files we deleted or would delete (dry-run) 50 | self.deleted = 0 51 | # Count of files we checked 52 | self.checked = 0 53 | # Sum of bytes in files we deleted or would delete (dry-run) 54 | self.disk_saved = 0 55 | # History of how many times a given mxid saved a file. 56 | self.excluded_count_by_user = {mxid: 0 for mxid in mxids_to_exclude} 57 | 58 | def cleanup(self) -> None: 59 | """ 60 | Check for rageshakes to remove according to settings. 61 | 62 | Do not run multiple times as statistics are generated internally during each call. 63 | """ 64 | today = datetime.today() 65 | for days_ago in self._days_to_check: 66 | target = today - timedelta(days=days_ago) 67 | folder_name = target.strftime("%Y-%m-%d") 68 | applications = set() 69 | for name in self._limits.keys(): 70 | if self._limits[name] < days_ago: 71 | applications.add(name) 72 | self._check_date(self._root_path + "/" + folder_name, applications) 73 | 74 | def _check_date(self, folder_name: str, applications_to_delete: Set[str]) -> None: 75 | """ 76 | Check all rageshakes on a given date (folder) 77 | """ 78 | if len(applications_to_delete) == 0: 79 | print(f"W Not checking {folder_name}, no applications would be removed") 80 | return 81 | 82 | if not os.path.exists(folder_name): 83 | print(f"W Not checking {folder_name}, not present or not a directory") 84 | return 85 | 86 | checked = 0 87 | deleted = 0 88 | with os.scandir(folder_name) as rageshakes: 89 | for rageshake in rageshakes: 90 | rageshake_path = folder_name + "/" + rageshake.name 91 | if rageshake.is_dir(): 92 | checked += 1 93 | try: 94 | if self._check_rageshake( 95 | rageshake_path, applications_to_delete 96 | ): 97 | deleted += 1 98 | except Exception as e: 99 | raise Exception( 100 | f"Error while checking rageshake {rageshake_path}" 101 | ) from e 102 | else: 103 | print( 104 | f"W File in rageshake tree {rageshake_path} is not a directory" 105 | ) 106 | 107 | print( 108 | f"I Checked {folder_name} for {applications_to_delete}, " 109 | f"{'would delete' if self._dry_run else 'deleted'} {deleted}/{checked} rageshakes" 110 | ) 111 | 112 | self.deleted += deleted 113 | self.checked += checked 114 | # optionally delete folder if we deleted 100% of rageshakes, but for now it' s fine. 115 | 116 | def _check_rageshake( 117 | self, rageshake_folder_path: str, applications_to_delete: Set[str] 118 | ) -> bool: 119 | """ 120 | Checks a given rageshake folder against the application and userid lists. 121 | 122 | If the folder matches, and dryrun mode is disabled, the folder is deleted. 123 | 124 | @returns: True if the rageshake matched, False if it was skipped. 125 | """ 126 | app_name = None 127 | mxid = None 128 | 129 | try: 130 | # TODO: use `details.json` instead of `details.log.gz`, which will avoid 131 | # this custom parsing 132 | with gzip.open(rageshake_folder_path + "/details.log.gz") as details: 133 | for line in details.readlines(): 134 | parts = line.decode("utf-8", errors="replace").split( 135 | ":", maxsplit=1 136 | ) 137 | if parts[0] == "Application": 138 | app_name = parts[1].strip() 139 | if parts[0] == "user_id": 140 | mxid = parts[1].strip() 141 | except FileNotFoundError as e: 142 | print( 143 | f"W Unable to open {e.filename} to check for application name. Ignoring this folder." 144 | ) 145 | return False 146 | 147 | if app_name in applications_to_delete: 148 | if mxid in self._mxids_to_exclude: 149 | self.excluded_count_by_user[mxid] += 1 150 | else: 151 | self._delete(rageshake_folder_path) 152 | return True 153 | 154 | return False 155 | 156 | def _delete(self, rageshake_folder_path: str) -> None: 157 | """ 158 | Delete a given rageshake folder, unless dryrun mode is enabled 159 | """ 160 | files = glob.glob(rageshake_folder_path + "/*") 161 | for file in files: 162 | self.disk_saved += os.stat(file).st_size 163 | if self._dry_run: 164 | print(f"I would delete {file}") 165 | else: 166 | print(f"I deleting {file}") 167 | os.unlink(file) 168 | 169 | if self._dry_run: 170 | print(f"I would remove directory {rageshake_folder_path}") 171 | else: 172 | print(f"I removing directory {rageshake_folder_path}") 173 | os.rmdir(rageshake_folder_path) 174 | 175 | 176 | def main(): 177 | parser = argparse.ArgumentParser(description="Cleanup rageshake files on disk") 178 | parser.add_argument( 179 | "limits", 180 | metavar="LIMIT", 181 | type=str, 182 | nargs="+", 183 | help="application_name retention limits in days (each formatted app-name:10)", 184 | ) 185 | group = parser.add_mutually_exclusive_group(required=True) 186 | group.add_argument( 187 | "--max-days", 188 | dest="max_days", 189 | type=int, 190 | help="Search all days until this maximum", 191 | ) 192 | group.add_argument( 193 | "--days-to-check", 194 | dest="days_to_check", 195 | type=str, 196 | help="Explicitly supply days in the past to check for deletion, eg '1,2,3,5'", 197 | ) 198 | parser.add_argument( 199 | "--exclude-mxids-file", 200 | dest="exclude_mxids_file", 201 | type=str, 202 | help="Supply a text file containing one mxid per line to exclude from cleanup. Blank lines and lines starting # are ignored.", 203 | ) 204 | parser.add_argument( 205 | "--dry-run", dest="dry_run", action="store_true", help="Dry run (do not delete)" 206 | ) 207 | parser.add_argument( 208 | "--path", 209 | dest="path", 210 | type=str, 211 | required=True, 212 | help="Root path of rageshakes (eg /home/rageshakes/bugs/)", 213 | ) 214 | 215 | parser.add_argument( 216 | "--repeat-delay-hours", 217 | dest="repeat_delay_hours", 218 | type=float, 219 | help=""" 220 | Instead of exiting after execution, run repeatedly, waiting this number of hours between each execution. 221 | An alternative to configuring a cronjob for ongoing cleanup. 222 | """, 223 | ) 224 | 225 | args = parser.parse_args() 226 | 227 | if args.repeat_delay_hours is not None: 228 | while True: 229 | execute(args) 230 | print("I =====================================================") 231 | print(f"I waiting {args.repeat_delay_hours}h for next execution") 232 | print("I =====================================================") 233 | time.sleep(args.repeat_delay_hours * 60 * 60) 234 | else: 235 | execute(args) 236 | 237 | 238 | def execute(args) -> None: 239 | """ 240 | Creation, configuration and execution of a cleanup task based on args. 241 | 242 | Allows exceptions to propagate to the caller for handling. 243 | """ 244 | 245 | application_limits: Dict[str, int] = {} 246 | for l in args.limits: 247 | parts = l.rsplit(":", 1) 248 | try: 249 | if len(parts) < 2: 250 | raise ValueError("missing :") 251 | limit = int(parts[1]) 252 | except ValueError as e: 253 | print(f"E Malformed --limits argument: {e}", file=sys.stderr) 254 | sys.exit(1) 255 | 256 | application_limits[parts[0]] = limit 257 | 258 | days_to_check: Iterable[int] = [] 259 | if args.max_days: 260 | days_to_check = range(args.max_days) 261 | if args.days_to_check: 262 | days_to_check = map(lambda x: int(x), args.days_to_check.split(",")) 263 | 264 | mxids_to_exclude = [] 265 | if args.exclude_mxids_file: 266 | with open(args.exclude_mxids_file) as file: 267 | for lineno, data in enumerate(file): 268 | data = data.strip() 269 | if len(data) == 0: 270 | # blank line, ignore 271 | pass 272 | elif data[0] == "#": 273 | # comment, ignore 274 | pass 275 | elif data[0] == "@": 276 | # mxid 277 | mxids_to_exclude.append(data) 278 | else: 279 | print( 280 | f"E Unable to parse --exclude-mxids-file on line {lineno + 1}: {data}", 281 | file=sys.stderr, 282 | ) 283 | sys.exit(1) 284 | 285 | cleanup = Cleanup( 286 | application_limits, days_to_check, args.dry_run, args.path, mxids_to_exclude 287 | ) 288 | 289 | cleanup.cleanup() 290 | print( 291 | f"I Deleted {cleanup.deleted} of {cleanup.checked} rageshakes, " 292 | f"saving {cleanup.disk_saved} bytes. Dry run? {cleanup._dry_run}" 293 | ) 294 | print(f"I excluded count by user {cleanup.excluded_count_by_user}") 295 | 296 | 297 | if __name__ == "__main__": 298 | main() 299 | -------------------------------------------------------------------------------- /scripts/lint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # check the go source for lint. This is run by CI, and the pre-commit hook. 4 | 5 | # we *don't* check gofmt here, following the advice at 6 | # https://golang.org/doc/go1.10#gofmt 7 | 8 | set -eu 9 | 10 | echo "golint:" 11 | golint -set_exit_status 12 | echo "go vet:" 13 | go vet -vettool=$(which shadow) 14 | echo "gocyclo:" 15 | gocyclo -over 12 . 16 | -------------------------------------------------------------------------------- /slack.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | type slackClient struct { 10 | webHook string 11 | name string 12 | face string 13 | } 14 | 15 | func newSlackClient(webHook string) *slackClient { 16 | return &slackClient{ 17 | webHook: webHook, 18 | name: "Notifier", 19 | face: "robot_face"} 20 | } 21 | 22 | func (slack *slackClient) Name(name string) { 23 | slack.name = name 24 | } 25 | 26 | func (slack *slackClient) Face(face string) { 27 | slack.face = face 28 | } 29 | 30 | func (slack slackClient) Notify(text string) error { 31 | json := buildRequest(text, slack) 32 | 33 | req, err := http.NewRequest("POST", slack.webHook, strings.NewReader(json)) 34 | if err != nil { 35 | return fmt.Errorf("Can't connect to host %s: %s", slack.webHook, err.Error()) 36 | } 37 | 38 | req.Header.Set("Content-Type", "application/json") 39 | 40 | client := http.Client{} 41 | _, err = client.Do(req) 42 | 43 | return err 44 | } 45 | 46 | func buildRequest(text string, slack slackClient) string { 47 | return fmt.Sprintf(`{"text":"%s", "username": "%s", "icon_emoji": ":%s:"}`, text, slack.name, slack.face) 48 | } 49 | -------------------------------------------------------------------------------- /submit.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Vector Creations Ltd 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "bufio" 21 | "bytes" 22 | "compress/gzip" 23 | "context" 24 | "encoding/base32" 25 | "encoding/json" 26 | "fmt" 27 | "github.com/ua-parser/uap-go/uaparser" 28 | "io" 29 | "io/ioutil" 30 | "log" 31 | "math/rand" 32 | "mime" 33 | "mime/multipart" 34 | "net" 35 | "net/http" 36 | "net/smtp" 37 | "os" 38 | "path/filepath" 39 | "regexp" 40 | "sort" 41 | "strconv" 42 | "strings" 43 | "text/template" 44 | "time" 45 | 46 | "github.com/google/go-github/github" 47 | "github.com/jordan-wright/email" 48 | "github.com/xanzy/go-gitlab" 49 | ) 50 | 51 | var maxPayloadSize = 1024 * 1024 * 55 // 55 MB 52 | 53 | type submitServer struct { 54 | // Template for building github and gitlab issues 55 | issueTemplate *template.Template 56 | 57 | // Template for building emails 58 | emailTemplate *template.Template 59 | 60 | // github client for reporting bugs. may be nil, in which case, 61 | // reporting is disabled. 62 | ghClient *github.Client 63 | glClient *gitlab.Client 64 | 65 | // External URI to /api 66 | apiPrefix string 67 | 68 | slack *slackClient 69 | 70 | genericWebhookClient *http.Client 71 | allowedAppNameMap map[string]bool 72 | cfg *config 73 | } 74 | 75 | // the type of payload which can be uploaded as JSON to the submit endpoint 76 | type jsonPayload struct { 77 | Text string `json:"text"` 78 | AppName string `json:"app"` 79 | Version string `json:"version"` 80 | UserAgent string `json:"user_agent"` 81 | Logs []jsonLogEntry `json:"logs"` 82 | Data map[string]string `json:"data"` 83 | Labels []string `json:"labels"` 84 | } 85 | 86 | type jsonLogEntry struct { 87 | ID string `json:"id"` 88 | Lines string `json:"lines"` 89 | } 90 | 91 | // `issueBodyTemplatePayload` contains the data made available to the `issue_body_template` and 92 | // `email_body_template`. 93 | // 94 | // !!! Keep in step with the documentation in `templates/README.md` !!! 95 | type issueBodyTemplatePayload struct { 96 | payload 97 | // Complete link to the listing URL that contains all uploaded logs 98 | ListingURL string 99 | } 100 | 101 | // `genericWebhookPayload` contains the data sent to webhooks configured with `generic_webhook_urls`, as 102 | // well as being written to `details.json` in the rageshake directory. 103 | // 104 | // See `docs/generic_webhook.md`. 105 | type genericWebhookPayload struct { 106 | payload 107 | // If a github/gitlab report is generated, this is set. 108 | ReportURL string `json:"report_url"` 109 | // Complete link to the listing URL that contains all uploaded logs 110 | ListingURL string `json:"listing_url"` 111 | } 112 | 113 | // `payload` stores information about a request made to this server. 114 | // 115 | // !!! Since this is inherited by `issueBodyTemplatePayload`, remember to keep it in step 116 | // with the documentation in `templates/README.md` !!! 117 | type payload struct { 118 | // A unique ID for this payload, generated within this server 119 | ID string `json:"id"` 120 | // A multi-line string containing the user description of the fault. 121 | UserText string `json:"user_text"` 122 | // A short slug to identify the app making the report 123 | AppName string `json:"app"` 124 | // Arbitrary data to annotate the report 125 | Data map[string]string `json:"data"` 126 | // Short labels to group reports 127 | Labels []string `json:"labels"` 128 | // A list of names of logs recognised by the server 129 | Logs []string `json:"logs"` 130 | // Set if there are log parsing errors 131 | LogErrors []string `json:"logErrors"` 132 | // A list of other files (not logs) uploaded as part of the rageshake 133 | Files []string `json:"files"` 134 | // Set if there are file parsing errors 135 | FileErrors []string `json:"fileErrors"` 136 | // The time the rageshake was submitted, in milliseconds since the epoch 137 | CreateTimeMillis int64 `json:"create_time"` 138 | } 139 | 140 | func (p payload) WriteTo(out io.Writer) { 141 | fmt.Fprintf( 142 | out, 143 | "%s\n\nNumber of logs: %d\nApplication: %s\n", 144 | p.UserText, len(p.Logs), p.AppName, 145 | ) 146 | fmt.Fprintf(out, "Labels: %s\n", strings.Join(p.Labels, ", ")) 147 | 148 | var dataKeys []string 149 | for k := range p.Data { 150 | dataKeys = append(dataKeys, k) 151 | } 152 | sort.Strings(dataKeys) 153 | for _, k := range dataKeys { 154 | v := p.Data[k] 155 | fmt.Fprintf(out, "%s: %s\n", k, v) 156 | } 157 | if len(p.LogErrors) > 0 { 158 | fmt.Fprint(out, "Log upload failures:\n") 159 | for _, e := range p.LogErrors { 160 | fmt.Fprintf(out, " %s\n", e) 161 | } 162 | } 163 | if len(p.FileErrors) > 0 { 164 | fmt.Fprint(out, "Attachment upload failures:\n") 165 | for _, e := range p.FileErrors { 166 | fmt.Fprintf(out, " %s\n", e) 167 | } 168 | } 169 | } 170 | 171 | type submitResponse struct { 172 | ReportURL string `json:"report_url,omitempty"` 173 | } 174 | 175 | type submitErrorResponse struct { 176 | Error string `json:"error"` 177 | ErrorCode string `json:"errcode"` 178 | PolicyURL string `json:"policy_url,omitempty"` 179 | } 180 | 181 | func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { 182 | // if we attempt to return a response without reading the request body, 183 | // apache gets upset and returns a 500. Let's try this. 184 | defer req.Body.Close() 185 | defer io.Copy(ioutil.Discard, req.Body) 186 | 187 | if req.Method != "POST" && req.Method != "OPTIONS" { 188 | writeError(w, 405, submitErrorResponse{Error: "Method not allowed. Use POST.", ErrorCode: ErrCodeMethodNotAllowed}) 189 | return 190 | } 191 | 192 | // Set CORS 193 | w.Header().Set("Access-Control-Allow-Origin", "*") 194 | w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS") 195 | w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept") 196 | if req.Method == "OPTIONS" { 197 | respond(200, w) 198 | return 199 | } 200 | s.handleSubmission(w, req) 201 | } 202 | 203 | func (s *submitServer) handleSubmission(w http.ResponseWriter, req *http.Request) { 204 | // create the report dir before parsing the request, so that we can dump 205 | // files straight in 206 | t := time.Now().UTC() 207 | prefix := t.Format("2006-01-02/150405") 208 | randBytes := make([]byte, 5) 209 | rand.Read(randBytes) 210 | prefix += "-" + base32.StdEncoding.EncodeToString(randBytes) 211 | reportDir := filepath.Join("bugs", prefix) 212 | if err := os.MkdirAll(reportDir, os.ModePerm); err != nil { 213 | log.Println("Unable to create report directory", err) 214 | writeError(w, 500, submitErrorResponse{Error: "Internal error", ErrorCode: ErrCodeUnknown}) 215 | return 216 | } 217 | 218 | listingURL := s.apiPrefix + "/listing/" + prefix 219 | log.Println("Handling report submission; listing URI will be", listingURL) 220 | 221 | p := parseRequest(w, req, reportDir) 222 | if p == nil { 223 | // parseRequest already wrote an error, but now let's delete the 224 | // useless report dir 225 | if err := os.RemoveAll(reportDir); err != nil { 226 | log.Printf("Unable to remove report dir %s after invalid upload: %v\n", 227 | reportDir, err) 228 | } 229 | return 230 | } 231 | 232 | // Filter out unwanted rageshakes, if a list is defined 233 | if len(s.allowedAppNameMap) != 0 && !s.allowedAppNameMap[p.AppName] { 234 | log.Printf("Blocking rageshake because app name %s not in list", p.AppName) 235 | if err := os.RemoveAll(reportDir); err != nil { 236 | log.Printf("Unable to remove report dir %s after rejected upload: %v\n", 237 | reportDir, err) 238 | } 239 | writeError(w, 400, submitErrorResponse{"This server does not accept rageshakes from your application.", ErrCodeDisallowedApp, "https://github.com/matrix-org/rageshake/blob/master/docs/blocked_rageshake.md"}) 240 | return 241 | } 242 | rejection, code := s.cfg.matchesRejectionCondition(p) 243 | if rejection != nil { 244 | log.Printf("Blocking rageshake from app %s because it matches a rejection_condition: %s", p.AppName, *rejection) 245 | if err := os.RemoveAll(reportDir); err != nil { 246 | log.Printf("Unable to remove report dir %s after rejected upload: %v\n", 247 | reportDir, err) 248 | } 249 | userErrorText := fmt.Sprintf("This server did not accept the rageshake because it matches a rejection condition: %s.", *rejection) 250 | writeError(w, 400, submitErrorResponse{userErrorText, *code, "https://github.com/matrix-org/rageshake/blob/master/docs/blocked_rageshake.md"}) 251 | return 252 | } 253 | 254 | // We use this prefix (eg, 2022-05-01/125223-abcde) as a unique identifier for this rageshake. 255 | // This is going to be used to uniquely identify rageshakes, even if they are not submitted to 256 | // an issue tracker for instance with automatic rageshakes that can be plentiful 257 | p.ID = prefix 258 | 259 | p.CreateTimeMillis = t.Unix() * 1e3 // TODO: drop support for Go 1.16, use UnixMilli 260 | 261 | resp, err := s.saveReport(req.Context(), *p, reportDir, listingURL) 262 | if err != nil { 263 | log.Println("Error handling report submission:", err) 264 | writeError(w, 500, submitErrorResponse{Error: "Could not save report", ErrorCode: ErrCodeUnknown}) 265 | return 266 | } 267 | 268 | w.Header().Set("Content-Type", "application/json") 269 | w.WriteHeader(200) 270 | json.NewEncoder(w).Encode(resp) 271 | } 272 | 273 | func writeError(w http.ResponseWriter, status int, response submitErrorResponse) { 274 | w.Header().Set("Content-Type", "application/json") 275 | w.WriteHeader(status) 276 | json.NewEncoder(w).Encode(response) 277 | } 278 | 279 | // parseRequest attempts to parse a received request as a bug report. If 280 | // the request cannot be parsed, it responds with an error and returns nil. 281 | func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *payload { 282 | length, err := strconv.Atoi(req.Header.Get("Content-Length")) 283 | if err != nil { 284 | log.Println("Couldn't parse content-length", err) 285 | writeError(w, 400, submitErrorResponse{Error: "Bad Content-Length header", ErrorCode: ErrCodeBadHeader}) 286 | return nil 287 | } 288 | if length > maxPayloadSize { 289 | log.Println("Content-length", length, "too large") 290 | writeError(w, 413, submitErrorResponse{Error: "Content too large", ErrorCode: ErrCodeContentTooLarge}) 291 | return nil 292 | } 293 | 294 | contentType := req.Header.Get("Content-Type") 295 | if contentType != "" { 296 | d, _, _ := mime.ParseMediaType(contentType) 297 | if d == "multipart/form-data" { 298 | p, err1 := parseMultipartRequest(w, req, reportDir) 299 | if err1 != nil { 300 | log.Println("Error parsing multipart data:", err1) 301 | writeError(w, 400, submitErrorResponse{Error: "Bad multipart data", ErrorCode: ErrCodeBadContent}) 302 | return nil 303 | } 304 | return p 305 | } 306 | } 307 | 308 | p, err := parseJSONRequest(w, req, reportDir) 309 | if err != nil { 310 | log.Println("Error parsing JSON body", err) 311 | writeError(w, 400, submitErrorResponse{Error: fmt.Sprintf("Could not decode payload: %s", err.Error()), ErrorCode: ErrCodeBadContent}) 312 | return nil 313 | } 314 | 315 | return p 316 | } 317 | 318 | var uaParser *uaparser.Parser = uaparser.NewFromSaved() 319 | 320 | func parseUserAgent(userAgent string) string { 321 | client := uaParser.Parse(userAgent) 322 | return fmt.Sprintf(`%s on %s running on %s device`, client.UserAgent.ToString(), client.Os.ToString(), client.Device.ToString()) 323 | } 324 | 325 | func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) { 326 | var p jsonPayload 327 | if err := json.NewDecoder(req.Body).Decode(&p); err != nil { 328 | return nil, err 329 | } 330 | 331 | parsed := payload{ 332 | UserText: strings.TrimSpace(p.Text), 333 | Data: make(map[string]string), 334 | Labels: p.Labels, 335 | } 336 | 337 | if p.Data != nil { 338 | parsed.Data = p.Data 339 | } 340 | 341 | for i, logfile := range p.Logs { 342 | buf := bytes.NewBufferString(logfile.Lines) 343 | leafName, err := saveLogPart(i, logfile.ID, buf, reportDir) 344 | if err != nil { 345 | log.Printf("Error saving log %s: %v", leafName, err) 346 | parsed.LogErrors = append(parsed.LogErrors, fmt.Sprintf("Error saving log %s: %v", leafName, err)) 347 | } else { 348 | parsed.Logs = append(parsed.Logs, leafName) 349 | } 350 | } 351 | 352 | parsed.AppName = p.AppName 353 | 354 | if p.UserAgent != "" { 355 | parsed.Data["Parsed-User-Agent"] = parseUserAgent(p.UserAgent) 356 | parsed.Data["User-Agent"] = p.UserAgent 357 | } 358 | if p.Version != "" { 359 | parsed.Data["Version"] = p.Version 360 | } 361 | 362 | return &parsed, nil 363 | } 364 | 365 | func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) { 366 | rdr, err := req.MultipartReader() 367 | if err != nil { 368 | return nil, err 369 | } 370 | 371 | p := payload{ 372 | Data: make(map[string]string), 373 | } 374 | 375 | for true { 376 | part, err := rdr.NextPart() 377 | if err == io.EOF { 378 | break 379 | } else if err != nil { 380 | return nil, err 381 | } 382 | 383 | if err = parseFormPart(part, &p, reportDir); err != nil { 384 | return nil, err 385 | } 386 | } 387 | return &p, nil 388 | } 389 | 390 | func parseFormPart(part *multipart.Part, p *payload, reportDir string) error { 391 | defer part.Close() 392 | field := part.FormName() 393 | partName := part.FileName() 394 | 395 | var partReader io.Reader 396 | if field == "compressed-log" { 397 | // decompress logs as we read them. 398 | // 399 | // we could save the log directly rather than unzipping and re-zipping, 400 | // but doing so conveys the benefit of checking the validity of the 401 | // gzip at upload time. 402 | zrdr, err := gzip.NewReader(part) 403 | if err != nil { 404 | // we don't reject the whole request if there is an 405 | // error reading one attachment. 406 | log.Printf("Error unzipping %s: %v", partName, err) 407 | 408 | p.LogErrors = append(p.LogErrors, fmt.Sprintf("Error unzipping %s: %v", partName, err)) 409 | return nil 410 | } 411 | defer zrdr.Close() 412 | partReader = zrdr 413 | } else { 414 | // read the field data directly from the multipart part 415 | partReader = part 416 | } 417 | 418 | if field == "file" { 419 | leafName, err := saveFormPart(partName, partReader, reportDir) 420 | if err != nil { 421 | log.Printf("Error saving %s %s: %v", field, partName, err) 422 | p.FileErrors = append(p.FileErrors, fmt.Sprintf("Error saving %s: %v", partName, err)) 423 | } else { 424 | p.Files = append(p.Files, leafName) 425 | } 426 | return nil 427 | } 428 | 429 | if field == "log" || field == "compressed-log" { 430 | leafName, err := saveLogPart(len(p.Logs), partName, partReader, reportDir) 431 | if err != nil { 432 | log.Printf("Error saving %s %s: %v", field, partName, err) 433 | p.LogErrors = append(p.LogErrors, fmt.Sprintf("Error saving %s: %v", partName, err)) 434 | } else { 435 | p.Logs = append(p.Logs, leafName) 436 | } 437 | return nil 438 | } 439 | 440 | b, err := ioutil.ReadAll(partReader) 441 | if err != nil { 442 | return err 443 | } 444 | data := string(b) 445 | formPartToPayload(field, data, p) 446 | return nil 447 | } 448 | 449 | // formPartToPayload updates the relevant part of *p from a name/value pair 450 | // read from the form data. 451 | func formPartToPayload(field, data string, p *payload) { 452 | if field == "text" { 453 | p.UserText = data 454 | } else if field == "app" { 455 | p.AppName = data 456 | } else if field == "version" { 457 | p.Data["Version"] = data 458 | } else if field == "user_agent" { 459 | p.Data["User-Agent"] = data 460 | p.Data["Parsed-User-Agent"] = parseUserAgent(data) 461 | } else if field == "label" { 462 | p.Labels = append(p.Labels, data) 463 | } else { 464 | p.Data[field] = data 465 | } 466 | } 467 | 468 | // we use a quite restrictive regexp for the filenames; in particular: 469 | // 470 | // - a limited set of extensions. We are careful to limit the content-types 471 | // we will serve the files with, but somebody might accidentally point an 472 | // Apache or nginx at the upload directory, which would serve js files as 473 | // application/javascript and open XSS vulnerabilities. We also allow gzipped 474 | // text and json on the same basis (there's really no sense allowing gzipped images). 475 | // 476 | // - no silly characters (/, ctrl chars, etc) 477 | // 478 | // - nothing starting with '.' 479 | var filenameRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-]+\.(jpg|png|txt|json|txt\.gz|json\.gz)$`) 480 | 481 | // saveFormPart saves a file upload to the report directory. 482 | // 483 | // Returns the leafname of the saved file. 484 | func saveFormPart(leafName string, reader io.Reader, reportDir string) (string, error) { 485 | if !filenameRegexp.MatchString(leafName) { 486 | return "", fmt.Errorf("Invalid upload filename") 487 | } 488 | 489 | fullName := filepath.Join(reportDir, leafName) 490 | 491 | log.Println("Saving uploaded file", leafName, "to", fullName) 492 | 493 | f, err := os.Create(fullName) 494 | if err != nil { 495 | return "", err 496 | } 497 | defer f.Close() 498 | 499 | _, err = io.Copy(f, reader) 500 | if err != nil { 501 | return "", err 502 | } 503 | 504 | return leafName, nil 505 | } 506 | 507 | // we require a sensible extension, and don't allow the filename to start with 508 | // '.' 509 | var logRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-][a-zA-Z0-9_.-]*\.(log|txt)(\.gz)?$`) 510 | 511 | // saveLogPart saves a log upload to the report directory. 512 | // 513 | // Returns the leafname of the saved file. 514 | func saveLogPart(logNum int, filename string, reader io.Reader, reportDir string) (string, error) { 515 | // pick a name to save the log file with. 516 | // 517 | // some clients use sensible names (foo.N.log), which we preserve. For 518 | // others, we just make up a filename. 519 | // 520 | // We append a ".gz" extension if not already present, as the final file we store on 521 | // disk will be gzipped. The original filename may or may not contain a '.gz' depending 522 | // on the client that uploaded it, and if it was uploaded already compressed. 523 | 524 | var leafName string 525 | if logRegexp.MatchString(filename) { 526 | leafName = filename 527 | if !strings.HasSuffix(filename, ".gz") { 528 | leafName += ".gz" 529 | } 530 | } else { 531 | leafName = fmt.Sprintf("logs-%04d.log.gz", logNum) 532 | } 533 | 534 | fullname := filepath.Join(reportDir, leafName) 535 | 536 | f, err := os.Create(fullname) 537 | if err != nil { 538 | return "", err 539 | } 540 | defer f.Close() 541 | 542 | gz := gzip.NewWriter(f) 543 | defer gz.Close() 544 | 545 | _, err = io.Copy(gz, reader) 546 | if err != nil { 547 | return "", err 548 | } 549 | 550 | return leafName, nil 551 | } 552 | 553 | func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, listingURL string) (*submitResponse, error) { 554 | var summaryBuf bytes.Buffer 555 | resp := submitResponse{} 556 | p.WriteTo(&summaryBuf) 557 | if err := gzipAndSave(summaryBuf.Bytes(), reportDir, "details.log.gz"); err != nil { 558 | return nil, err 559 | } 560 | 561 | if err := s.submitGithubIssue(ctx, p, listingURL, &resp); err != nil { 562 | return nil, err 563 | } 564 | 565 | if err := s.submitGitlabIssue(p, listingURL, &resp); err != nil { 566 | return nil, err 567 | } 568 | 569 | if err := s.submitSlackNotification(p, listingURL); err != nil { 570 | return nil, err 571 | } 572 | 573 | if err := s.sendEmail(p, reportDir, listingURL); err != nil { 574 | return nil, err 575 | } 576 | 577 | genericHookPayload := genericWebhookPayload{ 578 | payload: p, 579 | ReportURL: resp.ReportURL, 580 | ListingURL: listingURL, 581 | } 582 | 583 | if err := s.submitGenericWebhook(genericHookPayload); err != nil { 584 | return nil, err 585 | } 586 | 587 | // finally, write the details to details.json 588 | if err := s.writeJSONDetailsFile(reportDir, genericHookPayload); err != nil { 589 | return nil, err 590 | } 591 | 592 | return &resp, nil 593 | } 594 | 595 | // `writeJSONDetailsFile` records all the details of the rageshake in `details.json` in the report directory. 596 | func (s *submitServer) writeJSONDetailsFile(reportDir string, genericHookPayload genericWebhookPayload) error { 597 | f, err := os.Create(filepath.Join(reportDir, "details.json")) 598 | if err != nil { 599 | return err 600 | } 601 | defer f.Close() 602 | 603 | buffered := bufio.NewWriter(f) 604 | if err = json.NewEncoder(buffered).Encode(genericHookPayload); err != nil { 605 | return err 606 | } 607 | return buffered.Flush() 608 | } 609 | 610 | // submitGenericWebhook submits a basic JSON body to an endpoint configured in the config 611 | // 612 | // The request does not include the log body, only the metadata in the payload, 613 | // with the required listingURL to obtain the logs over http if required. 614 | // 615 | // If a github or gitlab issue was previously made, the reportURL will also be passed. 616 | // 617 | // Uses a goroutine to handle the http request asynchronously as by this point all critical 618 | // information has been stored. 619 | 620 | func (s *submitServer) submitGenericWebhook(genericHookPayload genericWebhookPayload) error { 621 | if s.genericWebhookClient == nil { 622 | return nil 623 | } 624 | for _, url := range s.cfg.GenericWebhookURLs { 625 | // Enrich the payload with a reportURL and listingURL, to convert a single struct 626 | // to JSON easily 627 | 628 | payloadBuffer := new(bytes.Buffer) 629 | json.NewEncoder(payloadBuffer).Encode(genericHookPayload) 630 | req, err := http.NewRequest("POST", url, payloadBuffer) 631 | req.Header.Set("Content-Type", "application/json") 632 | if err != nil { 633 | log.Println("Unable to submit to URL ", url, " ", err) 634 | return err 635 | } 636 | log.Println("Making generic webhook request to URL ", url) 637 | go s.sendGenericWebhook(req) 638 | } 639 | return nil 640 | } 641 | 642 | func (s *submitServer) sendGenericWebhook(req *http.Request) { 643 | resp, err := s.genericWebhookClient.Do(req) 644 | if err != nil { 645 | log.Println("Unable to submit notification", err) 646 | } else { 647 | defer resp.Body.Close() 648 | log.Println("Got response", resp.Status) 649 | } 650 | } 651 | 652 | func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listingURL string, resp *submitResponse) error { 653 | if s.ghClient == nil { 654 | return nil 655 | } 656 | 657 | // submit a github issue 658 | ghProj := s.cfg.GithubProjectMappings[p.AppName] 659 | if ghProj == "" { 660 | log.Println("Not creating GH issue for unknown app", p.AppName) 661 | return nil 662 | } 663 | splits := strings.SplitN(ghProj, "/", 2) 664 | if len(splits) < 2 { 665 | log.Println("Can't create GH issue for invalid repo", ghProj) 666 | } 667 | owner, repo := splits[0], splits[1] 668 | 669 | issueReq, err := buildGithubIssueRequest(p, listingURL, s.issueTemplate) 670 | if err != nil { 671 | return err 672 | } 673 | 674 | issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, issueReq) 675 | if err != nil { 676 | return err 677 | } 678 | 679 | log.Println("Created issue:", *issue.HTMLURL) 680 | 681 | resp.ReportURL = *issue.HTMLURL 682 | 683 | return nil 684 | } 685 | 686 | func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *submitResponse) error { 687 | if s.glClient == nil { 688 | return nil 689 | } 690 | 691 | glProj := s.cfg.GitlabProjectMappings[p.AppName] 692 | glLabels := s.cfg.GitlabProjectLabels[p.AppName] 693 | 694 | issueReq, err := buildGitlabIssueRequest(p, listingURL, s.issueTemplate, glLabels, s.cfg.GitlabIssueConfidential) 695 | if err != nil { 696 | return err 697 | } 698 | 699 | issue, _, err := s.glClient.Issues.CreateIssue(glProj, issueReq) 700 | 701 | if err != nil { 702 | return err 703 | } 704 | 705 | log.Println("Created issue:", issue.WebURL) 706 | 707 | resp.ReportURL = issue.WebURL 708 | 709 | return nil 710 | } 711 | 712 | func (s *submitServer) submitSlackNotification(p payload, listingURL string) error { 713 | if s.slack == nil { 714 | return nil 715 | } 716 | 717 | slackBuf := fmt.Sprintf( 718 | "%s\nApplication: %s\nReport: %s", 719 | p.UserText, p.AppName, listingURL, 720 | ) 721 | 722 | err := s.slack.Notify(slackBuf) 723 | if err != nil { 724 | return err 725 | } 726 | 727 | return nil 728 | } 729 | 730 | func buildReportTitle(p payload) string { 731 | // set the title to the first (non-empty) line of the user's report, if any 732 | trimmedUserText := strings.TrimSpace(p.UserText) 733 | if trimmedUserText == "" { 734 | return "Untitled report" 735 | } 736 | 737 | if i := strings.IndexAny(trimmedUserText, "\r\n"); i >= 0 { 738 | return trimmedUserText[0:i] 739 | } 740 | 741 | return trimmedUserText 742 | } 743 | 744 | func buildGenericIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (title string, body []byte, err error) { 745 | var bodyBuf bytes.Buffer 746 | 747 | issuePayload := issueBodyTemplatePayload{ 748 | payload: p, 749 | ListingURL: listingURL, 750 | } 751 | 752 | if err = bodyTemplate.Execute(&bodyBuf, issuePayload); err != nil { 753 | return 754 | } 755 | 756 | title = buildReportTitle(p) 757 | body = bodyBuf.Bytes() 758 | 759 | return 760 | } 761 | 762 | func buildGithubIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (*github.IssueRequest, error) { 763 | title, body, err := buildGenericIssueRequest(p, listingURL, bodyTemplate) 764 | if err != nil { 765 | return nil, err 766 | } 767 | 768 | labels := p.Labels 769 | // go-github doesn't like nils 770 | if labels == nil { 771 | labels = []string{} 772 | } 773 | bodyStr := string(body) 774 | return &github.IssueRequest{ 775 | Title: &title, 776 | Body: &bodyStr, 777 | Labels: &labels, 778 | }, nil 779 | } 780 | 781 | func buildGitlabIssueRequest(p payload, listingURL string, bodyTemplate *template.Template, labels []string, confidential bool) (*gitlab.CreateIssueOptions, error) { 782 | title, body, err := buildGenericIssueRequest(p, listingURL, bodyTemplate) 783 | if err != nil { 784 | return nil, err 785 | } 786 | 787 | if p.Labels != nil { 788 | labels = append(labels, p.Labels...) 789 | } 790 | 791 | bodyStr := string(body) 792 | return &gitlab.CreateIssueOptions{ 793 | Title: &title, 794 | Description: &bodyStr, 795 | Confidential: &confidential, 796 | Labels: labels, 797 | }, nil 798 | } 799 | 800 | func (s *submitServer) sendEmail(p payload, reportDir string, listingURL string) error { 801 | if len(s.cfg.EmailAddresses) == 0 { 802 | return nil 803 | } 804 | 805 | title, body, err := buildGenericIssueRequest(p, listingURL, s.emailTemplate) 806 | if err != nil { 807 | return err 808 | } 809 | 810 | e := email.NewEmail() 811 | 812 | e.From = "Rageshake " 813 | if s.cfg.EmailFrom != "" { 814 | e.From = s.cfg.EmailFrom 815 | } 816 | 817 | e.To = s.cfg.EmailAddresses 818 | e.Subject = fmt.Sprintf("[%s] %s", p.AppName, title) 819 | e.Text = body 820 | 821 | allFiles := append(p.Files, p.Logs...) 822 | for _, file := range allFiles { 823 | fullPath := filepath.Join(reportDir, file) 824 | e.AttachFile(fullPath) 825 | } 826 | 827 | var auth smtp.Auth = nil 828 | if s.cfg.SMTPPassword != "" || s.cfg.SMTPUsername != "" { 829 | host, _, _ := net.SplitHostPort(s.cfg.SMTPServer) 830 | auth = smtp.PlainAuth("", s.cfg.SMTPUsername, s.cfg.SMTPPassword, host) 831 | } 832 | err = e.Send(s.cfg.SMTPServer, auth) 833 | if err != nil { 834 | return err 835 | } 836 | 837 | return nil 838 | } 839 | 840 | func respond(code int, w http.ResponseWriter) { 841 | w.WriteHeader(code) 842 | w.Write([]byte("{}")) 843 | } 844 | 845 | func gzipAndSave(data []byte, dirname, fpath string) error { 846 | fpath = filepath.Join(dirname, fpath) 847 | 848 | if _, err := os.Stat(fpath); err == nil { 849 | return fmt.Errorf("file already exists") // the user can just retry 850 | } 851 | var b bytes.Buffer 852 | gz := gzip.NewWriter(&b) 853 | if _, err := gz.Write(data); err != nil { 854 | return err 855 | } 856 | if err := gz.Flush(); err != nil { 857 | return err 858 | } 859 | if err := gz.Close(); err != nil { 860 | return err 861 | } 862 | if err := ioutil.WriteFile(fpath, b.Bytes(), 0644); err != nil { 863 | return err 864 | } 865 | return nil 866 | } 867 | -------------------------------------------------------------------------------- /submit_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Vector Creations Ltd 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "bytes" 21 | "compress/gzip" 22 | "io" 23 | "io/ioutil" 24 | "net/http" 25 | "net/http/httptest" 26 | "os" 27 | "path/filepath" 28 | "strconv" 29 | "strings" 30 | "testing" 31 | "text/template" 32 | ) 33 | 34 | // testParsePayload builds a /submit request with the given body, and calls 35 | // parseRequest with it. 36 | // 37 | // if tempDir is empty, a new temp dir is created, and deleted when the test 38 | // completes. 39 | func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*payload, *http.Response) { 40 | req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body)) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | req.Header.Set("Content-Length", strconv.Itoa(len(body))) 45 | if contentType != "" { 46 | req.Header.Set("Content-Type", contentType) 47 | } 48 | 49 | // temporary dir for the uploaded files 50 | if tempDir == "" { 51 | tempDir = mkTempDir(t) 52 | defer os.RemoveAll(tempDir) 53 | } 54 | 55 | rr := httptest.NewRecorder() 56 | p := parseRequest(rr, req, tempDir) 57 | return p, rr.Result() 58 | } 59 | 60 | func submitSimpleRequestToServer(t *testing.T, allowedAppNameMap map[string]bool, body string) int { 61 | // Submit a request without files to the server and return statusCode 62 | // Could be extended with more complicated config; aimed here just to 63 | // test options for allowedAppNameMap 64 | 65 | req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body)) 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | req.Header.Set("Content-Length", strconv.Itoa(len(body))) 70 | w := httptest.NewRecorder() 71 | 72 | var cfg config 73 | s := &submitServer{nil, nil, nil, nil, "/", nil, nil, allowedAppNameMap, &cfg} 74 | 75 | s.ServeHTTP(w, req) 76 | rsp := w.Result() 77 | return rsp.StatusCode 78 | } 79 | 80 | func TestAppNames(t *testing.T) { 81 | body := `{ 82 | "app": "alice", 83 | "logs": [ ], 84 | "text": "test message", 85 | "user_agent": "Mozilla/0.9", 86 | "version": "0.9.9" 87 | }` 88 | validAppNameMap := map[string]bool{ 89 | "alice": true, 90 | } 91 | if submitSimpleRequestToServer(t, validAppNameMap, body) != 200 { 92 | t.Fatal("matching app was not accepted") 93 | } 94 | 95 | invalidAppNameMap := map[string]bool{ 96 | "bob": true, 97 | } 98 | if submitSimpleRequestToServer(t, invalidAppNameMap, body) != 400 { 99 | t.Fatal("nonmatching app was not rejected") 100 | } 101 | 102 | emptyAppNameMap := make(map[string]bool) 103 | if submitSimpleRequestToServer(t, emptyAppNameMap, body) != 200 { 104 | t.Fatal("empty map did not allow all") 105 | } 106 | } 107 | 108 | func TestEmptyJson(t *testing.T) { 109 | body := "{}" 110 | 111 | // we just test it is parsed without errors for now 112 | p, _ := testParsePayload(t, body, "application/json", "") 113 | if p == nil { 114 | t.Fatal("parseRequest returned nil") 115 | } 116 | if len(p.Labels) != 0 { 117 | t.Errorf("Labels: got %#v, want []", p.Labels) 118 | } 119 | } 120 | 121 | func TestJsonUpload(t *testing.T) { 122 | reportDir := mkTempDir(t) 123 | defer os.RemoveAll(reportDir) 124 | 125 | body := `{ 126 | "app": "riot-web", 127 | "logs": [ 128 | { 129 | "id": "instance-0.99152119701215051494400738905", 130 | "lines": "line1\nline2" 131 | } 132 | ], 133 | "text": "test message", 134 | "user_agent": "Mozilla", 135 | "version": "0.9.9" 136 | }` 137 | 138 | p, _ := testParsePayload(t, body, "application/json", reportDir) 139 | 140 | if p == nil { 141 | t.Fatal("parseRequest returned nil") 142 | } 143 | 144 | wanted := "test message" 145 | if p.UserText != wanted { 146 | t.Errorf("user text: got %s, want %s", p.UserText, wanted) 147 | } 148 | wanted = "riot-web" 149 | if p.AppName != wanted { 150 | t.Errorf("appname: got %s, want %s", p.AppName, wanted) 151 | } 152 | wanted = "0.9.9" 153 | if p.Data["Version"] != wanted { 154 | t.Errorf("version: got %s, want %s", p.Data["Version"], wanted) 155 | } 156 | 157 | checkUploadedFile(t, reportDir, "logs-0000.log.gz", true, "line1\nline2") 158 | } 159 | 160 | func TestMultipartUpload(t *testing.T) { 161 | reportDir := mkTempDir(t) 162 | defer os.RemoveAll(reportDir) 163 | 164 | p, _ := testParsePayload(t, multipartBody(), 165 | "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", 166 | reportDir, 167 | ) 168 | 169 | if p == nil { 170 | t.Fatal("parseRequest returned nil") 171 | } 172 | 173 | checkParsedMultipartUpload(t, p) 174 | 175 | // check logs uploaded correctly 176 | checkUploadedFile(t, reportDir, "logs-0000.log.gz", true, "log\nlog\nlog") 177 | checkUploadedFile(t, reportDir, "console.0.log.gz", true, "log") 178 | checkUploadedFile(t, reportDir, "logs-0002.log.gz", true, "test\n") 179 | 180 | // check file uploaded correctly 181 | checkUploadedFile(t, reportDir, "passwd.txt", false, "bibblybobbly") 182 | checkUploadedFile(t, reportDir, "crash.log.gz", true, "test\n") 183 | } 184 | 185 | func multipartBody() (body string) { 186 | body = `------WebKitFormBoundarySsdgl8Nq9voFyhdO 187 | Content-Disposition: form-data; name="text" 188 | 189 | test words. 190 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 191 | Content-Disposition: form-data; name="app" 192 | 193 | riot-web 194 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 195 | Content-Disposition: form-data; name="version" 196 | 197 | UNKNOWN 198 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 199 | Content-Disposition: form-data; name="user_agent" 200 | 201 | Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36 202 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 203 | Content-Disposition: form-data; name="test-field" 204 | 205 | Test data 206 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 207 | Content-Disposition: form-data; name="log"; filename="instance-0.215954445471346461492087122412" 208 | Content-Type: text/plain 209 | 210 | log 211 | log 212 | log 213 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 214 | Content-Disposition: form-data; name="log"; filename="console.0.log" 215 | Content-Type: text/plain 216 | 217 | log 218 | ` 219 | 220 | body += `------WebKitFormBoundarySsdgl8Nq9voFyhdO 221 | Content-Disposition: form-data; name="compressed-log"; filename="instance-0.0109372050779190651492004373866" 222 | Content-Type: application/octet-stream 223 | 224 | ` 225 | body += string([]byte{ 226 | 0x1f, 0x8b, 0x08, 0x00, 0xbf, 0xd8, 0xf5, 0x58, 0x00, 0x03, 227 | 0x2b, 0x49, 0x2d, 0x2e, 0xe1, 0x02, 0x00, 228 | 0xc6, 0x35, 0xb9, 0x3b, 0x05, 0x00, 0x00, 0x00, 229 | 0x0a, 230 | }) 231 | 232 | body += `------WebKitFormBoundarySsdgl8Nq9voFyhdO 233 | Content-Disposition: form-data; name="file"; filename="passwd.txt" 234 | Content-Type: application/octet-stream 235 | 236 | bibblybobbly 237 | ` 238 | body += `------WebKitFormBoundarySsdgl8Nq9voFyhdO 239 | Content-Disposition: form-data; name="compressed-log"; filename="crash.log.gz" 240 | Content-Type: application/octet-stream 241 | 242 | ` 243 | body += string([]byte{ 244 | 0x1f, 0x8b, 0x08, 0x00, 0xbf, 0xd8, 0xf5, 0x58, 0x00, 0x03, 245 | 0x2b, 0x49, 0x2d, 0x2e, 0xe1, 0x02, 0x00, 246 | 0xc6, 0x35, 0xb9, 0x3b, 0x05, 0x00, 0x00, 0x00, 247 | 0x0a, 248 | }) 249 | 250 | body += "------WebKitFormBoundarySsdgl8Nq9voFyhdO--\n" 251 | return 252 | } 253 | 254 | func checkParsedMultipartUpload(t *testing.T, p *payload) { 255 | wanted := "test words." 256 | if p.UserText != wanted { 257 | t.Errorf("User text: got %s, want %s", p.UserText, wanted) 258 | } 259 | if len(p.Logs) != 4 { 260 | t.Errorf("Log length: got %d, want 4", len(p.Logs)) 261 | } 262 | // One extra data field to account for User Agent being parsed into two fields 263 | if len(p.Data) != 4 { 264 | t.Errorf("Data length: got %d, want 4", len(p.Data)) 265 | } 266 | if len(p.Labels) != 0 { 267 | t.Errorf("Labels: got %#v, want []", p.Labels) 268 | } 269 | wanted = "Test data" 270 | if p.Data["test-field"] != wanted { 271 | t.Errorf("test-field: got %s, want %s", p.Data["test-field"], wanted) 272 | } 273 | wanted = "logs-0000.log.gz" 274 | if p.Logs[0] != wanted { 275 | t.Errorf("Log 0: got %s, want %s", p.Logs[0], wanted) 276 | } 277 | wanted = "console.0.log.gz" 278 | if p.Logs[1] != wanted { 279 | t.Errorf("Log 1: got %s, want %s", p.Logs[1], wanted) 280 | } 281 | wanted = "logs-0002.log.gz" 282 | if p.Logs[2] != wanted { 283 | t.Errorf("Log 2: got %s, want %s", p.Logs[2], wanted) 284 | } 285 | wanted = "crash.log.gz" 286 | if p.Logs[3] != wanted { 287 | t.Errorf("Log 3: got %s, want %s", p.Logs[3], wanted) 288 | } 289 | } 290 | 291 | func TestLabels(t *testing.T) { 292 | body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO 293 | Content-Disposition: form-data; name="label" 294 | 295 | label1 296 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 297 | Content-Disposition: form-data; name="label" 298 | 299 | label2 300 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO-- 301 | ` 302 | p, _ := testParsePayload(t, body, 303 | "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", 304 | "", 305 | ) 306 | 307 | if p == nil { 308 | t.Fatal("parseRequest returned nil") 309 | } 310 | 311 | wantedLabels := []string{"label1", "label2"} 312 | if !stringSlicesEqual(p.Labels, wantedLabels) { 313 | t.Errorf("Labels: got %v, want %v", p.Labels, wantedLabels) 314 | } 315 | } 316 | 317 | func stringSlicesEqual(got, want []string) bool { 318 | if len(got) != len(want) { 319 | return false 320 | } 321 | 322 | for i := range got { 323 | if got[i] != want[i] { 324 | return false 325 | } 326 | } 327 | return true 328 | } 329 | 330 | /* FIXME these should just give a message in the details file now 331 | func TestEmptyFilename(t *testing.T) { 332 | body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO 333 | Content-Disposition: form-data; name="file" 334 | 335 | file 336 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO-- 337 | ` 338 | p, resp := testParsePayload(t, body, "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", "") 339 | if p != nil { 340 | t.Error("parsePayload accepted upload with no filename") 341 | } 342 | 343 | if resp.StatusCode != 400 { 344 | t.Errorf("response code: got %v, want %v", resp.StatusCode, 400) 345 | } 346 | } 347 | 348 | func TestBadFilename(t *testing.T) { 349 | body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO 350 | Content-Disposition: form-data; name="file"; filename="etc/passwd" 351 | 352 | file 353 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO-- 354 | ` 355 | p, resp := testParsePayload(t, body, "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", "") 356 | if p != nil { 357 | t.Error("parsePayload accepted upload with bad filename") 358 | } 359 | 360 | if resp.StatusCode != 400 { 361 | t.Errorf("response code: got %v, want %v", resp.StatusCode, 400) 362 | } 363 | } 364 | */ 365 | 366 | func checkUploadedFile(t *testing.T, reportDir, leafName string, gzipped bool, wanted string) { 367 | fi, err := os.Open(filepath.Join(reportDir, leafName)) 368 | if err != nil { 369 | t.Errorf("unable to open uploaded file %s: %v", leafName, err) 370 | return 371 | } 372 | defer fi.Close() 373 | var rdr io.Reader 374 | if !gzipped { 375 | rdr = fi 376 | } else { 377 | gz, err2 := gzip.NewReader(fi) 378 | if err2 != nil { 379 | t.Errorf("unable to ungzip uploaded file %s: %v", leafName, err2) 380 | return 381 | } 382 | defer gz.Close() 383 | rdr = gz 384 | } 385 | dat, err := ioutil.ReadAll(rdr) 386 | if err != nil { 387 | t.Errorf("unable to read uploaded file %s: %v", leafName, err) 388 | return 389 | } 390 | 391 | datstr := string(dat) 392 | if datstr != wanted { 393 | t.Errorf("File %s: got %s, want %s", leafName, datstr, wanted) 394 | } 395 | } 396 | 397 | func mkTempDir(t *testing.T) string { 398 | td, err := ioutil.TempDir("", "rageshake_test") 399 | if err != nil { 400 | t.Fatal(err) 401 | } 402 | return td 403 | } 404 | 405 | /***************************************************************************** 406 | * 407 | * buildGithubIssueRequest tests 408 | */ 409 | 410 | // General test of Github issue formatting. 411 | func TestBuildGithubIssue(t *testing.T) { 412 | body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO 413 | Content-Disposition: form-data; name="text" 414 | 415 | 416 | test words. 417 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 418 | Content-Disposition: form-data; name="app" 419 | 420 | riot-web 421 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 422 | Content-Disposition: form-data; name="User-Agent" 423 | 424 | xxx 425 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 426 | Content-Disposition: form-data; name="user_id" 427 | 428 | id 429 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 430 | Content-Disposition: form-data; name="device_id" 431 | 432 | id 433 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 434 | Content-Disposition: form-data; name="version" 435 | 436 | 1 437 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 438 | Content-Disposition: form-data; name="file"; filename="passwd.txt" 439 | 440 | file 441 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO-- 442 | ` 443 | p, _ := testParsePayload(t, body, 444 | "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", 445 | "", 446 | ) 447 | 448 | if p == nil { 449 | t.Fatal("parseRequest returned nil") 450 | } 451 | 452 | parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) 453 | issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate) 454 | if err != nil { 455 | t.Fatalf("Error building issue request: %s", err) 456 | } 457 | 458 | if *issueReq.Title != "test words." { 459 | t.Errorf("Title: got %s, want %s", *issueReq.Title, "test words.") 460 | } 461 | expectedBody := "User message:\n\ntest words.\n\nUser-Agent: `xxx`\nVersion: `1`\ndevice_id: `id`\nuser_id: `id`\n\n[Logs](http://test/listing/foo) ([archive](http://test/listing/foo?format=tar.gz)) / [passwd.txt](http://test/listing/foo/passwd.txt)\n" 462 | if *issueReq.Body != expectedBody { 463 | t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody) 464 | } 465 | } 466 | 467 | func TestBuildGithubIssueLeadingNewline(t *testing.T) { 468 | body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO 469 | Content-Disposition: form-data; name="text" 470 | 471 | 472 | test words. 473 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO 474 | Content-Disposition: form-data; name="app" 475 | 476 | riot-web 477 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO-- 478 | ` 479 | p, _ := testParsePayload(t, body, 480 | "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", 481 | "", 482 | ) 483 | 484 | if p == nil { 485 | t.Fatal("parseRequest returned nil") 486 | } 487 | 488 | parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) 489 | issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate) 490 | if err != nil { 491 | t.Fatalf("Error building issue request: %s", err) 492 | } 493 | 494 | if *issueReq.Title != "test words." { 495 | t.Errorf("Title: got %s, want %s", *issueReq.Title, "test words.") 496 | } 497 | expectedBody := "User message:\n\ntest words.\n" 498 | if !strings.HasPrefix(*issueReq.Body, expectedBody) { 499 | t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody) 500 | } 501 | } 502 | 503 | func TestBuildGithubIssueEmptyBody(t *testing.T) { 504 | body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO 505 | Content-Disposition: form-data; name="text" 506 | 507 | ------WebKitFormBoundarySsdgl8Nq9voFyhdO-- 508 | ` 509 | p, _ := testParsePayload(t, body, 510 | "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", 511 | "", 512 | ) 513 | 514 | if p == nil { 515 | t.Fatal("parseRequest returned nil") 516 | } 517 | 518 | parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) 519 | issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate) 520 | if err != nil { 521 | t.Fatalf("Error building issue request: %s", err) 522 | } 523 | 524 | if *issueReq.Title != "Untitled report" { 525 | t.Errorf("Title: got %s, want %s", *issueReq.Title, "Untitled report") 526 | } 527 | expectedBody := "User message:\n\n\n" 528 | if !strings.HasPrefix(*issueReq.Body, expectedBody) { 529 | t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody) 530 | } 531 | } 532 | 533 | func TestSortDataKeys(t *testing.T) { 534 | expect := ` 535 | Number of logs: 0 536 | Application: 537 | Labels: 538 | User-Agent: xxx 539 | Version: 1 540 | device_id: id 541 | user_id: id 542 | ` 543 | expect = strings.TrimSpace(expect) 544 | sample := []struct { 545 | data map[string]string 546 | }{ 547 | { 548 | map[string]string{ 549 | "Version": "1", 550 | "User-Agent": "xxx", 551 | "user_id": "id", 552 | "device_id": "id", 553 | }, 554 | }, 555 | { 556 | map[string]string{ 557 | "user_id": "id", 558 | "device_id": "id", 559 | "Version": "1", 560 | "User-Agent": "xxx", 561 | }, 562 | }, 563 | } 564 | var buf bytes.Buffer 565 | for _, v := range sample { 566 | p := payload{Data: v.data} 567 | buf.Reset() 568 | p.WriteTo(&buf) 569 | got := strings.TrimSpace(buf.String()) 570 | if got != expect { 571 | t.Errorf("expected %s got %s", expect, got) 572 | } 573 | } 574 | 575 | parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) 576 | for k, v := range sample { 577 | p := payload{Data: v.data} 578 | res, err := buildGithubIssueRequest(p, "", parsedIssueTemplate) 579 | if err != nil { 580 | t.Fatalf("Error building issue request: %s", err) 581 | } 582 | got := *res.Body 583 | if k == 0 { 584 | expect = got 585 | continue 586 | } 587 | if got != expect { 588 | t.Errorf("expected %s got %s", expect, got) 589 | } 590 | } 591 | } 592 | 593 | func TestParseUserAgent(t *testing.T) { 594 | reportDir := mkTempDir(t) 595 | defer os.RemoveAll(reportDir) 596 | 597 | body := `{ 598 | "app": "riot-web", 599 | "logs": [], 600 | "text": "test message", 601 | "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.6723.91 Safari/537.3", 602 | "version": "0.9.9" 603 | }` 604 | 605 | p, _ := testParsePayload(t, body, "application/json", reportDir) 606 | 607 | if p == nil { 608 | t.Fatal("parseRequest returned nil") 609 | } 610 | 611 | wanted := "Chrome 130.0.6723 on Windows 10 running on Other device" 612 | if p.Data["Parsed-User-Agent"] != wanted { 613 | t.Errorf("user agent: got %s, want %s", p.Data["Parsed-User-Agent"], wanted) 614 | } 615 | } 616 | -------------------------------------------------------------------------------- /templates/README.md: -------------------------------------------------------------------------------- 1 | This directory contains the default templates that are used by the rageshake server. 2 | 3 | The templates can be overridden via settings in the config file. 4 | 5 | The templates are as follows: 6 | 7 | * `issue_body.tmpl`: Used when filing an issue at Github or Gitlab, and gives the issue description. Override via 8 | the `issue_body_template_file` setting in the configuration file. 9 | * `email_body.tmpl`: Used when sending an email. Override via the `email_body_template_file` configuration setting. 10 | 11 | See https://pkg.go.dev/text/template#pkg-overview for documentation of the template language. 12 | 13 | The following properties are defined on the input (accessible via `.` or `$`): 14 | 15 | | Name | Type | Description | 16 | |--------------|---------------------|---------------------------------------------------------------------------------------------------| 17 | | `ID` | `string` | The unique ID for this rageshake. | 18 | | `UserText` | `string` | A multi-line string containing the user description of the fault (from `text` in the submission). | 19 | | `AppName` | `string` | A short slug to identify the app making the report (from `app` in the submission). | 20 | | `Labels` | `[]string` | A list of labels requested by the application. | 21 | | `Data` | `map[string]string` | A map of other key/value pairs included in the submission. | 22 | | `Logs` | `[]string` | A list of log file names. | 23 | | `LogErrors` | `[]string` | Set if there are log parsing errors. | 24 | | `Files` | `[]string` | A list of other files (not logs) uploaded as part of the rageshake. | 25 | | `FileErrors` | `[]string` | Set if there are file parsing errors. | 26 | | `ListingURL` | `string` | Complete link to the listing URL that contains all uploaded logs. | 27 | -------------------------------------------------------------------------------- /templates/email_body.tmpl: -------------------------------------------------------------------------------- 1 | User message: 2 | {{ .UserText }} 3 | 4 | {{ range $key, $val := .Data -}} 5 | {{ $key }}: "{{ $val }}" 6 | {{ end }} 7 | [Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz)) 8 | {{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }}) 9 | {{- end }} 10 | -------------------------------------------------------------------------------- /templates/issue_body.tmpl: -------------------------------------------------------------------------------- 1 | User message: 2 | {{ .UserText }} 3 | 4 | {{ range $key, $val := .Data -}} 5 | {{ $key }}: `{{ $val }}` 6 | {{ end }} 7 | [Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz)) 8 | {{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }}) 9 | {{- end }} 10 | -------------------------------------------------------------------------------- /towncrier.toml: -------------------------------------------------------------------------------- 1 | [tool.towncrier] 2 | filename = "CHANGES.md" 3 | directory = "changelog.d" 4 | issue_format = "[\\#{issue}](https://github.com/matrix-org/rageshake/issues/{issue})" 5 | 6 | [[tool.towncrier.type]] 7 | directory = "feature" 8 | name = "Features" 9 | showcontent = true 10 | 11 | [[tool.towncrier.type]] 12 | directory = "bugfix" 13 | name = "Bugfixes" 14 | showcontent = true 15 | 16 | [[tool.towncrier.type]] 17 | directory = "misc" 18 | name = "Internal Changes" 19 | showcontent = true 20 | --------------------------------------------------------------------------------