├── .github
├── guide
│ ├── cloudflare-dns.webp
│ ├── hostinger-1.location.webp
│ ├── hostinger-2.os.webp
│ ├── hostinger-3.malware.webp
│ ├── hostinger-4.configuration.webp
│ ├── hostinger-5.configuration.webp
│ ├── hostinger-6.complete.webp
│ ├── hostinger-7.wait.webp
│ ├── hostinger-8.connect.webp
│ └── hostinger-9.panel.webp
└── workflows
│ ├── triage_issue.yml
│ └── triage_pr.yml
├── .gitignore
├── Caddyfile
├── README.md
├── compose.yml
├── generate_config.sh
└── migrations
├── .gitignore
├── 20240929-autumn-rewrite---prod-migration.mjs
└── 20240929-autumn-rewrite.mjs
/.github/guide/cloudflare-dns.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/revoltchat/self-hosted/99b0d743af8a70d71ce0a8a205500efff80330de/.github/guide/cloudflare-dns.webp
--------------------------------------------------------------------------------
/.github/guide/hostinger-1.location.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/revoltchat/self-hosted/99b0d743af8a70d71ce0a8a205500efff80330de/.github/guide/hostinger-1.location.webp
--------------------------------------------------------------------------------
/.github/guide/hostinger-2.os.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/revoltchat/self-hosted/99b0d743af8a70d71ce0a8a205500efff80330de/.github/guide/hostinger-2.os.webp
--------------------------------------------------------------------------------
/.github/guide/hostinger-3.malware.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/revoltchat/self-hosted/99b0d743af8a70d71ce0a8a205500efff80330de/.github/guide/hostinger-3.malware.webp
--------------------------------------------------------------------------------
/.github/guide/hostinger-4.configuration.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/revoltchat/self-hosted/99b0d743af8a70d71ce0a8a205500efff80330de/.github/guide/hostinger-4.configuration.webp
--------------------------------------------------------------------------------
/.github/guide/hostinger-5.configuration.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/revoltchat/self-hosted/99b0d743af8a70d71ce0a8a205500efff80330de/.github/guide/hostinger-5.configuration.webp
--------------------------------------------------------------------------------
/.github/guide/hostinger-6.complete.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/revoltchat/self-hosted/99b0d743af8a70d71ce0a8a205500efff80330de/.github/guide/hostinger-6.complete.webp
--------------------------------------------------------------------------------
/.github/guide/hostinger-7.wait.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/revoltchat/self-hosted/99b0d743af8a70d71ce0a8a205500efff80330de/.github/guide/hostinger-7.wait.webp
--------------------------------------------------------------------------------
/.github/guide/hostinger-8.connect.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/revoltchat/self-hosted/99b0d743af8a70d71ce0a8a205500efff80330de/.github/guide/hostinger-8.connect.webp
--------------------------------------------------------------------------------
/.github/guide/hostinger-9.panel.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/revoltchat/self-hosted/99b0d743af8a70d71ce0a8a205500efff80330de/.github/guide/hostinger-9.panel.webp
--------------------------------------------------------------------------------
/.github/workflows/triage_issue.yml:
--------------------------------------------------------------------------------
1 | name: Add Issue to Board
2 |
3 | on:
4 | issues:
5 | types: [opened]
6 |
7 | jobs:
8 | track_issue:
9 | runs-on: ubuntu-latest
10 | steps:
11 | - name: Get project data
12 | env:
13 | GITHUB_TOKEN: ${{ secrets.PAT }}
14 | run: |
15 | gh api graphql -f query='
16 | query {
17 | organization(login: "revoltchat"){
18 | projectV2(number: 3) {
19 | id
20 | fields(first:20) {
21 | nodes {
22 | ... on ProjectV2SingleSelectField {
23 | id
24 | name
25 | options {
26 | id
27 | name
28 | }
29 | }
30 | }
31 | }
32 | }
33 | }
34 | }' > project_data.json
35 |
36 | echo 'PROJECT_ID='$(jq '.data.organization.projectV2.id' project_data.json) >> $GITHUB_ENV
37 | echo 'STATUS_FIELD_ID='$(jq '.data.organization.projectV2.fields.nodes[] | select(.name== "Status") | .id' project_data.json) >> $GITHUB_ENV
38 | echo 'TODO_OPTION_ID='$(jq '.data.organization.projectV2.fields.nodes[] | select(.name== "Status") | .options[] | select(.name=="Todo") |.id' project_data.json) >> $GITHUB_ENV
39 |
40 | - name: Add issue to project
41 | env:
42 | GITHUB_TOKEN: ${{ secrets.PAT }}
43 | ISSUE_ID: ${{ github.event.issue.node_id }}
44 | run: |
45 | item_id="$( gh api graphql -f query='
46 | mutation($project:ID!, $issue:ID!) {
47 | addProjectV2ItemById(input: {projectId: $project, contentId: $issue}) {
48 | item {
49 | id
50 | }
51 | }
52 | }' -f project=$PROJECT_ID -f issue=$ISSUE_ID --jq '.data.addProjectV2ItemById.item.id')"
53 |
54 | echo 'ITEM_ID='$item_id >> $GITHUB_ENV
55 |
--------------------------------------------------------------------------------
/.github/workflows/triage_pr.yml:
--------------------------------------------------------------------------------
1 | name: Add PR to Board
2 |
3 | on:
4 | pull_request_target:
5 | types: [opened, synchronize, ready_for_review, review_requested]
6 |
7 | jobs:
8 | track_pr:
9 | runs-on: ubuntu-latest
10 | steps:
11 | - name: Get project data
12 | env:
13 | GITHUB_TOKEN: ${{ secrets.PAT }}
14 | run: |
15 | gh api graphql -f query='
16 | query {
17 | organization(login: "revoltchat"){
18 | projectV2(number: 5) {
19 | id
20 | fields(first:20) {
21 | nodes {
22 | ... on ProjectV2SingleSelectField {
23 | id
24 | name
25 | options {
26 | id
27 | name
28 | }
29 | }
30 | }
31 | }
32 | }
33 | }
34 | }' > project_data.json
35 |
36 | echo 'PROJECT_ID='$(jq '.data.organization.projectV2.id' project_data.json) >> $GITHUB_ENV
37 | echo 'STATUS_FIELD_ID='$(jq '.data.organization.projectV2.fields.nodes[] | select(.name== "Status") | .id' project_data.json) >> $GITHUB_ENV
38 | echo 'INCOMING_OPTION_ID='$(jq '.data.organization.projectV2.fields.nodes[] | select(.name== "Status") | .options[] | select(.name=="🆕 Untriaged") |.id' project_data.json) >> $GITHUB_ENV
39 |
40 | - name: Add PR to project
41 | env:
42 | GITHUB_TOKEN: ${{ secrets.PAT }}
43 | PR_ID: ${{ github.event.pull_request.node_id }}
44 | run: |
45 | item_id="$( gh api graphql -f query='
46 | mutation($project:ID!, $pr:ID!) {
47 | addProjectV2ItemById(input: {projectId: $project, contentId: $pr}) {
48 | item {
49 | id
50 | }
51 | }
52 | }' -f project=$PROJECT_ID -f pr=$PR_ID --jq '.data.addProjectV2ItemById.item.id')"
53 |
54 | echo 'ITEM_ID='$item_id >> $GITHUB_ENV
55 |
56 | - name: Set fields
57 | env:
58 | GITHUB_TOKEN: ${{ secrets.PAT }}
59 | run: |
60 | gh api graphql -f query='
61 | mutation (
62 | $project: ID!
63 | $item: ID!
64 | $status_field: ID!
65 | $status_value: String!
66 | ) {
67 | set_status: updateProjectV2ItemFieldValue(input: {
68 | projectId: $project
69 | itemId: $item
70 | fieldId: $status_field
71 | value: {
72 | singleSelectOptionId: $status_value
73 | }
74 | }) {
75 | projectV2Item {
76 | id
77 | }
78 | }
79 | }' -f project=$PROJECT_ID -f item=$ITEM_ID -f status_field=$STATUS_FIELD_ID -f status_value=${{ env.INCOMING_OPTION_ID }} --silent
80 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | data*
2 |
3 | .env
4 | .env.web
5 | Revolt.toml
6 |
7 | compose.override.yml
8 |
--------------------------------------------------------------------------------
/Caddyfile:
--------------------------------------------------------------------------------
1 | {$HOSTNAME} {
2 | route /api* {
3 | uri strip_prefix /api
4 | reverse_proxy http://api:14702
5 | }
6 |
7 | route /ws {
8 | uri strip_prefix /ws
9 | reverse_proxy http://events:14703
10 | }
11 |
12 | route /autumn* {
13 | uri strip_prefix /autumn
14 | reverse_proxy http://autumn:14704
15 | }
16 |
17 | route /january* {
18 | uri strip_prefix /january
19 | reverse_proxy http://january:14705
20 | }
21 |
22 | reverse_proxy http://web:5000
23 | }
24 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | Revolt Self-Hosted
4 |
5 | [](https://github.com/revoltchat/self-hosted/stargazers)
6 | [](https://github.com/revoltchat/self-hosted/network/members)
7 | [](https://github.com/revoltchat/self-hosted/pulls)
8 | [](https://github.com/revoltchat/self-hosted/issues)
9 | [](https://github.com/revoltchat/self-hosted/graphs/contributors)
10 | [](https://github.com/revoltchat/self-hosted/blob/main/LICENSE)
11 |
12 | Self-hosting Revolt using Docker
13 |
14 |
15 |
16 | This repository contains configurations and instructions that can be used for deploying Revolt.
17 |
18 | > [!WARNING]
19 | > If you are updating an instance from before November 28, 2024, please see the [notices section](#notices) at the bottom of this README!
20 |
21 | > [!IMPORTANT]
22 | > A list of security advisories is [provided at the bottom](#security-advisories).
23 |
24 | > [!NOTE]
25 | > Please consult _[What can I do with Revolt, and how do I self-host?](https://developers.revolt.chat/faq.html#admonition-what-can-i-do-with-revolt-and-how-do-i-self-host)_ on our developer site for information about licensing and brand use.
26 |
27 | > [!NOTE]
28 | > amd64 builds are not currently available for the web client.
29 |
30 | > [!NOTE]
31 | > This guide does not include working voice channels ([#138](https://github.com/revoltchat/self-hosted/pull/138#issuecomment-2762682655)). A [rework](https://github.com/revoltchat/backend/issues/313) is currently in progress.
32 |
33 | ## Table of Contents
34 | - [Deployment](#deployment)
35 | - [Updating](#updating)
36 | - [Advanced Deployment](#advanced-deployment)
37 | - [Additional Notes](#additional-notes)
38 | - [Custom Domain](#custom-domain)
39 | - [Placing Behind Another Reverse-Proxy or Another Port](#placing-behind-another-reverse-proxy-or-another-port)
40 | - [Insecurely Expose the Database](#insecurely-expose-the-database)
41 | - [Mongo Compatibility](#mongo-compatibility)
42 | - [Making Your Instance Invite-only](#making-your-instance-invite-only)
43 | - [Notices](#notices)
44 | - [Security Advisories](#security-advisories)
45 |
46 | ## Deployment
47 |
48 | To get started, find yourself a suitable server to deploy onto, we recommend starting with at least 2 vCPUs and 2 GB of memory.
49 |
50 | > [!TIP]
51 | >
52 | > **We've partnered with Hostinger to bring you a 20% discount off VPS hosting!**
53 | >
54 | > 👉 https://www.hostinger.com/vps-hosting?REFERRALCODE=REVOLTCHAT
55 | >
56 | > We recommend using the _KVM 2_ plan at minimum!\
57 | > Our testing environment for self-hosted currently sits on a KVM 2 instance, and we are happy to assist with issues.
58 |
59 | The instructions going forward will use Hostinger as an example hosting platform, but you should be able to adapt these to other platforms as necessary. There are important details throughout.
60 |
61 | 
62 |
63 | When asked, choose **Ubuntu Server** as your operating system; this is used by us in production, and we recommend its use.
64 |
65 | 
66 |
67 | If you've chosen to go with Hostinger, they include integrated malware scanning, which may be of interest:
68 |
69 | 
70 |
71 | You should set a secure root password for login (_or disable password login after setup, which is explained later! but you shouldn't make the password trivial until after this is secured at least!_) and we recommend that you configure an SSH key:
72 |
73 | 
74 | 
75 |
76 | Make sure to confirm everything is correct!
77 |
78 | 
79 |
80 | Wait for your VPS to be created...
81 |
82 | |  |  |
83 | | --------------------------------------------------------- | ------------------------------------------------------------ |
84 |
85 | After installation, SSH into the machine:
86 |
87 | ```bash
88 | # use the provided IP address to connect:
89 | ssh root@
90 | # .. if you have a SSH key configured
91 | ssh root@ -i path/to/id_rsa
92 | ```
93 |
94 | And now we can proceed with some basic configuration and securing the system:
95 |
96 | ```bash
97 | # update the system
98 | apt-get update && apt-get upgrade -y
99 |
100 | # configure firewall
101 | ufw allow ssh
102 | ufw allow http
103 | ufw allow https
104 | ufw default deny
105 | ufw enable
106 |
107 | # if you have configured an SSH key, disable password authentication:
108 | sudo sed -E -i 's|^#?(PasswordAuthentication)\s.*|\1 no|' /etc/ssh/sshd_config
109 | if ! grep '^PasswordAuthentication\s' /etc/ssh/sshd_config; then echo 'PasswordAuthentication no' |sudo tee -a /etc/ssh/sshd_config; fi
110 |
111 | # reboot to apply changes
112 | reboot
113 | ```
114 |
115 | Your system is now ready to proceed with installation, but before we continue, you should configure your domain.
116 |
117 | 
118 |
119 | Your domain (or a subdomain) should point to the server's IP (A and AAAA records) or CNAME to the hostname provided.
120 |
121 | Next, we must install the required dependencies:
122 |
123 | ```bash
124 | # ensure Git and Docker are installed
125 | apt-get update
126 | apt-get install ca-certificates curl git micro
127 | install -m 0755 -d /etc/apt/keyrings
128 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
129 | chmod a+r /etc/apt/keyrings/docker.asc
130 |
131 | echo \
132 | "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
133 | $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
134 | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
135 |
136 | apt-get update
137 | apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
138 | ```
139 |
140 | Now, we can pull in the configuration for Revolt:
141 |
142 | ```bash
143 | git clone https://github.com/revoltchat/self-hosted revolt
144 | cd revolt
145 | ```
146 |
147 | Generate a configuration file by running:
148 |
149 | ```bash
150 | chmod +x ./generate_config.sh
151 | ./generate_config.sh your.domain
152 | ```
153 |
154 | You can find [more options here](https://github.com/revoltchat/backend/blob/df074260196f5ed246e6360d8e81ece84d8d9549/crates/core/config/Revolt.toml), some noteworthy configuration options:
155 |
156 | - Email verification
157 | - Captcha
158 | - A custom S3 server
159 | - iOS & Android notifications (Requires Apple/Google developer accounts)
160 |
161 | If you'd like to edit the configuration, just run:
162 |
163 | ```bash
164 | micro Revolt.toml
165 | ```
166 |
167 | Finally, we can start up Revolt. First, run it in the foreground with:
168 |
169 | ```bash
170 | docker compose up
171 | ```
172 |
173 | If it runs without any critical errors, you can stop it with Ctrl + C and run it detached (in the background) by appending `-d`.
174 |
175 | ```bash
176 | docker compose up -d
177 | ```
178 |
179 | ## Updating
180 |
181 | Before updating, ensure you consult the notices at the top of this README, **as well as** [the notices](#notices) at the bottom, to check if there are any important changes to be aware of.
182 |
183 | Pull the latest version of this repository:
184 |
185 | ```bash
186 | git pull
187 | ```
188 |
189 | Check if your configuration file is correct by opening [the reference config file](https://github.com/revoltchat/backend/blob/df074260196f5ed246e6360d8e81ece84d8d9549/crates/core/config/Revolt.toml) and your `Revolt.toml` to compare changes.
190 |
191 | Then pull all the latest images:
192 |
193 | ```bash
194 | docker compose pull
195 | ```
196 |
197 | Then restart the services:
198 |
199 | ```bash
200 | docker compose up -d
201 | ```
202 |
203 | ## Advanced Deployment
204 |
205 | This guide assumes you know your way around a Linux terminal and Docker.
206 |
207 | Prerequisites before continuing:
208 |
209 | - [Git](https://git-scm.com)
210 | - [Docker](https://www.docker.com)
211 |
212 | Clone this repository.
213 |
214 | ```bash
215 | git clone https://github.com/revoltchat/self-hosted revolt
216 | cd revolt
217 | ```
218 |
219 | Create `.env.web` and download `Revolt.toml`, then modify them according to your requirements.
220 |
221 | > [!WARNING]
222 | > The default configurations are intended exclusively for testing and will only work locally. If you wish to deploy to a remote server, you **must** edit the URLs in `.env.web` and `Revolt.toml`. Please reference the section below on [configuring a custom domain](#custom-domain).
223 |
224 | ```bash
225 | echo "HOSTNAME=http://local.revolt.chat" > .env.web
226 | echo "REVOLT_PUBLIC_URL=http://local.revolt.chat/api" >> .env.web
227 | wget -O Revolt.toml https://raw.githubusercontent.com/revoltchat/backend/main/crates/core/config/Revolt.toml
228 | ```
229 |
230 | Then start Revolt:
231 |
232 | ```bash
233 | docker compose up -d
234 | ```
235 |
236 | ## Additional Notes
237 |
238 | ### Custom Domain
239 |
240 | To configure a custom domain, you will need to replace *all* instances of `local.revolt.chat` in `Revolt.toml` and `.env.web` to your chosen domain (here represented as `example.com`), like so:
241 |
242 | ```diff
243 | # .env.web
244 | - REVOLT_PUBLIC_URL=http://local.revolt.chat/api
245 | + REVOLT_PUBLIC_URL=http://example.com
246 | ```
247 |
248 | ```diff
249 | # Revolt.toml
250 | - app = "http://local.revolt.chat"
251 | + app = "http://example.com"
252 | ```
253 |
254 | In the case of `HOSTNAME`, you must strip the protocol prefix:
255 | ```diff
256 | # .env.web
257 | - HOSTNAME=http://example.com
258 | + HOSTNAME=example.com
259 | ```
260 |
261 | You will likely also want to change the protocols to enable HTTPS:
262 |
263 | ```diff
264 | # .env.web
265 | - REVOLT_PUBLIC_URL=http://example.com
266 | + REVOLT_PUBLIC_URL=https://example.com
267 | ```
268 |
269 | ```diff
270 | # Revolt.toml
271 | - app = "http://example.com"
272 | + app = "https://example.com"
273 |
274 | - events = "ws://example.com/ws"
275 | + events = "wss://example.com/ws"
276 | ```
277 |
278 | ### Placing Behind Another Reverse-Proxy or Another Port
279 |
280 | If you'd like to place Revolt behind another reverse proxy or on a non-standard port, you'll need to edit `compose.yml`.
281 |
282 | Override the port definitions on `caddy`:
283 |
284 | ```yml
285 | # compose.yml
286 | services:
287 | caddy:
288 | ports:
289 | - "1234:80"
290 | ```
291 |
292 | > [!WARNING]
293 | > This file is not included in `.gitignore`. It may be sufficient to use an override file, but that will not remove port `80` / `443` allocations.
294 |
295 | Update the hostname used by the web server:
296 |
297 | ```diff
298 | # .env.web
299 | - HOSTNAME=http://example.com
300 | + HOSTNAME=:80
301 | ```
302 |
303 | You can now reverse proxy to .
304 |
305 | ### Insecurely Expose the Database
306 |
307 | You can insecurely expose the database by adding a port definition:
308 |
309 | ```yml
310 | # compose.override.yml
311 | services:
312 | database:
313 | ports:
314 | - "27017:27017"
315 | ```
316 |
317 | For obvious reasons, be careful doing this.
318 |
319 | ### Mongo Compatibility
320 |
321 | Older processors may not support the latest MongoDB version; you may pin to MongoDB 4.4 as such:
322 |
323 | ```yml
324 | # compose.override.yml
325 | services:
326 | database:
327 | image: mongo:4.4
328 | ```
329 |
330 | ### Making Your Instance Invite-only
331 |
332 | Enable invite-only mode by setting `invite_only` in `Revolt.toml` to `true`.
333 |
334 | Create an invite:
335 |
336 | ```bash
337 | # drop into mongo shell
338 | docker compose exec database mongosh
339 |
340 | # create the invite
341 | use revolt
342 | db.invites.insertOne({ _id: "enter_an_invite_code_here" })
343 | ```
344 |
345 | ## Notices
346 |
347 | > [!IMPORTANT]
348 | > If you deployed Revolt before [2022-10-29](https://github.com/minio/docs/issues/624#issuecomment-1296608406), you may have to tag the `minio` image release if it's configured in "fs" mode.
349 | >
350 | > ```yml
351 | > image: minio/minio:RELEASE.2022-10-24T18-35-07Z
352 | > ```
353 |
354 | > [!IMPORTANT]
355 | > If you deployed Revolt before [2023-04-21](https://github.com/revoltchat/backend/commit/32542a822e3de0fc8cc7b29af46c54a9284ee2de), you may have to flush your Redis database.
356 | >
357 | > ```bash
358 | > # for stock Redis and older KeyDB images:
359 | > docker compose exec redis redis-cli
360 | > # ...or for newer KeyDB images:
361 | > docker compose exec redis keydb-cli
362 | >
363 | > # then run:
364 | > FLUSHDB
365 | > ```
366 |
367 | > [!IMPORTANT]
368 | > As of 30th September 2024, Autumn has undergone a major refactor, which requires a manual migration.
369 | >
370 | > To begin, add a temporary container that we can work from:
371 | >
372 | > ```yml
373 | > # compose.override.yml
374 | > services:
375 | > migration:
376 | > image: node:21
377 | > volumes:
378 | > - ./migrations:/cwd
379 | > command: "bash -c 'while true; do sleep 86400; done'"
380 | > ```
381 | >
382 | > Then switch to the shell:
383 | >
384 | > ```bash
385 | > docker compose up -d database migration
386 | > docker compose exec migration bash
387 | > ```
388 | >
389 | > Now we can run the migration:
390 | >
391 | > ```bash
392 | > cd /cwd
393 | > npm i mongodb
394 | > node ./20240929-autumn-rewrite.mjs
395 | > ```
396 |
397 | > [!IMPORTANT]
398 | > As of November 28, 2024, the following breaking changes have been applied:
399 | > - Rename config section `api.vapid` -> `pushd.vapid`
400 | > - Rename config section `api.fcm` -> `pushd.fcm`
401 | > - Rename config section `api.apn` -> `pushd.apn`
402 | >
403 | > These will NOT automatically be applied to your config and must be changed/added manually.
404 | >
405 | >
406 | > The following components have been added to the compose file:
407 | > - Added `rabbit` (RabbitMQ) and `pushd` (Revolt push daemon)
408 |
409 | ## Security Advisories
410 |
411 | - (`2024-06-21`) [GHSA-f26h-rqjq-qqjq revoltchat/backend: Unrestricted account creation.](https://github.com/revoltchat/backend/security/advisories/GHSA-f26h-rqjq-qqjq)
412 | - (`2024-12-17`) [GHSA-7f9x-pm3g-j7p4 revoltchat/january: January service can call itself recursively, causing heavy load.](https://github.com/revoltchat/january/security/advisories/GHSA-7f9x-pm3g-j7p4)
413 | - (`2025-02-10`) [GHSA-8684-rvfj-v3jq revoltchat/backend: Webhook tokens are freely accessible for users with read permissions.](https://github.com/revoltchat/backend/security/advisories/GHSA-8684-rvfj-v3jq)
414 | - (`2025-02-10`) [GHSA-h7h6-7pxm-mc66 revoltchat/backend: Nearby message fetch requests can be crafted to fetch entire message history.](https://github.com/revoltchat/backend/security/advisories/GHSA-h7h6-7pxm-mc66)
415 |
--------------------------------------------------------------------------------
/compose.yml:
--------------------------------------------------------------------------------
1 | name: revolt
2 |
3 | services:
4 | # MongoDB: Database
5 | database:
6 | image: docker.io/mongo
7 | restart: always
8 | volumes:
9 | - ./data/db:/data/db
10 | healthcheck:
11 | test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet
12 | interval: 10s
13 | timeout: 10s
14 | retries: 5
15 | start_period: 10s
16 |
17 | # Redis: Event message broker & KV store
18 | redis:
19 | image: docker.io/eqalpha/keydb
20 | restart: always
21 |
22 | # RabbitMQ: Internal message broker
23 | rabbit:
24 | image: docker.io/rabbitmq:4
25 | restart: always
26 | environment:
27 | RABBITMQ_DEFAULT_USER: rabbituser
28 | RABBITMQ_DEFAULT_PASS: rabbitpass
29 | volumes:
30 | - ./data/rabbit:/var/lib/rabbitmq
31 | healthcheck:
32 | test: rabbitmq-diagnostics -q ping
33 | interval: 10s
34 | timeout: 10s
35 | retries: 3
36 | start_period: 20s
37 |
38 | # MinIO: S3-compatible storage server
39 | minio:
40 | image: docker.io/minio/minio
41 | command: server /data
42 | volumes:
43 | - ./data/minio:/data
44 | environment:
45 | MINIO_ROOT_USER: minioautumn
46 | MINIO_ROOT_PASSWORD: minioautumn
47 | MINIO_DOMAIN: minio
48 | networks:
49 | default:
50 | aliases:
51 | - revolt-uploads.minio
52 | # legacy support:
53 | - attachments.minio
54 | - avatars.minio
55 | - backgrounds.minio
56 | - icons.minio
57 | - banners.minio
58 | - emojis.minio
59 | restart: always
60 |
61 | # Caddy: Web server
62 | caddy:
63 | image: docker.io/caddy
64 | restart: always
65 | env_file: .env.web
66 | ports:
67 | - "80:80"
68 | - "443:443"
69 | volumes:
70 | - ./Caddyfile:/etc/caddy/Caddyfile
71 | - ./data/caddy-data:/data
72 | - ./data/caddy-config:/config
73 |
74 | # API server
75 | api:
76 | image: ghcr.io/revoltchat/server:20250210-1
77 | depends_on:
78 | database:
79 | condition: service_healthy
80 | redis:
81 | condition: service_started
82 | rabbit:
83 | condition: service_healthy
84 | volumes:
85 | - type: bind
86 | source: ./Revolt.toml
87 | target: /Revolt.toml
88 | restart: always
89 |
90 | # Events service
91 | events:
92 | image: ghcr.io/revoltchat/bonfire:20250210-1
93 | depends_on:
94 | database:
95 | condition: service_healthy
96 | redis:
97 | condition: service_started
98 | volumes:
99 | - type: bind
100 | source: ./Revolt.toml
101 | target: /Revolt.toml
102 | restart: always
103 |
104 | # Web App
105 | web:
106 | image: ghcr.io/revoltchat/client:master
107 | restart: always
108 | env_file: .env.web
109 |
110 | # File server
111 | autumn:
112 | image: ghcr.io/revoltchat/autumn:20250210-1
113 | depends_on:
114 | database:
115 | condition: service_healthy
116 | createbuckets:
117 | condition: service_started
118 | volumes:
119 | - type: bind
120 | source: ./Revolt.toml
121 | target: /Revolt.toml
122 | restart: always
123 |
124 | # Metadata and image proxy
125 | january:
126 | image: ghcr.io/revoltchat/january:20250210-1
127 | volumes:
128 | - type: bind
129 | source: ./Revolt.toml
130 | target: /Revolt.toml
131 | restart: always
132 |
133 | # Regular task daemon
134 | crond:
135 | image: ghcr.io/revoltchat/crond:20250210-1-debug
136 | depends_on:
137 | database:
138 | condition: service_healthy
139 | minio:
140 | condition: service_started
141 | volumes:
142 | - type: bind
143 | source: ./Revolt.toml
144 | target: /Revolt.toml
145 | restart: always
146 |
147 | # Push notification daemon
148 | pushd:
149 | image: ghcr.io/revoltchat/pushd:20250210-1
150 | depends_on:
151 | database:
152 | condition: service_healthy
153 | redis:
154 | condition: service_started
155 | rabbit:
156 | condition: service_healthy
157 | volumes:
158 | - type: bind
159 | source: ./Revolt.toml
160 | target: /Revolt.toml
161 | restart: always
162 |
163 | # Create buckets for minio.
164 | createbuckets:
165 | image: docker.io/minio/mc
166 | depends_on:
167 | - minio
168 | entrypoint: >
169 | /bin/sh -c "
170 | while ! /usr/bin/mc ready minio; do
171 | /usr/bin/mc config host add minio http://minio:9000 minioautumn minioautumn;
172 | echo 'Waiting minio...' && sleep 1;
173 | done;
174 | /usr/bin/mc mb minio/revolt-uploads;
175 | exit 0;
176 | "
177 |
--------------------------------------------------------------------------------
/generate_config.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | # set hostname for Caddy
4 | echo "HOSTNAME=https://$1" > .env.web
5 | echo "REVOLT_PUBLIC_URL=https://$1/api" >> .env.web
6 |
7 | # hostnames
8 | echo "[hosts]" >> Revolt.toml
9 | echo "app = \"https://$1\"" >> Revolt.toml
10 | echo "api = \"https://$1/api\"" >> Revolt.toml
11 | echo "events = \"wss://$1/ws\"" >> Revolt.toml
12 | echo "autumn = \"https://$1/autumn\"" >> Revolt.toml
13 | echo "january = \"https://$1/january\"" >> Revolt.toml
14 |
15 | # VAPID keys
16 | echo "" >> Revolt.toml
17 | echo "[pushd.vapid]" >> Revolt.toml
18 | openssl ecparam -name prime256v1 -genkey -noout -out vapid_private.pem
19 | echo "private_key = \"$(base64 -i vapid_private.pem | tr -d '\n' | tr -d '=')\"" >> Revolt.toml
20 | echo "public_key = \"$(openssl ec -in vapid_private.pem -outform DER|tail -c 65|base64|tr '/+' '_-'|tr -d '\n'|tr -d '=')\"" >> Revolt.toml
21 | rm vapid_private.pem
22 |
23 | # encryption key for files
24 | echo "" >> Revolt.toml
25 | echo "[files]" >> Revolt.toml
26 | echo "encryption_key = \"$(openssl rand -base64 32)\"" >> Revolt.toml
27 |
--------------------------------------------------------------------------------
/migrations/.gitignore:
--------------------------------------------------------------------------------
1 | *.json
2 | node_modules
--------------------------------------------------------------------------------
/migrations/20240929-autumn-rewrite---prod-migration.mjs:
--------------------------------------------------------------------------------
1 | // THIS FILE IS TAILORED TO REVOLT PRODUCTION
2 | // MIGRATING FROM A BACKUP & EXISTING CDN NODE
3 | // INTO BACKBLAZE B2
4 | //
5 | // THIS IS ONLY INCLUDED FOR REFERENCE PURPOSES
6 |
7 | // NODE_EXTRA_CA_CERTS=~/projects/revolt-admin-panel/revolt.crt node index.mjs
8 | // NODE_EXTRA_CA_CERTS=/cwd/revolt.crt node /cwd/index.mjs
9 |
10 | import { readdir, readFile, writeFile } from "node:fs/promises";
11 | import { createCipheriv, createHash, randomBytes } from "node:crypto";
12 | import { resolve } from "node:path";
13 | import { MongoClient } from "mongodb";
14 | import { config } from "dotenv";
15 | import assert from "node:assert";
16 | import bfj from "bfj";
17 | config();
18 | config({ path: "/cwd/.env" });
19 |
20 | import BackBlazeB2 from "backblaze-b2";
21 | import axiosRetry from "axios-retry";
22 | import { decodeTime } from "ulid";
23 |
24 | // .env:
25 | // ENCRYPTION_KEY=
26 | // MONGODB=
27 | // B2_APP_KEYID=
28 | // B2_APP_KEY=
29 |
30 | /**
31 | * @type {string | null}
32 | */
33 | const USE_CACHE = "/cwd/cache.json";
34 | let processed_ids = new Set();
35 |
36 | async function dumpCache() {
37 | if (USE_CACHE) await bfj.write(USE_CACHE, [...processed_ids]);
38 | }
39 |
40 | if (USE_CACHE) {
41 | try {
42 | processed_ids = new Set(await bfj.read(USE_CACHE));
43 | } catch (err) {
44 | console.error(err);
45 | }
46 | }
47 |
48 | const b2 = new BackBlazeB2({
49 | applicationKeyId: process.env.B2_APP_KEYID,
50 | applicationKey: process.env.B2_APP_KEY,
51 | retry: {
52 | retryDelay: axiosRetry.exponentialDelay,
53 | },
54 | });
55 |
56 | await b2.authorize();
57 |
58 | //const encKey = Buffer.from(randomBytes(32), "utf8");
59 | //console.info(encKey.toString("base64"));
60 | const encKey = Buffer.from(process.env.ENCRYPTION_KEY, "base64");
61 |
62 | const mongo = new MongoClient(process.env.MONGODB);
63 | await mongo.connect();
64 |
65 | // TODO: set all existing files to current timestamp
66 | const dirs = [
67 | // "banners",
68 | // "emojis", // TODO: timestamps
69 | // "avatars",
70 | // "backgrounds",
71 | // "icons",
72 | "attachments", // https://stackoverflow.com/a/18777877
73 | ];
74 |
75 | async function encryptFile(data) {
76 | const iv = Buffer.from(randomBytes(12), "utf8");
77 | const cipher = createCipheriv("aes-256-gcm", encKey, iv);
78 |
79 | let enc = cipher.update(data, "utf8", "base64");
80 | enc += cipher.final("base64");
81 | // enc += cipher.getAuthTag();
82 |
83 | enc = Buffer.from(enc, "base64");
84 |
85 | return {
86 | iv,
87 | data: Buffer.concat([enc, cipher.getAuthTag()]),
88 | };
89 | }
90 |
91 | const cache = {};
92 |
93 | const objectLookup = {};
94 |
95 | /**
96 | * aaa
97 | */
98 | async function determineUploaderIdAndUse(f, v, i) {
99 | if (f.tag === "attachments" && v === "attachments") {
100 | if (typeof f.message_id !== "string") {
101 | console.warn(i, "No message id specified.");
102 | return null;
103 | }
104 |
105 | if (!objectLookup[f.message_id]) {
106 | objectLookup[f.message_id] = await mongo
107 | .db("revolt")
108 | .collection("messages")
109 | .findOne({
110 | _id: f.message_id,
111 | });
112 | }
113 |
114 | if (!objectLookup[f.message_id]) {
115 | console.warn(i, "Message", f.message_id, "doesn't exist anymore!");
116 | return null;
117 | }
118 |
119 | return {
120 | uploaded_at: new Date(decodeTime(f.message_id)),
121 | uploader_id: objectLookup[f.message_id].author,
122 | used_for: {
123 | type: "message",
124 | id: f.message_id,
125 | },
126 | };
127 | } else if (f.tag === "banners" && v === "banners") {
128 | if (typeof f.server_id !== "string") {
129 | console.warn(i, "No server id specified.");
130 | return null;
131 | }
132 |
133 | if (!objectLookup[f.server_id]) {
134 | objectLookup[f.server_id] = await mongo
135 | .db("revolt")
136 | .collection("servers")
137 | .findOne({
138 | _id: f.server_id,
139 | });
140 | }
141 |
142 | if (!objectLookup[f.server_id]) {
143 | console.warn(i, "Server", f.server_id, "doesn't exist anymore!");
144 | return null;
145 | }
146 |
147 | return {
148 | uploaded_at: new Date(),
149 | uploader_id: objectLookup[f.server_id].owner,
150 | used_for: {
151 | type: "serverBanner",
152 | id: f.server_id,
153 | },
154 | };
155 | } else if (f.tag === "emojis" && v === "emojis") {
156 | if (typeof f.object_id !== "string") {
157 | return null;
158 | }
159 |
160 | if (!objectLookup[f.object_id]) {
161 | objectLookup[f.object_id] = await mongo
162 | .db("revolt")
163 | .collection("emojis")
164 | .findOne({
165 | _id: f.object_id,
166 | });
167 | }
168 |
169 | if (!objectLookup[f.object_id]) {
170 | console.warn(i, "Emoji", f.object_id, "doesn't exist anymore!");
171 | return null;
172 | }
173 |
174 | return {
175 | uploaded_at: new Date(decodeTime(f.object_id)),
176 | uploader_id: objectLookup[f.object_id].creator_id,
177 | used_for: {
178 | type: "emoji",
179 | id: f.object_id,
180 | },
181 | };
182 | } else if (f.tag === "avatars" && v === "avatars") {
183 | if (typeof f.user_id !== "string") {
184 | return null;
185 | }
186 |
187 | if (!objectLookup[f.user_id]) {
188 | objectLookup[f.user_id] = await mongo
189 | .db("revolt")
190 | .collection("users")
191 | .findOne({
192 | _id: f.user_id,
193 | });
194 | }
195 |
196 | if (!objectLookup[f.user_id]) {
197 | console.warn(i, "User", f.user_id, "doesn't exist anymore!");
198 | return null;
199 | }
200 |
201 | if (objectLookup[f.user_id].avatar?._id !== f._id) {
202 | console.warn(
203 | i,
204 | "Attachment no longer in use.",
205 | f._id,
206 | "for",
207 | f.user_id,
208 | "current:",
209 | objectLookup[f.user_id].avatar?._id
210 | );
211 | return null;
212 | }
213 |
214 | return {
215 | uploaded_at: new Date(),
216 | uploader_id: f.user_id,
217 | used_for: {
218 | type: "userAvatar",
219 | id: f.user_id,
220 | },
221 | };
222 | } else if (f.tag === "backgrounds" && v === "backgrounds") {
223 | if (typeof f.user_id !== "string") {
224 | return null;
225 | }
226 |
227 | if (!objectLookup[f.user_id]) {
228 | objectLookup[f.user_id] = await mongo
229 | .db("revolt")
230 | .collection("users")
231 | .findOne({
232 | _id: f.user_id,
233 | });
234 | }
235 |
236 | if (!objectLookup[f.user_id]) {
237 | console.warn(i, "User", f.user_id, "doesn't exist anymore!");
238 | return null;
239 | }
240 |
241 | if (objectLookup[f.user_id].profile?.background?._id !== f._id) {
242 | console.warn(
243 | i,
244 | "Attachment no longer in use.",
245 | f._id,
246 | "for",
247 | f.user_id,
248 | "current:",
249 | objectLookup[f.user_id].profile?.background?._id
250 | );
251 | return null;
252 | }
253 |
254 | return {
255 | uploaded_at: new Date(),
256 | uploader_id: f.user_id,
257 | used_for: {
258 | type: "userProfileBackground",
259 | id: f.user_id,
260 | },
261 | };
262 | } else if (f.tag === "icons" && v === "icons") {
263 | if (typeof f.object_id !== "string") {
264 | return null;
265 | }
266 |
267 | // some bugged files at start
268 | // ... expensive to compute at worst case =(
269 | // so instead we can just disable it until everything is processed
270 | // then re-run on these!
271 | if (false) {
272 | objectLookup[f.object_id] = await mongo
273 | .db("revolt")
274 | .collection("users")
275 | .findOne({
276 | _id: f.object_id,
277 | });
278 |
279 | if (!objectLookup[f.object_id]) {
280 | console.warn(i, "No legacy match!");
281 | return null;
282 | }
283 |
284 | return {
285 | uploaded_at: new Date(),
286 | uploader_id: f.object_id,
287 | used_for: {
288 | type: "legacyGroupIcon",
289 | id: f.object_id,
290 | },
291 | };
292 | }
293 |
294 | if (!objectLookup[f.object_id]) {
295 | objectLookup[f.object_id] = await mongo
296 | .db("revolt")
297 | .collection("servers")
298 | .findOne({
299 | _id: f.object_id,
300 | });
301 | }
302 |
303 | if (
304 | !objectLookup[f.object_id] ||
305 | // heuristic for not server
306 | !objectLookup[f.object_id].channels
307 | ) {
308 | console.warn(i, "Server", f.object_id, "doesn't exist!");
309 |
310 | if (!objectLookup[f.object_id]) {
311 | objectLookup[f.object_id] = await mongo
312 | .db("revolt")
313 | .collection("channels")
314 | .findOne({
315 | _id: f.object_id,
316 | });
317 | }
318 |
319 | if (!objectLookup[f.object_id]) {
320 | console.warn(i, "Channel", f.object_id, "doesn't exist!");
321 | return null;
322 | }
323 |
324 | let server;
325 | const serverId = objectLookup[f.object_id].server;
326 | if (serverId) {
327 | server = objectLookup[serverId];
328 |
329 | if (!server) {
330 | server = await mongo.db("revolt").collection("servers").findOne({
331 | _id: serverId,
332 | });
333 |
334 | console.info(
335 | i,
336 | "Couldn't find matching server for channel " + f.object_id + "!"
337 | );
338 | if (!server) return null;
339 |
340 | objectLookup[serverId] = server;
341 | }
342 | }
343 |
344 | return {
345 | uploaded_at: new Date(),
346 | uploader_id: (server ?? objectLookup[f.object_id]).owner,
347 | used_for: {
348 | type: "channelIcon",
349 | id: f.object_id,
350 | },
351 | };
352 | }
353 |
354 | return {
355 | uploaded_at: new Date(),
356 | uploader_id: objectLookup[f.object_id].owner,
357 | used_for: {
358 | type: "serverIcon",
359 | id: f.object_id,
360 | },
361 | };
362 | } else {
363 | throw (
364 | "couldn't find uploader id for " +
365 | f._id +
366 | " expected " +
367 | v +
368 | " but got " +
369 | f.tag
370 | );
371 | }
372 | }
373 |
374 | const workerCount = 8;
375 | let workingOnHashes = [];
376 |
377 | for (const dir of dirs) {
378 | console.info(dir);
379 |
380 | // const RESUME = 869000 + 283000 + 772000;
381 |
382 | // UPLOAD FROM LOCAL FILE LISTING:
383 | // const RESUME = 0;
384 | // const files = (await readdir(dir)).slice(RESUME);
385 | // const total = files.length;
386 |
387 | // UPLOAD FROM DATABASE FILE LISTING:
388 | const files = await mongo
389 | .db("revolt")
390 | .collection("attachments")
391 | .find(
392 | {
393 | tag: dir,
394 | // don't upload delete files
395 | deleted: {
396 | $ne: true,
397 | },
398 | // don't upload already processed files
399 | hash: {
400 | $exists: false,
401 | },
402 | },
403 | {
404 | projection: { _id: 1 },
405 | }
406 | )
407 | .toArray()
408 | .then((arr) => arr.map((x) => x._id));
409 | const total = files.length;
410 |
411 | let i = 0;
412 | let skipsA = 0,
413 | skipsB = 0;
414 |
415 | await Promise.all(
416 | new Array(workerCount).fill(0).map(async (_) => {
417 | while (true) {
418 | const file = files.shift();
419 | if (!file) return;
420 |
421 | i++;
422 | console.info(i, files.length, file);
423 | // if (i < 869000) continue; // TODO
424 | // if (i > 3000) break;
425 |
426 | if (USE_CACHE) {
427 | if (processed_ids.has(file)) {
428 | console.info(i, "Skip, known file.");
429 | continue;
430 | }
431 | }
432 |
433 | const doc = await mongo
434 | .db("revolt")
435 | .collection("attachments")
436 | .findOne({
437 | _id: file,
438 | // don't upload delete files
439 | deleted: {
440 | $ne: true,
441 | },
442 | // don't upload already processed files
443 | hash: {
444 | $exists: false,
445 | },
446 | });
447 |
448 | if (!doc) {
449 | console.info(
450 | i,
451 | "Skipping as it does not exist in DB, is queued for deletion, or has already been processed!"
452 | );
453 | skipsA += 1;
454 | continue;
455 | }
456 |
457 | const metaUseInfo = await determineUploaderIdAndUse(doc, dir, i);
458 | if (!metaUseInfo) {
459 | if (USE_CACHE) {
460 | processed_ids.add(file);
461 | }
462 | console.info(i, "Skipping as it hasn't been attached to anything!");
463 | skipsB += 1;
464 | continue;
465 | }
466 |
467 | const start = +new Date();
468 |
469 | let buff;
470 | try {
471 | buff = await readFile(resolve(dir, file));
472 | } catch (err) {
473 | if (err.code === "ENOENT") {
474 | if (USE_CACHE) {
475 | processed_ids.add(file);
476 | }
477 | console.log(i, "File not found!");
478 | await mongo.db("revolt").collection("logs").insertOne({
479 | type: "missingFile",
480 | desc: "File doesn't exist!",
481 | file,
482 | });
483 | continue;
484 | } else {
485 | throw err;
486 | }
487 | }
488 |
489 | const hash = createHash("sha256").update(buff).digest("hex");
490 |
491 | while (workingOnHashes.includes(hash)) {
492 | console.log(
493 | "Waiting to avoid race condition... hash is already being processed..."
494 | );
495 |
496 | await new Promise((r) => setTimeout(r, 1000));
497 | }
498 |
499 | workingOnHashes.push(hash);
500 |
501 | // merge existing
502 | const existingHash = await mongo
503 | .db("revolt")
504 | .collection("attachment_hashes")
505 | .findOne({
506 | _id: hash,
507 | });
508 |
509 | if (existingHash) {
510 | console.info(i, "Hash already uploaded, merging!");
511 |
512 | await mongo
513 | .db("revolt")
514 | .collection("attachments")
515 | .updateOne(
516 | {
517 | _id: file,
518 | },
519 | {
520 | $set: {
521 | size: existingHash.size,
522 | hash,
523 | ...metaUseInfo,
524 | },
525 | }
526 | );
527 |
528 | await mongo.db("revolt").collection("logs").insertOne({
529 | type: "mergeHash",
530 | desc: "Merged an existing file!",
531 | hash: existingHash._id,
532 | size: existingHash.size,
533 | });
534 |
535 | workingOnHashes = workingOnHashes.filter((x) => x !== hash);
536 | continue;
537 | }
538 |
539 | // encrypt
540 | const { iv, data } = await encryptFile(buff);
541 | const end = +new Date();
542 |
543 | console.info(metaUseInfo); // + write hash
544 | console.info(
545 | file,
546 | hash,
547 | iv,
548 | `${end - start}ms`,
549 | buff.byteLength,
550 | "bytes"
551 | );
552 |
553 | let retry = true;
554 | while (retry) {
555 | try {
556 | const urlResp = await b2.getUploadUrl({
557 | bucketId: "---", // revolt-uploads
558 | });
559 |
560 | await b2.uploadFile({
561 | uploadUrl: urlResp.data.uploadUrl,
562 | uploadAuthToken: urlResp.data.authorizationToken,
563 | fileName: hash,
564 | data,
565 | onUploadProgress: (event) => console.info(event),
566 | });
567 |
568 | await mongo
569 | .db("revolt")
570 | .collection("attachment_hashes")
571 | .insertOne({
572 | _id: hash,
573 | processed_hash: hash,
574 |
575 | created_at: new Date(), // TODO on all
576 |
577 | bucket_id: "revolt-uploads",
578 | path: hash,
579 | iv: iv.toString("base64"),
580 |
581 | metadata: doc.metadata,
582 | content_type: doc.content_type,
583 | size: data.byteLength,
584 | });
585 |
586 | await mongo
587 | .db("revolt")
588 | .collection("attachments")
589 | .updateOne(
590 | {
591 | _id: file,
592 | },
593 | {
594 | $set: {
595 | size: data.byteLength,
596 | hash,
597 | ...metaUseInfo,
598 | },
599 | }
600 | );
601 |
602 | retry = false;
603 | } catch (err) {
604 | if (
605 | (err.isAxiosError &&
606 | (err.response?.status === 503 ||
607 | err.response?.status === 500)) ||
608 | (err?.code === "ENOTFOUND" && err?.syscall === "getaddrinfo") ||
609 | (err?.code === "ETIMEDOUT" && err?.syscall === "connect") ||
610 | (err?.code === "ECONNREFUSED" && err?.syscall === "connect")
611 | ) {
612 | console.error(i, err.response.status, "ERROR RETRYING");
613 |
614 | await mongo
615 | .db("revolt")
616 | .collection("logs")
617 | .insertOne({
618 | type: "upload503",
619 | desc:
620 | "Hit status " +
621 | (err?.code === "ETIMEDOUT" && err?.syscall === "connect"
622 | ? "Network issue (ETIMEDOUT connect)"
623 | : err?.code === "ECONNREFUSED" &&
624 | err?.syscall === "connect"
625 | ? "Network issue (ECONNREFUSED connect)"
626 | : err?.code === "ENOTFOUND" &&
627 | err?.syscall === "getaddrinfo"
628 | ? "DNS issue (ENOTFOUND getaddrinfo)"
629 | : err.response?.status) +
630 | ", trying a new URL!",
631 | hash,
632 | });
633 |
634 | await new Promise((r) => setTimeout(() => r(), 1500));
635 | } else {
636 | await dumpCache().catch(console.error);
637 | throw err;
638 | }
639 | }
640 | }
641 |
642 | console.info(i, "Successfully uploaded", file, "to S3!");
643 | console.info(
644 | "*** ➡️ Processed",
645 | i,
646 | "out of",
647 | total,
648 | "files",
649 | ((i / total) * 100).toFixed(2),
650 | "%"
651 | );
652 |
653 | workingOnHashes = workingOnHashes.filter((x) => x !== hash);
654 | }
655 | })
656 | );
657 |
658 | console.info("Skips (A):", skipsA, "(B):", skipsB);
659 | break;
660 | }
661 |
662 | await dumpCache().catch(console.error);
663 | process.exit(0);
664 |
--------------------------------------------------------------------------------
/migrations/20240929-autumn-rewrite.mjs:
--------------------------------------------------------------------------------
1 | // This script is intended for migrating to the new Autumn release.
2 | // Please read all TODOs in this file as they will help guide you
3 | // to migrate your data properly. Please do Ctrl + F "TODO".
4 |
5 | import { MongoClient } from "mongodb";
6 |
7 | /**
8 | * Map of tags to S3 bucket names
9 | *
10 | * TODO: if you've used AUTUMN_S3_BUCKET_PREFIX in the past
11 | * update the bucket names below to include the prefix
12 | *
13 | * NOTE: update `files.s3.default_bucket` in Revolt.toml!
14 | */
15 | const BUCKET_MAP = {
16 | attachments: "attachments",
17 | avatars: "avatars",
18 | backgrounds: "backgrounds",
19 | icons: "icons",
20 | banners: "banners",
21 | emojis: "emojis",
22 | };
23 |
24 | /**
25 | * Connection URL for MongoDB instance
26 | *
27 | * TODO: change if necessary
28 | */
29 | const CONNECTION_URL = "mongodb://database";
30 |
31 | const objectLookup = {};
32 | const mongo = new MongoClient(CONNECTION_URL);
33 | await mongo.connect();
34 |
35 | async function determineUploaderIdAndUse(f, v, i) {
36 | if (f.tag === "attachments" && v === "attachments") {
37 | if (typeof f.message_id !== "string") {
38 | console.warn(i, "No message id specified.");
39 | return null;
40 | }
41 |
42 | if (!objectLookup[f.message_id]) {
43 | objectLookup[f.message_id] = await mongo
44 | .db("revolt")
45 | .collection("messages")
46 | .findOne({
47 | _id: f.message_id,
48 | });
49 | }
50 |
51 | if (!objectLookup[f.message_id]) {
52 | console.warn(i, "Message", f.message_id, "doesn't exist anymore!");
53 | return null;
54 | }
55 |
56 | return {
57 | uploaded_at: new Date(decodeTime(f.message_id)),
58 | uploader_id: objectLookup[f.message_id].author,
59 | used_for: {
60 | type: "Message",
61 | id: f.message_id,
62 | },
63 | };
64 | } else if (f.tag === "banners" && v === "banners") {
65 | if (typeof f.server_id !== "string") {
66 | console.warn(i, "No server id specified.");
67 | return null;
68 | }
69 |
70 | if (!objectLookup[f.server_id]) {
71 | objectLookup[f.server_id] = await mongo
72 | .db("revolt")
73 | .collection("servers")
74 | .findOne({
75 | _id: f.server_id,
76 | });
77 | }
78 |
79 | if (!objectLookup[f.server_id]) {
80 | console.warn(i, "Server", f.server_id, "doesn't exist anymore!");
81 | return null;
82 | }
83 |
84 | return {
85 | uploaded_at: new Date(),
86 | uploader_id: objectLookup[f.server_id].owner,
87 | used_for: {
88 | type: "ServerBanner",
89 | id: f.server_id,
90 | },
91 | };
92 | } else if (f.tag === "emojis" && v === "emojis") {
93 | if (typeof f.object_id !== "string") {
94 | return null;
95 | }
96 |
97 | if (!objectLookup[f.object_id]) {
98 | objectLookup[f.object_id] = await mongo
99 | .db("revolt")
100 | .collection("emojis")
101 | .findOne({
102 | _id: f.object_id,
103 | });
104 | }
105 |
106 | if (!objectLookup[f.object_id]) {
107 | console.warn(i, "Emoji", f.object_id, "doesn't exist anymore!");
108 | return null;
109 | }
110 |
111 | return {
112 | uploaded_at: new Date(decodeTime(f.object_id)),
113 | uploader_id: objectLookup[f.object_id].creator_id,
114 | used_for: {
115 | type: "Emoji",
116 | id: f.object_id,
117 | },
118 | };
119 | } else if (f.tag === "avatars" && v === "avatars") {
120 | if (typeof f.user_id !== "string") {
121 | return null;
122 | }
123 |
124 | if (!objectLookup[f.user_id]) {
125 | objectLookup[f.user_id] = await mongo
126 | .db("revolt")
127 | .collection("users")
128 | .findOne({
129 | _id: f.user_id,
130 | });
131 | }
132 |
133 | if (!objectLookup[f.user_id]) {
134 | console.warn(i, "User", f.user_id, "doesn't exist anymore!");
135 | return null;
136 | }
137 |
138 | if (objectLookup[f.user_id].avatar?._id !== f._id) {
139 | console.warn(
140 | i,
141 | "Attachment no longer in use.",
142 | f._id,
143 | "for",
144 | f.user_id,
145 | "current:",
146 | objectLookup[f.user_id].avatar?._id
147 | );
148 | return null;
149 | }
150 |
151 | return {
152 | uploaded_at: new Date(),
153 | uploader_id: f.user_id,
154 | used_for: {
155 | type: "UserAvatar",
156 | id: f.user_id,
157 | },
158 | };
159 | } else if (f.tag === "backgrounds" && v === "backgrounds") {
160 | if (typeof f.user_id !== "string") {
161 | return null;
162 | }
163 |
164 | if (!objectLookup[f.user_id]) {
165 | objectLookup[f.user_id] = await mongo
166 | .db("revolt")
167 | .collection("users")
168 | .findOne({
169 | _id: f.user_id,
170 | });
171 | }
172 |
173 | if (!objectLookup[f.user_id]) {
174 | console.warn(i, "User", f.user_id, "doesn't exist anymore!");
175 | return null;
176 | }
177 |
178 | if (objectLookup[f.user_id].profile?.background?._id !== f._id) {
179 | console.warn(
180 | i,
181 | "Attachment no longer in use.",
182 | f._id,
183 | "for",
184 | f.user_id,
185 | "current:",
186 | objectLookup[f.user_id].profile?.background?._id
187 | );
188 | return null;
189 | }
190 |
191 | return {
192 | uploaded_at: new Date(),
193 | uploader_id: f.user_id,
194 | used_for: {
195 | type: "UserProfileBackground",
196 | id: f.user_id,
197 | },
198 | };
199 | } else if (f.tag === "icons" && v === "icons") {
200 | if (typeof f.object_id !== "string") {
201 | return null;
202 | }
203 |
204 | // some bugged files at start
205 | // ... expensive to compute at worst case =(
206 | // so instead we can just disable it until everything is processed
207 | // then re-run on these!
208 | if (false) {
209 | objectLookup[f.object_id] = await mongo
210 | .db("revolt")
211 | .collection("users")
212 | .findOne({
213 | _id: f.object_id,
214 | });
215 |
216 | if (!objectLookup[f.object_id]) {
217 | console.warn(i, "No legacy match!");
218 | return null;
219 | }
220 |
221 | return {
222 | uploaded_at: new Date(),
223 | uploader_id: f.object_id,
224 | used_for: {
225 | type: "LegacyGroupIcon",
226 | id: f.object_id,
227 | },
228 | };
229 | }
230 |
231 | if (!objectLookup[f.object_id]) {
232 | objectLookup[f.object_id] = await mongo
233 | .db("revolt")
234 | .collection("servers")
235 | .findOne({
236 | _id: f.object_id,
237 | });
238 | }
239 |
240 | if (
241 | !objectLookup[f.object_id] ||
242 | // heuristic for not server
243 | !objectLookup[f.object_id].channels
244 | ) {
245 | console.warn(i, "Server", f.object_id, "doesn't exist!");
246 |
247 | if (!objectLookup[f.object_id]) {
248 | objectLookup[f.object_id] = await mongo
249 | .db("revolt")
250 | .collection("channels")
251 | .findOne({
252 | _id: f.object_id,
253 | });
254 | }
255 |
256 | if (!objectLookup[f.object_id]) {
257 | console.warn(i, "Channel", f.object_id, "doesn't exist!");
258 | return null;
259 | }
260 |
261 | let server;
262 | const serverId = objectLookup[f.object_id].server;
263 | if (serverId) {
264 | server = objectLookup[serverId];
265 |
266 | if (!server) {
267 | server = await mongo.db("revolt").collection("servers").findOne({
268 | _id: serverId,
269 | });
270 |
271 | console.info(
272 | i,
273 | "Couldn't find matching server for channel " + f.object_id + "!"
274 | );
275 | if (!server) return null;
276 |
277 | objectLookup[serverId] = server;
278 | }
279 | }
280 |
281 | return {
282 | uploaded_at: new Date(),
283 | uploader_id: (server ?? objectLookup[f.object_id]).owner,
284 | used_for: {
285 | type: "ChannelIcon",
286 | id: f.object_id,
287 | },
288 | };
289 | }
290 |
291 | return {
292 | uploaded_at: new Date(),
293 | uploader_id: objectLookup[f.object_id].owner,
294 | used_for: {
295 | type: "ServerIcon",
296 | id: f.object_id,
297 | },
298 | };
299 | } else {
300 | throw (
301 | "couldn't find uploader id for " +
302 | f._id +
303 | " expected " +
304 | v +
305 | " but got " +
306 | f.tag
307 | );
308 | }
309 | }
310 |
311 | const dirs = [
312 | "banners",
313 | "emojis",
314 | "avatars",
315 | "backgrounds",
316 | "icons",
317 | "attachments", // https://stackoverflow.com/a/18777877
318 | ];
319 |
320 | // === add `used_for` field to files
321 | const files_pt1 = await mongo
322 | .db("revolt")
323 | .collection("attachments")
324 | .find({
325 | $or: [
326 | {
327 | used_for: {
328 | $exists: false,
329 | },
330 | },
331 | {
332 | uploader_id: {
333 | $exists: false,
334 | },
335 | },
336 | {
337 | uploader_at: {
338 | $exists: false,
339 | },
340 | },
341 | ],
342 | })
343 | .toArray();
344 |
345 | let i = 1;
346 | for (const file of files_pt1) {
347 | console.info(i++, files_pt1.length, file);
348 | const meta = determineUploaderIdAndUse(file, file.tag, i);
349 | if (meta) {
350 | await mongo.db("revolt").collection("attachments").updateOne(
351 | {
352 | _id: file._id,
353 | },
354 | {
355 | $set: meta,
356 | }
357 | );
358 | }
359 | }
360 |
361 | // === set hash to id and create relevant objects
362 | const files_pt2 = await mongo
363 | .db("revolt")
364 | .collection("attachments")
365 | .find({
366 | hash: {
367 | $exists: false,
368 | },
369 | })
370 | .toArray();
371 |
372 | await mongo
373 | .db("revolt")
374 | .collection("attachment_hashes")
375 | .insertMany(
376 | files_pt2.map((file) => ({
377 | _id: file._id,
378 | processed_hash: file._id,
379 |
380 | created_at: new Date(),
381 |
382 | bucket_id: BUCKET_MAP[file.tag],
383 | path: file._id,
384 | iv: "", // disable encryption for file
385 |
386 | metadata: file.metadata,
387 | content_type: file.content_type,
388 | size: file.size,
389 | }))
390 | );
391 |
392 | for (const file of files_pt2) {
393 | await mongo
394 | .db("revolt")
395 | .collection("attachments")
396 | .updateOne(
397 | {
398 | _id: file._id,
399 | },
400 | {
401 | $set: {
402 | hash: file._id,
403 | },
404 | }
405 | );
406 | }
407 |
--------------------------------------------------------------------------------