├── Signal-Server
├── personal-config
│ └── config
├── filtered-docker-compose.sh
├── Dockerfile
├── docker-compose-first-run.sh
├── README.md
├── docker-compose.yml
└── WhisperServerService.java
├── nginx-certbot
├── nginx-secrets
│ └── config
├── nginx-certbot.env
├── docker-compose.yml
├── signalcaptchas
│ ├── index.html
│ └── script.js
├── user_conf.d
│ └── personal.conf
└── README.md
├── registration-service
├── docker-compose.yml
├── Dockerfile
└── README.md
├── .gitignore
├── metadataproxy
├── docker-compose.yml
└── README.md
├── redis-cluster
├── docker-compose-first-run.sh
├── README.md
└── docker-compose.yml
└── README.md
/Signal-Server/personal-config/config:
--------------------------------------------------------------------------------
1 | config
--------------------------------------------------------------------------------
/nginx-certbot/nginx-secrets/config:
--------------------------------------------------------------------------------
1 | sample file so this folder shows up
--------------------------------------------------------------------------------
/registration-service/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3'
2 | services:
3 | registration-service:
4 | image: registration-service:1.0
5 | ports:
6 | - '50051:50051'
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .vscode
2 | .DS_Store
3 | Signal-Server/personal-config/*
4 | !Signal-Server/personal-config/config
5 | metadataproxy/env.env
6 | nginx-certbot/nginx-secrets/*
7 | !nginx-certbot/nginx-secrets/config
8 | registration-service/signal.pem
--------------------------------------------------------------------------------
/metadataproxy/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3'
2 | services:
3 | metadataproxy:
4 | build:
5 | context: ./metadataproxy
6 | ports:
7 | - "8080:8080"
8 | networks:
9 | - signal
10 | env_file:
11 | - env.env
--------------------------------------------------------------------------------
/nginx-certbot/nginx-certbot.env:
--------------------------------------------------------------------------------
1 | CERTBOT_EMAIL=jjtofflemire@gmail.com
2 | DHPARAM_SIZE=2048
3 | ELLIPTIC_CURVE=secp256r1
4 | RENEWAL_INTERVAL=8d
5 | RSA_KEY_SIZE=2048
6 | STAGING=0
7 | USE_ECDSA=1
8 | CERTBOT_AUTHENTICATOR=webroot
9 | CERTBOT_DNS_PROPAGATION_SECONDS=""
10 | DEBUG=0
11 | USE_LOCAL_CA=0
12 |
--------------------------------------------------------------------------------
/Signal-Server/filtered-docker-compose.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | sudo docker-compose up --no-log-prefix | awk '{
4 | gsub(/WARN /, "\033[33m&\033[0m");
5 | gsub(/ERROR/, "\033[31m&\033[0m");
6 | gsub(/INFO/, "\033[32m&\033[0m");
7 | }
8 | /Timing: [0-9]+ ms/,/<\/html>/ {next}
9 | !/^\s*$/ {
10 | print
11 | }'
12 |
--------------------------------------------------------------------------------
/metadataproxy/README.md:
--------------------------------------------------------------------------------
1 | [from here](https://github.com/lyft/metadataproxy)
2 |
3 | A back-burnered project that reroutes AWS traffic through a docker image proxy that runs alongside all your other containers
4 |
5 | - Requires host `iptables` configuration / port forwarding to manage the proxying, which ends up being the same amount of inconvenience as just using EC2 (though still possibly cheaper)
--------------------------------------------------------------------------------
/nginx-certbot/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3'
2 |
3 | services:
4 | nginx:
5 | image: jonasal/nginx-certbot:4.3.0
6 | restart: unless-stopped
7 | env_file:
8 | - ./nginx-certbot.env
9 | ports:
10 | - 80:80
11 | - 443:443
12 | - 442:442
13 | volumes:
14 | - nginx_secrets:/etc/letsencrypt
15 | - ./user_conf.d:/etc/nginx/user_conf.d
16 |
17 | volumes:
18 | nginx_secrets:
19 |
--------------------------------------------------------------------------------
/nginx-certbot/signalcaptchas/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | hCaptcha Demo
4 |
5 |
6 |
12 |
13 |
14 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/registration-service/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM eclipse-temurin:17
2 |
3 | WORKDIR /app
4 |
5 | RUN wget -q https://github.com/jtof-dev/registration-service/archive/45c2b87635185837aeb49e51afa1f9775d8e1341.zip && \
6 | jar xf 45c2b87635185837aeb49e51afa1f9775d8e1341.zip && \
7 | mv registration-service-45c2b87635185837aeb49e51afa1f9775d8e1341 registration-service && \
8 | chmod -R +777 registration-service && \
9 | rm 45c2b87635185837aeb49e51afa1f9775d8e1341.zip
10 |
11 | WORKDIR /app/registration-service
12 |
13 | # Downloads all the dependencies required and bundles it into the image
14 | RUN ./mvnw install -DskipTests -Dmicronaut.environments=dev
15 |
16 | CMD ./mvnw clean mn:run -Dmicronaut.environments=dev
--------------------------------------------------------------------------------
/Signal-Server/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM eclipse-temurin:17
2 |
3 | WORKDIR /app
4 |
5 | RUN wget -q https://github.com/signalapp/Signal-Server/archive/9c93d379a82428c27c50034a8ecd7eb36336e575.zip && jar xf 9c93d379a82428c27c50034a8ecd7eb36336e575.zip && mv Signal-Server-9c93d379a82428c27c50034a8ecd7eb36336e575 Signal-Server && chmod -R +777 Signal-Server && rm 9c93d379a82428c27c50034a8ecd7eb36336e575.zip
6 |
7 | COPY WhisperServerService.java /app/Signal-Server/service/src/main/java/org/whispersystems/textsecuregcm/
8 |
9 | WORKDIR /app/Signal-Server
10 |
11 | RUN ./mvnw clean install -DskipTests -Pexclude-spam-filter
12 |
13 | CMD jar_file=$(find service/target -name "TextSecureServer*.jar" ! -name "*-tests.jar" | head -n 1) && if [ -n "$jar_file" ] && [ -f "$jar_file" ]; then echo -e "\nStarting Signal-Server using $jar_file\n" && sleep 4 && java -jar -Dsecrets.bundle.filename=personal-config/config-secrets-bundle.yml "$jar_file" server personal-config/config.yml; else echo -e "\nNo valid Signal-Server JAR file found."; fi
--------------------------------------------------------------------------------
/registration-service/README.md:
--------------------------------------------------------------------------------
1 | # Registration-Service for the Signal-Project
2 |
3 | - For general information about `registration-service`, check out [the documentation in my fork](https://github.com/jtof-dev/registration-service)
4 | - This docker image currently doesn't work. I have no idea why - the deployment is incredibly simple but the server won't respond to the port it is opened on in the `docker-compose.yml`
5 |
6 | ## Building
7 |
8 | `registration-service` has a dev environment which reads an `application.yml` that can be configured to tell it to expect `https` requests. Grab the `fullchain.pem` and `privkey.pem` from your `nginx-certbot` docker image and put the files next to the `Dockerfile`
9 |
10 | ```
11 | docker exec -it bash
12 | cd /etc/letsencrypt/live//
13 | cat fullchain.pem
14 | cat privkey.pem
15 | ```
16 |
17 | - They will get passed in at build - I know that this hardcodes the values, but `volumes:` doesn't play nice with individual files
18 |
19 | Just build the `Dockerfile`. Nice and easy
20 |
21 | ```
22 | docker build -t registration-service:1.0 .
23 | ```
24 |
25 | ## Running
26 |
27 | ```
28 | sudo docker-compose up
29 | ```
--------------------------------------------------------------------------------
/Signal-Server/docker-compose-first-run.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | cd ..
4 |
5 | # get only the name of the working directory in lower case to ensure that the correct volumes are deleted
6 | folder=$(basename "$(pwd)" | tr '[:upper:]' '[:lower:]')
7 |
8 | sudo docker volume rm -f "$folder"_redis-cluster_data-0
9 | sudo docker volume rm -f "$folder"_redis-cluster_data-1
10 | sudo docker volume rm -f "$folder"_redis-cluster_data-2
11 | sudo docker volume rm -f "$folder"_redis-cluster_data-3
12 | sudo docker volume rm -f "$folder"_redis-cluster_data-4
13 | sudo docker volume rm -f "$folder"_redis-cluster_data-5
14 |
15 | # Download an unmodified Bitnami Redis-Cluster docker-comopse.yml file to generate the correct volumes to use with the modified docker-compose.yml
16 | wget -O docker-compose-first-run.yml https://raw.githubusercontent.com/bitnami/containers/fd15f56824528476ca6bd922d3f7ae8673f1cddd/bitnami/redis-cluster/7.0/debian-11/docker-compose.yml
17 |
18 | sudo docker-compose -f docker-compose-first-run.yml up -d && sudo docker-compose -f docker-compose-first-run.yml down
19 |
20 | rm docker-compose-first-run.yml
21 |
22 | cd scripts
23 |
24 | # backup in case you want to do this manually:
25 | #
26 | # docker-compose from here: https://github.com/bitnami/containers/blob/fd15f56824528476ca6bd922d3f7ae8673f1cddd/bitnami/redis-cluster/7.0/debian-11/docker-compose.yml
27 | #
28 | # then run this in the same directory as the main docker-compose.yml:
29 | #
30 | # sudo docker-compose -f docker-compose-first-run.yml up -d && sudo docker-compose -f docker-compose-first-run.yml down
--------------------------------------------------------------------------------
/redis-cluster/docker-compose-first-run.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # get only the name of the working directory in lower case to ensure that the correct volumes are deleted
4 | folder=$(basename "$(pwd)" | tr '[:upper:]' '[:lower:]')
5 |
6 | # delete any old volumes if they exst
7 | sudo docker volume rm -f "$folder"_redis-cluster_data-0
8 | sudo docker volume rm -f "$folder"_redis-cluster_data-1
9 | sudo docker volume rm -f "$folder"_redis-cluster_data-2
10 | sudo docker volume rm -f "$folder"_redis-cluster_data-3
11 | sudo docker volume rm -f "$folder"_redis-cluster_data-4
12 | sudo docker volume rm -f "$folder"_redis-cluster_data-5
13 |
14 | # Download an unmodified Bitnami Redis-Cluster docker-comopse.yml file to generate the correct volumes to use with the modified docker-compose.yml
15 | wget -O docker-compose-first-run.yml https://raw.githubusercontent.com/bitnami/containers/fd15f56824528476ca6bd922d3f7ae8673f1cddd/bitnami/redis-cluster/7.0/debian-11/docker-compose.yml
16 |
17 | sudo docker-compose -f docker-compose-first-run.yml up -d && sudo docker-compose -f docker-compose-first-run.yml down
18 |
19 | rm docker-compose-first-run.yml
20 |
21 | # backup in case you want to do this manually:
22 | #
23 | # docker-compose from here: https://github.com/bitnami/containers/blob/fd15f56824528476ca6bd922d3f7ae8673f1cddd/bitnami/redis-cluster/7.0/debian-11/docker-compose.yml
24 | #
25 | # then run this in the same directory as the main docker-compose.yml:
26 | #
27 | # sudo docker-compose -f docker-compose-first-run.yml up -d && sudo docker-compose -f docker-compose-first-run.yml down
--------------------------------------------------------------------------------
/nginx-certbot/signalcaptchas/script.js:
--------------------------------------------------------------------------------
1 | function onDataCallback(token) {
2 | console.log('Successful response')
3 |
4 | // const action = document.location.href.indexOf('challenge') !== -1 ?
5 | // 'challenge' : 'registration'
6 |
7 | const sitekey = 'c866ff6f-e3f6-4e9c-936e-73d268ec33d5'
8 | const action = 'registration' // HCaptcha goes with "registration"
9 | console.log({ sitekey, action, token })
10 |
11 | renderCallback('signal-hcaptcha', sitekey, action, token)
12 | }
13 |
14 | function redirect(solution) {
15 | var targetURL = 'signalcaptcha://' + solution
16 | var link = document.createElement('a')
17 | link.href = targetURL
18 | link.innerText = 'Open Signal'
19 |
20 | document.body.removeAttribute('class')
21 |
22 | setTimeout(function () {
23 | document.getElementById('container').appendChild(link)
24 | }, 2000)
25 |
26 | window.location.href = targetURL
27 | }
28 |
29 | function renderCallback(scheme, sitekey, action, token) {
30 | var fullSolution = [scheme, sitekey, action, token].join('.')
31 | if (fullSolution.length >= 2000 && window.navigator.userAgent && window.navigator.userAgent.toLowerCase().includes("windows")) {
32 | throw new Error(`solution is too long, but we don't have a fallback for windows yet.`)
33 | // fetch('/shortener', {
34 | // method: 'POST',
35 | // headers: { 'Content-Type': 'text/plain' },
36 | // body: token
37 | // })
38 | // .then(response => {
39 | // if (response.status !== 200) {
40 | // throw new Error('Shortening request failed with ' + response.status)
41 | // }
42 | // return response.text()
43 | // })
44 | // .then(shortCode => redirect([scheme + '-short', sitekey, action, shortCode].join(".")), _error => redirect(fullSolution))
45 | } else {
46 | redirect(fullSolution)
47 | }
48 | }
--------------------------------------------------------------------------------
/nginx-certbot/user_conf.d/personal.conf:
--------------------------------------------------------------------------------
1 | # A bare-bones server config for the main Signal-Server
2 |
3 | server {
4 | # Listen to port 443 on both IPv4 and IPv6.
5 | listen 443 ssl default_server reuseport;
6 | listen [::]:443 ssl default_server reuseport;
7 |
8 | # Domain names this server should respond to.
9 | server_name chat.middleman.foo;
10 |
11 | # Load the certificate files.
12 | ssl_certificate /etc/letsencrypt/live/test-name/fullchain.pem;
13 | ssl_certificate_key /etc/letsencrypt/live/test-name/privkey.pem;
14 | ssl_trusted_certificate /etc/letsencrypt/live/test-name/chain.pem;
15 |
16 | # Load the Diffie-Hellman parameter.
17 | ssl_dhparam /etc/letsencrypt/dhparams/dhparam.pem;
18 |
19 | # return 200 'Let\'s Encrypt certificate successfully installed!';
20 | # add_header Content-Type text/plain;
21 |
22 | location / {
23 | proxy_pass http://52.53.112.176:8080;
24 | proxy_set_header Host $host;
25 | proxy_set_header X-Real-IP $remote_addr;
26 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
27 | proxy_set_header X-Forwarded-Proto $scheme;
28 | }
29 |
30 | location /signalcaptchas {
31 | root /usr/share/nginx/html/;
32 | try_files $uri $uri/ =404;
33 | index index.html;
34 | }
35 |
36 | }
37 |
38 | # A bare-bones server config for registration-service (using gRPC)
39 |
40 | server {
41 | listen 442 ssl http2;
42 | server_name chat.middleman.foo;
43 |
44 | # Reuse the same certificates
45 | ssl_certificate /etc/letsencrypt/live/test-name/fullchain.pem;
46 | ssl_certificate_key /etc/letsencrypt/live/test-name/privkey.pem;
47 | ssl_trusted_certificate /etc/letsencrypt/live/test-name/chain.pem;
48 | ssl_dhparam /etc/letsencrypt/dhparams/dhparam.pem;
49 |
50 | # return 200 'Let\'s Encrypt certificate successfully installed!';
51 | # add_header Content-Type text/plain;
52 |
53 | location / {
54 | grpc_pass grpcs://52.53.112.176:50051;
55 | }
56 |
57 | }
58 |
--------------------------------------------------------------------------------
/Signal-Server/README.md:
--------------------------------------------------------------------------------
1 | # Signal-Server Dockerized!
2 |
3 | The source files running Signal-Server in a docker container. At the moment, Signal-Server expects to be ran in an EC2 container, so this is the skeleton of what a Signal-Server could look like. Most likely, this Docker image needs to be rewritten to use [metadataproxy](https://github.com/lyft/metadataproxy), which I don't plan on doing.
4 |
5 | ## Compilation
6 |
7 | `cd Signal-Server`
8 |
9 | Create a `signal-server` Docker image:
10 |
11 | ```
12 | docker build --no-cache -t signal-server:1.0 .
13 | ```
14 |
15 | If you need to reinstall, first run `docker rmi -f signal-server:1.0`
16 |
17 | Generate the correct cluster volumes with `bash docker-compose-first-run.sh`
18 |
19 | If you call the main `docker-compose.yml` instead of `docker-compose-first-run.yml`, the server will fail with an error related to not being able to connect to the redis cluster
20 |
21 | You can fix this by listing all volumes and deleting the ones you just generated:
22 |
23 | ```
24 | docker volume ls
25 |
26 | docker volume rm -f
27 | docker volume rm -f
28 | etc
29 | ```
30 |
31 | ## Configuration
32 |
33 | - Folllow [/signal-server-configuration.md` from Main](https://github.com/jtof-dev/Signal-Server/blob/main/docs/signal-server-configuration.md), and make sure to also follow the [Docker configuration](https://github.com/jtof-dev/Signal-Server/blob/main/docs/signal-server-configuration.md#dockerized-signal-server-documentation)
34 |
35 | - Place your completed `config.yml` and `config-secrets-bundle.yml` in `personal-config/`
36 |
37 | ## Starting the container
38 |
39 | Start the server:
40 |
41 | ```
42 | docker compose up
43 | ```
44 |
45 | ### Starting the container with [`filtered-docker-compose.sh`](filtered-docker-compose.sh)
46 |
47 | This script just calls a one-liner `docker-compose up --no-log-prefix` and runs it through some `awk` / `sed` filters
48 |
49 | - Currently the long datadog failed html output is the only thing omitted (since it throws 100 lines of code every couple of seconds and provides no useful info)
50 |
51 | - Also colors the words `INFO`, `WARN`, and `ERROR` to green, orange, and red respectively to make it easier to read the server's logs
52 |
53 | # To Do
54 |
55 | ## Signal-Server
56 |
57 | - Use [this EC2 spoofer tool](https://github.com/lyft/metadataproxy) to make the docker container work
58 |
59 | - Rewrite the docker container to just build a server, and have another java image run it
60 |
--------------------------------------------------------------------------------
/redis-cluster/README.md:
--------------------------------------------------------------------------------
1 | # Bitnami's Redis-Cluster for the Signal-Project
2 |
3 | - Adapted from [Bitnami's Redis-Cluster `docker-compose.yml`](https://github.com/bitnami/containers/blob/main/bitnami/redis-cluster/docker-compose.yml)
4 |
5 | ## Deploying
6 |
7 | - For some reason, just running `docker-compose up` with this repo's modified `docker-compose.yml` will cause the cluster to fail and need to be redone. So instead:
8 |
9 | ```
10 | bash docker-compose-first-run.sh
11 | docker-compose up -d
12 | ```
13 |
14 | ### Manually Deploying
15 |
16 | Or you can download the file from [here](https://github.com/bitnami/containers/blob/fd15f56824528476ca6bd922d3f7ae8673f1cddd/bitnami/redis-cluster/7.0/debian-11/docker-compose.yml), rename it to `docker-compose-first-run.yml`, place it next to the existing `docker-compose.yml` here and run it with:
17 |
18 | ```
19 | sudo docker-compose -f docker-compose-first-run.yml up -d && sudo docker-compose -f docker-compose-first-run.yml down
20 | ```
21 |
22 | If you want to verify that the first run has correctly started a redis cluster:
23 |
24 | - Start the container (`sudo docker-compose -f docker-compose-first-run.yml up -d`)
25 |
26 | - Find the name of a container to check the logs of with `sudo docker ps`
27 |
28 | - Run `sudo docker logs ` and look for a line like:
29 |
30 | ```
31 | 1:S 06 Jul 2023 22:53:49.430 * Connecting to MASTER 172.27.0.6:6379
32 | ```
33 |
34 | - Use that IP and port to connect to the server: `redis-cli -h 172.27.0.6 -p 6379`
35 |
36 | - Authenticate yourself with `AUTH bitnami`
37 |
38 | - Run `CLUSTER INFO`, and if it started correctly, will output:
39 |
40 | ```
41 | 172.27.0.6:6379> CLUSTER INFO
42 | cluster_state:ok
43 | cluster_slots_assigned:16384
44 | cluster_slots_ok:16384
45 | cluster_slots_pfail:0
46 | etc
47 | ```
48 |
49 | If you started the modified [docker-compose.yml](docker-compose.yml) before your first run, this test will fail. You can fix it with the `docker-compose-first-run.sh` or manually:
50 |
51 | ```
52 | docker volume rm signal-server_redis-cluster_data-0
53 | docker volume rm signal-server_redis-cluster_data-1
54 | docker volume rm signal-server_redis-cluster_data-2
55 | docker volume rm signal-server_redis-cluster_data-3
56 | docker volume rm signal-server_redis-cluster_data-4
57 | docker volume rm signal-server_redis-cluster_data-5
58 | ```
59 |
60 | Which assumes that you are in a folder named `Signal-Server`
61 |
62 | Which should erase all volumes created by the dockerized redis-cluster (and erease all data stored on the cluster)
63 |
64 | - Then rerun the first manual generation command
65 |
66 | - NOTE: there is a copy of redis-cluster's `docker-compose.yml` in both `redis-cluster/` and `Signal-Server/`. As far as I can tell (I don't remember why I did this), this folder is specifically for documentation, and redis cuslter should be ran from `Signal-Server/`
67 |
--------------------------------------------------------------------------------
/Signal-Server/docker-compose.yml:
--------------------------------------------------------------------------------
1 | # Copyright VMware, Inc.
2 | # SPDX-License-Identifier: APACHE-2.0
3 |
4 | version: '2'
5 | services:
6 |
7 | redis-node-0:
8 | image: docker.io/bitnami/redis-cluster:7.0
9 | volumes:
10 | - redis-cluster_data-0:/bitnami/redis/data
11 | environment:
12 | - 'ALLOW_EMPTY_PASSWORD=yes'
13 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
14 | networks:
15 | - signal
16 |
17 | redis-node-1:
18 | image: docker.io/bitnami/redis-cluster:7.0
19 | volumes:
20 | - redis-cluster_data-1:/bitnami/redis/data
21 | environment:
22 | - 'ALLOW_EMPTY_PASSWORD=yes'
23 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
24 | networks:
25 | - signal
26 |
27 | redis-node-2:
28 | image: docker.io/bitnami/redis-cluster:7.0
29 | volumes:
30 | - redis-cluster_data-2:/bitnami/redis/data
31 | environment:
32 | - 'ALLOW_EMPTY_PASSWORD=yes'
33 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
34 | networks:
35 | - signal
36 |
37 | redis-node-3:
38 | image: docker.io/bitnami/redis-cluster:7.0
39 | volumes:
40 | - redis-cluster_data-3:/bitnami/redis/data
41 | environment:
42 | - 'ALLOW_EMPTY_PASSWORD=yes'
43 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
44 | networks:
45 | - signal
46 |
47 | redis-node-4:
48 | image: docker.io/bitnami/redis-cluster:7.0
49 | volumes:
50 | - redis-cluster_data-4:/bitnami/redis/data
51 | environment:
52 | - 'ALLOW_EMPTY_PASSWORD=yes'
53 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
54 | networks:
55 | - signal
56 |
57 | redis-node-5:
58 | image: docker.io/bitnami/redis-cluster:7.0
59 | volumes:
60 | - redis-cluster_data-5:/bitnami/redis/data
61 | depends_on:
62 | - redis-node-0
63 | - redis-node-1
64 | - redis-node-2
65 | - redis-node-3
66 | - redis-node-4
67 | environment:
68 | - 'ALLOW_EMPTY_PASSWORD=yes'
69 | - 'REDIS_CLUSTER_REPLICAS=1'
70 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
71 | - 'REDIS_CLUSTER_CREATOR=yes'
72 | networks:
73 | - signal
74 |
75 | signal-server:
76 | image: signal-server:1.0
77 | working_dir: /app/Signal-Server/
78 | env_file:
79 | - personal-config/secrets.env
80 | networks:
81 | - signal
82 | volumes:
83 | - ./personal-config:/app/Signal-Server/personal-config
84 | ports:
85 | - '127.0.0.1:7006:7006'
86 |
87 | networks:
88 | signal:
89 | driver: bridge
90 |
91 | volumes:
92 | redis-cluster_data-0:
93 | driver: local
94 | redis-cluster_data-1:
95 | driver: local
96 | redis-cluster_data-2:
97 | driver: local
98 | redis-cluster_data-3:
99 | driver: local
100 | redis-cluster_data-4:
101 | driver: local
102 | redis-cluster_data-5:
103 | driver: local
--------------------------------------------------------------------------------
/redis-cluster/docker-compose.yml:
--------------------------------------------------------------------------------
1 | # Copyright VMware, Inc.
2 | # SPDX-License-Identifier: APACHE-2.0
3 |
4 | version: '2'
5 | services:
6 |
7 | redis-node-0:
8 | image: docker.io/bitnami/redis-cluster:7.0
9 | volumes:
10 | - redis-cluster_data-0:/bitnami/redis/data
11 | environment:
12 | - 'ALLOW_EMPTY_PASSWORD=yes'
13 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
14 | networks:
15 | - signal
16 |
17 | redis-node-1:
18 | image: docker.io/bitnami/redis-cluster:7.0
19 | volumes:
20 | - redis-cluster_data-1:/bitnami/redis/data
21 | environment:
22 | - 'ALLOW_EMPTY_PASSWORD=yes'
23 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
24 | networks:
25 | - signal
26 |
27 | redis-node-2:
28 | image: docker.io/bitnami/redis-cluster:7.0
29 | volumes:
30 | - redis-cluster_data-2:/bitnami/redis/data
31 | environment:
32 | - 'ALLOW_EMPTY_PASSWORD=yes'
33 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
34 | networks:
35 | - signal
36 |
37 | redis-node-3:
38 | image: docker.io/bitnami/redis-cluster:7.0
39 | volumes:
40 | - redis-cluster_data-3:/bitnami/redis/data
41 | environment:
42 | - 'ALLOW_EMPTY_PASSWORD=yes'
43 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
44 | networks:
45 | - signal
46 |
47 | redis-node-4:
48 | image: docker.io/bitnami/redis-cluster:7.0
49 | volumes:
50 | - redis-cluster_data-4:/bitnami/redis/data
51 | environment:
52 | - 'ALLOW_EMPTY_PASSWORD=yes'
53 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
54 | networks:
55 | - signal
56 |
57 | redis-node-5:
58 | image: docker.io/bitnami/redis-cluster:7.0
59 | volumes:
60 | - redis-cluster_data-5:/bitnami/redis/data
61 | depends_on:
62 | - redis-node-0
63 | - redis-node-1
64 | - redis-node-2
65 | - redis-node-3
66 | - redis-node-4
67 | environment:
68 | - 'ALLOW_EMPTY_PASSWORD=yes'
69 | - 'REDIS_CLUSTER_REPLICAS=1'
70 | - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
71 | - 'REDIS_CLUSTER_CREATOR=yes'
72 | networks:
73 | - signal
74 |
75 | signal-server:
76 | image: signal-server:1.0
77 | working_dir: /app/Signal-Server/
78 | env_file:
79 | - personal-config/secrets.env
80 | networks:
81 | - signal
82 | volumes:
83 | - ./personal-config:/app/Signal-Server/personal-config
84 | ports:
85 | - '127.0.0.1:7006:7006'
86 |
87 | networks:
88 | signal:
89 | driver: bridge
90 |
91 | volumes:
92 | redis-cluster_data-0:
93 | driver: local
94 | redis-cluster_data-1:
95 | driver: local
96 | redis-cluster_data-2:
97 | driver: local
98 | redis-cluster_data-3:
99 | driver: local
100 | redis-cluster_data-4:
101 | driver: local
102 | redis-cluster_data-5:
103 | driver: local
--------------------------------------------------------------------------------
/nginx-certbot/README.md:
--------------------------------------------------------------------------------
1 | # NGINX and Certbot for the Signal-Project
2 |
3 | ## Configuration
4 |
5 | ### External Work
6 |
7 | To get started, you'll need a (hopefully static) ip address and a domain
8 |
9 | - You'll need your [external ip address](https://wtfismyip.com/), though every time your router restarts this ip will change
10 |
11 | - The easiest implementation is [using an elastic ip](https://github.com/jtof-dev/Signal-Server/blob/main/docs/signal-server-configuration.md#aws-ec2) with your EC2 instance and running the server there (which is what this guide will assume)
12 |
13 | - For a domain, any provider works fine, but `Route 53` is probably the easiest since it's already integrated into AWS (I already had a domain with [njal.la](https://njal.la))
14 |
15 | - Go `Route 53` > `Hosted zones` > your domain > `Create record` > select `A` as the type and enter your subdomain (chat.website.com is the standard prefix used in Signal-Server) and your ip address (or elastic ip)
16 |
17 | ### In this repo
18 |
19 | If this repo or this folder gets stale, [you might want to check](https://github.com/JonasAlfredsson/docker-nginx-certbot/blob/master/docs/dockerhub_tags.md) and see if there is a much newer stable version of nginx-certbot's docker image
20 |
21 | Add your email to the [nginx-certbot.env](nginx-certbot.env) file:
22 |
23 | ```
24 | CERTBOT_EMAIL=sample@email.com
25 | ```
26 |
27 | Edit the [personal.conf](user_conf.d/personal.conf) into `user_conf.d` and add your domain and ip:
28 |
29 | ```
30 | server_name example.1.com example.2.com;
31 | proxy_pass http://your-ip:your-port;
32 | ```
33 |
34 | And change the `test-name` if you want a different name for the folder holding your keys:
35 |
36 | ```
37 | ssl_certificate /etc/letsencrypt/live/test-name/fullchain.pem;
38 | ssl_certificate_key /etc/letsencrypt/live/test-name/privkey.pem;
39 | ssl_trusted_certificate /etc/letsencrypt/live/test-name/chain.pem;
40 | ```
41 |
42 | And make sure you open all the relavent ports using port forwarding or `Security groups` in EC2, as well as in `docker-compose.yml`
43 |
44 | - The default values in this nginx container / Signal-Server and registration-service:
45 |
46 | - Signal-Server hosts on `localhost:8080` and nginx listens on `443`
47 |
48 | - registration-service hosts on `localhost:50051` and nginx listens on `442` (to prevent conflicts with Signal-Server)
49 |
50 | Signal also requires your to host your own `hcaptcha` landing page
51 |
52 | - There is an added `location` block inside `user_conf.d/personal.conf` that redirects `chat.your.domain/signalcaptchas` from the normal Signal-Server on port 443 to `signalcaptchas/index.html`
53 |
54 | - The only configuration you need to do is add your sitekey you got from `hcaptcha` - if you haven't done this already, check out [this section](https://github.com/jtof-dev/Signal-Server/blob/main/docs/signal-server-configuration.md#hcaptcha) of Signal-Server's documentation
55 |
56 | - Paste the `sitekey` into line 16 of `index.html`:
57 |
58 | ```
59 | data-sitekey="your-key"
60 | ```
61 |
62 | ## Running the container
63 |
64 | `docker-compose up`
65 |
66 | ## General Notes
67 |
68 | - Since this docker image has the `restart: unless-stopped` parameter, once it is set up for the first time you never need to worry about starting it with Signal-Server / registration-service
69 |
70 | - After updating the `personal.conf` or `docker-compose.yml`, run `docker-compose down && docker-compose up -d` to restart and apply the changes
71 |
72 | - You can get to your certificates like this:
73 |
74 | ```
75 | docker exec -it bash
76 | cd etc/letsencrypt/live/test-name/
77 | cat keys.pem
78 | exit
79 | ```
80 |
81 | ## To Do
82 |
83 | - Update `user_conf.d/personal.conf` and replace the deprecated `listen 442 ssl http2;`
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Signal-Docker
2 |
3 | ## What *is* Signal anyway?
4 |
5 | The full Signal-Project consists of two main branches: the user-facing apps and programs, and the messy server backend
6 |
7 | ### User-facing apps and programs
8 |
9 | - These are the android, ios, and desktop applications. They all are basic programs and have a slew (~12) of urls that ping different backend services
10 |
11 | ### The backend
12 |
13 | - The backend consists of the large [Signal-Server](https://github.com/jtof-dev/Signal-Server), the brains of the backend, and many small, scalable nodes that all listen on subdomains (chat.signal.org, storage.signal.org, etc)
14 |
15 | - The nodes handle parts of Signal like registration, sending images, voice and video calls, etc. Packaging these functions into seperate servers allows for easy scalability in AWS, but for a local implementation they can all be dockerized and ran with the main server on the same system
16 |
17 | ```
18 | Messaging Dependencies & Implementation
19 | ─────────────────────────────────────────
20 |
21 | ┌────────────────────┐ ┌───────────────────────────────────┐ ┌─────────────────┐
22 | │ │ │ │ │ │
23 | │ Signal-Server │◄──────►│ nginx │◄─────►│ Signal-Android │
24 | │ implemented in EC2 │ │ implemented in a docker container │ │ or iOS │
25 | │ │ │ │ └─────────────────┘
26 | └────────────────────┘ └───────────────────────────────────┘
27 | ▲ ▲ ▲ ▲
28 | │ │ │ │
29 | │ │ │ │
30 | │ │ │ │
31 | ┌─────────┘ │ │ └─────────┐
32 | │ │ │ │
33 | ▼ ▼ ▼ ▼
34 | ┌──────────────────────┐ ┌─────────────────┐ ┌─────────────────┐ ┌───────────────────────┐
35 | │ │ │ │ │ │ │ │
36 | │ registration-service │ │ storage-service │ │ backup-service │ │ SecureValueRecoveryV2 │
37 | │ implemented in EC2 │ │ not implemented │ │ not implemented │ │ not implemented │
38 | │ │ │ │ │ │ │ │
39 | └──────────────────────┘ └─────────────────┘ └─────────────────┘ └───────────────────────┘
40 | ```
41 |
42 | ## Signal-Project Roadmap
43 |
44 | **Goal:** Completely replicate and document the function of [Signal](https://signal.org/)
45 |
46 | **Progress:**
47 |
48 | - The [main server](https://github.com/jtof-dev/Signal-Server) that manages E2EE messaging compiles and runs without errors
49 |
50 | - Untested, but fully functional
51 |
52 | - The server itself works, receiving and sending api requests and registering phone numbers
53 |
54 | - Can't test messaging because other unconfigured services are required for messaging to work
55 |
56 | - `zkparams` couldn't be generated properly and was cut out, but is probably required in other self-hosted dependencies
57 |
58 | - [registration-service](https://github.com/jtof-dev/registration-service) is fully functional and you can register numbers over `https` using Signal-Android
59 |
60 | - Relies on the dev environment, which is probably impractical for deployment, but only requied for the actual handshake of registering a phone number (everything is stored in DynamoDB anyway)
61 |
62 | - [storage-service](https://github.com/signalapp/storage-service) is a Signal-Server dependency that handles the secure storage of various bits of user information and encrypted messages
63 |
64 | - Not started, but required to finish account creation, finding users to message, and possibly messaging as a whole
65 |
66 | - Extremely difficult to deploy - missing any documentation, including any build instructions or `sample.yml`'s to work off of
67 |
68 | - [SecureValueRecovery2](https://github.com/signalapp/SecureValueRecovery2) is the place where encrypted account recovery information is stored, locked behind the user's pin
69 |
70 | - Not started, but required to finish account creation
71 |
72 | - Relatively easy to deploy - comes with thorough enough build instructions and a `sample.yml` to fill out
73 |
74 | ## Signal-Project Backend
75 |
76 | ### Signal-Server
77 |
78 | [dockerized](Signal-Server)
79 |
80 | - This docker container has not been updated to use an IAM compatible container. This will be a back-burnered project, but you can check out or work on it [in the `metadataproxy` folder](metadataproxy/README.md)
81 |
82 | - This container also probably needs to change how it handles creating the image: as it is currently, it creates a new server with new server-specific certificates. Switching to a two-part build-then-run process would address this, as well as add the ability to pass in your pre-existing signal-server.jar at runtime
83 |
84 | [full instructions](https://github.com/jtof-dev/Signal-Server)
85 |
86 | - This is the guide to follow to deploy Signal-Server in an EC2 instance
87 |
88 | ### Registration Service
89 |
90 | [dockerized](registration-service)
91 |
92 | - The `Dockerfile` and `docker-compose.yml` appear to have the correct implementation, but even after exposing ports and port forwarding the server can't be pinged by public ip address
93 |
94 | - Currently not working despite being a very simple program to Dockerize. I am unsure why it doesn't work, but for the moment you will have to run `registration-service` on bare metal
95 |
96 | [full instructions](https://github.com/jtof-dev/registration-service)
97 |
98 | ## Others
99 |
100 | ### NGNIX with Certbot
101 |
102 | [dockerized docs](nginx-certbot)
103 |
104 | - [This docker image](https://github.com/JonasAlfredsson/docker-nginx-certbot/tree/master) handles the annoying bits of deploying NGINX and automates getting and renewing `https` certificates
105 |
106 | - The image passes in any custom configuration files at runtime, allowing for ease of use in addition to being dead simple to set up
107 |
108 | ### Redis-Cluster
109 |
110 | [dockerized docs](redis-cluster)
111 |
112 | - Very simple deployment (one script), with some added notes on manually verifying that everything works as intended
113 |
114 | ## To Do
115 |
116 | - Debug registration-service
117 |
118 | - Currently unresponsive to any pings despite building and running without errors and reading changes in the `application.yml`
119 |
120 | - Completely fill out Signal-Android
121 | - include reminders about gcloud oath2
--------------------------------------------------------------------------------
/Signal-Server/WhisperServerService.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2013 Signal Messenger, LLC
3 | * SPDX-License-Identifier: AGPL-3.0-only
4 | */
5 | package org.whispersystems.textsecuregcm;
6 |
7 | import static com.codahale.metrics.MetricRegistry.name;
8 | import static java.util.Objects.requireNonNull;
9 |
10 | import com.amazonaws.ClientConfiguration;
11 | import com.amazonaws.auth.AWSCredentialsProviderChain;
12 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
13 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
14 | import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier;
15 | import com.google.api.client.http.apache.v2.ApacheHttpTransport;
16 | import com.google.api.client.json.gson.GsonFactory;
17 | import com.google.auth.oauth2.GoogleCredentials;
18 | import com.google.cloud.logging.LoggingOptions;
19 | import com.google.common.collect.ImmutableMap;
20 | import com.google.common.collect.ImmutableSet;
21 | import com.google.common.collect.Lists;
22 | import io.dropwizard.Application;
23 | import io.dropwizard.auth.AuthFilter;
24 | import io.dropwizard.auth.PolymorphicAuthDynamicFeature;
25 | import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
26 | import io.dropwizard.auth.basic.BasicCredentialAuthFilter;
27 | import io.dropwizard.auth.basic.BasicCredentials;
28 | import io.dropwizard.setup.Bootstrap;
29 | import io.dropwizard.setup.Environment;
30 | import io.lettuce.core.resource.ClientResources;
31 | import io.micrometer.core.instrument.Metrics;
32 | import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics;
33 | import java.io.ByteArrayInputStream;
34 | import java.net.http.HttpClient;
35 | import java.nio.charset.StandardCharsets;
36 | import java.time.Clock;
37 | import java.time.Duration;
38 | import java.util.Collections;
39 | import java.util.EnumSet;
40 | import java.util.List;
41 | import java.util.Optional;
42 | import java.util.ServiceLoader;
43 | import java.util.concurrent.ArrayBlockingQueue;
44 | import java.util.concurrent.BlockingQueue;
45 | import java.util.concurrent.ExecutorService;
46 | import java.util.concurrent.LinkedBlockingQueue;
47 | import java.util.concurrent.ScheduledExecutorService;
48 | import java.util.concurrent.ThreadPoolExecutor;
49 | import java.util.concurrent.TimeUnit;
50 | import javax.servlet.DispatcherType;
51 | import javax.servlet.FilterRegistration;
52 | import javax.servlet.ServletRegistration;
53 | import org.eclipse.jetty.servlets.CrossOriginFilter;
54 | import org.glassfish.jersey.server.ServerProperties;
55 | import org.signal.event.AdminEventLogger;
56 | import org.signal.event.GoogleCloudAdminEventLogger;
57 | import org.signal.i18n.HeaderControlledResourceBundleLookup;
58 | import org.signal.libsignal.zkgroup.GenericServerSecretParams;
59 | import org.signal.libsignal.zkgroup.ServerSecretParams;
60 | import org.signal.libsignal.zkgroup.auth.ServerZkAuthOperations;
61 | import org.signal.libsignal.zkgroup.profiles.ServerZkProfileOperations;
62 | import org.signal.libsignal.zkgroup.receipts.ReceiptCredentialPresentation;
63 | import org.signal.libsignal.zkgroup.receipts.ServerZkReceiptOperations;
64 | import org.slf4j.Logger;
65 | import org.slf4j.LoggerFactory;
66 | import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
67 | import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
68 | import org.whispersystems.textsecuregcm.auth.CertificateGenerator;
69 | import org.whispersystems.textsecuregcm.auth.DisabledPermittedAccountAuthenticator;
70 | import org.whispersystems.textsecuregcm.auth.DisabledPermittedAuthenticatedAccount;
71 | import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator;
72 | import org.whispersystems.textsecuregcm.auth.PhoneVerificationTokenManager;
73 | import org.whispersystems.textsecuregcm.auth.RegistrationLockVerificationManager;
74 | import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator;
75 | import org.whispersystems.textsecuregcm.auth.WebsocketRefreshApplicationEventListener;
76 | import org.whispersystems.textsecuregcm.badges.ConfiguredProfileBadgeConverter;
77 | import org.whispersystems.textsecuregcm.badges.ResourceBundleLevelTranslator;
78 | import org.whispersystems.textsecuregcm.captcha.CaptchaChecker;
79 | import org.whispersystems.textsecuregcm.captcha.HCaptchaClient;
80 | import org.whispersystems.textsecuregcm.captcha.RecaptchaClient;
81 | import org.whispersystems.textsecuregcm.captcha.RegistrationCaptchaManager;
82 | import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
83 | import org.whispersystems.textsecuregcm.configuration.secrets.SecretStore;
84 | import org.whispersystems.textsecuregcm.configuration.secrets.SecretsModule;
85 | import org.whispersystems.textsecuregcm.controllers.AccountController;
86 | import org.whispersystems.textsecuregcm.controllers.AccountControllerV2;
87 | import org.whispersystems.textsecuregcm.controllers.ArtController;
88 | import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV2;
89 | import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV3;
90 | import org.whispersystems.textsecuregcm.controllers.CallLinkController;
91 | import org.whispersystems.textsecuregcm.controllers.CertificateController;
92 | import org.whispersystems.textsecuregcm.controllers.ChallengeController;
93 | import org.whispersystems.textsecuregcm.controllers.DeviceController;
94 | import org.whispersystems.textsecuregcm.controllers.DirectoryV2Controller;
95 | import org.whispersystems.textsecuregcm.controllers.DonationController;
96 | import org.whispersystems.textsecuregcm.controllers.KeepAliveController;
97 | import org.whispersystems.textsecuregcm.controllers.KeysController;
98 | import org.whispersystems.textsecuregcm.controllers.MessageController;
99 | import org.whispersystems.textsecuregcm.controllers.PaymentsController;
100 | import org.whispersystems.textsecuregcm.controllers.ProfileController;
101 | import org.whispersystems.textsecuregcm.controllers.ProvisioningController;
102 | import org.whispersystems.textsecuregcm.controllers.RegistrationController;
103 | import org.whispersystems.textsecuregcm.controllers.RemoteConfigController;
104 | import org.whispersystems.textsecuregcm.controllers.SecureBackupController;
105 | import org.whispersystems.textsecuregcm.controllers.SecureStorageController;
106 | import org.whispersystems.textsecuregcm.controllers.SecureValueRecovery2Controller;
107 | import org.whispersystems.textsecuregcm.controllers.StickerController;
108 | import org.whispersystems.textsecuregcm.controllers.SubscriptionController;
109 | import org.whispersystems.textsecuregcm.controllers.VerificationController;
110 | import org.whispersystems.textsecuregcm.currency.CoinMarketCapClient;
111 | import org.whispersystems.textsecuregcm.currency.CurrencyConversionManager;
112 | import org.whispersystems.textsecuregcm.currency.FixerClient;
113 | import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
114 | import org.whispersystems.textsecuregcm.filters.RemoteDeprecationFilter;
115 | import org.whispersystems.textsecuregcm.filters.RequestStatisticsFilter;
116 | import org.whispersystems.textsecuregcm.filters.TimestampResponseFilter;
117 | import org.whispersystems.textsecuregcm.limits.PushChallengeManager;
118 | import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager;
119 | import org.whispersystems.textsecuregcm.limits.RateLimiters;
120 | import org.whispersystems.textsecuregcm.mappers.CompletionExceptionMapper;
121 | import org.whispersystems.textsecuregcm.mappers.DeviceLimitExceededExceptionMapper;
122 | import org.whispersystems.textsecuregcm.mappers.IOExceptionMapper;
123 | import org.whispersystems.textsecuregcm.mappers.ImpossiblePhoneNumberExceptionMapper;
124 | import org.whispersystems.textsecuregcm.mappers.InvalidWebsocketAddressExceptionMapper;
125 | import org.whispersystems.textsecuregcm.mappers.JsonMappingExceptionMapper;
126 | import org.whispersystems.textsecuregcm.mappers.NonNormalizedPhoneNumberExceptionMapper;
127 | import org.whispersystems.textsecuregcm.mappers.RateLimitExceededExceptionMapper;
128 | import org.whispersystems.textsecuregcm.mappers.RegistrationServiceSenderExceptionMapper;
129 | import org.whispersystems.textsecuregcm.mappers.ServerRejectedExceptionMapper;
130 | import org.whispersystems.textsecuregcm.metrics.ApplicationShutdownMonitor;
131 | import org.whispersystems.textsecuregcm.metrics.BufferPoolGauges;
132 | import org.whispersystems.textsecuregcm.metrics.CpuUsageGauge;
133 | import org.whispersystems.textsecuregcm.metrics.FileDescriptorGauge;
134 | import org.whispersystems.textsecuregcm.metrics.FreeMemoryGauge;
135 | import org.whispersystems.textsecuregcm.metrics.GarbageCollectionGauges;
136 | import org.whispersystems.textsecuregcm.metrics.MaxFileDescriptorGauge;
137 | import org.whispersystems.textsecuregcm.metrics.MetricsApplicationEventListener;
138 | import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
139 | import org.whispersystems.textsecuregcm.metrics.NetworkReceivedGauge;
140 | import org.whispersystems.textsecuregcm.metrics.NetworkSentGauge;
141 | import org.whispersystems.textsecuregcm.metrics.OperatingSystemMemoryGauge;
142 | import org.whispersystems.textsecuregcm.metrics.ReportedMessageMetricsListener;
143 | import org.whispersystems.textsecuregcm.metrics.TrafficSource;
144 | import org.whispersystems.textsecuregcm.providers.MultiRecipientMessageProvider;
145 | import org.whispersystems.textsecuregcm.providers.RedisClusterHealthCheck;
146 | import org.whispersystems.textsecuregcm.push.APNSender;
147 | import org.whispersystems.textsecuregcm.push.ApnPushNotificationScheduler;
148 | import org.whispersystems.textsecuregcm.push.ClientPresenceManager;
149 | import org.whispersystems.textsecuregcm.push.FcmSender;
150 | import org.whispersystems.textsecuregcm.push.MessageSender;
151 | import org.whispersystems.textsecuregcm.push.ProvisioningManager;
152 | import org.whispersystems.textsecuregcm.push.PushLatencyManager;
153 | import org.whispersystems.textsecuregcm.push.PushNotificationManager;
154 | import org.whispersystems.textsecuregcm.push.ReceiptSender;
155 | import org.whispersystems.textsecuregcm.redis.ConnectionEventLogger;
156 | import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
157 | import org.whispersystems.textsecuregcm.registration.RegistrationServiceClient;
158 | import org.whispersystems.textsecuregcm.s3.PolicySigner;
159 | import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator;
160 | import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
161 | import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
162 | import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
163 | import org.whispersystems.textsecuregcm.spam.FilterSpam;
164 | import org.whispersystems.textsecuregcm.spam.RateLimitChallengeListener;
165 | import org.whispersystems.textsecuregcm.spam.ReportSpamTokenProvider;
166 | import org.whispersystems.textsecuregcm.spam.ScoreThresholdProvider;
167 | import org.whispersystems.textsecuregcm.spam.SpamFilter;
168 | import org.whispersystems.textsecuregcm.storage.AccountCleaner;
169 | import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler;
170 | import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache;
171 | import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener;
172 | import org.whispersystems.textsecuregcm.storage.AccountLockManager;
173 | import org.whispersystems.textsecuregcm.storage.Accounts;
174 | import org.whispersystems.textsecuregcm.storage.AccountsManager;
175 | import org.whispersystems.textsecuregcm.storage.ChangeNumberManager;
176 | import org.whispersystems.textsecuregcm.storage.DeletedAccounts;
177 | import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
178 | import org.whispersystems.textsecuregcm.storage.IssuedReceiptsManager;
179 | import org.whispersystems.textsecuregcm.storage.Keys;
180 | import org.whispersystems.textsecuregcm.storage.MessagePersister;
181 | import org.whispersystems.textsecuregcm.storage.MessagesCache;
182 | import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
183 | import org.whispersystems.textsecuregcm.storage.MessagesManager;
184 | import org.whispersystems.textsecuregcm.storage.NonNormalizedAccountCrawlerListener;
185 | import org.whispersystems.textsecuregcm.storage.PhoneNumberIdentifiers;
186 | import org.whispersystems.textsecuregcm.storage.Profiles;
187 | import org.whispersystems.textsecuregcm.storage.ProfilesManager;
188 | import org.whispersystems.textsecuregcm.storage.PushChallengeDynamoDb;
189 | import org.whispersystems.textsecuregcm.storage.PushFeedbackProcessor;
190 | import org.whispersystems.textsecuregcm.storage.RedeemedReceiptsManager;
191 | import org.whispersystems.textsecuregcm.storage.RegistrationRecoveryPasswords;
192 | import org.whispersystems.textsecuregcm.storage.RegistrationRecoveryPasswordsManager;
193 | import org.whispersystems.textsecuregcm.storage.RemoteConfigs;
194 | import org.whispersystems.textsecuregcm.storage.RemoteConfigsManager;
195 | import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb;
196 | import org.whispersystems.textsecuregcm.storage.ReportMessageManager;
197 | import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager;
198 | import org.whispersystems.textsecuregcm.storage.SubscriptionManager;
199 | import org.whispersystems.textsecuregcm.storage.VerificationCodeStore;
200 | import org.whispersystems.textsecuregcm.storage.VerificationSessionManager;
201 | import org.whispersystems.textsecuregcm.storage.VerificationSessions;
202 | import org.whispersystems.textsecuregcm.subscriptions.BraintreeManager;
203 | import org.whispersystems.textsecuregcm.subscriptions.StripeManager;
204 | import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
205 | import org.whispersystems.textsecuregcm.util.SystemMapper;
206 | import org.whispersystems.textsecuregcm.util.UsernameHashZkProofVerifier;
207 | import org.whispersystems.textsecuregcm.util.logging.LoggingUnhandledExceptionMapper;
208 | import org.whispersystems.textsecuregcm.util.logging.UncaughtExceptionHandler;
209 | import org.whispersystems.textsecuregcm.websocket.AuthenticatedConnectListener;
210 | import org.whispersystems.textsecuregcm.websocket.ProvisioningConnectListener;
211 | import org.whispersystems.textsecuregcm.websocket.WebSocketAccountAuthenticator;
212 | import org.whispersystems.textsecuregcm.workers.AssignUsernameCommand;
213 | import org.whispersystems.textsecuregcm.workers.CertificateCommand;
214 | import org.whispersystems.textsecuregcm.workers.CheckDynamicConfigurationCommand;
215 | import org.whispersystems.textsecuregcm.workers.CrawlAccountsCommand;
216 | import org.whispersystems.textsecuregcm.workers.DeleteUserCommand;
217 | import org.whispersystems.textsecuregcm.workers.MessagePersisterServiceCommand;
218 | import org.whispersystems.textsecuregcm.workers.ScheduledApnPushNotificationSenderServiceCommand;
219 | import org.whispersystems.textsecuregcm.workers.ServerVersionCommand;
220 | import org.whispersystems.textsecuregcm.workers.SetRequestLoggingEnabledTask;
221 | import org.whispersystems.textsecuregcm.workers.SetUserDiscoverabilityCommand;
222 | import org.whispersystems.textsecuregcm.workers.UnlinkDeviceCommand;
223 | import org.whispersystems.textsecuregcm.workers.ZkParamsCommand;
224 | import org.whispersystems.websocket.WebSocketResourceProviderFactory;
225 | import org.whispersystems.websocket.setup.WebSocketEnvironment;
226 | import reactor.core.scheduler.Scheduler;
227 | import reactor.core.scheduler.Schedulers;
228 | import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
229 | import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
230 | import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
231 | import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
232 | import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
233 | import software.amazon.awssdk.regions.Region;
234 | import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
235 | import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
236 | import software.amazon.awssdk.services.s3.S3Client;
237 |
238 | public class WhisperServerService extends Application {
239 |
240 | private static final Logger log = LoggerFactory.getLogger(WhisperServerService.class);
241 |
242 | public static final String SECRETS_BUNDLE_FILE_NAME_PROPERTY = "secrets.bundle.filename";
243 |
244 | public static final software.amazon.awssdk.auth.credentials.AwsCredentialsProvider AWSSDK_CREDENTIALS_PROVIDER =
245 | AwsCredentialsProviderChain.of(
246 | InstanceProfileCredentialsProvider.create(),
247 | WebIdentityTokenFileCredentialsProvider.create());
248 |
249 | public static final AWSCredentialsProviderChain AWSSDK_V1_CREDENTIALS_PROVIDER_CHAIN = new AWSCredentialsProviderChain(
250 | com.amazonaws.auth.InstanceProfileCredentialsProvider.getInstance(),
251 | com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create()
252 | );
253 |
254 |
255 | @Override
256 | public void initialize(final Bootstrap bootstrap) {
257 | // `SecretStore` needs to be initialized before Dropwizard reads the main application config file.
258 | final String secretsBundleFileName = requireNonNull(
259 | System.getProperty(SECRETS_BUNDLE_FILE_NAME_PROPERTY),
260 | "Application requires property [%s] to be provided".formatted(SECRETS_BUNDLE_FILE_NAME_PROPERTY));
261 | final SecretStore secretStore = SecretStore.fromYamlFileSecretsBundle(secretsBundleFileName);
262 | SecretsModule.INSTANCE.setSecretStore(secretStore);
263 |
264 | // Initializing SystemMapper here because parsing of the main application config happens before `run()` method is called.
265 | SystemMapper.configureMapper(bootstrap.getObjectMapper());
266 |
267 | bootstrap.addCommand(new DeleteUserCommand());
268 | bootstrap.addCommand(new CertificateCommand());
269 | bootstrap.addCommand(new ZkParamsCommand());
270 | bootstrap.addCommand(new ServerVersionCommand());
271 | bootstrap.addCommand(new CheckDynamicConfigurationCommand());
272 | bootstrap.addCommand(new SetUserDiscoverabilityCommand());
273 | bootstrap.addCommand(new AssignUsernameCommand());
274 | bootstrap.addCommand(new UnlinkDeviceCommand());
275 | bootstrap.addCommand(new CrawlAccountsCommand());
276 | bootstrap.addCommand(new ScheduledApnPushNotificationSenderServiceCommand());
277 | bootstrap.addCommand(new MessagePersisterServiceCommand());
278 | }
279 |
280 | @Override
281 | public String getName() {
282 | return "whisper-server";
283 | }
284 |
285 | @Override
286 | public void run(WhisperServerConfiguration config, Environment environment) throws Exception {
287 | final Clock clock = Clock.systemUTC();
288 | final int availableProcessors = Runtime.getRuntime().availableProcessors();
289 |
290 | UncaughtExceptionHandler.register();
291 |
292 | MetricsUtil.configureRegistries(config, environment);
293 |
294 | final boolean useSecondaryCredentialsJson = Optional.ofNullable(
295 | System.getenv("SIGNAL_USE_SECONDARY_CREDENTIALS_JSON"))
296 | .isPresent();
297 |
298 | HeaderControlledResourceBundleLookup headerControlledResourceBundleLookup =
299 | new HeaderControlledResourceBundleLookup();
300 | ConfiguredProfileBadgeConverter profileBadgeConverter = new ConfiguredProfileBadgeConverter(
301 | clock, config.getBadges(), headerControlledResourceBundleLookup);
302 | ResourceBundleLevelTranslator resourceBundleLevelTranslator = new ResourceBundleLevelTranslator(
303 | headerControlledResourceBundleLookup);
304 |
305 | DynamoDbAsyncClient dynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(config.getDynamoDbClientConfiguration(),
306 | AWSSDK_CREDENTIALS_PROVIDER);
307 |
308 | DynamoDbClient dynamoDbClient = DynamoDbFromConfig.client(config.getDynamoDbClientConfiguration(),
309 | AWSSDK_CREDENTIALS_PROVIDER);
310 |
311 | AmazonDynamoDB deletedAccountsLockDynamoDbClient = AmazonDynamoDBClientBuilder.standard()
312 | .withRegion(config.getDynamoDbClientConfiguration().getRegion())
313 | .withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(
314 | ((int) config.getDynamoDbClientConfiguration().getClientExecutionTimeout().toMillis()))
315 | .withRequestTimeout(
316 | (int) config.getDynamoDbClientConfiguration().getClientRequestTimeout().toMillis()))
317 | .withCredentials(AWSSDK_V1_CREDENTIALS_PROVIDER_CHAIN)
318 | .build();
319 |
320 | DeletedAccounts deletedAccounts = new DeletedAccounts(dynamoDbClient,
321 | config.getDynamoDbTables().getDeletedAccounts().getTableName());
322 |
323 | DynamicConfigurationManager dynamicConfigurationManager =
324 | new DynamicConfigurationManager<>(config.getAppConfig().getApplication(),
325 | config.getAppConfig().getEnvironment(),
326 | config.getAppConfig().getConfigurationName(),
327 | DynamicConfiguration.class);
328 |
329 | BlockingQueue messageDeletionQueue = new LinkedBlockingQueue<>();
330 | Metrics.gaugeCollectionSize(name(getClass(), "messageDeletionQueueSize"), Collections.emptyList(),
331 | messageDeletionQueue);
332 | ExecutorService messageDeletionAsyncExecutor = environment.lifecycle()
333 | .executorService(name(getClass(), "messageDeletionAsyncExecutor-%d")).maxThreads(16)
334 | .workQueue(messageDeletionQueue).build();
335 |
336 | Accounts accounts = new Accounts(
337 | dynamoDbClient,
338 | dynamoDbAsyncClient,
339 | config.getDynamoDbTables().getAccounts().getTableName(),
340 | config.getDynamoDbTables().getAccounts().getPhoneNumberTableName(),
341 | config.getDynamoDbTables().getAccounts().getPhoneNumberIdentifierTableName(),
342 | config.getDynamoDbTables().getAccounts().getUsernamesTableName(),
343 | config.getDynamoDbTables().getAccounts().getScanPageSize());
344 | PhoneNumberIdentifiers phoneNumberIdentifiers = new PhoneNumberIdentifiers(dynamoDbClient,
345 | config.getDynamoDbTables().getPhoneNumberIdentifiers().getTableName());
346 | Profiles profiles = new Profiles(dynamoDbClient, dynamoDbAsyncClient,
347 | config.getDynamoDbTables().getProfiles().getTableName());
348 | Keys keys = new Keys(dynamoDbClient,
349 | config.getDynamoDbTables().getEcKeys().getTableName(),
350 | config.getDynamoDbTables().getPqKeys().getTableName(),
351 | config.getDynamoDbTables().getPqLastResortKeys().getTableName());
352 | MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(dynamoDbClient, dynamoDbAsyncClient,
353 | config.getDynamoDbTables().getMessages().getTableName(),
354 | config.getDynamoDbTables().getMessages().getExpiration(),
355 | messageDeletionAsyncExecutor);
356 | RemoteConfigs remoteConfigs = new RemoteConfigs(dynamoDbClient,
357 | config.getDynamoDbTables().getRemoteConfig().getTableName());
358 | PushChallengeDynamoDb pushChallengeDynamoDb = new PushChallengeDynamoDb(dynamoDbClient,
359 | config.getDynamoDbTables().getPushChallenge().getTableName());
360 | ReportMessageDynamoDb reportMessageDynamoDb = new ReportMessageDynamoDb(dynamoDbClient,
361 | config.getDynamoDbTables().getReportMessage().getTableName(),
362 | config.getReportMessageConfiguration().getReportTtl());
363 | VerificationCodeStore pendingAccounts = new VerificationCodeStore(dynamoDbClient,
364 | config.getDynamoDbTables().getPendingAccounts().getTableName());
365 | VerificationCodeStore pendingDevices = new VerificationCodeStore(dynamoDbClient,
366 | config.getDynamoDbTables().getPendingDevices().getTableName());
367 | RegistrationRecoveryPasswords registrationRecoveryPasswords = new RegistrationRecoveryPasswords(
368 | config.getDynamoDbTables().getRegistrationRecovery().getTableName(),
369 | config.getDynamoDbTables().getRegistrationRecovery().getExpiration(),
370 | dynamoDbClient,
371 | dynamoDbAsyncClient
372 | );
373 |
374 | final VerificationSessions verificationSessions = new VerificationSessions(dynamoDbAsyncClient,
375 | config.getDynamoDbTables().getVerificationSessions().getTableName(), clock);
376 |
377 | ClientResources redisClientResources = ClientResources.builder().build();
378 | ConnectionEventLogger.logConnectionEvents(redisClientResources);
379 |
380 | FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", config.getCacheClusterConfiguration(), redisClientResources);
381 | FaultTolerantRedisCluster messagesCluster = new FaultTolerantRedisCluster("messages_cluster", config.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClientResources);
382 | FaultTolerantRedisCluster clientPresenceCluster = new FaultTolerantRedisCluster("client_presence_cluster", config.getClientPresenceClusterConfiguration(), redisClientResources);
383 | FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster", config.getMetricsClusterConfiguration(), redisClientResources);
384 | FaultTolerantRedisCluster pushSchedulerCluster = new FaultTolerantRedisCluster("push_scheduler", config.getPushSchedulerCluster(), redisClientResources);
385 | FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters", config.getRateLimitersCluster(), redisClientResources);
386 |
387 | final BlockingQueue keyspaceNotificationDispatchQueue = new ArrayBlockingQueue<>(100_000);
388 | Metrics.gaugeCollectionSize(name(getClass(), "keyspaceNotificationDispatchQueueSize"), Collections.emptyList(),
389 | keyspaceNotificationDispatchQueue);
390 | final BlockingQueue receiptSenderQueue = new LinkedBlockingQueue<>();
391 | Metrics.gaugeCollectionSize(name(getClass(), "receiptSenderQueue"), Collections.emptyList(), receiptSenderQueue);
392 | final BlockingQueue fcmSenderQueue = new LinkedBlockingQueue<>();
393 | Metrics.gaugeCollectionSize(name(getClass(), "fcmSenderQueue"), Collections.emptyList(), fcmSenderQueue);
394 | final BlockingQueue messageDeliveryQueue = new LinkedBlockingQueue<>();
395 | Metrics.gaugeCollectionSize(MetricsUtil.name(getClass(), "messageDeliveryQueue"), Collections.emptyList(),
396 | messageDeliveryQueue);
397 |
398 | ScheduledExecutorService recurringJobExecutor = environment.lifecycle()
399 | .scheduledExecutorService(name(getClass(), "recurringJob-%d")).threads(6).build();
400 | ScheduledExecutorService websocketScheduledExecutor = environment.lifecycle().scheduledExecutorService(name(getClass(), "websocket-%d")).threads(8).build();
401 | ExecutorService keyspaceNotificationDispatchExecutor = environment.lifecycle().executorService(name(getClass(), "keyspaceNotification-%d")).maxThreads(16).workQueue(keyspaceNotificationDispatchQueue).build();
402 | ExecutorService apnSenderExecutor = environment.lifecycle().executorService(name(getClass(), "apnSender-%d")).maxThreads(1).minThreads(1).build();
403 | ExecutorService fcmSenderExecutor = environment.lifecycle().executorService(name(getClass(), "fcmSender-%d"))
404 | .maxThreads(32).minThreads(32).workQueue(fcmSenderQueue).build();
405 | ExecutorService secureValueRecoveryServiceExecutor = environment.lifecycle()
406 | .executorService(name(getClass(), "secureValueRecoveryService-%d")).maxThreads(1).minThreads(1).build();
407 | ExecutorService storageServiceExecutor = environment.lifecycle()
408 | .executorService(name(getClass(), "storageService-%d")).maxThreads(1).minThreads(1).build();
409 | ExecutorService accountDeletionExecutor = environment.lifecycle().executorService(name(getClass(), "accountCleaner-%d")).maxThreads(16).minThreads(16).build();
410 |
411 | Scheduler messageDeliveryScheduler = Schedulers.fromExecutorService(
412 | ExecutorServiceMetrics.monitor(Metrics.globalRegistry,
413 | environment.lifecycle().executorService(name(getClass(), "messageDelivery-%d"))
414 | .minThreads(20)
415 | .maxThreads(20)
416 | .workQueue(messageDeliveryQueue)
417 | .build(),
418 | MetricsUtil.name(getClass(), "messageDeliveryExecutor"), MetricsUtil.PREFIX),
419 | "messageDelivery");
420 | // TODO: generally speaking this is a DynamoDB I/O executor for the accounts table; we should eventually have a general executor for speaking to the accounts table, but most of the server is still synchronous so this isn't widely useful yet
421 | ExecutorService batchIdentityCheckExecutor = environment.lifecycle().executorService(name(getClass(), "batchIdentityCheck-%d")).minThreads(32).maxThreads(32).build();
422 | ExecutorService multiRecipientMessageExecutor = environment.lifecycle()
423 | .executorService(name(getClass(), "multiRecipientMessage-%d")).minThreads(64).maxThreads(64).build();
424 | ExecutorService subscriptionProcessorExecutor = environment.lifecycle()
425 | .executorService(name(getClass(), "subscriptionProcessor-%d"))
426 | .maxThreads(availableProcessors) // mostly this is IO bound so tying to number of processors is tenuous at best
427 | .minThreads(availableProcessors) // mostly this is IO bound so tying to number of processors is tenuous at best
428 | .allowCoreThreadTimeOut(true).
429 | build();
430 | ExecutorService receiptSenderExecutor = environment.lifecycle()
431 | .executorService(name(getClass(), "receiptSender-%d"))
432 | .maxThreads(2)
433 | .minThreads(2)
434 | .workQueue(receiptSenderQueue)
435 | .rejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy())
436 | .build();
437 | ExecutorService registrationCallbackExecutor = environment.lifecycle()
438 | .executorService(name(getClass(), "registration-%d"))
439 | .maxThreads(2)
440 | .minThreads(2)
441 | .build();
442 |
443 | final AdminEventLogger adminEventLogger = new GoogleCloudAdminEventLogger(
444 | LoggingOptions.newBuilder().setProjectId(config.getAdminEventLoggingConfiguration().projectId())
445 | .setCredentials(GoogleCredentials.fromStream(new ByteArrayInputStream(
446 | useSecondaryCredentialsJson
447 | ? config.getAdminEventLoggingConfiguration().secondaryCredentials().getBytes(StandardCharsets.UTF_8)
448 | : config.getAdminEventLoggingConfiguration().credentials().getBytes(StandardCharsets.UTF_8))))
449 | .build().getService(),
450 | config.getAdminEventLoggingConfiguration().projectId(),
451 | config.getAdminEventLoggingConfiguration().logName());
452 |
453 | StripeManager stripeManager = new StripeManager(config.getStripe().apiKey().value(), subscriptionProcessorExecutor,
454 | config.getStripe().idempotencyKeyGenerator().value(), config.getStripe().boostDescription(), config.getStripe()
455 | .supportedCurrencies());
456 | BraintreeManager braintreeManager = new BraintreeManager(config.getBraintree().merchantId(),
457 | config.getBraintree().publicKey(), config.getBraintree().privateKey().value(), config.getBraintree().environment(),
458 | config.getBraintree().supportedCurrencies(), config.getBraintree().merchantAccounts(),
459 | config.getBraintree().graphqlUrl(), config.getBraintree().circuitBreaker(), subscriptionProcessorExecutor);
460 |
461 | ExternalServiceCredentialsGenerator directoryV2CredentialsGenerator = DirectoryV2Controller.credentialsGenerator(
462 | config.getDirectoryV2Configuration().getDirectoryV2ClientConfiguration());
463 | ExternalServiceCredentialsGenerator storageCredentialsGenerator = SecureStorageController.credentialsGenerator(
464 | config.getSecureStorageServiceConfiguration());
465 | ExternalServiceCredentialsGenerator backupCredentialsGenerator = SecureBackupController.credentialsGenerator(
466 | config.getSecureBackupServiceConfiguration());
467 | ExternalServiceCredentialsGenerator paymentsCredentialsGenerator = PaymentsController.credentialsGenerator(
468 | config.getPaymentsServiceConfiguration());
469 | ExternalServiceCredentialsGenerator artCredentialsGenerator = ArtController.credentialsGenerator(
470 | config.getArtServiceConfiguration());
471 | ExternalServiceCredentialsGenerator svr2CredentialsGenerator = SecureValueRecovery2Controller.credentialsGenerator(
472 | config.getSvr2Configuration());
473 |
474 | dynamicConfigurationManager.start();
475 |
476 | ExperimentEnrollmentManager experimentEnrollmentManager = new ExperimentEnrollmentManager(
477 | dynamicConfigurationManager);
478 | RegistrationRecoveryPasswordsManager registrationRecoveryPasswordsManager = new RegistrationRecoveryPasswordsManager(
479 | registrationRecoveryPasswords);
480 | UsernameHashZkProofVerifier usernameHashZkProofVerifier = new UsernameHashZkProofVerifier();
481 |
482 | RegistrationServiceClient registrationServiceClient = new RegistrationServiceClient(
483 | config.getRegistrationServiceConfiguration().host(),
484 | config.getRegistrationServiceConfiguration().port(),
485 | useSecondaryCredentialsJson
486 | ? config.getRegistrationServiceConfiguration().secondaryCredentialConfigurationJson()
487 | : config.getRegistrationServiceConfiguration().credentialConfigurationJson(),
488 | config.getRegistrationServiceConfiguration().identityTokenAudience(),
489 | config.getRegistrationServiceConfiguration().registrationCaCertificate(),
490 | registrationCallbackExecutor);
491 | SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator,
492 | secureValueRecoveryServiceExecutor, config.getSecureBackupServiceConfiguration());
493 | SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(svr2CredentialsGenerator,
494 | secureValueRecoveryServiceExecutor, config.getSvr2Configuration());
495 | SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator,
496 | storageServiceExecutor, config.getSecureStorageServiceConfiguration());
497 | ClientPresenceManager clientPresenceManager = new ClientPresenceManager(clientPresenceCluster, recurringJobExecutor,
498 | keyspaceNotificationDispatchExecutor);
499 | StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
500 | StoredVerificationCodeManager pendingDevicesManager = new StoredVerificationCodeManager(pendingDevices);
501 | ProfilesManager profilesManager = new ProfilesManager(profiles, cacheCluster);
502 | MessagesCache messagesCache = new MessagesCache(messagesCluster, messagesCluster,
503 | keyspaceNotificationDispatchExecutor, messageDeliveryScheduler, messageDeletionAsyncExecutor, clock);
504 | PushLatencyManager pushLatencyManager = new PushLatencyManager(metricsCluster, dynamicConfigurationManager);
505 | ReportMessageManager reportMessageManager = new ReportMessageManager(reportMessageDynamoDb, rateLimitersCluster,
506 | config.getReportMessageConfiguration().getCounterTtl());
507 | MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, reportMessageManager,
508 | messageDeletionAsyncExecutor);
509 | AccountLockManager accountLockManager = new AccountLockManager(deletedAccountsLockDynamoDbClient, config.getDynamoDbTables().getDeletedAccountsLock().getTableName());
510 | AccountsManager accountsManager = new AccountsManager(accounts, phoneNumberIdentifiers, cacheCluster,
511 | accountLockManager, deletedAccounts, keys, messagesManager, profilesManager,
512 | pendingAccountsManager, secureStorageClient, secureBackupClient, secureValueRecovery2Client,
513 | clientPresenceManager,
514 | experimentEnrollmentManager, registrationRecoveryPasswordsManager, clock);
515 | RemoteConfigsManager remoteConfigsManager = new RemoteConfigsManager(remoteConfigs);
516 | APNSender apnSender = new APNSender(apnSenderExecutor, config.getApnConfiguration());
517 | FcmSender fcmSender = new FcmSender(fcmSenderExecutor, config.getFcmConfiguration().credentials().value());
518 | ApnPushNotificationScheduler apnPushNotificationScheduler = new ApnPushNotificationScheduler(pushSchedulerCluster,
519 | apnSender, accountsManager, Optional.empty(), dynamicConfigurationManager);
520 | PushNotificationManager pushNotificationManager = new PushNotificationManager(accountsManager, apnSender, fcmSender,
521 | apnPushNotificationScheduler, pushLatencyManager);
522 | RateLimiters rateLimiters = RateLimiters.createAndValidate(config.getLimitsConfiguration(),
523 | dynamicConfigurationManager, rateLimitersCluster);
524 | ProvisioningManager provisioningManager = new ProvisioningManager(config.getPubsubCacheConfiguration().getUri(),
525 | redisClientResources, config.getPubsubCacheConfiguration().getTimeout(),
526 | config.getPubsubCacheConfiguration().getCircuitBreakerConfiguration());
527 | IssuedReceiptsManager issuedReceiptsManager = new IssuedReceiptsManager(
528 | config.getDynamoDbTables().getIssuedReceipts().getTableName(),
529 | config.getDynamoDbTables().getIssuedReceipts().getExpiration(),
530 | dynamoDbAsyncClient,
531 | config.getDynamoDbTables().getIssuedReceipts().getGenerator());
532 | RedeemedReceiptsManager redeemedReceiptsManager = new RedeemedReceiptsManager(clock,
533 | config.getDynamoDbTables().getRedeemedReceipts().getTableName(),
534 | dynamoDbAsyncClient,
535 | config.getDynamoDbTables().getRedeemedReceipts().getExpiration());
536 | SubscriptionManager subscriptionManager = new SubscriptionManager(
537 | config.getDynamoDbTables().getSubscriptions().getTableName(), dynamoDbAsyncClient);
538 |
539 | final RegistrationLockVerificationManager registrationLockVerificationManager = new RegistrationLockVerificationManager(
540 | accountsManager, clientPresenceManager, backupCredentialsGenerator, svr2CredentialsGenerator, registrationRecoveryPasswordsManager, pushNotificationManager, rateLimiters);
541 | final PhoneVerificationTokenManager phoneVerificationTokenManager = new PhoneVerificationTokenManager(
542 | registrationServiceClient, registrationRecoveryPasswordsManager);
543 |
544 | final ReportedMessageMetricsListener reportedMessageMetricsListener = new ReportedMessageMetricsListener(
545 | accountsManager);
546 | reportMessageManager.addListener(reportedMessageMetricsListener);
547 |
548 | final AccountAuthenticator accountAuthenticator = new AccountAuthenticator(accountsManager);
549 | final DisabledPermittedAccountAuthenticator disabledPermittedAccountAuthenticator = new DisabledPermittedAccountAuthenticator(
550 | accountsManager);
551 |
552 | final MessageSender messageSender = new MessageSender(clientPresenceManager, messagesManager,
553 | pushNotificationManager,
554 | pushLatencyManager);
555 | final ReceiptSender receiptSender = new ReceiptSender(accountsManager, messageSender, receiptSenderExecutor);
556 | final TurnTokenGenerator turnTokenGenerator = new TurnTokenGenerator(dynamicConfigurationManager);
557 |
558 | RecaptchaClient recaptchaClient = new RecaptchaClient(
559 | config.getRecaptchaConfiguration().projectPath(),
560 | useSecondaryCredentialsJson
561 | ? config.getRecaptchaConfiguration().secondaryCredentialConfigurationJson()
562 | : config.getRecaptchaConfiguration().credentialConfigurationJson(),
563 | dynamicConfigurationManager);
564 | HttpClient hcaptchaHttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2)
565 | .connectTimeout(Duration.ofSeconds(10)).build();
566 | HCaptchaClient hCaptchaClient = new HCaptchaClient(config.getHCaptchaConfiguration().apiKey().value(), hcaptchaHttpClient,
567 | dynamicConfigurationManager);
568 | CaptchaChecker captchaChecker = new CaptchaChecker(List.of(recaptchaClient, hCaptchaClient));
569 |
570 | PushChallengeManager pushChallengeManager = new PushChallengeManager(pushNotificationManager,
571 | pushChallengeDynamoDb);
572 | RateLimitChallengeManager rateLimitChallengeManager = new RateLimitChallengeManager(pushChallengeManager,
573 | captchaChecker, rateLimiters);
574 |
575 | MessagePersister messagePersister = new MessagePersister(messagesCache, messagesManager, accountsManager,
576 | dynamicConfigurationManager, Duration.ofMinutes(config.getMessageCacheConfiguration().getPersistDelayMinutes()),
577 | Optional.empty());
578 | ChangeNumberManager changeNumberManager = new ChangeNumberManager(messageSender, accountsManager);
579 |
580 | AccountDatabaseCrawlerCache accountCleanerAccountDatabaseCrawlerCache =
581 | new AccountDatabaseCrawlerCache(cacheCluster, AccountDatabaseCrawlerCache.ACCOUNT_CLEANER_PREFIX);
582 | AccountDatabaseCrawler accountCleanerAccountDatabaseCrawler = new AccountDatabaseCrawler("Account cleaner crawler",
583 | accountsManager,
584 | accountCleanerAccountDatabaseCrawlerCache,
585 | List.of(new AccountCleaner(accountsManager, accountDeletionExecutor)),
586 | config.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
587 | dynamicConfigurationManager
588 | );
589 |
590 | // TODO listeners must be ordered so that ones that directly update accounts come last, so that read-only ones are not working with stale data
591 | final List accountDatabaseCrawlerListeners = List.of(
592 | new NonNormalizedAccountCrawlerListener(accountsManager, metricsCluster),
593 | // PushFeedbackProcessor may update device properties
594 | new PushFeedbackProcessor(accountsManager));
595 |
596 | AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(cacheCluster,
597 | AccountDatabaseCrawlerCache.GENERAL_PURPOSE_PREFIX);
598 | AccountDatabaseCrawler accountDatabaseCrawler = new AccountDatabaseCrawler("General-purpose account crawler",
599 | accountsManager,
600 | accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners,
601 | config.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
602 | dynamicConfigurationManager
603 | );
604 |
605 | HttpClient currencyClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).connectTimeout(Duration.ofSeconds(10)).build();
606 | FixerClient fixerClient = new FixerClient(currencyClient, config.getPaymentsServiceConfiguration().fixerApiKey().value());
607 | CoinMarketCapClient coinMarketCapClient = new CoinMarketCapClient(currencyClient, config.getPaymentsServiceConfiguration().coinMarketCapApiKey().value(), config.getPaymentsServiceConfiguration().coinMarketCapCurrencyIds());
608 | CurrencyConversionManager currencyManager = new CurrencyConversionManager(fixerClient, coinMarketCapClient,
609 | cacheCluster, config.getPaymentsServiceConfiguration().paymentCurrencies(), Clock.systemUTC());
610 |
611 | environment.lifecycle().manage(apnSender);
612 | environment.lifecycle().manage(apnPushNotificationScheduler);
613 | environment.lifecycle().manage(provisioningManager);
614 | environment.lifecycle().manage(accountDatabaseCrawler);
615 | environment.lifecycle().manage(accountCleanerAccountDatabaseCrawler);
616 | environment.lifecycle().manage(messagesCache);
617 | environment.lifecycle().manage(messagePersister);
618 | environment.lifecycle().manage(clientPresenceManager);
619 | environment.lifecycle().manage(currencyManager);
620 | environment.lifecycle().manage(registrationServiceClient);
621 |
622 | final RegistrationCaptchaManager registrationCaptchaManager = new RegistrationCaptchaManager(captchaChecker,
623 | rateLimiters, config.getTestDevices(), dynamicConfigurationManager);
624 |
625 | StaticCredentialsProvider cdnCredentialsProvider = StaticCredentialsProvider
626 | .create(AwsBasicCredentials.create(
627 | config.getCdnConfiguration().accessKey().value(),
628 | config.getCdnConfiguration().accessSecret().value()));
629 | S3Client cdnS3Client = S3Client.builder()
630 | .credentialsProvider(cdnCredentialsProvider)
631 | .region(Region.of(config.getCdnConfiguration().region()))
632 | .build();
633 | PostPolicyGenerator profileCdnPolicyGenerator = new PostPolicyGenerator(config.getCdnConfiguration().region(),
634 | config.getCdnConfiguration().bucket(), config.getCdnConfiguration().accessKey().value());
635 | PolicySigner profileCdnPolicySigner = new PolicySigner(config.getCdnConfiguration().accessSecret().value(),
636 | config.getCdnConfiguration().region());
637 |
638 | ServerSecretParams zkSecretParams = new ServerSecretParams(config.getZkConfig().serverSecret().value());
639 | // GenericServerSecretParams genericZkSecretParams = new GenericServerSecretParams(config.getGenericZkConfig().serverSecret().value());
640 | ServerZkProfileOperations zkProfileOperations = new ServerZkProfileOperations(zkSecretParams);
641 | ServerZkAuthOperations zkAuthOperations = new ServerZkAuthOperations(zkSecretParams);
642 | ServerZkReceiptOperations zkReceiptOperations = new ServerZkReceiptOperations(zkSecretParams);
643 |
644 | AuthFilter accountAuthFilter = new BasicCredentialAuthFilter.Builder().setAuthenticator(
645 | accountAuthenticator).buildAuthFilter();
646 | AuthFilter disabledPermittedAccountAuthFilter = new BasicCredentialAuthFilter.Builder().setAuthenticator(
647 | disabledPermittedAccountAuthenticator).buildAuthFilter();
648 |
649 | environment.servlets()
650 | .addFilter("RemoteDeprecationFilter", new RemoteDeprecationFilter(dynamicConfigurationManager))
651 | .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/*");
652 |
653 | environment.jersey().register(new RequestStatisticsFilter(TrafficSource.HTTP));
654 | environment.jersey().register(MultiRecipientMessageProvider.class);
655 | environment.jersey().register(new MetricsApplicationEventListener(TrafficSource.HTTP));
656 | environment.jersey()
657 | .register(new PolymorphicAuthDynamicFeature<>(ImmutableMap.of(AuthenticatedAccount.class, accountAuthFilter,
658 | DisabledPermittedAuthenticatedAccount.class, disabledPermittedAccountAuthFilter)));
659 | environment.jersey().register(new PolymorphicAuthValueFactoryProvider.Binder<>(
660 | ImmutableSet.of(AuthenticatedAccount.class, DisabledPermittedAuthenticatedAccount.class)));
661 | environment.jersey().register(new WebsocketRefreshApplicationEventListener(accountsManager, clientPresenceManager));
662 | environment.jersey().register(new TimestampResponseFilter());
663 |
664 | ///
665 | WebSocketEnvironment webSocketEnvironment = new WebSocketEnvironment<>(environment,
666 | config.getWebSocketConfiguration(), 90000);
667 | webSocketEnvironment.setAuthenticator(new WebSocketAccountAuthenticator(accountAuthenticator));
668 | webSocketEnvironment.setConnectListener(
669 | new AuthenticatedConnectListener(receiptSender, messagesManager, pushNotificationManager,
670 | clientPresenceManager, websocketScheduledExecutor, messageDeliveryScheduler));
671 | webSocketEnvironment.jersey()
672 | .register(new WebsocketRefreshApplicationEventListener(accountsManager, clientPresenceManager));
673 | webSocketEnvironment.jersey().register(new RequestStatisticsFilter(TrafficSource.WEBSOCKET));
674 | webSocketEnvironment.jersey().register(MultiRecipientMessageProvider.class);
675 | webSocketEnvironment.jersey().register(new MetricsApplicationEventListener(TrafficSource.WEBSOCKET));
676 | webSocketEnvironment.jersey().register(new KeepAliveController(clientPresenceManager));
677 |
678 | // these should be common, but use @Auth DisabledPermittedAccount, which isn’t supported yet on websocket
679 | environment.jersey().register(
680 | new AccountController(pendingAccountsManager, accountsManager, rateLimiters,
681 | registrationServiceClient, dynamicConfigurationManager, turnTokenGenerator,
682 | registrationCaptchaManager, pushNotificationManager, changeNumberManager,
683 | registrationLockVerificationManager, registrationRecoveryPasswordsManager, usernameHashZkProofVerifier, clock));
684 |
685 | environment.jersey().register(new KeysController(rateLimiters, keys, accountsManager));
686 |
687 | boolean registeredSpamFilter = false;
688 | ReportSpamTokenProvider reportSpamTokenProvider = null;
689 |
690 | for (final SpamFilter filter : ServiceLoader.load(SpamFilter.class)) {
691 | if (filter.getClass().isAnnotationPresent(FilterSpam.class)) {
692 | try {
693 | filter.configure(config.getSpamFilterConfiguration().getEnvironment());
694 |
695 | ReportSpamTokenProvider thisProvider = filter.getReportSpamTokenProvider();
696 | if (reportSpamTokenProvider == null) {
697 | reportSpamTokenProvider = thisProvider;
698 | } else if (thisProvider != null) {
699 | log.info("Multiple spam report token providers found. Using the first.");
700 | }
701 |
702 | filter.getReportedMessageListeners().forEach(reportMessageManager::addListener);
703 |
704 | environment.lifecycle().manage(filter);
705 | environment.jersey().register(filter);
706 | webSocketEnvironment.jersey().register(filter);
707 |
708 | log.info("Registered spam filter: {}", filter.getClass().getName());
709 | registeredSpamFilter = true;
710 | } catch (final Exception e) {
711 | log.warn("Failed to register spam filter: {}", filter.getClass().getName(), e);
712 | }
713 | } else {
714 | log.warn("Spam filter {} not annotated with @FilterSpam and will not be installed",
715 | filter.getClass().getName());
716 | }
717 |
718 | if (filter instanceof RateLimitChallengeListener) {
719 | log.info("Registered rate limit challenge listener: {}", filter.getClass().getName());
720 | rateLimitChallengeManager.addListener((RateLimitChallengeListener) filter);
721 | }
722 | }
723 |
724 | if (!registeredSpamFilter) {
725 | log.warn("No spam filters installed");
726 | }
727 |
728 | if (reportSpamTokenProvider == null) {
729 | log.warn("No spam-reporting token providers found; using default (no-op) provider as a default");
730 | reportSpamTokenProvider = ReportSpamTokenProvider.noop();
731 | }
732 |
733 | final List