├── .github └── workflows │ ├── ci.yml │ ├── create-release.yaml │ ├── license-update.yml │ └── upload-to-release.yaml ├── .gitignore ├── .release-please-manifest.json ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── README.md ├── config.toml ├── diesel.toml ├── pgcdc.example.config ├── release-please-config.json ├── scripts └── test.sh └── src ├── api ├── auth.rs ├── mod.rs ├── query.rs ├── server.rs ├── ws_handler.rs └── ws_utils.rs ├── cdc ├── connection.rs ├── mod.rs └── replication.rs ├── forwarder └── mod.rs ├── inner.rs ├── main.rs └── utils ├── config.rs ├── mod.rs └── specific_filter.rs /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Code quality - linting 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | 8 | env: 9 | RUST_BACKTRACE: 1 10 | 11 | jobs: 12 | style: 13 | name: Check Style 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | - uses: dtolnay/rust-toolchain@stable 18 | - uses: Swatinem/rust-cache@v2 19 | - run: cargo fmt --all --check 20 | - run: cargo clippy --all-features --all -- -D "warnings" -------------------------------------------------------------------------------- /.github/workflows/create-release.yaml: -------------------------------------------------------------------------------- 1 | name: Release please flow 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | tags-ignore: 8 | - '**' 9 | 10 | env: 11 | CARGO_TERM_COLOR: always 12 | 13 | jobs: 14 | release_please_tag: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: googleapis/release-please-action@v4 18 | with: 19 | token: ${{ secrets.GH_PAT_RELEASE_ACCESS }} 20 | skip-github-pull-request: true 21 | 22 | release_please_pr: 23 | runs-on: ubuntu-latest 24 | needs: release_please_tag 25 | permissions: 26 | contents: write 27 | pull-requests: write 28 | steps: 29 | - uses: googleapis/release-please-action@v4 30 | with: 31 | skip-github-release: true -------------------------------------------------------------------------------- /.github/workflows/license-update.yml: -------------------------------------------------------------------------------- 1 | name: Update copyright year(s) in license file 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: "0 3 1 1 *" 7 | 8 | jobs: 9 | update-license-year: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | with: 14 | fetch-depth: 0 15 | - uses: FantasticFiasco/action-update-license-year@v2 16 | with: 17 | token: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/upload-to-release.yaml: -------------------------------------------------------------------------------- 1 | name: Create Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | build: 10 | name: Publish ${{ matrix.job.target }} (${{ matrix.job.os }}) 11 | runs-on: ${{ matrix.job.os }} 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | job: 16 | - { target: aarch64-unknown-linux-gnu , os: ubuntu-latest } 17 | - { target: arm-unknown-linux-gnueabihf , os: ubuntu-latest } 18 | - { target: x86_64-unknown-linux-gnu , os: ubuntu-latest } 19 | 20 | steps: 21 | - uses: actions/checkout@v4 22 | - uses: dtolnay/rust-toolchain@stable 23 | with: 24 | targets: ${{ matrix.job.target }} 25 | - uses: Swatinem/rust-cache@v2 26 | 27 | - name: Build 28 | run: cargo build --release --all-features 29 | 30 | - name: Upload binaries to release 31 | uses: svenstaro/upload-release-action@v2 32 | with: 33 | repo_token: ${{ secrets.GITHUB_TOKEN }} 34 | file: target/release/speculare-pgcdc 35 | asset_name: speculare-pgcdc-${{ github.ref_name }}-${{ matrix.job.target }} 36 | tag: ${{ github.ref }} 37 | overwrite: true -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | pgcdc.config 3 | pgauth.config 4 | *.lock 5 | xschema.rs -------------------------------------------------------------------------------- /.release-please-manifest.json: -------------------------------------------------------------------------------- 1 | {".":"0.1.1"} -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [0.1.1](https://github.com/speculare-cloud/speculare-pgcdc/compare/v0.0.10...v0.1.1) (2024-09-24) 4 | 5 | 6 | ### Bug Fixes 7 | 8 | * add correct flow for release ([4b8b6c6](https://github.com/speculare-cloud/speculare-pgcdc/commit/4b8b6c6e9a85f7c31aadac1625e62f2c68433dcb)) 9 | * **ci:** use ubuntu latest ([a57b2a3](https://github.com/speculare-cloud/speculare-pgcdc/commit/a57b2a3682193b4dc01483606d86de6e3d67f047)) 10 | * clippy & version upd of moka ([cdd4277](https://github.com/speculare-cloud/speculare-pgcdc/commit/cdd42774563cd347bd89e188aa2d17344da81c0e)) 11 | * clippy warn ([57d0c5d](https://github.com/speculare-cloud/speculare-pgcdc/commit/57d0c5d02a48aacee0bcf6b62b002861fcfc11a4)) 12 | * remove unused struct ([412fdd7](https://github.com/speculare-cloud/speculare-pgcdc/commit/412fdd7e0512b23b6ce1b5e4f3b54fe326432a3f)) 13 | * rustdoc ([adc6a3e](https://github.com/speculare-cloud/speculare-pgcdc/commit/adc6a3ec8a3c9d11c55f071a5501e0523569a182)) 14 | 15 | 16 | ### Miscellaneous Chores 17 | 18 | * release 0.1.1 ([8f314b9](https://github.com/speculare-cloud/speculare-pgcdc/commit/8f314b90de9a775fa8f02b011412b44ef13936be)) 19 | 20 | ## [0.0.10](https://github.com/speculare-cloud/speculare-pgcdc/compare/v0.0.9...v0.0.10) (2024-09-24) 21 | 22 | 23 | ### Bug Fixes 24 | 25 | * add correct flow for release ([4b8b6c6](https://github.com/speculare-cloud/speculare-pgcdc/commit/4b8b6c6e9a85f7c31aadac1625e62f2c68433dcb)) 26 | * clippy & version upd of moka ([cdd4277](https://github.com/speculare-cloud/speculare-pgcdc/commit/cdd42774563cd347bd89e188aa2d17344da81c0e)) 27 | * clippy warn ([57d0c5d](https://github.com/speculare-cloud/speculare-pgcdc/commit/57d0c5d02a48aacee0bcf6b62b002861fcfc11a4)) 28 | * remove unused struct ([412fdd7](https://github.com/speculare-cloud/speculare-pgcdc/commit/412fdd7e0512b23b6ce1b5e4f3b54fe326432a3f)) 29 | * rustdoc ([adc6a3e](https://github.com/speculare-cloud/speculare-pgcdc/commit/adc6a3ec8a3c9d11c55f071a5501e0523569a182)) 30 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "speculare-pgcdc" 3 | version = "0.1.1" 4 | authors = ["Martichou "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | [dependencies] 9 | sproot = { git = "https://github.com/speculare-cloud/sproot" } 10 | async-trait = "0.1" 11 | axum = { version = "0.7", features = ["ws", "http1", "http2"] } 12 | axum-extra = { version = "0.9", features = ["cookie-signed"], optional = true} 13 | axum-server = { version = "0.7", features = ["tls-rustls"] } 14 | bastion = "0.4" 15 | bastion-executor = { version = "0.4", features = ["tokio-runtime"] } 16 | bytes = "1.2" 17 | byteorder = "1.4" 18 | clap = { version = "4.2", features = ["derive"] } 19 | clap-verbosity-flag = "2.0" 20 | config = { version = "0.14", features = ["toml"] } 21 | diesel = { version = "2.0", features = ["postgres", "r2d2", "chrono"], optional = true } 22 | futures = "0.3" 23 | log = "0.4" 24 | moka = { version = "0.12", features = ["sync"], optional = true } 25 | once_cell = "1.14" 26 | openssl = "0.10" 27 | postgres-openssl = { git = "https://github.com/Martichou/rust-postgres", branch = "dev" } 28 | r2d2 = "0.8" 29 | serde = { version = "1.0", features = ["derive"] } 30 | serde_json = "1.0" 31 | simd-json = "0.14" 32 | tokio-postgres = { git = "https://github.com/Martichou/rust-postgres", branch = "dev" } 33 | tokio = { version = "1", features = ["macros", "rt-multi-thread"] } 34 | tokio-stream = "0.1" 35 | tower-http = { version = "0.6", features = ["trace"] } 36 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 37 | uuid-readable-rs = "0.1" 38 | uuid = { version = "1.10", features = ["v4"], optional = true } 39 | 40 | [features] 41 | default = ["timescale"] 42 | auth = ["moka", "uuid", "diesel", "axum-extra"] 43 | timescale = [] 44 | 45 | [profile.release] 46 | lto = true 47 | opt-level = 3 48 | codegen-units = 1 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2020-2022 Martin Andre 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

Speculare PGCDC

3 |

4 | Capture Data Change for Speculare 5 |

6 |

7 | 8 | [![Apache 2 License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE) 9 | [![CI](https://github.com/speculare-cloud/speculare-pgcdc/workflows/CI/badge.svg)](https://github.com/speculare-cloud/speculare-pgcdc/actions) 10 | [![Docs](https://img.shields.io/badge/Docs-latest-green.svg)](https://docs.speculare.cloud) 11 | 12 |

13 |
14 | 15 | Speculare PGCDC allows you to listen to changes in your PostgreSQL database via logical replication and then broadcast those changes over websockets. 16 | 17 | `Realtime` server works by: 18 | 1. listening to PostgreSQL's logical replication (here using Wal2Json). 19 | 2. filtering the incoming message 20 | 3. broadcasting the message over websocket 21 | 22 | Explaination 23 | -------------------------- 24 | 25 | You probably know that Postgresql is not a realtime databse. So if we want to stream the change of it to a websocket or any UI it's not possible by default. 26 | 27 | Hopefully Postgresql have that sweet feature named `Logical Replication`, which stream the change made into the database over a replication slot. We can use a multitude of plugins to format the output of this stream, but for Speculare-PGCDC we've chosen to use wal2json. 28 | 29 | This project create a replication slot on the targeted postgres instance and then stream the change from this slot to all the websockets connected. 30 | 31 | Server setup / Dev setup 32 | -------------------------- 33 | 34 | - Install all build dependencies 35 | ```bash 36 | $ sudo apt-get install cmake libssl-dev libpq-dev pkg-config build-essential 37 | ``` 38 | 39 | - Create a pgcdc.config file based on pgcdc.example.config 40 | 41 | > **⚠ WARNING: Check the [docs](https://docs.speculare.cloud) !** 42 | 43 | Usage 44 | -------------------------- 45 | 46 | ``` 47 | $ wss://server/ws?query=change_type:table:col.eq.val 48 | ``` 49 | will get `change_type` event from `table` where `col` is `equals` to `val`. 50 | 51 | The `change_type` and `table` parameters are mandatory, if you're missing them you'll get a 400 error. 52 | `change_type` can be any of those: *, insert, update, delete. 53 | `table` must be a valid table of your database. 54 | 55 | I decided to restrict the API in such way that a single websocket can only listen to one table. 56 | This might change in the future if needed, but as of now and in the current shape of Speculare, it's not needed. 57 | 58 | Contributing 59 | -------------------------- 60 | 61 | Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change. -------------------------------------------------------------------------------- /config.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = ["-C", "target-cpu=native"] -------------------------------------------------------------------------------- /diesel.toml: -------------------------------------------------------------------------------- 1 | # For documentation on how to configure this file, 2 | # see diesel.rs/guides/configuring-diesel-cli 3 | 4 | [print_schema] 5 | file = "src/xschema.rs" 6 | -------------------------------------------------------------------------------- /pgcdc.example.config: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # POSTGRESQL CONNECTION 3 | #------------------------------------------------------------------------------ 4 | 5 | database_host = "127.0.0.1" 6 | database_dbname = "speculare" 7 | database_user = "postgres" 8 | database_password = "azertyuiop" 9 | # database_tls = false 10 | 11 | #------------------------------------------------------------------------------ 12 | # AUTH POSTGRESQL CONNECTION (optional, needed if feature = ["auth"]) 13 | #------------------------------------------------------------------------------ 14 | 15 | # auth_database_url = "postgres://postgres:azerty@yourserver.instace.cloud/speculare?sslmode=require" 16 | # auth_database_max_connection = 10 17 | 18 | #------------------------------------------------------------------------------ 19 | # API SETTINGS 20 | #------------------------------------------------------------------------------ 21 | 22 | binding = "0.0.0.0:8080" 23 | # https = false 24 | # key_priv = "path/to/sslkey.key" 25 | # key_cert = "path/to/sslkey.cert" 26 | 27 | # (optional, need feature = ["auth"]) 28 | cookie_secret = "64_CHARS_LONG_SECRET" 29 | # (optional, needed if feature = ["auth"]) 30 | admin_secret = "64_CHARS_LONG_SECRET" -------------------------------------------------------------------------------- /release-please-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "prerelease": true, 3 | "packages": { 4 | ".": { 5 | "release-type": "simple" 6 | } 7 | }, 8 | "extra-files": [ 9 | { 10 | "type": "toml", 11 | "path": "Cargo.toml", 12 | "jsonpath": "package.version" 13 | }, 14 | { 15 | "type": "toml", 16 | "path": "Cargo.lock", 17 | "jsonpath": "$.package[?(@.name.value == 'speculare-pgcdc')].version" 18 | } 19 | ], 20 | "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" 21 | } -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # First create the TABLE we need 3 | PGPASSWORD=password psql -U postgres -h 0.0.0.0 -d pgcdc -c "CREATE TABLE IF NOT EXISTS test_table0(id serial primary key, name text);" 4 | PGPASSWORD=password psql -U postgres -h 0.0.0.0 -d pgcdc -c "CREATE TABLE IF NOT EXISTS test_table1(id serial primary key, name text);" 5 | # Then loop every 3s and insert something to trigger a change in the WAL 6 | x=1 7 | while [ true ] 8 | do 9 | name="W$x" 10 | y=$(($x%2)) 11 | # Insert into TABLE 12 | PGPASSWORD=password psql -U postgres -h 0.0.0.0 -d pgcdc -c "insert into test_table$y(name) values('$name');" 13 | echo "Inserted $name" 14 | x=$(( $x + 1 )) 15 | sleep 1 16 | PGPASSWORD=password psql -U postgres -h 0.0.0.0 -d pgcdc -c "update test_table$y set name='~~$name' where name='$name';" 17 | sleep 1 18 | done -------------------------------------------------------------------------------- /src/api/auth.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use crate::{ 4 | utils::specific_filter::{DataType, SpecificFilter}, 5 | CONFIG, 6 | }; 7 | 8 | use async_trait::async_trait; 9 | use axum::{ 10 | extract::{FromRef, FromRequestParts}, 11 | http::{request::Parts, StatusCode}, 12 | }; 13 | use axum_extra::extract::SignedCookieJar; 14 | use diesel::{r2d2::ConnectionManager, PgConnection}; 15 | use moka::sync::Cache; 16 | use once_cell::sync::Lazy; 17 | use serde::Deserialize; 18 | use sproot::{apierrors::ApiError, as_variant, models::ApiKey, Pool}; 19 | use uuid::Uuid; 20 | 21 | use super::AppState; 22 | 23 | const COOKIE_NAME: &str = "SP-CKS"; 24 | 25 | static CHECKSESSIONS_CACHE: Lazy> = Lazy::new(|| { 26 | Cache::builder() 27 | .time_to_live(Duration::from_secs(60 * 60)) 28 | .build() 29 | }); 30 | 31 | static CHECKAPI_CACHE: Lazy> = Lazy::new(|| { 32 | Cache::builder() 33 | .time_to_live(Duration::from_secs(60 * 60)) 34 | .build() 35 | }); 36 | 37 | pub static AUTHPOOL: Lazy = Lazy::new(|| { 38 | // Init the connection to the postgresql 39 | let manager = ConnectionManager::::new(&CONFIG.auth_database_url); 40 | // This step might spam for error CONFIG.database_max_connection of times, this is normal. 41 | match r2d2::Pool::builder() 42 | .max_size(CONFIG.auth_database_max_connection) 43 | .min_idle(Some((10 * CONFIG.auth_database_max_connection) / 100)) 44 | .build(manager) 45 | { 46 | Ok(pool) => { 47 | info!("R2D2 PostgreSQL pool created"); 48 | pool 49 | } 50 | Err(e) => { 51 | error!("Failed to create db pool: {}", e); 52 | std::process::exit(1); 53 | } 54 | } 55 | }); 56 | 57 | #[derive(Debug, Deserialize)] 58 | pub struct AuthCookie { 59 | pub user_id: String, 60 | } 61 | 62 | #[derive(Debug, Deserialize)] 63 | pub struct AuthInfo { 64 | pub is_admin: bool, 65 | pub auth_cookie: Option, 66 | } 67 | 68 | #[async_trait] 69 | impl FromRequestParts for AuthInfo 70 | where 71 | B: Send + Sync, 72 | AppState: FromRef, 73 | { 74 | type Rejection = (StatusCode, &'static str); 75 | 76 | async fn from_request_parts(req: &mut Parts, state: &B) -> Result { 77 | // dbg!("Cookies: {:?}", req.headers().get(COOKIE).split(';')); 78 | let cookies: SignedCookieJar = 79 | match SignedCookieJar::from_request_parts(req, state).await { 80 | Ok(cookies) => cookies, 81 | Err(_) => { 82 | return Err(( 83 | StatusCode::BAD_REQUEST, 84 | "you need to send a signed cookies along with your request", 85 | )) 86 | } 87 | }; 88 | 89 | let mut is_admin = false; 90 | let spcks = match cookies.get(COOKIE_NAME) { 91 | Some(cookie) => Some(cookie), 92 | None => { 93 | let adm = req.headers.get("SP-ADM"); 94 | if adm.is_none() || adm.unwrap().to_str().unwrap() != CONFIG.admin_secret { 95 | return Err((StatusCode::UNAUTHORIZED, "no `SP-CKS` found in cookies")); 96 | } 97 | 98 | is_admin = true; 99 | None 100 | } 101 | }; 102 | 103 | let auth_cookie = match spcks { 104 | Some(spcks) => { 105 | let mut value = spcks.value().to_owned().replace("\\\"", ""); 106 | match unsafe { simd_json::from_str::(&mut value) } { 107 | Ok(val) => Some(val), 108 | Err(_) => { 109 | return Err(( 110 | StatusCode::BAD_REQUEST, 111 | "cannot find the user_id inside the cookie", 112 | )) 113 | } 114 | } 115 | } 116 | None => None, 117 | }; 118 | 119 | Ok(Self { 120 | is_admin, 121 | auth_cookie, 122 | }) 123 | } 124 | } 125 | 126 | pub async fn restrict_auth(auth: AuthInfo, specific: SpecificFilter) -> Result<(), ApiError> { 127 | let auth_cookie = auth.auth_cookie.unwrap(); 128 | 129 | let sp_value = match as_variant!(specific.value, DataType::String) { 130 | Some(val) => val, 131 | None => { 132 | return Err(ApiError::InvalidRequestError(None)); 133 | } 134 | }; 135 | 136 | if specific.column == "host_uuid" || specific.column == "uuid" { 137 | // Parse the user_id into a UUID 138 | let uuid = match Uuid::parse_str(&auth_cookie.user_id) { 139 | Ok(uuid) => uuid, 140 | Err(_) => { 141 | return Err(ApiError::InvalidRequestError(None)); 142 | } 143 | }; 144 | 145 | // Check if value is present in the cache, otherwise check the database 146 | let cuid = auth_cookie.user_id.clone(); 147 | if CHECKSESSIONS_CACHE.get(&sp_value) == Some(auth_cookie.user_id) { 148 | trace!("CheckSessions: cache hit for {}", &sp_value); 149 | return Ok(()); 150 | } 151 | 152 | // Get a conn from the auth_db's pool 153 | let mut conn = match AUTHPOOL.get() { 154 | Ok(conn) => conn, 155 | Err(err) => { 156 | error!("failed to get a conn: {}", err); 157 | return Err(ApiError::ServerError(None)); 158 | } 159 | }; 160 | 161 | let csp = sp_value.clone(); 162 | let exists = tokio::task::spawn_blocking(move || { 163 | ApiKey::exists_by_owner_and_host(&mut conn, &uuid, &sp_value) 164 | }) 165 | .await 166 | .map_err(|_| ApiError::ServerError(None))??; 167 | 168 | if exists { 169 | CHECKSESSIONS_CACHE.insert(csp, cuid); 170 | return Ok(()); 171 | } 172 | 173 | return Err(ApiError::AuthorizationError(None)); 174 | } 175 | 176 | if specific.column == "customer_id" && sp_value == auth_cookie.user_id { 177 | return Ok(()); 178 | } 179 | 180 | if specific.column == "key" { 181 | // Parse the user_id into a UUID 182 | let uuid = match Uuid::parse_str(&auth_cookie.user_id) { 183 | Ok(uuid) => uuid, 184 | Err(_) => { 185 | return Err(ApiError::InvalidRequestError(None)); 186 | } 187 | }; 188 | 189 | // If the keys exists in the cache but it's not for the same user, error 190 | if let Some(cached) = CHECKAPI_CACHE.get(&sp_value) { 191 | if cached == uuid { 192 | trace!("CheckSessions: cache hit for {}", &sp_value); 193 | return Ok(()); 194 | } else { 195 | return Err(ApiError::AuthorizationError(None)); 196 | } 197 | } 198 | 199 | // Get a conn from the auth_db's pool 200 | let mut conn = match AUTHPOOL.get() { 201 | Ok(conn) => conn, 202 | Err(err) => { 203 | error!("failed to get a conn: {}", err); 204 | return Err(ApiError::ServerError(None)); 205 | } 206 | }; 207 | 208 | let csp = sp_value.clone(); 209 | let exists = tokio::task::spawn_blocking(move || { 210 | ApiKey::exists_by_owner_and_key(&mut conn, &uuid, &sp_value) 211 | }) 212 | .await 213 | .map_err(|_| ApiError::ServerError(None))??; 214 | 215 | if exists { 216 | CHECKAPI_CACHE.insert(csp, uuid); 217 | return Ok(()); 218 | } 219 | 220 | return Err(ApiError::AuthorizationError(None)); 221 | } 222 | 223 | Err(ApiError::AuthorizationError(None)) 224 | } 225 | -------------------------------------------------------------------------------- /src/api/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "auth")] 2 | use axum::extract::FromRef; 3 | #[cfg(feature = "auth")] 4 | use axum_extra::extract::cookie::Key; 5 | 6 | #[cfg(feature = "auth")] 7 | pub mod auth; 8 | pub mod query; 9 | pub mod server; 10 | pub mod ws_handler; 11 | pub mod ws_utils; 12 | 13 | #[cfg(feature = "auth")] 14 | #[derive(Clone)] 15 | struct AppState { 16 | key: Key, 17 | } 18 | 19 | #[cfg(feature = "auth")] 20 | impl FromRef for Key { 21 | fn from_ref(state: &AppState) -> Self { 22 | state.key.clone() 23 | } 24 | } 25 | 26 | #[cfg(feature = "auth")] 27 | impl From for Key { 28 | fn from(value: AppState) -> Self { 29 | value.key 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/api/query.rs: -------------------------------------------------------------------------------- 1 | use super::ws_utils::{self, WsWatchFor}; 2 | 3 | use crate::{ 4 | utils::specific_filter::{DataType, SpecificFilter}, 5 | TABLES, 6 | }; 7 | 8 | use sproot::apierrors::ApiError; 9 | 10 | pub fn parse_ws_query(query: &str) -> Result { 11 | let mut parts = query.split(':'); 12 | let mut change_flag = 0; 13 | 14 | // Apply bit operation to the change_flag based on the query type 15 | match parts.next() { 16 | Some(val) => val 17 | .split(',') 18 | .for_each(|ctype| ws_utils::apply_flag(&mut change_flag, ctype)), 19 | None => { 20 | return Err(ApiError::ExplicitError(String::from( 21 | "the change_type params is not present", 22 | ))); 23 | } 24 | } 25 | 26 | // If change_flag is 0, we have an error because we don't listen to any known event types 27 | if change_flag == 0 { 28 | return Err(ApiError::ExplicitError(String::from( 29 | "the change_type params does not match requirements", 30 | ))); 31 | } 32 | 33 | // Get the change_table and check if the table is valid 34 | let change_table = match parts.next() { 35 | Some(table) => { 36 | // Check if the table exists inside TABLES 37 | if !TABLES.read().unwrap().iter().any(|v| v == table) { 38 | return Err(ApiError::ExplicitError(String::from( 39 | "the table asked for does not exists", 40 | ))); 41 | } 42 | table.to_owned() 43 | } 44 | None => { 45 | return Err(ApiError::ExplicitError(String::from( 46 | "the change_table params is not present", 47 | ))); 48 | } 49 | }; 50 | 51 | // Construct the SpecificFilter from the request 52 | let specific: Option = if let Some(filter) = parts.next() { 53 | let mut fparts = filter.splitn(3, '.'); 54 | // let mut fparts = filter.splitn(2, ".eq."); 55 | match (fparts.next(), fparts.next(), fparts.next()) { 56 | (Some(col), Some(eq), Some(val)) => match eq { 57 | "eq" => Some(SpecificFilter { 58 | column: serde_json::Value::String(col.to_owned()), 59 | value: DataType::String(val.to_owned()), 60 | }), 61 | "in" => { 62 | let items = val 63 | .split(',') 64 | .map(|s| s.to_string()) 65 | .collect::>(); 66 | Some(SpecificFilter { 67 | column: serde_json::Value::String(col.to_owned()), 68 | value: DataType::Array(items), 69 | }) 70 | } 71 | _ => None, 72 | }, 73 | _ => None, 74 | } 75 | } else { 76 | None 77 | }; 78 | 79 | // Construct what the client is listening to 80 | Ok(WsWatchFor { 81 | change_table, 82 | change_flag, 83 | specific, 84 | }) 85 | } 86 | -------------------------------------------------------------------------------- /src/api/server.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "auth")] 2 | use super::AppState; 3 | use super::{ws_handler, ws_utils::ServerState}; 4 | 5 | use crate::CONFIG; 6 | 7 | use axum::{ 8 | routing::{any, get}, 9 | Extension, Router, 10 | }; 11 | #[cfg(feature = "auth")] 12 | use axum_extra::extract::cookie::Key; 13 | use axum_server::tls_rustls::RustlsConfig; 14 | use sproot::{apierrors::ApiError, field_isset}; 15 | use std::{net::SocketAddr, sync::Arc}; 16 | use tower_http::trace::{DefaultMakeSpan, TraceLayer}; 17 | 18 | pub async fn serve(serv_state: Arc) { 19 | #[cfg(feature = "auth")] 20 | let state = AppState { 21 | key: Key::from(CONFIG.cookie_secret.as_bytes()), 22 | }; 23 | 24 | // build our application with some routes 25 | let app = Router::new() 26 | .route("/ping", any(|| async { "zpour" })) 27 | .route("/ws", get(ws_handler::accept_conn)) 28 | // logging so we can see whats going on 29 | .layer(TraceLayer::new_for_http().make_span_with(DefaultMakeSpan::default())) 30 | .layer(Extension(serv_state)); 31 | 32 | #[cfg(feature = "auth")] 33 | let app = app.with_state(state); 34 | 35 | // Convert the binding into a SocketAddr 36 | let socket: SocketAddr = match CONFIG.binding.parse() { 37 | Ok(val) => val, 38 | Err(e) => { 39 | error!("The BINDING is not a valid SocketAddr: {}", e); 40 | std::process::exit(1); 41 | } 42 | }; 43 | 44 | // Run the axum server 45 | if CONFIG.https { 46 | info!("API served on {} (HTTPS)", socket); 47 | axum_server::bind_rustls( 48 | socket, 49 | RustlsConfig::from_pem_file( 50 | field_isset!(CONFIG.key_cert.as_ref(), "key_cert").unwrap(), 51 | field_isset!(CONFIG.key_priv.as_ref(), "key_priv").unwrap(), 52 | ) 53 | .await 54 | .unwrap(), 55 | ) 56 | .serve(app.into_make_service()) 57 | .await 58 | .unwrap(); 59 | } else { 60 | info!("API served on {} (HTTP)", socket); 61 | axum_server::bind(socket) 62 | .serve(app.into_make_service()) 63 | .await 64 | .unwrap(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/api/ws_handler.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "auth")] 2 | use super::auth::{self, AuthInfo}; 3 | 4 | use crate::{ 5 | api::ws_utils::{DELETE, UPDATE}, 6 | ID_COUNTER, 7 | }; 8 | 9 | use axum::{ 10 | extract::{ 11 | ws::{Message, WebSocket}, 12 | Query, WebSocketUpgrade, 13 | }, 14 | response::Response, 15 | Extension, 16 | }; 17 | use futures::{stream::SplitStream, FutureExt, StreamExt}; 18 | use sproot::apierrors::ApiError; 19 | use std::{collections::HashMap, sync::Arc}; 20 | use tokio::sync::mpsc::{self, UnboundedSender}; 21 | use tokio_stream::wrappers::UnboundedReceiverStream; 22 | 23 | use super::{ 24 | query, 25 | ws_utils::{ServerState, SessionInfo, WsWatchFor, INSERT}, 26 | }; 27 | 28 | pub async fn accept_conn( 29 | #[cfg(feature = "auth")] auth: AuthInfo, 30 | Extension(state): Extension>, 31 | Query(params): Query>, 32 | ws: WebSocketUpgrade, 33 | ) -> Result { 34 | // Extract the query params or return a bad request 35 | let query = match params.get("query") { 36 | Some(q) => q, 37 | None => { 38 | return Err(ApiError::ExplicitError(String::from( 39 | "missing the query params", 40 | ))) 41 | } 42 | }; 43 | 44 | // Construct the watch_for from the query and if error, bad request 45 | let watch_for = query::parse_ws_query(query)?; 46 | 47 | #[cfg(feature = "auth")] 48 | { 49 | if !auth.is_admin { 50 | if watch_for.specific.is_none() { 51 | return Err(ApiError::InvalidRequestError(None)); 52 | } 53 | 54 | let specific = watch_for.specific.clone().unwrap(); 55 | auth::restrict_auth(auth, specific).await?; 56 | } 57 | } 58 | 59 | Ok(ws.on_upgrade(|socket: WebSocket| async { 60 | let id = ID_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); 61 | trace!("Websocket: client connected: {}", id); 62 | 63 | // Split the socket into a sender and receive of messages. 64 | let (user_ws_tx, user_ws_rx) = socket.split(); 65 | 66 | // Use an bounded channel (256) to handle buffering and flushing of messages to the websocket. 67 | let (tx, rx) = mpsc::unbounded_channel(); 68 | let rx = UnboundedReceiverStream::new(rx); 69 | tokio::task::spawn(rx.forward(user_ws_tx).map(|result| { 70 | if let Err(err) = result { 71 | error!("Websocket: send error for: {}", err); 72 | } 73 | })); 74 | 75 | ws_connected(id, tx, user_ws_rx, watch_for, state).await; 76 | })) 77 | } 78 | 79 | async fn ws_connected( 80 | id: usize, 81 | tx: UnboundedSender>, 82 | mut user_ws_rx: SplitStream, 83 | watch_for: WsWatchFor, 84 | state: Arc, 85 | ) { 86 | let change_flag = watch_for.change_flag; 87 | let change_table = watch_for.change_table.to_owned(); 88 | // Save the sender in our list of connected clients. 89 | state.clients.write().unwrap().insert( 90 | id, 91 | SessionInfo { 92 | gate: tx.clone(), 93 | watch_for, 94 | }, 95 | ); 96 | 97 | // Insert in the right category depending on the ChangeType 98 | if has_bit!(change_flag, INSERT) { 99 | state 100 | .inserts 101 | .write() 102 | .unwrap() 103 | .entry(change_table.clone()) 104 | .or_default() 105 | .insert(id); 106 | } 107 | if has_bit!(change_flag, UPDATE) { 108 | state 109 | .updates 110 | .write() 111 | .unwrap() 112 | .entry(change_table.clone()) 113 | .or_default() 114 | .insert(id); 115 | } 116 | if has_bit!(change_flag, DELETE) { 117 | state 118 | .deletes 119 | .write() 120 | .unwrap() 121 | .entry(change_table) 122 | .or_default() 123 | .insert(id); 124 | } 125 | 126 | while let Some(event) = user_ws_rx.next().await { 127 | match event { 128 | Ok(payload) => { 129 | debug!("Websocket: msg: {:?}", payload); 130 | if let Message::Close(_) = payload { 131 | info!("Websocket: client closed"); 132 | break; 133 | } 134 | } 135 | Err(err) => { 136 | error!("Websocket: error: {}", err); 137 | break; 138 | } 139 | } 140 | } 141 | 142 | ws_disconnected(id, state, change_flag); 143 | } 144 | 145 | fn ws_disconnected(id: usize, state: Arc, change_flag: u8) { 146 | trace!("Websocket: client disconnected: {}", id); 147 | // Stream closed up, so remove from the user list 148 | state.clients.write().unwrap().remove(&id); 149 | 150 | if has_bit!(change_flag, INSERT) { 151 | // For each table entries, remove the id of the ws_session 152 | for list_sessions in state.inserts.write().unwrap().values_mut() { 153 | // Even if the event.id is not in the list_sessions, it will try 154 | list_sessions.remove(&id); 155 | } 156 | } 157 | if has_bit!(change_flag, UPDATE) { 158 | for list_sessions in state.updates.write().unwrap().values_mut() { 159 | list_sessions.remove(&id); 160 | } 161 | } 162 | if has_bit!(change_flag, DELETE) { 163 | for list_sessions in state.deletes.write().unwrap().values_mut() { 164 | list_sessions.remove(&id); 165 | } 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /src/api/ws_utils.rs: -------------------------------------------------------------------------------- 1 | use crate::utils::specific_filter::SpecificFilter; 2 | 3 | use axum::extract::ws::Message; 4 | use std::{ 5 | collections::{HashMap, HashSet}, 6 | sync::{Arc, RwLock}, 7 | }; 8 | use tokio::sync::mpsc; 9 | 10 | pub const INSERT: u8 = 1 << 1; 11 | pub const UPDATE: u8 = 1 << 2; 12 | pub const DELETE: u8 = 1 << 3; 13 | 14 | pub struct SessionInfo { 15 | pub gate: mpsc::UnboundedSender>, 16 | pub watch_for: WsWatchFor, 17 | } 18 | 19 | /// Our state of currently connected clients. 20 | /// 21 | /// - Key is their id 22 | /// - Value is a sender of `warp::ws::Message` 23 | type Clients = Arc>>; 24 | 25 | /// Our state of currently connected clients listening for a particular table. 26 | /// 27 | /// - Key is the table name 28 | /// - Value is a AHashSet containing all the client's id listening to that table. 29 | type TypeList = Arc>>>; 30 | 31 | /// Contains info for what does the Ws is listening to 32 | pub struct WsWatchFor { 33 | pub change_table: String, 34 | pub change_flag: u8, 35 | pub specific: Option, 36 | } 37 | 38 | pub fn apply_flag(flag: &mut u8, ctype: &str) { 39 | match ctype { 40 | "insert" => { 41 | *flag |= INSERT; 42 | } 43 | "update" => { 44 | *flag |= UPDATE; 45 | } 46 | "delete" => { 47 | *flag |= DELETE; 48 | } 49 | "*" => { 50 | *flag |= INSERT; 51 | *flag |= UPDATE; 52 | *flag |= DELETE; 53 | } 54 | _ => { 55 | error!("parts[0] (change_type) don't match any of the available types.") 56 | } 57 | } 58 | } 59 | 60 | /// Server state for WebSocket 61 | #[derive(Default, Clone)] 62 | pub struct ServerState { 63 | pub clients: Clients, 64 | pub inserts: TypeList, 65 | pub updates: TypeList, 66 | pub deletes: TypeList, 67 | } 68 | -------------------------------------------------------------------------------- /src/cdc/connection.rs: -------------------------------------------------------------------------------- 1 | use crate::CONFIG; 2 | 3 | use openssl::ssl::{SslConnector, SslMethod}; 4 | use postgres_openssl::MakeTlsConnector; 5 | use tokio_postgres::{Client, NoTls}; 6 | 7 | /// Connects to the Postgres server (using conn_string for server info) 8 | pub async fn db_client_start() -> Client { 9 | let conn_string = 10 | format!( 11 | "host={} user={} dbname={} replication=database password={} sslmode={} connect_timeout=10", 12 | CONFIG.database_host, 13 | CONFIG.database_user, 14 | CONFIG.database_dbname, 15 | CONFIG.database_password, 16 | if CONFIG.database_tls { "require" } else { "disable" } 17 | ); 18 | 19 | // TODO - Avoid all the duplicated code - Maybe using a macro ? 20 | // Someone can help if he want, as connector is a different type in both case 21 | // I don't know how to use common code for them. Unsafe here would be ok. 22 | let client = match CONFIG.database_tls { 23 | true => { 24 | let connector = 25 | MakeTlsConnector::new(SslConnector::builder(SslMethod::tls()).unwrap().build()); 26 | let (rc, rco) = match tokio_postgres::connect(&conn_string, connector).await { 27 | Ok((rc, rco)) => (rc, rco), 28 | Err(e) => { 29 | error!("Postgres: connection failed: {}, {:?}", e, e.as_db_error()); 30 | std::process::exit(1); 31 | } 32 | }; 33 | 34 | tokio::spawn(async move { 35 | if let Err(e) = rco.await { 36 | // Don't exit after this because we handle the "reconnection" 37 | error!("Postgres: connection broken due to: {}", e); 38 | } 39 | }); 40 | 41 | rc 42 | } 43 | false => { 44 | let connector = NoTls; 45 | let (rc, rco) = match tokio_postgres::connect(&conn_string, connector).await { 46 | Ok((rc, rco)) => (rc, rco), 47 | Err(e) => { 48 | error!("Postgres: connection failed: {}", e); 49 | std::process::exit(1); 50 | } 51 | }; 52 | 53 | tokio::spawn(async move { 54 | if let Err(e) = rco.await { 55 | // Don't exit after this because we handle the "reconnection" 56 | error!("Postgres: connection broken due to: {}", e); 57 | } 58 | }); 59 | 60 | rc 61 | } 62 | }; 63 | info!("Postgres: connection established"); 64 | 65 | client 66 | } 67 | -------------------------------------------------------------------------------- /src/cdc/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::TABLES; 2 | #[cfg(feature = "timescale")] 3 | use crate::TABLES_LOOKUP; 4 | 5 | use async_trait::async_trait; 6 | use tokio_postgres::{Client, SimpleQueryMessage}; 7 | 8 | pub mod connection; 9 | pub mod replication; 10 | 11 | #[cfg(feature = "timescale")] 12 | pub fn extract_hyper_idx(table_name: &str) -> Result { 13 | let mut parts = table_name.splitn(4, '_'); 14 | match parts.nth(2) { 15 | Some(val) => Ok(val.parse::().unwrap()), 16 | None => Err(()), 17 | } 18 | } 19 | 20 | #[async_trait] 21 | pub trait ExtConfig { 22 | async fn detect_tables(&self) {} 23 | #[cfg(feature = "timescale")] 24 | async fn detect_lookup(&self) {} 25 | } 26 | 27 | #[async_trait] 28 | impl ExtConfig for Client { 29 | /// Fill the global TABLES Vec with the tables available inside the database 30 | async fn detect_tables(&self) { 31 | let query = "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE' AND table_name!='__diesel_schema_migrations';"; 32 | TABLES.write().unwrap().clear(); 33 | 34 | match self.simple_query(query).await { 35 | Ok(res) => res.into_iter().for_each(|msg| { 36 | // And push them to the TABLES Vec 37 | if let SimpleQueryMessage::Row(row) = msg { 38 | if let Some(val) = row.get(0) { 39 | TABLES.write().unwrap().push(val.to_owned()) 40 | } 41 | } 42 | }), 43 | Err(err) => { 44 | error!("Cannot check the tables, continuing without them: {}", err); 45 | } 46 | } 47 | } 48 | 49 | #[cfg(feature = "timescale")] 50 | async fn detect_lookup(&self) { 51 | let query = 52 | "select table_name,associated_table_prefix from _timescaledb_catalog.hypertable;"; 53 | TABLES_LOOKUP.write().unwrap().clear(); 54 | 55 | match self.simple_query(query).await { 56 | Ok(res) => res.into_iter().for_each(|msg| { 57 | if let SimpleQueryMessage::Row(row) = msg { 58 | if let (Some(table), Some(prefix)) = (row.get(0), row.get(1)) { 59 | if let Ok(idx) = extract_hyper_idx(prefix) { 60 | TABLES_LOOKUP.write().unwrap().insert(idx, table.to_owned()); 61 | } 62 | } 63 | } 64 | }), 65 | Err(err) => { 66 | error!( 67 | "Cannot check the lookup tables, continuing without them: {}", 68 | err 69 | ); 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/cdc/replication.rs: -------------------------------------------------------------------------------- 1 | use byteorder::{BigEndian, ReadBytesExt}; 2 | use bytes::{BufMut, Bytes, BytesMut}; 3 | use futures::{SinkExt, StreamExt}; 4 | use once_cell::sync::Lazy; 5 | use std::{ 6 | io::{Cursor, Read}, 7 | pin::Pin, 8 | time::{Duration, SystemTime, UNIX_EPOCH}, 9 | }; 10 | use tokio::sync::mpsc::Sender; 11 | use tokio_postgres::{Client, CopyBothDuplex, SimpleQueryMessage, SimpleQueryRow}; 12 | 13 | const TIME_SEC_CONVERSION: u64 = 946_684_800; 14 | const XLOG_DATA_TAG: u8 = b'w'; 15 | const PRIMARY_KEEPALIVE_TAG: u8 = b'k'; 16 | 17 | static EPOCH: Lazy = 18 | Lazy::new(|| UNIX_EPOCH + Duration::from_secs(TIME_SEC_CONVERSION)); 19 | 20 | #[inline] 21 | pub fn current_time() -> u64 { 22 | EPOCH.elapsed().unwrap().as_micros() as u64 23 | } 24 | 25 | /// Send a CREATE_REPLICATION_SLOT ... TEMPORARY LOGICAL to the server. 26 | /// The response to the CREATE_REPLICATION is not documented but based 27 | /// on the code, it's an HashMap containing the following: 28 | /// 29 | /// 1. "slot_name": name of the slot that was created, as requested 30 | /// 2. "consistent_point": LSN at which we became consistent 31 | /// 3. "snapshot_name": exported snapshot's name 32 | /// 4. "output_plugin": name of the output plugin, as requested 33 | pub async fn replication_slot_create(client: &Client, slot_name: &str) -> String { 34 | let slot_query = &format!( 35 | "CREATE_REPLICATION_SLOT {} TEMPORARY LOGICAL wal2json NOEXPORT_SNAPSHOT", 36 | slot_name 37 | ); 38 | 39 | let resp: Vec = match client.simple_query(slot_query).await { 40 | Ok(result) => result 41 | .into_iter() 42 | .filter_map(|data| match data { 43 | SimpleQueryMessage::Row(row) => Some(row), 44 | _ => None, 45 | }) 46 | .collect(), 47 | Err(e) => { 48 | error!( 49 | "Replication: '{}' cannot get the consistent_point: {}", 50 | slot_name, e 51 | ); 52 | std::process::exit(1); 53 | } 54 | }; 55 | 56 | let lsn = resp[0].get("consistent_point").unwrap().to_owned(); 57 | 58 | trace!( 59 | "Replication: slot {} created and got lsn {}", 60 | slot_name, 61 | lsn 62 | ); 63 | 64 | lsn 65 | } 66 | 67 | /// Starts streaming logical changes from replication slot pgcdc_repl, 68 | /// starting from position start_lsn. 69 | pub async fn replication_stream_start( 70 | client: &Client, 71 | slot_name: &str, 72 | start_lsn: &str, 73 | ) -> CopyBothDuplex { 74 | let repl_query = format!("START_REPLICATION SLOT {} LOGICAL {}", slot_name, start_lsn); 75 | let copy_both_result = client.copy_both_simple::(&repl_query).await; 76 | let duplex_stream = match copy_both_result { 77 | Ok(result) => result, 78 | Err(e) => { 79 | error!("Replication: cannot get the a CopyBothDuplex: {}", e); 80 | std::process::exit(1); 81 | } 82 | }; 83 | 84 | trace!( 85 | "Replication: started successfully using slot {} from lsn {}", 86 | slot_name, 87 | start_lsn 88 | ); 89 | 90 | duplex_stream 91 | } 92 | 93 | /// Tries to read and process one message from a replication stream, using async I/O. 94 | pub async fn replication_stream_poll(duplex_stream: CopyBothDuplex, tx: Sender) { 95 | let mut boxed = Box::pin(duplex_stream); 96 | // PostgreSQL will default timeout at 1min so 10s is pretty much "ok". 97 | // Even in case where there's a lot of messages to handle, the tokio::select should 98 | // take the interval.tick() at least once :) 99 | let mut interval = tokio::time::interval(Duration::from_secs(10)); 100 | let mut sync_lsn: u64 = 0; 101 | 102 | loop { 103 | tokio::select! { 104 | _ = interval.tick() => { 105 | trace!("Replication: sending the keepalive to check the state of the connection"); 106 | match send_checkpoint(&mut boxed, sync_lsn).await { 107 | Ok(_) => {}, 108 | Err(e) => { 109 | error!("Replication: cannot contact the database: {}", e); 110 | return; 111 | } 112 | } 113 | }, 114 | Some(data) = boxed.next() => { 115 | match data { 116 | Ok(bytes) => { 117 | let mut buf = Cursor::new(bytes); 118 | let tag = match buf.read_u8() { 119 | Ok(tag) => tag, 120 | Err(e) => { 121 | error!("Replication: cannot read_u8 for tag: {}", e); 122 | continue; 123 | } 124 | }; 125 | 126 | match tag { 127 | XLOG_DATA_TAG => { 128 | parse_xlogdata_message(&mut buf, &mut sync_lsn, &tx).await; 129 | } 130 | // The keepalive here is not mandatory as we already send a keepalive every 10s 131 | // but for the sake of stableness, I keep it here. 132 | PRIMARY_KEEPALIVE_TAG => { 133 | // NOTE: Disabled because when the database is restarting -> will spam with reply 134 | // because it seems that PostgreSQL will send the request over and over again. 135 | // match parse_keepalive_message(&mut boxed, &mut buf, &mut sync_lsn).await { 136 | // Ok(_) => {}, 137 | // Err(e) => { 138 | // error!("Replication: parse_keepalive_message failed: {}", e); 139 | // return; 140 | // } 141 | // } 142 | } 143 | _ => { 144 | error!("Replication: Unknown streaming message type: `{}`", tag); 145 | continue; 146 | } 147 | } 148 | } 149 | Err(e) => { 150 | if e.is_closed() { 151 | error!("Replication: the connection has been closed"); 152 | return; 153 | } 154 | error!("Replication: unknown error: {}", e); 155 | } 156 | } 157 | }, 158 | } 159 | } 160 | } 161 | 162 | /// Parses a XLogData message received from the server. It is packed binary with the 163 | /// following structure: 164 | /// - u64: The starting point of the WAL data in this message. 165 | /// - u64: The current end of WAL on the server. 166 | /// - u64: The server's system clock at the time of transmission, as microseconds 167 | /// since midnight on 2000-01-01. 168 | /// - Byte(n): The output from the logical replication output plugin. 169 | async fn parse_xlogdata_message(buf: &mut Cursor, sync_lsn: &mut u64, tx: &Sender) { 170 | let wal_pos = match buf.read_u64::() { 171 | Ok(wal) => wal, 172 | Err(e) => { 173 | error!("XLogData: cannot read_u64 wal_pos: {}", e); 174 | return; 175 | } 176 | }; 177 | let _wal_end = buf.read_u64::(); 178 | let _ts = buf.read_u64::(); 179 | 180 | // trace!("XLogData: wal_pos {}/{:X}", wal_pos >> 32, wal_pos); 181 | 182 | let mut data: String = String::with_capacity(32); 183 | match buf.read_to_string(&mut data) { 184 | Ok(size) => { 185 | if size == 0 { 186 | return; 187 | } 188 | } 189 | Err(e) => { 190 | error!("XLogData: cannot read_to_string: {}", e); 191 | return; 192 | } 193 | }; 194 | // Broadcast data to the transmitter 195 | // send can fail if the other half of the channel is closed, either due to close 196 | // or because the Receiver has been dropped. In addition send will also block until 197 | // there is a room for the message into the queue. 198 | if let Err(e) = tx.send(data).await { 199 | error!("XLogData: can't send to the channel due to: {}", e); 200 | std::process::exit(1); 201 | } 202 | 203 | *sync_lsn = wal_pos; 204 | } 205 | 206 | /// Parses a "Primary keepalive message" received from the server. It is packed binary 207 | /// with the following structure: 208 | /// 209 | /// - u64: The current end of WAL on the server. 210 | /// - u64: The server's system clock at the time of transmission, as microseconds 211 | /// since midnight on 2000-01-01. 212 | /// - u8: 1 means that the client should reply to this message as soon as possible, 213 | /// to avoid a timeout disconnect. 0 otherwise. 214 | async fn _parse_keepalive_message( 215 | conn: &mut Pin>>, 216 | buf: &mut Cursor, 217 | sync_lsn: &mut u64, 218 | ) -> Result<(), tokio_postgres::Error> { 219 | let wal_pos = buf.read_u64::().unwrap(); 220 | let _ = buf.read_i64::(); // timestamp 221 | let reply_requested = match buf.read_u8() { 222 | Ok(reply) => reply == 1, 223 | Err(e) => { 224 | error!("Keepalive: cannot read_u8 reply_requested: {}", e); 225 | // This is not a fatal error that need to restart everything 226 | return Ok(()); 227 | } 228 | }; 229 | 230 | // Not 100% sure whether it's semantically correct to update our LSN position here -- 231 | // the keepalive message indicates the latest position on the server, which might not 232 | // necessarily correspond to the latest position on the client. But this is what 233 | // pg_recvlogical does, so it's probably ok. 234 | // UPDATE: disabling it as I truly think this is not correct. 235 | // *sync_lsn = std::cmp::max(wal_pos, *sync_lsn); 236 | 237 | if reply_requested { 238 | trace!( 239 | "Keepalive: wal_pos {}/{:X}, reply_requested {}", 240 | wal_pos >> 32, 241 | wal_pos, 242 | reply_requested 243 | ); 244 | 245 | // A failure here is not that big of a deal as 246 | // if we can't send the checkpoint, PostgreSQL 247 | // will cut the connection anyway and we'll just 248 | // restart it. 249 | return send_checkpoint(conn, *sync_lsn).await; 250 | } 251 | 252 | Ok(()) 253 | } 254 | 255 | /// Send a "Standby status update" message to server, indicating the LSN up to which we 256 | /// have received logs. This message is packed binary with the following structure: 257 | /// 258 | /// - u8('r'): Identifies the message as a receiver status update. 259 | /// - u64: The location of the last WAL byte + 1 received by the client. 260 | /// - u64: The location of the last WAL byte + 1 stored durably by the client. 261 | /// - u64: The location of the last WAL byte + 1 applied to the client DB. 262 | /// - u64: The client's system clock, as microseconds since midnight on 2000-01-01. 263 | /// - u8: If 1, the client requests the server to reply to this message immediately. 264 | async fn send_checkpoint( 265 | conn: &mut Pin>>, 266 | lsn: u64, 267 | ) -> Result<(), tokio_postgres::Error> { 268 | let mut ka_buf = BytesMut::with_capacity(34); 269 | 270 | ka_buf.put_u8(b'r'); 271 | ka_buf.put_u64(lsn); 272 | ka_buf.put_u64(lsn); 273 | ka_buf.put_u64(0); // Only used by physical replication 274 | ka_buf.put_u64(current_time()); 275 | ka_buf.put_u8(0); 276 | 277 | let res = (*conn).send(ka_buf.freeze()).await; 278 | 279 | trace!("Checkpoint: lsn: {}/{:X}", lsn >> 32, lsn); 280 | 281 | res 282 | } 283 | -------------------------------------------------------------------------------- /src/forwarder/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::api::ws_utils::{self, ServerState, DELETE, INSERT, UPDATE}; 2 | #[cfg(feature = "timescale")] 3 | use crate::{cdc::extract_hyper_idx, TABLES_LOOKUP}; 4 | 5 | use axum::extract::ws::Message; 6 | use serde_json::Value; 7 | use std::{collections::HashSet, sync::Arc}; 8 | use tokio::sync::mpsc::Receiver; 9 | 10 | /// Get the table name from an &str, returning a String 11 | /// This is used due to TimescaleDB renaming the hypertable using a 12 | /// pattern '_hyper_' with some number and all. If we can't convert the pattern 13 | /// back to it's original table name, return the pattern name. 14 | #[cfg(feature = "timescale")] 15 | fn get_table_name(table_name: &str) -> String { 16 | if table_name.starts_with("_hyper_") { 17 | let idx = match extract_hyper_idx(table_name) { 18 | Ok(idx) => idx, 19 | Err(_) => { 20 | error!( 21 | "Match table: table {} cannot be deconstructed into an idx", 22 | table_name 23 | ); 24 | return table_name.to_owned(); 25 | } 26 | }; 27 | // Get the table name from the index and return an owned String 28 | // or continue the loop and skip this value if not found. 29 | match TABLES_LOOKUP.read().unwrap().get(&idx) { 30 | Some(val) => return val.to_owned(), 31 | None => { 32 | error!( 33 | "Match table: table not found inside using index: {}:{}", 34 | idx, table_name, 35 | ); 36 | return table_name.to_owned(); 37 | } 38 | } 39 | } 40 | table_name.to_owned() 41 | } 42 | 43 | /// Send a message to a specific group of sessions (insert, update or delete) 44 | fn send_message( 45 | message: &serde_json::Value, 46 | sessions: Option<&HashSet>, 47 | server_state: &Arc, 48 | ) { 49 | // If no session were defined, skip 50 | if sessions.is_none() { 51 | return; 52 | } 53 | // For every sessions'id in the tables HashMap 54 | for id in sessions.unwrap() { 55 | // Get the client from the clients list inside the server_state 56 | if let Some(client) = server_state.clients.read().unwrap().get(id) { 57 | // Check if the client asked for a particular filter 58 | let to_send = match &client.watch_for.specific { 59 | Some(specific) => specific.match_filter(message), 60 | None => true, 61 | }; 62 | 63 | if to_send { 64 | // Send the message to the client 65 | if let Err(_disconnected) = client.gate.send(Ok(Message::Text(message.to_string()))) 66 | { 67 | error!("Send_message: client disconnected, should be removed soon"); 68 | } 69 | } 70 | } 71 | } 72 | } 73 | 74 | /// Start a new task which loop over the Receiver's value it may get and forward them to websockets. 75 | pub async fn start_forwarder(mut rx: Receiver, server_state: Arc) { 76 | trace!("Forwarder: Started and waiting for a message"); 77 | 78 | loop { 79 | match rx.recv().await { 80 | Some(mut value) => { 81 | // Convert the data to a Value enum of serde_json 82 | // Using simd optimization through simd_json crate. 83 | let data: Value = unsafe { simd_json::from_str(&mut value) }.unwrap(); 84 | // Extract what we really want and assert that it exists 85 | let changes = match data["change"].as_array() { 86 | Some(val) => val, 87 | None => { 88 | error!("Forwarder: The message is invalid: {}", data); 89 | continue; 90 | } 91 | }; 92 | // For each change inside of changes, we do the following treatment 93 | for change in changes { 94 | // Check the table (to str (using a match for safety)) 95 | if let (Some(table_name), Some(change_type)) = 96 | (change["table"].as_str(), change["kind"].as_str()) 97 | { 98 | // Get the table name from the _hyper_x_x_chunk 99 | // See comment in the main.rs for more information. 100 | #[cfg(feature = "timescale")] 101 | let table_name = get_table_name(table_name); 102 | #[cfg(not(feature = "timescale"))] 103 | let table_name = table_name.to_owned(); 104 | // Construct the change_flag 105 | let mut change_flag = 0u8; 106 | // At this stage, the change_flag can be only be one of INSERT, UPDATE, DELETE 107 | // but not multiple of them. 108 | ws_utils::apply_flag(&mut change_flag, change_type); 109 | // Only send the message to those interested in the change_type 110 | if has_bit!(change_flag, INSERT) { 111 | // First get the lock over the RwLock guard 112 | let lock = server_state.inserts.read().unwrap(); 113 | // Then get the sessions out of it 114 | let sessions = lock.get(&table_name); 115 | // And finally send the message to each client inside that sessions AHashSet 116 | send_message(change, sessions, &server_state); 117 | } else if has_bit!(change_flag, UPDATE) { 118 | let lock = server_state.updates.read().unwrap(); 119 | let sessions = lock.get(&table_name); 120 | send_message(change, sessions, &server_state); 121 | } else if has_bit!(change_flag, DELETE) { 122 | let lock = server_state.deletes.read().unwrap(); 123 | let sessions = lock.get(&table_name); 124 | send_message(change, sessions, &server_state); 125 | } else { 126 | error!("Forwarder: change_flag {:?} not handled.", change_flag); 127 | continue; 128 | }; 129 | } else { 130 | error!( 131 | "Forwarder: table ({:?}) or change_type ({:?}) not present.", 132 | change["table"], change["kind"] 133 | ); 134 | } 135 | } 136 | } 137 | None => { 138 | trace!("Channel returned None"); 139 | return; 140 | } 141 | } 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/inner.rs: -------------------------------------------------------------------------------- 1 | use crate::forwarder::start_forwarder; 2 | #[cfg(feature = "timescale")] 3 | use crate::TABLES_LOOKUP; 4 | use crate::{ 5 | api::ws_utils::ServerState, 6 | cdc::{ 7 | connection::db_client_start, 8 | replication::{replication_slot_create, replication_stream_poll, replication_stream_start}, 9 | ExtConfig, 10 | }, 11 | }; 12 | use crate::{SUPERVISOR, TABLES}; 13 | 14 | use bastion::prelude::BastionContext; 15 | use bastion::spawn; 16 | use std::sync::Arc; 17 | use tokio::select; 18 | use tokio::sync::mpsc; 19 | 20 | pub fn start_inner(server_state: Arc) { 21 | // Start the children in Bastion (allow for restart if fails) 22 | SUPERVISOR 23 | .children(|child| { 24 | child.with_exec(move |_: BastionContext| { 25 | trace!("Starting the replication forwarder & listener"); 26 | let server_state = server_state.clone(); 27 | 28 | async move { 29 | // A multi-producer, single-consumer channel queue. Using 128 buffers length. 30 | let (tx, rx) = mpsc::channel(128); 31 | 32 | // Start listening to the Sender & forward message when receiving one 33 | let handle = spawn! { 34 | start_forwarder(rx, server_state).await; 35 | }; 36 | 37 | // Form replication connection & keep the connection open 38 | let client = db_client_start().await; 39 | 40 | // Detect tables that we'll use to authorize or lookup with timescale 41 | client.detect_tables().await; 42 | trace!("Main: Allowed tables are: {:?}", &TABLES.read().unwrap()); 43 | #[cfg(feature = "timescale")] 44 | { 45 | client.detect_lookup().await; 46 | trace!( 47 | "Main: Tables lookup are: {:?}", 48 | &TABLES_LOOKUP.read().unwrap() 49 | ); 50 | } 51 | 52 | let slot_name = uuid_readable_rs::short().replace(' ', "_").to_lowercase(); 53 | let lsn = replication_slot_create(&client, &slot_name).await; 54 | let duplex_stream = replication_stream_start(&client, &slot_name, &lsn).await; 55 | 56 | // call to panic allow us to exit this children and restart a new one 57 | // in case any of the two (replication_stream_poll or handle) exit. 58 | select! { 59 | _ = replication_stream_poll(duplex_stream, tx.clone()) => { 60 | panic!("replication_stream_poll exited, panic to restart") 61 | } 62 | _ = handle => { 63 | panic!("start_forwarder exited, panic to restart") 64 | } 65 | } 66 | } 67 | }) 68 | }) 69 | .expect("Cannot create the Children for Bastion"); 70 | } 71 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | //! Quick note about the database table name due to Tailscale: 2 | //! Static array to hold the tables in the order of creation in the database. 3 | //! As we use TimescaleDB, each table get partitioned using a pattern like "_hyper_x_y_chunk", 4 | //! which don't give us the opportunity to detect which table is being updated/inserted. 5 | //! As the client will connect to the WS using the base table name, this array is used for lookup. 6 | //! The pattern always follow the same naming convention: "_hyper_(table_creation_order_from_1)_(partition_number)_chunk". 7 | //! So we use this array to derive the name of the table from the pattern naming chunk. 8 | 9 | #[macro_use] 10 | extern crate log; 11 | 12 | macro_rules! has_bit { 13 | ($a:expr,$b:expr) => { 14 | ($a & $b) != 0 15 | }; 16 | } 17 | 18 | use crate::api::server; 19 | use crate::utils::config::Config; 20 | 21 | use api::ws_utils::ServerState; 22 | use bastion::supervisor::{ActorRestartStrategy, RestartStrategy, SupervisorRef}; 23 | use bastion::Bastion; 24 | use clap::Parser; 25 | use clap_verbosity_flag::InfoLevel; 26 | use inner::start_inner; 27 | use once_cell::sync::Lazy; 28 | use sproot::prog; 29 | #[cfg(feature = "timescale")] 30 | use std::collections::HashMap; 31 | use std::sync::atomic::AtomicUsize; 32 | use std::sync::{Arc, RwLock}; 33 | use std::time::Duration; 34 | 35 | mod api; 36 | mod cdc; 37 | mod forwarder; 38 | mod inner; 39 | mod utils; 40 | 41 | #[derive(Parser, Debug)] 42 | #[clap(author, version, about)] 43 | struct Args { 44 | #[clap(short = 'c', long = "config")] 45 | config_path: Option, 46 | 47 | #[clap(flatten)] 48 | verbose: clap_verbosity_flag::Verbosity, 49 | } 50 | 51 | /// Our global unique client id counter. 52 | static ID_COUNTER: AtomicUsize = AtomicUsize::new(0); 53 | 54 | #[cfg(feature = "timescale")] 55 | // Used with TimescaleDB to lookup the table name (disks may be _hyper_1 for example) 56 | static TABLES_LOOKUP: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); 57 | 58 | // Lazy static of the Config which is loaded from the config file 59 | static CONFIG: Lazy = Lazy::new(|| match Config::new() { 60 | Ok(config) => config, 61 | Err(e) => { 62 | error!("Cannot build the Config: {}", e); 63 | std::process::exit(1); 64 | } 65 | }); 66 | 67 | // Which table are allowed (hard defined at startup for now) 68 | // Allow us to avoid accepting websocket which will never be triggered 69 | static TABLES: Lazy>> = Lazy::new(|| RwLock::new(Vec::new())); 70 | 71 | // Bastion supervisor used to define a custom restart policy for the children 72 | static SUPERVISOR: Lazy = Lazy::new(|| { 73 | match Bastion::supervisor(|sp| { 74 | sp.with_restart_strategy(RestartStrategy::default().with_actor_restart_strategy( 75 | ActorRestartStrategy::LinearBackOff { 76 | timeout: Duration::from_secs(3), 77 | }, 78 | )) 79 | }) { 80 | Ok(sp) => sp, 81 | Err(err) => { 82 | error!("Cannot create the Bastion supervisor: {:?}", err); 83 | std::process::exit(1); 84 | } 85 | } 86 | }); 87 | 88 | #[tokio::main] 89 | async fn main() { 90 | let args = Args::parse(); 91 | 92 | // Define log level 93 | if std::env::var("RUST_LOG").is_err() { 94 | std::env::set_var( 95 | "RUST_LOG", 96 | format!( 97 | "{}={level},tower_http={level}", 98 | &prog().map_or_else(|| "speculare_pgcdc".to_owned(), |f| f.replace('-', "_")), 99 | level = args.verbose.log_level_filter() 100 | ), 101 | ) 102 | } 103 | 104 | // Init logger/tracing 105 | tracing_subscriber::fmt::init(); 106 | 107 | // Construct our default server state 108 | let server_state = Arc::new(ServerState::default()); 109 | 110 | // Init and Start Bastion supervisor 111 | Bastion::init(); 112 | Bastion::start(); 113 | 114 | // Clone server_state for run_server (below) as we use server_state 115 | // in our SUPERVISOR.children. 116 | let cserver_state = server_state.clone(); 117 | 118 | // Start the inner work, replication, forwarder, ... 119 | start_inner(server_state); 120 | 121 | // Start the public api server 122 | server::serve(cserver_state).await 123 | } 124 | -------------------------------------------------------------------------------- /src/utils/config.rs: -------------------------------------------------------------------------------- 1 | use crate::Args; 2 | 3 | use clap::Parser; 4 | use config::ConfigError; 5 | use serde::Deserialize; 6 | 7 | #[derive(Debug, Deserialize, Clone)] 8 | 9 | pub struct Config { 10 | // POSTGRESQL DB CONFIGS 11 | pub database_host: String, 12 | pub database_dbname: String, 13 | pub database_user: String, 14 | pub database_password: String, 15 | #[serde(default = "default_dbtls")] 16 | pub database_tls: bool, 17 | 18 | // HTTP API CONFIGS 19 | #[serde(default = "default_binding")] 20 | pub binding: String, 21 | #[serde(default = "default_https")] 22 | pub https: bool, 23 | pub key_priv: Option, 24 | pub key_cert: Option, 25 | 26 | #[cfg(feature = "auth")] 27 | pub cookie_secret: String, 28 | #[cfg(feature = "auth")] 29 | pub admin_secret: String, 30 | 31 | // AUTH POSTGRESQL CONNECTION 32 | #[cfg(feature = "auth")] 33 | pub auth_database_url: String, 34 | #[cfg(feature = "auth")] 35 | #[serde(default = "default_maxconn")] 36 | pub auth_database_max_connection: u32, 37 | } 38 | 39 | impl Config { 40 | pub fn new() -> Result { 41 | let args = Args::parse(); 42 | 43 | let config_builder = config::Config::builder().add_source(config::File::new( 44 | &args 45 | .config_path 46 | .unwrap_or_else(|| "/etc/speculare/pgcdc.config".to_owned()), 47 | config::FileFormat::Toml, 48 | )); 49 | 50 | config_builder.build()?.try_deserialize() 51 | } 52 | } 53 | 54 | fn default_dbtls() -> bool { 55 | false 56 | } 57 | 58 | fn default_https() -> bool { 59 | false 60 | } 61 | 62 | #[cfg(feature = "auth")] 63 | fn default_maxconn() -> u32 { 64 | 10 65 | } 66 | 67 | fn default_binding() -> String { 68 | String::from("0.0.0.0:8080") 69 | } 70 | -------------------------------------------------------------------------------- /src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod config; 2 | pub mod specific_filter; 3 | -------------------------------------------------------------------------------- /src/utils/specific_filter.rs: -------------------------------------------------------------------------------- 1 | /// List of supported data type 2 | #[derive(Debug, Clone)] 3 | pub enum DataType { 4 | String(String), 5 | Array(Vec), 6 | } 7 | 8 | /// Contains the specific filter applied to the Ws 9 | #[derive(Debug, Clone)] 10 | pub struct SpecificFilter { 11 | pub column: serde_json::Value, 12 | pub value: DataType, 13 | } 14 | 15 | impl SpecificFilter { 16 | /// Determine if the filter match the message passed as parameter 17 | pub fn match_filter(&self, message: &serde_json::Value) -> bool { 18 | // Determine if the column is present in this change 19 | let columns = match message["columnnames"].as_array() { 20 | Some(val) => val, 21 | None => { 22 | error!("Specific: the message does not contains `columnnames`"); 23 | return false; 24 | } 25 | }; 26 | // Check if the cloumns we asked for exist in this data change 27 | let value_index = columns.iter().position(|c| c == &self.column); 28 | if value_index.is_none() { 29 | return false; 30 | } 31 | // Basically it just match, filter and sort around the criteria of the column value. 32 | if let Some(col_vals) = message["columnvalues"].as_array() { 33 | let targeted_value = &col_vals[value_index.unwrap()]; 34 | // If the value we asked for is a String or a Number 35 | return match &self.value { 36 | DataType::String(val) => { 37 | // Check if is_string, and if so, convert it then check 38 | match targeted_value.as_str() { 39 | Some(t) => t == val, 40 | None => false, 41 | } 42 | } 43 | DataType::Array(val) => match targeted_value.as_str() { 44 | Some(t) => val.iter().any(|x| x == t), 45 | None => false, 46 | }, 47 | }; 48 | } 49 | false 50 | } 51 | } 52 | --------------------------------------------------------------------------------