├── .gitignore ├── .rustfmt.toml ├── Cargo.toml ├── LICENSE ├── README.md ├── adapters └── synapse │ ├── Cargo.toml │ └── src │ ├── 65.md │ ├── extract.rs │ ├── insert.rs │ ├── lib.rs │ └── main.rs ├── cli ├── Cargo.toml └── src │ └── main.rs ├── graph.md └── lib ├── Cargo.toml └── src ├── lib.rs ├── traits.rs └── types.rs /.gitignore: -------------------------------------------------------------------------------- 1 | # File created using '.gitignore Generator' for Visual Studio Code: https://bit.ly/vscode-gig 2 | 3 | # Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,linux,intellij+all,rust 4 | # Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,linux,intellij+all,rust 5 | 6 | ### Intellij+all ### 7 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 8 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 9 | 10 | # User-specific stuff 11 | .idea/**/workspace.xml 12 | .idea/**/tasks.xml 13 | .idea/**/usage.statistics.xml 14 | .idea/**/dictionaries 15 | .idea/**/shelf 16 | 17 | # Generated files 18 | .idea/**/contentModel.xml 19 | 20 | # Sensitive or high-churn files 21 | .idea/**/dataSources/ 22 | .idea/**/dataSources.ids 23 | .idea/**/dataSources.local.xml 24 | .idea/**/sqlDataSources.xml 25 | .idea/**/dynamic.xml 26 | .idea/**/uiDesigner.xml 27 | .idea/**/dbnavigator.xml 28 | 29 | # Gradle 30 | .idea/**/gradle.xml 31 | .idea/**/libraries 32 | 33 | # Gradle and Maven with auto-import 34 | # When using Gradle or Maven with auto-import, you should exclude module files, 35 | # since they will be recreated, and may cause churn. Uncomment if using 36 | # auto-import. 37 | # .idea/artifacts 38 | # .idea/compiler.xml 39 | # .idea/jarRepositories.xml 40 | # .idea/modules.xml 41 | # .idea/*.iml 42 | # .idea/modules 43 | # *.iml 44 | # *.ipr 45 | 46 | # CMake 47 | cmake-build-*/ 48 | 49 | # Mongo Explorer plugin 50 | .idea/**/mongoSettings.xml 51 | 52 | # File-based project format 53 | *.iws 54 | 55 | # IntelliJ 56 | out/ 57 | 58 | # mpeltonen/sbt-idea plugin 59 | .idea_modules/ 60 | 61 | # JIRA plugin 62 | atlassian-ide-plugin.xml 63 | 64 | # Cursive Clojure plugin 65 | .idea/replstate.xml 66 | 67 | # Crashlytics plugin (for Android Studio and IntelliJ) 68 | com_crashlytics_export_strings.xml 69 | crashlytics.properties 70 | crashlytics-build.properties 71 | fabric.properties 72 | 73 | # Editor-based Rest Client 74 | .idea/httpRequests 75 | 76 | # Android studio 3.1+ serialized cache file 77 | .idea/caches/build_file_checksums.ser 78 | 79 | ### Intellij+all Patch ### 80 | # Ignores the whole .idea folder and all .iml files 81 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 82 | 83 | .idea/ 84 | 85 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 86 | 87 | *.iml 88 | modules.xml 89 | .idea/misc.xml 90 | *.ipr 91 | 92 | # Sonarlint plugin 93 | .idea/sonarlint 94 | 95 | ### Linux ### 96 | *~ 97 | 98 | # temporary files which can be created if a process still has a handle open of a deleted file 99 | .fuse_hidden* 100 | 101 | # KDE directory preferences 102 | .directory 103 | 104 | # Linux trash folder which might appear on any partition or disk 105 | .Trash-* 106 | 107 | # .nfs files are created when an open file is removed but is still being accessed 108 | .nfs* 109 | 110 | ### Rust ### 111 | # Generated by Cargo 112 | # will have compiled files and executables 113 | /target/ 114 | 115 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 116 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 117 | Cargo.lock 118 | 119 | ### VisualStudioCode ### 120 | .vscode/* 121 | !.vscode/settings.json 122 | !.vscode/tasks.json 123 | !.vscode/launch.json 124 | !.vscode/extensions.json 125 | *.code-workspace 126 | 127 | ### VisualStudioCode Patch ### 128 | # Ignore all local history of files 129 | .history 130 | .ionide 131 | 132 | # End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,linux,intellij+all,rust 133 | 134 | # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) 135 | 136 | 137 | 138 | # Added by cargo 139 | 140 | /target 141 | -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | comment_width = 100 2 | imports_granularity = "Crate" 3 | newline_style = "Unix" 4 | use_small_heuristics = "Max" 5 | wrap_comments = true 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["cli", "lib", "adapters/*"] 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | EUROPEAN UNION PUBLIC LICENCE v. 1.2 2 | EUPL © the European Union 2007, 2016 3 | 4 | This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined 5 | below) which is provided under the terms of this Licence. Any use of the Work, 6 | other than as authorised under this Licence is prohibited (to the extent such 7 | use is covered by a right of the copyright holder of the Work). 8 | 9 | The Work is provided under the terms of this Licence when the Licensor (as 10 | defined below) has placed the following notice immediately following the 11 | copyright notice for the Work: 12 | 13 | Licensed under the EUPL 14 | 15 | or has expressed by any other means his willingness to license under the EUPL. 16 | 17 | 1. Definitions 18 | 19 | In this Licence, the following terms have the following meaning: 20 | 21 | - ‘The Licence’: this Licence. 22 | 23 | - ‘The Original Work’: the work or software distributed or communicated by the 24 | Licensor under this Licence, available as Source Code and also as Executable 25 | Code as the case may be. 26 | 27 | - ‘Derivative Works’: the works or software that could be created by the 28 | Licensee, based upon the Original Work or modifications thereof. This Licence 29 | does not define the extent of modification or dependence on the Original Work 30 | required in order to classify a work as a Derivative Work; this extent is 31 | determined by copyright law applicable in the country mentioned in Article 15. 32 | 33 | - ‘The Work’: the Original Work or its Derivative Works. 34 | 35 | - ‘The Source Code’: the human-readable form of the Work which is the most 36 | convenient for people to study and modify. 37 | 38 | - ‘The Executable Code’: any code which has generally been compiled and which is 39 | meant to be interpreted by a computer as a program. 40 | 41 | - ‘The Licensor’: the natural or legal person that distributes or communicates 42 | the Work under the Licence. 43 | 44 | - ‘Contributor(s)’: any natural or legal person who modifies the Work under the 45 | Licence, or otherwise contributes to the creation of a Derivative Work. 46 | 47 | - ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of 48 | the Work under the terms of the Licence. 49 | 50 | - ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, 51 | renting, distributing, communicating, transmitting, or otherwise making 52 | available, online or offline, copies of the Work or providing access to its 53 | essential functionalities at the disposal of any other natural or legal 54 | person. 55 | 56 | 2. Scope of the rights granted by the Licence 57 | 58 | The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, 59 | sublicensable licence to do the following, for the duration of copyright vested 60 | in the Original Work: 61 | 62 | - use the Work in any circumstance and for all usage, 63 | - reproduce the Work, 64 | - modify the Work, and make Derivative Works based upon the Work, 65 | - communicate to the public, including the right to make available or display 66 | the Work or copies thereof to the public and perform publicly, as the case may 67 | be, the Work, 68 | - distribute the Work or copies thereof, 69 | - lend and rent the Work or copies thereof, 70 | - sublicense rights in the Work or copies thereof. 71 | 72 | Those rights can be exercised on any media, supports and formats, whether now 73 | known or later invented, as far as the applicable law permits so. 74 | 75 | In the countries where moral rights apply, the Licensor waives his right to 76 | exercise his moral right to the extent allowed by law in order to make effective 77 | the licence of the economic rights here above listed. 78 | 79 | The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to 80 | any patents held by the Licensor, to the extent necessary to make use of the 81 | rights granted on the Work under this Licence. 82 | 83 | 3. Communication of the Source Code 84 | 85 | The Licensor may provide the Work either in its Source Code form, or as 86 | Executable Code. If the Work is provided as Executable Code, the Licensor 87 | provides in addition a machine-readable copy of the Source Code of the Work 88 | along with each copy of the Work that the Licensor distributes or indicates, in 89 | a notice following the copyright notice attached to the Work, a repository where 90 | the Source Code is easily and freely accessible for as long as the Licensor 91 | continues to distribute or communicate the Work. 92 | 93 | 4. Limitations on copyright 94 | 95 | Nothing in this Licence is intended to deprive the Licensee of the benefits from 96 | any exception or limitation to the exclusive rights of the rights owners in the 97 | Work, of the exhaustion of those rights or of other applicable limitations 98 | thereto. 99 | 100 | 5. Obligations of the Licensee 101 | 102 | The grant of the rights mentioned above is subject to some restrictions and 103 | obligations imposed on the Licensee. Those obligations are the following: 104 | 105 | Attribution right: The Licensee shall keep intact all copyright, patent or 106 | trademarks notices and all notices that refer to the Licence and to the 107 | disclaimer of warranties. The Licensee must include a copy of such notices and a 108 | copy of the Licence with every copy of the Work he/she distributes or 109 | communicates. The Licensee must cause any Derivative Work to carry prominent 110 | notices stating that the Work has been modified and the date of modification. 111 | 112 | Copyleft clause: If the Licensee distributes or communicates copies of the 113 | Original Works or Derivative Works, this Distribution or Communication will be 114 | done under the terms of this Licence or of a later version of this Licence 115 | unless the Original Work is expressly distributed only under this version of the 116 | Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee 117 | (becoming Licensor) cannot offer or impose any additional terms or conditions on 118 | the Work or Derivative Work that alter or restrict the terms of the Licence. 119 | 120 | Compatibility clause: If the Licensee Distributes or Communicates Derivative 121 | Works or copies thereof based upon both the Work and another work licensed under 122 | a Compatible Licence, this Distribution or Communication can be done under the 123 | terms of this Compatible Licence. For the sake of this clause, ‘Compatible 124 | Licence’ refers to the licences listed in the appendix attached to this Licence. 125 | Should the Licensee's obligations under the Compatible Licence conflict with 126 | his/her obligations under this Licence, the obligations of the Compatible 127 | Licence shall prevail. 128 | 129 | Provision of Source Code: When distributing or communicating copies of the Work, 130 | the Licensee will provide a machine-readable copy of the Source Code or indicate 131 | a repository where this Source will be easily and freely available for as long 132 | as the Licensee continues to distribute or communicate the Work. 133 | 134 | Legal Protection: This Licence does not grant permission to use the trade names, 135 | trademarks, service marks, or names of the Licensor, except as required for 136 | reasonable and customary use in describing the origin of the Work and 137 | reproducing the content of the copyright notice. 138 | 139 | 6. Chain of Authorship 140 | 141 | The original Licensor warrants that the copyright in the Original Work granted 142 | hereunder is owned by him/her or licensed to him/her and that he/she has the 143 | power and authority to grant the Licence. 144 | 145 | Each Contributor warrants that the copyright in the modifications he/she brings 146 | to the Work are owned by him/her or licensed to him/her and that he/she has the 147 | power and authority to grant the Licence. 148 | 149 | Each time You accept the Licence, the original Licensor and subsequent 150 | Contributors grant You a licence to their contributions to the Work, under the 151 | terms of this Licence. 152 | 153 | 7. Disclaimer of Warranty 154 | 155 | The Work is a work in progress, which is continuously improved by numerous 156 | Contributors. It is not a finished work and may therefore contain defects or 157 | ‘bugs’ inherent to this type of development. 158 | 159 | For the above reason, the Work is provided under the Licence on an ‘as is’ basis 160 | and without warranties of any kind concerning the Work, including without 161 | limitation merchantability, fitness for a particular purpose, absence of defects 162 | or errors, accuracy, non-infringement of intellectual property rights other than 163 | copyright as stated in Article 6 of this Licence. 164 | 165 | This disclaimer of warranty is an essential part of the Licence and a condition 166 | for the grant of any rights to the Work. 167 | 168 | 8. Disclaimer of Liability 169 | 170 | Except in the cases of wilful misconduct or damages directly caused to natural 171 | persons, the Licensor will in no event be liable for any direct or indirect, 172 | material or moral, damages of any kind, arising out of the Licence or of the use 173 | of the Work, including without limitation, damages for loss of goodwill, work 174 | stoppage, computer failure or malfunction, loss of data or any commercial 175 | damage, even if the Licensor has been advised of the possibility of such damage. 176 | However, the Licensor will be liable under statutory product liability laws as 177 | far such laws apply to the Work. 178 | 179 | 9. Additional agreements 180 | 181 | While distributing the Work, You may choose to conclude an additional agreement, 182 | defining obligations or services consistent with this Licence. However, if 183 | accepting obligations, You may act only on your own behalf and on your sole 184 | responsibility, not on behalf of the original Licensor or any other Contributor, 185 | and only if You agree to indemnify, defend, and hold each Contributor harmless 186 | for any liability incurred by, or claims asserted against such Contributor by 187 | the fact You have accepted any warranty or additional liability. 188 | 189 | 10. Acceptance of the Licence 190 | 191 | The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ 192 | placed under the bottom of a window displaying the text of this Licence or by 193 | affirming consent in any other similar way, in accordance with the rules of 194 | applicable law. Clicking on that icon indicates your clear and irrevocable 195 | acceptance of this Licence and all of its terms and conditions. 196 | 197 | Similarly, you irrevocably accept this Licence and all of its terms and 198 | conditions by exercising any rights granted to You by Article 2 of this Licence, 199 | such as the use of the Work, the creation by You of a Derivative Work or the 200 | Distribution or Communication by You of the Work or copies thereof. 201 | 202 | 11. Information to the public 203 | 204 | In case of any Distribution or Communication of the Work by means of electronic 205 | communication by You (for example, by offering to download the Work from a 206 | remote location) the distribution channel or media (for example, a website) must 207 | at least provide to the public the information requested by the applicable law 208 | regarding the Licensor, the Licence and the way it may be accessible, concluded, 209 | stored and reproduced by the Licensee. 210 | 211 | 12. Termination of the Licence 212 | 213 | The Licence and the rights granted hereunder will terminate automatically upon 214 | any breach by the Licensee of the terms of the Licence. 215 | 216 | Such a termination will not terminate the licences of any person who has 217 | received the Work from the Licensee under the Licence, provided such persons 218 | remain in full compliance with the Licence. 219 | 220 | 13. Miscellaneous 221 | 222 | Without prejudice of Article 9 above, the Licence represents the complete 223 | agreement between the Parties as to the Work. 224 | 225 | If any provision of the Licence is invalid or unenforceable under applicable 226 | law, this will not affect the validity or enforceability of the Licence as a 227 | whole. Such provision will be construed or reformed so as necessary to make it 228 | valid and enforceable. 229 | 230 | The European Commission may publish other linguistic versions or new versions of 231 | this Licence or updated versions of the Appendix, so far this is required and 232 | reasonable, without reducing the scope of the rights granted by the Licence. New 233 | versions of the Licence will be published with a unique version number. 234 | 235 | All linguistic versions of this Licence, approved by the European Commission, 236 | have identical value. Parties can take advantage of the linguistic version of 237 | their choice. 238 | 239 | 14. Jurisdiction 240 | 241 | Without prejudice to specific agreement between parties, 242 | 243 | - any litigation resulting from the interpretation of this License, arising 244 | between the European Union institutions, bodies, offices or agencies, as a 245 | Licensor, and any Licensee, will be subject to the jurisdiction of the Court 246 | of Justice of the European Union, as laid down in article 272 of the Treaty on 247 | the Functioning of the European Union, 248 | 249 | - any litigation arising between other parties and resulting from the 250 | interpretation of this License, will be subject to the exclusive jurisdiction 251 | of the competent court where the Licensor resides or conducts its primary 252 | business. 253 | 254 | 15. Applicable Law 255 | 256 | Without prejudice to specific agreement between parties, 257 | 258 | - this Licence shall be governed by the law of the European Union Member State 259 | where the Licensor has his seat, resides or has his registered office, 260 | 261 | - this licence shall be governed by Belgian law if the Licensor has no seat, 262 | residence or registered office inside a European Union Member State. 263 | 264 | Appendix 265 | 266 | ‘Compatible Licences’ according to Article 5 EUPL are: 267 | 268 | - GNU General Public License (GPL) v. 2, v. 3 269 | - GNU Affero General Public License (AGPL) v. 3 270 | - Open Software License (OSL) v. 2.1, v. 3.0 271 | - Eclipse Public License (EPL) v. 1.0 272 | - CeCILL v. 2.0, v. 2.1 273 | - Mozilla Public Licence (MPL) v. 2 274 | - GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 275 | - Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for 276 | works other than software 277 | - European Union Public Licence (EUPL) v. 1.1, v. 1.2 278 | - Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong 279 | Reciprocity (LiLiQ-R+). 280 | 281 | The European Commission may update this Appendix to later versions of the above 282 | licences without producing a new version of the EUPL, as long as they provide 283 | the rights granted in Article 2 of this Licence and protect the covered Source 284 | Code from exclusive appropriation. 285 | 286 | All other changes or additions to this Appendix require the production of a new 287 | EUPL version. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Variate 2 | 3 | > *verb* (transitive, intransitive) To alter; to vary; to make or become different. 4 | 5 | 6 | > **NOTICE**: This repo is mothballed, and unlikely for anything to happen at this stage, i've lost motivation in working for matrix projects, and so feel free to take the idea and make your own homeserver migrator. 7 | -------------------------------------------------------------------------------- /adapters/synapse/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "variate_synapse" 3 | version = "0.1.0" 4 | edition = "2018" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | postgres = { version = "0.19.2", features = ["with-serde_json-1"] } 10 | serde_json = "1.0.71" 11 | serde = { version = "1.0.130", features = ["derive"] } 12 | variate_lib = { path = "../../lib" } 13 | postgres-types = { version = "0.2.2", features = ["derive"] } 14 | -------------------------------------------------------------------------------- /adapters/synapse/src/65.md: -------------------------------------------------------------------------------- 1 | `access_tokens`: yes 2 | 3 | `account_data`: yes 4 | `account_validity`: maybe? probably synapse-specific 5 | 6 | `application_services_state`: unsure, stream-based, so prob no 7 | `application_services_txns`: unsure, stream-based, so prob no 8 | 9 | `applied_module_schemas`: no 10 | `applied_schema_deltas`: no 11 | 12 | `appservice_room_list`: room list, so maybe 13 | `appservice_stream_position`: no 14 | 15 | `background_updates`: no 16 | 17 | `batch_events`: ??? 18 | 19 | `blocked_rooms`: unsure 20 | 21 | `cache_invalidation_stream_by_instance`: no 22 | 23 | `current_state_delta_stream`: convergence, no 24 | `current_state_events`: convergence, no 25 | 26 | `dehydrated_devices`: yes 27 | 28 | `deleted_pushers`: not needed, no 29 | 30 | `destination_rooms`: convergence, no 31 | `destinations`: convergence, no 32 | 33 | `device_federation_inbox`: not needed 34 | `device_federation_outbox`: yes 35 | `device_inbox`: yes 36 | `device_lists_outbound_last_success`: no 37 | `device_lists_outbound_pokes`: no 38 | `device_lists_remote_cache`: no 39 | `device_lists_remote_extremeties`: no 40 | `device_lists_remote_resync`: no 41 | `device_lists_stream`: no 42 | `devices`: yes 43 | 44 | `e2e_cross_signing_keys`: yes 45 | `e2e_cross_signing_signatures`: yes 46 | `e2e_device_keys_json`: yes 47 | `e2e_fallback_keys_json`: yes 48 | `e2e_one_time_keys_json`: yes 49 | `e2e_room_keys`: yes 50 | `e2e_room_keys_versions`: yes 51 | 52 | `erased_users`: no? 53 | 54 | `event_auth`: derived? 55 | `event_auth_chain_links`: derived? 56 | `event_auth_chain_to_calculate`: derived? 57 | `event_auth_chains`: derived? 58 | `event_backward_extremities`: ??? 59 | `event_edges`: derived? 60 | `event_expiry`: unsure 61 | `event_forward_extremities`: calculated? 62 | `event_json`: yes 63 | `event_labels`: ??? 64 | `event_push_actions`: yes 65 | `event_push_actions_staging`: ??? 66 | `event_push_summary`: maybe 67 | `event_push_summary_stream_ordering`: no 68 | `event_reference_hashes`: unsure 69 | `event_relations`: calculated? 70 | `event_reports`: maybe 71 | `event_search`: calculated, no 72 | `event_to_state_groups`: derived? 73 | `event_txn_id`: no 74 | `events`: yes 75 | 76 | `ex_outlier_stream`: ??? 77 | 78 | `federation_inbound_events_staging`: no 79 | `federation_stream_position`: no 80 | 81 | `group*`: no 82 | 83 | `instance_map`: no 84 | 85 | `local_current_membership`: derived? 86 | `local_group_*`: no 87 | `local_media_repository`: yes 88 | 89 | `monthly_active_users`: maybe 90 | 91 | `open_id_tokens`: unsure 92 | 93 | `presence`: unsure, probably not up-to-date 94 | `presence_stream`: no 95 | 96 | `profiles`: yes 97 | 98 | `public_room_list_stream`: yes 99 | 100 | `push_rules`: yes 101 | `push_rules_enable`: yes 102 | `push_rules_enable`: no 103 | `pusher_throttle`: unsure 104 | `pushers`: yes 105 | 106 | `ratelimit_override`: unsure 107 | 108 | `receipts_graph`: yes (local users only?) 109 | `receipts_linearized`: unsure? derived? 110 | `received_transactions`: no 111 | 112 | `redactions`: calculated? 113 | 114 | `refresh_tokens`: yes 115 | 116 | `registration_tokens`: yes 117 | 118 | `rejections`: calculated? 119 | 120 | `remote_media_cache`: maybe not 121 | `remote_media_cache_thumbnails`: no 122 | `remote_profile_cache`: maybe not 123 | 124 | `room_account_data`: yes 125 | `room_alias_servers`: maybe? 126 | `room_aliases`: yes 127 | `room_depth`: probably not, calculated? 128 | `room_memberships`: calculated 129 | `room_retention`: unsure, derived? 130 | `room_stats_current`: calculated, no 131 | `room_stats_earliest_token`: unsure 132 | `room_stats_historical`: unsure 133 | `room_stats_state`: unsure 134 | `room_tags`: yes 135 | `room_tags_revisions`: no 136 | `rooms`: yes 137 | 138 | `server_keys_json`: maybe? 139 | `server_signature_keys`: maybe? 140 | 141 | `sessions`: ??? 142 | 143 | `state_events`: calculated? 144 | `state_group_edges`: calculated 145 | `state_groups`: calculated 146 | `state_groups_state`: calculated 147 | 148 | `stats_incremental_position`: no 149 | 150 | `stream_ordering_to_exterm`: unsure 151 | `stream_positions`: no 152 | 153 | `threepid_guest_access_tokens`: ??? 154 | `threepid_guest_access_tokens`: ??? 155 | `threepid_validation_token`: ??? 156 | 157 | `ui_auth_sessions`: ??? 158 | `ui_auth_sessions_credentials`: ??? 159 | `ui_auth_sessions_ips`: ??? 160 | 161 | `user_daily_visits`: unsure 162 | `user_directory`: unsure, maybe not 163 | `user_directory_search`: calculated 164 | `user_directory_stream_pos`: no 165 | `user_external_ids`: unsure 166 | `user_filters`: yes 167 | `user_ips`: maybe not 168 | `user_signature_stream`: ??? 169 | `user_stats_current`: calculated 170 | `user_stats_historical`: likely not 171 | `user_threepid_id_server`: ??? 172 | `user_threepids`: ??? 173 | `users`: yes 174 | `users_in_public_rooms`: derived & calculated 175 | `users_pending_deactivation`: unsure 176 | `users_to_send_full_presence_to`: probably 177 | `users_who_share_private_rooms`: unsure, calculated? 178 | 179 | `worker_locks`: no 180 | 181 | --- 182 | 183 | - Users 184 | - Registration Token 185 | - Access Tokens 186 | - Refresh Tokens 187 | - Account Data 188 | - Room Data 189 | - Room Tags 190 | - Devices 191 | - E2E 192 | - Dehydrated 193 | - Profile 194 | - Filters 195 | - Pusher 196 | - Rooms 197 | - PDUs 198 | - Read Receipts 199 | - Aliases 200 | - Room list 201 | - Media 202 | - Signing Key 203 | -------------------------------------------------------------------------------- /adapters/synapse/src/extract.rs: -------------------------------------------------------------------------------- 1 | use std::{cell::RefMut, collections::HashMap}; 2 | 3 | use postgres::{fallible_iterator::FallibleIterator, Client, RowIter}; 4 | use postgres_types::{FromSql, Json, ToSql}; 5 | use serde::Deserialize; 6 | use serde_json::{map::Map, value::Value}; 7 | use variate_lib::{ 8 | traits::{ 9 | PduExtractor, ReadReceiptsExtractor, RoomExtractor, ToDeviceMessageExtractor, UserExtractor, 10 | }, 11 | types::{BIter, DeviceId, EventId, Pdu, RoomReadReceipts, RoomId, ToDeviceMessage, UserId}, 12 | Extractor, 13 | }; 14 | 15 | use crate::DatabasePuck; 16 | 17 | pub struct SynapseExtractor { 18 | puck: DatabasePuck, 19 | } 20 | 21 | impl SynapseExtractor { 22 | pub(crate) fn new(puck: DatabasePuck) -> Self { 23 | Self { puck } 24 | } 25 | } 26 | 27 | impl Extractor for SynapseExtractor { 28 | fn pdu_e(&mut self) -> &mut dyn PduExtractor { 29 | self 30 | } 31 | 32 | fn rr_e(&mut self) -> &mut dyn ReadReceiptsExtractor { 33 | self 34 | } 35 | 36 | fn room_e(&mut self) -> &mut dyn RoomExtractor { 37 | self 38 | } 39 | 40 | fn usr_e(&mut self) -> &mut dyn UserExtractor { 41 | self 42 | } 43 | 44 | fn td_e(&mut self) -> &mut dyn ToDeviceMessageExtractor { 45 | self 46 | } 47 | } 48 | 49 | fn empty() -> std::iter::Empty { 50 | std::iter::empty::() 51 | } 52 | 53 | impl RoomExtractor for SynapseExtractor { 54 | fn all_known_ids(&mut self) -> Vec { 55 | let mut it = self.puck.pg.query_raw("SELECT room_id FROM rooms", empty()).unwrap(); 56 | 57 | std::iter::from_fn(move || it.next().ok().flatten().map(|r| r.get("room_id"))).collect() 58 | } 59 | } 60 | 61 | impl UserExtractor for SynapseExtractor { 62 | fn all_local_ids(&mut self) -> Vec { 63 | let mut it = self.puck.pg.query_raw("SELECT name FROM users", empty()).unwrap(); 64 | 65 | std::iter::from_fn(move || it.next().ok().flatten().map(|r| r.get("name"))).collect() 66 | } 67 | } 68 | 69 | impl ToDeviceMessageExtractor for SynapseExtractor { 70 | fn all(&mut self) -> Vec { 71 | let mut messages = vec![]; 72 | let mut client = &mut self.puck.pg; 73 | 74 | if false { 75 | let mut it = client 76 | .query_raw("SELECT user_id, device_id, message_json FROM device_inbox", empty()) 77 | .unwrap(); 78 | 79 | #[derive(Deserialize)] 80 | struct MessageJson { 81 | pub content: Value, 82 | pub r#type: String, 83 | pub sender: String, 84 | } 85 | 86 | while let Some(row) = it.next().unwrap() { 87 | let user_id: UserId = row.get("user_id"); 88 | let device_id: DeviceId = row.get("device_id"); 89 | let message_json: String = row.get("message_json"); 90 | let message = serde_json::from_str::(&message_json).unwrap(); 91 | 92 | messages.push(ToDeviceMessage { 93 | user: user_id, 94 | device: device_id, 95 | sender: message.sender, 96 | r#type: message.r#type, 97 | message: message.content, 98 | }) 99 | } 100 | } 101 | 102 | { 103 | let mut it = client 104 | .query_raw("SELECT messages_json FROM device_federation_outbox", empty()) 105 | .unwrap(); 106 | 107 | #[derive(Deserialize)] 108 | struct MessagesJson { 109 | pub messages: HashMap>, 110 | pub sender: UserId, 111 | pub r#type: String, 112 | } 113 | 114 | while let Some(row) = it.next().unwrap() { 115 | let messages_json: String = row.get("messages_json"); 116 | let messages_data = serde_json::from_str::(&messages_json).unwrap(); 117 | 118 | for (user, devices) in messages_data.messages { 119 | for (device, message) in devices { 120 | messages.push(ToDeviceMessage { 121 | user: user.clone(), 122 | device, 123 | sender: messages_data.sender.clone(), 124 | r#type: messages_data.r#type.clone(), 125 | message, 126 | }) 127 | } 128 | } 129 | } 130 | } 131 | 132 | messages 133 | } 134 | } 135 | 136 | impl PduExtractor for SynapseExtractor { 137 | fn for_room<'a>(&'a mut self, room: &RoomId) -> BIter<'a, Pdu> { 138 | struct TryThis<'a> { 139 | iter: RowIter<'a>, 140 | } 141 | 142 | impl<'a, 'b> Iterator for TryThis<'a> { 143 | type Item = Pdu; 144 | 145 | fn next(&mut self) -> Option { 146 | self.iter.next().ok().flatten().map(|r| { 147 | let id = r.get::<_, String>("event_id"); 148 | let version: i32 = r.get("format_version"); 149 | let json: String = r.get("json"); 150 | let value: Map = serde_json::from_str(&json).unwrap(); 151 | 152 | // a bunch of sanity checks 153 | assert!(value.contains_key("type"), "event did not have type"); 154 | if version == 1 { 155 | assert!( 156 | value.contains_key("event_id"), 157 | "event version 1 did not have event_id: {} ({}), {:#?}", 158 | id, 159 | version, 160 | value 161 | ); 162 | } else { 163 | assert!( 164 | !value.contains_key("event_id"), 165 | "event version !1 did have event_id: {} ({}), {:#?}", 166 | id, 167 | version, 168 | value 169 | ); 170 | } 171 | assert!(value.contains_key("content")); 172 | assert!(value.contains_key("room_id")); 173 | 174 | Pdu { id, value } 175 | }) 176 | } 177 | } 178 | 179 | let iter: RowIter<'a> = self 180 | .puck 181 | .pg 182 | .query_raw( 183 | // todo remove 1 = 1 184 | "SELECT event_id, json, format_version FROM event_json WHERE room_id = $1 OR 1 = 1", 185 | &[room], 186 | ) 187 | .unwrap(); 188 | 189 | Box::new(TryThis { iter }) 190 | } 191 | } 192 | 193 | impl ReadReceiptsExtractor for SynapseExtractor { 194 | fn for_room(&mut self, room: &RoomId) -> RoomReadReceipts { 195 | todo!() 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /adapters/synapse/src/insert.rs: -------------------------------------------------------------------------------- 1 | use variate_lib::{ 2 | traits::{ 3 | PduInserter, ReadReceiptsInserter, RoomInserter, ToDeviceMessageInserter, UserInserter, 4 | }, 5 | Inserter, 6 | }; 7 | 8 | use crate::DatabasePuck; 9 | 10 | pub struct SynapseInserter { 11 | puck: DatabasePuck, 12 | } 13 | 14 | impl Inserter for SynapseInserter { 15 | fn pdu_i(&mut self) -> &mut dyn PduInserter { 16 | self 17 | } 18 | 19 | fn rr_i(&mut self) -> &mut dyn ReadReceiptsInserter { 20 | self 21 | } 22 | 23 | fn room_i(&mut self) -> &mut dyn RoomInserter { 24 | self 25 | } 26 | 27 | fn usr_i(&mut self) -> &mut dyn UserInserter { 28 | self 29 | } 30 | 31 | fn td_i(&mut self) -> &mut dyn ToDeviceMessageInserter { 32 | self 33 | } 34 | } 35 | 36 | impl PduInserter for SynapseInserter { 37 | fn add_to_room(&mut self, room: &variate_lib::types::RoomId, pdu: variate_lib::types::Pdu) { 38 | todo!() 39 | } 40 | } 41 | 42 | impl ReadReceiptsInserter for SynapseInserter { 43 | fn into_room( 44 | &mut self, 45 | room: &variate_lib::types::RoomId, 46 | rr: variate_lib::types::ReadReceipts, 47 | ) { 48 | todo!() 49 | } 50 | } 51 | 52 | impl RoomInserter for SynapseInserter { 53 | fn skeleton_room(&mut self, room: variate_lib::types::RoomId) { 54 | todo!() 55 | } 56 | } 57 | 58 | impl UserInserter for SynapseInserter { 59 | fn skeleton_user(&mut self, id: variate_lib::types::UserId) { 60 | todo!() 61 | } 62 | } 63 | 64 | impl ToDeviceMessageInserter for SynapseInserter { 65 | fn add(&mut self, message: variate_lib::types::ToDeviceMessage) { 66 | todo!() 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /adapters/synapse/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::cell::RefCell; 2 | 3 | pub mod extract; 4 | // pub mod insert; 5 | 6 | use postgres::{Client, Config}; 7 | use variate_lib::{Extractor, Inserter}; 8 | 9 | pub fn make_config( 10 | host: &str, 11 | port: Option, 12 | user: &str, 13 | password: &str, 14 | database: &str, 15 | ) -> Config { 16 | let mut c = Config::new(); 17 | 18 | c.host(host).user(user).password(password).dbname(database); 19 | 20 | if let Some(port) = port { 21 | c.port(port); 22 | } 23 | 24 | c 25 | } 26 | 27 | pub fn make_extractor(c: Config) -> impl Extractor { 28 | let mut client = c.connect(postgres::NoTls).unwrap(); 29 | 30 | let version: i32 = 31 | client.query_one("SELECT version FROM schema_version", &[]).unwrap().get("version"); 32 | 33 | assert_eq!(version, 65); // cant accept other schemas at the moment 34 | 35 | let puck = DatabasePuck { pg: client, version }; 36 | 37 | extract::SynapseExtractor::new(puck) 38 | } 39 | 40 | pub fn make_inserter() -> Box { 41 | todo!() 42 | } 43 | 44 | struct DatabasePuck { 45 | pg: Client, 46 | version: i32, 47 | } 48 | -------------------------------------------------------------------------------- /adapters/synapse/src/main.rs: -------------------------------------------------------------------------------- 1 | use variate_lib::Extractor; 2 | use variate_synapse; 3 | 4 | // const PASSWORD: &str = "CUZBABYYOUREAFIRE"; 5 | const PASSWORD: &str = "Cuzbabytonight"; 6 | 7 | fn main() { 8 | let c = variate_synapse::make_config("localhost", None, "synapse", PASSWORD, "synapse"); 9 | 10 | let mut e = variate_synapse::make_extractor(c); 11 | 12 | // let ids = e.room_e().all_known_ids(); 13 | 14 | // dbg!(ids); 15 | 16 | let rooms = e.room_e().all_known_ids(); 17 | let in_rooms = &rooms[..10]; 18 | 19 | for room in in_rooms { 20 | dbg!(room); 21 | 22 | let mut counter = 0; 23 | for event in e.pdu_e().for_room(room) { 24 | counter += 1; 25 | if counter % 1000 == 0 { 26 | dbg!(counter); 27 | } 28 | } 29 | 30 | if counter % 1000 != 0 { 31 | dbg!(counter); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "variate-cli" 3 | version = "0.0.1" 4 | authors = ["Jonathan de Jong "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | -------------------------------------------------------------------------------- /cli/src/main.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | println!("Hello, world!"); 3 | } 4 | -------------------------------------------------------------------------------- /graph.md: -------------------------------------------------------------------------------- 1 | lib -> adapters -> cli -------------------------------------------------------------------------------- /lib/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "variate_lib" 3 | version = "0.1.0" 4 | edition = "2018" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | serde = { version = "1.0.130", features = ["derive"] } 10 | serde_json = "1.0.67" 11 | -------------------------------------------------------------------------------- /lib/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod traits; 2 | pub mod types; 3 | 4 | use traits::*; 5 | 6 | pub trait Extractor { 7 | fn pdu_e(&mut self) -> &mut dyn PduExtractor; 8 | fn rr_e(&mut self) -> &mut dyn ReadReceiptsExtractor; 9 | fn room_e(&mut self) -> &mut dyn RoomExtractor; 10 | fn usr_e(&mut self) -> &mut dyn UserExtractor; 11 | fn td_e(&mut self) -> &mut dyn ToDeviceMessageExtractor; 12 | } 13 | 14 | pub trait Inserter { 15 | fn pdu_i(&mut self) -> &mut dyn PduInserter; 16 | fn rr_i(&mut self) -> &mut dyn ReadReceiptsInserter; 17 | fn room_i(&mut self) -> &mut dyn RoomInserter; 18 | fn usr_i(&mut self) -> &mut dyn UserInserter; 19 | fn td_i(&mut self) -> &mut dyn ToDeviceMessageInserter; 20 | } 21 | -------------------------------------------------------------------------------- /lib/src/traits.rs: -------------------------------------------------------------------------------- 1 | use super::types; 2 | 3 | // user -> room -> PDU -> RR 4 | 5 | pub use extr::*; 6 | 7 | mod extr { 8 | use crate::types::BIter; 9 | 10 | use super::types; 11 | 12 | pub trait PduExtractor { 13 | fn for_room(&mut self, room: &types::RoomId) -> BIter; 14 | } 15 | pub trait RoomExtractor { 16 | fn all_known_ids(&mut self) -> Vec; 17 | } 18 | 19 | pub trait UserExtractor { 20 | fn all_local_ids(&mut self) -> Vec; 21 | } 22 | 23 | pub trait ReadReceiptsExtractor { 24 | fn for_room(&mut self, room: &types::RoomId) -> types::RoomReadReceipts; 25 | } 26 | pub trait ToDeviceMessageExtractor { 27 | /// Gets all to-device messages still to be forwarded. 28 | /// 29 | /// This includes outbox messages, and inbox messages. 30 | fn all(&mut self) -> Vec; 31 | } 32 | } 33 | 34 | pub use ins::*; 35 | 36 | mod ins { 37 | use super::types; 38 | 39 | pub trait PduInserter { 40 | fn add_to_room(&mut self, room: &types::RoomId, pdu: types::Pdu); 41 | } 42 | 43 | pub trait RoomInserter { 44 | fn skeleton_room(&mut self, room: types::RoomId); 45 | } 46 | 47 | pub trait UserInserter { 48 | fn skeleton_user(&mut self, id: types::UserId); 49 | } 50 | 51 | pub trait ReadReceiptsInserter { 52 | fn into_room(&mut self, room: &types::RoomId, rr: types::RoomReadReceipts); 53 | } 54 | 55 | pub trait ToDeviceMessageInserter { 56 | fn add(&mut self, message: types::ToDeviceMessage); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /lib/src/types.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use serde_json::{map::Map, value::Value}; 4 | 5 | pub type BIter<'a, T> = Box + 'a>; 6 | 7 | // unvetted user ID 8 | // #[repr(transparent)] 9 | // #[derive(Debug, Hash, PartialEq, Eq, Deserialize, Serialize, ToSql)] 10 | pub type UserId = String; 11 | 12 | // #[repr(transparent)] 13 | // #[derive(Debug)] 14 | pub type EventId = String; 15 | 16 | // #[repr(transparent)] 17 | // #[derive(Debug)] 18 | pub type RoomId = String; 19 | 20 | // #[repr(transparent)] 21 | // #[derive(Debug, Hash, PartialEq, Eq, Deserialize, Serialize)] 22 | // pub struct DeviceId(pub String); 23 | pub type DeviceId = String; 24 | 25 | pub enum Untrusted { 26 | Re(T), 27 | Un(Value), 28 | } 29 | 30 | #[derive(Debug)] 31 | pub struct Pdu { 32 | pub id: EventId, 33 | pub value: Map, 34 | } 35 | 36 | impl Pdu { 37 | // None = invalid 38 | pub fn is_state(&self) -> Option { 39 | match self.value.get("state_key") { 40 | Some(Value::String(_)) => Some(true), 41 | Some(_) => None, 42 | None => Some(false), 43 | } 44 | } 45 | } 46 | 47 | #[derive(Debug)] 48 | pub struct RoomReadReceipts(pub HashMap>); 49 | 50 | #[derive(Debug)] 51 | pub struct ToDeviceMessage { 52 | pub user: UserId, 53 | pub device: DeviceId, 54 | pub sender: UserId, 55 | 56 | pub r#type: String, 57 | 58 | pub message: Value, 59 | } 60 | --------------------------------------------------------------------------------