├── .DS_Store ├── .github └── workflows │ └── test.yml ├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md ├── RELEASE.md ├── examples └── iot-service │ ├── Cargo.toml │ ├── README.md │ └── src │ ├── database.rs │ ├── http_server.rs │ ├── main.rs │ └── udp_server.rs ├── streambed-codec ├── .gitignore ├── Cargo.toml ├── README.md └── src │ └── lib.rs ├── streambed-confidant-cli ├── Cargo.toml ├── README.md └── src │ ├── cryptor.rs │ ├── errors.rs │ └── main.rs ├── streambed-confidant ├── Cargo.toml ├── README.md ├── examples │ ├── README.md │ └── confidant.rs └── src │ ├── args.rs │ └── lib.rs ├── streambed-kafka ├── Cargo.toml ├── README.md ├── src │ ├── args.rs │ └── lib.rs └── tests │ ├── commit_log.rs │ └── mod.rs ├── streambed-logged-cli ├── Cargo.toml ├── README.md └── src │ ├── errors.rs │ ├── main.rs │ ├── producer.rs │ └── subscriber.rs ├── streambed-logged ├── Cargo.toml ├── README.md ├── benches │ └── benches.rs └── src │ ├── args.rs │ ├── compaction.rs │ ├── lib.rs │ └── topic_file_op.rs ├── streambed-patterns ├── Cargo.toml ├── README.md ├── examples │ └── ask.rs └── src │ ├── ask.rs │ └── lib.rs ├── streambed-storage ├── Cargo.toml ├── README.md └── src │ ├── args.rs │ └── lib.rs ├── streambed-test ├── Cargo.toml ├── README.md └── src │ ├── lib.rs │ └── server.rs ├── streambed-vault ├── Cargo.toml ├── README.md ├── examples │ ├── README.md │ └── vault.rs ├── src │ ├── args.rs │ └── lib.rs └── tests │ ├── mod.rs │ └── secret_store.rs └── streambed ├── Cargo.toml ├── README.md └── src ├── commit_log.rs ├── crypto.rs ├── delayer.rs ├── lib.rs └── secret_store.rs /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/streambed/streambed-rs/31074bac46d6bee4e66584f37e0f5db24d13b103/.DS_Store -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | RUSTFLAGS: -Dwarnings 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-20.04 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Update Rust 20 | run: | 21 | rustup update 22 | - uses: Swatinem/rust-cache@v2 23 | 24 | - name: Format, lint and test 25 | run: | 26 | cargo fmt -- --check 27 | cargo clippy --verbose 28 | cargo test --verbose 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | Cargo.lock 3 | target 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | 4 | members = [ 5 | "streambed", 6 | "streambed-codec", 7 | "streambed-confidant", 8 | "streambed-confidant-cli", 9 | "streambed-kafka", 10 | "streambed-logged", 11 | "streambed-logged-cli", 12 | "streambed-patterns", 13 | "streambed-storage", 14 | "streambed-test", 15 | "streambed-vault", 16 | "examples/iot-service", 17 | ] 18 | 19 | [workspace.package] 20 | version = "0.13.0" # WHEN CHANGING THIS, CHANGE THE "STREAMBED" DEPENDENCIES BELOW ALSO. 21 | edition = "2021" 22 | rust-version = "1.70.0" 23 | license = "Apache-2.0" 24 | repository = "https://github.com/streambed/streambed-rs.git" 25 | 26 | [workspace.dependencies] 27 | aes = "0.8.2" 28 | async-stream = "0.3.3" 29 | async-trait = "0.1.60" 30 | base64 = "0.20.0" 31 | bytes = "1.3.0" 32 | cache_loader_async = "0.2.1" 33 | chrono = "0.4.23" 34 | ciborium = "0.2" 35 | clap = "4.0.29" 36 | crc = "3.0.1" 37 | criterion = "0.4.0" 38 | env_logger = "0.10.0" 39 | exponential-backoff = "1.1.0" 40 | futures-util = { version = "0.3", default-features = false } 41 | hex = "0.4.3" 42 | http = "0.2.8" 43 | hmac = "0.12.1" 44 | humantime = "2.1.0" 45 | hyper = { version = "0.14.23", default-features = false } 46 | log = "0.4.17" 47 | metrics = { version = "0.20.1", default-features = false } 48 | postcard = { version = "1.0.8", default-features = false } 49 | rand = "0.8.5" 50 | reqwest = "0.11.1" 51 | scopeguard = "1.1" 52 | serde = "1.0.151" 53 | serde_json = "1.0.90" 54 | serde_with = "2.0.1" 55 | sha2 = "0.10.6" 56 | smol_str = "0.3.2" 57 | test-log = "0.2.11" 58 | tokio = "1.23.0" 59 | tokio-stream = "0.1.11" 60 | tokio-util = "0.7.4" 61 | warp = "0.3" 62 | 63 | streambed = { path = "streambed", version = "0.13.0" } 64 | streambed-confidant = { path = "streambed-confidant", version = "0.13.0" } 65 | streambed-logged = { path = "streambed-logged", version = "0.13.0" } 66 | streambed-test = { path = "streambed-test", version = "0.13.0" } 67 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # streambed-rs - An efficient and curated toolkit for writing event-driven services 2 | 3 | _Being event-driven closes the gap between understanding a problem domain and expressing that problem in code._ 4 | 5 | Streambed is a curated set of dependencies and a toolkit for writing asynchronous event-driven services that aim to run on the 6 | smallest "std" targets supported by Rust. Streambed-based programs presently use a single core MIPS32 OpenWrt device running at around 500MHz 7 | and 128MiB as a baseline target. 8 | 9 | A commit log modelled on [Apache Kafka](https://kafka.apache.org/) is provided, along with partial and extended support for the Kafka HTTP API. 10 | 11 | A secret store modelled on [Hashicorp Vault](https://www.vaultproject.io/) is provided along with partial support for the Vault HTTP API. 12 | 13 | File-based implementations of the commit log and secret store suitable for use on embedded Linux are provided with 14 | [logged](streambed-logged/README.md) and [confidant](streambed-confidant/README.md) respectively. 15 | 16 | Production services using the commit log and secret store have been shown to use less than 3MiB of resident memory 17 | while also offering good performance. 18 | 19 | ## Streambed's characteristics 20 | ### Event-driven services 21 | 22 | Event-driven services promote responsiveness as events can be pushed to where they need to be consumed; by the user 23 | of a system. Event-driven services are also resilient to failure as they can use "event sourcing" to 24 | quickly rebuild their state through replaying events. 25 | 26 | ### Efficient 27 | 28 | Streambed based applications are designed to run at the edge on embedded computers as well as in the cloud and so efficient CPU, memory and disk usage are a primary concern. 29 | 30 | ### Secure 31 | 32 | Security is also a primary consideration throughout the design of Streambed. For example, in the world of the Internet of Things, if an individual sensor becomes compromised then its blast-radius can be minimized. 33 | 34 | ### Built for integration 35 | 36 | Streambed is a toolkit that promotes the consented sharing of data between many third-party applications. No more silos of data. Improved data availability leads to better decision making, which leads to better business. 37 | 38 | ### Standing on the shoulders of giants, leveraging existing communities 39 | 40 | Streambed is an assemblage of proven approaches and technologies that already have strong communities. Should you have a problem there are many people and resources you can call on. 41 | 42 | ### Open source and open standards 43 | 44 | Streambed is entirely open source providing cost benefits, fast-time-to-market, the avoidance of vendor lock-in, improved security and more. 45 | 46 | ### Rust 47 | 48 | Streambed leverages Rust's characteristics of writing fast and efficient software correctly. 49 | 50 | ## Minimum supported Rust 51 | 52 | streambed-rs requires a minimum of Rust version 1.70.0 stable (June 2023), but the most recent stable version of Rust is recommended. 53 | 54 | ## A brief history and why Rust 55 | 56 | Streambed-jvm first manifested itself as a Scala based-project with similar goals to streambed-rs and 57 | targeted on larger machines that could run a JVM. Cisco Inc. sponsored Titan Class Pty Ltd with the development of 58 | streambed-jvm targeting edge-based routers on farms. Titan Class continues to run streambed at 59 | the edge on several Australian farms and has done so now for several years. This experience has proven out the 60 | event-driven approach that underpins Streambed. It also highlighted that more energy-efficient solutions needed 61 | to be sought given that power at the edge is a challenge. Hence the re-writing of streambed-jvm in Rust. 62 | 63 | ## Contribution policy 64 | 65 | Contributions via GitHub pull requests are gladly accepted from their original author. Along with any pull requests, please state that the contribution is your original work and that you license the work to the project under the project's open source license. Whether or not you state this explicitly, by submitting any copyrighted material via pull request, email, or other means you agree to license the material under the project's open source license and warrant that you have the legal authority to do so. 66 | 67 | ## License 68 | 69 | This code is open source software licensed under the [Apache-2.0 license](./LICENSE). 70 | 71 | © Copyright [Titan Class P/L](https://www.titanclass.com.au/), 2022 72 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | Releasing 2 | === 3 | 4 | 1. Update the cargo.toml and change the `workspace.package.version`. 5 | 2. Change the `dependency.streambed` and `dependency.streambed-test` versions to be the same version number as per step 1. 6 | 3. Commit the changes and tag with the version using `v` as a prefix e.g. 1.0.0 would be "v1.0.0". 7 | 4. Perform the commands below, and in the same order... 8 | 9 | ``` 10 | cargo publish -p streambed 11 | cargo publish -p streambed-test 12 | cargo publish -p streambed-confidant 13 | cargo publish -p streambed-confidant-cli 14 | cargo publish -p streambed-kafka 15 | cargo publish -p streambed-logged 16 | cargo publish -p streambed-logged-cli 17 | cargo publish -p streambed-patterns 18 | cargo publish -p streambed-storage 19 | cargo publish -p streambed-vault 20 | cargo publish -p streambed-codec 21 | ``` -------------------------------------------------------------------------------- /examples/iot-service/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "iot-service" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | 7 | [dependencies] 8 | async-stream = { workspace = true } 9 | chrono = { workspace = true } 10 | ciborium = { workspace = true } 11 | clap = { workspace = true, features = ["derive", "env"] } 12 | env_logger = { workspace = true } 13 | hex = { workspace = true } 14 | humantime = { workspace = true } 15 | log = { workspace = true } 16 | postcard = { workspace = true, default-features = false, features = ["use-std"] } 17 | scopeguard = { workspace = true } 18 | serde = { workspace = true, features = ["derive"] } 19 | tokio = { workspace = true, features = ["full"] } 20 | tokio-stream = { workspace = true } 21 | tokio-util = { workspace = true, features = ["codec"] } 22 | warp = { workspace = true } 23 | 24 | streambed = { path = "../../streambed" } 25 | streambed-logged = { path = "../../streambed-logged" } 26 | streambed-patterns = { path = "../../streambed-patterns" } 27 | 28 | [dev-dependencies] 29 | env_logger = { workspace = true } 30 | test-log = { workspace = true } 31 | -------------------------------------------------------------------------------- /examples/iot-service/README.md: -------------------------------------------------------------------------------- 1 | IoT service example 2 | === 3 | 4 | This is an example that illustrates the use of streambed that writes telemetry 5 | to an append log. A simple HTTP API is provided to query and scan the commit log. 6 | 7 | The service is not a complete example given that encryption has been left out so 8 | that the example remains simple. Encryption should normally be applied to data 9 | at rest (persisted by the commit log) and in flight (http and UDP). 10 | 11 | Running 12 | --- 13 | 14 | To run via cargo, first `cd` into this directory and then: 15 | 16 | ``` 17 | mkdir -p /tmp/iot-service/var/lib/logged 18 | RUST_LOG=debug cargo run -- \ 19 | --cl-root-path=/tmp/iot-service/var/lib/logged 20 | ``` 21 | 22 | You should now be able to query for database events: 23 | 24 | ``` 25 | curl -v "127.0.0.1:8080/api/database/events?id=1" 26 | ``` 27 | 28 | You should also be able to post database events to the UDP socket. Note that 29 | we're using Postcard to deserialize binary data. Postcard uses variable length 30 | integers where the top bit, when set, indicates that the next byte also contains 31 | data. See [Postcard](https://docs.rs/postcard/latest/postcard/) for more details. 32 | 33 | ``` 34 | echo -n "\x01\x02" | nc -w0 127.0.0.1 -u 8081 35 | ``` 36 | 37 | You should see a `DEBUG` log indicating that the post has been received. You should 38 | also be able to query the database again with the id that was sent (`1`). 39 | 40 | You should also be able to see events being written to the log file store itself: 41 | 42 | ``` 43 | hexdump /tmp/iot-service/var/lib/logged/device-events 44 | ``` 45 | 46 | Compaction 47 | ---- 48 | 49 | If you would like to see compaction at work then you can drive UDP traffic with 50 | a script such as the following: 51 | 52 | ```bash 53 | #!/bin/bash 54 | for i in {0..2000} 55 | do 56 | printf "\x01\x02" | nc -w0 127.0.0.1 -u 8081 57 | done 58 | ``` 59 | 60 | The above will send 2,000 UDP messages to the service. Toward the end, if you watch 61 | the log of the service, you will see various messages from the compactor at work. 62 | When it has finished, you can observe that they are two log files for our topic e.g.: 63 | 64 | ``` 65 | ls -al /tmp/iot-service/var/lib/logged 66 | 67 | drwxr-xr-x 4 huntc wheel 128 14 Feb 14:31 . 68 | drwxr-xr-x 3 huntc wheel 96 14 Feb 10:49 .. 69 | -rw-r--r-- 1 huntc wheel 495 14 Feb 14:31 device-events 70 | -rw-r--r-- 1 huntc wheel 330 14 Feb 14:31 device-events.history 71 | ``` 72 | 73 | The `history` file contains the compacted log. As each record is 33 bytes, this means 74 | that compaction retained the last 10 records (330 bytes). The active file, or `device-events`, 75 | contains 15 additional records that continued to be written while compaction was 76 | in progress. The compactor is designed to avoid back-pressuring the production of 77 | records. That said, if the production of events overwhelms compaction then 78 | it will back-pressure on the producer. It is up to you, as the application developer, 79 | to decide whether to always await the completion of producing a record. In some 80 | real-time scenarios, waiting on confirmation that an event is written may be 81 | undesirable. However, with the correct dimensioning of the commit log in terms of 82 | its buffers and how the compaction strategies are composed, compaction back-pressure 83 | can also be avoided. -------------------------------------------------------------------------------- /examples/iot-service/src/database.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::VecDeque, time::Duration}; 2 | 3 | use chrono::{DateTime, Utc}; 4 | use serde::{Deserialize, Serialize}; 5 | use streambed::commit_log::{CommitLog, ProducerRecord, Subscription, Topic}; 6 | use streambed_logged::{compaction::NthKeyBasedRetention, FileLog}; 7 | use tokio::sync::mpsc; 8 | use tokio_stream::StreamExt; 9 | 10 | const DEVICE_EVENTS_TOPIC: &str = "device-events"; 11 | const IDLE_TIMEOUT: Duration = Duration::from_millis(50); 12 | const MAX_EVENTS_TO_REPLY: usize = 10; 13 | // Size the following to the typical number of devices we expect to have in the system. 14 | // Note though that it will impact memory, so there is a trade-off. Let's suppose this 15 | // was some LoRaWAN system and that our gateway cannot handle more than 1,000 devices 16 | // being connected. We can work out that 1,000 is therefore a reasonable limit. We can 17 | // have less or more. The overhead is small, but should be calculated and measured for 18 | // a production app. 19 | const MAX_TOPIC_COMPACTION_KEYS: usize = 1_000; 20 | 21 | pub type GetReplyToFnOnce = dyn FnOnce(VecDeque<(DateTime, Event)>) + Send; 22 | 23 | pub enum Command { 24 | Get(u64, Box), 25 | Post(u64, Event), 26 | } 27 | 28 | #[derive(Deserialize, Serialize)] 29 | pub enum Event { 30 | TemperatureRead(u32), 31 | } 32 | 33 | /// Manage the database. We use CBOR as the serialization type for the data 34 | /// we persist to the log. We do this as it has the benefits of JSON in terms 35 | /// of schema evolution, but is faster to serialize and represents itself as 36 | /// smaller on disk. 37 | pub async fn task(mut cl: FileLog, mut database_command_rx: mpsc::Receiver) { 38 | // We register a compaction strategy for our topic such that when we use up 39 | // 64KB of disk space (the default), we will run compaction so that unwanted 40 | // events are removed. In our scenario, unwanted events can be removed when 41 | // the exceed MAX_EVENTS_TO_REPLY as we do not have a requirement to ever 42 | // return more than that. 43 | cl.register_compaction( 44 | Topic::from(DEVICE_EVENTS_TOPIC), 45 | NthKeyBasedRetention::new(MAX_TOPIC_COMPACTION_KEYS, MAX_EVENTS_TO_REPLY), 46 | ) 47 | .await 48 | .unwrap(); 49 | 50 | while let Some(c) = database_command_rx.recv().await { 51 | match c { 52 | Command::Get(id, reply_to) => { 53 | let offsets = vec![]; 54 | 55 | let subscriptions = vec![Subscription { 56 | topic: Topic::from(DEVICE_EVENTS_TOPIC), 57 | }]; 58 | 59 | let mut records = 60 | cl.scoped_subscribe("iot-service", offsets, subscriptions, Some(IDLE_TIMEOUT)); 61 | 62 | let mut events = VecDeque::with_capacity(MAX_EVENTS_TO_REPLY); 63 | while let Some(record) = records.next().await { 64 | if record.key == id { 65 | let Some(timestamp) = record.timestamp else { 66 | continue; 67 | }; 68 | let Ok(database_event) = 69 | ciborium::de::from_reader::(&*record.value) 70 | else { 71 | continue; 72 | }; 73 | if events.len() == MAX_EVENTS_TO_REPLY { 74 | events.pop_front(); 75 | } 76 | events.push_back((timestamp, database_event)); 77 | 78 | // We perform an additional check to return if the 79 | // event is recent so that we don't get stuck consuming 80 | // when appending is happening simultaneously at a very 81 | // high frequency. 82 | if (timestamp - Utc::now()) 83 | .to_std() 84 | .map(|r| r < IDLE_TIMEOUT) 85 | .unwrap_or_default() 86 | { 87 | break; 88 | } 89 | } 90 | } 91 | 92 | reply_to(events); 93 | } 94 | Command::Post(id, event) => { 95 | let mut buf = Vec::new(); 96 | if ciborium::ser::into_writer(&event, &mut buf).is_ok() { 97 | let record = ProducerRecord { 98 | topic: Topic::from(DEVICE_EVENTS_TOPIC), 99 | headers: vec![], 100 | timestamp: Some(Utc::now()), 101 | key: id, 102 | value: buf, 103 | partition: 0, 104 | }; 105 | let _ = cl.produce(record).await; 106 | } 107 | } 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /examples/iot-service/src/http_server.rs: -------------------------------------------------------------------------------- 1 | //! Handle http serving concerns 2 | //! 3 | use crate::database; 4 | use std::collections::HashMap; 5 | use streambed_patterns::ask::Ask; 6 | use tokio::sync::mpsc; 7 | use warp::{hyper::StatusCode, Filter, Rejection, Reply}; 8 | 9 | /// Declares routes to serve our HTTP interface. 10 | pub fn routes( 11 | database_command_tx: mpsc::Sender, 12 | ) -> impl Filter + Clone { 13 | let get_database_route = { 14 | warp::get() 15 | .and(warp::path("events")) 16 | .and(warp::path::end()) 17 | .and(warp::query()) 18 | .then(move |query: HashMap| { 19 | let task_database_command_tx = database_command_tx.clone(); 20 | async move { 21 | let Some(id) = query.get("id") else { 22 | return warp::reply::with_status( 23 | warp::reply::json(&"An id is required"), 24 | StatusCode::BAD_REQUEST, 25 | ); 26 | }; 27 | 28 | let Ok(id) = id.parse() else { 29 | return warp::reply::with_status( 30 | warp::reply::json(&"Invalid id - must be a number"), 31 | StatusCode::BAD_REQUEST, 32 | ); 33 | }; 34 | 35 | let Ok(events) = task_database_command_tx 36 | .ask(|reply_to| database::Command::Get(id, reply_to)) 37 | .await 38 | else { 39 | return warp::reply::with_status( 40 | warp::reply::json(&"Service unavailable"), 41 | StatusCode::SERVICE_UNAVAILABLE, 42 | ); 43 | }; 44 | 45 | warp::reply::with_status(warp::reply::json(&events), StatusCode::OK) 46 | } 47 | }) 48 | }; 49 | 50 | warp::path("api").and(warp::path("database").and(get_database_route)) 51 | } 52 | -------------------------------------------------------------------------------- /examples/iot-service/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{error::Error, net::SocketAddr}; 2 | 3 | use clap::Parser; 4 | use log::info; 5 | use streambed_logged::{args::CommitLogArgs, FileLog}; 6 | use tokio::{net::UdpSocket, sync::mpsc}; 7 | 8 | mod database; 9 | mod http_server; 10 | mod udp_server; 11 | 12 | /// This service receives IoT data re. temperature, stores it in the 13 | /// commit log keyed by its sensor id, and provides an HTTP interface 14 | /// to access it. 15 | #[derive(Parser, Debug)] 16 | #[clap(author, about, long_about = None, version)] 17 | struct Args { 18 | /// Logged commit log args 19 | #[clap(flatten)] 20 | cl_args: CommitLogArgs, 21 | 22 | /// A socket address for serving our HTTP web service requests. 23 | #[clap(env, long, default_value = "127.0.0.1:8080")] 24 | http_addr: SocketAddr, 25 | 26 | /// A socket address for receiving telemetry from our ficticious 27 | /// sensor. 28 | #[clap(env, long, default_value = "127.0.0.1:8081")] 29 | udp_addr: SocketAddr, 30 | } 31 | 32 | const MAX_DATABASE_MANAGER_COMMANDS: usize = 10; 33 | 34 | #[tokio::main] 35 | async fn main() -> Result<(), Box> { 36 | let args = Args::parse(); 37 | 38 | env_logger::builder().format_timestamp_millis().init(); 39 | 40 | // Set up the commit log 41 | let cl = FileLog::new(args.cl_args.cl_root_path.clone()); 42 | 43 | // Establish channels for the database 44 | let (database_command_tx, database_command_rx) = mpsc::channel(MAX_DATABASE_MANAGER_COMMANDS); 45 | 46 | // Start up the http service 47 | let routes = http_server::routes(database_command_tx.clone()); 48 | tokio::spawn(warp::serve(routes).run(args.http_addr)); 49 | info!("HTTP listening on {}", args.http_addr); 50 | 51 | // Start up the UDP service 52 | let socket = UdpSocket::bind(args.udp_addr).await?; 53 | tokio::spawn(udp_server::task(socket, database_command_tx)); 54 | info!("UDP listening on {}", args.udp_addr); 55 | 56 | // All things started up but our database. We're running 57 | // that in our main task. 58 | 59 | info!("IoT service ready"); 60 | 61 | tokio::spawn(async { database::task(cl, database_command_rx).await }) 62 | .await 63 | .map_err(|e| e.into()) 64 | } 65 | -------------------------------------------------------------------------------- /examples/iot-service/src/udp_server.rs: -------------------------------------------------------------------------------- 1 | use log::debug; 2 | use serde::{Deserialize, Serialize}; 3 | use tokio::{net::UdpSocket, sync::mpsc}; 4 | 5 | use crate::database::{self, Command}; 6 | 7 | const MAX_DATAGRAM_SIZE: usize = 12; 8 | 9 | #[derive(Debug, Deserialize, Serialize)] 10 | struct TemperatureUpdated(u64, u32); 11 | 12 | pub async fn task(socket: UdpSocket, database_command_tx: mpsc::Sender) { 13 | let mut recv_buf = [0; MAX_DATAGRAM_SIZE]; 14 | while let Ok((len, _remote_addr)) = socket.recv_from(&mut recv_buf).await { 15 | if let Ok(event) = 16 | postcard::from_bytes::(&recv_buf[..len.min(MAX_DATAGRAM_SIZE)]) 17 | { 18 | debug!("Posting : {:?}", event); 19 | 20 | let _ = database_command_tx 21 | .send(Command::Post( 22 | event.0, 23 | database::Event::TemperatureRead(event.1), 24 | )) 25 | .await; 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /streambed-codec/.gitignore: -------------------------------------------------------------------------------- 1 | /test_data/ -------------------------------------------------------------------------------- /streambed-codec/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed-codec" 3 | description = "Codecs and a typed wrapped for streambed CommitLog" 4 | 5 | version.workspace = true 6 | edition.workspace = true 7 | rust-version.workspace = true 8 | license.workspace = true 9 | repository.workspace = true 10 | 11 | [dependencies] 12 | streambed.workspace = true 13 | rand.workspace = true 14 | ciborium.workspace = true 15 | serde.workspace = true 16 | futures-util.workspace = true 17 | async-stream.workspace = true 18 | 19 | 20 | [dev-dependencies] 21 | tokio = { workspace = true, features = ["full"] } 22 | streambed-logged.workspace = true 23 | streambed-confidant.workspace = true 24 | -------------------------------------------------------------------------------- /streambed-codec/README.md: -------------------------------------------------------------------------------- 1 | # Codec 2 | 3 | This crate provides a trait `Codec` and two (initial) implementations, `Cbor` and `CborEncrypted`. 4 | A `Codec` value is a convenient abstraction over the lower level serialisation and crypto functions 5 | in `streambed`. 6 | 7 | A `Codec` is also closely associated with a `CommitLog` and a `Topic` since values stored on the commit 8 | log under a given topic will all be encoded the same way. 9 | 10 | The type `LogAdapter` is provided to wrap a `Codec`, `CommitLog` and `Topic`. A `LogAdapter` carries 11 | a type parameter for the decoded log values. It can be viewed as a typed counterpart to `CommitLog` 12 | 13 | The `produce` method on `LogAdapter` accepts a typed value, encodes it and appends it to the log. 14 | The `history` method returns a `Stream` of typed values from the commit log. 15 | 16 | -------------------------------------------------------------------------------- /streambed-codec/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | 3 | use async_stream::stream; 4 | use futures_util::{Stream, StreamExt}; 5 | use rand::thread_rng; 6 | use serde::{de::DeserializeOwned, Serialize}; 7 | use std::{marker::PhantomData, pin::Pin, vec::Vec}; 8 | use streambed::{ 9 | commit_log::{Offset, ProducerRecord, Subscription, Topic}, 10 | decrypt_buf_with_secret, encrypt_struct_with_secret, get_secret_value, 11 | secret_store::SecretStore, 12 | }; 13 | 14 | pub use streambed::commit_log::{CommitLog, ProducerError}; 15 | 16 | /// Wraps a `CommitLog` and specializes it for a specific payload type. 17 | /// This adds the type, topic and the encoding and encryption scheme. 18 | #[derive(Debug)] 19 | pub struct LogAdapter { 20 | commit_log: L, 21 | codec: C, 22 | topic: Topic, 23 | marker: PhantomData, 24 | } 25 | 26 | /// Provides a method on `CommitLog` to specialize it for a payload type. 27 | pub trait CommitLogExt 28 | where 29 | Self: CommitLog + Sized, 30 | { 31 | /// Specialize this commit log for items of type `A` 32 | /// The topic and group names are given and a `Codec` 33 | /// for encoding and decoding values of type `A`. 34 | fn adapt( 35 | self, 36 | topic: impl Into, 37 | codec: impl Codec, 38 | ) -> LogAdapter, A> { 39 | LogAdapter { 40 | commit_log: self, 41 | codec, 42 | topic: topic.into(), 43 | marker: PhantomData, 44 | } 45 | } 46 | } 47 | 48 | impl CommitLogExt for L where L: CommitLog {} 49 | 50 | impl LogAdapter 51 | where 52 | L: CommitLog, 53 | C: Codec, 54 | A: 'static, 55 | { 56 | /// Send one item to the underlying commit log. 57 | pub async fn produce(&self, item: A) -> Result { 58 | let topic = self.topic.clone(); 59 | 60 | if let Some(value) = self.codec.encode(item) { 61 | self.commit_log 62 | .produce(ProducerRecord { 63 | topic, 64 | headers: Vec::new(), 65 | timestamp: None, 66 | key: 0, 67 | value, 68 | partition: 0, 69 | }) 70 | .await 71 | .map(|r| r.offset) 72 | } else { 73 | Err(ProducerError::CannotProduce) 74 | } 75 | } 76 | 77 | /// Return an async stream of items representing the 78 | /// history up to the time of the call. 79 | #[allow(clippy::needless_lifetimes)] 80 | pub async fn history<'a>(&'a self) -> Pin + 'a>> { 81 | let last_offset = self 82 | .commit_log 83 | .offsets(self.topic.clone(), 0) 84 | .await 85 | .map(|lo| lo.end_offset); 86 | let subscriptions = Vec::from([Subscription { 87 | topic: self.topic.clone(), 88 | }]); 89 | 90 | let mut records = 91 | self.commit_log 92 | .scoped_subscribe("EDFSM", Vec::new(), subscriptions, None); 93 | 94 | Box::pin(stream! { 95 | if let Some(last_offset) = last_offset { 96 | while let Some(mut r) = records.next().await { 97 | if r.offset <= last_offset { 98 | if let Some(item) = self.codec.decode(&mut r.value) { 99 | yield item; 100 | } 101 | if r.offset == last_offset { 102 | break; 103 | } 104 | } else { 105 | break; 106 | } 107 | } 108 | } 109 | }) 110 | } 111 | } 112 | 113 | /// A trait for codecs for a specic type `A`, usually an event type stored on a CommitLog. 114 | pub trait Codec { 115 | /// Encode a value. 116 | fn encode(&self, item: A) -> Option>; 117 | /// Decode a value. 118 | fn decode(&self, bytes: &mut [u8]) -> Option; 119 | } 120 | 121 | /// A `Codec` for encripted CBOR 122 | #[derive(Debug)] 123 | pub struct CborEncrypted { 124 | secret: String, 125 | } 126 | 127 | impl CborEncrypted { 128 | /// Create an encrypted CBOR codec with the given secret store. 129 | pub async fn new(secret_store: &S, secret_path: &str) -> Option 130 | where 131 | S: SecretStore, 132 | { 133 | Some(Self { 134 | secret: get_secret_value(secret_store, secret_path).await?, 135 | }) 136 | } 137 | } 138 | 139 | impl Codec for CborEncrypted 140 | where 141 | A: Serialize + DeserializeOwned + Send, 142 | { 143 | fn encode(&self, item: A) -> Option> { 144 | let serialize = |item: &A| { 145 | let mut buf = Vec::new(); 146 | ciborium::ser::into_writer(item, &mut buf).map(|_| buf) 147 | }; 148 | encrypt_struct_with_secret(self.secret.clone(), serialize, thread_rng, &item) 149 | } 150 | 151 | fn decode(&self, bytes: &mut [u8]) -> Option { 152 | decrypt_buf_with_secret(self.secret.clone(), bytes, |b| { 153 | ciborium::de::from_reader::(b) 154 | }) 155 | } 156 | } 157 | 158 | /// A `Codec` for CBOR 159 | #[derive(Debug)] 160 | pub struct Cbor; 161 | 162 | impl Codec for Cbor 163 | where 164 | A: Serialize + DeserializeOwned + Send, 165 | { 166 | fn encode(&self, item: A) -> Option> { 167 | let mut buf = Vec::new(); 168 | ciborium::ser::into_writer(&item, &mut buf).ok()?; 169 | Some(buf) 170 | } 171 | 172 | fn decode(&self, bytes: &mut [u8]) -> Option { 173 | ciborium::de::from_reader::(bytes).ok() 174 | } 175 | } 176 | 177 | #[cfg(test)] 178 | mod test { 179 | 180 | use crate::{Cbor, CborEncrypted, CommitLogExt}; 181 | use futures_util::StreamExt; 182 | use serde::{Deserialize, Serialize}; 183 | use std::{path::Path, time::Duration}; 184 | use streambed_confidant::FileSecretStore; 185 | use streambed_logged::FileLog; 186 | use tokio::{task::yield_now, time::sleep}; 187 | 188 | // use std::time::Duration; 189 | // use tokio::time::sleep; 190 | 191 | const TEST_DATA: &str = "test_data"; 192 | const TOPIC: &str = "event_series"; 193 | 194 | #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] 195 | pub enum Event { 196 | Num(u32), 197 | } 198 | 199 | fn fixture_store() -> FileSecretStore { 200 | todo!() 201 | } 202 | 203 | fn fixture_data() -> impl Iterator { 204 | (1..100).map(Event::Num) 205 | } 206 | 207 | #[tokio::test] 208 | async fn cbor_history() { 209 | cbor_produce().await; 210 | sleep(Duration::from_secs(1)).await; 211 | let mut data = fixture_data(); 212 | let log = FileLog::new(TEST_DATA).adapt::(TOPIC, Cbor); 213 | let mut history = log.history().await; 214 | while let Some(event) = history.next().await { 215 | println!("{event:?}"); 216 | assert_eq!(event, data.next().unwrap()); 217 | } 218 | assert!(data.next().is_none()); 219 | } 220 | 221 | async fn cbor_produce() { 222 | let topic_file = [TEST_DATA, TOPIC].join("/"); 223 | let _ = std::fs::remove_file(&topic_file); 224 | let _ = std::fs::create_dir(TEST_DATA); 225 | let log = FileLog::new(TEST_DATA).adapt::(TOPIC, Cbor); 226 | for e in fixture_data() { 227 | log.produce(e).await.expect("failed to produce a log entry"); 228 | } 229 | assert!(Path::new(&topic_file).exists()); 230 | drop(log); 231 | yield_now().await; 232 | } 233 | 234 | #[tokio::test] 235 | async fn cbor_produce_test() { 236 | cbor_produce().await; 237 | } 238 | 239 | #[tokio::test] 240 | #[ignore] 241 | async fn cbor_encrypted_history() { 242 | let codec = CborEncrypted::new(&fixture_store(), "secret_path") 243 | .await 244 | .unwrap(); 245 | let log = FileLog::new(TEST_DATA).adapt::(TOPIC, codec); 246 | let mut history = log.history().await; 247 | while let Some(event) = history.next().await { 248 | println!("{event:?}") 249 | } 250 | } 251 | 252 | #[tokio::test] 253 | #[ignore] 254 | async fn cbor_encrypted_produce() { 255 | let codec = CborEncrypted::new(&fixture_store(), "secret_path") 256 | .await 257 | .unwrap(); 258 | let log = FileLog::new(TEST_DATA).adapt::(TOPIC, codec); 259 | for i in 1..100 { 260 | let _ = log.produce(Event::Num(i)).await; 261 | } 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /streambed-confidant-cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed-confidant-cli" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "A CLI for a file-system-based secret store that applies streambed-crypto to data" 9 | 10 | [dependencies] 11 | base64 = { workspace = true } 12 | clap = { workspace = true } 13 | env_logger = { workspace = true } 14 | hex = { workspace = true } 15 | humantime = { workspace = true } 16 | rand = { workspace = true } 17 | serde_json = { workspace = true } 18 | streambed = { workspace = true } 19 | streambed-confidant = { workspace = true } 20 | tokio = { workspace = true, features = ["full"] } 21 | tokio-stream = { workspace = true } 22 | 23 | [[bin]] 24 | name = "confidant" 25 | path = "src/main.rs" 26 | -------------------------------------------------------------------------------- /streambed-confidant-cli/README.md: -------------------------------------------------------------------------------- 1 | confidant 2 | === 3 | 4 | The `confidant` command provides a utility for conveniently operating on file-based secret store. It is 5 | often used in conjunction `logged` to encrypt payloads as a pre-stage, or decrypt as a post-stage. No 6 | assumption is made regarding the nature of a payload beyond it being encrypted using streambed crypto 7 | functions. 8 | 9 | Running with an example encrypt followed by a decrypt 10 | --- 11 | 12 | First get the executable: 13 | 14 | ``` 15 | cargo install streambed-confidant-cli 16 | ``` 17 | 18 | ...or build it from this repo: 19 | 20 | ``` 21 | cargo build --bin confidant --release 22 | ``` 23 | 24 | ...and make a build available on the PATH (only for `cargo build` and just once per shell session): 25 | 26 | ``` 27 | export PATH="$PWD/target/release":$PATH 28 | ``` 29 | 30 | Before you can use `confidant`, you must provide it with a root secret and a "secret id" (password) 31 | to authenticate the session. Here's an example with some dummy data: 32 | 33 | ``` 34 | echo "1800af9b273e4b9ea71ec723426933a4" > /tmp/root-secret 35 | ``` 36 | 37 | We also need to create a directory for `confidant` to read and write its secrets. A security feature 38 | of the `confidant` library is that the directory must have a permission of `600` for the owner user. 39 | ACLs should then be used to control access for individual processes. Here's how the directory can be 40 | created: 41 | 42 | ``` 43 | mkdir /tmp/confidant 44 | chmod 700 /tmp/confidant 45 | ``` 46 | 47 | You would normally source the above secrets from your production system, preferably without 48 | them leaving your production system. 49 | 50 | Given the root secret, encrypt some data : 51 | 52 | ``` 53 | echo '{"value":"SGkgdGhlcmU="}' | \ 54 | confidant --root-path=/tmp/confidant encrypt --file - --path="default/my-secret-path" 55 | ``` 56 | 57 | ...which would output: 58 | 59 | ``` 60 | {"value":"EZy4HLnFC4c/W63Qtp288WWFj8U="} 61 | ``` 62 | 63 | That value is now encrypted with a salt. 64 | 65 | We can also decrypt in a similar fashion: 66 | 67 | ``` 68 | echo '{"value":"EZy4HLnFC4c/W63Qtp288WWFj8U="}' | \ 69 | confidant --root-path=/tmp/confidant decrypt --file - --path="default/my-secret-path" 70 | ``` 71 | 72 | ...which will yield the original BASE64 value that we encrypted. 73 | 74 | Use `--help` to discover all of the options. 75 | -------------------------------------------------------------------------------- /streambed-confidant-cli/src/cryptor.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::HashMap, 3 | io::{self, Write}, 4 | time::Duration, 5 | }; 6 | 7 | use rand::RngCore; 8 | use serde_json::Value; 9 | use streambed::{ 10 | crypto::{self, SALT_SIZE}, 11 | get_secret_value, 12 | secret_store::{SecretData, SecretStore}, 13 | }; 14 | use tokio::{sync::mpsc::channel, time}; 15 | 16 | use crate::errors::Errors; 17 | 18 | const FLUSH_DELAY: Duration = Duration::from_millis(100); 19 | const OUTPUT_QUEUE_SIZE: usize = 10; 20 | 21 | pub async fn process_records( 22 | ss: impl SecretStore, 23 | mut line_reader: impl FnMut() -> Result, io::Error>, 24 | mut output: impl Write, 25 | path: &str, 26 | process: fn(Vec, Vec) -> Option>, 27 | select: &str, 28 | ) -> Result<(), Errors> { 29 | let (output_tx, mut output_rx) = channel(OUTPUT_QUEUE_SIZE); 30 | 31 | let processor = async move { 32 | while let Some(line) = line_reader()? { 33 | let mut record = serde_json::from_str::(&line).map_err(Errors::from)?; 34 | let Some(value) = record.get_mut(select) else { 35 | return Err(Errors::CannotSelectValue); 36 | }; 37 | let Some(str_value) = value.as_str() else { 38 | return Err(Errors::CannotGetValue); 39 | }; 40 | let Ok(bytes) = base64::decode(str_value) else { 41 | return Err(Errors::CannotDecodeValue); 42 | }; 43 | let decrypted_bytes = get_secret_value(&ss, path) 44 | .await 45 | .and_then(|secret_value| { 46 | let key = hex::decode(secret_value).ok()?; 47 | process(key, bytes) 48 | }) 49 | .ok_or(Errors::CannotDecryptValue)?; 50 | let encoded_decrypted_str = base64::encode(decrypted_bytes); 51 | *value = Value::String(encoded_decrypted_str); 52 | let _ = output_tx.send(record).await; 53 | } 54 | 55 | Ok(()) 56 | }; 57 | 58 | let outputter = async move { 59 | let mut timeout = FLUSH_DELAY; 60 | loop { 61 | tokio::select! { 62 | record = output_rx.recv() => { 63 | if let Some(record) = record { 64 | let buf = serde_json::to_string(&record)?; 65 | output.write_all(buf.as_bytes())?; 66 | output.write_all(b"\n")?; 67 | timeout = FLUSH_DELAY; 68 | } else { 69 | break; 70 | } 71 | } 72 | _ = time::sleep(timeout) => { 73 | output.flush()?; 74 | timeout = Duration::MAX; 75 | } 76 | } 77 | } 78 | Ok(()) 79 | }; 80 | 81 | tokio::try_join!(processor, outputter).map(|_| ()) 82 | } 83 | 84 | pub async fn decrypt( 85 | ss: impl SecretStore, 86 | line_reader: impl FnMut() -> Result, io::Error>, 87 | output: impl Write, 88 | path: &str, 89 | select: &str, 90 | ) -> Result<(), Errors> { 91 | fn process(key: Vec, mut bytes: Vec) -> Option> { 92 | let (salt, data_bytes) = bytes.split_at_mut(crypto::SALT_SIZE); 93 | crypto::decrypt(data_bytes, &key.try_into().ok()?, &salt.try_into().ok()?); 94 | Some(data_bytes.to_vec()) 95 | } 96 | process_records(ss, line_reader, output, path, process, select).await 97 | } 98 | 99 | pub async fn encrypt( 100 | ss: impl SecretStore, 101 | line_reader: impl FnMut() -> Result, io::Error>, 102 | output: impl Write, 103 | path: &str, 104 | select: &str, 105 | ) -> Result<(), Errors> { 106 | // As a convenience, we create the secret when encrypting if there 107 | // isn't one. 108 | if get_secret_value(&ss, path).await.is_none() { 109 | let mut key = vec![0; 16]; 110 | rand::thread_rng().fill_bytes(&mut key); 111 | let data = HashMap::from([("value".to_string(), hex::encode(key))]); 112 | ss.create_secret(path, SecretData { data }) 113 | .await 114 | .map_err(Errors::SecretStore)?; 115 | } 116 | 117 | fn process(key: Vec, mut data_bytes: Vec) -> Option> { 118 | let salt = crypto::salt(&mut rand::thread_rng()); 119 | crypto::encrypt(&mut data_bytes, &key.try_into().ok()?, &salt); 120 | let mut buf = Vec::with_capacity(SALT_SIZE + data_bytes.len()); 121 | buf.extend(salt); 122 | buf.extend(data_bytes); 123 | Some(buf) 124 | } 125 | process_records(ss, line_reader, output, path, process, select).await 126 | } 127 | -------------------------------------------------------------------------------- /streambed-confidant-cli/src/errors.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | error::Error, 3 | fmt::{self, Debug}, 4 | io, 5 | }; 6 | 7 | use streambed::secret_store; 8 | 9 | #[derive(Debug)] 10 | pub enum Errors { 11 | CannotDecodeRootSecretAsHex, 12 | CannotDecodeValue, 13 | CannotDecryptValue, 14 | CannotEncryptValue, 15 | CannotGetValue, 16 | CannotSelectValue, 17 | EmptyRootSecretFile, 18 | EmptySecretIdFile, 19 | InvalidRootSecret, 20 | InvalidSaltLen, 21 | Io(io::Error), 22 | RootSecretFileIo(io::Error), 23 | SecretIdFileIo(io::Error), 24 | SecretStore(secret_store::Error), 25 | Serde(serde_json::Error), 26 | } 27 | 28 | impl From for Errors { 29 | fn from(value: io::Error) -> Self { 30 | Self::Io(value) 31 | } 32 | } 33 | 34 | impl From for Errors { 35 | fn from(value: serde_json::Error) -> Self { 36 | Self::Serde(value) 37 | } 38 | } 39 | 40 | impl fmt::Display for Errors { 41 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 42 | match self { 43 | Self::CannotDecodeRootSecretAsHex => { 44 | f.write_str("Cannot decode the root-secret as hex") 45 | } 46 | Self::CannotDecodeValue => { 47 | f.write_str("Cannot decode the selected value of the JSON object") 48 | } 49 | Self::CannotDecryptValue => { 50 | f.write_str("Cannot decrypt the selected value of the JSON object") 51 | } 52 | Self::CannotEncryptValue => { 53 | f.write_str("Cannot encrypt the selected value of the JSON object") 54 | } 55 | Self::CannotGetValue => { 56 | f.write_str("Cannot get the selected value of the selected field") 57 | } 58 | Self::CannotSelectValue => { 59 | f.write_str("Cannot select the value within the JSON object") 60 | } 61 | Self::EmptyRootSecretFile => f.write_str("Empty root-secret file"), 62 | Self::EmptySecretIdFile => f.write_str("Empty secret-id file"), 63 | Self::InvalidRootSecret => { 64 | f.write_str("Invalid root secret - must be a hex string of 32 characters") 65 | } 66 | Self::InvalidSaltLen => f.write_str("Invalid salt length - must be 12 bytes"), 67 | Self::Io(e) => fmt::Display::fmt(&e, f), 68 | Self::SecretStore(_) => f.write_str("Unauthorised access"), 69 | Self::RootSecretFileIo(_) => f.write_str("root-secret file problem"), 70 | Self::SecretIdFileIo(_) => f.write_str("secret-id file problem"), 71 | Self::Serde(e) => fmt::Display::fmt(&e, f), 72 | } 73 | } 74 | } 75 | 76 | impl Error for Errors { 77 | fn source(&self) -> Option<&(dyn Error + 'static)> { 78 | match self { 79 | Self::CannotDecodeValue 80 | | Self::CannotDecodeRootSecretAsHex 81 | | Self::CannotDecryptValue 82 | | Self::CannotEncryptValue 83 | | Self::CannotGetValue 84 | | Self::CannotSelectValue 85 | | Self::EmptyRootSecretFile 86 | | Self::EmptySecretIdFile 87 | | Self::InvalidRootSecret 88 | | Self::InvalidSaltLen 89 | | Self::SecretStore(_) => None, 90 | Self::Io(e) => e.source(), 91 | Self::RootSecretFileIo(e) => e.source(), 92 | Self::SecretIdFileIo(e) => e.source(), 93 | Self::Serde(e) => e.source(), 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /streambed-confidant-cli/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | error::Error, 3 | fs::{self, File}, 4 | io::{self, BufRead, BufReader, BufWriter, Read, Write}, 5 | path::{Path, PathBuf}, 6 | time::Duration, 7 | }; 8 | 9 | use clap::{Args, Parser, Subcommand}; 10 | use errors::Errors; 11 | use streambed::secret_store::SecretStore; 12 | use streambed_confidant::FileSecretStore; 13 | 14 | pub mod cryptor; 15 | pub mod errors; 16 | 17 | /// The `confidant` command provides a utility for conveniently operating on file-based secret store. It is 18 | /// often used in conjunction `logged` to encrypt payloads as a pre-stage, or decrypt as a post-stage. JSON objects 19 | /// followed by a newline are expected to be streamed in and out. The value to be encrypted/decypted is defaulted to a 20 | /// field named "value", which can be altered via the `select` option. This value is expected to be encoded as BASE64 and 21 | /// will output as encoded BASE64. No assumption is made regarding the nature of a value passed in beyond it to be 22 | /// encrypted/decrypted using streambed crypto functions. 23 | #[derive(Parser, Debug)] 24 | #[clap(author, about, long_about = None, version)] 25 | struct ProgramArgs { 26 | /// In order to initialise the secret store, a root secret is also required. A credentials-directory path can be provided 27 | /// where a `root-secret`` file is expected. This argument corresponds conveniently with systemd's CREDENTIALS_DIRECTORY 28 | /// environment variable and is used by various services we have written. 29 | /// Also associated with this argument is an optional "secret id" file` for role-based authentication with the secret store. 30 | /// This secret is expected to be found in a ss-secret-id file of the directory and, if not provided, will default to 31 | /// an "unusedid" value. 32 | #[clap(env, long, default_value = "/tmp")] 33 | pub credentials_directory: PathBuf, 34 | 35 | /// The max number of Vault Secret Store secrets to retain by our cache at any time. 36 | /// Least Recently Used (LRU) secrets will be evicted from our cache once this value 37 | /// is exceeded. 38 | #[clap(env, long, default_value_t = 10_000)] 39 | pub max_secrets: usize, 40 | 41 | /// The Secret Store role_id to use for approle authentication. 42 | #[clap(env, long, default_value = "streambed-confidant-cli")] 43 | pub role_id: String, 44 | 45 | /// The location of all secrets belonging to confidant. The recommendation is to 46 | /// create a user for confidant and a requirement is to remove group and world permissions. 47 | /// Then, use ACLs to express further access conditions. 48 | #[clap(env, long, default_value = "/var/lib/confidant")] 49 | pub root_path: PathBuf, 50 | 51 | /// A data field to used in place of Vault's lease_duration field. Time 52 | /// will be interpreted as a humantime string e.g. "1m", "1s" etc. Note 53 | /// that v2 of the Vault server does not appear to populate the lease_duration 54 | /// field for the KV secret store any longer. Instead, we can use a "ttl" field 55 | /// from the data. 56 | #[clap(env, long)] 57 | pub ttl_field: Option, 58 | 59 | /// How long we wait until re-requesting the Secret Store for an 60 | /// unauthorized secret again. 61 | #[clap(env, long, default_value = "1m")] 62 | pub unauthorized_timeout: humantime::Duration, 63 | 64 | #[command(subcommand)] 65 | pub command: Command, 66 | } 67 | 68 | #[derive(Subcommand, Debug)] 69 | enum Command { 70 | Decrypt(DecryptCommand), 71 | Encrypt(EncryptCommand), 72 | } 73 | 74 | /// Consume a JSON object followed by a newline character, from a stream until EOF and decrypt it. 75 | #[derive(Args, Debug)] 76 | struct DecryptCommand { 77 | /// The file to consume JSON objects with newlines from, or `-` to indicate STDIN. 78 | #[clap(short, long)] 79 | pub file: PathBuf, 80 | 81 | /// The max number utf-8 characters to read from the file. 82 | #[clap(long, default_value_t = 8 * 1024)] 83 | pub max_line_len: u64, 84 | 85 | /// By default, records are output to STDOUT as JSON followed newlines. 86 | /// This option can be used to write to a file. 87 | #[clap(short, long)] 88 | pub output: Option, 89 | 90 | /// The JSON field to select that holds the BASE64 value for decryption. 91 | /// The first 12 bytes of a decoded BASE64 value is expected to be a salt 92 | /// value. 93 | #[clap(long, default_value = "value")] 94 | pub select: String, 95 | 96 | /// The path to the secret e.g. "default/secrets.configurator-events.key" 97 | #[clap(long)] 98 | pub path: String, 99 | } 100 | 101 | /// Consume a JSON object followed by a newline character, from a stream until EOF and encrypt it. 102 | #[derive(Args, Debug)] 103 | struct EncryptCommand { 104 | /// The file to consume JSON objects with newlines from, or `-` to indicate STDIN. 105 | #[clap(short, long)] 106 | pub file: PathBuf, 107 | 108 | /// The max number utf-8 characters to read from the file. 109 | #[clap(long, default_value_t = 8 * 1024)] 110 | pub max_line_len: u64, 111 | 112 | /// By default, records are output to STDOUT as a JSON followed newlines. 113 | /// This option can be used to write to a file. 114 | #[clap(short, long)] 115 | pub output: Option, 116 | 117 | /// The JSON field to select that holds the BASE64 value for encryption. 118 | /// The first 12 bytes of a decoded BASE64 value is expected to be a salt 119 | /// value. 120 | #[clap(long, default_value = "value")] 121 | pub select: String, 122 | 123 | /// The path to the secret e.g. "default/secrets.configurator-events.key". 124 | /// NOTE: as a convenience, in the case where the there is no secret at 125 | /// this path, then one will be attempted to be created. 126 | #[clap(long)] 127 | pub path: String, 128 | } 129 | 130 | async fn secret_store( 131 | credentials_directory: PathBuf, 132 | max_secrets: usize, 133 | root_path: PathBuf, 134 | role_id: String, 135 | ttl_field: Option<&str>, 136 | unauthorized_timeout: Duration, 137 | ) -> Result { 138 | let ss = { 139 | let (root_secret, ss_secret_id) = { 140 | let f = File::open(credentials_directory.join("root-secret")) 141 | .map_err(Errors::RootSecretFileIo)?; 142 | let f = BufReader::new(f); 143 | let root_secret = f 144 | .lines() 145 | .next() 146 | .ok_or(Errors::EmptyRootSecretFile)? 147 | .map_err(Errors::RootSecretFileIo)?; 148 | 149 | let ss_secret_id = if Path::exists(&credentials_directory.join("ss-secret-id")) { 150 | let f = File::open(credentials_directory.join("ss-secret-id")) 151 | .map_err(Errors::SecretIdFileIo)?; 152 | let f = BufReader::new(f); 153 | f.lines() 154 | .next() 155 | .ok_or(Errors::EmptyRootSecretFile)? 156 | .map_err(Errors::SecretIdFileIo)? 157 | } else { 158 | String::from("unusedid") 159 | }; 160 | 161 | (root_secret, ss_secret_id) 162 | }; 163 | 164 | let root_secret = 165 | hex::decode(root_secret).map_err(|_| Errors::CannotDecodeRootSecretAsHex)?; 166 | 167 | let ss = FileSecretStore::new( 168 | root_path, 169 | &root_secret 170 | .try_into() 171 | .map_err(|_| Errors::InvalidRootSecret)?, 172 | unauthorized_timeout, 173 | max_secrets, 174 | ttl_field, 175 | ); 176 | 177 | ss.approle_auth(&role_id, &ss_secret_id) 178 | .await 179 | .map_err(Errors::SecretStore)?; 180 | 181 | ss 182 | }; 183 | Ok(ss) 184 | } 185 | 186 | type LineReaderFn = Box Result, io::Error> + Send>; 187 | type OutputFn = Box; 188 | 189 | fn pipeline( 190 | file: PathBuf, 191 | output: Option, 192 | max_line_len: u64, 193 | ) -> Result<(LineReaderFn, OutputFn), Errors> { 194 | // A line reader function is provided rather than something based on the 195 | // Read trait as reading a line from stdin involves locking it via its 196 | // own read_line function. A Read trait will cause a lock to occur for 197 | // each access of the Read methods, and we cannot be guaranteed which of those will 198 | // be used - serde, for example, may only read a byte at a time. This would cause 199 | // many locks. 200 | let line_reader: LineReaderFn = if file.as_os_str() == "-" { 201 | let stdin = io::stdin(); 202 | let mut line = String::new(); 203 | Box::new(move || { 204 | line.clear(); 205 | if stdin.lock().take(max_line_len).read_line(&mut line)? > 0 { 206 | Ok(Some(line.clone())) 207 | } else { 208 | Ok(None) 209 | } 210 | }) 211 | } else { 212 | let mut buf = 213 | BufReader::new(fs::File::open(file).map_err(Errors::from)?).take(max_line_len); 214 | let mut line = String::new(); 215 | Box::new(move || { 216 | line.clear(); 217 | if buf.read_line(&mut line)? > 0 { 218 | Ok(Some(line.clone())) 219 | } else { 220 | Ok(None) 221 | } 222 | }) 223 | }; 224 | let output: OutputFn = if let Some(output) = output { 225 | Box::new(BufWriter::new( 226 | fs::File::create(output).map_err(Errors::from)?, 227 | )) 228 | } else { 229 | Box::new(io::stdout()) 230 | }; 231 | Ok((line_reader, output)) 232 | } 233 | 234 | #[tokio::main] 235 | async fn main() -> Result<(), Box> { 236 | let args = ProgramArgs::parse(); 237 | 238 | env_logger::builder().format_timestamp_millis().init(); 239 | 240 | let ss = secret_store( 241 | args.credentials_directory, 242 | args.max_secrets, 243 | args.root_path, 244 | args.role_id, 245 | args.ttl_field.as_deref(), 246 | args.unauthorized_timeout.into(), 247 | ) 248 | .await?; 249 | 250 | let task = tokio::spawn(async move { 251 | match args.command { 252 | Command::Decrypt(command) => { 253 | let (line_reader, output) = 254 | pipeline(command.file, command.output, command.max_line_len)?; 255 | cryptor::decrypt(ss, line_reader, output, &command.path, &command.select).await 256 | } 257 | Command::Encrypt(command) => { 258 | let (line_reader, output) = 259 | pipeline(command.file, command.output, command.max_line_len)?; 260 | cryptor::encrypt(ss, line_reader, output, &command.path, &command.select).await 261 | } 262 | } 263 | }); 264 | 265 | task.await 266 | .map_err(|e| e.into()) 267 | .and_then(|r: Result<(), Errors>| r.map_err(|e| e.into())) 268 | } 269 | -------------------------------------------------------------------------------- /streambed-confidant/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed-confidant" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "Confidant is a small library that implements a file-system-based secret store" 9 | 10 | [dependencies] 11 | async-trait = { workspace = true } 12 | base64 = { workspace = true } 13 | cache_loader_async = { workspace = true, features = ["lru-cache", "ttl-cache"] } 14 | clap = { workspace = true, features = ["derive", "env"] } 15 | hex = { workspace = true } 16 | humantime = { workspace = true } 17 | postcard = { workspace = true, default-features = false, features = ["use-std"] } 18 | rand = { workspace = true } 19 | serde = { workspace = true, features = ["derive"] } 20 | serde_json = { workspace = true } 21 | streambed = { workspace = true } 22 | tokio = { workspace = true, features = [ "fs", "io-util" ] } 23 | 24 | [dev-dependencies] 25 | env_logger = { workspace = true } 26 | hex = { workspace = true } 27 | log = { workspace = true } 28 | test-log = { workspace = true } 29 | -------------------------------------------------------------------------------- /streambed-confidant/README.md: -------------------------------------------------------------------------------- 1 | # confidant 2 | 3 | Confidant is a small library that implements a file-system-based 4 | secret store function for autonomous systems that often live 5 | at the edge of a wider network. 6 | 7 | Confidant implements the Streambed secret store API and stored 8 | secrets are encrypted. 9 | 10 | ## An quick introduction 11 | 12 | Nothing beats code for a quick introduction! Here is an example of 13 | writing a secret and retrieving it. Please refer to the various tests 14 | for more complete examples. 15 | 16 | ```rs 17 | // Let's set up the correct permissions for where all secrets will live 18 | fs::set_permissions(&confidant_dir, PermissionsExt::from_mode(0o700)) 19 | .await 20 | .unwrap(); 21 | 22 | let ss = FileSecretStore::new( 23 | confidant_dir.clone(), 24 | &[0; crypto::KEY_SIZE], // A key to encrypt the stored secrets with 25 | Duration::from_secs(1), // Timeout for unauthorized secrets before we try again 26 | 10, // The number of secrets we cache 27 | None, // A data field to be used to indicate a TTL - if any 28 | ); 29 | 30 | let mut data = HashMap::new(); 31 | data.insert("key".to_string(), "value".to_string()); 32 | let data = SecretData { data }; 33 | 34 | // Write the secret out. 35 | assert!(ss.create_secret("some.secret", data.clone()).await.is_ok()); 36 | 37 | // Read the secret 38 | assert!(ss.get_secret("some.secret").await.unwrap().is_some()); 39 | ``` 40 | 41 | ## Why confidant? 42 | 43 | The primary functional use-cases of confidant are: 44 | 45 | * retrieve and store a user's secrets (user meaning operating system 46 | user); and 47 | * share secrets with other users. 48 | 49 | The primary operational use-cases of confidant are: 50 | 51 | * to be hosted by resource-constrained devices, typically with less 52 | than 128MiB memory and 8GB of storage. 53 | 54 | ## What is confidant? 55 | 56 | ### No networking 57 | 58 | Confidant has no notion of what a network is and relies on the 59 | file system along with operating system permissions 60 | 61 | ### No authentication or authorization 62 | 63 | ...over and above what the operating system provides. 64 | 65 | ### The Vault model 66 | 67 | Confidant is modelled with the same concepts as Hashicorp Vault. 68 | Services using confidant may therefore lend themselves to portability 69 | toward Vault. 70 | 71 | ## How is confidant implemented? 72 | 73 | Confidant is implemented as a library avoiding the need for a server process. 74 | Please note that we had no requirements to serve Windows and wanted to leverage 75 | Unix file permissions explicitly. When a secret is written is to written with 76 | the same mode as the directory given to Confidant when instantiated. This then 77 | ensures that the same permissions, including ACLs, are passed to each secret 78 | file. 79 | 80 | The file system is used to store secrets and the host operating system 81 | permissions, including users, groups and ACLs, are leverage. 82 | [Tokio](https://tokio.rs/) is used for file read/write operations so that any stalled 83 | operations permit other tasks to continue running. [Postcard](https://docs.rs/postcard/latest/postcard/) 84 | is used for serialization as it is able to conveniently represent in-memory structures 85 | and is optimized for resource-constrained targets. 86 | 87 | A TTL cache is maintained so that IO is minimized for those secrets that are often retrieved. 88 | 89 | ## When should you not use confidant 90 | 91 | When you have Hashicorp Vault. -------------------------------------------------------------------------------- /streambed-confidant/examples/README.md: -------------------------------------------------------------------------------- 1 | # confidant authenticator 2 | 3 | This example illustrates how confidant can be used to set up a service, create a user 4 | and then authenticate against that user. 5 | 6 | To run, first create a directory that confidant can use to store its encrypted secrets: 7 | 8 | ``` 9 | mkdir /tmp/confidant 10 | chmod 700 /tmp/confidant 11 | ``` 12 | 13 | The service can then be invoked: 14 | 15 | ``` 16 | echo -n "01234567890123456789012345678912some-secret-id" | \ 17 | RUST_LOG=debug cargo run --example confidant -- \ 18 | --ss-role-id="my-great-service" \ 19 | --ss-root-path="/tmp/confidant" 20 | ``` 21 | 22 | You should then receive an output like: 23 | 24 | ``` 25 | [2022-12-20T14:27:48Z INFO main] Confidant authenticator started 26 | [2022-12-20T14:27:48Z INFO main] User created 27 | [2022-12-20T14:27:48Z INFO main] User pass auth token: UserPassAuthReply { auth: AuthToken { client_token: "eyJkYXRhIjp7InVzZXJuYW1lIjoibWl0Y2hlbGxoIiwiZXhwaXJlcyI6MTY3MjE1MTI2ODc3Nn0sInNpZ25hdHVyZSI6ImJjYTdmZTY4ZWI0NDcwN2QwMGI1MmUzZjgwZDE4MmFlMTcxYjE5MDZkYzdjYTJjZTUwN2EwNTk2OTY0MDQ4MGMifQ==", lease_duration: 604800 } } 28 | [2022-12-20T14:27:48Z INFO main] Token auth result: Ok(()) 29 | ``` -------------------------------------------------------------------------------- /streambed-confidant/examples/confidant.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use clap::Parser; 4 | use log::info; 5 | use streambed::secret_store::SecretStore; 6 | use streambed_confidant::{args::SsArgs, FileSecretStore}; 7 | 8 | /// Service CLI that brings in the args we need for configuring access to a secret store 9 | #[derive(Parser, Debug)] 10 | #[clap(author, about, long_about = None, version)] 11 | struct Args { 12 | #[clap(flatten)] 13 | ss_args: SsArgs, 14 | } 15 | 16 | #[tokio::main] 17 | async fn main() -> Result<(), Box> { 18 | let args = Args::parse(); 19 | 20 | env_logger::init(); 21 | info!("Confidant authenticator started"); 22 | 23 | // Setup and authenticate our service with the secret store 24 | 25 | let line = streambed::read_line(std::io::stdin()).unwrap(); 26 | assert!(!line.is_empty(), "Cannot source a line from stdin"); 27 | let (root_secret, ss_secret_id) = line.split_at(32); 28 | let root_secret = hex::decode(root_secret).unwrap(); 29 | assert!( 30 | !ss_secret_id.is_empty(), 31 | "Cannot source a secret id from stdin" 32 | ); 33 | 34 | let ss = FileSecretStore::new( 35 | args.ss_args.ss_root_path, 36 | &root_secret.try_into().unwrap(), 37 | args.ss_args.ss_unauthorized_timeout.into(), 38 | args.ss_args.ss_max_secrets, 39 | args.ss_args.ss_ttl_field.as_deref(), 40 | ); 41 | 42 | ss.approle_auth(&args.ss_args.ss_role_id, ss_secret_id) 43 | .await 44 | .unwrap(); 45 | 46 | // We now create a user so that we can authenticate with it later 47 | 48 | if ss 49 | .userpass_create_update_user("mitchellh", "mitchellh", "foo") 50 | .await 51 | .is_ok() 52 | { 53 | info!("User created"); 54 | 55 | // Now authenticate as a human i.e. with a username and password. These credentials 56 | // would normally come from the outside e.g. via an http login. 57 | // To authenticate, we base a new store on our existing one and authenticate again. 58 | // The idea is that we create and drop secret store instances in association with 59 | // user sessions. A secret store instance will share resources with the other 60 | // instances it has been associated with, and have its own cache of secrets. 61 | 62 | let user_ss = FileSecretStore::with_new_auth_prepared(&ss); 63 | let auth_token = user_ss.userpass_auth("mitchellh", "foo").await.unwrap(); 64 | info!("User pass auth token: {:?}", auth_token); 65 | 66 | // Now suppose that we have a token received from the outside world. We establish 67 | // a new token store and then authenticate the token against it. If we choose to 68 | // store secret stores in relation to the same user id, then we could also do that 69 | // and share secret key caching. For now though, we'll just create a new from from 70 | // our service's secret store. 71 | 72 | let user_ss = FileSecretStore::with_new_auth_prepared(&ss); 73 | let result = user_ss.token_auth(&auth_token.auth.client_token).await; 74 | info!("Token auth result: {:?}", result); 75 | } 76 | 77 | Ok(()) 78 | } 79 | -------------------------------------------------------------------------------- /streambed-confidant/src/args.rs: -------------------------------------------------------------------------------- 1 | //! Provides command line arguments that are typically used for all services using this module. 2 | 3 | use std::path::PathBuf; 4 | 5 | use clap::Parser; 6 | 7 | #[derive(Parser, Debug)] 8 | pub struct SsArgs { 9 | /// The max number of Secret Store secrets to retain by our cache at any time. 10 | /// Least Recently Used (LRU) secrets will be evicted from our cache once this value 11 | /// is exceeded. 12 | #[clap(env, long, default_value_t = 10_000)] 13 | pub ss_max_secrets: usize, 14 | 15 | /// A namespace to use when communicating with the Vault Secret Store 16 | #[clap(env, long, default_value = "default")] 17 | pub ss_ns: String, 18 | 19 | /// The Secret Store role_id to use for approle authentication. 20 | #[clap(env, long)] 21 | pub ss_role_id: String, 22 | 23 | /// The location of all secrets belonging to confidant. The recommendation is to 24 | /// create a user for confidant and a requirement is to remove group and world permissions. 25 | /// Then, use ACLs to express further access conditions. 26 | #[clap(env, long, default_value = "/var/lib/confidant")] 27 | pub ss_root_path: PathBuf, 28 | 29 | /// A data field to used in place of Vault's lease_duration field. Time 30 | /// will be interpreted as a humantime string e.g. "1m", "1s" etc. Note 31 | /// that v2 of the Vault server does not appear to populate the lease_duration 32 | /// field for the KV secret store any longer. Instead, we can use a "ttl" field 33 | /// from the data. 34 | #[clap(env, long)] 35 | pub ss_ttl_field: Option, 36 | 37 | /// How long we wait until re-requesting the Vault Secret Store server for an 38 | /// unauthorized secret again. 39 | #[clap(env, long, default_value = "1m")] 40 | pub ss_unauthorized_timeout: humantime::Duration, 41 | } 42 | -------------------------------------------------------------------------------- /streambed-kafka/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed-kafka" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "Provides commit log functionality to connect with the Kafka HTTP API" 9 | 10 | [dependencies] 11 | async-stream = { workspace = true } 12 | async-trait = { workspace = true } 13 | clap = { workspace = true, features = ["derive", "env"] } 14 | humantime = { workspace = true } 15 | log = { workspace = true } 16 | metrics = { workspace = true, default-features = false } 17 | reqwest = { workspace = true, features = ["json"] } 18 | serde = { workspace = true, features = ["derive"] } 19 | serde_json = { workspace = true } 20 | smol_str = { workspace = true, features = ["serde"] } 21 | streambed = { workspace = true, features = ["reqwest"] } 22 | tokio = { workspace = true } 23 | tokio-stream = { workspace = true } 24 | 25 | [dev-dependencies] 26 | env_logger ={ workspace = true } 27 | http = { workspace = true } 28 | hyper = { workspace = true, default-features = false, features = ["tcp", "stream", "http1", "http2", "client", "server", "runtime"] } 29 | streambed-test = { workspace = true } 30 | test-log = { workspace = true } 31 | -------------------------------------------------------------------------------- /streambed-kafka/README.md: -------------------------------------------------------------------------------- 1 | # kafka 2 | 3 | Provides commit log functionality to connect with the Kafka HTTP API, with one difference... 4 | Kafka's HTTP API does not appear to provide a means of streaming consumer records. We have 5 | therefore introduced a new API call to subscribe to topics and consume their events by 6 | assuming that the server will keep the connection open. -------------------------------------------------------------------------------- /streambed-kafka/src/args.rs: -------------------------------------------------------------------------------- 1 | //! Provides command line arguments that are typically used for all services using this module. 2 | 3 | use std::path::PathBuf; 4 | 5 | use clap::Parser; 6 | use reqwest::Url; 7 | 8 | #[derive(Parser, Debug)] 9 | pub struct CommitLogArgs { 10 | /// The amount of time to indicate that no more events are immediately 11 | /// available from the Commit Log endpoint 12 | #[clap(env, long, default_value = "100ms")] 13 | pub cl_idle_timeout: humantime::Duration, 14 | 15 | /// A namespace to use when communicating with the Commit Log 16 | #[clap(env, long, default_value = "default")] 17 | pub cl_ns: String, 18 | 19 | /// A URL of the Commit Log RESTful Kafka server to communicate with 20 | #[clap(env, long, default_value = "http://localhost:8082")] 21 | pub cl_server: Url, 22 | 23 | /// A path to a TLS cert pem file to be used for connecting with the Commit Log server. 24 | #[clap(env, long)] 25 | pub cl_server_cert_path: Option, 26 | 27 | /// Insecurely trust the Commit Log's TLS certificate 28 | #[clap(env, long)] 29 | pub cl_server_insecure: bool, 30 | } 31 | -------------------------------------------------------------------------------- /streambed-kafka/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | 3 | pub mod args; 4 | 5 | use std::collections::HashMap; 6 | use std::pin::Pin; 7 | use std::time::Duration; 8 | 9 | use async_stream::stream; 10 | use async_trait::async_trait; 11 | use log::{debug, trace}; 12 | use metrics::increment_counter; 13 | use reqwest::Certificate; 14 | use reqwest::{Client, Url}; 15 | use serde::Deserialize; 16 | use serde::Serialize; 17 | use streambed::commit_log::CommitLog; 18 | use streambed::commit_log::Consumer; 19 | use streambed::commit_log::ConsumerOffset; 20 | use streambed::commit_log::ConsumerRecord; 21 | use streambed::commit_log::PartitionOffsets; 22 | use streambed::commit_log::ProducedOffset; 23 | use streambed::commit_log::ProducerError; 24 | use streambed::commit_log::ProducerRecord; 25 | use streambed::commit_log::Subscription; 26 | use streambed::commit_log::Topic; 27 | use streambed::delayer::Delayer; 28 | use tokio::time; 29 | use tokio_stream::Stream; 30 | 31 | /// A commit log holds topics and can be appended to and tailed. 32 | #[derive(Clone)] 33 | pub struct KafkaRestCommitLog { 34 | client: Client, 35 | server: Url, 36 | } 37 | 38 | const CONSUMER_GROUP_NAME_LABEL: &str = "consumer_group_name"; 39 | const TOPIC_LABEL: &str = "topic"; 40 | 41 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 42 | struct ProduceRequest { 43 | pub records: Vec, 44 | } 45 | 46 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 47 | struct ProduceReply { 48 | pub offsets: Vec, 49 | } 50 | 51 | type OffsetMap = HashMap<(Topic, u32), u64>; 52 | 53 | impl KafkaRestCommitLog { 54 | /// Establish a new commit log session. 55 | pub fn new(server: Url, server_cert: Option, tls_insecure: bool) -> Self { 56 | let client = Client::builder().danger_accept_invalid_certs(tls_insecure); 57 | 58 | let client = if let Some(cert) = server_cert { 59 | client.add_root_certificate(cert) 60 | } else { 61 | client 62 | }; 63 | 64 | Self { 65 | client: client.build().unwrap(), 66 | server, 67 | } 68 | } 69 | } 70 | 71 | #[async_trait] 72 | impl CommitLog for KafkaRestCommitLog { 73 | async fn offsets(&self, topic: Topic, partition: u32) -> Option { 74 | let mut delayer = Delayer::default(); 75 | loop { 76 | match self 77 | .client 78 | .get( 79 | self.server 80 | .join(&format!("/topics/{topic}/partitions/{partition}/offsets")) 81 | .unwrap(), 82 | ) 83 | .send() 84 | .await 85 | { 86 | Ok(response) => { 87 | if response.status().is_success() { 88 | trace!("Retrieved offsets for: {} {}", topic, partition); 89 | increment_counter!("offset_replies", TOPIC_LABEL => topic.to_string()); 90 | break response.json::().await.ok(); 91 | } else { 92 | debug!( 93 | "Commit log failure status while retrieving offsets: {:?}", 94 | response.status() 95 | ); 96 | increment_counter!("offsets_other_reply_failures"); 97 | break None; 98 | } 99 | } 100 | Err(e) => { 101 | debug!( 102 | "Commit log is unavailable while retrieving offsets. Error: {:?}", 103 | e 104 | ); 105 | increment_counter!("offsets_unavailables"); 106 | } 107 | } 108 | delayer.delay().await; 109 | } 110 | } 111 | 112 | async fn produce(&self, record: ProducerRecord) -> Result { 113 | let mut delayer = Delayer::default(); 114 | loop { 115 | match self 116 | .client 117 | .post( 118 | self.server 119 | .join(&format!("/topics/{}", record.topic)) 120 | .unwrap(), 121 | ) 122 | .json(&ProduceRequest { 123 | records: vec![record.clone()], 124 | }) 125 | .send() 126 | .await 127 | { 128 | Ok(response) => { 129 | if response.status().is_success() { 130 | trace!("Produced record: {:?}", record); 131 | increment_counter!("producer_replies", TOPIC_LABEL => record.topic.to_string()); 132 | break response 133 | .json::() 134 | .await 135 | .map_err(|_| ProducerError::CannotProduce) 136 | .and_then(|r| { 137 | r.offsets.first().map(|o| o.to_owned()).ok_or_else(|| { 138 | debug!( 139 | "Commit log failure reply with no offset while producing" 140 | ); 141 | increment_counter!("producer_other_reply_failures"); 142 | ProducerError::CannotProduce 143 | }) 144 | }); 145 | } else { 146 | debug!( 147 | "Commit log failure status while producing: {:?}", 148 | response.status() 149 | ); 150 | increment_counter!("producer_other_reply_failures"); 151 | break Err(ProducerError::CannotProduce); 152 | } 153 | } 154 | Err(e) => { 155 | debug!("Commit log is unavailable while producing. Error: {:?}", e); 156 | increment_counter!("producer_unavailables"); 157 | } 158 | } 159 | delayer.delay().await; 160 | } 161 | } 162 | 163 | /// Subscribe to one or more topics for a given consumer group 164 | /// having committed zero or more topics. Connections are 165 | /// retried if they cannot be established, or become lost. 166 | /// Once a connection is established then records are streamed 167 | /// back indefinitely unless an idle timeout argument is provided. 168 | /// In the case of an idle timeout, if no record is received 169 | /// within that period, None is returned to end the stream. 170 | fn scoped_subscribe<'a>( 171 | &'a self, 172 | consumer_group_name: &str, 173 | offsets: Vec, 174 | subscriptions: Vec, 175 | idle_timeout: Option, 176 | ) -> Pin + Send + 'a>> { 177 | let consumer_group_name = consumer_group_name.to_string(); 178 | let mut offsets: OffsetMap = offsets 179 | .iter() 180 | .map(|e| ((e.topic.to_owned(), e.partition), e.offset)) 181 | .collect::(); 182 | let subscriptions: Vec = subscriptions.iter().map(|e| e.to_owned()).collect(); 183 | Box::pin(stream!({ 184 | let mut delayer = Delayer::default(); 185 | 'stream_loop: loop { 186 | increment_counter!("consumer_group_requests", CONSUMER_GROUP_NAME_LABEL => consumer_group_name.to_string()); 187 | let consumer = Consumer { 188 | offsets: offsets 189 | .clone() 190 | .iter() 191 | .map(|((topic, partition), offset)| ConsumerOffset { 192 | offset: *offset, 193 | partition: *partition, 194 | topic: topic.clone(), 195 | }) 196 | .collect(), 197 | subscriptions: subscriptions.clone(), 198 | }; 199 | let response = self 200 | .client 201 | .post( 202 | self.server 203 | .join(&format!("/consumers/{}", consumer_group_name)) 204 | .unwrap(), 205 | ) 206 | .json(&consumer) 207 | .send() 208 | .await; 209 | match response { 210 | Ok(mut r) => loop { 211 | let chunk = if let Some(it) = idle_timeout { 212 | match time::timeout(it, r.chunk()).await { 213 | Ok(c) => c, 214 | Err(_) => break 'stream_loop, 215 | } 216 | } else { 217 | r.chunk().await 218 | }; 219 | match chunk { 220 | Ok(Some(c)) => { 221 | if let Ok(record) = serde_json::from_slice::(&c) { 222 | trace!("Received record: {:?}", record); 223 | let topic = record.topic.to_owned(); 224 | let partition = record.partition; 225 | let record_offset = record.offset; 226 | increment_counter!("consumer_group_replies", CONSUMER_GROUP_NAME_LABEL => consumer_group_name.to_string(), TOPIC_LABEL => topic.to_string()); 227 | yield record; 228 | 229 | let _ = offsets.insert((topic, partition), record_offset); 230 | } else { 231 | debug!("Unable to decode record"); 232 | } 233 | } 234 | Ok(None) => { 235 | debug!("Unable to receive chunk"); 236 | delayer = Delayer::default(); 237 | continue 'stream_loop; 238 | } 239 | Err(e) => debug!("Error receiving chunk {:?}", e), 240 | } 241 | }, 242 | Err(e) => { 243 | debug!( 244 | "Commit log is unavailable while subscribing. Error: {:?}", 245 | e 246 | ); 247 | increment_counter!("consumer_group_request_failures", CONSUMER_GROUP_NAME_LABEL => consumer_group_name.to_string()); 248 | } 249 | } 250 | delayer.delay().await; 251 | } 252 | })) 253 | } 254 | } 255 | -------------------------------------------------------------------------------- /streambed-kafka/tests/commit_log.rs: -------------------------------------------------------------------------------- 1 | use std::{convert::Infallible, time::Duration}; 2 | 3 | use serde::Deserialize; 4 | use streambed_kafka::KafkaRestCommitLog; 5 | 6 | use async_stream::stream; 7 | use reqwest::Url; 8 | use serde_json::json; 9 | use streambed::commit_log::*; 10 | use streambed_test::server; 11 | use tokio_stream::StreamExt; 12 | 13 | #[tokio::test] 14 | async fn kafka_rest_scoped_subscribe() { 15 | let server = server::http(move |mut req| async move { 16 | assert_eq!(req.uri(), "/consumers/farmo-integrator"); 17 | 18 | let mut req_body = Vec::new(); 19 | while let Some(item) = req.body_mut().next().await { 20 | req_body.extend(&*item.unwrap()); 21 | } 22 | 23 | let req_body = serde_json::from_slice::(&req_body).unwrap(); 24 | assert_eq!( 25 | req_body, 26 | Consumer { 27 | offsets: vec!(ConsumerOffset { 28 | topic: Topic::from("default:end-device-events"), 29 | partition: 0, 30 | offset: 0, 31 | }), 32 | subscriptions: vec!(Subscription { 33 | topic: Topic::from("default:end-device-events") 34 | }), 35 | } 36 | ); 37 | 38 | let stream = stream! { 39 | yield Result::<_, Infallible>::Ok(serde_json::to_vec(&ConsumerRecord { 40 | topic: Topic::from("default:end-device-events"), 41 | headers: vec![], 42 | timestamp: None, 43 | key: 0, 44 | value: b"hi there".to_vec(), 45 | partition: 0, 46 | offset: 1, 47 | }).unwrap()); 48 | 49 | yield Result::<_, Infallible>::Ok(serde_json::to_vec(&ConsumerRecord { 50 | topic: Topic::from("default:end-device-events"), 51 | headers: vec![], 52 | timestamp: None, 53 | key: 0, 54 | value: b"hi there again".to_vec(), 55 | partition: 0, 56 | offset: 2, 57 | }).unwrap()); 58 | 59 | tokio::time::sleep(Duration::from_millis(500)).await; 60 | }; 61 | 62 | let body = hyper::Body::wrap_stream(stream); 63 | 64 | http::Response::new(body) 65 | }); 66 | 67 | let server_addr = server.addr(); 68 | 69 | let cl = KafkaRestCommitLog::new( 70 | Url::parse(&format!( 71 | "http://{}:{}", 72 | server_addr.ip(), 73 | server_addr.port() 74 | )) 75 | .unwrap(), 76 | None, 77 | false, 78 | ); 79 | 80 | let events = cl.scoped_subscribe( 81 | "farmo-integrator", 82 | vec![ConsumerOffset { 83 | topic: Topic::from("default:end-device-events"), 84 | partition: 0, 85 | offset: 0, 86 | }], 87 | vec![Subscription { 88 | topic: Topic::from("default:end-device-events"), 89 | }], 90 | Some(Duration::from_millis(100)), 91 | ); 92 | tokio::pin!(events); 93 | 94 | assert_eq!( 95 | events.next().await, 96 | Some(ConsumerRecord { 97 | topic: Topic::from("default:end-device-events"), 98 | headers: vec![], 99 | timestamp: None, 100 | key: 0, 101 | value: b"hi there".to_vec(), 102 | partition: 0, 103 | offset: 1, 104 | }) 105 | ); 106 | assert_eq!( 107 | events.next().await, 108 | Some(ConsumerRecord { 109 | topic: Topic::from("default:end-device-events"), 110 | headers: vec![], 111 | timestamp: None, 112 | key: 0, 113 | value: b"hi there again".to_vec(), 114 | partition: 0, 115 | offset: 2, 116 | }) 117 | ); 118 | assert_eq!(events.next().await, None); 119 | } 120 | 121 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] 122 | struct ProduceRequest { 123 | pub records: Vec, 124 | } 125 | 126 | #[tokio::test] 127 | async fn kafka_publish() { 128 | let server = server::http(move |mut req| async move { 129 | assert_eq!(req.uri(), "/topics/default:end-device-events"); 130 | 131 | let mut req_body = Vec::new(); 132 | while let Some(item) = req.body_mut().next().await { 133 | req_body.extend(&*item.unwrap()); 134 | } 135 | 136 | let req_body = serde_json::from_slice::(&req_body).unwrap(); 137 | assert_eq!( 138 | req_body, 139 | ProduceRequest { 140 | records: vec![ProducerRecord { 141 | topic: Topic::from("default:end-device-events"), 142 | headers: vec![], 143 | timestamp: None, 144 | key: 0, 145 | value: b"hi there".to_vec(), 146 | partition: 0 147 | }] 148 | } 149 | ); 150 | 151 | let body = json!( 152 | { 153 | "key_schema_id": null, 154 | "value_schema_id": null, 155 | "offsets": [ 156 | { 157 | "partition": 2, 158 | "offset": 100 159 | }, 160 | { 161 | "partition": 1, 162 | "offset": 101 163 | }, 164 | { 165 | "partition": 2, 166 | "offset": 102 167 | } 168 | ] 169 | } 170 | ); 171 | 172 | http::Response::new(body.to_string().into()) 173 | }); 174 | 175 | let server_addr = server.addr(); 176 | 177 | let cl = KafkaRestCommitLog::new( 178 | Url::parse(&format!( 179 | "http://{}:{}", 180 | server_addr.ip(), 181 | server_addr.port() 182 | )) 183 | .unwrap(), 184 | None, 185 | false, 186 | ); 187 | 188 | let record = ProducerRecord { 189 | topic: Topic::from("default:end-device-events"), 190 | headers: vec![], 191 | timestamp: None, 192 | key: 0, 193 | value: b"hi there".to_vec(), 194 | partition: 0, 195 | }; 196 | let result = cl.produce(record).await.unwrap(); 197 | assert_eq!(result.offset, 100); 198 | } 199 | 200 | #[tokio::test] 201 | async fn kafka_offsets() { 202 | let server = server::http(move |req| async move { 203 | assert_eq!( 204 | req.uri(), 205 | "/topics/default:end-device-events/partitions/0/offsets" 206 | ); 207 | 208 | let body = json!( 209 | { 210 | "beginning_offset": 0, 211 | "end_offset": 1 212 | } 213 | ); 214 | 215 | http::Response::new(body.to_string().into()) 216 | }); 217 | 218 | let server_addr = server.addr(); 219 | 220 | let cl = KafkaRestCommitLog::new( 221 | Url::parse(&format!( 222 | "http://{}:{}", 223 | server_addr.ip(), 224 | server_addr.port() 225 | )) 226 | .unwrap(), 227 | None, 228 | false, 229 | ); 230 | 231 | let result = cl 232 | .offsets(Topic::from("default:end-device-events"), 0) 233 | .await 234 | .unwrap(); 235 | assert_eq!( 236 | result, 237 | PartitionOffsets { 238 | beginning_offset: 0, 239 | end_offset: 1, 240 | } 241 | ); 242 | } 243 | -------------------------------------------------------------------------------- /streambed-kafka/tests/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod commit_log; 3 | -------------------------------------------------------------------------------- /streambed-logged-cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed-logged-cli" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "A CLI for a file-system-based commit log" 9 | 10 | [dependencies] 11 | clap = { workspace = true } 12 | env_logger = { workspace = true } 13 | humantime = { workspace = true } 14 | serde_json = { workspace = true } 15 | smol_str = { workspace = true } 16 | streambed = { workspace = true } 17 | streambed-logged = { workspace = true } 18 | tokio = { workspace = true, features = ["full"] } 19 | tokio-stream = { workspace = true } 20 | 21 | [[bin]] 22 | name = "logged" 23 | path = "src/main.rs" 24 | -------------------------------------------------------------------------------- /streambed-logged-cli/README.md: -------------------------------------------------------------------------------- 1 | logged 2 | === 3 | 4 | The `logged` command provides a utility for conveniently operating on file-based commit logs. Functions such as the ability to consume a JSON file of records, or produce them, are available. No assumptions are made regarding the structure of a record's value (payload), or whether it is encrypted or not. The expectation is that a separate tool for that concern is used in a pipeline if required. 5 | 6 | Running with an example write followed by a read 7 | --- 8 | 9 | First get the executable: 10 | 11 | ``` 12 | cargo install streambed-logged-cli 13 | ``` 14 | 15 | ...or build it from this repo: 16 | 17 | ``` 18 | cargo build --bin logged --release 19 | ``` 20 | 21 | ...and make a build available on the PATH (only for `cargo build` and just once per shell session): 22 | 23 | ``` 24 | export PATH="$PWD/target/release":$PATH 25 | ``` 26 | 27 | ...then write some data to a topic named `my-topic`: 28 | 29 | ``` 30 | echo '{"topic":"my-topic","headers":[],"key":0,"value":"SGkgdGhlcmU=","partition":0}' | \ 31 | logged --root-path=/tmp produce --file - 32 | ``` 33 | 34 | ...then read it back: 35 | 36 | ``` 37 | logged --root-path=/tmp subscribe --subscription my-topic --idle-timeout=0ms 38 | ``` 39 | 40 | ...which would output: 41 | 42 | ``` 43 | {"topic":"my-topic","headers":[],"timestamp":null,"key":0,"value":"SGkgdGhlcmU=","partition":0,"offset":0} 44 | ``` 45 | 46 | If you're using [nushell](https://www.nushell.sh/) then you can do nice things like base64 decode payload 47 | values along the way: 48 | 49 | ``` 50 | logged --root-path=/tmp subscribe --subscription my-topic --idle-timeout=0m | from json --objects | update value {decode base64} 51 | ``` 52 | 53 | ...which would output: 54 | 55 | ``` 56 | ╭────┬──────────┬────────────────┬───────────┬─────┬──────────┬───────────┬────────╮ 57 | │ # │ topic │ headers │ timestamp │ key │ value │ partition │ offset │ 58 | ├────┼──────────┼────────────────┼───────────┼─────┼──────────┼───────────┼────────┤ 59 | │ 0 │ my-topic │ [list 0 items] │ │ 0 │ Hi there │ 0 │ 0 │ 60 | ╰────┴──────────┴────────────────┴───────────┴─────┴──────────┴───────────┴────────╯ 61 | ``` 62 | 63 | Use `--help` to discover all of the options. 64 | -------------------------------------------------------------------------------- /streambed-logged-cli/src/errors.rs: -------------------------------------------------------------------------------- 1 | use std::{error::Error, fmt, io}; 2 | 3 | use streambed::commit_log::ProducerError; 4 | 5 | #[derive(Debug)] 6 | pub enum Errors { 7 | Io(io::Error), 8 | Producer(ProducerError), 9 | Serde(serde_json::Error), 10 | } 11 | 12 | impl From for Errors { 13 | fn from(value: io::Error) -> Self { 14 | Self::Io(value) 15 | } 16 | } 17 | 18 | impl From for Errors { 19 | fn from(value: serde_json::Error) -> Self { 20 | Self::Serde(value) 21 | } 22 | } 23 | 24 | impl fmt::Display for Errors { 25 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 26 | match self { 27 | Self::Io(e) => e.fmt(f), 28 | Self::Producer(_) => f.write_str("CannotProduce"), 29 | Self::Serde(e) => e.fmt(f), 30 | } 31 | } 32 | } 33 | 34 | impl Error for Errors { 35 | fn source(&self) -> Option<&(dyn Error + 'static)> { 36 | match self { 37 | Self::Io(e) => e.source(), 38 | Self::Producer(_) => None, 39 | Self::Serde(e) => e.source(), 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /streambed-logged-cli/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | error::Error, 3 | fmt, fs, 4 | io::{self, BufRead, BufReader, BufWriter, Read, Write}, 5 | path::PathBuf, 6 | }; 7 | 8 | use clap::{Args, Parser, Subcommand}; 9 | use errors::Errors; 10 | use streambed::commit_log::{ConsumerOffset, Subscription}; 11 | use streambed_logged::FileLog; 12 | 13 | pub mod errors; 14 | pub mod producer; 15 | pub mod subscriber; 16 | 17 | /// A utility for conveniently operating on file-based commit logs. 18 | /// Functions such as the ability to consume a JSON file of records, 19 | /// or produce them, are available. 20 | /// No assumptions are made regarding the structure of a record's 21 | /// value (payload), or whether it is encrypted or not. The expectation 22 | /// is that a separate tool for that concern is used in a pipeline. 23 | #[derive(Parser, Debug)] 24 | #[clap(author, about, long_about = None, version)] 25 | struct ProgramArgs { 26 | /// The location of all topics in the Commit Log 27 | #[clap(env, long, default_value = "/var/lib/logged")] 28 | pub root_path: PathBuf, 29 | 30 | #[command(subcommand)] 31 | pub command: Command, 32 | } 33 | 34 | #[derive(Subcommand, Debug)] 35 | enum Command { 36 | Produce(ProduceCommand), 37 | Subscribe(SubscribeCommand), 38 | } 39 | 40 | /// Consume JSON records from a stream until EOF and append them to the log. 41 | /// Records may be represented using JSON-line, or not. 42 | /// JSON-line is JSON with a newline character between each object streamed. 43 | #[derive(Args, Debug)] 44 | struct ProduceCommand { 45 | /// The file to consume records from, or `-` to indicate STDIN. 46 | #[clap(short, long)] 47 | pub file: PathBuf, 48 | 49 | /// The max number utf-8 characters to read from the file. 50 | #[clap(long, default_value_t = 8 * 1024)] 51 | pub max_line_len: u64, 52 | } 53 | 54 | /// Subscribe to topics and consume from them producing JSON-line records to a stream. 55 | /// JSON-line is JSON with a newline character between each object streamed. 56 | #[derive(Args, Debug)] 57 | struct SubscribeCommand { 58 | /// The amount of time to indicate that no more events are immediately 59 | /// available from the Commit Log endpoint. If unspecified then the 60 | /// CLI will wait indefinitely for records to appear. 61 | #[clap(long)] 62 | pub idle_timeout: Option, 63 | 64 | /// In the case that an offset is supplied, it is 65 | /// associated with their respective topics such that any 66 | /// subsequent subscription will source from the offset. 67 | /// The fields are topic name, partition and offset which 68 | /// are separated by commas with no spaces e.g. "--offset=my-topic,0,1000". 69 | #[clap(long)] 70 | #[arg(value_parser = parse_offset)] 71 | pub offset: Vec, 72 | 73 | /// By default, records of the topic are consumed and output to STDOUT. 74 | /// This option can be used to write to a file. Records are output as JSON. 75 | #[clap(short, long)] 76 | pub output: Option, 77 | 78 | /// In the case where a subscription topic names are supplied, the consumer 79 | /// instance will subscribe and reply with a stream of records 80 | /// ending only when the connection to the topic is severed. 81 | /// Topics may be namespaced by prefixing with characters followed by 82 | /// a `:`. For example, "my-ns:my-topic". 83 | #[clap(long, required = true)] 84 | pub subscription: Vec, 85 | } 86 | 87 | #[derive(Clone, Debug)] 88 | struct Offset { 89 | pub topic: String, 90 | pub partition: u32, 91 | pub offset: u64, 92 | } 93 | 94 | impl From for ConsumerOffset { 95 | fn from(value: Offset) -> Self { 96 | ConsumerOffset { 97 | topic: value.topic.into(), 98 | partition: value.partition, 99 | offset: value.offset, 100 | } 101 | } 102 | } 103 | 104 | #[derive(Debug)] 105 | enum OffsetParseError { 106 | MissingTopic, 107 | MissingPartition, 108 | InvalidPartition, 109 | MissingOffset, 110 | InvalidOffset, 111 | } 112 | 113 | impl fmt::Display for OffsetParseError { 114 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 115 | match self { 116 | OffsetParseError::MissingTopic => { 117 | f.write_str("Missing the topic as the first part of the argument") 118 | } 119 | OffsetParseError::MissingPartition => { 120 | f.write_str("Missing the partition number as the second part to the argument") 121 | } 122 | OffsetParseError::InvalidPartition => { 123 | f.write_str("An invalid partition number was provided") 124 | } 125 | OffsetParseError::MissingOffset => { 126 | f.write_str("Missing the offset as the third part to the argument") 127 | } 128 | OffsetParseError::InvalidOffset => f.write_str("An invalid offset number was provided"), 129 | } 130 | } 131 | } 132 | 133 | impl Error for OffsetParseError {} 134 | 135 | fn parse_offset(arg: &str) -> Result { 136 | let mut iter = arg.split(','); 137 | let Some(topic) = iter.next().map(|s| s.to_string()) else { 138 | return Err(OffsetParseError::MissingTopic); 139 | }; 140 | let Some(partition) = iter.next() else { 141 | return Err(OffsetParseError::MissingPartition); 142 | }; 143 | let Ok(partition) = partition.parse() else { 144 | return Err(OffsetParseError::InvalidPartition); 145 | }; 146 | let Some(offset) = iter.next() else { 147 | return Err(OffsetParseError::MissingOffset); 148 | }; 149 | let Ok(offset) = offset.parse() else { 150 | return Err(OffsetParseError::InvalidOffset); 151 | }; 152 | Ok(Offset { 153 | topic, 154 | partition, 155 | offset, 156 | }) 157 | } 158 | 159 | type LineReaderFn = Box Result, io::Error> + Send>; 160 | type OutputFn = Box; 161 | 162 | #[tokio::main] 163 | async fn main() -> Result<(), Box> { 164 | let args = ProgramArgs::parse(); 165 | 166 | env_logger::builder().format_timestamp_millis().init(); 167 | 168 | let cl = FileLog::new(args.root_path); 169 | 170 | let task = tokio::spawn(async move { 171 | match args.command { 172 | Command::Produce(command) => { 173 | // A line reader function is provided rather than something based on the 174 | // Read trait as reading a line from stdin involves locking it via its 175 | // own read_line function. A Read trait will cause a lock to occur for 176 | // each access of the Read methods, and we cannot be guaranteed which of those will 177 | // be used - serde, for example, may only read a byte at a time. This would cause 178 | // many locks. 179 | let line_reader: LineReaderFn = if command.file.as_os_str() == "-" { 180 | let stdin = io::stdin(); 181 | let mut line = String::new(); 182 | Box::new(move || { 183 | line.clear(); 184 | if stdin 185 | .lock() 186 | .take(command.max_line_len) 187 | .read_line(&mut line)? 188 | > 0 189 | { 190 | Ok(Some(line.clone())) 191 | } else { 192 | Ok(None) 193 | } 194 | }) 195 | } else { 196 | let mut buf = 197 | BufReader::new(fs::File::open(command.file).map_err(Errors::from)?) 198 | .take(command.max_line_len); 199 | let mut line = String::new(); 200 | Box::new(move || { 201 | line.clear(); 202 | if buf.read_line(&mut line)? > 0 { 203 | Ok(Some(line.clone())) 204 | } else { 205 | Ok(None) 206 | } 207 | }) 208 | }; 209 | 210 | producer::produce(cl, line_reader).await 211 | } 212 | Command::Subscribe(command) => { 213 | let output: OutputFn = if let Some(output) = command.output { 214 | Box::new(BufWriter::new( 215 | fs::File::create(output).map_err(Errors::from)?, 216 | )) 217 | } else { 218 | Box::new(io::stdout()) 219 | }; 220 | subscriber::subscribe( 221 | cl, 222 | command.idle_timeout.map(|d| d.into()), 223 | command.offset.into_iter().map(|o| o.into()).collect(), 224 | output, 225 | command 226 | .subscription 227 | .into_iter() 228 | .map(|s| Subscription { topic: s.into() }) 229 | .collect(), 230 | ) 231 | .await 232 | } 233 | } 234 | }); 235 | 236 | task.await 237 | .map_err(|e| e.into()) 238 | .and_then(|r: Result<(), Errors>| r.map_err(|e| e.into())) 239 | } 240 | -------------------------------------------------------------------------------- /streambed-logged-cli/src/producer.rs: -------------------------------------------------------------------------------- 1 | use std::io::{self}; 2 | 3 | use streambed::commit_log::{CommitLog, ProducerRecord}; 4 | 5 | use crate::errors::Errors; 6 | 7 | pub async fn produce( 8 | cl: impl CommitLog, 9 | mut line_reader: impl FnMut() -> Result, io::Error>, 10 | ) -> Result<(), Errors> { 11 | while let Some(line) = line_reader()? { 12 | let record = serde_json::from_str::(&line)?; 13 | let record = ProducerRecord { 14 | topic: record.topic, 15 | headers: record.headers, 16 | timestamp: record.timestamp, 17 | key: record.key, 18 | value: record.value, 19 | partition: record.partition, 20 | }; 21 | cl.produce(record).await.map_err(Errors::Producer)?; 22 | } 23 | Ok(()) 24 | } 25 | -------------------------------------------------------------------------------- /streambed-logged-cli/src/subscriber.rs: -------------------------------------------------------------------------------- 1 | use std::{io::Write, time::Duration}; 2 | 3 | use streambed::commit_log::{CommitLog, ConsumerOffset, Subscription}; 4 | use tokio::time; 5 | use tokio_stream::StreamExt; 6 | 7 | use crate::errors::Errors; 8 | 9 | const FLUSH_DELAY: Duration = Duration::from_millis(100); 10 | 11 | pub async fn subscribe( 12 | cl: impl CommitLog, 13 | idle_timeout: Option, 14 | offsets: Vec, 15 | mut output: impl Write, 16 | subscriptions: Vec, 17 | ) -> Result<(), Errors> { 18 | let mut records = 19 | cl.scoped_subscribe("streambed-logged-cli", offsets, subscriptions, idle_timeout); 20 | let mut timeout = FLUSH_DELAY; 21 | loop { 22 | tokio::select! { 23 | record = records.next() => { 24 | if let Some(record) = record { 25 | let buf = serde_json::to_string(&record).map_err(|e| Errors::Io(e.into()))?; 26 | output.write_all(buf.as_bytes())?; 27 | output.write_all(b"\n")?; 28 | timeout = FLUSH_DELAY; 29 | } else { 30 | break; 31 | } 32 | } 33 | _ = time::sleep(timeout) => { 34 | output.flush()?; 35 | timeout = Duration::MAX; 36 | } 37 | } 38 | } 39 | Ok(()) 40 | } 41 | -------------------------------------------------------------------------------- /streambed-logged/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed-logged" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "Logged is a small library that implements a file-system-based commit log" 9 | 10 | [dependencies] 11 | async-stream = { workspace = true } 12 | async-trait = { workspace = true } 13 | bytes = { workspace = true } 14 | chrono = { workspace = true } 15 | clap = { workspace = true, features = ["derive", "env"] } 16 | crc = { workspace = true } 17 | log = { workspace = true } 18 | postcard = { workspace = true, default-features = false, features = ["use-std", "use-crc"] } 19 | serde = { workspace = true, features = ["derive"] } 20 | serde_with = { workspace = true, features = ["chrono_0_4"] } 21 | smol_str = { workspace = true, features = ["serde"] } 22 | streambed = { workspace = true } 23 | tokio = { workspace = true, features = [ "fs" ] } 24 | tokio-stream = { workspace = true } 25 | tokio-util = { workspace = true, features = ["codec"] } 26 | 27 | [dev-dependencies] 28 | criterion = { workspace = true, features = ["async_tokio", "html_reports"] } 29 | env_logger = { workspace = true } 30 | test-log = { workspace = true } 31 | 32 | [[bench]] 33 | name = "benches" 34 | harness = false 35 | -------------------------------------------------------------------------------- /streambed-logged/README.md: -------------------------------------------------------------------------------- 1 | # logged 2 | 3 | Logged is a small library that implements a file-system-based 4 | commit log function for autonomous systems that often live 5 | at the edge of a wider network. 6 | 7 | Logged implements the Streambed commit log API which, in turn, models 8 | itself on the Kafka API. 9 | 10 | ## An quick introduction 11 | 12 | Nothing beats code for a quick introduction! Here is an example of 13 | establishing the commit log, producing a record and then consuming 14 | it. Please refer to the various tests for more complete examples. 15 | 16 | ```rs 17 | let cl = FileLog::new(logged_dir); 18 | 19 | let topic = Topic::from("my-topic"_; 20 | 21 | cl.produce(ProducerRecord { 22 | topic: topic.clone(), 23 | headers: None, 24 | timestamp: None, 25 | key: 0, 26 | value: b"some-value".to_vec(), 27 | partition: 0, 28 | }) 29 | .await 30 | .unwrap(); 31 | 32 | let subscriptions = vec![Subscription { 33 | topic: topic.clone(), 34 | }]; 35 | let mut records = cl.scoped_subscribe("some-consumer", None, subscriptions, None); 36 | 37 | assert_eq!( 38 | records.next().await, 39 | Some(ConsumerRecord { 40 | topic, 41 | headers: None, 42 | timestamp: None, 43 | key: 0, 44 | value: b"some-value".to_vec(), 45 | partition: 0, 46 | offset: 0 47 | }) 48 | ); 49 | ``` 50 | 51 | ## Why logged? 52 | 53 | The primary functional use-cases of logged are: 54 | 55 | * event sourcing - to re-constitute state from events; and 56 | * observing - to communicate events reliably for one or more consumers. 57 | 58 | The primary operational use-cases of logged are: 59 | 60 | * to be hosted by resource-constrained devices, typically with less 61 | than 128MiB memory and 8GB of storage. 62 | 63 | ## What is logged? 64 | 65 | ### No networking 66 | 67 | Logged has no notion of what a network is. Any replication of the 68 | data managed by logged is to be handled outside of it. 69 | 70 | ### Memory vs speed 71 | 72 | Logged is optimized for small memory usage over speed and also 73 | recognizes that storage is limited. 74 | 75 | ### Single writer/multiple reader 76 | 77 | Logged deliberately constrains the appending of a commit log to 78 | one process. This is known as the [single writer principle](https://mechanical-sympathy.blogspot.com/2011/09/single-writer-principle.html), and 79 | greatly simplifies the design of logged. 80 | 81 | ### Multiple reader processes 82 | 83 | Readers can exist in processes other than the one that writes to a 84 | commit log topic. There is no broker process involved 85 | as both readers and writers read the same files on the file system. 86 | No locking is required for writing due to the single 87 | writer principle. 88 | 89 | ### No encryption, compression or authentication 90 | 91 | Encryption and compression are an application concern, with encryption 92 | being encouraged. Compression may be less important, depending on the 93 | volume of data that an application needs to retain. 94 | 95 | Authentication is bypassed given the assumption of encryption. If a reader is unable 96 | to decrypt a message then this is seen to be as good as not gaining 97 | access in the first place. Avoiding authentication yields an additional 98 | side-effect of keeping logged free of complexity by not having to manage 99 | identity. 100 | 101 | ### Data retention is an application concern 102 | 103 | Logged provides the tools to the application so that an application 104 | may perform compaction. Our experience with applying commit logs 105 | has shown that data retention strategies are an application concern 106 | often subject to the contents of the data. 107 | 108 | ### The Kafka model 109 | 110 | Logged is modelled with the same concepts as Apache Kafka including 111 | topics, partitions, keys, records, offsets, timestamps and headers. 112 | Services using logged may therefore lend themselves to portability 113 | toward Kafka and others. 114 | 115 | ### Data integrity 116 | 117 | A CRC32C checksum is used to ensure data integrity against the harsh reality 118 | of a host being powered off abruptly. Any errors detected, including 119 | incomplete writes or compactions, will be automatically recovered. 120 | 121 | ## How is logged implemented? 122 | 123 | Logged is implemented as a library avoiding the need for a broker process. This 124 | is similar to [Chronicle Queue](https://github.com/OpenHFT/Chronicle-Queue) on the JVM. 125 | 126 | The file system is used to store records in relation to a topic and a single 127 | partition. [Tokio](https://tokio.rs/) is used for file read/write operations so that any stalled 128 | operations permit other tasks to continue running. [Postcard](https://docs.rs/postcard/latest/postcard/) 129 | is used for serialization as it is able to conveniently represent in-memory structures 130 | and is optimized for resource-constrained targets. 131 | 132 | When seeking an offset into a topic, logged will read through records sequentially 133 | as there is no indexing. This simple approach relies on the ability 134 | for processors to scan memory fast (which they do), and also for the application-level 135 | compaction to be effective. By "effective" we mean that scanning can avoid traversing 136 | thousands of records. 137 | 138 | The compaction of topics is an application concern and as it is able 139 | to consider the contents of a record. Logged provides functions to atomically "split" 140 | an existing topic log at its head and yield a new topic to compact to. An 141 | application-based compactor can then consume the existing topic, retain the records 142 | it needs in memory. Having consumed the topic, the application-based compactor will 143 | append the retained records to the topic to compact to. Once appended, logged can be 144 | instructed to replace the compacted log for the old one. Meanwhile, any new appends post 145 | splitting the log will be written to another new log for the topic. As a result, 146 | there can be two files representing the topic, and sometimes three during a compaction 147 | activity. Logged takes care of interpreting the file system and various files used 148 | when an application operates with a topic. 149 | 150 | ## When should you not use logged 151 | 152 | When you have Kafka. -------------------------------------------------------------------------------- /streambed-logged/benches/benches.rs: -------------------------------------------------------------------------------- 1 | use std::thread; 2 | use std::time::Duration; 3 | use std::{env, fs}; 4 | 5 | use criterion::{criterion_group, criterion_main, Criterion}; 6 | use streambed::commit_log::ProducerRecord; 7 | use streambed::commit_log::Subscription; 8 | use streambed::commit_log::{CommitLog, Topic}; 9 | use streambed_logged::FileLog; 10 | use tokio_stream::StreamExt; 11 | 12 | const SAMPLE_SIZE: usize = 30_000; 13 | 14 | fn criterion_benchmark(c: &mut Criterion) { 15 | c.bench_function("produce records", move |b| { 16 | let logged_dir = env::temp_dir().join("benches"); 17 | let _ = fs::remove_dir_all(&logged_dir); 18 | let _ = fs::create_dir_all(&logged_dir); 19 | 20 | let cl = FileLog::new(logged_dir); 21 | 22 | let topic = Topic::from("my-topic"); 23 | 24 | let rt = tokio::runtime::Runtime::new().unwrap(); 25 | 26 | rt.block_on(async { 27 | cl.produce(ProducerRecord { 28 | topic: topic.clone(), 29 | headers: vec![], 30 | timestamp: None, 31 | key: 0, 32 | value: b"some-value".to_vec(), 33 | partition: 0, 34 | }) 35 | .await 36 | .unwrap(); 37 | }); 38 | 39 | b.to_async(&rt).iter(|| { 40 | let task_cl = cl.clone(); 41 | let task_topic = topic.clone(); 42 | async move { 43 | tokio::spawn(async move { 44 | for _ in 1..SAMPLE_SIZE { 45 | task_cl 46 | .produce(ProducerRecord { 47 | topic: task_topic.clone(), 48 | headers: vec![], 49 | timestamp: None, 50 | key: 0, 51 | value: b"some-value".to_vec(), 52 | partition: 0, 53 | }) 54 | .await 55 | .unwrap(); 56 | } 57 | }) 58 | .await 59 | } 60 | }) 61 | }); 62 | 63 | c.bench_function("consume records", move |b| { 64 | let logged_dir = env::temp_dir().join("benches"); 65 | let _ = fs::remove_dir_all(&logged_dir); 66 | let _ = fs::create_dir_all(&logged_dir); 67 | 68 | let cl = FileLog::new(logged_dir); 69 | 70 | let topic = Topic::from("my-topic"); 71 | 72 | let rt = tokio::runtime::Runtime::new().unwrap(); 73 | 74 | rt.block_on(async { 75 | for _ in 0..SAMPLE_SIZE { 76 | cl.produce(ProducerRecord { 77 | topic: topic.clone(), 78 | headers: vec![], 79 | timestamp: None, 80 | key: 0, 81 | value: b"some-value".to_vec(), 82 | partition: 0, 83 | }) 84 | .await 85 | .unwrap(); 86 | } 87 | }); 88 | 89 | // Provide time for the writes to be flushed 90 | thread::sleep(Duration::from_secs(1)); 91 | 92 | b.to_async(&rt).iter(|| { 93 | let task_cl = cl.clone(); 94 | let task_topic = topic.clone(); 95 | async move { 96 | tokio::spawn(async move { 97 | let offsets = vec![]; 98 | let subscriptions = vec![Subscription { 99 | topic: task_topic.clone(), 100 | }]; 101 | let mut records = 102 | task_cl.scoped_subscribe("some-consumer", offsets, subscriptions, None); 103 | 104 | for _ in 0..SAMPLE_SIZE { 105 | let _ = records.next().await; 106 | } 107 | }) 108 | .await 109 | } 110 | }) 111 | }); 112 | } 113 | 114 | criterion_group! { 115 | name = benches; 116 | config = Criterion::default().sample_size(10); 117 | targets = criterion_benchmark 118 | } 119 | criterion_main!(benches); 120 | -------------------------------------------------------------------------------- /streambed-logged/src/args.rs: -------------------------------------------------------------------------------- 1 | //! Provides command line arguments that are typically used for all services using this module. 2 | 3 | use std::path::PathBuf; 4 | 5 | use clap::Parser; 6 | 7 | #[derive(Parser, Debug)] 8 | pub struct CommitLogArgs { 9 | /// A namespace to use when communicating with the Commit Log 10 | #[clap(env, long, default_value = "default")] 11 | pub cl_ns: String, 12 | 13 | /// The location of all topics in the Commit Log 14 | #[clap(env, long, default_value = "/var/lib/logged")] 15 | pub cl_root_path: PathBuf, 16 | } 17 | -------------------------------------------------------------------------------- /streambed-logged/src/topic_file_op.rs: -------------------------------------------------------------------------------- 1 | //! Provides atomic operations on the files associated with a specific topic of a commit log. 2 | //! We can have zero or more files representing the entire commit log for a topic. 3 | //! There are history files plus an "active" file. The active file is the one that 4 | //! can be appended to. History files are immutable. 5 | //! Two types of history file are available - history and ancient history. A history 6 | //! file signifies that compaction has taken or is taking place. Ancient history represents 7 | //! that compaction is taking place and is removed once compaction is done. Both can exist. 8 | //! The first elements of the returned vec, sans the last one, will represent file handles 9 | //! to history files. The last element will always represent the present commit log. 10 | //! Note that these operations return file handles so that their operations may 11 | //! succeed post subsequent file renames, deletions and so forth. 12 | 13 | use std::{ 14 | error::Error, 15 | fmt::Display, 16 | fs::{self, File, OpenOptions}, 17 | io::{self, BufWriter, Write}, 18 | path::PathBuf, 19 | sync::{Arc, Mutex}, 20 | time::SystemTime, 21 | }; 22 | 23 | use streambed::commit_log::Topic; 24 | 25 | pub(crate) const ANCIENT_HISTORY_FILE_EXTENSION: &str = "ancient_history"; 26 | pub(crate) const HISTORY_FILE_EXTENSION: &str = "history"; 27 | pub(crate) const WORK_FILE_EXTENSION: &str = "work"; 28 | 29 | #[derive(Debug)] 30 | pub(crate) enum TopicFileOpError { 31 | CannotLock, 32 | CannotSerialize, 33 | #[allow(dead_code)] 34 | IoError(io::Error), 35 | } 36 | impl Display for TopicFileOpError { 37 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 38 | match self { 39 | TopicFileOpError::CannotLock => write!(f, "CannotLock"), 40 | TopicFileOpError::CannotSerialize => write!(f, "CannotSerialize"), 41 | e @ TopicFileOpError::IoError(_) => e.fmt(f), 42 | } 43 | } 44 | } 45 | impl Error for TopicFileOpError {} 46 | 47 | #[derive(Clone)] 48 | pub(crate) struct TopicFileOp { 49 | root_path: PathBuf, 50 | topic: Topic, 51 | write_handle: Arc>>>, 52 | } 53 | 54 | impl TopicFileOp { 55 | pub fn new(root_path: PathBuf, topic: Topic) -> Self { 56 | Self { 57 | root_path, 58 | topic, 59 | write_handle: Arc::new(Mutex::new(None)), 60 | } 61 | } 62 | 63 | pub fn active_file_size( 64 | &mut self, 65 | open_options: &OpenOptions, 66 | write_buffer_size: usize, 67 | ) -> Result { 68 | // Ensure the active file is opened for appending first. 69 | let _ = self.with_active_file(open_options, write_buffer_size, |_| Ok(0))?; 70 | 71 | let Ok(mut locked_write_handle) = self.write_handle.lock() else { 72 | return Err(TopicFileOpError::CannotLock); 73 | }; 74 | let r = if let Some(write_handle) = locked_write_handle.as_mut() { 75 | write_handle.get_ref().metadata().map(|m| m.len()) 76 | } else { 77 | Ok(0) 78 | }; 79 | r.map_err(TopicFileOpError::IoError) 80 | } 81 | 82 | pub fn age_active_file(&mut self) -> Result<(), TopicFileOpError> { 83 | let Ok(mut locked_write_handle) = self.write_handle.lock() else { 84 | return Err(TopicFileOpError::CannotLock); 85 | }; 86 | let present_path = self.root_path.join(self.topic.as_str()); 87 | let ancient_history_path = present_path.with_extension(ANCIENT_HISTORY_FILE_EXTENSION); 88 | let history_path = present_path.with_extension(HISTORY_FILE_EXTENSION); 89 | 90 | if let Some(write_handle) = locked_write_handle.as_mut() { 91 | write_handle.flush().map_err(TopicFileOpError::IoError)?; 92 | } 93 | 94 | if !ancient_history_path.exists() { 95 | let r = fs::rename(&history_path, ancient_history_path); 96 | if let Err(e) = r { 97 | if e.kind() != io::ErrorKind::NotFound { 98 | return Err(TopicFileOpError::IoError(e)); 99 | } 100 | } 101 | let r = fs::rename(present_path, history_path); 102 | *locked_write_handle = None; 103 | if let Err(e) = r { 104 | if e.kind() != io::ErrorKind::NotFound { 105 | return Err(TopicFileOpError::IoError(e)); 106 | } 107 | } 108 | } 109 | Ok(()) 110 | } 111 | 112 | pub fn flush_active_file(&self) -> Result<(), TopicFileOpError> { 113 | let Ok(mut locked_write_handle) = self.write_handle.lock() else { 114 | return Err(TopicFileOpError::CannotLock); 115 | }; 116 | let r = if let Some(write_handle) = locked_write_handle.as_mut() { 117 | write_handle.flush() 118 | } else { 119 | Ok(()) 120 | }; 121 | r.map(|_| ()).map_err(TopicFileOpError::IoError) 122 | } 123 | 124 | pub fn open_active_file(&self, open_options: OpenOptions) -> Result { 125 | let Ok(locked_write_handle) = self.write_handle.lock() else { 126 | return Err(TopicFileOpError::CannotLock); 127 | }; 128 | let present_path = self.root_path.join(self.topic.as_str()); 129 | let r = open_options.open(present_path); 130 | drop(locked_write_handle); 131 | r.map_err(TopicFileOpError::IoError) 132 | } 133 | 134 | pub fn modification_time(&self) -> Option { 135 | let locked_write_handle = self.write_handle.lock().ok()?; 136 | 137 | let present_path = self.root_path.join(self.topic.as_str()); 138 | let modification_time = fs::metadata(&present_path) 139 | .ok() 140 | .and_then(|m| m.modified().ok()) 141 | .or_else(|| { 142 | let history_path = present_path.with_extension(HISTORY_FILE_EXTENSION); 143 | fs::metadata(&history_path) 144 | .ok() 145 | .and_then(|m| m.modified().ok()) 146 | }) 147 | .or_else(|| { 148 | let ancient_history_path = 149 | present_path.with_extension(ANCIENT_HISTORY_FILE_EXTENSION); 150 | fs::metadata(&ancient_history_path) 151 | .ok() 152 | .and_then(|m| m.modified().ok()) 153 | }); 154 | 155 | drop(locked_write_handle); 156 | modification_time 157 | } 158 | 159 | pub fn open_files( 160 | &self, 161 | open_options: OpenOptions, 162 | exclude_active_file: bool, 163 | ) -> Vec> { 164 | let Ok(locked_write_handle) = self.write_handle.lock() else { 165 | return vec![]; 166 | }; 167 | 168 | let mut files = Vec::with_capacity(3); 169 | let present_path = self.root_path.join(self.topic.as_str()); 170 | let ancient_history_path = present_path.with_extension(ANCIENT_HISTORY_FILE_EXTENSION); 171 | let history_path = present_path.with_extension(HISTORY_FILE_EXTENSION); 172 | match open_options.open(ancient_history_path) { 173 | Ok(f) => files.push(Ok(f)), 174 | Err(e) if e.kind() == io::ErrorKind::NotFound => (), 175 | Err(e) => files.push(Err(e)), 176 | } 177 | match open_options.open(history_path) { 178 | Ok(f) => files.push(Ok(f)), 179 | Err(e) if e.kind() == io::ErrorKind::NotFound => (), 180 | Err(e) => files.push(Err(e)), 181 | } 182 | if !exclude_active_file { 183 | match open_options.open(present_path) { 184 | Ok(f) => files.push(Ok(f)), 185 | Err(e) if e.kind() == io::ErrorKind::NotFound => (), 186 | e @ Err(_) => files.push(e), 187 | } 188 | } 189 | drop(locked_write_handle); 190 | files 191 | } 192 | 193 | pub fn new_work_file( 194 | &self, 195 | write_buffer_size: usize, 196 | ) -> Result, TopicFileOpError> { 197 | let present_path = self.root_path.join(self.topic.as_str()); 198 | let work_path = present_path.with_extension(WORK_FILE_EXTENSION); 199 | let r = File::create(work_path); 200 | r.map(|write_handle| BufWriter::with_capacity(write_buffer_size, write_handle)) 201 | .map_err(TopicFileOpError::IoError) 202 | } 203 | 204 | pub fn recover_history_files(&self) -> Result<(), TopicFileOpError> { 205 | let present_path = self.root_path.join(self.topic.as_str()); 206 | let work_path = present_path.with_extension(WORK_FILE_EXTENSION); 207 | let ancient_history_path = present_path.with_extension(ANCIENT_HISTORY_FILE_EXTENSION); 208 | let history_path = present_path.with_extension(HISTORY_FILE_EXTENSION); 209 | 210 | let _ = fs::rename(ancient_history_path, history_path); 211 | let _ = fs::remove_file(work_path); 212 | Ok(()) 213 | } 214 | 215 | pub fn replace_history_files(&self) -> Result<(), TopicFileOpError> { 216 | let present_path = self.root_path.join(self.topic.as_str()); 217 | let work_path = present_path.with_extension(WORK_FILE_EXTENSION); 218 | let ancient_history_path = present_path.with_extension(ANCIENT_HISTORY_FILE_EXTENSION); 219 | let history_path = present_path.with_extension(HISTORY_FILE_EXTENSION); 220 | 221 | let r = fs::rename(work_path, history_path); 222 | if let Err(e) = r { 223 | return Err(TopicFileOpError::IoError(e)); 224 | } 225 | let _ = fs::remove_file(ancient_history_path); 226 | Ok(()) 227 | } 228 | 229 | pub fn with_active_file( 230 | &mut self, 231 | open_options: &OpenOptions, 232 | write_buffer_size: usize, 233 | f: F, 234 | ) -> Result<(usize, bool), TopicFileOpError> 235 | where 236 | F: FnOnce(&mut BufWriter) -> Result, 237 | { 238 | let Ok(mut locked_write_handle) = self.write_handle.lock() else { 239 | return Err(TopicFileOpError::CannotLock); 240 | }; 241 | if let Some(write_handle) = locked_write_handle.as_mut() { 242 | f(write_handle).map(|size| (size, false)) 243 | } else { 244 | let present_path = self.root_path.join(self.topic.as_str()); 245 | let r = open_options.clone().open(present_path); 246 | if let Ok(write_handle) = r { 247 | let mut write_handle = BufWriter::with_capacity(write_buffer_size, write_handle); 248 | let r = f(&mut write_handle); 249 | *locked_write_handle = Some(write_handle); 250 | r.map(|size| (size, true)) 251 | } else { 252 | r.map(|_| (0, true)).map_err(TopicFileOpError::IoError) 253 | } 254 | } 255 | } 256 | } 257 | 258 | #[cfg(test)] 259 | mod tests { 260 | use std::env; 261 | 262 | use test_log::test; 263 | use tokio::fs; 264 | 265 | use super::*; 266 | 267 | #[test(tokio::test)] 268 | async fn test_open_topic_files() { 269 | let root_path = env::temp_dir().join("test_open_topic_files"); 270 | let _ = fs::remove_dir_all(&root_path).await; 271 | let _ = fs::create_dir_all(&root_path).await; 272 | 273 | let current_path = root_path.join("topic"); 274 | assert!(fs::File::create(¤t_path).await.is_ok()); 275 | 276 | let topic = Topic::from("topic"); 277 | 278 | let file_ops = TopicFileOp::new(root_path, topic); 279 | 280 | let mut open_options = OpenOptions::new(); 281 | open_options.read(true); 282 | 283 | assert_eq!( 284 | file_ops 285 | .open_files(open_options.clone(), false) 286 | .into_iter() 287 | .map(|f| f.is_ok()) 288 | .collect::>(), 289 | [true] 290 | ); 291 | 292 | let history_path = current_path.with_extension(HISTORY_FILE_EXTENSION); 293 | println!("{history_path:?}"); 294 | assert!(fs::File::create(&history_path).await.is_ok()); 295 | assert_eq!( 296 | file_ops 297 | .open_files(open_options.clone(), false) 298 | .into_iter() 299 | .map(|f| f.is_ok()) 300 | .collect::>(), 301 | [true, true] 302 | ); 303 | assert_eq!( 304 | file_ops 305 | .open_files(open_options.clone(), true) 306 | .into_iter() 307 | .map(|f| f.is_ok()) 308 | .collect::>(), 309 | [true] 310 | ); 311 | } 312 | } 313 | -------------------------------------------------------------------------------- /streambed-patterns/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed-patterns" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "Patterns for working with streambed" 9 | 10 | [dependencies] 11 | async-trait = { workspace = true } 12 | tokio = { workspace = true, features = [ "sync" ] } 13 | 14 | [dev-dependencies.tokio] 15 | workspace = true 16 | features = [ "full" ] -------------------------------------------------------------------------------- /streambed-patterns/README.md: -------------------------------------------------------------------------------- 1 | # patterns 2 | 3 | Patterns for working with streambed. 4 | 5 | ## Ask Pattern 6 | 7 | The ask pattern is based on the concept of the same name from [Akka](https://doc.akka.io/docs/akka/current/stream/operators/Source-or-Flow/ask.html). 8 | It encodes async request-reply semantics which can be used, for example, to perform request-reply operations across a `tokio::sync::mpsc::channel` or 9 | similar. The provided implementation sends a message using a `tokio::sync::mpsc::Sender<_>` and uses a `tokio::sync::oneshot` channel internally to convey the 10 | response. -------------------------------------------------------------------------------- /streambed-patterns/examples/ask.rs: -------------------------------------------------------------------------------- 1 | use crate::ExampleCommand::{GetState, SetState}; 2 | use std::error::Error; 3 | use streambed_patterns::ask::Ask; 4 | 5 | enum ExampleCommand { 6 | GetState { 7 | reply_to: Box, 8 | }, 9 | SetState { 10 | new_state: String, 11 | }, 12 | } 13 | 14 | #[tokio::main] 15 | async fn main() -> Result<(), Box> { 16 | let (tx, mut rx) = tokio::sync::mpsc::channel::(1); 17 | 18 | tokio::spawn(async move { 19 | let mut state = "initial state".to_string(); 20 | 21 | while let Some(msg) = rx.recv().await { 22 | match msg { 23 | GetState { reply_to } => { 24 | reply_to(state.clone()); 25 | } 26 | SetState { new_state } => { 27 | state = new_state; 28 | } 29 | } 30 | } 31 | }); 32 | 33 | // Use the ask pattern to get the state 34 | let state = tx.ask(|reply_to| GetState { reply_to }).await.unwrap(); 35 | println!("state: {state}"); 36 | 37 | // Set the state to something different 38 | let _ = tx 39 | .send(SetState { 40 | new_state: "new state".to_string(), 41 | }) 42 | .await; 43 | println!("state updated"); 44 | 45 | // Use the ask pattern to get the state again 46 | let state = tx.ask(|reply_to| GetState { reply_to }).await.unwrap(); 47 | println!("state: {state}"); 48 | 49 | Ok(()) 50 | } 51 | -------------------------------------------------------------------------------- /streambed-patterns/src/ask.rs: -------------------------------------------------------------------------------- 1 | //! This is an implementation of the ask pattern that sends a request to a [tokio::sync::mpsc::Sender<_>] 2 | //! and uses a [tokio::sync::oneshot] channel internally to convey the reply. 3 | 4 | use async_trait::async_trait; 5 | use std::error::Error; 6 | use std::fmt::{Debug, Display, Formatter}; 7 | use tokio::sync::{mpsc, oneshot}; 8 | 9 | #[derive(Debug)] 10 | pub enum AskError { 11 | SendError, 12 | ReceiveError, 13 | } 14 | 15 | impl Display for AskError { 16 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 17 | match self { 18 | AskError::SendError => write!(f, "SendError"), 19 | AskError::ReceiveError => write!(f, "ReceiveError"), 20 | } 21 | } 22 | } 23 | 24 | impl Error for AskError {} 25 | 26 | #[async_trait] 27 | pub trait Ask { 28 | /// The ask pattern is a way to send a request and get a reply back. 29 | async fn ask(&self, f: F) -> Result 30 | where 31 | F: FnOnce(Box) -> A + Send, 32 | R: Send + 'static; 33 | } 34 | 35 | #[async_trait] 36 | impl Ask for mpsc::Sender 37 | where 38 | A: Send, 39 | { 40 | async fn ask(&self, f: F) -> Result 41 | where 42 | F: FnOnce(Box) -> A + Send, 43 | R: Send + 'static, 44 | { 45 | let (tx, rx) = oneshot::channel(); 46 | let reply_to = Box::new(|r| { 47 | let _ = tx.send(r); 48 | }); 49 | 50 | let request = f(reply_to); 51 | self.send(request).await.map_err(|_| AskError::SendError)?; 52 | 53 | rx.await.map_err(|_| AskError::ReceiveError) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /streambed-patterns/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | 3 | pub mod ask; 4 | -------------------------------------------------------------------------------- /streambed-storage/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed-storage" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "Functionality for loading an persisting structures" 9 | 10 | [dependencies] 11 | async-trait = { workspace = true } 12 | clap = { workspace = true, features = ["derive", "env"] } 13 | humantime = { workspace = true } 14 | rand = { workspace = true } 15 | serde = { workspace = true, features = ["derive"] } 16 | streambed = { workspace = true } 17 | tokio = { workspace = true, features = ["fs", "io-util"] } 18 | -------------------------------------------------------------------------------- /streambed-storage/README.md: -------------------------------------------------------------------------------- 1 | # storage 2 | 3 | Functionality for loading an persisting structures. 4 | -------------------------------------------------------------------------------- /streambed-storage/src/args.rs: -------------------------------------------------------------------------------- 1 | //! Provides command line arguments that are typically used for all services using this module. 2 | 3 | use std::path::PathBuf; 4 | 5 | use clap::Parser; 6 | 7 | #[derive(Parser, Debug)] 8 | pub struct StateStorageArgs { 9 | /// A file system path to where state may be stored 10 | #[clap(env, long, default_value = "/var/run")] 11 | pub state_storage_path: PathBuf, 12 | 13 | /// The interval between connecting to the commit-log-server and storing our state. 14 | #[clap(env, long, default_value = "1m")] 15 | pub state_store_interval: humantime::Duration, 16 | } 17 | -------------------------------------------------------------------------------- /streambed-storage/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | 3 | pub mod args; 4 | 5 | use rand::RngCore; 6 | use serde::{de::DeserializeOwned, Serialize}; 7 | use std::{error::Error, path::Path}; 8 | use streambed::{decrypt_buf, encrypt_struct, secret_store}; 9 | use tokio::{ 10 | fs, 11 | io::{AsyncReadExt, AsyncWriteExt}, 12 | }; 13 | 14 | /// Loads and deserializes a structure described by T, if the file is present. 15 | /// If there are IO issues outside of the file not being there, they will be returned 16 | /// as an error. Beyond IO, state is attempted to be decrypted and deserialized when present. 17 | /// Any issues there cause the default representation of the structure to be returned. The default 18 | /// structure is also returned where there is no file present in the first place. 19 | pub async fn load_struct( 20 | state_storage_path: &Path, 21 | ss: &impl secret_store::SecretStore, 22 | secret_path: &str, 23 | deserialize: D, 24 | ) -> Result> 25 | where 26 | T: Default + DeserializeOwned, 27 | D: FnOnce(&[u8]) -> Result, 28 | { 29 | if let Ok(mut f) = fs::File::open(state_storage_path).await { 30 | let mut buf = vec![]; 31 | f.read_to_end(&mut buf).await?; 32 | Ok(decrypt_buf(ss, secret_path, &mut buf, deserialize) 33 | .await 34 | .unwrap_or_default()) 35 | } else { 36 | Ok(T::default()) 37 | } 38 | } 39 | 40 | /// Saves an encrypted structure described by T. Any IO errors are returned. 41 | pub async fn save_struct( 42 | state_storage_path: &Path, 43 | ss: &impl secret_store::SecretStore, 44 | secret_path: &str, 45 | serialize: S, 46 | rng: F, 47 | state: &T, 48 | ) -> Result<(), Box> 49 | where 50 | T: Serialize, 51 | S: FnOnce(&T) -> Result, SE>, 52 | F: FnOnce() -> U, 53 | U: RngCore, 54 | { 55 | if let Some(buf) = encrypt_struct(ss, secret_path, serialize, rng, state).await { 56 | let mut f = fs::File::create(state_storage_path).await?; 57 | f.write_all(&buf).await?; 58 | } 59 | Ok(()) 60 | } 61 | -------------------------------------------------------------------------------- /streambed-test/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed-test" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "A collection of utilities that facilitate unit and integration testing" 9 | 10 | [dependencies] 11 | http = { workspace = true } 12 | hyper = { workspace = true, default-features = false, features = ["tcp", "stream", "http1", "http2", "client", "server", "runtime"] } 13 | tokio = { workspace = true } 14 | -------------------------------------------------------------------------------- /streambed-test/README.md: -------------------------------------------------------------------------------- 1 | # Test 2 | 3 | A collection of utilities that facilitate unit and integration testing. -------------------------------------------------------------------------------- /streambed-test/src/lib.rs: -------------------------------------------------------------------------------- 1 | /// Taken from https://github.com/seanmonstar/reqwest/blob/master/tests/support/mod.rs 2 | pub mod server; 3 | 4 | #[allow(unused)] 5 | pub static DEFAULT_USER_AGENT: &str = 6 | concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); 7 | -------------------------------------------------------------------------------- /streambed-test/src/server.rs: -------------------------------------------------------------------------------- 1 | /// Taken from https://github.com/seanmonstar/reqwest/blob/master/tests/support/server.rs 2 | use std::convert::Infallible; 3 | use std::future::Future; 4 | use std::net; 5 | use std::sync::mpsc as std_mpsc; 6 | use std::thread; 7 | use std::time::Duration; 8 | 9 | use tokio::sync::oneshot; 10 | 11 | pub use http::Response; 12 | use tokio::runtime; 13 | 14 | pub struct Server { 15 | addr: net::SocketAddr, 16 | panic_rx: std_mpsc::Receiver<()>, 17 | shutdown_tx: Option>, 18 | } 19 | 20 | impl Server { 21 | pub fn addr(&self) -> net::SocketAddr { 22 | self.addr 23 | } 24 | } 25 | 26 | impl Drop for Server { 27 | fn drop(&mut self) { 28 | if let Some(tx) = self.shutdown_tx.take() { 29 | let _ = tx.send(()); 30 | } 31 | 32 | if !::std::thread::panicking() { 33 | self.panic_rx 34 | .recv_timeout(Duration::from_secs(3)) 35 | .expect("test server should not panic"); 36 | } 37 | } 38 | } 39 | 40 | #[allow(clippy::async_yields_async)] 41 | pub fn http(func: F) -> Server 42 | where 43 | F: Fn(http::Request) -> Fut + Clone + Send + 'static, 44 | Fut: Future> + Send + 'static, 45 | { 46 | //Spawn new runtime in thread to prevent reactor execution context conflict 47 | thread::spawn(move || { 48 | let rt = runtime::Builder::new_current_thread() 49 | .enable_all() 50 | .build() 51 | .expect("new rt"); 52 | let srv = rt.block_on(async move { 53 | hyper::Server::bind(&([127, 0, 0, 1], 0).into()).serve(hyper::service::make_service_fn( 54 | move |_| { 55 | let func = func.clone(); 56 | async move { 57 | Ok::<_, Infallible>(hyper::service::service_fn(move |req| { 58 | let fut = func(req); 59 | async move { Ok::<_, Infallible>(fut.await) } 60 | })) 61 | } 62 | }, 63 | )) 64 | }); 65 | 66 | let addr = srv.local_addr(); 67 | let (shutdown_tx, shutdown_rx) = oneshot::channel(); 68 | let srv = srv.with_graceful_shutdown(async move { 69 | let _ = shutdown_rx.await; 70 | }); 71 | 72 | let (panic_tx, panic_rx) = std_mpsc::channel(); 73 | let tname = format!( 74 | "test({})-support-server", 75 | thread::current().name().unwrap_or("") 76 | ); 77 | thread::Builder::new() 78 | .name(tname) 79 | .spawn(move || { 80 | rt.block_on(srv).unwrap(); 81 | let _ = panic_tx.send(()); 82 | }) 83 | .expect("thread spawn"); 84 | 85 | Server { 86 | addr, 87 | panic_rx, 88 | shutdown_tx: Some(shutdown_tx), 89 | } 90 | }) 91 | .join() 92 | .unwrap() 93 | } 94 | -------------------------------------------------------------------------------- /streambed-vault/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed-vault" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "Provides an implementation of the Streambed secret store to be used with the Hashicorp Vault API" 9 | 10 | [dependencies] 11 | async-trait = { workspace = true } 12 | cache_loader_async = { workspace = true, features = ["lru-cache", "ttl-cache"] } 13 | clap = { workspace = true, features = ["derive", "env"] } 14 | humantime = { workspace = true } 15 | log = { workspace = true } 16 | metrics = { workspace = true, default-features = false } 17 | reqwest = { workspace = true, features = ["json"] } 18 | serde = { workspace = true, features = ["derive"] } 19 | streambed = { workspace = true } 20 | tokio = { workspace = true } 21 | 22 | [dev-dependencies] 23 | env_logger = { workspace = true } 24 | http = { workspace = true } 25 | hyper = { workspace = true, default-features = false, features = ["tcp", "stream", "http1", "http2", "client", "server", "runtime"] } 26 | streambed = { workspace = true, features = ["reqwest"] } 27 | streambed-test = { workspace = true } 28 | test-log = { workspace = true } 29 | tokio-stream = { workspace = true } 30 | -------------------------------------------------------------------------------- /streambed-vault/README.md: -------------------------------------------------------------------------------- 1 | # Vault 2 | 3 | Provides an implementation of the Streambed secret store to be used with the 4 | [Hashicorp Vault API](https://www.vaultproject.io/api-docs). 5 | 6 | At this time V1 of the KV store is supported. -------------------------------------------------------------------------------- /streambed-vault/examples/README.md: -------------------------------------------------------------------------------- 1 | # vault authenticator 2 | 3 | This example illustrates the essential setup of a Vault secret store and authenticates a service. 4 | It then goes on to authenticate a username/password. 5 | 6 | > Note that you'll need both [`vault`](https://www.vaultproject.io/) and [`jq`](https://stedolan.github.io/jq/). 7 | 8 | To begin with, you must start Vault in development mode: 9 | 10 | ``` 11 | vault server -dev 12 | ``` 13 | 14 | In another terminal, enter the following commands to enable app role authentication, configure policies, 15 | and the obtain a role_id and secret_id for our example service: 16 | 17 | ``` 18 | export VAULT_ADDR='http://127.0.0.1:8200' 19 | 20 | vault kv put secret/default/secrets.some-secret value=some-secret-value 21 | 22 | vault auth enable approle 23 | 24 | vault policy write my-example-policy -< Result<(), Box> { 18 | let args = Args::parse(); 19 | 20 | env_logger::init(); 21 | info!("Vault authenticator started"); 22 | 23 | // Setup and authenticate our service with the secret store 24 | 25 | let ss_secret_id = streambed::read_line(std::io::stdin()).unwrap(); 26 | assert!( 27 | !ss_secret_id.is_empty(), 28 | "Cannot source a secret id from stdin" 29 | ); 30 | 31 | let ss_server_cert = if let Some(path) = args.ss_args.ss_server_cert_path { 32 | Some( 33 | streambed::read_pem(std::fs::File::open(&path)?) 34 | .await 35 | .unwrap_or_else(|_| panic!("Cannot find ss_server_cert_path at {path:?}")), 36 | ) 37 | } else { 38 | None 39 | }; 40 | 41 | let ss = VaultSecretStore::new( 42 | args.ss_args.ss_server, 43 | ss_server_cert, 44 | args.ss_args.ss_server_insecure, 45 | args.ss_args.ss_unauthorized_timeout.into(), 46 | args.ss_args.ss_max_secrets, 47 | args.ss_args.ss_ttl_field.as_deref(), 48 | ); 49 | 50 | streambed::reauthenticate_secret_store( 51 | ss.clone(), 52 | &args.ss_args.ss_role_id, 53 | &ss_secret_id, 54 | args.ss_args.ss_unauthenticated_timeout.into(), 55 | args.ss_args.ss_max_lease_duration.into(), 56 | ) 57 | .await; 58 | 59 | // At this point, the secret store may not be available to us (it may not be online). 60 | // This is a normal condition - the above `streambed::authenticate_secret_store` 61 | // call will spawn a task to retry connecting and then authentication, plus 62 | // re-authenticate when a token has expired. 63 | // 64 | // We are going to assume that the secret store is available though for the purposes of this example. 65 | // If it isn't then you'll see errors. 66 | 67 | // Let's now read a secret that our role should permit reading 68 | 69 | let secret_path = format!("{}/secrets.some-secret", args.ss_args.ss_ns); 70 | 71 | if let Some(value) = streambed::get_secret_value(&ss, &secret_path).await { 72 | info!("Read secrets.some-secret: {:?}", value); 73 | } 74 | 75 | // Now authenticate as a human i.e. with a username and password. These credentials 76 | // would normally come from the outside e.g. via an http login. 77 | // To authenticate, we base a new store on our existing one and authenticate again. 78 | // The idea is that we create and drop secret store instances in association with 79 | // user sessions. A secret store instance will share HTTP resources with the other 80 | // instances it has been associated with, and its own cache of secrets. 81 | 82 | let user_ss = VaultSecretStore::with_new_auth_prepared(&ss); 83 | let auth_token = user_ss.userpass_auth("mitchellh", "foo").await; 84 | info!("User pass auth token: {:?}", auth_token); 85 | 86 | // Given our new identity, we should no longer be able to access that secret we 87 | // accessed before. 88 | 89 | if streambed::get_secret_value(&user_ss, &secret_path) 90 | .await 91 | .is_none() 92 | { 93 | info!("Correctly disallows access to the secret given the user access"); 94 | } 95 | 96 | // For fun, let's update our credentials to the same values. Note how we do 97 | // this from our service's identity, not the user one. For our example, our user 98 | // actually doesn't have the role assigned such that it is able to. 99 | if ss 100 | .userpass_create_update_user("mitchellh", "mitchellh", "foo") 101 | .await 102 | .is_ok() 103 | { 104 | info!("User pass updated"); 105 | } 106 | 107 | Ok(()) 108 | } 109 | -------------------------------------------------------------------------------- /streambed-vault/src/args.rs: -------------------------------------------------------------------------------- 1 | //! Provides command line arguments that are typically used for all services using this module. 2 | 3 | use std::path::PathBuf; 4 | 5 | use clap::Parser; 6 | use reqwest::Url; 7 | 8 | #[derive(Parser, Debug)] 9 | pub struct SsArgs { 10 | /// The maximum time we allow for a lease to expire. This is important to curb 11 | /// situations where the secret store is not honoring a previously advised 12 | /// lease duration for some reason. While this should never happen, given the 13 | /// nature of the secret store being in a separate process, it technically 14 | /// can happen (and has!). 15 | #[clap(env, long, default_value = "30m")] 16 | pub ss_max_lease_duration: humantime::Duration, 17 | 18 | /// The max number of Vault Secret Store secrets to retain by our cache at any time. 19 | /// Least Recently Used (LRU) secrets will be evicted from our cache once this value 20 | /// is exceeded. 21 | #[clap(env, long, default_value_t = 10_000)] 22 | pub ss_max_secrets: usize, 23 | 24 | /// A namespace to use when communicating with the Vault Secret Store 25 | #[clap(env, long, default_value = "default")] 26 | pub ss_ns: String, 27 | 28 | /// The Vault Secret Store role_id to use for approle authentication. 29 | #[clap(env, long)] 30 | pub ss_role_id: String, 31 | 32 | /// A URL of the Vault Secret Store server to communicate with 33 | #[clap(env, long, default_value = "http://localhost:9876")] 34 | pub ss_server: Url, 35 | 36 | /// A path to a TLS cert pem file to be used for connecting with the Vault Secret Store server. 37 | #[clap(env, long)] 38 | pub ss_server_cert_path: Option, 39 | 40 | /// Insecurely trust the Secret Store's TLS certificate 41 | #[clap(env, long)] 42 | pub ss_server_insecure: bool, 43 | 44 | /// A data field to used in place of Vault's lease_duration field. Time 45 | /// will be interpreted as a humantime string e.g. "1m", "1s" etc. Note 46 | /// that v2 of the Vault server does not appear to populate the lease_duration 47 | /// field for the KV secret store any longer. Instead, we can use a "ttl" field 48 | /// from the data. 49 | #[clap(env, long)] 50 | pub ss_ttl_field: Option, 51 | 52 | /// How long we wait until re-requesting the Vault Secret Store server for an 53 | /// authentication given a bad auth prior. 54 | #[clap(env, long, default_value = "1m")] 55 | pub ss_unauthenticated_timeout: humantime::Duration, 56 | 57 | /// How long we wait until re-requesting the Vault Secret Store server for an 58 | /// unauthorized secret again. 59 | #[clap(env, long, default_value = "1m")] 60 | pub ss_unauthorized_timeout: humantime::Duration, 61 | } 62 | -------------------------------------------------------------------------------- /streambed-vault/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | 3 | pub mod args; 4 | 5 | use std::{sync::Arc, time::Duration}; 6 | 7 | use async_trait::async_trait; 8 | use cache_loader_async::{ 9 | backing::{LruCacheBacking, TtlCacheBacking, TtlMeta}, 10 | cache_api::{CacheEntry, LoadingCache, WithMeta}, 11 | }; 12 | use log::debug; 13 | use metrics::increment_counter; 14 | use reqwest::{Certificate, Client, StatusCode, Url}; 15 | use serde::{Deserialize, Serialize}; 16 | use tokio::{sync::Mutex, time::Instant}; 17 | 18 | use streambed::{ 19 | delayer::Delayer, 20 | secret_store::{ 21 | AppRoleAuthReply, Error, GetSecretReply, SecretData, SecretStore, UserPassAuthReply, 22 | }, 23 | }; 24 | 25 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 26 | struct AppRoleAuthRequest { 27 | pub role_id: String, 28 | pub secret_id: String, 29 | } 30 | 31 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 32 | struct UserPassAuthRequest { 33 | pub password: String, 34 | } 35 | 36 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 37 | struct UserPassCreateUpdateRequest { 38 | pub username: String, 39 | pub password: String, 40 | } 41 | 42 | type TtlCache = LoadingCache< 43 | String, 44 | Option, 45 | Error, 46 | TtlCacheBacking< 47 | String, 48 | CacheEntry, Error>, 49 | LruCacheBacking, Error>, Instant)>, 50 | >, 51 | >; 52 | 53 | /// A client interface that uses the Hashicorp Vault HTTP API. 54 | #[derive(Clone)] 55 | pub struct VaultSecretStore { 56 | cache: TtlCache, 57 | client: Client, 58 | client_token: Arc>>, 59 | max_secrets_cached: usize, 60 | server: Url, 61 | ttl_field: Option, 62 | unauthorized_timeout: Duration, 63 | } 64 | 65 | const APPROLE_AUTH_LABEL: &str = "approle_auth"; 66 | const SECRET_PATH_LABEL: &str = "secret_path"; 67 | const USERPASS_AUTH_LABEL: &str = "userpass_auth"; 68 | const USERPASS_CREATE_UPDATE_LABEL: &str = "userpass_create_update"; 69 | 70 | impl VaultSecretStore { 71 | /// Establish a new client to Hashicorp Vault. In the case where TLS is required, 72 | /// a root certificate may be provided e.g. when using self-signed certificates. TLS 73 | /// connections are encouraged. 74 | /// An unauthorized_timeout determines how long the server should wait before being 75 | /// requested again. 76 | /// A max_secrets_cached arg limits the number of secrets that can be held at any time. 77 | /// 78 | /// Avoid creating many new Vault secret stores and clone them instead so that HTTP 79 | /// connection pools can be shared. 80 | pub fn new( 81 | server: Url, 82 | server_cert: Option, 83 | tls_insecure: bool, 84 | unauthorized_timeout: Duration, 85 | max_secrets_cached: usize, 86 | ttl_field: Option<&str>, 87 | ) -> Self { 88 | let client = Client::builder().danger_accept_invalid_certs(tls_insecure); 89 | let client = if let Some(cert) = server_cert { 90 | client.add_root_certificate(cert) 91 | } else { 92 | client 93 | }; 94 | 95 | Self::with_new_cache( 96 | client.build().unwrap(), 97 | server, 98 | unauthorized_timeout, 99 | max_secrets_cached, 100 | ttl_field.map(|s| s.to_string()), 101 | ) 102 | } 103 | 104 | fn with_new_cache( 105 | client: Client, 106 | server: Url, 107 | unauthorized_timeout: Duration, 108 | max_secrets_cached: usize, 109 | ttl_field: Option, 110 | ) -> Self { 111 | let client_token: Arc>> = Arc::new(Mutex::new(None)); 112 | let retained_client_token = Arc::clone(&client_token); 113 | 114 | let retained_client = client.clone(); 115 | let retained_server = server.clone(); 116 | let retained_ttl_field = ttl_field.clone(); 117 | let retained_unauthorized_timeout = unauthorized_timeout; 118 | 119 | let cache: TtlCache = LoadingCache::with_meta_loader( 120 | TtlCacheBacking::with_backing( 121 | unauthorized_timeout, 122 | LruCacheBacking::new(max_secrets_cached), 123 | ), 124 | move |secret_path| { 125 | let task_client_token = Arc::clone(&client_token); 126 | 127 | let task_client = client.clone(); 128 | let task_server = server.clone(); 129 | let task_ttl_field = ttl_field.clone(); 130 | 131 | async move { 132 | let mut delayer = Delayer::default(); 133 | loop { 134 | increment_counter!("ss_get_secret_requests", SECRET_PATH_LABEL => secret_path.clone()); 135 | 136 | let mut builder = task_client.get( 137 | task_server 138 | .join(&format!("{task_server}v1/secret/data/{secret_path}")) 139 | .unwrap(), 140 | ); 141 | if let Some(client_token) = task_client_token.lock().await.as_deref() { 142 | builder = builder.header("X-Vault-Token", client_token) 143 | } 144 | 145 | let result = builder.send().await; 146 | match result { 147 | Ok(response) => { 148 | if response.status() == StatusCode::FORBIDDEN { 149 | increment_counter!("ss_unauthorized", SECRET_PATH_LABEL => secret_path.clone()); 150 | break Err(Error::Unauthorized); 151 | } else { 152 | let secret_reply = if response.status().is_success() { 153 | response.json::().await.ok() 154 | } else { 155 | debug!( 156 | "Secret store failure status while getting secret: {:?}", 157 | response.status() 158 | ); 159 | increment_counter!("ss_other_reply_failures"); 160 | None 161 | }; 162 | let lease_duration = secret_reply.as_ref().map(|sr| { 163 | let mut lease_duration = None; 164 | if let Some(ttl_field) = task_ttl_field.as_ref() { 165 | if let Some(ttl) = sr.data.data.get(ttl_field) { 166 | if let Ok(ttl_duration) = 167 | ttl.parse::() 168 | { 169 | lease_duration = Some(ttl_duration.into()); 170 | } 171 | } 172 | } 173 | lease_duration.unwrap_or_else(|| { 174 | Duration::from_secs(sr.lease_duration) 175 | }) 176 | }); 177 | break Ok(secret_reply) 178 | .with_meta(lease_duration.map(TtlMeta::from)); 179 | } 180 | } 181 | Err(e) => { 182 | debug!( 183 | "Secret store is unavailable while getting secret. Error: {:?}", 184 | e 185 | ); 186 | increment_counter!("ss_unavailables"); 187 | } 188 | } 189 | delayer.delay().await; 190 | } 191 | } 192 | }, 193 | ); 194 | 195 | Self { 196 | cache, 197 | client: retained_client, 198 | client_token: retained_client_token, 199 | max_secrets_cached, 200 | ttl_field: retained_ttl_field, 201 | server: retained_server, 202 | unauthorized_timeout: retained_unauthorized_timeout, 203 | } 204 | } 205 | 206 | pub fn with_new_auth_prepared(ss: &Self) -> Self { 207 | Self::with_new_cache( 208 | ss.client.clone(), 209 | ss.server.clone(), 210 | ss.unauthorized_timeout, 211 | ss.max_secrets_cached, 212 | ss.ttl_field.clone(), 213 | ) 214 | } 215 | } 216 | 217 | #[async_trait] 218 | impl SecretStore for VaultSecretStore { 219 | async fn approle_auth( 220 | &self, 221 | role_id: &str, 222 | secret_id: &str, 223 | ) -> Result { 224 | loop { 225 | let role_id = role_id.to_string(); 226 | 227 | increment_counter!("ss_approle_auth_requests", APPROLE_AUTH_LABEL => role_id.clone()); 228 | 229 | let task_client_token = Arc::clone(&self.client_token); 230 | let result = self 231 | .client 232 | .post( 233 | self.server 234 | .join(&format!("{}v1/auth/approle/login", self.server)) 235 | .unwrap(), 236 | ) 237 | .json(&AppRoleAuthRequest { 238 | role_id: role_id.to_string(), 239 | secret_id: secret_id.to_string(), 240 | }) 241 | .send() 242 | .await; 243 | match result { 244 | Ok(response) => { 245 | if response.status() == StatusCode::FORBIDDEN { 246 | increment_counter!("ss_unauthorized", APPROLE_AUTH_LABEL => role_id.clone()); 247 | break Err(Error::Unauthorized); 248 | } else { 249 | let secret_reply = if response.status().is_success() { 250 | let approle_auth_reply = response 251 | .json::() 252 | .await 253 | .map_err(|_| Error::Unauthorized); 254 | if let Ok(r) = &approle_auth_reply { 255 | let mut client_token = task_client_token.lock().await; 256 | *client_token = Some(r.auth.client_token.clone()); 257 | } 258 | approle_auth_reply 259 | } else { 260 | debug!( 261 | "Secret store failure status while authenticating: {:?}", 262 | response.status() 263 | ); 264 | increment_counter!("ss_other_reply_failures", APPROLE_AUTH_LABEL => role_id.clone()); 265 | Err(Error::Unauthorized) 266 | }; 267 | break secret_reply; 268 | } 269 | } 270 | Err(e) => { 271 | debug!( 272 | "Secret store is unavailable while authenticating. Error: {:?}", 273 | e 274 | ); 275 | increment_counter!("ss_unavailables"); 276 | } 277 | } 278 | } 279 | } 280 | 281 | async fn create_secret( 282 | &self, 283 | _secret_path: &str, 284 | _secret_data: SecretData, 285 | ) -> Result<(), Error> { 286 | todo!() 287 | } 288 | 289 | async fn get_secret(&self, secret_path: &str) -> Result, Error> { 290 | self.cache 291 | .get(secret_path.to_owned()) 292 | .await 293 | .map_err(|e| e.as_loading_error().unwrap().clone()) // Unsure how we can deal with caching issues 294 | } 295 | 296 | async fn userpass_auth( 297 | &self, 298 | username: &str, 299 | password: &str, 300 | ) -> Result { 301 | let username = username.to_string(); 302 | 303 | increment_counter!("ss_userpass_auth_requests", USERPASS_AUTH_LABEL => username.clone()); 304 | 305 | let task_client_token = Arc::clone(&self.client_token); 306 | let result = self 307 | .client 308 | .post( 309 | self.server 310 | .join(&format!( 311 | "{}v1/auth/userpass/login/{}", 312 | self.server, username 313 | )) 314 | .unwrap(), 315 | ) 316 | .json(&UserPassAuthRequest { 317 | password: password.to_string(), 318 | }) 319 | .send() 320 | .await; 321 | match result { 322 | Ok(response) => { 323 | if response.status() == StatusCode::FORBIDDEN { 324 | increment_counter!("ss_unauthorized", USERPASS_AUTH_LABEL => username.clone()); 325 | Err(Error::Unauthorized) 326 | } else if response.status().is_success() { 327 | let userpass_auth_reply = response 328 | .json::() 329 | .await 330 | .map_err(|_| Error::Unauthorized); 331 | if let Ok(r) = &userpass_auth_reply { 332 | let mut client_token = task_client_token.lock().await; 333 | *client_token = Some(r.auth.client_token.clone()); 334 | } 335 | userpass_auth_reply 336 | } else { 337 | debug!( 338 | "Secret store failure status while authenticating: {:?}", 339 | response.status() 340 | ); 341 | increment_counter!("ss_other_reply_failures", USERPASS_AUTH_LABEL => username.clone()); 342 | Err(Error::Unauthorized) 343 | } 344 | } 345 | Err(e) => { 346 | debug!( 347 | "Secret store is unavailable while authenticating. Error: {:?}", 348 | e 349 | ); 350 | increment_counter!("ss_unavailables"); 351 | Err(Error::Unauthorized) 352 | } 353 | } 354 | } 355 | 356 | async fn token_auth(&self, _token: &str) -> Result<(), Error> { 357 | todo!() 358 | } 359 | 360 | async fn userpass_create_update_user( 361 | &self, 362 | current_username: &str, 363 | username: &str, 364 | password: &str, 365 | ) -> Result<(), Error> { 366 | let username = username.to_string(); 367 | 368 | increment_counter!("ss_userpass_create_updates", USERPASS_CREATE_UPDATE_LABEL => username.clone()); 369 | 370 | let mut builder = self 371 | .client 372 | .post( 373 | self.server 374 | .join(&format!( 375 | "{}v1/auth/userpass/users/{}", 376 | self.server, current_username 377 | )) 378 | .unwrap(), 379 | ) 380 | .json(&UserPassCreateUpdateRequest { 381 | username: username.to_string(), 382 | password: password.to_string(), 383 | }); 384 | if let Some(client_token) = self.client_token.lock().await.as_deref() { 385 | builder = builder.header("X-Vault-Token", client_token) 386 | } 387 | 388 | let result = builder.send().await; 389 | match result { 390 | Ok(response) => { 391 | if response.status() == StatusCode::FORBIDDEN { 392 | increment_counter!("ss_unauthorized", USERPASS_CREATE_UPDATE_LABEL => username.clone()); 393 | Err(Error::Unauthorized) 394 | } else if response.status().is_success() { 395 | let _ = response.json::<()>().await.map_err(|_| Error::Unauthorized); 396 | Ok(()) 397 | } else { 398 | debug!( 399 | "Secret store failure status while creating/updating userpass: {:?}", 400 | response.status() 401 | ); 402 | increment_counter!("ss_other_reply_failures", USERPASS_CREATE_UPDATE_LABEL => username.clone()); 403 | Err(Error::Unauthorized) 404 | } 405 | } 406 | Err(e) => { 407 | debug!( 408 | "Secret store is unavailable while creating/updating userpass. Error: {:?}", 409 | e 410 | ); 411 | increment_counter!("ss_unavailables"); 412 | Err(Error::Unauthorized) 413 | } 414 | } 415 | } 416 | } 417 | -------------------------------------------------------------------------------- /streambed-vault/tests/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod secret_store; 3 | -------------------------------------------------------------------------------- /streambed-vault/tests/secret_store.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::HashMap, 3 | sync::atomic::{AtomicUsize, Ordering}, 4 | time::Duration, 5 | }; 6 | 7 | use http::Method; 8 | use streambed_test::server; 9 | use streambed_vault::VaultSecretStore; 10 | 11 | use reqwest::Url; 12 | use std::str; 13 | use streambed::secret_store::*; 14 | use tokio_stream::StreamExt; 15 | 16 | #[tokio::test] 17 | async fn get_http_vault_secret_with_auth() { 18 | static HTTP_REQUESTS: AtomicUsize = AtomicUsize::new(0); 19 | 20 | let server = server::http(move |mut req| async move { 21 | let _ = HTTP_REQUESTS.fetch_add(1, Ordering::Relaxed); 22 | let body = match (req.method(), req.uri().to_string().as_ref()) { 23 | (&Method::POST, "/v1/auth/approle/login") => { 24 | assert!(req.headers().get("X-Vault-Token").is_none()); 25 | 26 | let mut req_body: Vec = Vec::new(); 27 | while let Some(item) = req.body_mut().next().await { 28 | req_body.extend(&*item.unwrap()); 29 | } 30 | 31 | assert_eq!( 32 | str::from_utf8(&req_body), 33 | Ok( 34 | r#"{"role_id":"0935c840-c870-dc18-a6a3-0af0f3421df7","secret_id":"7d63c6a5-75eb-1edb-d6ee-e7c657861a5f"}"# 35 | ) 36 | ); 37 | 38 | r#" 39 | { 40 | "request_id": "a65a6f27-0ad7-fbcd-3987-9cb70ba179a0", 41 | "lease_id": "", 42 | "renewable": false, 43 | "lease_duration": 0, 44 | "data": null, 45 | "wrap_info": null, 46 | "warnings": null, 47 | "auth": { 48 | "client_token": "hvs.CAESICKLN-7ZJbKHLWZFcJ-XVYIUdeCyb7ZUEoFz7vqq856VGh4KHGh2cy52UnhKaVdJMmtjazV5aVNFdFdGWndJM0c", 49 | "accessor": "Zd6UeKSyKIGQl3jZjGFLpj9u", 50 | "policies": [ 51 | "default", 52 | "farmo-integrator" 53 | ], 54 | "token_policies": [ 55 | "default", 56 | "farmo-integrator" 57 | ], 58 | "metadata": { 59 | "role_name": "integrator" 60 | }, 61 | "lease_duration": 0, 62 | "renewable": true, 63 | "entity_id": "170c0b57-96ea-ac97-ccd7-d79a9537b642", 64 | "token_type": "service", 65 | "orphan": true, 66 | "mfa_requirement": null, 67 | "num_uses": 0 68 | } 69 | } 70 | "# 71 | } 72 | (&Method::GET, "/v1/secret/data/secrets.end-device-events.key.0000000000018879") => { 73 | assert_eq!(req.headers()["X-Vault-Token"], "hvs.CAESICKLN-7ZJbKHLWZFcJ-XVYIUdeCyb7ZUEoFz7vqq856VGh4KHGh2cy52UnhKaVdJMmtjazV5aVNFdFdGWndJM0c"); 74 | r#" 75 | { 76 | "request_id": "3b6168d5-88a4-369e-97a5-fcb217c634a1", 77 | "lease_id": "", 78 | "renewable": false, 79 | "lease_duration": 0, 80 | "data": { 81 | "data": { 82 | "ttl": "1m", 83 | "value": "f9418ed3daaff8f808fe5f268bee6139" 84 | }, 85 | "metadata": { 86 | "created_time": "2022-05-25T05:22:24.138956Z", 87 | "custom_metadata": null, 88 | "deletion_time": "", 89 | "destroyed": false, 90 | "version": 1 91 | } 92 | }, 93 | "wrap_info": null, 94 | "warnings": null, 95 | "auth": null 96 | } 97 | "# 98 | } 99 | _ => panic!("Unexpected uri"), 100 | }; 101 | http::Response::new(body.into()) 102 | }); 103 | 104 | let server_addr = server.addr(); 105 | 106 | let cl = VaultSecretStore::new( 107 | Url::parse(&format!( 108 | "http://{}:{}", 109 | server_addr.ip(), 110 | server_addr.port() 111 | )) 112 | .unwrap(), 113 | None, 114 | false, 115 | Duration::from_secs(60), 116 | 1, 117 | Some("ttl"), 118 | ); 119 | 120 | let result = cl 121 | .approle_auth( 122 | "0935c840-c870-dc18-a6a3-0af0f3421df7", 123 | "7d63c6a5-75eb-1edb-d6ee-e7c657861a5f", 124 | ) 125 | .await; 126 | assert_eq!( 127 | result, 128 | Ok(AppRoleAuthReply { 129 | auth: AuthToken { 130 | client_token: "hvs.CAESICKLN-7ZJbKHLWZFcJ-XVYIUdeCyb7ZUEoFz7vqq856VGh4KHGh2cy52UnhKaVdJMmtjazV5aVNFdFdGWndJM0c".to_string(), 131 | lease_duration: 0, 132 | } 133 | }) 134 | ); 135 | 136 | let result = cl 137 | .get_secret("secrets.end-device-events.key.0000000000018879") 138 | .await; 139 | assert_eq!( 140 | result, 141 | Ok(Some(GetSecretReply { 142 | lease_duration: 0, 143 | data: SecretData { 144 | data: HashMap::from([ 145 | ("ttl".to_string(), "1m".to_string()), 146 | ( 147 | "value".to_string(), 148 | "f9418ed3daaff8f808fe5f268bee6139".to_string() 149 | ) 150 | ]) 151 | } 152 | })) 153 | ); 154 | let result = cl 155 | .get_secret("secrets.end-device-events.key.0000000000018879") 156 | .await; 157 | assert!(result.is_ok()); 158 | 159 | assert_eq!(HTTP_REQUESTS.load(Ordering::Relaxed), 2); 160 | } 161 | 162 | #[tokio::test] 163 | async fn userpass_auth() { 164 | static HTTP_REQUESTS: AtomicUsize = AtomicUsize::new(0); 165 | 166 | let server = server::http(move |mut req| async move { 167 | let _ = HTTP_REQUESTS.fetch_add(1, Ordering::Relaxed); 168 | let body = match (req.method(), req.uri().to_string().as_ref()) { 169 | (&Method::POST, "/v1/auth/userpass/login/some-user") => { 170 | assert!(req.headers().get("X-Vault-Token").is_none()); 171 | 172 | let mut req_body: Vec = Vec::new(); 173 | while let Some(item) = req.body_mut().next().await { 174 | req_body.extend(&*item.unwrap()); 175 | } 176 | 177 | assert_eq!(str::from_utf8(&req_body), Ok(r#"{"password":"some-pass"}"#)); 178 | 179 | r#" 180 | { 181 | "request_id": "a65a6f27-0ad7-fbcd-3987-9cb70ba179a0", 182 | "lease_id": "", 183 | "renewable": false, 184 | "lease_duration": 0, 185 | "data": null, 186 | "wrap_info": null, 187 | "warnings": null, 188 | "auth": { 189 | "client_token": "hvs.CAESICKLN-7ZJbKHLWZFcJ-XVYIUdeCyb7ZUEoFz7vqq856VGh4KHGh2cy52UnhKaVdJMmtjazV5aVNFdFdGWndJM0c", 190 | "accessor": "Zd6UeKSyKIGQl3jZjGFLpj9u", 191 | "policies": [ 192 | "default", 193 | "farmo-integrator" 194 | ], 195 | "token_policies": [ 196 | "default", 197 | "farmo-integrator" 198 | ], 199 | "metadata": { 200 | "role_name": "integrator" 201 | }, 202 | "lease_duration": 0, 203 | "renewable": true, 204 | "entity_id": "170c0b57-96ea-ac97-ccd7-d79a9537b642", 205 | "token_type": "service", 206 | "orphan": true, 207 | "mfa_requirement": null, 208 | "num_uses": 0 209 | } 210 | } 211 | "# 212 | } 213 | (&Method::POST, "/v1/auth/userpass/users/some-user") => { 214 | assert_eq!(req.headers()["X-Vault-Token"], "hvs.CAESICKLN-7ZJbKHLWZFcJ-XVYIUdeCyb7ZUEoFz7vqq856VGh4KHGh2cy52UnhKaVdJMmtjazV5aVNFdFdGWndJM0c"); 215 | 216 | let mut req_body: Vec = Vec::new(); 217 | while let Some(item) = req.body_mut().next().await { 218 | req_body.extend(&*item.unwrap()); 219 | } 220 | 221 | assert_eq!( 222 | str::from_utf8(&req_body), 223 | Ok(r#"{"username":"some-new-user","password":"some-new-pass"}"#) 224 | ); 225 | 226 | r#""# 227 | } 228 | _ => panic!("Unexpected uri"), 229 | }; 230 | http::Response::new(body.into()) 231 | }); 232 | 233 | let server_addr = server.addr(); 234 | 235 | let cl = VaultSecretStore::new( 236 | Url::parse(&format!( 237 | "http://{}:{}", 238 | server_addr.ip(), 239 | server_addr.port() 240 | )) 241 | .unwrap(), 242 | None, 243 | false, 244 | Duration::from_secs(60), 245 | 1, 246 | Some("ttl"), 247 | ); 248 | 249 | let result = cl.userpass_auth("some-user", "some-pass").await; 250 | assert_eq!( 251 | result, 252 | Ok(UserPassAuthReply { 253 | auth: AuthToken { 254 | client_token: "hvs.CAESICKLN-7ZJbKHLWZFcJ-XVYIUdeCyb7ZUEoFz7vqq856VGh4KHGh2cy52UnhKaVdJMmtjazV5aVNFdFdGWndJM0c".to_string(), 255 | lease_duration: 0, 256 | } 257 | }) 258 | ); 259 | 260 | let result = cl 261 | .userpass_create_update_user("some-user", "some-new-user", "some-new-pass") 262 | .await; 263 | assert!(result.is_ok()); 264 | 265 | assert_eq!(HTTP_REQUESTS.load(Ordering::Relaxed), 2); 266 | } 267 | -------------------------------------------------------------------------------- /streambed/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "streambed" 3 | version.workspace = true 4 | edition.workspace = true 5 | rust-version.workspace = true 6 | license.workspace = true 7 | repository.workspace = true 8 | description = "Streambed core library components" 9 | 10 | [dependencies] 11 | aes = { workspace = true } 12 | async-trait = { workspace = true } 13 | chrono = { workspace = true, features = ["serde"] } 14 | exponential-backoff = { workspace = true } 15 | hex = { workspace = true } 16 | hmac = { workspace = true } 17 | log = { workspace = true } 18 | rand = { workspace = true } 19 | reqwest = { workspace = true, optional = true } 20 | serde = { workspace = true, features = ["derive"] } 21 | serde_with = { workspace = true, features = ["base64"] } 22 | sha2 = { workspace = true } 23 | smol_str = { workspace = true, features = ["serde"] } 24 | tokio = { workspace = true, features = [ "macros", "rt" ] } 25 | tokio-stream = { workspace = true } 26 | 27 | [dev-dependencies] 28 | serde_json = { workspace = true } 29 | 30 | [features] 31 | # Use the reqwest library with its default features incl. TLS 32 | reqwest = ["dep:reqwest"] -------------------------------------------------------------------------------- /streambed/README.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /streambed/src/commit_log.rs: -------------------------------------------------------------------------------- 1 | //! [A commit log is an append-only data structure that can be used 2 | //! in a variety of use-cases, such as tracking sequences of events, 3 | //! transactions or replicated state machines](https://docs.rs/commitlog/latest/commitlog/). 4 | //! 5 | //! Commit log functionality that is modelled on [Apache Kafka's](https://kafka.apache.org/) 6 | //! API, and can be implemented with multiple types of backend 7 | //! e.g. one that uses the Kafka HTTP REST API. 8 | 9 | use std::{pin::Pin, time::Duration}; 10 | 11 | use async_trait::async_trait; 12 | use chrono::{DateTime, Utc}; 13 | use serde::{Deserialize, Deserializer, Serialize}; 14 | use serde_with::{base64::Base64, serde_as}; 15 | use smol_str::SmolStr; 16 | use tokio_stream::Stream; 17 | 18 | /// An offset into a commit log. Offsets are used to address 19 | /// records and can be relied on to have an ascending order. 20 | pub type Offset = u64; 21 | 22 | /// Each record in a commit log has a key. How the key is formed 23 | /// is an application concern. By way of an example, keys can be 24 | /// used to associate to an entity. 25 | pub type Key = u64; 26 | 27 | /// Topics can be distributed into partitions which, in turn, 28 | /// enable scaling. 29 | pub type Partition = u32; 30 | 31 | /// A topic to subscribe to or has been subscribed to. Topics 32 | /// may be namespaced by prefixing with characters followed by 33 | /// a `:`. For example, "my-ns:my-topic". In the absence of 34 | /// a namespace, the server will assume a default namespace. 35 | pub type Topic = SmolStr; 36 | 37 | /// Header key type 38 | pub type HeaderKey = SmolStr; 39 | 40 | /// A header provides a means of augmenting a record with 41 | /// meta-data. 42 | #[serde_as] 43 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 44 | pub struct Header { 45 | pub key: HeaderKey, 46 | #[serde_as(as = "Base64")] 47 | pub value: Vec, 48 | } 49 | 50 | /// A declaration of an offset to be committed to a topic. 51 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 52 | pub struct ConsumerOffset { 53 | pub topic: Topic, 54 | pub partition: Partition, 55 | pub offset: Offset, 56 | } 57 | 58 | /// A declaration of a topic to subscribe to 59 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 60 | pub struct Subscription { 61 | pub topic: Topic, 62 | } 63 | 64 | /// A declaration of a consumer group session to connect with. 65 | /// In the case that offsets are supplied, these offsets are 66 | /// associated with their respective topics such that any 67 | /// subsequent subscription will source from the offset. 68 | /// In the case where subscriptions are supplied, the consumer 69 | /// instance will subscribe and reply with a stream of records 70 | /// ending only when the connection to the topic is severed. 71 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 72 | pub struct Consumer { 73 | #[serde(deserialize_with = "nullable_vec")] 74 | pub offsets: Vec, 75 | pub subscriptions: Vec, 76 | } 77 | 78 | /// A declaration of a record produced by a subscription 79 | #[serde_as] 80 | #[derive(Clone, Deserialize, Debug, Eq, PartialEq, Serialize)] 81 | pub struct ConsumerRecord { 82 | pub topic: Topic, 83 | #[serde(deserialize_with = "nullable_vec")] 84 | pub headers: Vec
, 85 | pub timestamp: Option>, 86 | pub key: Key, 87 | #[serde_as(as = "Base64")] 88 | pub value: Vec, 89 | pub partition: Partition, 90 | pub offset: Offset, 91 | } 92 | 93 | /// The reply to an offsets request 94 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 95 | pub struct PartitionOffsets { 96 | pub beginning_offset: Offset, 97 | pub end_offset: Offset, 98 | } 99 | 100 | /// A declaration of a record to be produced to a topic 101 | #[serde_as] 102 | #[derive(Clone, Deserialize, Debug, Eq, PartialEq, Serialize)] 103 | pub struct ProducerRecord { 104 | pub topic: Topic, 105 | #[serde(deserialize_with = "nullable_vec")] 106 | pub headers: Vec
, 107 | pub timestamp: Option>, 108 | pub key: Key, 109 | #[serde_as(as = "Base64")] 110 | pub value: Vec, 111 | pub partition: Partition, 112 | } 113 | 114 | /// The reply to a publish request 115 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 116 | pub struct ProducedOffset { 117 | pub offset: Offset, 118 | } 119 | 120 | /// There was a problem producing a record 121 | #[derive(Clone, Debug, Eq, PartialEq)] 122 | pub enum ProducerError { 123 | /// The commit log received the request but was unable to process it. 124 | CannotProduce, 125 | } 126 | 127 | /// A commit log holds topics and can be appended to and tailed. 128 | /// Connections are managed and retried if they cannot be established. 129 | #[async_trait] 130 | pub trait CommitLog: Clone + Send + Sync { 131 | /// Retrieve the current offsets of a topic if they are present. 132 | async fn offsets(&self, topic: Topic, partition: Partition) -> Option; 133 | 134 | /// Publish a record and return the offset that was assigned. 135 | async fn produce(&self, record: ProducerRecord) -> Result; 136 | 137 | /// Subscribe to one or more topics for a given consumer group 138 | /// having committed zero or more topics. The records are streamed 139 | /// back indefinitely unless an idle timeout argument is provided. 140 | /// In the case of an idle timeout, if no record is received 141 | /// within that period, None is returned to end the stream. 142 | fn scoped_subscribe<'a>( 143 | &'a self, 144 | consumer_group_name: &str, 145 | offsets: Vec, 146 | subscriptions: Vec, 147 | idle_timeout: Option, 148 | ) -> Pin + Send + 'a>>; 149 | } 150 | 151 | fn nullable_vec<'de, D, T>(d: D) -> Result, D::Error> 152 | where 153 | D: Deserializer<'de>, 154 | T: Deserialize<'de>, 155 | { 156 | Deserialize::deserialize(d).map(|x: Option<_>| x.unwrap_or_default()) 157 | } 158 | 159 | #[cfg(test)] 160 | mod tests { 161 | use super::*; 162 | 163 | #[test] 164 | fn test_nullable_vec_handles_null() { 165 | let json = r#" 166 | { 167 | "offsets": null, 168 | "subscriptions": [] 169 | } 170 | "#; 171 | assert_eq!( 172 | serde_json::from_str::(json).unwrap(), 173 | Consumer { 174 | offsets: vec![], 175 | subscriptions: vec![] 176 | } 177 | ); 178 | } 179 | 180 | #[test] 181 | fn test_nullable_vec_handles_empty_vec() { 182 | let json = r#" 183 | { 184 | "offsets": [], 185 | "subscriptions": [] 186 | } 187 | "#; 188 | assert_eq!( 189 | serde_json::from_str::(json).unwrap(), 190 | Consumer { 191 | offsets: vec![], 192 | subscriptions: vec![] 193 | } 194 | ); 195 | } 196 | 197 | #[test] 198 | fn test_nullable_vec_handles_vec() { 199 | let json = r#" 200 | { 201 | "offsets": [{"topic": "topic", "partition": 0, "offset": 0}], 202 | "subscriptions": [] 203 | } 204 | "#; 205 | assert_eq!( 206 | serde_json::from_str::(json).unwrap(), 207 | Consumer { 208 | offsets: vec![ConsumerOffset { 209 | topic: Topic::from("topic"), 210 | partition: 0, 211 | offset: 0 212 | }], 213 | subscriptions: vec![] 214 | } 215 | ); 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /streambed/src/crypto.rs: -------------------------------------------------------------------------------- 1 | //! Helpers to encrypt and decrypt things, particularly message payloads. 2 | 3 | use aes::{ 4 | cipher::{generic_array::GenericArray, BlockEncrypt}, 5 | Aes128, 6 | }; 7 | use rand::RngCore; 8 | use sha2::{Digest, Sha256, Sha512}; 9 | 10 | use hmac::Hmac; 11 | type HmacSha256 = Hmac; 12 | 13 | const BLOCKSIZE: usize = 16; 14 | 15 | pub const KEY_SIZE: usize = 16; 16 | pub const SALT_SIZE: usize = 12; 17 | 18 | /// Generate a random salt for the required SALT_SIZE. 19 | pub fn salt(rng: &mut T) -> [u8; SALT_SIZE] 20 | where 21 | T: RngCore, 22 | { 23 | let mut bytes = [0; SALT_SIZE]; 24 | rng.fill_bytes(&mut bytes); 25 | bytes 26 | } 27 | 28 | /// Performs an AES-128 CTR-blockmode-style decryption as used by 29 | /// LoRaWAN FRMPayloads. For more information on CTR: 30 | /// 31 | /// This is the same as the `encrypt` function, but named differently 32 | /// to convey intent. 33 | #[inline] 34 | pub fn decrypt(bytes: &mut [u8], key: &[u8; KEY_SIZE], salt: &[u8; SALT_SIZE]) { 35 | encrypt(bytes, key, salt) 36 | } 37 | 38 | /// Performs an AES-128 CTR-blockmode-style encryption as used by 39 | /// LoRaWAN FRMPayloads. For more information on CTR: 40 | /// 41 | pub fn encrypt(bytes: &mut [u8], key: &[u8; KEY_SIZE], salt: &[u8; SALT_SIZE]) { 42 | use aes::cipher::KeyInit; 43 | let key = GenericArray::from_slice(key); 44 | let cipher = Aes128::new(key); 45 | 46 | let bytes_len = bytes.len(); 47 | 48 | let block_count = (bytes_len.max(1) - 1) / BLOCKSIZE + 1; 49 | let mut blocks = Vec::with_capacity(block_count); 50 | let mut block = Vec::::with_capacity(BLOCKSIZE); 51 | for i in 1..=block_count { 52 | block.clear(); 53 | let bytes = (i as u32).to_be_bytes(); 54 | block.extend_from_slice(&bytes); 55 | block.extend_from_slice(salt); 56 | blocks.push(GenericArray::clone_from_slice(&block)); 57 | } 58 | cipher.encrypt_blocks(&mut blocks); 59 | 60 | let mut offset = 0; 61 | 'outer: for block in blocks { 62 | for b in block { 63 | if offset < bytes_len { 64 | bytes[offset] ^= b; 65 | offset += 1; 66 | } else { 67 | break 'outer; 68 | } 69 | } 70 | } 71 | } 72 | 73 | /// Sign an array of bytes using a key. Given the same bytes and key, this will always 74 | /// result in the same output. 75 | pub fn sign(bytes: &[u8], key: &[u8]) -> Vec { 76 | use hmac::Mac; 77 | let mut mac = HmacSha256::new_from_slice(key).unwrap(); 78 | mac.update(bytes); 79 | mac.finalize().into_bytes().to_vec() 80 | } 81 | 82 | /// Verify that a signed value is equal to an original array of bytes 83 | pub fn verify(bytes: &[u8], key: &[u8], compare_bytes: &[u8]) -> bool { 84 | use hmac::Mac; 85 | let mut mac = HmacSha256::new_from_slice(key).unwrap(); 86 | mac.update(bytes); 87 | mac.verify_slice(compare_bytes).is_ok() 88 | } 89 | 90 | /// Hash an array of bytes with a salt. Yields a SHA-512 hash. 91 | pub fn hash(bytes: &[u8], salt: &[u8; SALT_SIZE]) -> Vec { 92 | let mut hasher = Sha512::new(); 93 | hasher.update(salt); 94 | hasher.update(bytes); 95 | let mut hash = salt.to_vec(); 96 | hash.extend(hasher.finalize()); 97 | hash 98 | } 99 | 100 | #[cfg(test)] 101 | mod tests { 102 | use super::*; 103 | 104 | #[test] 105 | fn test_encrypt_decrypt() { 106 | let bytes = b"The quick brown fox jumps over the lazy dog."; 107 | let key = [0_u8; KEY_SIZE]; 108 | let salt = [0_u8; SALT_SIZE]; 109 | let mut encrypted_bytes = *bytes; 110 | encrypt(&mut encrypted_bytes, &key, &salt); 111 | assert_eq!(hex::encode( 112 | encrypted_bytes 113 | ), "bafc2e0f979c95ebeb6202ffc61631558b5b052e6b5d836a00b2cea50fc5aadd461c7ee09b333a0761a0796e"); 114 | let mut decrypted_bytes = encrypted_bytes; 115 | decrypt(&mut decrypted_bytes, &key, &salt); 116 | assert_eq!(&decrypted_bytes, bytes); 117 | } 118 | 119 | #[test] 120 | fn test_sign() { 121 | let bytes = b"The quick brown fox jumps over the lazy dog."; 122 | let key = b"changeme"; 123 | let hashed = sign(bytes, key); 124 | assert_eq!( 125 | hex::encode(hashed.clone()), 126 | "ed31e94c161aea6ff2300c72b17741f71b616463f294dac0542324bbdbf8a2de" 127 | ); 128 | assert!(verify(bytes, key, &hashed)); 129 | assert!(!verify(bytes, key, b"rubbish")); 130 | } 131 | 132 | #[test] 133 | fn test_hash() { 134 | let bytes = b"The quick brown fox jumps over the lazy dog."; 135 | let salt = [0_u8; SALT_SIZE]; 136 | let hashed = hash(bytes, &salt); 137 | assert_eq!(hex::encode( 138 | hashed 139 | ), "00000000000000000000000027cbc61af1b821ec863e77076df08016f41eb583d668426520b8ce5c85c150ada36f89061955c1bd12340764ee471f370528326dfa060a8c98978fb25774b35d"); 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /streambed/src/delayer.rs: -------------------------------------------------------------------------------- 1 | // A utility for delaying with exponential backoff. Once retries have 2 | // been attained then we start again. There are a hard set of values 3 | // here as it is private within Streambed. 4 | 5 | use std::time::Duration; 6 | 7 | use exponential_backoff::Backoff; 8 | use tokio::time; 9 | 10 | pub struct Delayer { 11 | backoff: Backoff, 12 | retry_attempt: u32, 13 | } 14 | 15 | impl Delayer { 16 | pub async fn delay(&mut self) { 17 | let delay = if let Some(d) = self.backoff.next(self.retry_attempt) { 18 | d 19 | } else { 20 | self.retry_attempt = 0; 21 | self.backoff.next(self.retry_attempt).unwrap() 22 | }; 23 | time::sleep(delay).await; 24 | self.retry_attempt = self.retry_attempt.wrapping_add(1); 25 | } 26 | } 27 | 28 | impl Default for Delayer { 29 | fn default() -> Self { 30 | let backoff = Backoff::new(8, Duration::from_millis(100), Duration::from_secs(10)); 31 | Self { 32 | backoff, 33 | retry_attempt: 0, 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /streambed/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | 3 | use std::io::{self, BufRead, BufReader, Read}; 4 | use std::sync::Arc; 5 | use std::time::Duration; 6 | 7 | use crypto::SALT_SIZE; 8 | use log::{debug, info, warn}; 9 | use rand::RngCore; 10 | #[cfg(feature = "reqwest")] 11 | use reqwest::Certificate; 12 | use serde::{Deserialize, Serialize}; 13 | use tokio::sync::Notify; 14 | use tokio::task::JoinHandle; 15 | use tokio::time; 16 | 17 | pub mod commit_log; 18 | pub mod crypto; 19 | pub mod delayer; 20 | pub mod secret_store; 21 | 22 | /// Read a line from an async reader. 23 | pub fn read_line(reader: R) -> Result { 24 | let mut line = String::new(); 25 | let mut reader = BufReader::new(reader); 26 | reader.read_line(&mut line)?; 27 | let len = line.trim_end_matches(&['\r', '\n'][..]).len(); 28 | line.truncate(len); 29 | Ok(line) 30 | } 31 | 32 | /// Read a pem file and return its corresponding Reqwest certificate. 33 | #[cfg(feature = "reqwest")] 34 | pub async fn read_pem(mut reader: R) -> Result> { 35 | let mut buf = vec![]; 36 | reader.read_to_end(&mut buf)?; 37 | Certificate::from_pem(&buf).map_err(|e| e.into()) 38 | } 39 | 40 | /// A handle to the task created by `authenticate_secret_store` that 41 | /// can be used to subsequently cancel it. 42 | pub struct AuthenticationTask { 43 | join_handle: Option>, 44 | termination: Arc, 45 | } 46 | 47 | /// Performs an initial authentication with the secret store and also spawns a 48 | /// task to re-authenticate on token expiry. A timeout is provided to cause the 49 | /// re-authentication to sleep between non-successful authentication attempts. 50 | pub async fn reauthenticate_secret_store( 51 | ss: impl secret_store::SecretStore + 'static, 52 | role_id: &str, 53 | secret_id: &str, 54 | unauthenticated_timeout: Duration, 55 | max_lease_duration: Duration, 56 | ) -> AuthenticationTask { 57 | let mut approle_auth_reply = ss.approle_auth(role_id, secret_id).await; 58 | let auth_role_id = role_id.to_string(); 59 | let auth_secret_id = secret_id.to_string(); 60 | let auth_unauthenticated_timeout = unauthenticated_timeout; 61 | let termination = Arc::new(Notify::new()); 62 | let task_termination = termination.clone(); 63 | let join_handle = tokio::spawn(async move { 64 | let mut never_reported_info = true; 65 | let mut never_reported_warn = true; 66 | loop { 67 | match &approle_auth_reply { 68 | Ok(approle_auth) => { 69 | if never_reported_info { 70 | info!("Initially authenticated with the secret store"); 71 | never_reported_info = false; 72 | } 73 | tokio::select! { 74 | _ = task_termination.notified() => break, 75 | _ = time::sleep(Duration::from_secs(approle_auth.auth.lease_duration).min(max_lease_duration)) => { 76 | approle_auth_reply = ss.approle_auth(&auth_role_id, &auth_secret_id).await; 77 | } 78 | } 79 | } 80 | Err(e) => { 81 | if never_reported_warn { 82 | warn!( 83 | "Unable to initially authenticate with the secret store. Error: {:?}", 84 | e 85 | ); 86 | never_reported_warn = false; 87 | } 88 | tokio::select! { 89 | _ = task_termination.notified() => break, 90 | _ = time::sleep(auth_unauthenticated_timeout) => (), 91 | } 92 | } 93 | } 94 | } 95 | }); 96 | AuthenticationTask { 97 | join_handle: Some(join_handle), 98 | termination, 99 | } 100 | } 101 | 102 | impl AuthenticationTask { 103 | /// Cancels a previous authentication task and waits for it to 104 | /// finish. The method may be called multiple times, although 105 | /// it is effective only on the first call. 106 | pub async fn cancel(&mut self) { 107 | if let Some(join_handle) = self.join_handle.take() { 108 | debug!("Cancelling the original secret store authentication"); 109 | self.termination.notify_one(); 110 | let _ = join_handle.await; 111 | } 112 | } 113 | } 114 | 115 | /// Given a secret store, a path to a secret, get a secret. 116 | /// The secret is expected to reside in a data field named "value". 117 | pub async fn get_secret_value( 118 | ss: &impl secret_store::SecretStore, 119 | secret_path: &str, 120 | ) -> Option { 121 | let result = ss.get_secret(secret_path).await; 122 | if let Ok(Some(secret_reply)) = result { 123 | secret_reply.data.data.get("value").cloned() 124 | } else { 125 | None 126 | } 127 | } 128 | 129 | /// Given a secret store, a path to a secret, and a byte buffer to be decrypted, 130 | /// decrypt it in place. Returns a decoded structure if decryption 131 | /// was successful. 132 | /// The secret is expected to reside in a data field named "value" and 133 | /// is encoded as a hex string of 32 characters (16 bytes) 134 | /// The buffer is expected to contain both the salt and the bytes to be decrypted. 135 | pub async fn decrypt_buf<'a, T, D, DE>( 136 | ss: &impl secret_store::SecretStore, 137 | secret_path: &str, 138 | buf: &'a mut [u8], 139 | deserialize: D, 140 | ) -> Option 141 | where 142 | T: Deserialize<'a>, 143 | D: FnOnce(&'a [u8]) -> Result, 144 | { 145 | get_secret_value(ss, secret_path) 146 | .await 147 | .and_then(|secret_value| decrypt_buf_with_secret(secret_value, buf, deserialize)) 148 | } 149 | 150 | /// Given a secret, and a byte buffer to be decrypted, 151 | /// decrypt it in place. Returns a decoded structure if decryption 152 | /// was successful. 153 | /// The buffer is expected to contain both the salt and the bytes to be decrypted. 154 | pub fn decrypt_buf_with_secret<'a, T, D, DE>( 155 | secret_value: String, 156 | buf: &'a mut [u8], 157 | deserialize: D, 158 | ) -> Option 159 | where 160 | T: Deserialize<'a>, 161 | D: FnOnce(&'a [u8]) -> Result, 162 | { 163 | if buf.len() >= crypto::SALT_SIZE { 164 | if let Ok(s) = hex::decode(secret_value) { 165 | let (salt, bytes) = buf.split_at_mut(crypto::SALT_SIZE); 166 | crypto::decrypt(bytes, &s.try_into().ok()?, &salt.try_into().ok()?); 167 | return deserialize(bytes).ok(); 168 | } 169 | } 170 | None 171 | } 172 | 173 | /// Given a secret store, a path to a secret, and a type to be encrypted, 174 | /// serialize and then encrypt it. 175 | /// Returns an encrypted buffer prefixed with a random salt if successful. 176 | /// The secret is expected to reside in a data field named "value" and 177 | /// is encoded as a hex string of 32 characters (16 bytes) 178 | /// is encoded as a hex string. Any non alpha-numeric characters are 179 | /// also filtered out. 180 | pub async fn encrypt_struct( 181 | ss: &impl secret_store::SecretStore, 182 | secret_path: &str, 183 | serialize: S, 184 | rng: F, 185 | t: &T, 186 | ) -> Option> 187 | where 188 | T: Serialize, 189 | S: FnOnce(&T) -> Result, SE>, 190 | F: FnOnce() -> U, 191 | U: RngCore, 192 | { 193 | get_secret_value(ss, secret_path) 194 | .await 195 | .and_then(|secret_value| encrypt_struct_with_secret(secret_value, serialize, rng, t)) 196 | } 197 | 198 | /// Given secret, and a type to be encrypted, 199 | /// serialize and then encrypt it. 200 | /// Returns an encrypted buffer prefixed with a random salt if successful. 201 | pub fn encrypt_struct_with_secret( 202 | secret_value: String, 203 | serialize: S, 204 | rng: F, 205 | t: &T, 206 | ) -> Option> 207 | where 208 | T: Serialize, 209 | S: FnOnce(&T) -> Result, SE>, 210 | F: FnOnce() -> U, 211 | U: RngCore, 212 | { 213 | if let Ok(s) = hex::decode(secret_value) { 214 | if let Ok(mut bytes) = serialize(t) { 215 | let salt = crypto::salt(&mut (rng)()); 216 | crypto::encrypt(&mut bytes, &s.try_into().ok()?, &salt); 217 | let mut buf = Vec::with_capacity(SALT_SIZE + bytes.len()); 218 | buf.extend(salt); 219 | buf.extend(bytes); 220 | return Some(buf); 221 | } 222 | } 223 | None 224 | } 225 | -------------------------------------------------------------------------------- /streambed/src/secret_store.rs: -------------------------------------------------------------------------------- 1 | //! Secret store behavior modelled off of the Hashicorp Vault HTTP API, 2 | //! but that which facilitates various backend implementations (including 3 | //! those that do not use HTTP). 4 | 5 | use std::collections::HashMap; 6 | 7 | use async_trait::async_trait; 8 | use serde::{Deserialize, Serialize}; 9 | 10 | /// The reply to a approle authentication 11 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 12 | pub struct AppRoleAuthReply { 13 | pub auth: AuthToken, 14 | } 15 | 16 | /// An authentication token 17 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 18 | pub struct AuthToken { 19 | pub client_token: String, 20 | pub lease_duration: u64, 21 | } 22 | 23 | /// The reply to a get secret request - includes the actual secret data. 24 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 25 | pub struct GetSecretReply { 26 | pub lease_duration: u64, 27 | pub data: SecretData, 28 | } 29 | 30 | /// Secrets and metadata 31 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 32 | pub struct SecretData { 33 | pub data: HashMap, 34 | } 35 | 36 | /// The reply to a userpass authentication 37 | #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] 38 | pub struct UserPassAuthReply { 39 | pub auth: AuthToken, 40 | } 41 | 42 | #[derive(Clone, Debug, Eq, PartialEq)] 43 | pub enum Error { 44 | /// The secret store is not authenticated. 45 | Unauthorized, 46 | } 47 | 48 | /// Describes a secret store modelled on the Hashicorp Vault API, 49 | /// but one that can be backended with other implementations. 50 | /// Connections are managed and retried if they cannot be established. 51 | #[async_trait] 52 | pub trait SecretStore: Clone + Send + Sync { 53 | /// Perform an app authentication given a role and secret. If successful, then the 54 | /// secret store will be updated with a client token thereby permitting subsequent 55 | /// operations including getting secrets. 56 | async fn approle_auth(&self, role_id: &str, secret_id: &str) 57 | -> Result; 58 | 59 | /// Attempt to create/update a secret. 60 | async fn create_secret(&self, secret_path: &str, secret_data: SecretData) -> Result<(), Error>; 61 | 62 | /// Attempt to access a secret. An optional value of None in reply means that 63 | /// the client is unauthorized to obtain it - either due to authorization 64 | /// or it may just not exist. 65 | async fn get_secret(&self, secret_path: &str) -> Result, Error>; 66 | 67 | /// Given a token, authenticate the secret store. 68 | async fn token_auth(&self, token: &str) -> Result<(), Error>; 69 | 70 | /// Perform an app authentication given a username and password. 71 | async fn userpass_auth( 72 | &self, 73 | username: &str, 74 | password: &str, 75 | ) -> Result; 76 | 77 | /// Updates a username and password. 78 | async fn userpass_create_update_user( 79 | &self, 80 | current_username: &str, 81 | username: &str, 82 | password: &str, 83 | ) -> Result<(), Error>; 84 | } 85 | --------------------------------------------------------------------------------