├── .github ├── .well-known │ └── funding-manifest-urls ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug-report.md │ ├── config.yml │ └── feature-request.md └── workflows │ ├── release-bot.yml │ └── rust.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── README_MORE.md ├── benchmark ├── Cargo.toml ├── README.md └── src │ └── bin │ ├── baseline.rs │ ├── consumer.rs │ ├── producer.rs │ └── relay.rs ├── build-tools ├── cargo-clippy.sh ├── cargo-fmt.sh ├── cargo-publish.sh ├── docker-kafka.yml ├── docker-redis.yml ├── docker-redpanda.yml ├── readme.sh └── sync-files.sh ├── docs ├── SeaQL icon.png └── SeaStreamer Banner.png ├── examples ├── Cargo.toml ├── README.md ├── price-feed │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── sea-orm-sink │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── Screenshot.png │ └── src │ │ ├── main.rs │ │ └── spread.rs └── src │ └── bin │ ├── blocking.rs │ ├── buffered.rs │ ├── consumer.rs │ ├── processor.rs │ ├── producer.rs │ └── resumable.rs ├── sea-streamer-file ├── Cargo.toml ├── README.md ├── sea-streamer-file-reader │ ├── .eslintrc.js │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── jest.config.js │ ├── package.json │ ├── src │ │ ├── buffer.ts │ │ ├── crc.test.ts │ │ ├── crc.ts │ │ ├── decoder.ts │ │ ├── dyn_file.ts │ │ ├── error.ts │ │ ├── file.ts │ │ ├── format.ts │ │ ├── index.ts │ │ ├── is_utf8.ts │ │ ├── message.test.ts │ │ ├── message.ts │ │ ├── source.ts │ │ ├── subprocess.ts │ │ └── types.ts │ ├── testcases │ │ ├── consumer.ss │ │ └── quicksort.ss │ └── tsconfig.json ├── src │ ├── bin │ │ ├── clock.rs │ │ ├── decoder.rs │ │ ├── sink.rs │ │ ├── stdin-to-file.rs │ │ └── tail.rs │ ├── buffer.rs │ ├── consumer │ │ ├── future.rs │ │ ├── group.rs │ │ └── mod.rs │ ├── crc.rs │ ├── dyn_file.rs │ ├── error.rs │ ├── export.rs │ ├── file.rs │ ├── format.rs │ ├── lib.rs │ ├── messages.rs │ ├── producer │ │ ├── backend.rs │ │ └── mod.rs │ ├── sink.rs │ ├── source.rs │ ├── streamer.rs │ ├── surveyor.rs │ └── watcher.rs └── tests │ ├── consumer.rs │ ├── loopback.rs │ ├── producer.rs │ ├── sample-1.ss │ ├── sample.rs │ ├── streamer.rs │ ├── surveyor.rs │ └── util.rs ├── sea-streamer-fuse ├── Cargo.toml └── src │ └── lib.rs ├── sea-streamer-kafka ├── Cargo.toml ├── NOTES.md ├── README.md ├── src │ ├── bin │ │ ├── consumer.rs │ │ └── producer.rs │ ├── cluster.rs │ ├── consumer.rs │ ├── error.rs │ ├── host.rs │ ├── lib.rs │ ├── producer.rs │ ├── runtime │ │ ├── async_std_impl.rs │ │ ├── mod.rs │ │ └── no_rt.rs │ └── streamer.rs └── tests │ └── consumer.rs ├── sea-streamer-redis ├── Cargo.toml ├── README.md ├── docs │ └── sea-streamer-concurrency.svg ├── redis-streams-dump │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── main.rs ├── src │ ├── bin │ │ ├── consumer.rs │ │ ├── manager-test.rs │ │ ├── producer.rs │ │ └── trim-stream.rs │ ├── cluster.rs │ ├── connection.rs │ ├── consumer │ │ ├── cluster.rs │ │ ├── future.rs │ │ ├── mod.rs │ │ ├── node.rs │ │ ├── options.rs │ │ └── shard.rs │ ├── error.rs │ ├── host.rs │ ├── lib.rs │ ├── manager.rs │ ├── message.rs │ ├── producer.rs │ └── streamer.rs └── tests │ ├── consumer-group.rs │ ├── load-balanced.rs │ ├── realtime.rs │ ├── resumable.rs │ ├── seek-rewind.rs │ ├── sharding.rs │ └── util.rs ├── sea-streamer-runtime ├── Cargo.toml ├── README.md └── src │ ├── file │ ├── mod.rs │ └── no_rt_file.rs │ ├── lib.rs │ ├── mutex.rs │ ├── sleep.rs │ ├── task │ ├── async_std_task.rs │ ├── mod.rs │ ├── no_rt_task.rs │ └── tokio_task.rs │ └── timeout │ ├── async_std_timeout.rs │ ├── mod.rs │ ├── no_rt_timeout.rs │ └── tokio_timeout.rs ├── sea-streamer-socket ├── Cargo.toml ├── README.md └── src │ ├── backend.rs │ ├── bin │ └── relay.rs │ ├── connect_options.rs │ ├── consumer.rs │ ├── consumer_options.rs │ ├── error.rs │ ├── lib.rs │ ├── message.rs │ ├── producer.rs │ ├── producer_options.rs │ └── streamer.rs ├── sea-streamer-stdio ├── Cargo.toml ├── README.md ├── src │ ├── bin │ │ ├── README.md │ │ ├── clock.rs │ │ ├── complex.rs │ │ └── relay.rs │ ├── consumer.rs │ ├── consumer_group.rs │ ├── error.rs │ ├── lib.rs │ ├── parser.rs │ ├── producer.rs │ ├── streamer.rs │ └── util.rs └── tests │ ├── group.rs │ └── loopback.rs ├── sea-streamer-types ├── Cargo.toml ├── README.md └── src │ ├── components │ └── mod.rs │ ├── consumer.rs │ ├── error.rs │ ├── export.rs │ ├── lib.rs │ ├── message.rs │ ├── options.rs │ ├── producer.rs │ ├── stream.rs │ └── streamer.rs └── src └── lib.rs /.github/.well-known/funding-manifest-urls: -------------------------------------------------------------------------------- 1 | https://www.sea-ql.org/funding.json 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: SeaQL -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Report a bug or feature flaw 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 20 | 21 | ## Description 22 | 23 | 24 | 25 | ## Steps to Reproduce 26 | 27 | 1. 28 | 2. 29 | 3. 30 | 31 | ### Expected Behavior 32 | 33 | 34 | 35 | ### Actual Behavior 36 | 37 | 38 | 39 | ### Reproduces How Often 40 | 41 | 42 | 43 | ## Versions 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Q & A 4 | url: https://github.com/SeaQL/sea-streamer/discussions/new?category=q-a 5 | about: Ask a question or look for help. Try to provide sufficient context, snippets to reproduce and error messages. 6 | - name: SeaQL Discord Server 7 | url: https://discord.com/invite/uCPdDXzbdv 8 | about: Join our Discord server to chat with others in the SeaQL community! 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Suggest a new feature for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 22 | 23 | ## Motivation 24 | 25 | 26 | 27 | ## Proposed Solutions 28 | 29 | 30 | 31 | ## Additional Information 32 | 33 | 34 | -------------------------------------------------------------------------------- /.github/workflows/release-bot.yml: -------------------------------------------------------------------------------- 1 | name: Release Bot 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | comment: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | issues: write 12 | pull-requests: write 13 | steps: 14 | - name: Commenting on `${{ github.event.release.tag_name }}` release 15 | uses: billy1624/release-comment-on-pr@master 16 | with: 17 | release-tag: ${{ github.event.release.tag_name }} 18 | token: ${{ github.token }} 19 | message: | 20 | ### :tada: Released In [${releaseTag}](${releaseUrl}) :tada: 21 | 22 | Thank you everyone for the contribution! 23 | This feature is now available in the latest release. Now is a good time to upgrade! 24 | Your participation is what makes us unique; your adoption is what drives us forward. 25 | You can support SeaQL 🌊 by starring our repos, sharing our libraries and becoming a sponsor ⭐. 26 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | 3 | on: 4 | pull_request: 5 | paths-ignore: 6 | - '**.md' 7 | - '.github/ISSUE_TEMPLATE/**' 8 | push: 9 | paths-ignore: 10 | - '**.md' 11 | - '.github/ISSUE_TEMPLATE/**' 12 | branches: 13 | - main 14 | - 0.*.x 15 | - pr/**/ci 16 | 17 | concurrency: 18 | group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} 19 | cancel-in-progress: true 20 | 21 | env: 22 | CARGO_TERM_COLOR: always 23 | 24 | jobs: 25 | 26 | test: 27 | name: Unit Test, Clippy & Fmt 28 | runs-on: ubuntu-latest 29 | steps: 30 | - uses: actions/checkout@v4 31 | - uses: dtolnay/rust-toolchain@master 32 | with: 33 | toolchain: stable 34 | components: clippy 35 | - run: cargo fmt --all -- --check 36 | - run: cargo build --all 37 | - run: cargo build --manifest-path "sea-streamer-file/Cargo.toml" --features executables 38 | - run: cargo test --all 39 | - run: cargo test --package sea-streamer-stdio --test '*' --features test -- --nocapture 40 | - run: cargo test --package sea-streamer-file --test '*' --features test,runtime-async-std -- --nocapture 41 | - run: cargo test --package sea-streamer-file --test '*' --features test,runtime-tokio -- --nocapture 42 | - run: cargo clippy --features file,redis,kafka,stdio,socket,json -- -D warnings 43 | - run: cargo clippy --all 44 | 45 | kafka: 46 | name: Kafka 47 | runs-on: ubuntu-latest 48 | env: 49 | HOST_ID: DUMMY_HOST_ID 50 | RUST_LOG: rdkafka=trace 51 | strategy: 52 | fail-fast: false 53 | matrix: 54 | kafka_version: [3.3.1, 2.8] 55 | runtime: [async-std, tokio] 56 | services: 57 | zookeeper: 58 | image: bitnami/zookeeper:latest 59 | ports: 60 | - 2181:2181 61 | env: 62 | ALLOW_ANONYMOUS_LOGIN: yes 63 | kafka: 64 | image: bitnami/kafka:${{ matrix.kafka_version }} 65 | ports: 66 | - 9092:9092 67 | env: 68 | KAFKA_BROKER_ID: 1 69 | KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 70 | KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092 71 | KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 72 | ALLOW_PLAINTEXT_LISTENER: yes 73 | steps: 74 | - uses: actions/checkout@v4 75 | - uses: dtolnay/rust-toolchain@stable 76 | - run: cargo test --package sea-streamer-kafka --features test,runtime-${{ matrix.runtime }} -- --nocapture 77 | 78 | redpanda: 79 | name: Redpanda 80 | runs-on: ubuntu-latest 81 | env: 82 | HOST_ID: DUMMY_HOST_ID 83 | RUST_LOG: rdkafka=trace 84 | strategy: 85 | fail-fast: false 86 | matrix: 87 | redpanda_version: [latest] 88 | runtime: [async-std, tokio] 89 | steps: 90 | - uses: actions/checkout@v4 91 | - uses: dtolnay/rust-toolchain@stable 92 | - run: docker run -d -p 9092:9092 -p 9644:9644 redpandadata/redpanda:${{ matrix.redpanda_version }} redpanda start --overprovisioned --smp 1 --memory 1G --reserve-memory 0M --node-id 0 --check=false 93 | - run: cargo test --package sea-streamer-kafka --features test,runtime-${{ matrix.runtime }} -- --nocapture 94 | 95 | redis: 96 | name: Redis 97 | runs-on: ubuntu-latest 98 | env: 99 | HOST_ID: DUMMY_HOST_ID 100 | strategy: 101 | fail-fast: false 102 | matrix: 103 | redis_version: [latest] 104 | runtime: [async-std, tokio] 105 | services: 106 | redis: 107 | image: redis:${{ matrix.redis_version }} 108 | ports: 109 | - 6379:6379 110 | steps: 111 | - uses: actions/checkout@v4 112 | - uses: dtolnay/rust-toolchain@stable 113 | - run: cargo test --package sea-streamer-redis --no-default-features --features test,runtime-${{ matrix.runtime }} -- --nocapture 114 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | firedbg/ 3 | Cargo.lock -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | ".", 4 | "examples", 5 | "examples/price-feed", 6 | "benchmark", 7 | "sea-streamer-file", 8 | "sea-streamer-fuse", 9 | "sea-streamer-kafka", 10 | "sea-streamer-redis", 11 | "sea-streamer-redis/redis-streams-dump", 12 | "sea-streamer-runtime", 13 | "sea-streamer-socket", 14 | "sea-streamer-stdio", 15 | "sea-streamer-types", 16 | ] 17 | 18 | [package] 19 | name = "sea-streamer" 20 | version = "0.5.2" 21 | authors = ["Chris Tsang "] 22 | edition = "2021" 23 | description = "🌊 The stream processing toolkit for Rust" 24 | license = "MIT OR Apache-2.0" 25 | documentation = "https://docs.rs/sea-streamer" 26 | repository = "https://github.com/SeaQL/sea-streamer" 27 | categories = ["concurrency"] 28 | keywords = ["async", "stream", "kafka", "stream-processing"] 29 | rust-version = "1.60" 30 | 31 | [package.metadata.docs.rs] 32 | features = ["json", "kafka", "stdio", "socket"] 33 | rustdoc-args = ["--cfg", "docsrs"] 34 | 35 | [dependencies] 36 | sea-streamer-types = { version = "0.5", path = "sea-streamer-types" } 37 | sea-streamer-kafka = { version = "0.5", path = "sea-streamer-kafka", optional = true } 38 | sea-streamer-redis = { version = "0.5", path = "sea-streamer-redis", optional = true } 39 | sea-streamer-stdio = { version = "0.5", path = "sea-streamer-stdio", optional = true } 40 | sea-streamer-file = { version = "0.5", path = "sea-streamer-file", optional = true } 41 | sea-streamer-socket = { version = "0.5", path = "sea-streamer-socket", optional = true } 42 | sea-streamer-runtime = { version = "0.5", path = "sea-streamer-runtime", optional = true } 43 | 44 | [features] 45 | json = ["sea-streamer-types/json"] 46 | kafka = ["sea-streamer-kafka", "sea-streamer-socket?/backend-kafka"] 47 | redis = ["sea-streamer-redis", "sea-streamer-socket?/backend-redis"] 48 | stdio = ["sea-streamer-stdio", "sea-streamer-socket?/backend-stdio"] 49 | file = ["sea-streamer-file", "sea-streamer-socket?/backend-file"] 50 | socket = ["sea-streamer-socket"] 51 | runtime = ["sea-streamer-runtime"] 52 | runtime-async-std = ["sea-streamer-socket?/runtime-async-std", "sea-streamer-kafka?/runtime-async-std", "sea-streamer-redis?/runtime-async-std", "sea-streamer-runtime/runtime-async-std"] 53 | runtime-tokio = ["sea-streamer-socket?/runtime-tokio", "sea-streamer-kafka?/runtime-tokio", "sea-streamer-redis?/runtime-tokio", "sea-streamer-runtime/runtime-tokio"] -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2023 Seafire Software Limited 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README_MORE.md: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | 5 | - Apache License, Version 2.0 6 | ([LICENSE-APACHE](LICENSE-APACHE) or ) 7 | - MIT license 8 | ([LICENSE-MIT](LICENSE-MIT) or ) 9 | 10 | at your option. 11 | 12 | Unless you explicitly state otherwise, any contribution intentionally submitted 13 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall be 14 | dual licensed as above, without any additional terms or conditions. 15 | 16 | ## Sponsor 17 | 18 | [SeaQL.org](https://www.sea-ql.org/) is an independent open-source organization run by passionate developers. If you enjoy using our libraries, please star and share our repositories. If you feel generous, a small donation via [GitHub Sponsor](https://github.com/sponsors/SeaQL) will be greatly appreciated, and goes a long way towards sustaining the organization. 19 | 20 | We invite you to participate, contribute and together help build Rust's future. 21 | -------------------------------------------------------------------------------- /benchmark/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-benchmark" 3 | version = "0.5.0" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "🌊 The stream processing toolkit for Rust" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/sea-streamer" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | rust-version = "1.60" 12 | 13 | [dependencies] 14 | anyhow = { version = "1" } 15 | async-std = { version = "1", features = ["attributes"], optional = true } 16 | env_logger = { version = "0.9" } 17 | flume = { version = "0.11", default-features = false, features = ["async"] } 18 | clap = { version = "4.5", features = ["derive", "env"] } 19 | tokio = { version = "1.10", features = ["full"], optional = true } 20 | 21 | [dependencies.sea-streamer] 22 | path = ".." # remove this line in your own project 23 | version = "0.5" 24 | features = ["redis", "stdio", "file", "socket"] 25 | 26 | [features] 27 | default = ["runtime-tokio"] 28 | runtime-tokio = ["tokio", "sea-streamer/runtime-tokio"] 29 | runtime-async-std = ["async-std", "sea-streamer/runtime-async-std"] 30 | -------------------------------------------------------------------------------- /benchmark/README.md: -------------------------------------------------------------------------------- 1 | # SeaStreamer Benchmark 2 | 3 | Some micro benchmarks. Note: it is only meaningful for these numbers to compare with itself. It's only intended for measuring the performance improvements (and hopefully not regressions) over the course of development. 4 | 5 | ## Commands 6 | 7 | ```sh 8 | time cargo run --release --bin baseline -- --stream stdio:///test 9 | 10 | time cargo run --release --bin producer -- --stream redis://localhost/clock 11 | rm clock.ss; touch clock.ss; time cargo run --release --bin producer -- --stream file://clock.ss/clock 12 | time cargo run --release --bin producer -- --stream stdio:///clock > clock.log 13 | 14 | time cargo run --release --bin consumer -- --stream redis://localhost/clock 15 | time cargo run --release --bin consumer -- --stream file://clock.ss/clock 16 | time ( cargo run --release --bin producer -- --stream stdio:///clock | cargo run --release --bin relay -- --input clock --output relay | cargo run --release --bin consumer -- --stream stdio:///relay ) 17 | ``` 18 | 19 | ## Summary 20 | 21 | 100k messages. 22 | 23 | Each message has a payload of 256 bytes. 24 | 25 | Dump is around 30MB in size. -------------------------------------------------------------------------------- /benchmark/src/bin/baseline.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer::{runtime::sleep, StreamUrl}; 4 | use std::time::Duration; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap( 9 | long, 10 | help = "Streamer URI with stream key, i.e. try `kafka://localhost:9092/my_topic`", 11 | env = "STREAM_URL" 12 | )] 13 | stream: StreamUrl, 14 | } 15 | 16 | #[cfg_attr(feature = "runtime-tokio", tokio::main)] 17 | #[cfg_attr(feature = "runtime-async-std", async_std::main)] 18 | async fn main() -> Result<()> { 19 | env_logger::init(); 20 | 21 | let Args { stream } = Args::parse(); 22 | std::hint::black_box(stream); 23 | 24 | for i in 0..100_000 { 25 | let message = format!("The this the message payload {i:0>5}: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo"); 26 | std::hint::black_box(message); 27 | if i % 1000 == 0 { 28 | sleep(Duration::from_nanos(1)).await; 29 | } 30 | } 31 | 32 | Ok(()) 33 | } 34 | -------------------------------------------------------------------------------- /benchmark/src/bin/consumer.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer::{ 4 | Buffer, Consumer, ConsumerMode, ConsumerOptions, Message, SeaConsumer, SeaConsumerOptions, 5 | SeaMessage, SeaStreamReset, SeaStreamer, StreamUrl, Streamer, 6 | }; 7 | 8 | #[derive(Debug, Parser)] 9 | struct Args { 10 | #[clap( 11 | long, 12 | help = "Streamer URI with stream key(s), i.e. try `kafka://localhost:9092/my_topic`", 13 | env = "STREAM_URL" 14 | )] 15 | stream: StreamUrl, 16 | } 17 | 18 | #[cfg_attr(feature = "runtime-tokio", tokio::main)] 19 | #[cfg_attr(feature = "runtime-async-std", async_std::main)] 20 | async fn main() -> Result<()> { 21 | env_logger::init(); 22 | 23 | let Args { stream } = Args::parse(); 24 | 25 | let streamer = SeaStreamer::connect(stream.streamer(), Default::default()).await?; 26 | 27 | let mut options = SeaConsumerOptions::new(ConsumerMode::RealTime); 28 | options.set_auto_stream_reset(SeaStreamReset::Earliest); 29 | 30 | let consumer: SeaConsumer = streamer 31 | .create_consumer(stream.stream_keys(), options) 32 | .await?; 33 | 34 | let mut mess: Option = None; 35 | for i in 0..100_000 { 36 | mess = Some(consumer.next().await?); 37 | if i % 1000 == 0 { 38 | let mess = mess.as_ref().unwrap(); 39 | println!("[{}] {}", mess.timestamp(), mess.message().as_str()?); 40 | } 41 | } 42 | let mess = mess.as_ref().unwrap(); 43 | println!("[{}] {}", mess.timestamp(), mess.message().as_str()?); 44 | 45 | Ok(()) 46 | } 47 | -------------------------------------------------------------------------------- /benchmark/src/bin/producer.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer::{Producer, SeaProducer, SeaStreamer, StreamUrl, Streamer}; 4 | use std::time::Duration; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap( 9 | long, 10 | help = "Streamer URI with stream key, i.e. try `kafka://localhost:9092/my_topic`", 11 | env = "STREAM_URL" 12 | )] 13 | stream: StreamUrl, 14 | } 15 | 16 | #[cfg_attr(feature = "runtime-tokio", tokio::main)] 17 | #[cfg_attr(feature = "runtime-async-std", async_std::main)] 18 | async fn main() -> Result<()> { 19 | env_logger::init(); 20 | 21 | let Args { stream } = Args::parse(); 22 | 23 | let streamer = SeaStreamer::connect(stream.streamer(), Default::default()).await?; 24 | 25 | let producer: SeaProducer = streamer 26 | .create_producer(stream.stream_key()?, Default::default()) 27 | .await?; 28 | 29 | for i in 0..100_000 { 30 | let message = format!("The this the message payload {i:0>5}: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo"); 31 | producer.send(message)?; 32 | if i % 1000 == 0 { 33 | tokio::time::sleep(Duration::from_nanos(1)).await; 34 | } 35 | } 36 | 37 | producer.end().await?; // flush 38 | 39 | Ok(()) 40 | } 41 | -------------------------------------------------------------------------------- /benchmark/src/bin/relay.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer::stdio::StdioStreamer; 4 | use sea_streamer::{Consumer, Message, Producer, StreamKey, Streamer, StreamerUri}; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap(long, help = "Stream key of input")] 9 | input: StreamKey, 10 | #[clap(long, help = "Stream key of output")] 11 | output: StreamKey, 12 | } 13 | 14 | #[tokio::main] 15 | async fn main() -> Result<()> { 16 | env_logger::init(); 17 | 18 | let Args { input, output } = Args::parse(); 19 | 20 | let streamer = StdioStreamer::connect(StreamerUri::zero(), Default::default()).await?; 21 | let consumer = streamer 22 | .create_consumer(&[input], Default::default()) 23 | .await?; 24 | let producer = streamer.create_producer(output, Default::default()).await?; 25 | 26 | for _ in 0..100_000 { 27 | let mess = consumer.next().await?; 28 | producer.send(mess.message())?; 29 | } 30 | 31 | producer.end().await?; // flush 32 | 33 | Ok(()) 34 | } 35 | -------------------------------------------------------------------------------- /build-tools/cargo-clippy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | if [ -d ./build-tools ]; then 4 | crates=(`find . -type f -name 'Cargo.toml'`) 5 | for crate in "${crates[@]}"; do 6 | echo "cargo clippy --manifest-path ${crate} --fix --allow-dirty --allow-staged" 7 | cargo clippy --manifest-path "${crate}" --fix --allow-dirty --allow-staged 8 | done 9 | else 10 | echo "Please execute this script from the repository root." 11 | fi 12 | -------------------------------------------------------------------------------- /build-tools/cargo-fmt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | if [ -d ./build-tools ]; then 4 | crates=(`find . -type f -name 'Cargo.toml'`) 5 | for crate in "${crates[@]}"; do 6 | echo "cargo +nightly fmt --manifest-path ${crate} --all" 7 | cargo +nightly fmt --manifest-path "${crate}" --all 8 | done 9 | else 10 | echo "Please execute this script from the repository root." 11 | fi 12 | -------------------------------------------------------------------------------- /build-tools/cargo-publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd sea-streamer-types 5 | cargo publish 6 | cd .. 7 | 8 | cd sea-streamer-runtime 9 | cargo publish 10 | cd .. 11 | 12 | cd sea-streamer-stdio 13 | cargo publish 14 | cd .. 15 | 16 | cd sea-streamer-file 17 | cargo publish 18 | cd .. 19 | 20 | cd sea-streamer-kafka 21 | cargo publish 22 | cd .. 23 | 24 | cd sea-streamer-redis 25 | cargo publish 26 | cd .. 27 | 28 | cd sea-streamer-socket 29 | cargo publish 30 | cd .. 31 | 32 | # publish `sea-streamer` 33 | cargo publish 34 | 35 | cd examples 36 | cargo publish -------------------------------------------------------------------------------- /build-tools/docker-kafka.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | zookeeper: 5 | image: 'bitnami/zookeeper:latest' 6 | ports: 7 | - '2181:2181' 8 | environment: 9 | - ALLOW_ANONYMOUS_LOGIN=yes 10 | kafka: 11 | image: 'bitnami/kafka:3.3.1' 12 | ports: 13 | - '9092:9092' 14 | environment: 15 | - KAFKA_BROKER_ID=1 16 | - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 17 | - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 18 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 19 | - ALLOW_PLAINTEXT_LISTENER=yes 20 | depends_on: 21 | - zookeeper 22 | -------------------------------------------------------------------------------- /build-tools/docker-redis.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | redis: 5 | image: redis:latest 6 | ports: 7 | - 6379:6379 8 | -------------------------------------------------------------------------------- /build-tools/docker-redpanda.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | redpanda: 5 | image: redpandadata/redpanda:latest 6 | command: 7 | - redpanda 8 | - start 9 | - --smp 10 | - '1' 11 | - --reserve-memory 12 | - 0M 13 | - --overprovisioned 14 | - --node-id 15 | - '0' 16 | - --kafka-addr 17 | - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092 18 | - --advertise-kafka-addr 19 | - PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092 20 | - --pandaproxy-addr 21 | - PLAINTEXT://0.0.0.0:28082,OUTSIDE://0.0.0.0:8082 22 | - --advertise-pandaproxy-addr 23 | - PLAINTEXT://redpanda:28082,OUTSIDE://localhost:8082 24 | ports: 25 | - 8081:8081 26 | - 8082:8082 27 | - 9092:9092 28 | - 28082:28082 29 | - 29092:29092 30 | -------------------------------------------------------------------------------- /build-tools/readme.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | # cargo install cargo-readme 3 | alias readme='cargo readme --no-badges --no-indent-headings --no-license --no-template --no-title' 4 | readme > README.md 5 | cd sea-streamer-types 6 | readme > README.md 7 | echo '' >> ../README.md 8 | readme >> ../README.md 9 | cd ../sea-streamer-socket 10 | readme > README.md 11 | echo '' >> ../README.md 12 | readme >> ../README.md 13 | cd ../sea-streamer-kafka 14 | readme > README.md 15 | echo '' >> ../README.md 16 | readme >> ../README.md 17 | cd ../sea-streamer-redis 18 | readme > README.md 19 | echo '' >> ../README.md 20 | readme >> ../README.md 21 | cd ../sea-streamer-stdio 22 | readme > README.md 23 | echo '' >> ../README.md 24 | readme >> ../README.md 25 | cd ../sea-streamer-file 26 | readme > README.md 27 | echo '' >> ../README.md 28 | readme >> ../README.md 29 | cd ../sea-streamer-runtime 30 | readme > README.md 31 | echo '' >> ../README.md 32 | readme >> ../README.md 33 | cd .. 34 | echo '' >> README.md 35 | cat README_MORE.md >> README.md -------------------------------------------------------------------------------- /build-tools/sync-files.sh: -------------------------------------------------------------------------------- 1 | cp sea-streamer-kafka/src/host.rs sea-streamer-redis/src/host.rs -------------------------------------------------------------------------------- /docs/SeaQL icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SeaQL/sea-streamer/07241fb987e8dc73cc9d3237bbdc077a358f48e2/docs/SeaQL icon.png -------------------------------------------------------------------------------- /docs/SeaStreamer Banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SeaQL/sea-streamer/07241fb987e8dc73cc9d3237bbdc077a358f48e2/docs/SeaStreamer Banner.png -------------------------------------------------------------------------------- /examples/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-examples" 3 | version = "0.5.0" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "🌊 The stream processing toolkit for Rust" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/sea-streamer" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | rust-version = "1.60" 12 | 13 | [dependencies] 14 | anyhow = { version = "1" } 15 | async-std = { version = "1", features = ["attributes"], optional = true } 16 | env_logger = { version = "0.9" } 17 | flume = { version = "0.11", default-features = false, features = ["async"] } 18 | clap = { version = "4.5", features = ["derive", "env"] } 19 | tokio = { version = "1.10", features = ["full"], optional = true } 20 | 21 | [dependencies.sea-streamer] 22 | path = ".." # remove this line in your own project 23 | version = "0.5" 24 | features = ["kafka", "redis", "stdio", "file", "socket"] 25 | 26 | [features] 27 | default = ["runtime-tokio"] 28 | runtime-tokio = ["tokio", "sea-streamer/runtime-tokio"] 29 | runtime-async-std = ["async-std", "sea-streamer/runtime-async-std"] 30 | -------------------------------------------------------------------------------- /examples/price-feed/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-price-feed" 3 | version = "0.1.0" 4 | edition = "2021" 5 | publish = false 6 | 7 | [dependencies] 8 | anyhow = "1" 9 | async-tungstenite = { version = "0.24", features = ["tokio-runtime", "tokio-native-tls"] } 10 | clap = { version = "4.5", features = ["derive"] } 11 | rust_decimal = "1.34" 12 | serde = { version = "1", features = ["derive"] } 13 | serde_json = "1" 14 | tokio = { version = "1", features = ["full"] } 15 | 16 | [dependencies.sea-streamer] 17 | path = "../.." # remove this line in your own project 18 | version = "0.5" 19 | features = ["redis", "socket", "json"] -------------------------------------------------------------------------------- /examples/price-feed/README.md: -------------------------------------------------------------------------------- 1 | # Price Feed 2 | 3 | This example demonstrates how to subscribe to a real-time websocket data feed and stream to Redis / Kafka. 4 | 5 | As an example, we subscribe to the `GBP/USD` price feed from Kraken, documentation can be found at https://docs.kraken.com/websockets/#message-spread. 6 | 7 | It will stream to localhost Redis by default. Stream key will be named `GBP_USD`. 8 | 9 | ```sh 10 | cargo run 11 | ``` 12 | 13 | Here is a sample message serialized to JSON: 14 | 15 | ```json 16 | {"spread":{"bid":"1.23150","ask":"1.23166","timestamp":"2024-04-22T11:24:41.461661","bid_vol":"40.55300552","ask_vol":"315.04699448"},"channel_name":"spread","pair":"GBP/USD"} 17 | ``` 18 | 19 | #### NOT FINANCIAL ADVICE: FOR EDUCATIONAL AND INFORMATIONAL PURPOSES ONLY -------------------------------------------------------------------------------- /examples/price-feed/src/main.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{bail, Result}; 2 | use async_tungstenite::tungstenite::Message; 3 | use clap::Parser; 4 | use rust_decimal::Decimal; 5 | use sea_streamer::{ 6 | export::futures::{SinkExt, StreamExt}, 7 | Producer, SeaProducer, SeaStreamer, Streamer, StreamerUri, Timestamp, TIMESTAMP_FORMAT, 8 | }; 9 | use serde::{Deserialize, Serialize}; 10 | 11 | #[derive(Debug, Parser)] 12 | struct Args { 13 | #[clap(long, help = "Streamer URI", default_value = "redis://localhost")] 14 | streamer: StreamerUri, 15 | } 16 | 17 | #[derive(Debug, Serialize, Deserialize)] 18 | struct SpreadMessage { 19 | #[allow(dead_code)] 20 | #[serde(skip_serializing)] 21 | channel_id: u32, 22 | spread: Spread, 23 | channel_name: String, 24 | pair: String, 25 | } 26 | 27 | #[derive(Debug, Serialize, Deserialize)] 28 | struct Spread { 29 | bid: Decimal, 30 | ask: Decimal, 31 | #[serde(with = "timestamp_serde")] 32 | timestamp: Timestamp, 33 | bid_vol: Decimal, 34 | ask_vol: Decimal, 35 | } 36 | 37 | #[tokio::main] 38 | async fn main() -> Result<()> { 39 | let Args { streamer } = Args::parse(); 40 | 41 | println!("Connecting .."); 42 | let (mut ws, _) = async_tungstenite::tokio::connect_async("wss://ws.kraken.com/").await?; 43 | println!("Connected."); 44 | 45 | ws.send(Message::Text( 46 | r#"{ 47 | "event": "subscribe", 48 | "pair": [ 49 | "GBP/USD" 50 | ], 51 | "subscription": { 52 | "name": "spread" 53 | } 54 | }"# 55 | .to_owned(), 56 | )) 57 | .await?; 58 | 59 | loop { 60 | match ws.next().await { 61 | Some(Ok(Message::Text(data))) => { 62 | println!("{data}"); 63 | if data.contains(r#""status":"subscribed""#) { 64 | println!("Subscribed."); 65 | break; 66 | } 67 | } 68 | e => bail!("Unexpected message {e:?}"), 69 | } 70 | } 71 | 72 | let streamer = SeaStreamer::connect(streamer, Default::default()).await?; 73 | let producer: SeaProducer = streamer 74 | .create_producer("GBP_USD".parse()?, Default::default()) 75 | .await?; 76 | 77 | loop { 78 | match ws.next().await { 79 | Some(Ok(Message::Text(data))) => { 80 | if data == r#"{"event":"heartbeat"}"# { 81 | continue; 82 | } 83 | let spread: SpreadMessage = serde_json::from_str(&data)?; 84 | let message = serde_json::to_string(&spread)?; 85 | println!("{message}"); 86 | producer.send(message)?; 87 | } 88 | Some(Err(e)) => bail!("Socket error: {e}"), 89 | None => bail!("Stream ended"), 90 | e => bail!("Unexpected message {e:?}"), 91 | } 92 | } 93 | } 94 | 95 | mod timestamp_serde { 96 | use super::*; 97 | 98 | pub fn deserialize<'de, D>(deserializer: D) -> Result 99 | where 100 | D: serde::Deserializer<'de>, 101 | { 102 | let s = <&str>::deserialize(deserializer)?; 103 | let value: Decimal = s.parse().map_err(serde::de::Error::custom)?; 104 | Timestamp::from_unix_timestamp_nanos( 105 | (value * Decimal::from(1_000_000_000)).try_into().unwrap(), 106 | ) 107 | .map_err(serde::de::Error::custom) 108 | } 109 | 110 | pub fn serialize(v: &Timestamp, serializer: S) -> Result 111 | where 112 | S: serde::Serializer, 113 | { 114 | serializer.serialize_str( 115 | &v.format(TIMESTAMP_FORMAT) 116 | .map_err(serde::ser::Error::custom)?, 117 | ) 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /examples/sea-orm-sink/.gitignore: -------------------------------------------------------------------------------- 1 | *.sqlite -------------------------------------------------------------------------------- /examples/sea-orm-sink/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | 3 | [package] 4 | name = "sea-streamer-sea-orm-sink" 5 | version = "0.1.0" 6 | edition = "2021" 7 | publish = false 8 | 9 | [dependencies] 10 | anyhow = { version = "1" } 11 | clap = { version = "4.5", features = ["derive"] } 12 | env_logger = { version = "0.9" } 13 | log = { version = "0.4", default-features = false } 14 | serde = { version = "1", features = ["derive"] } 15 | serde_json = { version = "1" } 16 | tokio = { version = "1", features = ["full"] } 17 | 18 | [dependencies.sea-orm] 19 | version = "1.0.0-rc.3" 20 | features = ["sqlx-sqlite", "runtime-tokio-native-tls"] 21 | 22 | [dependencies.sea-streamer] 23 | path = "../.." # remove this line in your own project 24 | version = "0.5" 25 | features = ["redis", "socket", "json", "runtime-tokio"] -------------------------------------------------------------------------------- /examples/sea-orm-sink/README.md: -------------------------------------------------------------------------------- 1 | # SeaORM Data Sink 2 | 3 | This example demonstrates how to consume a stream from Redis / Kafka and store the data to MySQL / Postgres / SQLite / SQL Server. 4 | 5 | It will create the table automatically. You have to run the `price-feed` example first. It will subscribe to `GBP_USD` and saves to `GBP_USD.sqlite` by default. Incoming JSON messages will be deserialized and inserted into database. 6 | 7 | ```sh 8 | RUST_LOG=info cargo run 9 | ``` 10 | 11 | A more complex example with buffering and periodic flush can be found at https://github.com/SeaQL/FireDBG.for.Rust/blob/main/indexer/src/main.rs 12 | 13 | ![](Screenshot.png) -------------------------------------------------------------------------------- /examples/sea-orm-sink/Screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SeaQL/sea-streamer/07241fb987e8dc73cc9d3237bbdc077a358f48e2/examples/sea-orm-sink/Screenshot.png -------------------------------------------------------------------------------- /examples/sea-orm-sink/src/main.rs: -------------------------------------------------------------------------------- 1 | mod spread; 2 | 3 | use anyhow::Result; 4 | use clap::Parser; 5 | use sea_orm::{ 6 | ActiveModelTrait, ConnectOptions, ConnectionTrait, Database, DbConn, DbErr, IntoActiveModel, 7 | NotSet, Schema, 8 | }; 9 | use sea_streamer::{Buffer, Consumer, Message, SeaStreamer, StreamKey, Streamer, StreamerUri}; 10 | use serde::Deserialize; 11 | 12 | #[derive(Debug, Parser)] 13 | struct Args { 14 | #[clap(long, help = "Streamer URI", default_value = "redis://localhost")] 15 | streamer: StreamerUri, 16 | #[clap(long, help = "Stream Key", default_value = "GBP_USD")] 17 | stream_key: StreamKey, 18 | } 19 | 20 | #[derive(Deserialize)] 21 | struct Item { 22 | spread: spread::Model, 23 | } 24 | 25 | #[tokio::main] 26 | async fn main() -> Result<()> { 27 | env_logger::init(); 28 | 29 | let Args { 30 | streamer, 31 | stream_key, 32 | } = Args::parse(); 33 | 34 | let mut opt = ConnectOptions::new(format!("sqlite://{}.sqlite?mode=rwc", stream_key)); 35 | opt.max_connections(1).sqlx_logging(false); 36 | let db = Database::connect(opt).await?; 37 | create_tables(&db).await?; 38 | 39 | let streamer = SeaStreamer::connect(streamer, Default::default()).await?; 40 | let consumer = streamer 41 | .create_consumer(&[stream_key], Default::default()) 42 | .await?; 43 | 44 | loop { 45 | let message = consumer.next().await?; 46 | let payload = message.message(); 47 | let json = payload.as_str()?; 48 | log::info!("{json}"); 49 | let item: Item = serde_json::from_str(json)?; 50 | let mut spread = item.spread.into_active_model(); 51 | spread.id = NotSet; 52 | spread.save(&db).await?; 53 | } 54 | } 55 | 56 | async fn create_tables(db: &DbConn) -> Result<(), DbErr> { 57 | let builder = db.get_database_backend(); 58 | let schema = Schema::new(builder); 59 | 60 | let stmt = builder.build( 61 | schema 62 | .create_table_from_entity(spread::Entity) 63 | .if_not_exists(), 64 | ); 65 | log::info!("{stmt}"); 66 | db.execute(stmt).await?; 67 | 68 | Ok(()) 69 | } 70 | -------------------------------------------------------------------------------- /examples/sea-orm-sink/src/spread.rs: -------------------------------------------------------------------------------- 1 | use sea_orm::entity::prelude::*; 2 | use serde::Deserialize; 3 | 4 | #[derive(Debug, Clone, PartialEq, Eq, DeriveEntityModel, Deserialize)] 5 | #[sea_orm(table_name = "event")] 6 | pub struct Model { 7 | #[sea_orm(primary_key)] 8 | #[serde(default)] 9 | pub id: i32, 10 | pub timestamp: String, 11 | pub bid: String, 12 | pub ask: String, 13 | pub bid_vol: String, 14 | pub ask_vol: String, 15 | } 16 | 17 | #[derive(Debug, Copy, Clone, EnumIter, DeriveRelation)] 18 | pub enum Relation {} 19 | 20 | impl ActiveModelBehavior for ActiveModel {} 21 | -------------------------------------------------------------------------------- /examples/src/bin/blocking.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use flume::bounded; 4 | use std::time::Duration; 5 | 6 | use sea_streamer::{ 7 | runtime::{sleep, spawn_task}, 8 | Buffer, Consumer, ConsumerMode, ConsumerOptions, Message, Producer, SeaConsumer, 9 | SeaConsumerOptions, SeaMessage, SeaProducer, SeaStreamer, SharedMessage, StreamUrl, Streamer, 10 | }; 11 | 12 | #[derive(Debug, Parser)] 13 | struct Args { 14 | #[clap( 15 | long, 16 | help = "Streamer URI with stream key(s), i.e. try `kafka://localhost:9092/my_topic`" 17 | )] 18 | input: StreamUrl, 19 | #[clap( 20 | long, 21 | help = "Streamer URI with stream key, i.e. try `stdio:///my_stream`" 22 | )] 23 | output: StreamUrl, 24 | } 25 | 26 | const NUM_THREADS: usize = 4; // Every one has at least 2 cores with 2 hyperthreads these days ... right? RIGHT? 27 | 28 | #[cfg_attr(feature = "runtime-tokio", tokio::main)] 29 | #[cfg_attr(feature = "runtime-async-std", async_std::main)] 30 | async fn main() -> Result<()> { 31 | env_logger::init(); 32 | 33 | let Args { input, output } = Args::parse(); 34 | 35 | // The queue 36 | let (sender, receiver) = bounded(1024); 37 | 38 | let streamer = SeaStreamer::connect(input.streamer(), Default::default()).await?; 39 | let options = SeaConsumerOptions::new(ConsumerMode::RealTime); 40 | let consumer: SeaConsumer = streamer 41 | .create_consumer(input.stream_keys(), options) 42 | .await?; 43 | 44 | // This will consume as quickly as possible. But when the queue is full, it will back off. 45 | // So the bounded queue also acts as a rate-limiter. 46 | spawn_task::<_, Result<()>>(async move { 47 | loop { 48 | let message: SeaMessage = consumer.next().await?; 49 | // If the queue is full, we'll wait 50 | sender.send_async(message.to_owned()).await?; 51 | } 52 | }); 53 | 54 | let streamer = SeaStreamer::connect(output.streamer(), Default::default()).await?; 55 | let producer: SeaProducer = streamer 56 | .create_producer(output.stream_key()?, Default::default()) 57 | .await?; 58 | 59 | // Spawn some threads 60 | let mut threads: Vec<_> = (0..NUM_THREADS) 61 | .map(|i| { 62 | let producer = producer.clone(); 63 | let receiver = receiver.clone(); 64 | // This is an OS thread, so it can use up 100% of a pseudo CPU core 65 | std::thread::spawn::<_, Result<()>>(move || { 66 | loop { 67 | let message = receiver.recv()?; 68 | let message = process(i, message)?; 69 | producer.send(message)?; // send is non-blocking 70 | } 71 | }) 72 | }) 73 | .collect(); 74 | 75 | // Handle errors if the threads exit unexpectedly 76 | loop { 77 | watch(&mut threads); 78 | // We can still do async IO here 79 | sleep(Duration::from_secs(1)).await; 80 | } 81 | } 82 | 83 | // Here we simulate a slow, blocking function 84 | fn process(i: usize, message: SharedMessage) -> Result { 85 | std::thread::sleep(Duration::from_secs(1)); 86 | Ok(format!( 87 | "[thread {i}] {m} processed", 88 | m = message.message().as_str()? 89 | )) 90 | } 91 | 92 | fn watch(threads: &mut Vec>>) { 93 | for (i, thread) in threads.iter().enumerate() { 94 | if thread.is_finished() { 95 | panic!("thread {i} exited: {err:?}", err = threads.remove(i).join()); 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /examples/src/bin/buffered.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use flume::bounded; 4 | use std::time::Duration; 5 | 6 | use sea_streamer::{ 7 | runtime::{sleep, spawn_task}, 8 | Buffer, Consumer, ConsumerMode, ConsumerOptions, Message, Producer, SeaConsumer, 9 | SeaConsumerOptions, SeaMessage, SeaProducer, SeaStreamer, SharedMessage, StreamUrl, Streamer, 10 | }; 11 | 12 | #[derive(Debug, Parser)] 13 | struct Args { 14 | #[clap( 15 | long, 16 | help = "Streamer URI with stream key(s), i.e. try `kafka://localhost:9092/my_topic`" 17 | )] 18 | input: StreamUrl, 19 | #[clap( 20 | long, 21 | help = "Streamer URI with stream key, i.e. try `stdio:///my_stream`" 22 | )] 23 | output: StreamUrl, 24 | } 25 | 26 | #[cfg_attr(feature = "runtime-tokio", tokio::main)] 27 | #[cfg_attr(feature = "runtime-async-std", async_std::main)] 28 | async fn main() -> Result<()> { 29 | env_logger::init(); 30 | 31 | let Args { input, output } = Args::parse(); 32 | 33 | // The queue 34 | let (sender, receiver) = bounded(1024); 35 | 36 | let streamer = SeaStreamer::connect(input.streamer(), Default::default()).await?; 37 | let options = SeaConsumerOptions::new(ConsumerMode::RealTime); 38 | let consumer: SeaConsumer = streamer 39 | .create_consumer(input.stream_keys(), options) 40 | .await?; 41 | 42 | // This will consume as quickly as possible. But when the queue is full, it will back off. 43 | // So the bounded queue also acts as a rate-limiter. 44 | spawn_task::<_, Result<()>>(async move { 45 | loop { 46 | let message: SeaMessage = consumer.next().await?; 47 | // If the queue is full, we'll wait 48 | sender.send_async(message.to_owned()).await?; 49 | } 50 | }); 51 | 52 | let streamer = SeaStreamer::connect(output.streamer(), Default::default()).await?; 53 | let producer: SeaProducer = streamer 54 | .create_producer(output.stream_key()?, Default::default()) 55 | .await?; 56 | 57 | for batch in 0..usize::MAX { 58 | // Take all messages currently buffered in the queue, but do not wait 59 | let mut messages: Vec = receiver.drain().collect(); 60 | if messages.is_empty() { 61 | // Queue is empty, so we wait until there is something 62 | messages.push(receiver.recv_async().await?) 63 | } 64 | for message in process(batch, messages).await? { 65 | // Send is non-blocking so it does not slow down the loop 66 | producer.send(message)?; 67 | } 68 | } 69 | 70 | Ok(()) 71 | } 72 | 73 | // Process the messages in batch 74 | async fn process(batch: usize, messages: Vec) -> Result> { 75 | // Here we simulate a slow operation 76 | sleep(Duration::from_secs(1)).await; 77 | messages 78 | .into_iter() 79 | .map(|message| { 80 | Ok(format!( 81 | "[batch {}] {} processed", 82 | batch, 83 | message.message().as_str()? 84 | )) 85 | }) 86 | .collect() 87 | } 88 | -------------------------------------------------------------------------------- /examples/src/bin/consumer.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer::{ 4 | Buffer, Consumer, ConsumerMode, ConsumerOptions, Message, SeaConsumer, SeaConsumerOptions, 5 | SeaMessage, SeaStreamReset, SeaStreamer, StreamUrl, Streamer, 6 | }; 7 | 8 | #[derive(Debug, Parser)] 9 | struct Args { 10 | #[clap( 11 | long, 12 | help = "Streamer URI with stream key(s), i.e. try `kafka://localhost:9092/my_topic`", 13 | env = "STREAM_URL" 14 | )] 15 | stream: StreamUrl, 16 | } 17 | 18 | #[cfg_attr(feature = "runtime-tokio", tokio::main)] 19 | #[cfg_attr(feature = "runtime-async-std", async_std::main)] 20 | async fn main() -> Result<()> { 21 | env_logger::init(); 22 | 23 | let Args { stream } = Args::parse(); 24 | 25 | let streamer = SeaStreamer::connect(stream.streamer(), Default::default()).await?; 26 | 27 | let mut options = SeaConsumerOptions::new(ConsumerMode::RealTime); 28 | options.set_auto_stream_reset(SeaStreamReset::Earliest); 29 | 30 | let consumer: SeaConsumer = streamer 31 | .create_consumer(stream.stream_keys(), options) 32 | .await?; 33 | 34 | loop { 35 | let mess: SeaMessage = consumer.next().await?; 36 | println!("[{}] {}", mess.timestamp(), mess.message().as_str()?); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /examples/src/bin/processor.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer::{ 4 | Buffer, Consumer, ConsumerMode, ConsumerOptions, Message, Producer, SeaConsumer, 5 | SeaConsumerOptions, SeaMessage, SeaProducer, SeaStreamer, StreamUrl, Streamer, 6 | }; 7 | 8 | #[derive(Debug, Parser)] 9 | struct Args { 10 | #[clap( 11 | long, 12 | help = "Streamer URI with stream key(s), i.e. try `kafka://localhost:9092/my_topic`" 13 | )] 14 | input: StreamUrl, 15 | #[clap( 16 | long, 17 | help = "Streamer URI with stream key, i.e. try `stdio:///my_stream`" 18 | )] 19 | output: StreamUrl, 20 | } 21 | 22 | #[cfg_attr(feature = "runtime-tokio", tokio::main)] 23 | #[cfg_attr(feature = "runtime-async-std", async_std::main)] 24 | async fn main() -> Result<()> { 25 | env_logger::init(); 26 | 27 | let Args { input, output } = Args::parse(); 28 | 29 | let streamer = SeaStreamer::connect(input.streamer(), Default::default()).await?; 30 | let options = SeaConsumerOptions::new(ConsumerMode::RealTime); 31 | let consumer: SeaConsumer = streamer 32 | .create_consumer(input.stream_keys(), options) 33 | .await?; 34 | 35 | let streamer = SeaStreamer::connect(output.streamer(), Default::default()).await?; 36 | let producer: SeaProducer = streamer 37 | .create_producer(output.stream_key()?, Default::default()) 38 | .await?; 39 | 40 | loop { 41 | let message: SeaMessage = consumer.next().await?; 42 | let message = process(message).await?; 43 | eprintln!("{message}"); 44 | producer.send(message)?; // send is non-blocking 45 | } 46 | } 47 | 48 | // Of course this will be a complex async function 49 | async fn process(message: SeaMessage<'_>) -> Result { 50 | Ok(format!("{} processed", message.message().as_str()?)) 51 | } 52 | -------------------------------------------------------------------------------- /examples/src/bin/producer.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer::{Producer, SeaProducer, SeaStreamer, StreamUrl, Streamer}; 4 | use std::time::Duration; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap( 9 | long, 10 | help = "Streamer URI with stream key, i.e. try `kafka://localhost:9092/my_topic`", 11 | env = "STREAM_URL" 12 | )] 13 | stream: StreamUrl, 14 | } 15 | 16 | #[cfg_attr(feature = "runtime-tokio", tokio::main)] 17 | #[cfg_attr(feature = "runtime-async-std", async_std::main)] 18 | async fn main() -> Result<()> { 19 | env_logger::init(); 20 | 21 | let Args { stream } = Args::parse(); 22 | 23 | let streamer = SeaStreamer::connect(stream.streamer(), Default::default()).await?; 24 | 25 | let producer: SeaProducer = streamer 26 | .create_producer(stream.stream_key()?, Default::default()) 27 | .await?; 28 | 29 | for tick in 0..100 { 30 | let message = format!(r#""tick {tick}""#); 31 | eprintln!("{message}"); 32 | producer.send(message)?; 33 | tokio::time::sleep(Duration::from_secs(1)).await; 34 | } 35 | 36 | producer.end().await?; // flush 37 | 38 | Ok(()) 39 | } 40 | -------------------------------------------------------------------------------- /examples/src/bin/resumable.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer::{ 4 | kafka::AutoOffsetReset, 5 | redis::{AutoCommit, AutoStreamReset}, 6 | Buffer, Consumer, ConsumerMode, ConsumerOptions, Message, Producer, SeaConsumer, 7 | SeaConsumerOptions, SeaMessage, SeaProducer, SeaStreamer, SeaStreamerBackend, StreamUrl, 8 | Streamer, 9 | }; 10 | use std::time::Duration; 11 | 12 | const TRANSACTION: bool = true; 13 | 14 | #[derive(Debug, Parser)] 15 | struct Args { 16 | #[clap( 17 | long, 18 | help = "Streamer URI with stream key(s), i.e. try `kafka://localhost:9092/my_topic`" 19 | )] 20 | input: StreamUrl, 21 | #[clap( 22 | long, 23 | help = "Streamer URI with stream key, i.e. try `stdio:///my_stream`" 24 | )] 25 | output: StreamUrl, 26 | } 27 | 28 | #[cfg_attr(feature = "runtime-tokio", tokio::main)] 29 | #[cfg_attr(feature = "runtime-async-std", async_std::main)] 30 | async fn main() -> Result<()> { 31 | env_logger::init(); 32 | 33 | let Args { input, output } = Args::parse(); 34 | 35 | let streamer = SeaStreamer::connect(input.streamer(), Default::default()).await?; 36 | let mut options = SeaConsumerOptions::new(ConsumerMode::Resumable); 37 | options.set_kafka_consumer_options(|options| { 38 | options.set_auto_offset_reset(AutoOffsetReset::Earliest); 39 | options.set_enable_auto_commit(true); 40 | options.set_auto_commit_interval(Duration::from_secs(1)); 41 | options.set_enable_auto_offset_store(false); 42 | }); 43 | options.set_redis_consumer_options(|options| { 44 | options.set_auto_stream_reset(AutoStreamReset::Earliest); 45 | options.set_auto_commit(if TRANSACTION { 46 | AutoCommit::Disabled 47 | } else { 48 | AutoCommit::Rolling 49 | }); 50 | // demo only; choose a larger number in your processor 51 | options.set_auto_commit_interval(Duration::from_secs(0)); 52 | // demo only; choose a larger number in your processor 53 | options.set_batch_size(1); 54 | }); 55 | let mut consumer: SeaConsumer = streamer 56 | .create_consumer(input.stream_keys(), options) 57 | .await?; 58 | 59 | let streamer = SeaStreamer::connect(output.streamer(), Default::default()).await?; 60 | let producer: SeaProducer = streamer 61 | .create_producer(output.stream_key()?, Default::default()) 62 | .await?; 63 | 64 | loop { 65 | let message: SeaMessage = consumer.next().await?; 66 | let identifier = message.identifier(); 67 | // wait for the delivery receipt 68 | producer.send(process(message).await?)?.await?; 69 | if let Some(consumer) = consumer.get_kafka() { 70 | if TRANSACTION { 71 | // wait until committed 72 | consumer.commit_with(&identifier).await?; 73 | } else { 74 | // don't wait, so it may or may not have committed 75 | consumer.store_offset_with(&identifier)?; 76 | } 77 | } 78 | if let Some(consumer) = consumer.get_redis() { 79 | consumer.ack_with(&identifier)?; 80 | if TRANSACTION { 81 | // wait until committed 82 | consumer.commit()?.await?; 83 | } 84 | } 85 | } 86 | } 87 | 88 | // Of course this will be a complex async function 89 | async fn process(message: SeaMessage<'_>) -> Result { 90 | Ok(format!("{} processed", message.message().as_str()?)) 91 | } 92 | -------------------------------------------------------------------------------- /sea-streamer-file/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-file" 3 | version = "0.5.2" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "🌊 SeaStreamer File Backend" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/sea-streamer-file" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | keywords = ["async", "stream", "stream-processing"] 12 | rust-version = "1.60" 13 | 14 | [package.metadata.docs.rs] 15 | features = [] 16 | rustdoc-args = ["--cfg", "docsrs"] 17 | 18 | [dependencies] 19 | anyhow = { version = "1", optional = true } 20 | async-std = { version = "1", optional = true } 21 | env_logger = { version = "0.9", optional = true } 22 | fastrand = { version = "1" } 23 | flume = { version = "0.11", default-features = false, features = ["async"] } 24 | lazy_static = { version = "1.4" } 25 | log = { version = "0.4", default-features = false } 26 | notify = { version = "6" } 27 | sea-streamer-types = { version = "0.5", path = "../sea-streamer-types" } 28 | sea-streamer-runtime = { version = "0.5", path = "../sea-streamer-runtime", features = ["file"]} 29 | serde = { version = "1", optional = true, features = ["derive"] } 30 | serde_json = { version = "1", optional = true } 31 | clap = { version = "4.5", features = ["derive"], optional = true } 32 | thiserror = { version = "1", default-features = false } 33 | tokio = { version = "1.10.0", optional = true } 34 | 35 | [dev-dependencies] 36 | 37 | [features] 38 | default = [] 39 | test = ["anyhow", "async-std?/attributes", "tokio?/full", "env_logger"] 40 | executables = ["anyhow", "tokio/full", "env_logger", "clap", "sea-streamer-runtime/runtime-tokio", "serde", "serde_json", "sea-streamer-types/serde"] 41 | runtime-async-std = ["async-std", "sea-streamer-runtime/runtime-async-std"] 42 | runtime-tokio = ["tokio", "sea-streamer-runtime/runtime-tokio"] 43 | 44 | [[bin]] 45 | name = "clock" 46 | path = "src/bin/clock.rs" 47 | required-features = ["executables"] 48 | 49 | [[bin]] 50 | name = "ss-decode" 51 | path = "src/bin/decoder.rs" 52 | required-features = ["executables"] 53 | 54 | [[bin]] 55 | name = "sink" 56 | path = "src/bin/sink.rs" 57 | required-features = ["executables"] 58 | 59 | [[bin]] 60 | name = "tail" 61 | path = "src/bin/tail.rs" 62 | required-features = ["executables"] 63 | 64 | [[bin]] 65 | name = "stdin-to-file" 66 | path = "src/bin/stdin-to-file.rs" 67 | required-features = ["executables"] -------------------------------------------------------------------------------- /sea-streamer-file/README.md: -------------------------------------------------------------------------------- 1 | ### `sea-streamer-file`: File Backend 2 | 3 | This is very similar to `sea-streamer-stdio`, but the difference is SeaStreamerStdio works in real-time, 4 | while `sea-streamer-file` works in real-time and replay. That means, SeaStreamerFile has the ability to 5 | traverse a `.ss` (sea-stream) file and seek/rewind to a particular timestamp/offset. 6 | 7 | In addition, Stdio can only work with UTF-8 text data, while File is able to work with binary data. 8 | In Stdio, there is only one Streamer per process. In File, there can be multiple independent Streamers 9 | in the same process. Afterall, a Streamer is just a file. 10 | 11 | The basic idea of SeaStreamerFile is like a `tail -f` with one message per line, with a custom message frame 12 | carrying binary payloads. The interesting part is, in SeaStreamer, we do not use delimiters to separate messages. 13 | This removes the overhead of encoding/decoding message payloads. But it added some complexity to the file format. 14 | 15 | The SeaStreamerFile format is designed for efficient fast-forward and seeking. This is enabled by placing an array 16 | of Beacons at fixed interval in the file. A Beacon contains a summary of the streams, so it acts like an inplace 17 | index. It also allows readers to align with the message boundaries. To learn more about the file format, read 18 | [`src/format.rs`](https://github.com/SeaQL/sea-streamer/blob/main/sea-streamer-file/src/format.rs). 19 | 20 | On top of that, are the high-level SeaStreamer multi-producer, multi-consumer stream semantics, resembling 21 | the behaviour of other SeaStreamer backends. In particular, the load-balancing behaviour is same as Stdio, 22 | i.e. round-robin. 23 | 24 | ### Decoder 25 | 26 | We provide a small utility to decode `.ss` files: 27 | 28 | ```sh 29 | cargo install sea-streamer-file --features=executables --bin ss-decode 30 | # local version 31 | alias ss-decode='cargo run --package sea-streamer-file --features=executables --bin ss-decode' 32 | ss-decode -- --file --format 33 | ``` 34 | 35 | Pro tip: pipe it to `less` for pagination 36 | 37 | ```sh 38 | ss-decode --file mystream.ss | less 39 | ``` 40 | 41 | Example `log` format: 42 | 43 | ```log 44 | # header 45 | [2023-06-05T13:55:53.001 | hello | 1 | 0] message-1 46 | # beacon 47 | ``` 48 | 49 | Example `ndjson` format: 50 | 51 | ```json 52 | /* header */ 53 | {"header":{"stream_key":"hello","shard_id":0,"sequence":1,"timestamp":"2023-06-05T13:55:53.001"},"payload":"message-1"} 54 | /* beacon */ 55 | ``` 56 | 57 | There is also a Typescript implementation under [`sea-streamer-file-reader`](https://github.com/SeaQL/sea-streamer/tree/main/sea-streamer-file/sea-streamer-file-reader). 58 | 59 | ### TODO 60 | 61 | 1. Resumable: currently unimplemented. A potential implementation might be to commit into a local SQLite database. 62 | 2. Sharding: currently it only streams to Shard ZERO. 63 | 3. Verify: a utility program to verify and repair SeaStreamer binary file. 64 | -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/.eslintrc.js: -------------------------------------------------------------------------------- 1 | /**@type {import('eslint').Linter.Config} */ 2 | // eslint-disable-next-line no-undef 3 | module.exports = { 4 | root: true, 5 | parser: '@typescript-eslint/parser', 6 | plugins: [ 7 | '@typescript-eslint', 8 | ], 9 | extends: [ 10 | 'eslint:recommended', 11 | 'plugin:@typescript-eslint/recommended', 12 | ], 13 | ignorePatterns: [ 14 | 'media' 15 | ], 16 | rules: { 17 | 'semi': [2, "always"], 18 | '@typescript-eslint/no-unused-vars': 0, 19 | '@typescript-eslint/no-explicit-any': 0, 20 | '@typescript-eslint/explicit-module-boundary-types': 0, 21 | '@typescript-eslint/no-non-null-assertion': 0, 22 | } 23 | }; -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/.gitignore: -------------------------------------------------------------------------------- 1 | package-lock.json 2 | node_modules/ 3 | out/ 4 | dist/ -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2023 Seafire Software Limited 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/README.md: -------------------------------------------------------------------------------- 1 | ### `sea-streamer-file-reader`: File Reader for node.js 2 | 3 | This library implements a decoder for the [SeaStreamer file format](https://github.com/SeaQL/sea-streamer/tree/main/sea-streamer-file). 4 | 5 | It does not provide the high-level Consumer interface. But the reason it is so complex in Rust is due to multi-threaded concurrency. Since node.js is basically single threaded, this made the implementation much simpler. 6 | 7 | ### Features 8 | 9 | Lightweight. There are 0 dependencies. It only depends on the node.js standard library. 10 | 11 | ### Install 12 | 13 | This package is published on npm as [`sea-streamer-file-reader`](https://www.npmjs.com/package/sea-streamer-file-reader). 14 | 15 | ### Running the `decoder` 16 | 17 | ```sh 18 | npm run decoder -- --file ./testcases/consumer.ss 19 | ``` 20 | 21 | ```log 22 | # {"fileName":"consumer-1691686154741","createdAt":"2023-08-10T16:49:14.742Z","beaconInterval":1024} 23 | [2023-08-10T16:49:14.745Z | hello | 0 | 1] hello-0 24 | [2023-08-10T16:49:14.746Z | hello | 1 | 1] hello-1 25 | [2023-08-10T16:49:14.748Z | hello | 2 | 1] hello-2 26 | [2023-08-10T16:49:14.750Z | hello | 3 | 1] hello-3 27 | [2023-08-10T16:49:14.751Z | hello | 4 | 1] hello-4 28 | [2023-08-10T16:49:14.753Z | hello | 5 | 1] hello-5 29 | [2023-08-10T16:49:14.754Z | hello | 6 | 1] hello-6 30 | [2023-08-10T16:49:14.755Z | hello | 7 | 1] hello-7 31 | [2023-08-10T16:49:14.757Z | hello | 8 | 1] hello-8 32 | [2023-08-10T16:49:14.758Z | hello | 9 | 1] hello-9 33 | [2023-08-10T16:49:14.759Z | hello | 10 | 1] hello-10 34 | [2023-08-10T16:49:14.761Z | hello | 11 | 1] hello-11 35 | [2023-08-10T16:49:14.762Z | hello | 12 | 1] hello-12 36 | [2023-08-10T16:49:14.763Z | hello | 13 | 1] hello-13 37 | [2023-08-10T16:49:14.765Z | hello | 14 | 1] hello-14 38 | [2023-08-10T16:49:14.766Z | hello | 15 | 1] hello-15 39 | [2023-08-10T16:49:14.768Z | hello | 16 | 1] hello-16 40 | [2023-08-10T16:49:14.769Z | hello | 17 | 1] hello-17 41 | [2023-08-10T16:49:14.770Z | hello | 18 | 1] hello-18 42 | [2023-08-10T16:49:14.771Z | hello | 19 | 1] hello-19 43 | [2023-08-10T16:49:14.773Z | hello | 20 | 1] hello-20 44 | # [{"header":{"streamKey":"hello","shardId":1,"sequence":20,"timestamp":"2023-08-10T16:49:14.773Z"},"runningChecksum":2887}] 45 | [2023-08-10T16:49:14.775Z | hello | 21 | 1] hello-21 46 | [2023-08-10T16:49:14.777Z | hello | 22 | 1] hello-22 47 | [2023-08-10T16:49:14.778Z | hello | 23 | 1] hello-23 48 | [2023-08-10T16:49:14.780Z | hello | 24 | 1] hello-24 49 | [2023-08-10T16:49:14.781Z | hello | 25 | 1] hello-25 50 | ... 51 | ``` -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | }; -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sea-streamer-file-reader", 3 | "version": "0.3.1", 4 | "author": "Chris Tsang ", 5 | "description": "Decoder for the SeaStreamer file format", 6 | "license": "MIT", 7 | "scripts": { 8 | "compile": "tsc -p ./", 9 | "lint": "eslint \"src/**/*.ts\"", 10 | "watch": "tsc -w -p ./", 11 | "test": "jest", 12 | "build": "npx esbuild src/subprocess.ts --bundle --platform=node --outfile=dist/streamer.js", 13 | "clean": "rm -rf dist", 14 | "decoder": "npx ts-node src/decoder.ts" 15 | }, 16 | "devDependencies": { 17 | "@types/jest": "^29.5.3", 18 | "@types/node": "^20.6.2", 19 | "esbuild": "0.19.3", 20 | "jest": "^29.6.2", 21 | "ts-jest": "^29.1.1", 22 | "ts-node": "^10.9.1", 23 | "typescript": "^5.1.6" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/src/buffer.ts: -------------------------------------------------------------------------------- 1 | import { Buffer as SystemBuffer } from "node:buffer"; 2 | import { FileErr, FileErrType } from "./error"; 3 | import { ByteSource } from "./source"; 4 | 5 | export class Buffer implements ByteSource { 6 | buffer: SystemBuffer; 7 | 8 | constructor() { 9 | this.buffer = SystemBuffer.alloc(0); 10 | } 11 | 12 | size(): bigint { 13 | return BigInt(this.buffer.length); 14 | } 15 | 16 | byteAt(at: number): number { 17 | return this.buffer.readUInt8(at); 18 | } 19 | 20 | consume(size: bigint): Buffer { 21 | const bytes = SystemBuffer.from(this.buffer.subarray(0, Number(size))); 22 | this.buffer = this.buffer.subarray(Number(size)); 23 | const result = new Buffer(); 24 | result.buffer = bytes; 25 | return result; 26 | } 27 | 28 | append(bytes: SystemBuffer | Buffer) { 29 | if (bytes instanceof SystemBuffer) { 30 | this.buffer = SystemBuffer.concat([this.buffer, bytes]); 31 | } else if (bytes instanceof Buffer) { 32 | this.buffer = SystemBuffer.concat([this.buffer, bytes.buffer]); 33 | } 34 | } 35 | 36 | toString(): string { 37 | return this.buffer.toString("utf8"); 38 | } 39 | 40 | clear() { 41 | this.buffer = SystemBuffer.alloc(0); 42 | } 43 | 44 | async requestBytes(size: bigint): Promise { 45 | if (size <= this.size()) { 46 | return this.consume(size); 47 | } else { 48 | return new FileErr(FileErrType.NotEnoughBytes); 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/src/crc.test.ts: -------------------------------------------------------------------------------- 1 | import { crc16Cdma2000 } from "./crc"; 2 | 3 | test('crc16Cdma2000', () => { 4 | expect(crc16Cdma2000(Buffer.from("123456789"))).toStrictEqual(0x4C06); 5 | }); -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/src/crc.ts: -------------------------------------------------------------------------------- 1 | const CRC_TABLE = [ 2 | 0x0000, 0xc867, 0x58a9, 0x90ce, 0xb152, 0x7935, 0xe9fb, 0x219c, 0xaac3, 0x62a4, 0xf26a, 0x3a0d, 3 | 0x1b91, 0xd3f6, 0x4338, 0x8b5f, 0x9de1, 0x5586, 0xc548, 0x0d2f, 0x2cb3, 0xe4d4, 0x741a, 0xbc7d, 4 | 0x3722, 0xff45, 0x6f8b, 0xa7ec, 0x8670, 0x4e17, 0xded9, 0x16be, 0xf3a5, 0x3bc2, 0xab0c, 0x636b, 5 | 0x42f7, 0x8a90, 0x1a5e, 0xd239, 0x5966, 0x9101, 0x01cf, 0xc9a8, 0xe834, 0x2053, 0xb09d, 0x78fa, 6 | 0x6e44, 0xa623, 0x36ed, 0xfe8a, 0xdf16, 0x1771, 0x87bf, 0x4fd8, 0xc487, 0x0ce0, 0x9c2e, 0x5449, 7 | 0x75d5, 0xbdb2, 0x2d7c, 0xe51b, 0x2f2d, 0xe74a, 0x7784, 0xbfe3, 0x9e7f, 0x5618, 0xc6d6, 0x0eb1, 8 | 0x85ee, 0x4d89, 0xdd47, 0x1520, 0x34bc, 0xfcdb, 0x6c15, 0xa472, 0xb2cc, 0x7aab, 0xea65, 0x2202, 9 | 0x039e, 0xcbf9, 0x5b37, 0x9350, 0x180f, 0xd068, 0x40a6, 0x88c1, 0xa95d, 0x613a, 0xf1f4, 0x3993, 10 | 0xdc88, 0x14ef, 0x8421, 0x4c46, 0x6dda, 0xa5bd, 0x3573, 0xfd14, 0x764b, 0xbe2c, 0x2ee2, 0xe685, 11 | 0xc719, 0x0f7e, 0x9fb0, 0x57d7, 0x4169, 0x890e, 0x19c0, 0xd1a7, 0xf03b, 0x385c, 0xa892, 0x60f5, 12 | 0xebaa, 0x23cd, 0xb303, 0x7b64, 0x5af8, 0x929f, 0x0251, 0xca36, 0x5e5a, 0x963d, 0x06f3, 0xce94, 13 | 0xef08, 0x276f, 0xb7a1, 0x7fc6, 0xf499, 0x3cfe, 0xac30, 0x6457, 0x45cb, 0x8dac, 0x1d62, 0xd505, 14 | 0xc3bb, 0x0bdc, 0x9b12, 0x5375, 0x72e9, 0xba8e, 0x2a40, 0xe227, 0x6978, 0xa11f, 0x31d1, 0xf9b6, 15 | 0xd82a, 0x104d, 0x8083, 0x48e4, 0xadff, 0x6598, 0xf556, 0x3d31, 0x1cad, 0xd4ca, 0x4404, 0x8c63, 16 | 0x073c, 0xcf5b, 0x5f95, 0x97f2, 0xb66e, 0x7e09, 0xeec7, 0x26a0, 0x301e, 0xf879, 0x68b7, 0xa0d0, 17 | 0x814c, 0x492b, 0xd9e5, 0x1182, 0x9add, 0x52ba, 0xc274, 0x0a13, 0x2b8f, 0xe3e8, 0x7326, 0xbb41, 18 | 0x7177, 0xb910, 0x29de, 0xe1b9, 0xc025, 0x0842, 0x988c, 0x50eb, 0xdbb4, 0x13d3, 0x831d, 0x4b7a, 19 | 0x6ae6, 0xa281, 0x324f, 0xfa28, 0xec96, 0x24f1, 0xb43f, 0x7c58, 0x5dc4, 0x95a3, 0x056d, 0xcd0a, 20 | 0x4655, 0x8e32, 0x1efc, 0xd69b, 0xf707, 0x3f60, 0xafae, 0x67c9, 0x82d2, 0x4ab5, 0xda7b, 0x121c, 21 | 0x3380, 0xfbe7, 0x6b29, 0xa34e, 0x2811, 0xe076, 0x70b8, 0xb8df, 0x9943, 0x5124, 0xc1ea, 0x098d, 22 | 0x1f33, 0xd754, 0x479a, 0x8ffd, 0xae61, 0x6606, 0xf6c8, 0x3eaf, 0xb5f0, 0x7d97, 0xed59, 0x253e, 23 | 0x04a2, 0xccc5, 0x5c0b, 0x946c, 24 | ]; 25 | 26 | function crc_update(crc: number, data: Buffer): number { 27 | for (let i = 0; i < data.length; i++) { 28 | let d = data.readUInt8(i); 29 | let tbl_idx = ((crc >> 8) ^ d) & 0xff; 30 | crc = (CRC_TABLE[tbl_idx] ^ (crc << 8)) & 0xffff; 31 | } 32 | return crc & 0xffff; 33 | } 34 | 35 | export function crc16Cdma2000(data: Buffer): number { 36 | let crc = 0xffff; 37 | crc = crc_update(crc, data); 38 | return crc; 39 | } 40 | -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/src/decoder.ts: -------------------------------------------------------------------------------- 1 | import { FileErr } from "./error"; 2 | import { MessageSource, isEndOfStream } from "./message"; 3 | import { SeqPos, StreamMode } from "./types"; 4 | import { isUtf8 } from "./is_utf8"; 5 | 6 | async function main() { 7 | let path; 8 | for (let i = 0; i < process.argv.length; i++) { 9 | if (process.argv[i] == "--file") { 10 | path = process.argv[i + 1]; 11 | } 12 | } 13 | if (!path) { 14 | throw new Error("Please specify the file path with `--file`"); 15 | } 16 | const source = await MessageSource.new(path, StreamMode.LiveReplay); 17 | if (source instanceof FileErr) { throw new Error("Failed to read file header"); } 18 | console.log("#", JSON.stringify(source.fileHeader().toJson())); 19 | 20 | let beacon = 0; 21 | while (true) { 22 | const message = await source.next(); 23 | if (message instanceof FileErr) { 24 | console.error(message); 25 | return; 26 | } 27 | const h = message.header; 28 | console.log( 29 | `[${h.timestamp.toISOString()} | ${h.streamKey.name} | ${h.sequence.no} | ${h.shardId.id}]`, 30 | isUtf8(message.payload.buffer) ? message.payload.toString() : "" 31 | ); 32 | 33 | if (source.getBeacon()[0] != beacon) { 34 | beacon = source.getBeacon()[0]; 35 | console.log("#", JSON.stringify(source.getBeacon()[1].map((x) => x.toJson()))); 36 | } 37 | 38 | if (isEndOfStream(message)) { 39 | break; 40 | } 41 | } 42 | } 43 | 44 | main(); -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/src/dyn_file.ts: -------------------------------------------------------------------------------- 1 | import { FileErr } from "./error"; 2 | import { SeqPosEnum } from "./types"; 3 | 4 | export enum FileSourceType { 5 | FileReader = "FileReader", 6 | FileSource = "FileSource", 7 | } 8 | 9 | export interface DynFileSource { 10 | sourceType(): FileSourceType; 11 | seek(to: SeqPosEnum): Promise; 12 | resize(): Promise; 13 | switchTo(type: FileSourceType): DynFileSource; 14 | getOffset(): bigint; 15 | fileSize(): bigint; 16 | setTimeout(ms: number): void; 17 | close(): Promise; 18 | } -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/src/error.ts: -------------------------------------------------------------------------------- 1 | type Meta = { [key: string]: string | number | boolean }; 2 | 3 | export class FileErr { 4 | type: FileErrType; 5 | meta: Meta | undefined; 6 | 7 | constructor(type: FileErrType, meta?: Meta) { 8 | this.type = type; 9 | this.meta = meta; 10 | } 11 | 12 | toString(): string { 13 | return `${this.type}${this.meta !== undefined ? ": " + this.meta : ""}`; 14 | } 15 | } 16 | 17 | export enum FileErrType { 18 | TimedOut = "TimedOut", 19 | NotEnoughBytes = "NotEnoughBytes", 20 | FormatErr__ByteMark = "FormatErr::ByteMark", 21 | FormatErr__Version = "FormatErr::Version", 22 | FormatErr__ChecksumErr = "FormatErr::ChecksumErr", 23 | } -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./buffer"; 2 | export * from "./error"; 3 | export * from "./format"; 4 | export * from "./is_utf8"; 5 | export * from "./message"; 6 | export * from "./source"; 7 | export * from "./types"; -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/src/is_utf8.ts: -------------------------------------------------------------------------------- 1 | /* 2 | https://tools.ietf.org/html/rfc3629 3 | 4 | UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 5 | 6 | UTF8-1 = %x00-7F 7 | 8 | UTF8-2 = %xC2-DF UTF8-tail 9 | 10 | UTF8-3 = %xE0 %xA0-BF UTF8-tail 11 | %xE1-EC 2( UTF8-tail ) 12 | %xED %x80-9F UTF8-tail 13 | %xEE-EF 2( UTF8-tail ) 14 | 15 | UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) 16 | %xF1-F3 3( UTF8-tail ) 17 | %xF4 %x80-8F 2( UTF8-tail ) 18 | 19 | UTF8-tail = %x80-BF 20 | */ 21 | 22 | /** 23 | * Check if a Node.js Buffer or Uint8Array is UTF-8. 24 | * 25 | * Copied from https://github.com/hcodes/isutf8 (MIT license) 26 | * 27 | * Copyright (c) 2021 Denis Seleznev, hcodes@yandex.ru 28 | */ 29 | export function isUtf8(buf?: Buffer | Uint8Array): boolean { 30 | if (!buf) { 31 | return false; 32 | } 33 | 34 | let i = 0; 35 | const len = buf.length; 36 | 37 | while (i < len) { 38 | // UTF8-1 = %x00-7F 39 | if (buf[i] <= 0x7F) { 40 | i++; 41 | 42 | continue; 43 | } 44 | 45 | // UTF8-2 = %xC2-DF UTF8-tail 46 | if (buf[i] >= 0xC2 && buf[i] <= 0xDF) { 47 | // if(buf[i + 1] >= 0x80 && buf[i + 1] <= 0xBF) { 48 | if (buf[i + 1] >> 6 === 2) { 49 | i += 2; 50 | 51 | continue; 52 | } else { 53 | return false; 54 | } 55 | } 56 | 57 | // UTF8-3 = %xE0 %xA0-BF UTF8-tail 58 | // UTF8-3 = %xED %x80-9F UTF8-tail 59 | if ( 60 | ( 61 | (buf[i] === 0xE0 && buf[i + 1] >= 0xA0 && buf[i + 1] <= 0xBF) || 62 | (buf[i] === 0xED && buf[i + 1] >= 0x80 && buf[i + 1] <= 0x9F) 63 | ) && buf[i + 2] >> 6 === 2 64 | ) { 65 | i += 3; 66 | 67 | continue; 68 | } 69 | 70 | // UTF8-3 = %xE1-EC 2( UTF8-tail ) 71 | // UTF8-3 = %xEE-EF 2( UTF8-tail ) 72 | if ( 73 | ( 74 | (buf[i] >= 0xE1 && buf[i] <= 0xEC) || 75 | (buf[i] >= 0xEE && buf[i] <= 0xEF) 76 | ) && 77 | buf[i + 1] >> 6 === 2 && 78 | buf[i + 2] >> 6 === 2 79 | ) { 80 | i += 3; 81 | 82 | continue; 83 | } 84 | 85 | // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) 86 | // %xF1-F3 3( UTF8-tail ) 87 | // %xF4 %x80-8F 2( UTF8-tail ) 88 | if ( 89 | ( 90 | (buf[i] === 0xF0 && buf[i + 1] >= 0x90 && buf[i + 1] <= 0xBF) || 91 | (buf[i] >= 0xF1 && buf[i] <= 0xF3 && buf[i + 1] >> 6 === 2) || 92 | (buf[i] === 0xF4 && buf[i + 1] >= 0x80 && buf[i + 1] <= 0x8F) 93 | ) && 94 | buf[i + 2] >> 6 === 2 && 95 | buf[i + 3] >> 6 === 2 96 | ) { 97 | i += 4; 98 | 99 | continue; 100 | } 101 | 102 | return false; 103 | } 104 | 105 | return true; 106 | } -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/src/message.test.ts: -------------------------------------------------------------------------------- 1 | import { FileErr, FileErrType } from "./error"; 2 | import { MessageSource, isEndOfStream } from "./message"; 3 | import { SeqPos, StreamMode } from "./types"; 4 | 5 | test('rewind', async () => testRewind(StreamMode.Replay)); 6 | 7 | test('rewind', async () => testRewind(StreamMode.LiveReplay)); 8 | 9 | async function testRewind(mode: StreamMode) { 10 | const path = './testcases/consumer.ss'; 11 | const source = await MessageSource.new(path, mode); 12 | if (source instanceof FileErr) { throwNewError(source); } 13 | 14 | await expectNext(0, 100); 15 | 16 | let nth; 17 | nth = await source.rewind(new SeqPos.At(1n)); if (nth instanceof FileErr) { throwNewError(nth); } 18 | expect(nth).toStrictEqual(1); 19 | 20 | expect(await source.isStreamEnded()).toBe(true); 21 | await expectNext(21, 50); 22 | 23 | nth = await source.rewind(new SeqPos.At(4n)); if (nth instanceof FileErr) { throwNewError(nth); } 24 | expect(nth).toStrictEqual(4); 25 | 26 | await expectNext(86, 99); 27 | 28 | nth = await source.rewind(new SeqPos.Beginning); if (nth instanceof FileErr) { throwNewError(nth); } 29 | expect(nth).toStrictEqual(0); 30 | 31 | await expectNext(0, 100); 32 | 33 | nth = await source.rewind(new SeqPos.At(0n)); if (nth instanceof FileErr) { throwNewError(nth); } 34 | expect(nth).toStrictEqual(0); 35 | 36 | await expectNext(0, 100); 37 | 38 | const eos = await source.next(); if (eos instanceof FileErr) { throwNewError(eos); } 39 | expect(isEndOfStream(eos)).toBe(true); 40 | 41 | if (mode === StreamMode.Replay) { 42 | const err = await source.next(); 43 | expect(err).toStrictEqual(new FileErr(FileErrType.NotEnoughBytes)); 44 | } else if (mode === StreamMode.LiveReplay) { 45 | source.setTimeout(100); 46 | const err = await source.next(); 47 | expect(err).toStrictEqual(new FileErr(FileErrType.TimedOut)); 48 | } 49 | 50 | async function expectNext(from: number, to: number) { 51 | if (source instanceof FileErr) { throwNewError(source); } 52 | for (let i = from; i < to; i++) { 53 | const message = await source.next(); if (message instanceof FileErr) { throwNewError(message); } 54 | expect(message.payload.toString()).toStrictEqual(`hello-${i}`); 55 | } 56 | } 57 | } 58 | 59 | function throwNewError(err: FileErr): never { 60 | throw new Error(err.toString()); 61 | } 62 | -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/src/types.ts: -------------------------------------------------------------------------------- 1 | export const SEA_STREAMER_INTERNAL: string = "SEA_STREAMER_INTERNAL"; 2 | export const PULSE_MESSAGE: string = "PULSE"; 3 | export const END_OF_STREAM: string = "EOS"; 4 | export const EOS_MESSAGE_SIZE: bigint = 56n; 5 | 6 | export type Timestamp = Date; 7 | export class StreamKey { 8 | name: string; 9 | constructor(name: string) { 10 | this.name = name; 11 | } 12 | } 13 | export class SeqNo { 14 | no: bigint; 15 | constructor(no: bigint) { 16 | this.no = no; 17 | } 18 | } 19 | export class ShardId { 20 | id: bigint; 21 | constructor(id: bigint) { 22 | this.id = id; 23 | } 24 | } 25 | 26 | class SeqPosBeginning { } 27 | class SeqPosEnd { } 28 | class SeqPosAt { 29 | at: bigint; 30 | constructor(at: bigint) { 31 | this.at = at; 32 | } 33 | } 34 | const SeqPos = { 35 | Beginning: SeqPosBeginning, 36 | End: SeqPosEnd, 37 | At: SeqPosAt, 38 | } 39 | export { SeqPos } 40 | export type SeqPosEnum = SeqPosBeginning | SeqPosEnd | SeqPosAt; 41 | 42 | export enum StreamMode { 43 | /** 44 | * Streaming from a file at the end 45 | */ 46 | Live = "Live", 47 | /** 48 | * Replaying a dead file 49 | */ 50 | Replay = "Replay", 51 | /** 52 | * Replaying a live file, might catch up to live 53 | */ 54 | LiveReplay = "LiveReplay", 55 | } -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/testcases/consumer.ss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SeaQL/sea-streamer/07241fb987e8dc73cc9d3237bbdc077a358f48e2/sea-streamer-file/sea-streamer-file-reader/testcases/consumer.ss -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/testcases/quicksort.ss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SeaQL/sea-streamer/07241fb987e8dc73cc9d3237bbdc077a358f48e2/sea-streamer-file/sea-streamer-file-reader/testcases/quicksort.ss -------------------------------------------------------------------------------- /sea-streamer-file/sea-streamer-file-reader/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "es2020", 5 | "lib": ["es2020", "dom"], 6 | "outDir": "out", 7 | "sourceMap": true, 8 | "strict": true, 9 | "rootDir": "src" 10 | }, 11 | "exclude": ["node_modules", ".vscode-test"] 12 | } 13 | -------------------------------------------------------------------------------- /sea-streamer-file/src/bin/clock.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | use clap::Parser; 3 | use sea_streamer_file::{FileId, FileStreamer}; 4 | use sea_streamer_types::{Producer, StreamKey, Streamer}; 5 | use std::time::Duration; 6 | 7 | #[derive(Debug, Parser)] 8 | struct Args { 9 | #[clap(long, help = "Stream to this file")] 10 | file: FileId, 11 | #[clap(long, value_parser = parse_duration, help = "Period of the clock. e.g. 1s, 100ms")] 12 | interval: Duration, 13 | } 14 | 15 | fn parse_duration(src: &str) -> Result { 16 | if let Some(s) = src.strip_suffix("ms") { 17 | Ok(Duration::from_millis(s.parse()?)) 18 | } else if let Some(s) = src.strip_suffix('s') { 19 | Ok(Duration::from_secs(s.parse()?)) 20 | } else if let Some(s) = src.strip_suffix('m') { 21 | Ok(Duration::from_secs(s.parse::()? * 60)) 22 | } else { 23 | Err(anyhow!("Failed to parse {} as Duration", src)) 24 | } 25 | } 26 | 27 | #[tokio::main] 28 | async fn main() -> Result<()> { 29 | env_logger::init(); 30 | 31 | let Args { file, interval } = Args::parse(); 32 | 33 | let stream_key = StreamKey::new("clock")?; 34 | let streamer = FileStreamer::connect(file.to_streamer_uri()?, Default::default()).await?; 35 | let producer = streamer 36 | .create_producer(stream_key, Default::default()) 37 | .await?; 38 | 39 | for i in 0..u64::MAX { 40 | producer.send(format!("tick-{i}"))?; 41 | tokio::time::sleep(interval).await; 42 | } 43 | 44 | producer.end().await?; 45 | 46 | Ok(()) 47 | } 48 | -------------------------------------------------------------------------------- /sea-streamer-file/src/bin/sink.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | use clap::Parser; 3 | use sea_streamer_file::{FileId, MessageSink, DEFAULT_BEACON_INTERVAL, DEFAULT_FILE_SIZE_LIMIT}; 4 | use sea_streamer_types::{MessageHeader, OwnedMessage, ShardId, StreamKey, Timestamp}; 5 | use std::time::Duration; 6 | 7 | #[derive(Debug, Parser)] 8 | struct Args { 9 | #[clap(long, help = "Stream to this file")] 10 | file: FileId, 11 | #[clap(long, value_parser = parse_duration, help = "Period of the clock. e.g. 1s, 100ms")] 12 | interval: Duration, 13 | } 14 | 15 | fn parse_duration(src: &str) -> Result { 16 | if let Some(s) = src.strip_suffix("ms") { 17 | Ok(Duration::from_millis(s.parse()?)) 18 | } else if let Some(s) = src.strip_suffix('s') { 19 | Ok(Duration::from_secs(s.parse()?)) 20 | } else if let Some(s) = src.strip_suffix('m') { 21 | Ok(Duration::from_secs(s.parse::()? * 60)) 22 | } else { 23 | Err(anyhow!("Failed to parse {} as Duration", src)) 24 | } 25 | } 26 | 27 | #[tokio::main] 28 | async fn main() -> Result<()> { 29 | env_logger::init(); 30 | 31 | let Args { file, interval } = Args::parse(); 32 | let mut sink = MessageSink::new( 33 | file.clone(), 34 | DEFAULT_BEACON_INTERVAL, 35 | DEFAULT_FILE_SIZE_LIMIT, 36 | ) 37 | .await?; 38 | let stream_key = StreamKey::new("clock")?; 39 | let shard = ShardId::new(0); 40 | 41 | for i in 0..u64::MAX { 42 | let header = MessageHeader::new(stream_key.clone(), shard, i, Timestamp::now_utc()); 43 | let message = OwnedMessage::new(header, format!("tick-{i}").into_bytes()); 44 | sink.write(message)?; 45 | tokio::time::sleep(interval).await; 46 | } 47 | 48 | sink.flush().await?; 49 | 50 | Ok(()) 51 | } 52 | -------------------------------------------------------------------------------- /sea-streamer-file/src/bin/stdin-to-file.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer_file::export::flume::{unbounded, Receiver}; 4 | use sea_streamer_file::{AsyncFile, FileId}; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap(long, help = "Write to this file", default_value = "output.log")] 9 | file: FileId, 10 | } 11 | 12 | fn main() -> Result<()> { 13 | env_logger::init(); 14 | log::info!("Please type something into the console and press enter:"); 15 | 16 | let (sender, receiver) = unbounded(); 17 | 18 | let handle = std::thread::spawn(move || { 19 | let rt = tokio::runtime::Builder::new_current_thread() 20 | .enable_all() 21 | .build() 22 | .unwrap(); 23 | rt.block_on(file_sink(receiver)) 24 | }); 25 | 26 | for _ in 0..10 { 27 | let mut line = String::new(); 28 | match std::io::stdin().read_line(&mut line) { 29 | Ok(0) => break, // this means stdin is closed 30 | Ok(_) => (), 31 | Err(e) => panic!("{e:?}"), 32 | } 33 | sender.send(line)?; 34 | } 35 | 36 | std::mem::drop(sender); 37 | handle.join().unwrap().unwrap(); 38 | Ok(()) 39 | } 40 | 41 | async fn file_sink(receiver: Receiver) -> Result<()> { 42 | let Args { file } = Args::parse(); 43 | let mut file = AsyncFile::new_ow(file).await?; 44 | 45 | while let Ok(line) = receiver.recv_async().await { 46 | file.write_all(line.as_bytes()).await?; 47 | } 48 | 49 | Ok(()) 50 | } 51 | -------------------------------------------------------------------------------- /sea-streamer-file/src/bin/tail.rs: -------------------------------------------------------------------------------- 1 | //! A for demo `tail -f` program. 2 | use anyhow::Result; 3 | use clap::Parser; 4 | use sea_streamer_file::{FileId, FileSource, ReadFrom}; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap(long, help = "File path")] 9 | file: FileId, 10 | } 11 | 12 | #[tokio::main] 13 | async fn main() -> Result<()> { 14 | env_logger::init(); 15 | 16 | let Args { file } = Args::parse(); 17 | let mut stream = FileSource::new(file, ReadFrom::End).await?; 18 | 19 | loop { 20 | let bytes = stream.stream_bytes().await?; 21 | print!("{}", std::str::from_utf8(&bytes.bytes())?); 22 | std::io::Write::flush(&mut std::io::stdout()).unwrap(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /sea-streamer-file/src/consumer/future.rs: -------------------------------------------------------------------------------- 1 | //! Basically copied from sea-streamer-redis 2 | use super::{FileConsumer, NextFuture}; 3 | use crate::FileResult; 4 | use sea_streamer_types::{export::futures::Stream, Consumer, SharedMessage}; 5 | use std::{fmt::Debug, future::Future}; 6 | 7 | pub struct StreamFuture<'a> { 8 | con: &'a FileConsumer, 9 | fut: NextFuture<'a>, 10 | } 11 | 12 | impl Debug for StreamFuture<'_> { 13 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 14 | f.debug_struct("StreamFuture").finish() 15 | } 16 | } 17 | 18 | impl<'a> StreamFuture<'a> { 19 | pub fn new(con: &'a FileConsumer) -> Self { 20 | let fut = con.next(); 21 | Self { con, fut } 22 | } 23 | } 24 | 25 | impl Stream for StreamFuture<'_> { 26 | type Item = FileResult; 27 | 28 | fn poll_next( 29 | mut self: std::pin::Pin<&mut Self>, 30 | cx: &mut std::task::Context<'_>, 31 | ) -> std::task::Poll> { 32 | use std::task::Poll::{Pending, Ready}; 33 | match std::pin::Pin::new(&mut self.fut).poll(cx) { 34 | Ready(res) => { 35 | self.fut = self.con.next(); 36 | Ready(Some(res)) 37 | } 38 | Pending => Pending, 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /sea-streamer-file/src/crc.rs: -------------------------------------------------------------------------------- 1 | //! Generated on by pycrc https://pycrc.org 2 | //! 3 | //! using the configuration: 4 | //! - Width = 16 5 | //! - Poly = 0xc867 6 | //! - XorIn = 0xffff 7 | //! - ReflectIn = False 8 | //! - XorOut = 0x0000 9 | //! - ReflectOut = False 10 | //! - Algorithm = table-driven 11 | 12 | static CRC_TABLE: [u16; 256] = [ 13 | 0x0000, 0xc867, 0x58a9, 0x90ce, 0xb152, 0x7935, 0xe9fb, 0x219c, 0xaac3, 0x62a4, 0xf26a, 0x3a0d, 14 | 0x1b91, 0xd3f6, 0x4338, 0x8b5f, 0x9de1, 0x5586, 0xc548, 0x0d2f, 0x2cb3, 0xe4d4, 0x741a, 0xbc7d, 15 | 0x3722, 0xff45, 0x6f8b, 0xa7ec, 0x8670, 0x4e17, 0xded9, 0x16be, 0xf3a5, 0x3bc2, 0xab0c, 0x636b, 16 | 0x42f7, 0x8a90, 0x1a5e, 0xd239, 0x5966, 0x9101, 0x01cf, 0xc9a8, 0xe834, 0x2053, 0xb09d, 0x78fa, 17 | 0x6e44, 0xa623, 0x36ed, 0xfe8a, 0xdf16, 0x1771, 0x87bf, 0x4fd8, 0xc487, 0x0ce0, 0x9c2e, 0x5449, 18 | 0x75d5, 0xbdb2, 0x2d7c, 0xe51b, 0x2f2d, 0xe74a, 0x7784, 0xbfe3, 0x9e7f, 0x5618, 0xc6d6, 0x0eb1, 19 | 0x85ee, 0x4d89, 0xdd47, 0x1520, 0x34bc, 0xfcdb, 0x6c15, 0xa472, 0xb2cc, 0x7aab, 0xea65, 0x2202, 20 | 0x039e, 0xcbf9, 0x5b37, 0x9350, 0x180f, 0xd068, 0x40a6, 0x88c1, 0xa95d, 0x613a, 0xf1f4, 0x3993, 21 | 0xdc88, 0x14ef, 0x8421, 0x4c46, 0x6dda, 0xa5bd, 0x3573, 0xfd14, 0x764b, 0xbe2c, 0x2ee2, 0xe685, 22 | 0xc719, 0x0f7e, 0x9fb0, 0x57d7, 0x4169, 0x890e, 0x19c0, 0xd1a7, 0xf03b, 0x385c, 0xa892, 0x60f5, 23 | 0xebaa, 0x23cd, 0xb303, 0x7b64, 0x5af8, 0x929f, 0x0251, 0xca36, 0x5e5a, 0x963d, 0x06f3, 0xce94, 24 | 0xef08, 0x276f, 0xb7a1, 0x7fc6, 0xf499, 0x3cfe, 0xac30, 0x6457, 0x45cb, 0x8dac, 0x1d62, 0xd505, 25 | 0xc3bb, 0x0bdc, 0x9b12, 0x5375, 0x72e9, 0xba8e, 0x2a40, 0xe227, 0x6978, 0xa11f, 0x31d1, 0xf9b6, 26 | 0xd82a, 0x104d, 0x8083, 0x48e4, 0xadff, 0x6598, 0xf556, 0x3d31, 0x1cad, 0xd4ca, 0x4404, 0x8c63, 27 | 0x073c, 0xcf5b, 0x5f95, 0x97f2, 0xb66e, 0x7e09, 0xeec7, 0x26a0, 0x301e, 0xf879, 0x68b7, 0xa0d0, 28 | 0x814c, 0x492b, 0xd9e5, 0x1182, 0x9add, 0x52ba, 0xc274, 0x0a13, 0x2b8f, 0xe3e8, 0x7326, 0xbb41, 29 | 0x7177, 0xb910, 0x29de, 0xe1b9, 0xc025, 0x0842, 0x988c, 0x50eb, 0xdbb4, 0x13d3, 0x831d, 0x4b7a, 30 | 0x6ae6, 0xa281, 0x324f, 0xfa28, 0xec96, 0x24f1, 0xb43f, 0x7c58, 0x5dc4, 0x95a3, 0x056d, 0xcd0a, 31 | 0x4655, 0x8e32, 0x1efc, 0xd69b, 0xf707, 0x3f60, 0xafae, 0x67c9, 0x82d2, 0x4ab5, 0xda7b, 0x121c, 32 | 0x3380, 0xfbe7, 0x6b29, 0xa34e, 0x2811, 0xe076, 0x70b8, 0xb8df, 0x9943, 0x5124, 0xc1ea, 0x098d, 33 | 0x1f33, 0xd754, 0x479a, 0x8ffd, 0xae61, 0x6606, 0xf6c8, 0x3eaf, 0xb5f0, 0x7d97, 0xed59, 0x253e, 34 | 0x04a2, 0xccc5, 0x5c0b, 0x946c, 35 | ]; 36 | 37 | pub(crate) fn crc_update(mut crc: u16, data: &[u8]) -> u16 { 38 | for d in data.iter() { 39 | let tbl_idx = (crc >> 8) as u8 ^ *d; 40 | crc = CRC_TABLE[tbl_idx as usize] ^ (crc << 8); 41 | } 42 | crc 43 | } 44 | 45 | pub fn crc16_cdma2000(data: &[u8]) -> u16 { 46 | let mut crc = 0xffff; 47 | crc = crc_update(crc, data); 48 | crc 49 | } 50 | 51 | #[cfg(test)] 52 | mod test { 53 | use super::*; 54 | 55 | #[test] 56 | fn test_crc16_cdma2000() { 57 | assert_eq!(crc16_cdma2000("123456789".as_bytes()), 0x4C06); 58 | let str = "hello, world"; 59 | assert_eq!(crc16_cdma2000(str.as_bytes()), 0x8028); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /sea-streamer-file/src/error.rs: -------------------------------------------------------------------------------- 1 | use crate::{format::FormatErr, ConfigErr}; 2 | use sea_streamer_types::{StreamErr, StreamResult}; 3 | use std::str::Utf8Error; 4 | use thiserror::Error; 5 | 6 | #[derive(Error, Debug)] 7 | pub enum FileErr { 8 | #[error("ConfigErr: {0}")] 9 | ConfigErr(#[source] ConfigErr), 10 | #[error("Utf8Error: {0}")] 11 | Utf8Error(#[source] Utf8Error), 12 | #[error("IO Error: {0}")] 13 | IoError(#[source] std::io::Error), 14 | #[error("Duplicate IoError")] 15 | DuplicateIoError, 16 | #[error("Watch Error: {0}")] 17 | WatchError(String), 18 | #[error("FormatErr: {0}")] 19 | FormatErr(#[source] FormatErr), 20 | #[error("SeekErr: {0}")] 21 | SeekErr(#[source] SeekErr), 22 | #[error("File Removed")] 23 | FileRemoved, 24 | #[error("File Limit Exceeded")] 25 | FileLimitExceeded, 26 | #[error("Task Dead ({0})")] 27 | TaskDead(&'static str), 28 | #[error("Not Enough Bytes: the file might be truncated.")] 29 | NotEnoughBytes, 30 | #[error("Stream Ended: the file might have been removed or an EOS message.")] 31 | StreamEnded, 32 | #[error("Producer Ended: the file might have been removed or was ended intentionally.")] 33 | ProducerEnded, 34 | } 35 | 36 | #[derive(Error, Debug, Clone, Copy)] 37 | pub enum SeekErr { 38 | #[error("Out Of Bound: what you are seeking is probably not in this file")] 39 | OutOfBound, 40 | #[error("Exhausted: there must be an algorithmic error")] 41 | Exhausted, 42 | } 43 | 44 | pub type FileResult = StreamResult; 45 | 46 | impl FileErr { 47 | /// Take ownership of this Err, leaving a clone in place. 48 | pub fn take(&mut self) -> Self { 49 | let mut copy = match self { 50 | FileErr::ConfigErr(e) => FileErr::ConfigErr(*e), 51 | FileErr::Utf8Error(e) => FileErr::Utf8Error(*e), 52 | FileErr::IoError(_) => FileErr::DuplicateIoError, 53 | FileErr::DuplicateIoError => FileErr::DuplicateIoError, 54 | FileErr::WatchError(e) => FileErr::WatchError(e.clone()), 55 | FileErr::FormatErr(e) => FileErr::FormatErr(*e), 56 | FileErr::SeekErr(e) => FileErr::SeekErr(*e), 57 | FileErr::FileRemoved => FileErr::FileRemoved, 58 | FileErr::FileLimitExceeded => FileErr::FileLimitExceeded, 59 | FileErr::TaskDead(e) => FileErr::TaskDead(e), 60 | FileErr::NotEnoughBytes => FileErr::NotEnoughBytes, 61 | FileErr::StreamEnded => FileErr::StreamEnded, 62 | FileErr::ProducerEnded => FileErr::ProducerEnded, 63 | }; 64 | std::mem::swap(self, &mut copy); 65 | copy 66 | } 67 | } 68 | 69 | impl From for StreamErr { 70 | fn from(err: FileErr) -> Self { 71 | StreamErr::Backend(err) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /sea-streamer-file/src/export.rs: -------------------------------------------------------------------------------- 1 | pub use flume; 2 | -------------------------------------------------------------------------------- /sea-streamer-file/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! ### `sea-streamer-file`: File Backend 2 | //! 3 | //! This is very similar to `sea-streamer-stdio`, but the difference is SeaStreamerStdio works in real-time, 4 | //! while `sea-streamer-file` works in real-time and replay. That means, SeaStreamerFile has the ability to 5 | //! traverse a `.ss` (sea-stream) file and seek/rewind to a particular timestamp/offset. 6 | //! 7 | //! In addition, Stdio can only work with UTF-8 text data, while File is able to work with binary data. 8 | //! In Stdio, there is only one Streamer per process. In File, there can be multiple independent Streamers 9 | //! in the same process. Afterall, a Streamer is just a file. 10 | //! 11 | //! The basic idea of SeaStreamerFile is like a `tail -f` with one message per line, with a custom message frame 12 | //! carrying binary payloads. The interesting part is, in SeaStreamer, we do not use delimiters to separate messages. 13 | //! This removes the overhead of encoding/decoding message payloads. But it added some complexity to the file format. 14 | //! 15 | //! The SeaStreamerFile format is designed for efficient fast-forward and seeking. This is enabled by placing an array 16 | //! of Beacons at fixed interval in the file. A Beacon contains a summary of the streams, so it acts like an inplace 17 | //! index. It also allows readers to align with the message boundaries. To learn more about the file format, read 18 | //! [`src/format.rs`](https://github.com/SeaQL/sea-streamer/blob/main/sea-streamer-file/src/format.rs). 19 | //! 20 | //! On top of that, are the high-level SeaStreamer multi-producer, multi-consumer stream semantics, resembling 21 | //! the behaviour of other SeaStreamer backends. In particular, the load-balancing behaviour is same as Stdio, 22 | //! i.e. round-robin. 23 | //! 24 | //! ### Decoder 25 | //! 26 | //! We provide a small utility to decode `.ss` files: 27 | //! 28 | //! ```sh 29 | //! cargo install sea-streamer-file --features=executables --bin ss-decode 30 | //! # local version 31 | //! alias ss-decode='cargo run --package sea-streamer-file --features=executables --bin ss-decode' 32 | //! ss-decode -- --file --format 33 | //! ``` 34 | //! 35 | //! Pro tip: pipe it to `less` for pagination 36 | //! 37 | //! ```sh 38 | //! ss-decode --file mystream.ss | less 39 | //! ``` 40 | //! 41 | //! Example `log` format: 42 | //! 43 | //! ```log 44 | //! # header 45 | //! [2023-06-05T13:55:53.001 | hello | 1 | 0] message-1 46 | //! # beacon 47 | //! ``` 48 | //! 49 | //! Example `ndjson` format: 50 | //! 51 | //! ```json 52 | //! /* header */ 53 | //! {"header":{"stream_key":"hello","shard_id":0,"sequence":1,"timestamp":"2023-06-05T13:55:53.001"},"payload":"message-1"} 54 | //! /* beacon */ 55 | //! ``` 56 | //! 57 | //! There is also a Typescript implementation under [`sea-streamer-file-reader`](https://github.com/SeaQL/sea-streamer/tree/main/sea-streamer-file/sea-streamer-file-reader). 58 | //! 59 | //! ### TODO 60 | //! 61 | //! 1. Resumable: currently unimplemented. A potential implementation might be to commit into a local SQLite database. 62 | //! 2. Sharding: currently it only streams to Shard ZERO. 63 | //! 3. Verify: a utility program to verify and repair SeaStreamer binary file. 64 | mod buffer; 65 | mod consumer; 66 | mod crc; 67 | mod dyn_file; 68 | mod error; 69 | pub mod export; 70 | mod file; 71 | pub mod format; 72 | mod messages; 73 | mod producer; 74 | mod sink; 75 | mod source; 76 | mod streamer; 77 | mod surveyor; 78 | mod watcher; 79 | 80 | pub use buffer::*; 81 | pub use consumer::*; 82 | pub use dyn_file::*; 83 | pub use error::*; 84 | pub use file::*; 85 | pub use messages::*; 86 | pub use producer::*; 87 | pub use sink::*; 88 | pub use source::*; 89 | pub use streamer::*; 90 | pub use surveyor::*; 91 | 92 | pub const DEFAULT_BEACON_INTERVAL: u32 = 1024 * 1024; // 1MB 93 | pub const DEFAULT_FILE_SIZE_LIMIT: u64 = 16 * 1024 * 1024 * 1024; // 16GB 94 | pub const DEFAULT_PREFETCH_MESSAGE: usize = 1000; 95 | 96 | /// Reserved by SeaStreamer. Avoid using this as StreamKey. 97 | pub const SEA_STREAMER_WILDCARD: &str = "SEA_STREAMER_WILDCARD"; 98 | -------------------------------------------------------------------------------- /sea-streamer-file/tests/sample-1.ss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SeaQL/sea-streamer/07241fb987e8dc73cc9d3237bbdc077a358f48e2/sea-streamer-file/tests/sample-1.ss -------------------------------------------------------------------------------- /sea-streamer-file/tests/sample.rs: -------------------------------------------------------------------------------- 1 | static INIT: std::sync::Once = std::sync::Once::new(); 2 | 3 | // cargo test --test sample --features=test,runtime-tokio -- --nocapture 4 | // cargo test --test sample --features=test,runtime-async-std -- --nocapture 5 | #[cfg(feature = "test")] 6 | #[cfg_attr(feature = "runtime-tokio", tokio::test)] 7 | #[cfg_attr(feature = "runtime-async-std", async_std::test)] 8 | async fn sample_1() -> anyhow::Result<()> { 9 | use sea_streamer_file::{AutoStreamReset, FileConsumerOptions, FileErr, FileId, FileStreamer}; 10 | use sea_streamer_types::{Consumer, Message, StreamErr, StreamKey, Streamer}; 11 | 12 | INIT.call_once(env_logger::init); 13 | 14 | let file_id: FileId = "tests/sample-1.ss".parse().unwrap(); 15 | let streamer = FileStreamer::connect(file_id.to_streamer_uri()?, Default::default()).await?; 16 | let mut options = FileConsumerOptions::default(); 17 | options.set_auto_stream_reset(AutoStreamReset::Earliest); 18 | let consumer = streamer 19 | .create_consumer(&[StreamKey::new("event")?], options) 20 | .await?; 21 | 22 | for i in 1..=22 { 23 | let mess = consumer.next().await?; 24 | assert_eq!(mess.sequence(), i); 25 | } 26 | let err = consumer.next().await; 27 | assert!(matches!(err, Err(StreamErr::Backend(FileErr::StreamEnded)))); 28 | 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /sea-streamer-file/tests/surveyor.rs: -------------------------------------------------------------------------------- 1 | // cargo test --test surveyor --features=test,runtime-tokio -- --nocapture 2 | #[cfg(feature = "test")] 3 | #[cfg_attr(feature = "runtime-tokio", tokio::test)] 4 | #[cfg_attr(feature = "runtime-async-std", async_std::test)] 5 | async fn surveyor() -> anyhow::Result<()> { 6 | use sea_streamer_file::{format::Beacon, MockBeacon, SurveyResult, Surveyor}; 7 | use std::cmp::Ordering; 8 | 9 | env_logger::init(); 10 | 11 | const TARGET: u32 = 3; 12 | let finder = |b: &Beacon| { 13 | if b.remaining_messages_bytes == 0 { 14 | SurveyResult::Undecided 15 | } else { 16 | match b.remaining_messages_bytes.cmp(&TARGET) { 17 | Ordering::Less | Ordering::Equal => SurveyResult::Left, 18 | Ordering::Greater => SurveyResult::Right, 19 | } 20 | } 21 | }; 22 | 23 | // baseline, no beacon at all 24 | let mut beacon = MockBeacon::new(10); 25 | let surveyor = Surveyor::new(&mut beacon, finder).await?; 26 | assert_eq!(surveyor.run().await?, (0, u32::MAX)); // no scope at all 27 | 28 | // what we are looking for is between 3 & 4 29 | let mut beacon = MockBeacon::new(10); 30 | for i in 1..=10 { 31 | add(&mut beacon, i); 32 | } 33 | let surveyor = Surveyor::new(&mut beacon, finder).await?; 34 | assert_eq!(surveyor.run().await?, (3, 4)); 35 | 36 | // 4 is left empty 37 | let mut beacon = MockBeacon::new(10); 38 | for i in 1..=10 { 39 | if i != 4 { 40 | add(&mut beacon, i); 41 | } 42 | } 43 | let surveyor = Surveyor::new(&mut beacon, finder).await?; 44 | assert_eq!(surveyor.run().await?, (3, 5)); 45 | 46 | // 3 & 4 is left empty 47 | let mut beacon = MockBeacon::new(10); 48 | for i in 1..=10 { 49 | if !matches!(i, 3 | 4) { 50 | add(&mut beacon, i); 51 | } 52 | } 53 | let surveyor = Surveyor::new(&mut beacon, finder).await?; 54 | assert_eq!(surveyor.run().await?, (2, 5)); 55 | 56 | // there is only 1, 8 & 9 57 | let mut beacon = MockBeacon::new(10); 58 | add(&mut beacon, 1); 59 | add(&mut beacon, 8); 60 | add(&mut beacon, 9); 61 | let surveyor = Surveyor::new(&mut beacon, finder).await?; 62 | assert_eq!(surveyor.run().await?, (1, 8)); 63 | 64 | // there is only 8 & 9 65 | let mut beacon = MockBeacon::new(10); 66 | add(&mut beacon, 8); 67 | add(&mut beacon, 9); 68 | let surveyor = Surveyor::new(&mut beacon, finder).await?; 69 | assert_eq!(surveyor.run().await?, (0, 8)); 70 | 71 | // there is only 1 beacon 72 | let mut beacon = MockBeacon::new(10); 73 | add(&mut beacon, 3); 74 | let surveyor = Surveyor::new(&mut beacon, finder).await?; 75 | assert_eq!(surveyor.run().await?, (3, u32::MAX)); 76 | 77 | // there is only 1 beacon 78 | let mut beacon = MockBeacon::new(10); 79 | add(&mut beacon, 8); 80 | let surveyor = Surveyor::new(&mut beacon, finder).await?; 81 | assert_eq!(surveyor.run().await?, (0, 8)); 82 | 83 | fn add(beacon: &mut MockBeacon, i: u32) { 84 | beacon.add( 85 | i, 86 | Beacon { 87 | remaining_messages_bytes: i, 88 | items: Vec::new(), 89 | }, 90 | ) 91 | } 92 | 93 | Ok(()) 94 | } 95 | -------------------------------------------------------------------------------- /sea-streamer-file/tests/util.rs: -------------------------------------------------------------------------------- 1 | use sea_streamer_file::FileId; 2 | use sea_streamer_types::Timestamp; 3 | use std::fs::OpenOptions; 4 | 5 | pub fn temp_file(name: &str) -> Result { 6 | let path = format!("/tmp/{name}"); 7 | let _file = OpenOptions::new() 8 | .read(true) 9 | .write(true) 10 | .create_new(true) 11 | .open(&path)?; 12 | 13 | Ok(FileId::new(path)) 14 | } 15 | 16 | pub fn millis_of(ts: &Timestamp) -> i64 { 17 | (ts.unix_timestamp_nanos() / 1_000_000) as i64 18 | } 19 | -------------------------------------------------------------------------------- /sea-streamer-fuse/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-fuse" 3 | version = "0.5.2" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "Stream processing toolbox" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/sea-streamer-fuse" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | keywords = ["async", "stream", "stream-processing"] 12 | rust-version = "1.60" 13 | 14 | [package.metadata.docs.rs] 15 | all-features = true 16 | rustdoc-args = ["--cfg", "docsrs"] 17 | 18 | [dependencies] 19 | thiserror = { version = "1", default-features = false } 20 | pin-project = { version = "1.1" } 21 | 22 | sea-streamer-types = { version = "0.5", path = "../sea-streamer-types" } 23 | 24 | [dev-dependencies] 25 | tokio = { version = "1", features = ["full"] } 26 | sea-streamer-socket = { version = "0.5", path = "../sea-streamer-socket" } 27 | -------------------------------------------------------------------------------- /sea-streamer-kafka/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-kafka" 3 | version = "0.5.0" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "🌊 SeaStreamer Kafka / Redpanda Backend" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/sea-streamer-kafka" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | keywords = ["async", "stream", "kafka", "stream-processing"] 12 | rust-version = "1.60" 13 | 14 | [package.metadata.docs.rs] 15 | features = [] 16 | rustdoc-args = ["--cfg", "docsrs"] 17 | 18 | [dependencies] 19 | anyhow = { version = "1", optional = true } 20 | async-std = { version = "1", optional = true } 21 | env_logger = { version = "0.9", optional = true } 22 | lazy_static = { version = "1.4" } 23 | mac_address = { version = "1" } 24 | rdkafka = { version = "0.36", default-features = false, features = ["libz"] } 25 | sea-streamer-types = { version = "0.5", path = "../sea-streamer-types" } 26 | sea-streamer-runtime = { version = "0.5", path = "../sea-streamer-runtime" } 27 | clap = { version = "4.5", features = ["derive", "env"], optional = true } 28 | tokio = { version = "1.10.0", optional = true } 29 | 30 | [dev-dependencies] 31 | 32 | [features] 33 | test = ["anyhow", "async-std?/attributes", "tokio?/full", "env_logger"] 34 | executables = ["anyhow", "env_logger", "clap", "runtime-tokio", "tokio/full"] 35 | runtime-async-std = ["async-std", "sea-streamer-runtime/runtime-async-std"] 36 | runtime-tokio = ["tokio", "rdkafka/tokio", "sea-streamer-runtime/runtime-tokio"] 37 | # passthru of rdkafka features 38 | libz = ["rdkafka/libz"] 39 | libz-static = ["rdkafka/libz-static"] 40 | ssl = ["rdkafka/ssl"] 41 | ssl-vendored = ["rdkafka/ssl-vendored"] 42 | gssapi-vendored = ["rdkafka/gssapi-vendored"] 43 | 44 | [[bin]] 45 | name = "consumer" 46 | path = "src/bin/consumer.rs" 47 | required-features = ["executables"] 48 | 49 | [[bin]] 50 | name = "producer" 51 | path = "src/bin/producer.rs" 52 | required-features = ["executables"] 53 | -------------------------------------------------------------------------------- /sea-streamer-kafka/NOTES.md: -------------------------------------------------------------------------------- 1 | docker run -d --rm -p 9092:9092 -p 9644:9644 redpandadata/redpanda:latest redpanda start --overprovisioned --smp 1 --memory 1G --reserve-memory 0M --node-id 0 --check=false -------------------------------------------------------------------------------- /sea-streamer-kafka/README.md: -------------------------------------------------------------------------------- 1 | ### `sea-streamer-kafka`: Kafka / Redpanda Backend 2 | 3 | This is the Kafka / Redpanda backend implementation for SeaStreamer. 4 | This crate provides a comprehensive type system that makes working with Kafka easier and safer. 5 | 6 | First of all, all API (many are sync) are properly wrapped as async. Methods are also marked `&mut` to eliminate possible race conditions. 7 | 8 | `KafkaConsumerOptions` has typed parameters. 9 | 10 | `KafkaConsumer` allows you to `seek` to point in time, `rewind` to particular offset, and `commit` message read. 11 | 12 | `KafkaProducer` allows you to `await` a send `Receipt` or discard it if you are uninterested. You can also flush the Producer. 13 | 14 | `KafkaStreamer` allows you to flush all producers on `disconnect`. 15 | 16 | See [tests](https://github.com/SeaQL/sea-streamer/blob/main/sea-streamer-kafka/tests/consumer.rs) for an illustration of the stream semantics. 17 | 18 | This crate depends on [`rdkafka`](https://docs.rs/rdkafka), 19 | which in turn depends on [librdkafka-sys](https://docs.rs/librdkafka-sys), which itself is a wrapper of 20 | [librdkafka](https://docs.confluent.io/platform/current/clients/librdkafka/html/index.html). 21 | 22 | Configuration Reference: 23 | -------------------------------------------------------------------------------- /sea-streamer-kafka/src/bin/consumer.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer_kafka::{AutoOffsetReset, KafkaConsumerOptions, KafkaStreamer}; 4 | use sea_streamer_types::{ 5 | Buffer, Consumer, ConsumerMode, ConsumerOptions, Message, StreamUrl, Streamer, 6 | }; 7 | 8 | #[derive(Debug, Parser)] 9 | struct Args { 10 | #[clap( 11 | long, 12 | help = "Streamer URI with stream key(s), i.e. try `kafka://localhost:9092/hello`", 13 | env = "STREAM_URL" 14 | )] 15 | stream: StreamUrl, 16 | } 17 | 18 | #[tokio::main] 19 | async fn main() -> Result<()> { 20 | env_logger::init(); 21 | 22 | let Args { stream } = Args::parse(); 23 | 24 | let streamer = KafkaStreamer::connect(stream.streamer(), Default::default()).await?; 25 | let mut options = KafkaConsumerOptions::new(ConsumerMode::RealTime); 26 | options.set_auto_offset_reset(AutoOffsetReset::Earliest); 27 | let consumer = streamer 28 | .create_consumer(stream.stream_keys(), options) 29 | .await?; 30 | 31 | loop { 32 | let mess = consumer.next().await?; 33 | println!("{}", mess.message().as_str()?); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /sea-streamer-kafka/src/bin/producer.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer_kafka::KafkaStreamer; 4 | use sea_streamer_types::{Producer, StreamUrl, Streamer}; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap( 9 | long, 10 | help = "Streamer URI with stream key, i.e. try `kafka://localhost:9092/hello`", 11 | env = "STREAM_URL" 12 | )] 13 | stream: StreamUrl, 14 | } 15 | 16 | #[tokio::main] 17 | async fn main() -> Result<()> { 18 | env_logger::init(); 19 | 20 | let Args { stream } = Args::parse(); 21 | 22 | let streamer = KafkaStreamer::connect(stream.streamer(), Default::default()).await?; 23 | let producer = streamer 24 | .create_producer(stream.stream_key()?, Default::default()) 25 | .await?; 26 | 27 | for i in 0..1000 { 28 | let message = format!("{{\"hello\": {i}}}"); 29 | producer.send(message)?; 30 | } 31 | 32 | streamer.disconnect().await?; 33 | 34 | Ok(()) 35 | } 36 | -------------------------------------------------------------------------------- /sea-streamer-kafka/src/cluster.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Write; 2 | 3 | use crate::KafkaErr; 4 | use sea_streamer_types::StreamerUri; 5 | 6 | pub(crate) fn cluster_uri(streamer: &StreamerUri) -> Result { 7 | let mut string = String::new(); 8 | for (i, node) in streamer.nodes().iter().enumerate() { 9 | write!( 10 | string, 11 | "{comma}{node}", 12 | comma = if i != 0 { "," } else { "" }, 13 | node = node.as_str().trim_start_matches("kafka://") 14 | ) 15 | .unwrap(); 16 | } 17 | if string.is_empty() { 18 | return Err(KafkaErr::ClientCreation( 19 | "StreamerUri has no nodes".to_owned(), 20 | )); 21 | } 22 | Ok(string) 23 | } 24 | 25 | #[cfg(test)] 26 | #[test] 27 | fn test_cluster_uri() { 28 | let uri: StreamerUri = "kafka://localhost:9092".parse().unwrap(); 29 | assert_eq!(cluster_uri(&uri).unwrap(), "localhost:9092"); 30 | let uri: StreamerUri = "localhost:9092".parse().unwrap(); 31 | assert_eq!(cluster_uri(&uri).unwrap(), "localhost:9092"); 32 | let uri: StreamerUri = "kafka://host-a:9092,host-b:9092".parse().unwrap(); 33 | assert_eq!(cluster_uri(&uri).unwrap(), "host-a:9092,host-b:9092"); 34 | } 35 | -------------------------------------------------------------------------------- /sea-streamer-kafka/src/error.rs: -------------------------------------------------------------------------------- 1 | pub use rdkafka::error::KafkaError as KafkaErr; 2 | use sea_streamer_types::{StreamErr, StreamResult}; 3 | 4 | pub type KafkaResult = StreamResult; 5 | 6 | pub(crate) fn stream_err(err: KafkaErr) -> StreamErr { 7 | StreamErr::Backend(err) 8 | } 9 | -------------------------------------------------------------------------------- /sea-streamer-kafka/src/host.rs: -------------------------------------------------------------------------------- 1 | //! This module is shared between `sea-streamer-kafka` and `sea-streamer-redis` 2 | use mac_address::get_mac_address; 3 | use std::{ 4 | fs::File, 5 | io::{BufRead, BufReader}, 6 | }; 7 | 8 | lazy_static::lazy_static! { 9 | static ref HOST_ID: String = init(); 10 | } 11 | 12 | const LEN: usize = 12; 13 | 14 | fn init() -> String { 15 | if let Ok(host_id) = std::env::var("HOST_ID") { 16 | return host_id; 17 | } 18 | if let Ok(file) = File::open("/proc/self/cgroup") { 19 | // check whether this is a docker container 20 | let last = BufReader::new(file) 21 | .lines() 22 | .last() 23 | .expect("Empty file?") 24 | .expect("IO Error"); 25 | if let Some((_, remaining)) = last.split_once("0::/docker/") { 26 | if remaining.is_empty() { 27 | panic!("Failed to get docker container ID"); 28 | } 29 | let (container_id, _) = remaining.split_at(LEN); 30 | return container_id.to_owned(); 31 | } 32 | } 33 | let mac = get_mac_address() 34 | .expect("Failed to get MAC address") 35 | .expect("There is no MAC address on this host"); 36 | let mac = mac.to_string().replace(':', ""); 37 | let (mac, _) = mac.split_at(LEN); 38 | mac.to_owned() 39 | } 40 | 41 | /// Get the host id. Inside Docker, it's the container ID. Otherwise it's the MAC address. 42 | /// You can override it via the `HOST_ID` env var. 43 | pub fn host_id() -> &'static str { 44 | &HOST_ID 45 | } 46 | -------------------------------------------------------------------------------- /sea-streamer-kafka/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! ### `sea-streamer-kafka`: Kafka / Redpanda Backend 2 | //! 3 | //! This is the Kafka / Redpanda backend implementation for SeaStreamer. 4 | //! This crate provides a comprehensive type system that makes working with Kafka easier and safer. 5 | //! 6 | //! First of all, all API (many are sync) are properly wrapped as async. Methods are also marked `&mut` to eliminate possible race conditions. 7 | //! 8 | //! `KafkaConsumerOptions` has typed parameters. 9 | //! 10 | //! `KafkaConsumer` allows you to `seek` to point in time, `rewind` to particular offset, and `commit` message read. 11 | //! 12 | //! `KafkaProducer` allows you to `await` a send `Receipt` or discard it if you are uninterested. You can also flush the Producer. 13 | //! 14 | //! `KafkaStreamer` allows you to flush all producers on `disconnect`. 15 | //! 16 | //! See [tests](https://github.com/SeaQL/sea-streamer/blob/main/sea-streamer-kafka/tests/consumer.rs) for an illustration of the stream semantics. 17 | //! 18 | //! This crate depends on [`rdkafka`](https://docs.rs/rdkafka), 19 | //! which in turn depends on [librdkafka-sys](https://docs.rs/librdkafka-sys), which itself is a wrapper of 20 | //! [librdkafka](https://docs.confluent.io/platform/current/clients/librdkafka/html/index.html). 21 | //! 22 | //! Configuration Reference: 23 | 24 | #![cfg_attr(docsrs, feature(doc_cfg))] 25 | #![deny(missing_debug_implementations)] 26 | #![doc( 27 | html_logo_url = "https://raw.githubusercontent.com/SeaQL/sea-streamer/main/docs/SeaQL icon.png" 28 | )] 29 | 30 | /// The default Kafka port number 31 | pub const KAFKA_PORT: u16 = 9092; 32 | 33 | /// The default timeout, if needed but unspecified 34 | pub const DEFAULT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); 35 | 36 | #[cfg(all(feature = "runtime-async-std", feature = "runtime-tokio"))] 37 | compile_error!("'runtime-async-std' and 'runtime-tokio' cannot be enabled at the same time"); 38 | 39 | mod cluster; 40 | mod consumer; 41 | mod error; 42 | mod host; 43 | mod producer; 44 | mod runtime; 45 | mod streamer; 46 | 47 | use cluster::*; 48 | pub use consumer::*; 49 | pub use error::*; 50 | pub use host::*; 51 | pub use producer::*; 52 | pub use runtime::*; 53 | pub use streamer::*; 54 | 55 | /// Re-export types from `rdkafka` 56 | pub mod export { 57 | pub use rdkafka; 58 | } 59 | 60 | macro_rules! impl_into_string { 61 | ($name:ident) => { 62 | impl From<$name> for String { 63 | fn from(o: $name) -> Self { 64 | o.as_str().to_owned() 65 | } 66 | } 67 | }; 68 | } 69 | 70 | pub(crate) use impl_into_string; 71 | -------------------------------------------------------------------------------- /sea-streamer-kafka/src/runtime/async_std_impl.rs: -------------------------------------------------------------------------------- 1 | use rdkafka::util::AsyncRuntime; 2 | use std::{future::Future, pin::Pin, time::Duration}; 3 | 4 | #[derive(Debug)] 5 | pub struct AsyncStdRuntime; 6 | 7 | impl AsyncRuntime for AsyncStdRuntime { 8 | // Sadly I don't see an easy way to remove the box here 9 | type Delay = Pin + Send>>; 10 | 11 | fn spawn(task: T) 12 | where 13 | T: Future + Send + 'static, 14 | { 15 | async_std::task::spawn(task); 16 | } 17 | 18 | fn delay_for(duration: Duration) -> Self::Delay { 19 | Box::pin(async_std::task::sleep(duration)) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /sea-streamer-kafka/src/runtime/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "runtime-async-std")] 2 | mod async_std_impl; 3 | 4 | #[cfg(feature = "runtime-async-std")] 5 | pub use async_std_impl::AsyncStdRuntime as KafkaAsyncRuntime; 6 | 7 | #[cfg(feature = "runtime-tokio")] 8 | pub use rdkafka::util::TokioRuntime as KafkaAsyncRuntime; 9 | 10 | #[cfg(not(any(feature = "runtime-tokio", feature = "runtime-async-std")))] 11 | mod no_rt; 12 | 13 | #[cfg(not(any(feature = "runtime-tokio", feature = "runtime-async-std")))] 14 | pub use no_rt::NoRuntime as KafkaAsyncRuntime; 15 | -------------------------------------------------------------------------------- /sea-streamer-kafka/src/runtime/no_rt.rs: -------------------------------------------------------------------------------- 1 | use rdkafka::util::AsyncRuntime; 2 | use std::{ 3 | future::{Future, Ready}, 4 | time::Duration, 5 | }; 6 | 7 | #[derive(Debug)] 8 | pub struct NoRuntime; 9 | 10 | impl AsyncRuntime for NoRuntime { 11 | type Delay = Ready<()>; 12 | 13 | fn spawn(_: T) 14 | where 15 | T: Future + Send + 'static, 16 | { 17 | panic!("Please enable a runtime"); 18 | } 19 | 20 | fn delay_for(_: Duration) -> Self::Delay { 21 | panic!("Please enable a runtime"); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /sea-streamer-redis/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-redis" 3 | version = "0.5.2" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "🌊 SeaStreamer Redis Backend" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/sea-streamer-redis" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | keywords = ["async", "stream", "redis", "stream-processing"] 12 | rust-version = "1.60" 13 | 14 | [package.metadata.docs.rs] 15 | features = [] 16 | rustdoc-args = ["--cfg", "docsrs"] 17 | 18 | [dependencies] 19 | anyhow = { version = "1", optional = true } 20 | async-std = { version = "1", optional = true } 21 | env_logger = { version = "0.9", optional = true } 22 | flume = { version = "0.11", default-features = false, features = ["async"] } 23 | lazy_static = { version = "1.4" } 24 | log = { version = "0.4", default-features = false } 25 | mac_address = { version = "1" } 26 | redis = { version = "0.25", default-features = false, features = ["acl", "streams"] } 27 | sea-streamer-types = { version = "0.5", path = "../sea-streamer-types" } 28 | sea-streamer-runtime = { version = "0.5", path = "../sea-streamer-runtime" } 29 | clap = { version = "4.5", features = ["derive", "env"], optional = true } 30 | thiserror = { version = "1", default-features = false } 31 | tokio = { version = "1.10.0", optional = true } 32 | 33 | [dev-dependencies] 34 | 35 | [features] 36 | default = ["runtime-tokio"] # sadly redis does not compile without a runtime 37 | test = ["anyhow", "async-std?/attributes", "tokio?/full", "env_logger"] 38 | executables = ["anyhow", "env_logger", "clap", "runtime-tokio", "tokio/full"] 39 | runtime-async-std = ["async-std", "redis/async-std-comp", "sea-streamer-runtime/runtime-async-std"] 40 | runtime-tokio = ["tokio", "redis/tokio-comp", "sea-streamer-runtime/runtime-tokio"] 41 | runtime-async-std-native-tls = ["runtime-async-std", "redis/async-std-native-tls-comp"] 42 | runtime-tokio-native-tls = ["runtime-tokio", "redis/tokio-native-tls-comp"] 43 | nanosecond-timestamp = ["sea-streamer-types/wide-seq-no"] 44 | 45 | [[bin]] 46 | name = "consumer" 47 | path = "src/bin/consumer.rs" 48 | required-features = ["executables"] 49 | 50 | [[bin]] 51 | name = "producer" 52 | path = "src/bin/producer.rs" 53 | required-features = ["executables"] 54 | 55 | [[bin]] 56 | name = "trim-stream" 57 | path = "src/bin/trim-stream.rs" 58 | required-features = ["executables"] 59 | 60 | [[bin]] 61 | name = "manager-test" 62 | path = "src/bin/manager-test.rs" 63 | required-features = ["executables"] 64 | -------------------------------------------------------------------------------- /sea-streamer-redis/README.md: -------------------------------------------------------------------------------- 1 | ### `sea-streamer-redis`: Redis Backend 2 | 3 | This is the Redis backend implementation for SeaStreamer. 4 | This crate provides a high-level async API on top of Redis that makes working with Redis Streams fool-proof: 5 | 6 | + Implements the familiar SeaStreamer abstract interface 7 | + A comprehensive type system that guides/restricts you with the API 8 | + High-level API, so you don't call `XADD`, `XREAD` or `XACK` anymore 9 | + Mutex-free implementation: concurrency achieved by message passing 10 | + Pipelined `XADD` and paged `XREAD`, with a throughput in the realm of 100k messages per second 11 | 12 | While we'd like to provide a Kafka-like client experience, there are some fundamental differences between Redis and Kafka: 13 | 14 | 1. In Redis sequence numbers are not contiguous 15 | 1. In Kafka sequence numbers are contiguous 16 | 2. In Redis messages are dispatched to consumers among group members in a first-ask-first-served manner, which leads to the next point 17 | 1. In Kafka consumer <-> shard is 1 to 1 in a consumer group 18 | 3. In Redis `ACK` has to be done per message 19 | 1. In Kafka only 1 Ack (read-up-to) is needed for a series of reads 20 | 21 | What's already implemented: 22 | 23 | + RealTime mode with AutoStreamReset 24 | + Resumable mode with auto-ack and/or auto-commit 25 | + LoadBalanced mode with failover behaviour 26 | + Seek/rewind to point in time 27 | + Basic stream sharding: split a stream into multiple sub-streams 28 | 29 | It's best to look through the [tests](https://github.com/SeaQL/sea-streamer/tree/main/sea-streamer-redis/tests) 30 | for an illustration of the different streaming behaviour. 31 | 32 | How SeaStreamer offers better concurrency? 33 | 34 | Consider the following simple stream processor: 35 | 36 | ```rust 37 | loop { 38 | let input = XREAD.await; 39 | let output = process(input).await; 40 | XADD(output).await; 41 | } 42 | ``` 43 | 44 | When it's reading or writing, it's not processing. So it's wasting time idle and reading messages with a higher delay, which in turn limits the throughput. 45 | In addition, the ideal batch size for reads may not be the ideal batch size for writes. 46 | 47 | With SeaStreamer, the read and write loops are separated from your process loop, so they can all happen in parallel (async in Rust is multi-threaded, so it is truely parallel)! 48 | 49 | ![](https://raw.githubusercontent.com/SeaQL/sea-streamer/main/sea-streamer-redis/docs/sea-streamer-concurrency.svg) 50 | 51 | If you are reading from a consumer group, you also have to consider when to ACK and how many ACKs to batch in one request. SeaStreamer can commit in the background on a regular interval, or you can commit asynchronously without blocking your process loop. 52 | 53 | In the future, we'd like to support Redis Cluster, because sharding without clustering is not very useful. 54 | Right now it's pretty much a work-in-progress. 55 | It's quite a difficult task, because clients have to take responsibility when working with a cluster. 56 | In Redis, shards and nodes is a M-N mapping - shards can be moved among nodes *at any time*. 57 | It makes testing much more difficult. 58 | Let us know if you'd like to help! 59 | 60 | You can quickly start a Redis instance via Docker: 61 | 62 | ```sh 63 | docker run -d --rm --name redis -p 6379:6379 redis 64 | ``` 65 | 66 | There is also a [small utility](https://github.com/SeaQL/sea-streamer/tree/main/sea-streamer-redis/redis-streams-dump) to dump Redis Streams messages into a SeaStreamer file. 67 | 68 | This crate is built on top of [`redis`](https://docs.rs/redis). 69 | -------------------------------------------------------------------------------- /sea-streamer-redis/docs/sea-streamer-concurrency.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 12 | 14 | 15 | READ 16 | 17 | PROCESS 18 | 19 | WRITE 20 | 21 | READ 22 | 23 | PROCESS 24 | 25 | WRITE 26 | 27 | ... 28 | 29 | ... 30 | 31 | ... 32 | 33 | ... 34 | 35 | READ 36 | 37 | PROCESS 38 | 39 | WRITE 40 | 41 | READ 42 | 43 | PROCESS 44 | 45 | WRITE 46 | 47 | 48 | 49 | 50 | 51 | SeaStreamer 52 | 53 | -------------------------------------------------------------------------------- /sea-streamer-redis/redis-streams-dump/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "redis-streams-dump" 3 | version = "0.5.0" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "A small utility to dump Redis Streams content into a SeaStreamer file" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/redis-streams-dump" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | keywords = ["async", "stream", "redis", "stream-processing"] 12 | rust-version = "1.60" 13 | 14 | [dependencies] 15 | anyhow = { version = "1" } 16 | env_logger = { version = "0.9" } 17 | log = { version = "0.4", default-features = false } 18 | sea-streamer-file = { version = "0.5", path = "../../sea-streamer-file", features = ["runtime-tokio"] } 19 | sea-streamer-redis = { version = "0.5", path = "../../sea-streamer-redis", features = ["runtime-tokio"] } 20 | sea-streamer-types = { version = "0.5", path = "../../sea-streamer-types" } 21 | clap = { version = "4.5", features = ["derive"] } 22 | time = { version = "0.3", default-features = false, features = ["std", "parsing"] } 23 | tokio = { version = "1.10.0", features = ["full"]} 24 | 25 | [dev-dependencies] 26 | 27 | [features] 28 | runtime-tokio-native-tls = ["sea-streamer-redis/runtime-tokio-native-tls"] 29 | -------------------------------------------------------------------------------- /sea-streamer-redis/redis-streams-dump/README.md: -------------------------------------------------------------------------------- 1 | # Redis Streams Dump 2 | 3 | A small utility to dump Redis Streams messages into a SeaStreamer file. 4 | 5 | ```sh 6 | cargo install redis-streams-dump 7 | ``` 8 | 9 | ```sh 10 | redis-streams-dump --stream redis://localhost/clock --output ./clock.ss --since 2023-09-05T13:30:00.7 --until 2023-09-05T13:30:00.8 11 | # OR in the workspace 12 | cargo run --package redis-streams-dump -- .. 13 | ``` 14 | 15 | ```sh 16 | redis-streams-dump 17 | 18 | USAGE: 19 | redis-streams-dump [OPTIONS] --output --stream 20 | 21 | FLAGS: 22 | -h, --help Prints help information 23 | -V, --version Prints version information 24 | 25 | OPTIONS: 26 | --output Output file. Overwrites if exist 27 | --since Timestamp start of range 28 | --stream Streamer URI with stream key, i.e. try `redis://localhost/hello` 29 | --until Timestamp end of range 30 | ``` -------------------------------------------------------------------------------- /sea-streamer-redis/redis-streams-dump/src/main.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer_file::{FileId, MessageSink, DEFAULT_BEACON_INTERVAL, DEFAULT_FILE_SIZE_LIMIT}; 4 | use sea_streamer_redis::{AutoStreamReset, RedisConsumerOptions, RedisStreamer}; 5 | use sea_streamer_types::{ 6 | Consumer, ConsumerMode, ConsumerOptions, Message, StreamUrl, Streamer, Timestamp, 7 | TIMESTAMP_FORMAT, 8 | }; 9 | use std::time::Duration; 10 | use time::PrimitiveDateTime; 11 | 12 | #[derive(Parser)] 13 | struct Args { 14 | #[clap( 15 | long, 16 | help = "Streamer URI with stream key, i.e. try `redis://localhost/hello`" 17 | )] 18 | stream: StreamUrl, 19 | #[clap(long, help = "Output file. Overwrites if exist")] 20 | output: FileId, 21 | #[clap(long, help = "Timestamp start of range")] 22 | since: Option, 23 | #[clap(long, help = "Timestamp end of range")] 24 | until: Option, 25 | } 26 | 27 | #[tokio::main] 28 | async fn main() -> Result<()> { 29 | env_logger::init(); 30 | 31 | let Args { 32 | stream, 33 | output, 34 | since, 35 | until, 36 | } = Args::parse(); 37 | 38 | let since = since.map(|s| parse_timestamp(&s).unwrap()); 39 | let until = until.map(|s| parse_timestamp(&s).unwrap()); 40 | 41 | let streamer = RedisStreamer::connect(stream.streamer(), Default::default()).await?; 42 | let mut options = RedisConsumerOptions::new(ConsumerMode::RealTime); 43 | options.set_auto_stream_reset(AutoStreamReset::Earliest); 44 | 45 | let mut consumer = streamer 46 | .create_consumer(stream.stream_keys(), options) 47 | .await?; 48 | if let Some(since) = since { 49 | consumer.seek(since).await?; 50 | } 51 | 52 | let mut sink = MessageSink::new( 53 | output.clone(), 54 | DEFAULT_BEACON_INTERVAL, 55 | DEFAULT_FILE_SIZE_LIMIT, 56 | ) 57 | .await?; 58 | 59 | let dur = Duration::from_secs(1); 60 | let mut count = 0; 61 | while let Ok(Ok(mess)) = tokio::time::timeout(dur, consumer.next()).await { 62 | if let Some(until) = &until { 63 | if &mess.timestamp() > until { 64 | break; 65 | } 66 | } 67 | sink.write(mess.to_owned_message())?; 68 | count += 1; 69 | } 70 | 71 | sink.flush().await?; 72 | log::info!("Written {count} messages to {output}"); 73 | 74 | Ok(()) 75 | } 76 | 77 | fn parse_timestamp(input: &str) -> Result { 78 | let ts = PrimitiveDateTime::parse(input, &TIMESTAMP_FORMAT)?; 79 | Ok(ts.assume_utc()) 80 | } 81 | -------------------------------------------------------------------------------- /sea-streamer-redis/src/bin/consumer.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer_redis::{AutoStreamReset, RedisConsumerOptions, RedisStreamer}; 4 | use sea_streamer_types::{ 5 | Buffer, Consumer, ConsumerMode, ConsumerOptions, Message, StreamUrl, Streamer, 6 | }; 7 | 8 | #[derive(Debug, Parser)] 9 | struct Args { 10 | #[clap( 11 | long, 12 | help = "Streamer URI with stream key, i.e. try `redis://localhost/hello`", 13 | env = "STREAM_URL" 14 | )] 15 | stream: StreamUrl, 16 | } 17 | 18 | #[tokio::main] 19 | async fn main() -> Result<()> { 20 | env_logger::init(); 21 | 22 | let Args { stream } = Args::parse(); 23 | 24 | let streamer = RedisStreamer::connect(stream.streamer(), Default::default()).await?; 25 | let mut options = RedisConsumerOptions::new(ConsumerMode::RealTime); 26 | options.set_auto_stream_reset(AutoStreamReset::Earliest); 27 | let consumer = streamer 28 | .create_consumer(stream.stream_keys(), options) 29 | .await?; 30 | 31 | loop { 32 | let mess = consumer.next().await?; 33 | println!("{}", mess.message().as_str()?); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /sea-streamer-redis/src/bin/manager-test.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer_redis::{IdRange, RedisStreamer}; 4 | use sea_streamer_types::{Buffer, Message, StreamKey, Streamer, StreamerUri}; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap( 9 | long, 10 | help = "Streamer URI", 11 | default_value = "redis://localhost", 12 | env = "STREAMER_URI" 13 | )] 14 | streamer: StreamerUri, 15 | } 16 | 17 | #[tokio::main] 18 | async fn main() -> Result<()> { 19 | env_logger::init(); 20 | 21 | let Args { streamer } = Args::parse(); 22 | 23 | let streamer = RedisStreamer::connect(streamer, Default::default()).await?; 24 | let mut manager = streamer.create_manager().await?; 25 | let streams = manager.scan("0").await?.streams; 26 | log::info!("{:#?}", streams); 27 | 28 | let messages = manager 29 | .range( 30 | StreamKey::new(&streams[0])?, 31 | IdRange::Minus, 32 | IdRange::Plus, 33 | Some(1), 34 | ) 35 | .await?; 36 | 37 | if messages.is_empty() { 38 | log::info!("No messages"); 39 | } 40 | 41 | for msg in messages { 42 | log::info!( 43 | "[{}] [{}] {}", 44 | msg.timestamp(), 45 | msg.stream_key(), 46 | msg.message().as_str().unwrap() 47 | ); 48 | } 49 | 50 | Ok(()) 51 | } 52 | -------------------------------------------------------------------------------- /sea-streamer-redis/src/bin/producer.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer_redis::RedisStreamer; 4 | use sea_streamer_types::{Producer, StreamUrl, Streamer}; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap( 9 | long, 10 | help = "Streamer URI with stream key, i.e. try `redis://localhost/hello`", 11 | env = "STREAM_URL" 12 | )] 13 | stream: StreamUrl, 14 | } 15 | 16 | #[tokio::main] 17 | async fn main() -> Result<()> { 18 | env_logger::init(); 19 | 20 | let Args { stream } = Args::parse(); 21 | 22 | let streamer = RedisStreamer::connect(stream.streamer(), Default::default()).await?; 23 | let mut producer = streamer 24 | .create_producer(stream.stream_key()?, Default::default()) 25 | .await?; 26 | 27 | for i in 0..10 { 28 | let message = format!("{{\"hello\": {i}}}"); 29 | producer.send(message)?; 30 | } 31 | 32 | producer.flush().await?; 33 | 34 | Ok(()) 35 | } 36 | -------------------------------------------------------------------------------- /sea-streamer-redis/src/bin/trim-stream.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer_redis::RedisStreamer; 4 | use sea_streamer_types::{StreamUrl, Streamer}; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap( 9 | long, 10 | help = "Streamer URI with stream key, i.e. try `redis://localhost/hello`", 11 | env = "STREAM_URL" 12 | )] 13 | stream: StreamUrl, 14 | #[clap( 15 | long, 16 | help = "Trim the stream down to this number of items (not exact)" 17 | )] 18 | maxlen: u32, 19 | } 20 | 21 | #[tokio::main] 22 | async fn main() -> Result<()> { 23 | env_logger::init(); 24 | 25 | let Args { stream, maxlen } = Args::parse(); 26 | let stream_key = stream.stream_key()?; 27 | 28 | let streamer = RedisStreamer::connect(stream.streamer(), Default::default()).await?; 29 | let producer = streamer.create_generic_producer(Default::default()).await?; 30 | 31 | match producer.trim(&stream_key, maxlen).await { 32 | Ok(trimmed) => log::info!("XTRIM {stream_key} trimmed {trimmed} entries"), 33 | Err(err) => log::error!("{err:?}"), 34 | } 35 | 36 | Ok(()) 37 | } 38 | -------------------------------------------------------------------------------- /sea-streamer-redis/src/consumer/future.rs: -------------------------------------------------------------------------------- 1 | use super::RedisConsumer; 2 | use crate::{consumer::cluster::CtrlMsg, RedisErr, RedisResult}; 3 | use flume::r#async::RecvFut; 4 | use sea_streamer_types::{ 5 | export::futures::{FutureExt, Stream}, 6 | Consumer, SharedMessage, StreamErr, 7 | }; 8 | use std::{fmt::Debug, future::Future}; 9 | 10 | pub struct NextFuture<'a> { 11 | pub(super) con: &'a RedisConsumer, 12 | pub(super) fut: RecvFut<'a, RedisResult>, 13 | pub(super) read: bool, 14 | } 15 | 16 | impl Debug for NextFuture<'_> { 17 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 18 | f.debug_struct("NextFuture").finish() 19 | } 20 | } 21 | 22 | impl Future for NextFuture<'_> { 23 | type Output = RedisResult; 24 | 25 | fn poll( 26 | mut self: std::pin::Pin<&mut Self>, 27 | cx: &mut std::task::Context<'_>, 28 | ) -> std::task::Poll { 29 | use std::task::Poll::{Pending, Ready}; 30 | if !self.read && !self.con.config.pre_fetch { 31 | self.read = true; 32 | self.con.handle.try_send(CtrlMsg::Read).ok(); 33 | } 34 | match self.fut.poll_unpin(cx) { 35 | Ready(res) => match res { 36 | Ok(Ok(msg)) => { 37 | if self.con.config.auto_ack && self.con.auto_ack(msg.header()).is_err() { 38 | return Ready(Err(StreamErr::Backend(RedisErr::ConsumerDied))); 39 | } 40 | self.read = false; 41 | Ready(Ok(msg)) 42 | } 43 | Ok(Err(err)) => Ready(Err(err)), 44 | Err(_) => Ready(Err(StreamErr::Backend(RedisErr::ConsumerDied))), 45 | }, 46 | Pending => Pending, 47 | } 48 | } 49 | } 50 | 51 | impl Drop for NextFuture<'_> { 52 | fn drop(&mut self) { 53 | if self.read { 54 | self.con.handle.try_send(CtrlMsg::Unread).ok(); 55 | } 56 | } 57 | } 58 | 59 | pub struct StreamFuture<'a> { 60 | con: &'a RedisConsumer, 61 | fut: NextFuture<'a>, 62 | } 63 | 64 | impl Debug for StreamFuture<'_> { 65 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 66 | f.debug_struct("StreamFuture").finish() 67 | } 68 | } 69 | 70 | impl<'a> StreamFuture<'a> { 71 | pub fn new(con: &'a RedisConsumer) -> Self { 72 | let fut = con.next(); 73 | Self { con, fut } 74 | } 75 | } 76 | 77 | impl Stream for StreamFuture<'_> { 78 | type Item = RedisResult; 79 | 80 | fn poll_next( 81 | mut self: std::pin::Pin<&mut Self>, 82 | cx: &mut std::task::Context<'_>, 83 | ) -> std::task::Poll> { 84 | use std::task::Poll::{Pending, Ready}; 85 | match std::pin::Pin::new(&mut self.fut).poll(cx) { 86 | Ready(res) => { 87 | self.fut = self.con.next(); 88 | Ready(Some(res)) 89 | } 90 | Pending => Pending, 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /sea-streamer-redis/src/consumer/shard.rs: -------------------------------------------------------------------------------- 1 | use super::PendingAck; 2 | use crate::{get_message_id, map_err, MessageId, RedisCluster, RedisResult, ZERO}; 3 | use redis::AsyncCommands; 4 | use sea_streamer_types::{MessageHeader, ShardId, StreamKey, Timestamp}; 5 | 6 | pub type StreamShard = (StreamKey, ShardId); 7 | 8 | #[derive(Debug)] 9 | pub struct ShardState { 10 | pub stream: StreamShard, 11 | pub key: String, 12 | pub id: Option, 13 | pub pending_ack: Vec, 14 | } 15 | 16 | impl ShardState { 17 | pub fn update(&mut self, header: &MessageHeader) { 18 | self.id = Some(get_message_id(header)); 19 | } 20 | 21 | pub fn ack_message(&mut self, id: MessageId, ts: Timestamp) { 22 | self.pending_ack.push(PendingAck(id, ts)); 23 | } 24 | 25 | pub fn key(&self) -> &StreamShard { 26 | &self.stream 27 | } 28 | } 29 | 30 | pub fn format_stream_shard((a, b): &StreamShard) -> String { 31 | format!("{a}:{b}") 32 | } 33 | 34 | pub async fn discover_shards( 35 | cluster: &mut RedisCluster, 36 | stream: StreamKey, 37 | ) -> RedisResult> { 38 | let (_node, conn) = cluster.get_any()?; 39 | let shard_keys: Vec = conn 40 | .keys(format!("{}:*", stream.name())) 41 | .await 42 | .map_err(map_err)?; 43 | 44 | Ok(if shard_keys.is_empty() { 45 | vec![ShardState { 46 | stream: (stream.clone(), ZERO), 47 | key: stream.name().to_owned(), 48 | id: None, 49 | pending_ack: Default::default(), 50 | }] 51 | } else { 52 | let mut shards: Vec<_> = shard_keys 53 | .into_iter() 54 | .filter_map(|key| match key.split_once(':') { 55 | Some((_, tail)) => { 56 | // make sure we can parse the tail 57 | if let Ok(s) = tail.parse() { 58 | Some(ShardState { 59 | stream: (stream.clone(), ShardId::new(s)), 60 | key, 61 | id: None, 62 | pending_ack: Default::default(), 63 | }) 64 | } else { 65 | log::warn!("Ignoring `{key}`"); 66 | None 67 | } 68 | } 69 | None => unreachable!(), 70 | }) 71 | .collect(); 72 | shards.sort_by_key(|s| s.stream.1); 73 | shards 74 | }) 75 | } 76 | -------------------------------------------------------------------------------- /sea-streamer-redis/src/error.rs: -------------------------------------------------------------------------------- 1 | use redis::{ErrorKind, RedisError}; 2 | use sea_streamer_types::{StreamErr, StreamResult}; 3 | use thiserror::Error; 4 | 5 | #[derive(Error, Debug, Clone)] 6 | /// Different types of Redis errors. 7 | pub enum RedisErr { 8 | #[error("Failed to parse message ID: {0}")] 9 | MessageId(String), 10 | #[error("Failed to parse StreamReadReply: {0:?}")] 11 | StreamReadReply(String), 12 | #[error("The Producer task died")] 13 | ProducerDied, 14 | #[error("Consumer died with unrecoverable error. Check the log for details.")] 15 | ConsumerDied, 16 | #[error("The server generated an invalid response: {0}")] 17 | ResponseError(String), 18 | #[error("The authentication with the server failed: {0}")] 19 | AuthenticationFailed(String), 20 | #[error("Operation failed because of a type mismatch: {0}")] 21 | TypeError(String), 22 | #[error("A script execution was aborted: {0}")] 23 | ExecAbortError(String), 24 | #[error("The server cannot response because it's loading a dump: {0}")] 25 | BusyLoadingError(String), 26 | #[error("A script that was requested does not actually exist: {0}")] 27 | NoScriptError(String), 28 | #[error("An error that was caused because the parameter to the client were wrong: {0}")] 29 | InvalidClientConfig(String), 30 | #[error("Raised if a key moved to a different node: {0}")] 31 | Moved(String), 32 | #[error("Raised if a key moved to a different node but we need to ask: {0}")] 33 | Ask(String), 34 | #[error("Raised if a request needs to be retried: {0}")] 35 | TryAgain(String), 36 | #[error("Raised if a redis cluster is down: {0}")] 37 | ClusterDown(String), 38 | #[error("A request spans multiple slots: {0}")] 39 | CrossSlot(String), 40 | #[error("A cluster master is unavailable: {0}")] 41 | MasterDown(String), 42 | #[error("IO error: {0}")] 43 | IoError(String), 44 | #[error("An error raised that was identified on the client before execution: {0}")] 45 | ClientError(String), 46 | #[error("Extension error: {0}")] 47 | ExtensionError(String), 48 | #[error("Attempt to write to a read-only server: {0}")] 49 | ReadOnly(String), 50 | #[error("Unknown error: {0}")] 51 | Unknown(String), 52 | } 53 | 54 | /// A type alias for convenience. 55 | pub type RedisResult = StreamResult; 56 | 57 | pub(crate) fn map_err(err: RedisError) -> StreamErr { 58 | let e = format!("{err}"); 59 | StreamErr::Backend(match err.kind() { 60 | ErrorKind::ResponseError => RedisErr::ResponseError(e), 61 | ErrorKind::AuthenticationFailed => RedisErr::AuthenticationFailed(e), 62 | ErrorKind::TypeError => RedisErr::TypeError(e), 63 | ErrorKind::ExecAbortError => RedisErr::ExecAbortError(e), 64 | ErrorKind::BusyLoadingError => RedisErr::BusyLoadingError(e), 65 | ErrorKind::NoScriptError => RedisErr::NoScriptError(e), 66 | ErrorKind::InvalidClientConfig => RedisErr::InvalidClientConfig(e), 67 | ErrorKind::Moved => RedisErr::Moved(e), 68 | ErrorKind::Ask => RedisErr::Ask(e), 69 | ErrorKind::TryAgain => RedisErr::TryAgain(e), 70 | ErrorKind::ClusterDown => RedisErr::ClusterDown(e), 71 | ErrorKind::CrossSlot => RedisErr::CrossSlot(e), 72 | ErrorKind::MasterDown => RedisErr::MasterDown(e), 73 | ErrorKind::IoError => RedisErr::IoError(e), 74 | ErrorKind::ClientError => RedisErr::ClientError(e), 75 | ErrorKind::ExtensionError => RedisErr::ExtensionError(e), 76 | ErrorKind::ReadOnly => RedisErr::ReadOnly(e), 77 | _ => RedisErr::Unknown(e), 78 | }) 79 | } 80 | -------------------------------------------------------------------------------- /sea-streamer-redis/src/host.rs: -------------------------------------------------------------------------------- 1 | //! This module is shared between `sea-streamer-kafka` and `sea-streamer-redis` 2 | use mac_address::get_mac_address; 3 | use std::{ 4 | fs::File, 5 | io::{BufRead, BufReader}, 6 | }; 7 | 8 | lazy_static::lazy_static! { 9 | static ref HOST_ID: String = init(); 10 | } 11 | 12 | const LEN: usize = 12; 13 | 14 | fn init() -> String { 15 | if let Ok(host_id) = std::env::var("HOST_ID") { 16 | return host_id; 17 | } 18 | if let Ok(file) = File::open("/proc/self/cgroup") { 19 | // check whether this is a docker container 20 | let last = BufReader::new(file) 21 | .lines() 22 | .last() 23 | .expect("Empty file?") 24 | .expect("IO Error"); 25 | if let Some((_, remaining)) = last.split_once("0::/docker/") { 26 | if remaining.is_empty() { 27 | panic!("Failed to get docker container ID"); 28 | } 29 | let (container_id, _) = remaining.split_at(LEN); 30 | return container_id.to_owned(); 31 | } 32 | } 33 | let mac = get_mac_address() 34 | .expect("Failed to get MAC address") 35 | .expect("There is no MAC address on this host"); 36 | let mac = mac.to_string().replace(':', ""); 37 | let (mac, _) = mac.split_at(LEN); 38 | mac.to_owned() 39 | } 40 | 41 | /// Get the host id. Inside Docker, it's the container ID. Otherwise it's the MAC address. 42 | /// You can override it via the `HOST_ID` env var. 43 | pub fn host_id() -> &'static str { 44 | &HOST_ID 45 | } 46 | -------------------------------------------------------------------------------- /sea-streamer-redis/tests/consumer-group.rs: -------------------------------------------------------------------------------- 1 | mod util; 2 | use util::*; 3 | 4 | static INIT: std::sync::Once = std::sync::Once::new(); 5 | 6 | // cargo test --test consumer-group --features=test,runtime-tokio -- --nocapture 7 | // cargo test --test consumer-group --no-default-features --features=test,runtime-async-std -- --nocapture 8 | #[cfg(feature = "test")] 9 | #[cfg_attr(feature = "runtime-tokio", tokio::test)] 10 | #[cfg_attr(feature = "runtime-async-std", async_std::test)] 11 | async fn consumer_group() -> anyhow::Result<()> { 12 | use sea_streamer_redis::{ 13 | AutoStreamReset, RedisConnectOptions, RedisConsumerOptions, RedisStreamer, 14 | }; 15 | use sea_streamer_types::{ 16 | Consumer, ConsumerMode, ConsumerOptions, Producer, StreamKey, Streamer, Timestamp, 17 | }; 18 | 19 | const TEST: &str = "group-1"; 20 | INIT.call_once(env_logger::init); 21 | 22 | test(false).await?; 23 | test(true).await?; 24 | 25 | async fn test(mkstream: bool) -> anyhow::Result<()> { 26 | println!("mkstream = {mkstream:?} ..."); 27 | 28 | let options = RedisConnectOptions::default(); 29 | let streamer = RedisStreamer::connect( 30 | std::env::var("BROKERS_URL") 31 | .unwrap_or_else(|_| "redis://localhost".to_owned()) 32 | .parse() 33 | .unwrap(), 34 | options, 35 | ) 36 | .await?; 37 | let now = Timestamp::now_utc(); 38 | let stream = StreamKey::new(format!( 39 | "{}-{}", 40 | TEST, 41 | now.unix_timestamp_nanos() / 1_000_000 42 | ))?; 43 | println!("stream = {stream}"); 44 | 45 | let mut producer = streamer 46 | .create_producer(stream.clone(), Default::default()) 47 | .await?; 48 | 49 | let mut options = RedisConsumerOptions::new(ConsumerMode::LoadBalanced); 50 | options.set_mkstream(mkstream); 51 | options.set_auto_stream_reset(AutoStreamReset::Earliest); 52 | 53 | let mut consumer = streamer 54 | .create_consumer(&[stream.clone()], options.clone()) 55 | .await?; 56 | 57 | if !mkstream { 58 | assert!(consumer.next().await.is_err()); 59 | } 60 | 61 | let mut last = 0; 62 | for i in 0..5 { 63 | let message = format!("{i}"); 64 | let receipt = producer.send(message)?.await?; 65 | assert!(*receipt.sequence() > last); 66 | last = *receipt.sequence(); 67 | } 68 | producer.flush().await?; 69 | 70 | if mkstream { 71 | let seq = consume(&mut consumer, 5).await?; 72 | assert_eq!(seq, [0, 1, 2, 3, 4]); 73 | } 74 | 75 | Ok(()) 76 | } 77 | 78 | Ok(()) 79 | } 80 | -------------------------------------------------------------------------------- /sea-streamer-redis/tests/sharding.rs: -------------------------------------------------------------------------------- 1 | mod util; 2 | use util::*; 3 | 4 | // cargo test --test sharding --features=test,runtime-tokio -- --nocapture 5 | // cargo test --test sharding --features=test,runtime-async-std -- --nocapture 6 | #[cfg(feature = "test")] 7 | #[cfg_attr(feature = "runtime-tokio", tokio::test)] 8 | #[cfg_attr(feature = "runtime-async-std", async_std::test)] 9 | async fn main() -> anyhow::Result<()> { 10 | use sea_streamer_redis::{ 11 | AutoStreamReset, RedisConnectOptions, RedisConsumerOptions, RedisProducerOptions, 12 | RedisStreamer, RoundRobinSharder, 13 | }; 14 | use sea_streamer_runtime::sleep; 15 | use sea_streamer_types::{ 16 | ConsumerMode, ConsumerOptions, Producer, StreamKey, Streamer, Timestamp, 17 | }; 18 | use std::time::Duration; 19 | 20 | const TEST: &str = "sharding"; 21 | const SHARDS: u32 = 3; 22 | env_logger::init(); 23 | 24 | test(ConsumerMode::RealTime).await?; 25 | test(ConsumerMode::Resumable).await?; 26 | 27 | async fn test(mode: ConsumerMode) -> anyhow::Result<()> { 28 | println!("ConsumerMode = {mode:?} ..."); 29 | 30 | let options = RedisConnectOptions::default(); 31 | let streamer = RedisStreamer::connect( 32 | std::env::var("BROKERS_URL") 33 | .unwrap_or_else(|_| "redis://localhost".to_owned()) 34 | .parse() 35 | .unwrap(), 36 | options, 37 | ) 38 | .await?; 39 | 40 | let now = Timestamp::now_utc(); 41 | let stream = StreamKey::new(format!( 42 | "{}-{}", 43 | TEST, 44 | now.unix_timestamp_nanos() / 1_000_000 45 | ))?; 46 | 47 | let mut options = RedisProducerOptions::default(); 48 | options.set_sharder(RoundRobinSharder::new(SHARDS)); 49 | let mut producer = streamer.create_producer(stream.clone(), options).await?; 50 | 51 | let mut sequence = 0; 52 | for i in 0..10 { 53 | let message = format!("{i}"); 54 | let receipt = producer.send(message)?.await?; 55 | assert_eq!(receipt.stream_key(), &stream); 56 | // should always increase 57 | assert!(receipt.sequence() > &sequence); 58 | sequence = *receipt.sequence(); 59 | assert_eq!(receipt.shard_id().id(), i % SHARDS as u64); 60 | sleep(Duration::from_millis(1)).await; 61 | } 62 | 63 | println!("Stream to shards ... ok"); 64 | 65 | let mut options = RedisConsumerOptions::new(mode); 66 | options.set_auto_stream_reset(AutoStreamReset::Earliest); 67 | 68 | let mut full = streamer.create_consumer(&[stream.clone()], options).await?; 69 | 70 | let mut seq = consume(&mut full, 10).await?; 71 | seq.sort(); 72 | assert_eq!(seq, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); 73 | 74 | println!("Stream from shards ... ok"); 75 | 76 | for i in 10..20 { 77 | let message = format!("{i}"); 78 | producer.send(message)?; 79 | } 80 | 81 | producer.flush().await?; 82 | let mut seq = consume(&mut full, 10).await?; 83 | seq.sort(); 84 | assert_eq!(seq, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]); 85 | 86 | println!("Stream more shards ... ok"); 87 | 88 | producer.end().await?; 89 | full.end().await?; 90 | 91 | println!("End test case."); 92 | Ok(()) 93 | } 94 | 95 | Ok(()) 96 | } 97 | -------------------------------------------------------------------------------- /sea-streamer-redis/tests/util.rs: -------------------------------------------------------------------------------- 1 | use sea_streamer_redis::{RedisConsumer, RedisResult}; 2 | use sea_streamer_runtime::timeout; 3 | use sea_streamer_types::{Buffer, Consumer, Message}; 4 | use std::time::Duration; 5 | 6 | #[allow(dead_code)] 7 | pub async fn consume(consumer: &mut RedisConsumer, max: usize) -> RedisResult> { 8 | consume_impl(consumer, max, false).await 9 | } 10 | 11 | #[allow(dead_code)] 12 | pub async fn consume_and_ack(consumer: &mut RedisConsumer, max: usize) -> RedisResult> { 13 | consume_impl(consumer, max, true).await 14 | } 15 | 16 | async fn consume_impl( 17 | consumer: &mut RedisConsumer, 18 | max: usize, 19 | ack: bool, 20 | ) -> RedisResult> { 21 | let mut numbers = Vec::new(); 22 | for _ in 0..max { 23 | match timeout(Duration::from_secs(60), consumer.next()).await { 24 | Ok(mess) => { 25 | let mess = mess?; 26 | if ack { 27 | consumer.ack(&mess)?; 28 | } 29 | numbers.push(mess.message().as_str().unwrap().parse::().unwrap()); 30 | } 31 | Err(_) => panic!("Timed out when streaming up to {numbers:?}"), 32 | } 33 | } 34 | Ok(numbers) 35 | } 36 | -------------------------------------------------------------------------------- /sea-streamer-runtime/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-runtime" 3 | version = "0.5.0" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "🌊 SeaStreamer async runtime abstraction" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/sea-streamer-runtime" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | keywords = ["async", "async-std", "tokio"] 12 | rust-version = "1.60" 13 | 14 | [package.metadata.docs.rs] 15 | features = [] 16 | rustdoc-args = ["--cfg", "docsrs"] 17 | 18 | [dependencies] 19 | async-std = { version = "1.12", optional = true } 20 | futures = { version = "0.3", default-features = false } 21 | tokio = { version = "1.10", optional = true, features = ["time", "sync", "rt"] } 22 | 23 | [dev-dependencies] 24 | 25 | [features] 26 | runtime-async-std = ["async-std"] 27 | runtime-tokio = ["tokio"] 28 | file = ["tokio?/fs", "tokio?/io-util"] -------------------------------------------------------------------------------- /sea-streamer-runtime/README.md: -------------------------------------------------------------------------------- 1 | ### `sea-streamer-runtime`: Async runtime abstraction 2 | 3 | This crate provides a small set of functions aligning the type signatures between `async-std` and `tokio`, 4 | so that you can build applications generic to both runtimes. 5 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/file/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "runtime-tokio")] 2 | pub use tokio::{ 3 | fs::{File, OpenOptions}, 4 | io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, 5 | }; 6 | 7 | #[cfg(feature = "runtime-async-std")] 8 | pub use async_std::{ 9 | fs::{File, OpenOptions}, 10 | io::{prelude::SeekExt as AsyncSeekExt, ReadExt as AsyncReadExt, WriteExt as AsyncWriteExt}, 11 | }; 12 | 13 | #[cfg(not(any(feature = "runtime-tokio", feature = "runtime-async-std")))] 14 | mod no_rt_file; 15 | #[cfg(not(any(feature = "runtime-tokio", feature = "runtime-async-std")))] 16 | pub use no_rt_file::*; 17 | 18 | pub use std::io::SeekFrom; 19 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/file/no_rt_file.rs: -------------------------------------------------------------------------------- 1 | use super::SeekFrom; 2 | use futures::future::{ready, Future, Ready}; 3 | use std::{ 4 | fs::Metadata, 5 | io::{Error as IoError, ErrorKind}, 6 | path::Path, 7 | }; 8 | 9 | pub struct File; 10 | 11 | pub struct OpenOptions; 12 | 13 | pub trait AsyncReadExt { 14 | type Future: Future>; 15 | 16 | fn read(&mut self, _: &mut [u8]) -> Self::Future; 17 | } 18 | 19 | pub trait AsyncWriteExt { 20 | type Future: Future>; 21 | 22 | fn write_all(&mut self, _: &[u8]) -> Self::Future; 23 | } 24 | 25 | pub trait AsyncSeekExt { 26 | type Future: Future>; 27 | 28 | fn seek(&mut self, _: SeekFrom) -> Self::Future; 29 | } 30 | 31 | impl File { 32 | pub async fn open>(_: P) -> Result { 33 | unimplemented!("Please enable a runtime") 34 | } 35 | 36 | pub async fn metadata(&self) -> Result { 37 | unimplemented!("Please enable a runtime") 38 | } 39 | 40 | pub async fn flush(&self) -> Result<(), IoError> { 41 | unimplemented!("Please enable a runtime") 42 | } 43 | 44 | pub async fn sync_all(&self) -> Result<(), IoError> { 45 | unimplemented!("Please enable a runtime") 46 | } 47 | } 48 | 49 | impl AsyncReadExt for File { 50 | type Future = Ready>; 51 | 52 | fn read(&mut self, _: &mut [u8]) -> Self::Future { 53 | ready(Err(IoError::new( 54 | ErrorKind::Other, 55 | "Please enable a runtime", 56 | ))) 57 | } 58 | } 59 | 60 | impl AsyncWriteExt for File { 61 | type Future = Ready>; 62 | 63 | fn write_all(&mut self, _: &[u8]) -> Self::Future { 64 | ready(Err(IoError::new( 65 | ErrorKind::Other, 66 | "Please enable a runtime", 67 | ))) 68 | } 69 | } 70 | 71 | impl AsyncSeekExt for File { 72 | type Future = Ready>; 73 | 74 | fn seek(&mut self, _: SeekFrom) -> Self::Future { 75 | ready(Err(IoError::new( 76 | ErrorKind::Other, 77 | "Please enable a runtime", 78 | ))) 79 | } 80 | } 81 | 82 | impl Default for OpenOptions { 83 | fn default() -> Self { 84 | Self::new() 85 | } 86 | } 87 | 88 | impl OpenOptions { 89 | pub fn new() -> Self { 90 | Self 91 | } 92 | 93 | pub fn read(&mut self, _: bool) -> &mut OpenOptions { 94 | self 95 | } 96 | 97 | pub fn write(&mut self, _: bool) -> &mut OpenOptions { 98 | self 99 | } 100 | 101 | pub fn truncate(&mut self, _: bool) -> &mut OpenOptions { 102 | self 103 | } 104 | 105 | pub fn append(&mut self, _: bool) -> &mut OpenOptions { 106 | self 107 | } 108 | 109 | pub fn create(&mut self, _: bool) -> &mut OpenOptions { 110 | self 111 | } 112 | 113 | pub fn create_new(&mut self, _: bool) -> &mut OpenOptions { 114 | self 115 | } 116 | 117 | pub async fn open(&self, _: impl AsRef) -> Result { 118 | Err(IoError::new(ErrorKind::Other, "Please enable a runtime")) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! ### `sea-streamer-runtime`: Async runtime abstraction 2 | //! 3 | //! This crate provides a small set of functions aligning the type signatures between `async-std` and `tokio`, 4 | //! so that you can build applications generic to both runtimes. 5 | 6 | #[cfg(all(feature = "runtime-async-std", feature = "runtime-tokio"))] 7 | compile_error!("'runtime-async-std' and 'runtime-tokio' cannot be enabled at the same time"); 8 | 9 | #[cfg(feature = "file")] 10 | pub mod file; 11 | mod mutex; 12 | mod sleep; 13 | mod task; 14 | mod timeout; 15 | 16 | pub use mutex::*; 17 | pub use sleep::*; 18 | pub use task::*; 19 | pub use timeout::*; 20 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/mutex.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "runtime-tokio")] 2 | pub use tokio::sync::Mutex as AsyncMutex; 3 | 4 | #[cfg(feature = "runtime-async-std")] 5 | pub use async_std::sync::Mutex as AsyncMutex; 6 | 7 | #[cfg(not(any(feature = "runtime-tokio", feature = "runtime-async-std")))] 8 | mod no_rt_mutex { 9 | use std::ops::{Deref, DerefMut}; 10 | 11 | pub struct AsyncMutex { 12 | m: std::marker::PhantomData, 13 | } 14 | 15 | impl AsyncMutex { 16 | pub fn new(_: T) -> Self { 17 | Self { 18 | m: Default::default(), 19 | } 20 | } 21 | 22 | pub async fn lock(&self) -> Self { 23 | Self { 24 | m: Default::default(), 25 | } 26 | } 27 | } 28 | 29 | impl Deref for AsyncMutex { 30 | type Target = T; 31 | 32 | fn deref(&self) -> &Self::Target { 33 | unimplemented!("Please enable a runtime") 34 | } 35 | } 36 | 37 | impl DerefMut for AsyncMutex { 38 | fn deref_mut(&mut self) -> &mut Self::Target { 39 | unimplemented!("Please enable a runtime") 40 | } 41 | } 42 | } 43 | 44 | #[cfg(not(any(feature = "runtime-tokio", feature = "runtime-async-std")))] 45 | pub use no_rt_mutex::*; 46 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/sleep.rs: -------------------------------------------------------------------------------- 1 | #![allow(unreachable_code)] 2 | use std::time::Duration; 3 | 4 | #[inline] 5 | pub async fn sleep(_s: Duration) { 6 | #[cfg(feature = "runtime-async-std")] 7 | return async_std::task::sleep(_s).await; 8 | 9 | #[cfg(feature = "runtime-tokio")] 10 | return tokio::time::sleep(_s).await; 11 | 12 | panic!("Please enable a runtime"); 13 | } 14 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/task/async_std_task.rs: -------------------------------------------------------------------------------- 1 | use futures::future::{Future, FutureExt}; 2 | 3 | pub struct TaskHandle(async_std::task::JoinHandle); 4 | 5 | #[derive(Debug)] 6 | pub struct JoinError; 7 | 8 | pub fn spawn_task(future: F) -> TaskHandle 9 | where 10 | F: Future + Send + 'static, 11 | T: Send + 'static, 12 | { 13 | TaskHandle(async_std::task::spawn(future)) 14 | } 15 | 16 | pub fn spawn_blocking(future: F) -> TaskHandle 17 | where 18 | F: FnOnce() -> T + Send + 'static, 19 | T: Send + 'static, 20 | { 21 | TaskHandle(async_std::task::spawn_blocking(future)) 22 | } 23 | 24 | impl std::fmt::Display for JoinError { 25 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 26 | write!(f, "JoinError") 27 | } 28 | } 29 | 30 | impl std::error::Error for JoinError {} 31 | 32 | impl Future for TaskHandle { 33 | type Output = Result; 34 | 35 | fn poll( 36 | mut self: std::pin::Pin<&mut Self>, 37 | cx: &mut std::task::Context<'_>, 38 | ) -> std::task::Poll { 39 | match self.0.poll_unpin(cx) { 40 | std::task::Poll::Ready(res) => std::task::Poll::Ready(Ok(res)), 41 | std::task::Poll::Pending => std::task::Poll::Pending, 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/task/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "runtime-tokio")] 2 | mod tokio_task; 3 | 4 | #[cfg(feature = "runtime-tokio")] 5 | pub use tokio_task::*; 6 | 7 | #[cfg(feature = "runtime-async-std")] 8 | mod async_std_task; 9 | 10 | #[cfg(feature = "runtime-async-std")] 11 | pub use async_std_task::*; 12 | 13 | #[cfg(not(any(feature = "runtime-tokio", feature = "runtime-async-std")))] 14 | mod no_rt_task; 15 | 16 | #[cfg(not(any(feature = "runtime-tokio", feature = "runtime-async-std")))] 17 | pub use no_rt_task::*; 18 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/task/no_rt_task.rs: -------------------------------------------------------------------------------- 1 | use futures::future::{ready, Ready}; 2 | use std::future::Future; 3 | 4 | #[derive(Debug)] 5 | pub struct JoinError; 6 | 7 | pub type TaskHandle = Ready>; 8 | 9 | pub fn spawn_task(_: F) -> TaskHandle 10 | where 11 | F: Future + Send + 'static, 12 | T: Send + 'static, 13 | { 14 | ready(Err(JoinError)) 15 | } 16 | 17 | pub fn spawn_blocking(_: F) -> TaskHandle 18 | where 19 | F: FnOnce() -> T + Send + 'static, 20 | T: Send + 'static, 21 | { 22 | ready(Err(JoinError)) 23 | } 24 | 25 | impl std::fmt::Display for JoinError { 26 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 27 | write!(f, "Please enable a runtime") 28 | } 29 | } 30 | 31 | impl std::error::Error for JoinError {} 32 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/task/tokio_task.rs: -------------------------------------------------------------------------------- 1 | use futures::future::Future; 2 | 3 | pub use tokio::task::{spawn_blocking, JoinError, JoinHandle as TaskHandle}; 4 | 5 | pub fn spawn_task(future: F) -> TaskHandle 6 | where 7 | F: Future + Send + 'static, 8 | T: Send + 'static, 9 | { 10 | tokio::task::spawn(future) 11 | } 12 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/timeout/async_std_timeout.rs: -------------------------------------------------------------------------------- 1 | pub use async_std::future::{timeout, TimeoutError}; 2 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/timeout/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "runtime-tokio")] 2 | mod tokio_timeout; 3 | 4 | #[cfg(feature = "runtime-tokio")] 5 | pub use tokio_timeout::*; 6 | 7 | #[cfg(feature = "runtime-async-std")] 8 | mod async_std_timeout; 9 | 10 | #[cfg(feature = "runtime-async-std")] 11 | pub use async_std_timeout::*; 12 | 13 | #[cfg(not(any(feature = "runtime-tokio", feature = "runtime-async-std")))] 14 | mod no_rt_timeout; 15 | 16 | #[cfg(not(any(feature = "runtime-tokio", feature = "runtime-async-std")))] 17 | pub use no_rt_timeout::*; 18 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/timeout/no_rt_timeout.rs: -------------------------------------------------------------------------------- 1 | use std::{future::Future, time::Duration}; 2 | 3 | #[derive(Debug)] 4 | pub struct TimeoutError; 5 | 6 | pub async fn timeout(_: Duration, _f: F) -> Result 7 | where 8 | F: Future, 9 | { 10 | Err(TimeoutError) 11 | } 12 | 13 | impl std::fmt::Display for TimeoutError { 14 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 15 | write!(f, "Please enable a runtime") 16 | } 17 | } 18 | 19 | impl std::error::Error for TimeoutError {} 20 | -------------------------------------------------------------------------------- /sea-streamer-runtime/src/timeout/tokio_timeout.rs: -------------------------------------------------------------------------------- 1 | use futures::future::Future; 2 | use std::time::Duration; 3 | pub use tokio::time::error::Elapsed as TimeoutError; 4 | 5 | pub async fn timeout(dur: Duration, f: F) -> Result 6 | where 7 | F: Future, 8 | { 9 | tokio::time::timeout(dur, f).await 10 | } 11 | -------------------------------------------------------------------------------- /sea-streamer-socket/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-socket" 3 | version = "0.5.2" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "🌊 SeaStreamer backend-agnostic Socket API" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/sea-streamer-socket" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | keywords = ["async", "stream", "kafka", "stream-processing"] 12 | rust-version = "1.60" 13 | 14 | [package.metadata.docs.rs] 15 | features = ["default"] 16 | rustdoc-args = ["--cfg", "docsrs"] 17 | 18 | [dependencies] 19 | anyhow = { version = "1", optional = true } 20 | env_logger = { version = "0.9", optional = true } 21 | sea-streamer-kafka = { version = "0.5", path = "../sea-streamer-kafka", optional = true } 22 | sea-streamer-redis = { version = "0.5", path = "../sea-streamer-redis", optional = true } 23 | sea-streamer-stdio = { version = "0.5", path = "../sea-streamer-stdio", optional = true } 24 | sea-streamer-file = { version = "0.5", path = "../sea-streamer-file", optional = true } 25 | sea-streamer-types = { version = "0.5", path = "../sea-streamer-types" } 26 | clap = { version = "4.5", features = ["derive"], optional = true } 27 | thiserror = { version = "1", default-features = false } 28 | tokio = { version = "1.10.0", optional = true } 29 | 30 | [features] 31 | default = ["backend-stdio"] # sadly cannot compile without one backend 32 | executables = ["anyhow", "env_logger", "clap", "tokio/full", "runtime-tokio"] 33 | runtime-async-std = ["sea-streamer-kafka?/runtime-async-std", "sea-streamer-redis?/runtime-async-std", "sea-streamer-stdio?/runtime-async-std", "sea-streamer-file?/runtime-async-std"] 34 | runtime-tokio = ["sea-streamer-kafka?/runtime-tokio", "sea-streamer-redis?/runtime-tokio", "sea-streamer-stdio?/runtime-tokio", "sea-streamer-file?/runtime-tokio"] 35 | backend-kafka = ["sea-streamer-kafka"] 36 | backend-redis = ["sea-streamer-redis"] 37 | backend-stdio = ["sea-streamer-stdio"] 38 | backend-file = ["sea-streamer-file"] 39 | 40 | [[bin]] 41 | name = "relay" 42 | path = "src/bin/relay.rs" 43 | required-features = ["executables"] 44 | -------------------------------------------------------------------------------- /sea-streamer-socket/README.md: -------------------------------------------------------------------------------- 1 | ### `sea-streamer-socket`: Backend-agnostic Socket API 2 | 3 | Akin to how SeaORM allows you to build applications for different databases, SeaStreamer allows you to build 4 | stream processors for different streaming servers. 5 | 6 | While the `sea-streamer-types` crate provides a nice trait-based abstraction, this crates provides a concrete-type API, 7 | so that your program can stream from/to any SeaStreamer backend selected by the user *on runtime*. 8 | 9 | This allows you to do neat things, like generating data locally and then stream them to Redis / Kafka. Or in the other 10 | way, sink data from server to work on them locally. All _without recompiling_ the stream processor. 11 | 12 | If you only ever work with one backend, feel free to depend on `sea-streamer-redis` / `sea-streamer-kafka` directly. 13 | 14 | A small number of cli programs are provided for demonstration. Let's set them up first: 15 | 16 | ```shell 17 | # The `clock` program generate messages in the form of `{ "tick": N }` 18 | alias clock='cargo run --package sea-streamer-stdio --features=executables --bin clock' 19 | # The `relay` program redirect messages from `input` to `output` 20 | alias relay='cargo run --package sea-streamer-socket --features=executables,backend-kafka,backend-redis --bin relay' 21 | ``` 22 | 23 | Here is how to stream from Stdio ➡️ Redis / Kafka. We generate messages using `clock` and then pipe it to `relay`, 24 | which then streams to Redis / Kafka: 25 | 26 | ```shell 27 | # Stdio -> Redis 28 | clock -- --stream clock --interval 1s | \ 29 | relay -- --input stdio:///clock --output redis://localhost:6379/clock 30 | # Stdio -> Kafka 31 | clock -- --stream clock --interval 1s | \ 32 | relay -- --input stdio:///clock --output kafka://localhost:9092/clock 33 | ``` 34 | 35 | Here is how to stream between Redis ↔️ Kafka: 36 | 37 | ```shell 38 | # Redis -> Kafka 39 | relay -- --input redis://localhost:6379/clock --output kafka://localhost:9092/clock 40 | # Kafka -> Redis 41 | relay -- --input kafka://localhost:9092/clock --output redis://localhost:6379/clock 42 | ``` 43 | 44 | Here is how to *replay* the stream from Kafka / Redis: 45 | 46 | ```shell 47 | relay -- --input redis://localhost:6379/clock --output stdio:///clock --offset start 48 | relay -- --input kafka://localhost:9092/clock --output stdio:///clock --offset start 49 | ``` 50 | -------------------------------------------------------------------------------- /sea-streamer-socket/src/backend.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, Copy, Clone, PartialEq, Eq)] 2 | /// `sea-streamer-socket` Enum for identifying the underlying backend. 3 | pub enum Backend { 4 | #[cfg(feature = "backend-kafka")] 5 | Kafka, 6 | #[cfg(feature = "backend-redis")] 7 | Redis, 8 | #[cfg(feature = "backend-stdio")] 9 | Stdio, 10 | #[cfg(feature = "backend-file")] 11 | File, 12 | } 13 | 14 | /// `sea-streamer-socket` methods shared by `Sea*` types. 15 | pub trait SeaStreamerBackend { 16 | #[cfg(feature = "backend-kafka")] 17 | type Kafka; 18 | #[cfg(feature = "backend-redis")] 19 | type Redis; 20 | #[cfg(feature = "backend-stdio")] 21 | type Stdio; 22 | #[cfg(feature = "backend-file")] 23 | type File; 24 | 25 | /// Identifies the underlying backend 26 | fn backend(&self) -> Backend; 27 | 28 | #[cfg(feature = "backend-kafka")] 29 | /// Get the concrete type for the Kafka backend. None if it's another Backend 30 | fn get_kafka(&mut self) -> Option<&mut Self::Kafka>; 31 | 32 | #[cfg(feature = "backend-redis")] 33 | /// Get the concrete type for the Redis backend. None if it's another Backend 34 | fn get_redis(&mut self) -> Option<&mut Self::Redis>; 35 | 36 | #[cfg(feature = "backend-stdio")] 37 | /// Get the concrete type for the Stdio backend. None if it's another Backend 38 | fn get_stdio(&mut self) -> Option<&mut Self::Stdio>; 39 | 40 | #[cfg(feature = "backend-file")] 41 | /// Get the concrete type for the File backend. None if it's another Backend 42 | fn get_file(&mut self) -> Option<&mut Self::File>; 43 | } 44 | -------------------------------------------------------------------------------- /sea-streamer-socket/src/bin/relay.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{bail, Result}; 2 | use clap::Parser; 3 | use sea_streamer_socket::{SeaConsumerOptions, SeaStreamer}; 4 | use sea_streamer_types::{ 5 | Consumer, ConsumerMode, ConsumerOptions, Message, Producer, StreamUrl, Streamer, 6 | }; 7 | use std::str::FromStr; 8 | 9 | #[cfg(feature = "backend-kafka")] 10 | use sea_streamer_kafka::AutoOffsetReset; 11 | #[cfg(feature = "backend-redis")] 12 | use sea_streamer_redis::AutoStreamReset; 13 | 14 | #[derive(Debug, Parser)] 15 | struct Args { 16 | #[clap( 17 | long, 18 | help = "Streamer Source Uri, i.e. try `kafka://localhost:9092/stream_key`" 19 | )] 20 | input: StreamUrl, 21 | #[clap(long, help = "Streamer Sink Uri, i.e. try `stdio:///stream_key`")] 22 | output: StreamUrl, 23 | #[clap(long, help = "Stream from `start` or `end`", default_value = "end")] 24 | offset: Offset, 25 | } 26 | 27 | #[derive(Debug)] 28 | enum Offset { 29 | Start, 30 | End, 31 | } 32 | 33 | impl FromStr for Offset { 34 | type Err = &'static str; 35 | 36 | fn from_str(s: &str) -> std::result::Result { 37 | match s { 38 | "start" => Ok(Self::Start), 39 | "end" => Ok(Self::End), 40 | _ => Err("unknown Offset"), 41 | } 42 | } 43 | } 44 | 45 | #[tokio::main] 46 | async fn main() -> Result<()> { 47 | env_logger::init(); 48 | 49 | let Args { 50 | input, 51 | output, 52 | offset, 53 | } = Args::parse(); 54 | 55 | if input == output && input.streamer().protocol() != Some("stdio") { 56 | bail!("input == output !!!"); 57 | } 58 | 59 | let source = SeaStreamer::connect(input.streamer(), Default::default()).await?; 60 | let mut options = SeaConsumerOptions::new(ConsumerMode::RealTime); 61 | #[cfg(feature = "backend-kafka")] 62 | options.set_kafka_consumer_options(|options| { 63 | options.set_auto_offset_reset(match offset { 64 | Offset::Start => AutoOffsetReset::Earliest, 65 | Offset::End => AutoOffsetReset::Latest, 66 | }); 67 | }); 68 | #[cfg(feature = "backend-redis")] 69 | options.set_redis_consumer_options(|options| { 70 | options.set_auto_stream_reset(match offset { 71 | Offset::Start => AutoStreamReset::Earliest, 72 | Offset::End => AutoStreamReset::Latest, 73 | }); 74 | }); 75 | let consumer = source.create_consumer(input.stream_keys(), options).await?; 76 | 77 | let sink = SeaStreamer::connect(output.streamer(), Default::default()).await?; 78 | let producer = sink 79 | .create_producer(output.stream_key()?, Default::default()) 80 | .await?; 81 | 82 | loop { 83 | let mess = consumer.next().await?; 84 | producer.send(mess.message())?; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /sea-streamer-socket/src/connect_options.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "backend-file")] 2 | use sea_streamer_file::FileConnectOptions; 3 | #[cfg(feature = "backend-kafka")] 4 | use sea_streamer_kafka::KafkaConnectOptions; 5 | #[cfg(feature = "backend-redis")] 6 | use sea_streamer_redis::RedisConnectOptions; 7 | #[cfg(feature = "backend-stdio")] 8 | use sea_streamer_stdio::StdioConnectOptions; 9 | 10 | use crate::{map_err, BackendErr, SeaResult}; 11 | use sea_streamer_types::ConnectOptions; 12 | use std::time::Duration; 13 | 14 | #[derive(Debug, Default, Clone)] 15 | /// `sea-streamer-socket` concrete type of ConnectOptions. 16 | pub struct SeaConnectOptions { 17 | #[cfg(feature = "backend-kafka")] 18 | kafka: KafkaConnectOptions, 19 | #[cfg(feature = "backend-redis")] 20 | redis: RedisConnectOptions, 21 | #[cfg(feature = "backend-stdio")] 22 | stdio: StdioConnectOptions, 23 | #[cfg(feature = "backend-file")] 24 | file: FileConnectOptions, 25 | } 26 | 27 | impl SeaConnectOptions { 28 | #[cfg(feature = "backend-kafka")] 29 | pub fn into_kafka_connect_options(self) -> KafkaConnectOptions { 30 | self.kafka 31 | } 32 | 33 | #[cfg(feature = "backend-redis")] 34 | pub fn into_redis_connect_options(self) -> RedisConnectOptions { 35 | self.redis 36 | } 37 | 38 | #[cfg(feature = "backend-stdio")] 39 | pub fn into_stdio_connect_options(self) -> StdioConnectOptions { 40 | self.stdio 41 | } 42 | 43 | #[cfg(feature = "backend-file")] 44 | pub fn into_file_connect_options(self) -> FileConnectOptions { 45 | self.file 46 | } 47 | 48 | #[cfg(feature = "backend-kafka")] 49 | /// Set options that only applies to Kafka 50 | pub fn set_kafka_connect_options(&mut self, func: F) { 51 | func(&mut self.kafka) 52 | } 53 | 54 | #[cfg(feature = "backend-redis")] 55 | /// Set options that only applies to Redis 56 | pub fn set_redis_connect_options(&mut self, func: F) { 57 | func(&mut self.redis) 58 | } 59 | 60 | #[cfg(feature = "backend-stdio")] 61 | /// Set options that only applies to Stdio 62 | pub fn set_stdio_connect_options(&mut self, func: F) { 63 | func(&mut self.stdio) 64 | } 65 | 66 | #[cfg(feature = "backend-file")] 67 | /// Set options that only applies to File 68 | pub fn set_file_connect_options(&mut self, func: F) { 69 | func(&mut self.file) 70 | } 71 | } 72 | 73 | impl ConnectOptions for SeaConnectOptions { 74 | type Error = BackendErr; 75 | 76 | fn timeout(&self) -> SeaResult { 77 | #![allow(unreachable_code)] 78 | 79 | #[cfg(feature = "backend-kafka")] 80 | return self.kafka.timeout().map_err(map_err); 81 | #[cfg(feature = "backend-redis")] 82 | return self.redis.timeout().map_err(map_err); 83 | #[cfg(feature = "backend-stdio")] 84 | return self.stdio.timeout().map_err(map_err); 85 | #[cfg(feature = "backend-file")] 86 | return self.file.timeout().map_err(map_err); 87 | } 88 | 89 | fn set_timeout(&mut self, d: Duration) -> SeaResult<&mut Self> { 90 | #[cfg(feature = "backend-kafka")] 91 | self.kafka.set_timeout(d).map_err(map_err)?; 92 | #[cfg(feature = "backend-redis")] 93 | self.redis.set_timeout(d).map_err(map_err)?; 94 | #[cfg(feature = "backend-stdio")] 95 | self.stdio.set_timeout(d).map_err(map_err)?; 96 | #[cfg(feature = "backend-file")] 97 | self.file.set_timeout(d).map_err(map_err)?; 98 | 99 | Ok(self) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /sea-streamer-socket/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! ### `sea-streamer-socket`: Backend-agnostic Socket API 2 | //! 3 | //! Akin to how SeaORM allows you to build applications for different databases, SeaStreamer allows you to build 4 | //! stream processors for different streaming servers. 5 | //! 6 | //! While the `sea-streamer-types` crate provides a nice trait-based abstraction, this crates provides a concrete-type API, 7 | //! so that your program can stream from/to any SeaStreamer backend selected by the user *on runtime*. 8 | //! 9 | //! This allows you to do neat things, like generating data locally and then stream them to Redis / Kafka. Or in the other 10 | //! way, sink data from server to work on them locally. All _without recompiling_ the stream processor. 11 | //! 12 | //! If you only ever work with one backend, feel free to depend on `sea-streamer-redis` / `sea-streamer-kafka` directly. 13 | //! 14 | //! A small number of cli programs are provided for demonstration. Let's set them up first: 15 | //! 16 | //! ```shell 17 | //! # The `clock` program generate messages in the form of `{ "tick": N }` 18 | //! alias clock='cargo run --package sea-streamer-stdio --features=executables --bin clock' 19 | //! # The `relay` program redirect messages from `input` to `output` 20 | //! alias relay='cargo run --package sea-streamer-socket --features=executables,backend-kafka,backend-redis --bin relay' 21 | //! ``` 22 | //! 23 | //! Here is how to stream from Stdio ➡️ Redis / Kafka. We generate messages using `clock` and then pipe it to `relay`, 24 | //! which then streams to Redis / Kafka: 25 | //! 26 | //! ```shell 27 | //! # Stdio -> Redis 28 | //! clock -- --stream clock --interval 1s | \ 29 | //! relay -- --input stdio:///clock --output redis://localhost:6379/clock 30 | //! # Stdio -> Kafka 31 | //! clock -- --stream clock --interval 1s | \ 32 | //! relay -- --input stdio:///clock --output kafka://localhost:9092/clock 33 | //! ``` 34 | //! 35 | //! Here is how to stream between Redis ↔️ Kafka: 36 | //! 37 | //! ```shell 38 | //! # Redis -> Kafka 39 | //! relay -- --input redis://localhost:6379/clock --output kafka://localhost:9092/clock 40 | //! # Kafka -> Redis 41 | //! relay -- --input kafka://localhost:9092/clock --output redis://localhost:6379/clock 42 | //! ``` 43 | //! 44 | //! Here is how to *replay* the stream from Kafka / Redis: 45 | //! 46 | //! ```shell 47 | //! relay -- --input redis://localhost:6379/clock --output stdio:///clock --offset start 48 | //! relay -- --input kafka://localhost:9092/clock --output stdio:///clock --offset start 49 | //! ``` 50 | 51 | #![cfg_attr(docsrs, feature(doc_cfg))] 52 | #![deny(missing_debug_implementations)] 53 | #![doc( 54 | html_logo_url = "https://raw.githubusercontent.com/SeaQL/sea-streamer/main/docs/SeaQL icon.png" 55 | )] 56 | 57 | mod backend; 58 | mod connect_options; 59 | mod consumer; 60 | mod consumer_options; 61 | mod error; 62 | mod message; 63 | mod producer; 64 | mod producer_options; 65 | mod streamer; 66 | 67 | pub use backend::*; 68 | pub use connect_options::*; 69 | pub use consumer::*; 70 | pub use consumer_options::*; 71 | pub use error::*; 72 | pub use message::*; 73 | pub use producer::*; 74 | pub use producer_options::*; 75 | pub use streamer::*; 76 | -------------------------------------------------------------------------------- /sea-streamer-socket/src/producer_options.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "backend-file")] 2 | use sea_streamer_file::FileProducerOptions; 3 | #[cfg(feature = "backend-kafka")] 4 | use sea_streamer_kafka::KafkaProducerOptions; 5 | #[cfg(feature = "backend-redis")] 6 | use sea_streamer_redis::RedisProducerOptions; 7 | #[cfg(feature = "backend-stdio")] 8 | use sea_streamer_stdio::StdioProducerOptions; 9 | 10 | use sea_streamer_types::ProducerOptions; 11 | 12 | #[derive(Debug, Default, Clone)] 13 | /// `sea-streamer-socket` concrete type of ProducerOptions. 14 | pub struct SeaProducerOptions { 15 | #[cfg(feature = "backend-kafka")] 16 | kafka: KafkaProducerOptions, 17 | #[cfg(feature = "backend-redis")] 18 | redis: RedisProducerOptions, 19 | #[cfg(feature = "backend-stdio")] 20 | stdio: StdioProducerOptions, 21 | #[cfg(feature = "backend-file")] 22 | file: FileProducerOptions, 23 | } 24 | 25 | impl SeaProducerOptions { 26 | #[cfg(feature = "backend-kafka")] 27 | pub fn into_kafka_producer_options(self) -> KafkaProducerOptions { 28 | self.kafka 29 | } 30 | 31 | #[cfg(feature = "backend-redis")] 32 | pub fn into_redis_producer_options(self) -> RedisProducerOptions { 33 | self.redis 34 | } 35 | 36 | #[cfg(feature = "backend-stdio")] 37 | pub fn into_stdio_producer_options(self) -> StdioProducerOptions { 38 | self.stdio 39 | } 40 | 41 | #[cfg(feature = "backend-file")] 42 | pub fn into_file_producer_options(self) -> FileProducerOptions { 43 | self.file 44 | } 45 | 46 | #[cfg(feature = "backend-kafka")] 47 | /// Set options that only applies to Kafka 48 | pub fn set_kafka_producer_options(&mut self, func: F) { 49 | func(&mut self.kafka) 50 | } 51 | 52 | #[cfg(feature = "backend-redis")] 53 | /// Set options that only applies to Redis 54 | pub fn set_redis_producer_options(&mut self, func: F) { 55 | func(&mut self.redis) 56 | } 57 | 58 | #[cfg(feature = "backend-stdio")] 59 | /// Set options that only applies to Stdio 60 | pub fn set_stdio_producer_options(&mut self, func: F) { 61 | func(&mut self.stdio) 62 | } 63 | 64 | #[cfg(feature = "backend-file")] 65 | /// Set options that only applies to File 66 | pub fn set_file_producer_options(&mut self, func: F) { 67 | func(&mut self.file) 68 | } 69 | } 70 | 71 | impl ProducerOptions for SeaProducerOptions {} 72 | -------------------------------------------------------------------------------- /sea-streamer-stdio/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-stdio" 3 | version = "0.5.0" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "🌊 SeaStreamer Standard I/O Backend" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/sea-streamer-stdio" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | keywords = ["async", "stream", "stream-processing"] 12 | rust-version = "1.60" 13 | 14 | [package.metadata.docs.rs] 15 | features = [] 16 | rustdoc-args = ["--cfg", "docsrs"] 17 | 18 | [dependencies] 19 | anyhow = { version = "1", optional = true } 20 | env_logger = { version = "0.9", optional = true } 21 | flume = { version = "0.11", default-features = false, features = ["async"] } 22 | lazy_static = { version = "1.4" } 23 | log = { version = "0.4", default-features = false } 24 | nom = { version = "7" } 25 | sea-streamer-types = { version = "0.5", path = "../sea-streamer-types" } 26 | sea-streamer-runtime = { version = "0.5", path = "../sea-streamer-runtime" } 27 | serde_json = { version = "1", optional = true } 28 | clap = { version = "4.5", features = ["derive"], optional = true } 29 | thiserror = { version = "1", default-features = false } 30 | time = { version = "0.3", default-features = false, features = ["std", "parsing"] } 31 | tokio = { version = "1.10.0", optional = true } 32 | 33 | [dev-dependencies] 34 | 35 | [features] 36 | default = [] 37 | test = ["anyhow", "tokio/full", "env_logger", "sea-streamer-runtime/runtime-tokio"] 38 | executables = ["anyhow", "tokio/full", "env_logger", "clap", "serde_json", "sea-streamer-types/json", "sea-streamer-runtime/runtime-tokio"] 39 | runtime-async-std = ["sea-streamer-runtime/runtime-async-std"] 40 | runtime-tokio = ["sea-streamer-runtime/runtime-tokio"] 41 | 42 | [[bin]] 43 | name = "clock" 44 | path = "src/bin/clock.rs" 45 | required-features = ["executables"] 46 | 47 | [[bin]] 48 | name = "complex" 49 | path = "src/bin/complex.rs" 50 | required-features = ["executables"] 51 | 52 | [[bin]] 53 | name = "relay" 54 | path = "src/bin/relay.rs" 55 | required-features = ["executables"] 56 | -------------------------------------------------------------------------------- /sea-streamer-stdio/README.md: -------------------------------------------------------------------------------- 1 | ### `sea-streamer-stdio`: Standard I/O Backend 2 | 3 | This is the `stdio` backend implementation for SeaStreamer. It is designed to be connected together with unix pipes, 4 | enabling great flexibility when developing stream processors or processing data locally. 5 | 6 | You can connect processors together with pipes: `processor_a | processor_b`. 7 | 8 | You can also connect them asynchronously: 9 | 10 | ```shell 11 | touch stream # set up an empty file 12 | tail -f stream | processor_b # program b can be spawned anytime 13 | processor_a >> stream # append to the file 14 | ``` 15 | 16 | You can also use `cat` to replay a file, but it runs from start to end as fast as possible then stops, 17 | which may or may not be the desired behavior. 18 | 19 | You can write any valid UTF-8 string to stdin and each line will be considered a message. In addition, you can write some message meta in a simple format: 20 | 21 | ```log 22 | [timestamp | stream_key | sequence | shard_id] payload 23 | ``` 24 | 25 | Note: the square brackets are literal `[` `]`. 26 | 27 | The following are all valid: 28 | 29 | ```log 30 | a plain, raw message 31 | [2022-01-01T00:00:00] { "payload": "anything" } 32 | [2022-01-01T00:00:00.123 | my_topic] "a string payload" 33 | [2022-01-01T00:00:00 | my-topic-2 | 123] ["array", "of", "values"] 34 | [2022-01-01T00:00:00 | my-topic-2 | 123 | 4] { "payload": "anything" } 35 | [my_topic] a string payload 36 | [my_topic | 123] { "payload": "anything" } 37 | [my_topic | 123 | 4] { "payload": "anything" } 38 | ``` 39 | 40 | The following are all invalid: 41 | 42 | ```log 43 | [Jan 1, 2022] { "payload": "anything" } 44 | [2022-01-01T00:00:00] 12345 45 | ``` 46 | 47 | If no stream key is given, it will be assigned the name `broadcast` and sent to all consumers. 48 | 49 | You can create consumers that subscribe to only a subset of the topics. 50 | 51 | Consumers in the same `ConsumerGroup` will be load balanced (in a round-robin fashion), meaning you can spawn multiple async tasks to process messages in parallel. 52 | -------------------------------------------------------------------------------- /sea-streamer-stdio/src/bin/README.md: -------------------------------------------------------------------------------- 1 | # SeaStreamer Stdio Utils 2 | 3 | ## clock 4 | 5 | + Generate a stream of ticks 6 | 7 | Usage: 8 | 9 | ```shell 10 | cargo run --bin clock -- --interval 1s --stream clock 11 | 12 | [2022-12-06T18:31:23.285852 | clock | 0] { "tick": 0 } 13 | [2022-12-06T18:31:24.287452 | clock | 1] { "tick": 1 } 14 | [2022-12-06T18:31:25.289144 | clock | 2] { "tick": 2 } 15 | [2022-12-06T18:31:26.289846 | clock | 3] { "tick": 3 } 16 | ``` 17 | 18 | ## relay 19 | 20 | + Stream from input and redirect to output (not really useful without filters) 21 | 22 | Usage (using unix pipe): 23 | 24 | ```shell 25 | cargo run --bin clock -- --interval 1s --stream clock | cargo run --bin relay -- --input clock --output relay 26 | 27 | [2022-12-06T18:34:19.348575 | relay | 0] {"relay":true,"tick":0} 28 | [2022-12-06T18:34:20.351214 | relay | 1] {"relay":true,"tick":1} 29 | [2022-12-06T18:34:21.353177 | relay | 2] {"relay":true,"tick":2} 30 | [2022-12-06T18:34:22.354132 | relay | 3] {"relay":true,"tick":3} 31 | ``` 32 | 33 | ## complex 34 | 35 | ```shell 36 | cargo run --bin clock -- --interval 1s --stream clock | cargo run --bin complex -- --input clock --output relay 37 | 38 | [2023-02-24T08:13:10Z DEBUG sea_streamer_stdio::producer] [57811] stdout thread spawned 39 | [2023-02-24T08:13:10 | relay | 0] {"relay":1,"tick":0} 40 | [2023-02-24T08:13:11 | relay | 1] {"relay":2,"tick":1} 41 | [2023-02-24T08:13:12 | relay | 2] {"relay":1,"tick":2} 42 | [2023-02-24T08:13:13 | relay | 3] {"relay":2,"tick":3} 43 | [2023-02-24T08:13:14 | relay | 4] {"relay":1,"tick":4} 44 | [2023-02-24T08:13:15 | relay | 5] {"relay":2,"tick":5} 45 | [2023-02-24T08:13:16 | relay | 6] {"relay":1,"tick":6} 46 | [2023-02-24T08:13:17 | relay | 7] {"relay":2,"tick":7} 47 | [2023-02-24T08:13:18 | relay | 8] {"relay":1,"tick":8} 48 | [2023-02-24T08:13:19 | relay | 9] {"relay":2,"tick":9} 49 | [2023-02-24T08:13:19Z DEBUG sea_streamer_stdio::producer] [57812] stdout thread exit 50 | [2023-02-24T08:13:19Z DEBUG sea_streamer_stdio::producer] [57812] stdout thread spawned 51 | [2023-02-24T08:13:20 | relay | 10] {"relay":0,"tick":10} 52 | [2023-02-24T08:13:21 | relay | 11] {"relay":0,"tick":11} 53 | [2023-02-24T08:13:22 | relay | 12] {"relay":0,"tick":12} 54 | [2023-02-24T08:13:23 | relay | 13] {"relay":0,"tick":13} 55 | [2023-02-24T08:13:24 | relay | 14] {"relay":0,"tick":14} 56 | [2023-02-24T08:13:25 | relay | 15] {"relay":0,"tick":15} 57 | [2023-02-24T08:13:26 | relay | 16] {"relay":0,"tick":16} 58 | [2023-02-24T08:13:27 | relay | 17] {"relay":0,"tick":17} 59 | [2023-02-24T08:13:28 | relay | 18] {"relay":0,"tick":18} 60 | [2023-02-24T08:13:29 | relay | 19] {"relay":0,"tick":19} 61 | [2023-02-24T08:13:30 | relay | 20] {"relay":0,"tick":20} 62 | ``` -------------------------------------------------------------------------------- /sea-streamer-stdio/src/bin/clock.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | use clap::Parser; 3 | use sea_streamer_stdio::StdioStreamer; 4 | use sea_streamer_types::{Producer, StreamKey, Streamer, StreamerUri}; 5 | use std::time::Duration; 6 | 7 | #[derive(Debug, Parser)] 8 | struct Args { 9 | #[clap(long, help = "Stream key")] 10 | stream: StreamKey, 11 | #[clap(long, value_parser = parse_duration, help = "Period of the clock. e.g. 1s, 100ms")] 12 | interval: Duration, 13 | } 14 | 15 | fn parse_duration(src: &str) -> Result { 16 | if let Some(s) = src.strip_suffix("ns") { 17 | Ok(Duration::from_nanos(s.parse()?)) 18 | } else if let Some(s) = src.strip_suffix("us") { 19 | Ok(Duration::from_micros(s.parse()?)) 20 | } else if let Some(s) = src.strip_suffix("ms") { 21 | Ok(Duration::from_millis(s.parse()?)) 22 | } else if let Some(s) = src.strip_suffix('s') { 23 | Ok(Duration::from_secs(s.parse()?)) 24 | } else if let Some(s) = src.strip_suffix('m') { 25 | Ok(Duration::from_secs(s.parse::()? * 60)) 26 | } else { 27 | Err(anyhow!("Failed to parse {} as Duration", src)) 28 | } 29 | } 30 | 31 | #[tokio::main] 32 | async fn main() -> Result<()> { 33 | env_logger::init(); 34 | 35 | let Args { stream, interval } = Args::parse(); 36 | 37 | let streamer = StdioStreamer::connect(StreamerUri::zero(), Default::default()).await?; 38 | let producer = streamer.create_producer(stream, Default::default()).await?; 39 | let mut tick: u64 = 0; 40 | 41 | loop { 42 | producer.send(format!(r#"{{ "tick": {tick} }}"#))?; 43 | tick += 1; 44 | tokio::time::sleep(interval).await; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /sea-streamer-stdio/src/bin/complex.rs: -------------------------------------------------------------------------------- 1 | //! This is just to demonstrate the more complex behaviour of the streamer. 2 | //! Should later put this under a test framework that can manage subprocesses. 3 | use anyhow::Result; 4 | use clap::Parser; 5 | use sea_streamer_stdio::{StdioConsumerOptions, StdioStreamer}; 6 | use sea_streamer_types::{ 7 | Consumer, ConsumerGroup, ConsumerMode, ConsumerOptions, Message, Producer, StreamKey, Streamer, 8 | StreamerUri, 9 | }; 10 | 11 | #[derive(Debug, Parser)] 12 | struct Args { 13 | #[clap(long, help = "Stream key of input")] 14 | input: StreamKey, 15 | #[clap(long, help = "Stream key of output")] 16 | output: StreamKey, 17 | } 18 | 19 | #[tokio::main] 20 | async fn main() -> Result<()> { 21 | env_logger::init(); 22 | 23 | let Args { input, output } = Args::parse(); 24 | 25 | let streamer = StdioStreamer::connect(StreamerUri::zero(), Default::default()).await?; 26 | let mut consumer_opt = StdioConsumerOptions::new(ConsumerMode::LoadBalanced); 27 | consumer_opt.set_consumer_group(ConsumerGroup::new("abc".to_owned()))?; 28 | let producer = streamer 29 | .create_producer(output.clone(), Default::default()) 30 | .await?; 31 | 32 | { 33 | let consumer1 = streamer 34 | .create_consumer(&[input.clone()], consumer_opt.clone()) 35 | .await?; 36 | let consumer2 = streamer 37 | .create_consumer(&[input.clone()], consumer_opt.clone()) 38 | .await?; 39 | 40 | for _ in 0..5 { 41 | let mess = consumer1.next().await?; 42 | let mut value: serde_json::Value = mess.message().deserialize_json()?; 43 | if let serde_json::Value::Object(object) = &mut value { 44 | object.insert("relay".to_owned(), serde_json::Value::Number(1.into())); 45 | } 46 | producer.send(serde_json::to_string(&value)?.as_str()).ok(); 47 | 48 | let mess = consumer2.next().await?; 49 | let mut value: serde_json::Value = mess.message().deserialize_json()?; 50 | if let serde_json::Value::Object(object) = &mut value { 51 | object.insert("relay".to_owned(), serde_json::Value::Number(2.into())); 52 | } 53 | producer.send(serde_json::to_string(&value)?.as_str()).ok(); 54 | } 55 | } 56 | 57 | streamer.disconnect().await?; 58 | let streamer = StdioStreamer::connect(StreamerUri::zero(), Default::default()).await?; 59 | let consumer = streamer 60 | .create_consumer(&[input.clone()], consumer_opt.clone()) 61 | .await?; 62 | // if we don't create a new producer, `send` will return Disconnected error 63 | let producer = streamer.create_producer(output, Default::default()).await?; 64 | 65 | loop { 66 | let mess = consumer.next().await?; 67 | let mut value: serde_json::Value = mess.message().deserialize_json()?; 68 | if let serde_json::Value::Object(object) = &mut value { 69 | object.insert("relay".to_owned(), serde_json::Value::Number(0.into())); 70 | } 71 | producer.send(serde_json::to_string(&value)?.as_str())?; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /sea-streamer-stdio/src/bin/relay.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use sea_streamer_stdio::StdioStreamer; 4 | use sea_streamer_types::{Consumer, Message, Producer, StreamKey, Streamer, StreamerUri}; 5 | 6 | #[derive(Debug, Parser)] 7 | struct Args { 8 | #[clap(long, help = "Stream key of input")] 9 | input: StreamKey, 10 | #[clap(long, help = "Stream key of output")] 11 | output: StreamKey, 12 | } 13 | 14 | #[tokio::main] 15 | async fn main() -> Result<()> { 16 | env_logger::init(); 17 | 18 | let Args { input, output } = Args::parse(); 19 | 20 | let streamer = StdioStreamer::connect(StreamerUri::zero(), Default::default()).await?; 21 | let consumer = streamer 22 | .create_consumer(&[input], Default::default()) 23 | .await?; 24 | let producer = streamer.create_producer(output, Default::default()).await?; 25 | 26 | loop { 27 | let mess = consumer.next().await?; 28 | let mut value: serde_json::Value = mess.message().deserialize_json()?; 29 | if let serde_json::Value::Object(object) = &mut value { 30 | object.insert("relay".to_owned(), serde_json::Value::Bool(true)); 31 | } 32 | producer.send(serde_json::to_string(&value)?.as_str())?; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /sea-streamer-stdio/src/error.rs: -------------------------------------------------------------------------------- 1 | use sea_streamer_types::StreamResult; 2 | use thiserror::Error; 3 | 4 | #[derive(Error, Debug)] 5 | pub enum StdioErr { 6 | #[error("Flume RecvError: {0}")] 7 | RecvError(flume::RecvError), 8 | #[error("IO Error: {0}")] 9 | IoError(std::io::Error), 10 | #[error("StdioStreamer has been disconnected")] 11 | Disconnected, 12 | } 13 | 14 | pub type StdioResult = StreamResult; 15 | -------------------------------------------------------------------------------- /sea-streamer-stdio/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! ### `sea-streamer-stdio`: Standard I/O Backend 2 | //! 3 | //! This is the `stdio` backend implementation for SeaStreamer. It is designed to be connected together with unix pipes, 4 | //! enabling great flexibility when developing stream processors or processing data locally. 5 | //! 6 | //! You can connect processors together with pipes: `processor_a | processor_b`. 7 | //! 8 | //! You can also connect them asynchronously: 9 | //! 10 | //! ```shell 11 | //! touch stream # set up an empty file 12 | //! tail -f stream | processor_b # program b can be spawned anytime 13 | //! processor_a >> stream # append to the file 14 | //! ``` 15 | //! 16 | //! You can also use `cat` to replay a file, but it runs from start to end as fast as possible then stops, 17 | //! which may or may not be the desired behavior. 18 | //! 19 | //! You can write any valid UTF-8 string to stdin and each line will be considered a message. In addition, you can write some message meta in a simple format: 20 | //! 21 | //! ```log 22 | //! [timestamp | stream_key | sequence | shard_id] payload 23 | //! ``` 24 | //! 25 | //! Note: the square brackets are literal `[` `]`. 26 | //! 27 | //! The following are all valid: 28 | //! 29 | //! ```log 30 | //! a plain, raw message 31 | //! [2022-01-01T00:00:00] { "payload": "anything" } 32 | //! [2022-01-01T00:00:00.123 | my_topic] "a string payload" 33 | //! [2022-01-01T00:00:00 | my-topic-2 | 123] ["array", "of", "values"] 34 | //! [2022-01-01T00:00:00 | my-topic-2 | 123 | 4] { "payload": "anything" } 35 | //! [my_topic] a string payload 36 | //! [my_topic | 123] { "payload": "anything" } 37 | //! [my_topic | 123 | 4] { "payload": "anything" } 38 | //! ``` 39 | //! 40 | //! The following are all invalid: 41 | //! 42 | //! ```log 43 | //! [Jan 1, 2022] { "payload": "anything" } 44 | //! [2022-01-01T00:00:00] 12345 45 | //! ``` 46 | //! 47 | //! If no stream key is given, it will be assigned the name `broadcast` and sent to all consumers. 48 | //! 49 | //! You can create consumers that subscribe to only a subset of the topics. 50 | //! 51 | //! Consumers in the same `ConsumerGroup` will be load balanced (in a round-robin fashion), meaning you can spawn multiple async tasks to process messages in parallel. 52 | 53 | #![cfg_attr(docsrs, feature(doc_cfg))] 54 | #![deny(missing_debug_implementations)] 55 | #![doc( 56 | html_logo_url = "https://raw.githubusercontent.com/SeaQL/sea-streamer/main/docs/SeaQL icon.png" 57 | )] 58 | 59 | /// Default stream key 60 | pub const BROADCAST: &str = "broadcast"; 61 | 62 | use time::{format_description::FormatItem, macros::format_description}; 63 | 64 | /// Canonical time format 65 | pub const TIMESTAMP_FORMAT: &[FormatItem<'static>] = 66 | format_description!("[year]-[month]-[day]T[hour]:[minute]:[second]"); 67 | // have no idea to how to make subsecond optional 68 | /// Canonical time format with sub-seconds 69 | pub const TIMESTAMP_FORMAT_SUBSEC: &[FormatItem<'static>] = 70 | format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]"); 71 | 72 | mod consumer; 73 | mod consumer_group; 74 | mod error; 75 | pub(crate) mod parser; 76 | mod producer; 77 | mod streamer; 78 | mod util; 79 | 80 | pub use consumer::*; 81 | pub use error::*; 82 | pub(crate) use parser::*; 83 | pub use producer::*; 84 | pub use streamer::*; 85 | -------------------------------------------------------------------------------- /sea-streamer-stdio/src/util.rs: -------------------------------------------------------------------------------- 1 | pub struct PanicGuard; 2 | 3 | impl Drop for PanicGuard { 4 | fn drop(&mut self) { 5 | if std::thread::panicking() { 6 | panic!("PanicGuard dropped while thread panicking"); 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /sea-streamer-stdio/tests/group.rs: -------------------------------------------------------------------------------- 1 | // cargo test --test group --features=test -- --nocapture 2 | #[cfg(feature = "test")] 3 | #[tokio::test] 4 | async fn main() -> anyhow::Result<()> { 5 | use sea_streamer_stdio::{ 6 | StdioConnectOptions, StdioConsumer, StdioConsumerOptions, StdioStreamer, 7 | }; 8 | use sea_streamer_types::{ 9 | export::futures::StreamExt, Buffer, Consumer, ConsumerGroup, ConsumerMode, ConsumerOptions, 10 | Message, Producer, StreamKey, Streamer, StreamerUri, 11 | }; 12 | 13 | env_logger::init(); 14 | 15 | let stream = StreamKey::new("hello")?; 16 | 17 | let mut options = StdioConnectOptions::default(); 18 | options.set_loopback(true); 19 | let streamer = StdioStreamer::connect(StreamerUri::zero(), options).await?; 20 | 21 | let producer = streamer 22 | .create_producer(stream.clone(), Default::default()) 23 | .await?; 24 | 25 | let mut consumer_opt = StdioConsumerOptions::new(ConsumerMode::LoadBalanced); 26 | consumer_opt.set_consumer_group(ConsumerGroup::new("abc".to_owned()))?; 27 | let mut first = streamer 28 | .create_consumer(&[stream.clone()], consumer_opt.clone()) 29 | .await?; 30 | 31 | for i in 0..4 { 32 | let mess = format!("{}", i); 33 | producer.send(mess)?; 34 | } 35 | 36 | let seq = consume(&mut first, 4).await; 37 | assert_eq!(seq, [0, 1, 2, 3]); 38 | 39 | let mut second = streamer 40 | .create_consumer(&[stream.clone()], consumer_opt) 41 | .await?; 42 | 43 | for i in 0..10 { 44 | let mess = format!("{}", i); 45 | producer.send(mess)?; 46 | } 47 | 48 | let seq = consume(&mut first, 5).await; 49 | assert_eq!(seq, [0, 2, 4, 6, 8]); 50 | 51 | let seq = consume(&mut second, 5).await; 52 | assert_eq!(seq, [1, 3, 5, 7, 9]); 53 | 54 | streamer.disconnect().await?; 55 | 56 | async fn consume(consumer: &mut StdioConsumer, num: usize) -> Vec { 57 | consumer 58 | .stream() 59 | .take(num) 60 | .map(|mess| { 61 | mess.unwrap() 62 | .message() 63 | .as_str() 64 | .unwrap() 65 | .parse::() 66 | .unwrap() 67 | }) 68 | .collect::>() 69 | .await 70 | } 71 | 72 | Ok(()) 73 | } 74 | -------------------------------------------------------------------------------- /sea-streamer-stdio/tests/loopback.rs: -------------------------------------------------------------------------------- 1 | // cargo test --test loopback --features=test -- --nocapture 2 | #[cfg(feature = "test")] 3 | #[tokio::test] 4 | async fn main() -> anyhow::Result<()> { 5 | use sea_streamer_stdio::{StdioConnectOptions, StdioConsumer, StdioStreamer}; 6 | use sea_streamer_types::{ 7 | export::futures::StreamExt, Buffer, Consumer, Message, Producer, StreamKey, Streamer, 8 | StreamerUri, 9 | }; 10 | 11 | env_logger::init(); 12 | 13 | let stream = StreamKey::new("hello")?; 14 | let streamer = StdioStreamer::connect(StreamerUri::zero(), Default::default()).await?; 15 | let consumer = streamer 16 | .create_consumer(&[stream.clone()], Default::default()) 17 | .await?; 18 | let producer = streamer 19 | .create_producer(stream.clone(), Default::default()) 20 | .await?; 21 | 22 | for _ in 0..5 { 23 | let mess = format!("{}", -1); 24 | // these are not looped back 25 | producer.send(mess)?; 26 | } 27 | 28 | streamer.disconnect().await?; 29 | assert!(consumer.next().await.is_err()); 30 | assert!(producer.send("").is_err()); 31 | 32 | let mut options = StdioConnectOptions::default(); 33 | options.set_loopback(true); 34 | let streamer = StdioStreamer::connect(StreamerUri::zero(), options).await?; 35 | let producer = streamer 36 | .create_producer(stream.clone(), Default::default()) 37 | .await?; 38 | let mut consumer = streamer 39 | .create_consumer(&[stream.clone()], Default::default()) 40 | .await?; 41 | 42 | for i in 0..5 { 43 | let mess = format!("{}", i); 44 | producer.send(mess)?; 45 | } 46 | 47 | let seq = consume(&mut consumer, 5).await; 48 | assert_eq!(seq, [0, 1, 2, 3, 4]); 49 | 50 | streamer.disconnect().await?; 51 | 52 | async fn consume(consumer: &mut StdioConsumer, num: usize) -> Vec { 53 | consumer 54 | .stream() 55 | .take(num) 56 | .map(|mess| { 57 | mess.unwrap() 58 | .message() 59 | .as_str() 60 | .unwrap() 61 | .parse::() 62 | .unwrap() 63 | }) 64 | .collect::>() 65 | .await 66 | } 67 | 68 | Ok(()) 69 | } 70 | -------------------------------------------------------------------------------- /sea-streamer-types/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sea-streamer-types" 3 | version = "0.5.2" 4 | authors = ["Chris Tsang "] 5 | edition = "2021" 6 | description = "🌊 SeaStreamer Traits & Types" 7 | license = "MIT OR Apache-2.0" 8 | documentation = "https://docs.rs/sea-streamer-types" 9 | repository = "https://github.com/SeaQL/sea-streamer" 10 | categories = ["concurrency"] 11 | keywords = ["async", "stream", "stream-processing"] 12 | rust-version = "1.60" 13 | 14 | [package.metadata.docs.rs] 15 | all-features = true 16 | rustdoc-args = ["--cfg", "docsrs"] 17 | 18 | [dependencies] 19 | futures = { version = "0.3", default-features = false, features = ["std", "alloc", "async-await"] } 20 | thiserror = { version = "1", default-features = false } 21 | time = { version = "0.3", default-features = false, features = ["std", "macros", "formatting"] } 22 | url = { version = "2.2", default-features = false } 23 | serde = { version = "1", default-features = false, optional = true, features = ["derive"] } 24 | serde_json = { version = "1", optional = true } 25 | 26 | [features] 27 | json = ["serde", "serde_json"] 28 | wide-seq-no = [] -------------------------------------------------------------------------------- /sea-streamer-types/README.md: -------------------------------------------------------------------------------- 1 | ### `sea-streamer-types`: Traits & Types 2 | 3 | This crate defines all the traits and types for the SeaStreamer API, but does not provide any implementation. 4 | -------------------------------------------------------------------------------- /sea-streamer-types/src/components/mod.rs: -------------------------------------------------------------------------------- 1 | // Use flume for in-process concurrency -------------------------------------------------------------------------------- /sea-streamer-types/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::str::Utf8Error; 2 | use thiserror::Error; 3 | 4 | /// Type alias of the [`Result`] type specific to `sea-streamer`. 5 | pub type StreamResult = std::result::Result>; 6 | 7 | #[derive(Error, Debug)] 8 | /// Common errors that may occur. 9 | pub enum StreamErr { 10 | #[error("Connection Error: {0}")] 11 | Connect(String), 12 | #[error("Timeout has not yet been set")] 13 | TimeoutNotSet, 14 | #[error("Producer has already been anchored")] 15 | AlreadyAnchored, 16 | #[error("Producer has not yet been anchored")] 17 | NotAnchored, 18 | #[error("Consumer group is set; but not expected")] 19 | ConsumerGroupIsSet, 20 | #[error("Consumer group has not yet been set")] 21 | ConsumerGroupNotSet, 22 | #[error("Stream key set is empty")] 23 | StreamKeyEmpty, 24 | #[error("Stream key not found")] 25 | StreamKeyNotFound, 26 | #[error("You cannot commit on a real-time consumer")] 27 | CommitNotAllowed, 28 | #[error("Utf8Error: {0}")] 29 | Utf8Error(Utf8Error), 30 | #[error("StreamUrlErr {0}")] 31 | StreamUrlErr(#[from] StreamUrlErr), 32 | #[error("StreamKeyErr {0}")] 33 | StreamKeyErr(#[from] StreamKeyErr), 34 | #[error("Unsupported feature: {0}")] 35 | Unsupported(String), 36 | #[error("Backend error: {0}")] 37 | Backend(E), 38 | #[error("Runtime error: {0}")] 39 | Runtime(Box), 40 | } 41 | 42 | #[cfg(feature = "json")] 43 | #[cfg_attr(docsrs, doc(cfg(feature = "json")))] 44 | #[derive(Error, Debug)] 45 | /// Errors that may happen when processing JSON 46 | pub enum JsonErr { 47 | #[error("Utf8Error {0}")] 48 | Utf8Error(#[from] std::str::Utf8Error), 49 | #[error("serde_json::Error {0}")] 50 | SerdeJson(#[from] serde_json::Error), 51 | } 52 | 53 | #[derive(Error, Debug, Clone)] 54 | /// Errors that may happen when parsing stream URL 55 | pub enum StreamUrlErr { 56 | #[error("UrlParseError {0}")] 57 | UrlParseError(#[from] url::ParseError), 58 | #[error("StreamKeyErr {0}")] 59 | StreamKeyErr(#[from] StreamKeyErr), 60 | #[error("Expected one stream key, found zero or more than one")] 61 | NotOneStreamKey, 62 | #[error("No node has been specified")] 63 | ZeroNode, 64 | #[error("Protocol is required")] 65 | ProtocolRequired, 66 | #[error("URL must have an ending slash, even when streams is empty")] 67 | NoEndingSlash, 68 | } 69 | 70 | #[derive(Error, Debug, Clone, Copy)] 71 | /// Errors that may happen when handling StreamKey 72 | pub enum StreamKeyErr { 73 | #[error("Invalid stream key: valid pattern is [a-zA-Z0-9._-]{{1, 249}}")] 74 | InvalidStreamKey, 75 | } 76 | 77 | /// Function to construct a [`StreamErr::Runtime`] error variant. 78 | pub fn runtime_error( 79 | e: E, 80 | ) -> StreamErr { 81 | StreamErr::Runtime(Box::new(e)) 82 | } 83 | -------------------------------------------------------------------------------- /sea-streamer-types/src/export.rs: -------------------------------------------------------------------------------- 1 | pub use futures; 2 | pub use time; 3 | pub use url; 4 | -------------------------------------------------------------------------------- /sea-streamer-types/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! ### `sea-streamer-types`: Traits & Types 2 | //! 3 | //! This crate defines all the traits and types for the SeaStreamer API, but does not provide any implementation. 4 | 5 | #![cfg_attr(docsrs, feature(doc_cfg))] 6 | #![deny(missing_debug_implementations)] 7 | #![doc( 8 | html_logo_url = "https://raw.githubusercontent.com/SeaQL/sea-streamer/main/docs/SeaQL icon.png" 9 | )] 10 | 11 | mod consumer; 12 | mod error; 13 | mod message; 14 | mod options; 15 | mod producer; 16 | mod stream; 17 | mod streamer; 18 | 19 | pub use consumer::*; 20 | pub use error::*; 21 | pub use message::*; 22 | pub use options::*; 23 | pub use producer::*; 24 | pub use stream::*; 25 | pub use streamer::*; 26 | 27 | /// Re-export types from related libraries 28 | pub mod export; 29 | -------------------------------------------------------------------------------- /sea-streamer-types/src/options.rs: -------------------------------------------------------------------------------- 1 | use crate::StreamResult; 2 | use std::time::Duration; 3 | 4 | /// Common options when connecting to a streamer. 5 | pub trait ConnectOptions: Default + Clone + Send { 6 | type Error: std::error::Error; 7 | 8 | fn timeout(&self) -> StreamResult; 9 | fn set_timeout(&mut self, d: Duration) -> StreamResult<&mut Self, Self::Error>; 10 | } 11 | -------------------------------------------------------------------------------- /sea-streamer-types/src/producer.rs: -------------------------------------------------------------------------------- 1 | use futures::Future; 2 | 3 | use crate::{Buffer, MessageHeader, StreamKey, StreamResult}; 4 | 5 | /// Common options of a Producer. 6 | pub trait ProducerOptions: Default + Clone + Send {} 7 | 8 | /// Delivery receipt. 9 | pub type Receipt = MessageHeader; 10 | 11 | /// Common interface of producers, to be implemented by all backends. 12 | pub trait Producer: Clone + Send + Sync { 13 | type Error: std::error::Error; 14 | type SendFuture: Future>; 15 | 16 | /// Send a message to a particular stream. This function is non-blocking. 17 | /// You don't have to await the future if you are not interested in the Receipt. 18 | fn send_to( 19 | &self, 20 | stream: &StreamKey, 21 | payload: S, 22 | ) -> StreamResult; 23 | 24 | /// Send a message to the already anchored stream. This function is non-blocking. 25 | /// You don't have to await the future if you are not interested in the Receipt. 26 | /// 27 | /// If the producer is not anchored, this will return `StreamErr::NotAnchored` error. 28 | fn send(&self, payload: S) -> StreamResult { 29 | self.send_to(self.anchored()?, payload) 30 | } 31 | 32 | /// End this producer, only after flushing all it's pending messages. 33 | fn end(self) -> impl Future> + Send; 34 | 35 | /// Flush all pending messages. 36 | fn flush(&mut self) -> impl Future> + Send; 37 | 38 | /// Lock this producer to a particular stream. This function can only be called once. 39 | /// Subsequent calls should return `StreamErr::AlreadyAnchored` error. 40 | fn anchor(&mut self, stream: StreamKey) -> StreamResult<(), Self::Error>; 41 | 42 | /// If the producer is already anchored, return a reference to the StreamKey. 43 | /// If the producer is not anchored, this will return `StreamErr::NotAnchored` error. 44 | fn anchored(&self) -> StreamResult<&StreamKey, Self::Error>; 45 | } 46 | -------------------------------------------------------------------------------- /sea-streamer-types/src/stream.rs: -------------------------------------------------------------------------------- 1 | use std::{fmt::Display, str::FromStr, sync::Arc}; 2 | pub use time::OffsetDateTime as Timestamp; 3 | 4 | use crate::StreamKeyErr; 5 | 6 | /// Maximum string length of a stream key. 7 | pub const MAX_STREAM_KEY_LEN: usize = 249; 8 | 9 | /// Reserved by SeaStreamer. Avoid using this as StreamKey. 10 | pub const SEA_STREAMER_INTERNAL: &str = "SEA_STREAMER_INTERNAL"; 11 | 12 | /// Canonical display format for Timestamp. 13 | pub const TIMESTAMP_FORMAT: &[time::format_description::FormatItem<'static>] = 14 | time::macros::format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]"); 15 | 16 | #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] 17 | /// Identifies a stream. Aka. topic. 18 | pub struct StreamKey { 19 | name: Arc, 20 | } 21 | 22 | #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 23 | /// Identifies a shard. Aka. partition. 24 | pub struct ShardId { 25 | id: u64, 26 | } 27 | 28 | /// The tuple (StreamKey, ShardId, SeqNo) uniquely identifies a message. Aka. offset. 29 | #[cfg(not(feature = "wide-seq-no"))] 30 | pub type SeqNo = u64; 31 | #[cfg(feature = "wide-seq-no")] 32 | pub type SeqNo = u128; 33 | 34 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 35 | /// Identifies a position in a stream. 36 | pub enum SeqPos { 37 | Beginning, 38 | End, 39 | At(SeqNo), 40 | } 41 | 42 | impl StreamKey { 43 | pub fn new>(key: S) -> Result { 44 | let key = key.as_ref(); 45 | if is_valid_stream_key(key) { 46 | Ok(Self { 47 | name: Arc::from(key), 48 | }) 49 | } else { 50 | Err(StreamKeyErr::InvalidStreamKey) 51 | } 52 | } 53 | 54 | pub fn name(&self) -> &str { 55 | &self.name 56 | } 57 | } 58 | 59 | impl ShardId { 60 | pub const fn new(id: u64) -> Self { 61 | Self { id } 62 | } 63 | 64 | pub fn id(&self) -> u64 { 65 | self.id 66 | } 67 | } 68 | 69 | impl Display for StreamKey { 70 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 71 | write!(f, "{}", self.name) 72 | } 73 | } 74 | 75 | impl Display for ShardId { 76 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 77 | write!(f, "{self:?}") 78 | } 79 | } 80 | 81 | impl FromStr for StreamKey { 82 | type Err = StreamKeyErr; 83 | 84 | fn from_str(s: &str) -> Result { 85 | StreamKey::new(s) 86 | } 87 | } 88 | 89 | pub fn is_valid_stream_key(s: &str) -> bool { 90 | s.len() <= MAX_STREAM_KEY_LEN && s.chars().all(is_valid_stream_key_char) 91 | } 92 | 93 | /// Returns true if this character can be used in a stream key. 94 | pub fn is_valid_stream_key_char(c: char) -> bool { 95 | // https://stackoverflow.com/questions/37062904/what-are-apache-kafka-topic-name-limitations 96 | c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') 97 | } 98 | 99 | #[cfg(feature = "serde")] 100 | mod impl_serde { 101 | use super::StreamKey; 102 | 103 | impl<'de> serde::Deserialize<'de> for StreamKey { 104 | fn deserialize(deserializer: D) -> Result 105 | where 106 | D: serde::Deserializer<'de>, 107 | { 108 | let s = <&str>::deserialize(deserializer)?; 109 | s.parse().map_err(serde::de::Error::custom) 110 | } 111 | } 112 | 113 | impl serde::Serialize for StreamKey { 114 | fn serialize(&self, serializer: S) -> Result 115 | where 116 | S: serde::Serializer, 117 | { 118 | serializer.serialize_str(self.name()) 119 | } 120 | } 121 | } 122 | --------------------------------------------------------------------------------