├── .cargo └── config.toml ├── .dockerignore ├── .github └── workflows │ └── cargo-test.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── README.md ├── coerce ├── Cargo.toml ├── README.md ├── benches │ ├── actor_creation.rs │ └── actor_messaging.rs ├── macros │ ├── Cargo.toml │ └── src │ │ ├── lib.rs │ │ ├── message │ │ ├── json.rs │ │ └── mod.rs │ │ └── snapshot │ │ ├── json.rs │ │ └── mod.rs ├── src │ ├── actor │ │ ├── blocking.rs │ │ ├── context.rs │ │ ├── describe │ │ │ └── mod.rs │ │ ├── event │ │ │ ├── dispatcher.rs │ │ │ └── mod.rs │ │ ├── lifecycle.rs │ │ ├── message │ │ │ └── mod.rs │ │ ├── metrics │ │ │ └── mod.rs │ │ ├── mod.rs │ │ ├── refs │ │ │ └── mod.rs │ │ ├── scheduler │ │ │ ├── mod.rs │ │ │ └── timer.rs │ │ ├── supervised.rs │ │ ├── system │ │ │ ├── builder.rs │ │ │ └── mod.rs │ │ ├── watch │ │ │ ├── mod.rs │ │ │ └── watchers.rs │ │ └── worker.rs │ ├── lib.rs │ ├── persistent │ │ ├── actor.rs │ │ ├── batch.rs │ │ ├── context.rs │ │ ├── failure.rs │ │ ├── inspect │ │ │ └── mod.rs │ │ ├── journal │ │ │ ├── mod.rs │ │ │ ├── proto │ │ │ │ ├── journal.rs │ │ │ │ └── mod.rs │ │ │ ├── provider.rs │ │ │ ├── snapshot.rs │ │ │ ├── storage.rs │ │ │ └── types.rs │ │ ├── mod.rs │ │ └── recovery.rs │ ├── protocol │ │ ├── network.proto │ │ ├── persistent │ │ │ └── journal.proto │ │ ├── sharding.proto │ │ └── singleton.proto │ ├── remote │ │ ├── actor │ │ │ ├── clients.rs │ │ │ ├── message.rs │ │ │ ├── mod.rs │ │ │ └── registry.rs │ │ ├── actor_ref.rs │ │ ├── api │ │ │ ├── builder.rs │ │ │ ├── cluster │ │ │ │ └── mod.rs │ │ │ ├── metrics │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ ├── openapi.rs │ │ │ ├── sharding │ │ │ │ ├── cluster.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── node.rs │ │ │ │ └── routes.rs │ │ │ └── system │ │ │ │ ├── actors.rs │ │ │ │ └── mod.rs │ │ ├── cluster │ │ │ ├── builder │ │ │ │ ├── client.rs │ │ │ │ ├── mod.rs │ │ │ │ └── worker.rs │ │ │ ├── client │ │ │ │ └── mod.rs │ │ │ ├── discovery │ │ │ │ ├── dns.rs │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ └── node.rs │ │ ├── config │ │ │ └── mod.rs │ │ ├── handler.rs │ │ ├── heartbeat │ │ │ ├── health.rs │ │ │ └── mod.rs │ │ ├── mod.rs │ │ ├── net │ │ │ ├── client │ │ │ │ ├── connect.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── ping.rs │ │ │ │ ├── receive.rs │ │ │ │ └── send.rs │ │ │ ├── message.rs │ │ │ ├── metrics.rs │ │ │ ├── mod.rs │ │ │ ├── proto │ │ │ │ ├── mod.rs │ │ │ │ └── network.rs │ │ │ ├── security │ │ │ │ ├── auth │ │ │ │ │ ├── jwt.rs │ │ │ │ │ └── mod.rs │ │ │ │ └── mod.rs │ │ │ └── server │ │ │ │ ├── mod.rs │ │ │ │ └── session │ │ │ │ ├── mod.rs │ │ │ │ └── store.rs │ │ ├── stream │ │ │ ├── alerts.rs │ │ │ ├── mediator │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ ├── pubsub.rs │ │ │ └── system.rs │ │ ├── system │ │ │ ├── actor.rs │ │ │ ├── builder.rs │ │ │ ├── cluster.rs │ │ │ ├── mod.rs │ │ │ └── rpc.rs │ │ └── tracing │ │ │ └── mod.rs │ ├── sharding │ │ ├── builder.rs │ │ ├── coordinator │ │ │ ├── allocation.rs │ │ │ ├── balancing.rs │ │ │ ├── discovery.rs │ │ │ ├── factory.rs │ │ │ ├── mod.rs │ │ │ ├── stats.rs │ │ │ └── stream.rs │ │ ├── host │ │ │ ├── mod.rs │ │ │ ├── request.rs │ │ │ └── stats.rs │ │ ├── mod.rs │ │ ├── proto │ │ │ ├── mod.rs │ │ │ └── sharding.rs │ │ └── shard │ │ │ ├── message.rs │ │ │ ├── mod.rs │ │ │ ├── passivation │ │ │ └── mod.rs │ │ │ ├── recovery.rs │ │ │ └── stats.rs │ └── singleton │ │ ├── factory.rs │ │ ├── manager │ │ ├── lease.rs │ │ ├── mod.rs │ │ ├── start.rs │ │ └── status.rs │ │ ├── mod.rs │ │ ├── proto │ │ ├── mod.rs │ │ └── singleton.rs │ │ └── proxy │ │ ├── mod.rs │ │ └── send.rs ├── tests │ ├── sharding │ │ └── mod.rs │ ├── system │ │ └── mod.rs │ ├── test_actor_describe.rs │ ├── test_actor_lifecycle.rs │ ├── test_actor_messaging.rs │ ├── test_actor_supervision.rs │ ├── test_actor_system.rs │ ├── test_actor_system_blocking.rs │ ├── test_actor_watching.rs │ ├── test_cluster_singleton.rs │ ├── test_persistent_failures.rs │ ├── test_persistent_recovery.rs │ ├── test_remote_actor_creation.rs │ ├── test_remote_actor_err.rs │ ├── test_remote_actor_locate.rs │ ├── test_remote_api.rs │ ├── test_remote_cluster_client.rs │ ├── test_remote_cluster_formation.rs │ ├── test_remote_cluster_heartbeat.rs │ ├── test_remote_codec.rs │ ├── test_remote_handle.rs │ ├── test_remote_net.rs │ ├── test_remote_pubsub.rs │ ├── test_remote_sharding.rs │ ├── test_remote_sharding_rebalancing.rs │ ├── test_remote_system_health.rs │ ├── test_remote_system_security.rs │ ├── test_timer.rs │ ├── test_tracing.rs │ ├── test_worker.rs │ └── util │ │ └── mod.rs └── tools │ ├── coerce-proto-build │ ├── Cargo.toml │ └── src │ │ └── main.rs │ └── coerce-test │ ├── Cargo.toml │ └── src │ ├── event │ ├── filter.rs │ └── mod.rs │ └── lib.rs ├── examples ├── README.md ├── coerce-cluster-example │ ├── Cargo.toml │ └── src │ │ ├── actor.rs │ │ ├── main.rs │ │ └── worker.rs └── coerce-sharded-chat-example │ ├── Cargo.toml │ ├── Dockerfile-server │ ├── README.md │ ├── bin │ ├── client.rs │ └── server.rs │ ├── kustomize │ └── sharded-chat-server │ │ ├── base │ │ ├── kustomization.yaml │ │ ├── service.yaml │ │ ├── serviceaccount.yaml │ │ └── statefulset.yaml │ │ └── kustomization.yaml │ ├── scripts │ ├── run-client-user-1.ps1 │ ├── run-client-user-2.ps1 │ ├── run-client-user-3.ps1 │ ├── run-client-user-4.ps1 │ ├── run-server-1.ps1 │ ├── run-server-2.ps1 │ ├── run-server-3.ps1 │ └── run-server-4.ps1 │ ├── src │ ├── actor │ │ ├── mod.rs │ │ ├── peer │ │ │ ├── message.rs │ │ │ └── mod.rs │ │ ├── pubsub.rs │ │ └── stream │ │ │ └── mod.rs │ ├── app.rs │ ├── lib.rs │ ├── protocol │ │ ├── chat.proto │ │ ├── chat.rs │ │ ├── internal.proto │ │ └── mod.rs │ └── websocket │ │ ├── client │ │ └── mod.rs │ │ └── mod.rs │ └── tests │ └── sharded_chat_tests.rs └── providers ├── discovery └── coerce-k8s │ ├── Cargo.toml │ └── src │ ├── config.rs │ └── lib.rs └── persistence └── coerce-redis ├── Cargo.toml ├── src ├── journal │ ├── actor.rs │ └── mod.rs └── lib.rs └── tests └── test_redis_journal_storage.rs /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [env] 2 | LOG_LEVEL = "TRACE" 3 | 4 | ## Required when running coerce with `--all-features` or enabling `tracing-unstable` feature. 5 | [build] 6 | rustflags = ["--cfg", "tracing_unstable"] -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | target 2 | Dockerfile 3 | .dockerignore 4 | .git 5 | .gitignore 6 | .fleet -------------------------------------------------------------------------------- /.github/workflows/cargo-test.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - master 5 | - feature/** 6 | - dev 7 | pull_request: 8 | branches: 9 | - master 10 | workflow_dispatch: 11 | 12 | name: coerce-rs tests 13 | 14 | jobs: 15 | build_and_test: 16 | name: Coerce Runtime Test 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@master 20 | - uses: supercharge/redis-github-action@1.2.0 21 | with: 22 | redis-version: 6 23 | - uses: actions-rs/cargo@v1 24 | with: 25 | command: test 26 | args: --all-features 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | .idea/ 4 | /coerce-remote/extensions/ 5 | .fleet/ -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "coerce", 4 | "coerce/macros", 5 | "examples/coerce-cluster-example", 6 | "examples/coerce-sharded-chat-example", 7 | "coerce/tools/coerce-proto-build", 8 | "providers/persistence/coerce-redis", 9 | "providers/discovery/coerce-k8s" 10 | ] 11 | 12 | resolver = "2" -------------------------------------------------------------------------------- /coerce/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "coerce" 3 | description = "Async actor runtime and distributed systems framework" 4 | license = "Apache-2.0" 5 | version = "0.8.12" 6 | authors = ["Leon Hartley "] 7 | edition = "2021" 8 | readme = "README.md" 9 | repository = "https://github.com/leonhartley/coerce-rs" 10 | 11 | [features] 12 | default = [] 13 | 14 | full = [ 15 | "remote", 16 | "persistence", 17 | "metrics", 18 | "sharding", 19 | "api", 20 | "actor-tracing", 21 | "actor-tracing-info", 22 | "client-auth-jwt", 23 | "singleton", 24 | ] 25 | 26 | remote = [ 27 | "dep:hashring", 28 | "dep:protobuf", 29 | "dep:chrono", 30 | "dep:tokio-stream", 31 | "dep:parking_lot", 32 | "dep:bytes", 33 | "dep:byteorder" 34 | ] 35 | 36 | persistence = [ 37 | "dep:protobuf", 38 | "dep:anyhow", 39 | "dep:parking_lot" 40 | ] 41 | 42 | metrics = [ 43 | "dep:metrics", 44 | "dep:metrics-exporter-prometheus", 45 | "dep:metrics-util" 46 | ] 47 | 48 | sharding = [ 49 | "remote", 50 | "persistence" 51 | ] 52 | 53 | actor-tracing = [] 54 | 55 | actor-events = [] 56 | 57 | tracing-unstable = [ 58 | "tracing/valuable" 59 | ] 60 | 61 | # When this feature is enabled, actor spans will be created at INFO level 62 | actor-tracing-info = ["actor-tracing"] 63 | 64 | # When this feature is enabled, actor spans will be created at TRACE level 65 | actor-tracing-trace = ["actor-tracing"] 66 | 67 | # When this feature is enabled, actor spans will be created at DEBUG level 68 | actor-tracing-debug = ["actor-tracing"] 69 | 70 | api = ["remote", "dep:axum", "dep:utoipa", "dep:utoipa-swagger-ui"] 71 | 72 | client-auth-jwt = ["dep:jwt", "dep:hmac", "dep:sha2"] 73 | 74 | singleton = [] 75 | 76 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 77 | 78 | [dependencies] 79 | tokio = { version = "1.32.0", features = ["full"] } 80 | tokio-util = { version = "0.7.8", features = ["full"] } 81 | tokio-stream = { version = "0.1.14", optional = true } 82 | tracing = { version = "0.1.37" } 83 | uuid = { version = "1.1.2", features = ["serde", "v4"] } 84 | lazy_static = "1.4.0" 85 | serde = { version = "1.0", features = ["derive", "rc"] } 86 | serde_json = "1.0" 87 | futures = "0.3.28" 88 | async-trait = { version = "0.1" } 89 | hashring = { version = "0.3.0", optional = true } 90 | bytes = { version = "1.4.0", optional = true } 91 | byteorder = { version = "1.4.3", optional = true } 92 | chrono = { version = "0.4", features = ["serde"], optional = true } 93 | protobuf = { version = "=3.2.0", optional = true } 94 | anyhow = { version = "1.0.71", optional = true } 95 | rand = "0.8.5" 96 | parking_lot = { version = "0.12.1", optional = true } 97 | metrics = { version = "0.21.0", optional = true } 98 | valuable = { version = "0.1", features = ["derive"] } 99 | metrics-exporter-prometheus = { version = "0.12.1", optional = true } 100 | metrics-util = { version = "0.15.0", optional = true } 101 | jwt = { version = "0.16.0", optional = true } 102 | hmac = { version = "0.12.1", optional = true } 103 | sha2 = { version = "0.10.6", optional = true } 104 | 105 | # API dependencies 106 | axum = { version = "0.6.18", features = ["query"], optional = true } 107 | utoipa = { version = "3", features = ["axum_extras", "chrono"], optional = true } 108 | utoipa-swagger-ui = { version = "3", features = ["axum"], optional = true } 109 | 110 | [dev-dependencies] 111 | coerce-macros = { version = "0.2.0" } 112 | bencher = { version = "0.1.5" } 113 | tracing-subscriber = { features = ["json"], version = "0.3.17" } 114 | 115 | [[bench]] 116 | name = "actor_messaging" 117 | harness = false 118 | 119 | [[bench]] 120 | name = "actor_creation" 121 | harness = false 122 | 123 | [package.metadata.docs.rs] 124 | all-features = true -------------------------------------------------------------------------------- /coerce/benches/actor_creation.rs: -------------------------------------------------------------------------------- 1 | use bencher::{benchmark_group, benchmark_main, Bencher}; 2 | use coerce::actor::scheduler::ActorType::Anonymous; 3 | use coerce::actor::system::ActorSystem; 4 | use coerce::actor::{Actor, IntoActorId, LocalActorRef}; 5 | use tokio::runtime::Runtime; 6 | 7 | fn rt() -> Runtime { 8 | tokio::runtime::Builder::new_multi_thread().build().unwrap() 9 | } 10 | 11 | fn create_1000_actors(bench: &mut Bencher) { 12 | let runtime = rt(); 13 | 14 | bench.iter(|| { 15 | runtime.block_on(async { 16 | for _ in 0..1000 { 17 | let _ = actor().await; 18 | } 19 | }) 20 | }) 21 | } 22 | 23 | struct BenchmarkActor; 24 | 25 | impl Actor for BenchmarkActor {} 26 | 27 | async fn actor() -> LocalActorRef { 28 | let system = ActorSystem::new(); 29 | system 30 | .new_actor("actor".into_actor_id(), BenchmarkActor, Anonymous) 31 | .await 32 | .expect("unable to create actor") 33 | } 34 | 35 | benchmark_group!(actor_creation, create_1000_actors); 36 | benchmark_main!(actor_creation); 37 | -------------------------------------------------------------------------------- /coerce/benches/actor_messaging.rs: -------------------------------------------------------------------------------- 1 | //! These benchmarks exist as a way to find performance regressions, not as a demonstration of real world 2 | //! performance 3 | 4 | #[macro_use] 5 | extern crate bencher; 6 | 7 | use bencher::Bencher; 8 | use coerce::actor::context::ActorContext; 9 | use coerce::actor::message::{Handler, Message}; 10 | use coerce::actor::scheduler::ActorType::Anonymous; 11 | use coerce::actor::system::ActorSystem; 12 | use coerce::actor::{Actor, IntoActorId, LocalActorRef, ToActorId}; 13 | use tokio::runtime::Runtime; 14 | 15 | struct BenchmarkActor; 16 | 17 | impl Actor for BenchmarkActor {} 18 | 19 | struct Msg; 20 | 21 | impl Message for Msg { 22 | type Result = (); 23 | } 24 | 25 | #[async_trait::async_trait] 26 | impl Handler for BenchmarkActor { 27 | async fn handle(&mut self, _message: Msg, _ctx: &mut ActorContext) {} 28 | } 29 | 30 | async fn actor_1000_send_and_wait(actor: &LocalActorRef) { 31 | for _ in 0..1000 { 32 | let _ = actor.send(Msg).await.unwrap(); 33 | } 34 | } 35 | 36 | async fn actor_999_notify_1_send_and_wait(actor: &LocalActorRef) { 37 | for _ in 0..999 { 38 | let _ = actor.notify(Msg); 39 | } 40 | 41 | let _ = actor.send(Msg).await.unwrap(); 42 | } 43 | 44 | fn actor_send_1000_benchmark(bench: &mut Bencher) { 45 | let runtime = rt(); 46 | let actor = runtime.block_on(async { actor().await }); 47 | 48 | bench.iter(|| runtime.block_on(actor_1000_send_and_wait(&actor))); 49 | } 50 | 51 | fn actor_notify_1000_benchmark(bench: &mut Bencher) { 52 | let runtime = rt(); 53 | let actor = runtime.block_on(async { actor().await }); 54 | 55 | bench.iter(|| runtime.block_on(actor_999_notify_1_send_and_wait(&actor))) 56 | } 57 | 58 | fn rt() -> Runtime { 59 | tokio::runtime::Builder::new_multi_thread().build().unwrap() 60 | } 61 | 62 | async fn actor() -> LocalActorRef { 63 | let system = ActorSystem::new(); 64 | system 65 | .new_actor("actor".into_actor_id(), BenchmarkActor, Anonymous) 66 | .await 67 | .expect("unable to create actor") 68 | } 69 | 70 | benchmark_group!( 71 | actor_messaging, 72 | actor_send_1000_benchmark, 73 | actor_notify_1000_benchmark 74 | ); 75 | benchmark_main!(actor_messaging); 76 | -------------------------------------------------------------------------------- /coerce/macros/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "coerce-macros" 3 | version = "0.2.0" 4 | authors = ["Leon Hartley "] 5 | edition = "2021" 6 | description = "Utility macros for Coerce" 7 | license = "Apache-2.0" 8 | repository = "https://github.com/leonhartley/coerce-rs" 9 | 10 | [lib] 11 | proc-macro = true 12 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 13 | 14 | [dependencies] 15 | quote = "1" 16 | syn = { version = "1.0", features = ["full"] } 17 | proc-macro2 = "1.0" 18 | 19 | -------------------------------------------------------------------------------- /coerce/macros/src/message/json.rs: -------------------------------------------------------------------------------- 1 | use proc_macro2::{Span, TokenStream}; 2 | use quote::{quote, ToTokens}; 3 | 4 | use crate::get_attribute_type_multiple; 5 | 6 | const RESULT_ATTR: &str = "result"; 7 | 8 | pub(crate) fn expand(ast: &syn::DeriveInput) -> TokenStream { 9 | let item_type = { 10 | match get_attribute_type_multiple(ast, RESULT_ATTR) { 11 | Ok(ty) => match ty.len() { 12 | 1 => ty[0].clone(), 13 | _ => { 14 | return syn::Error::new( 15 | Span::call_site(), 16 | format!( 17 | "#[{}(type)] takes 1 parameters, given {}", 18 | RESULT_ATTR, 19 | ty.len() 20 | ), 21 | ) 22 | .to_compile_error() 23 | } 24 | }, 25 | Err(err) => return err.to_compile_error(), 26 | } 27 | }; 28 | 29 | let name = &ast.ident; 30 | let (impl_generics, ty_generics, where_clause) = ast.generics.split_for_impl(); 31 | 32 | let item_type = item_type 33 | .map(ToTokens::into_token_stream) 34 | .unwrap_or_else(|| quote! { () }); 35 | 36 | quote! { 37 | impl #impl_generics ::coerce::actor::message::Message for #name #ty_generics #where_clause { 38 | type Result = #item_type; 39 | 40 | fn as_bytes(&self) -> Result, coerce::actor::message::MessageWrapErr> { 41 | serde_json::to_vec(&self) 42 | .map_err(|_e| coerce::actor::message::MessageWrapErr::SerializationErr) 43 | } 44 | 45 | fn from_bytes(bytes: Vec) -> Result { 46 | serde_json::from_slice(bytes.as_slice()).map_err(|_e| coerce::actor::message::MessageUnwrapErr::DeserializationErr) 47 | } 48 | 49 | fn read_remote_result(bytes: Vec) -> Result { 50 | serde_json::from_slice(bytes.as_slice()).map_err(|_e| coerce::actor::message::MessageUnwrapErr::DeserializationErr) 51 | } 52 | 53 | fn write_remote_result(res: Self::Result) -> Result, coerce::actor::message::MessageWrapErr> { 54 | serde_json::to_vec(&res).map_err(|_e| coerce::actor::message::MessageWrapErr::SerializationErr) 55 | } 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /coerce/macros/src/message/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod json; 2 | -------------------------------------------------------------------------------- /coerce/macros/src/snapshot/json.rs: -------------------------------------------------------------------------------- 1 | use proc_macro2::TokenStream; 2 | use quote::quote; 3 | 4 | pub(crate) fn expand(ast: &syn::DeriveInput) -> TokenStream { 5 | let name = &ast.ident; 6 | let (impl_generics, ty_generics, where_clause) = ast.generics.split_for_impl(); 7 | 8 | quote! { 9 | impl #impl_generics ::coerce::persistent::journal::snapshot::Snapshot for #name #ty_generics #where_clause { 10 | fn into_remote_envelope(self) -> Result, coerce::actor::message::MessageWrapErr> { 11 | serde_json::to_vec(&self) 12 | .map_err(|_e| coerce::actor::message::MessageWrapErr::SerializationErr) 13 | .map(|bytes| coerce::actor::message::Envelope::Remote(bytes)) 14 | } 15 | 16 | fn from_remote_envelope(bytes: Vec) -> Result { 17 | serde_json::from_slice(bytes.as_slice()).map_err(|_e| coerce::actor::message::MessageUnwrapErr::DeserializationErr) 18 | } 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /coerce/macros/src/snapshot/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod json; 2 | -------------------------------------------------------------------------------- /coerce/src/actor/blocking.rs: -------------------------------------------------------------------------------- 1 | //! Blocking Actor creation and communication APIs 2 | 3 | use crate::actor::lifecycle::Stop; 4 | use crate::actor::message::{ActorMessage, Exec, Handler, Message}; 5 | use crate::actor::metrics::ActorMetrics; 6 | use crate::actor::scheduler::{start_actor, ActorType, RegisterActor}; 7 | use crate::actor::system::ActorSystem; 8 | use crate::actor::{Actor, ActorRefErr, IntoActorId, LocalActorRef}; 9 | use tokio::sync::oneshot; 10 | 11 | impl ActorSystem { 12 | pub fn new_actor_blocking( 13 | &self, 14 | id: I, 15 | actor: A, 16 | actor_type: ActorType, 17 | ) -> Result, ActorRefErr> { 18 | let id = id.into_actor_id(); 19 | let (tx, rx) = oneshot::channel(); 20 | let actor_ref = start_actor( 21 | actor, 22 | id.clone(), 23 | actor_type, 24 | Some(tx), 25 | Some(self.clone()), 26 | None, 27 | self.system_name().clone(), 28 | ); 29 | 30 | if actor_type.is_tracked() { 31 | let _ = self.scheduler().send_blocking(RegisterActor { 32 | id: id.clone(), 33 | actor_ref: actor_ref.clone(), 34 | }); 35 | } 36 | 37 | match rx.blocking_recv() { 38 | Ok(_) => Ok(actor_ref), 39 | Err(_e) => { 40 | error!( 41 | actor_id = id.as_ref(), 42 | actor_type = A::type_name(), 43 | "actor not started", 44 | ); 45 | Err(ActorRefErr::ActorStartFailed) 46 | } 47 | } 48 | } 49 | } 50 | 51 | impl LocalActorRef { 52 | pub fn send_blocking(&self, msg: Msg) -> Result 53 | where 54 | A: Handler, 55 | { 56 | let message_type = msg.name(); 57 | let actor_type = A::type_name(); 58 | 59 | ActorMetrics::incr_messages_sent(A::type_name(), msg.name()); 60 | 61 | let (tx, rx) = oneshot::channel(); 62 | match self 63 | .sender() 64 | .send(Box::new(ActorMessage::new(msg, Some(tx)))) 65 | { 66 | Ok(_) => match rx.blocking_recv() { 67 | Ok(res) => { 68 | trace!( 69 | "recv result (msg_type={msg_type} actor_type={actor_type})", 70 | msg_type = message_type, 71 | actor_type = actor_type 72 | ); 73 | 74 | Ok(res) 75 | } 76 | Err(_e) => Err(ActorRefErr::ResultChannelClosed), 77 | }, 78 | Err(_e) => Err(ActorRefErr::InvalidRef), 79 | } 80 | } 81 | 82 | pub fn stop_blocking(&self) -> Result<(), ActorRefErr> { 83 | let (tx, rx) = oneshot::channel(); 84 | let res = self.notify(Stop(Some(tx))); 85 | if res.is_ok() { 86 | rx.blocking_recv().map_err(|_| ActorRefErr::InvalidRef) 87 | } else { 88 | res 89 | } 90 | } 91 | 92 | pub fn exec_blocking(&self, f: F) -> Result 93 | where 94 | F: (FnMut(&mut A) -> R) + 'static + Send + Sync, 95 | R: 'static + Send + Sync, 96 | { 97 | self.send_blocking(Exec::new(f)) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /coerce/src/actor/event/dispatcher.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::Actor; 2 | 3 | pub struct Dispatcher {} 4 | 5 | #[async_trait] 6 | impl Actor for Dispatcher {} 7 | -------------------------------------------------------------------------------- /coerce/src/actor/event/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::message::Message; 2 | /// 3 | /// Actor events provide the ability to monitor the creation of actors, what types of messages 4 | /// they receive and monitor actors that are stopped. 5 | /// 6 | /// This provides the ability to create pluggable actor monitoring without the need to manually 7 | /// implement tracing inside each actor, it can be enabled at system level. 8 | /// 9 | /// Applications such as dashboards can be created that can receive events as they happen, rather 10 | /// than having to poll. 11 | /// 12 | use crate::actor::{ActorId, BoxedActorRef}; 13 | 14 | pub mod dispatcher; 15 | 16 | pub struct ActorStarted { 17 | timestamp: u64, 18 | actor_id: ActorId, 19 | actor_ref: BoxedActorRef, 20 | } 21 | 22 | pub struct ActorStopped { 23 | timestamp: u64, 24 | actor_ref: BoxedActorRef, 25 | } 26 | 27 | pub struct ActorReceived { 28 | timestamp: u64, 29 | actor_ref: BoxedActorRef, 30 | message_type: &'static str, 31 | } 32 | 33 | pub enum ActorEvent { 34 | Started(ActorStarted), 35 | Stopped(ActorStopped), 36 | Received(ActorReceived), 37 | } 38 | -------------------------------------------------------------------------------- /coerce/src/actor/metrics/mod.rs: -------------------------------------------------------------------------------- 1 | //! Actor Metrics 2 | use std::time::Duration; 3 | 4 | pub const METRIC_ACTOR_CREATED: &str = "coerce_actor_created"; 5 | pub const METRIC_ACTOR_STOPPED: &str = "coerce_actor_stopped"; 6 | pub const METRIC_ACTOR_MESSAGES_SENT_TOTAL: &str = "coerce_actor_msg_sent_total"; 7 | pub const METRIC_ACTOR_MESSAGE_WAIT_TIME: &str = "coerce_actor_msg_wait_time"; 8 | pub const METRIC_ACTOR_MESSAGE_PROCESSING_TIME: &str = "coerce_actor_msg_processing_time"; 9 | pub const METRIC_ACTOR_MESSAGES_PROCESSED_TOTAL: &str = "coerce_actor_msg_processed_total"; 10 | 11 | pub const LABEL_ACTOR_TYPE: &str = "actor_type"; 12 | pub const LABEL_MESSAGE_TYPE: &str = "msg_type"; 13 | 14 | pub struct ActorMetrics; 15 | 16 | impl ActorMetrics { 17 | #[inline] 18 | pub fn incr_actor_created(actor_type: &'static str) { 19 | #[cfg(feature = "metrics")] 20 | increment_counter!(METRIC_ACTOR_CREATED, 21 | LABEL_ACTOR_TYPE => actor_type, 22 | ); 23 | } 24 | 25 | #[inline] 26 | pub fn incr_actor_stopped(actor_type: &'static str) { 27 | #[cfg(feature = "metrics")] 28 | increment_counter!(METRIC_ACTOR_STOPPED, 29 | LABEL_ACTOR_TYPE => actor_type, 30 | ); 31 | } 32 | 33 | #[inline] 34 | pub fn incr_messages_sent(actor_type: &'static str, msg_type: &'static str) { 35 | #[cfg(feature = "metrics")] 36 | increment_counter!(METRIC_ACTOR_MESSAGES_SENT_TOTAL, 37 | LABEL_ACTOR_TYPE => actor_type, 38 | LABEL_MESSAGE_TYPE => msg_type 39 | ); 40 | } 41 | 42 | #[inline] 43 | pub fn incr_messages_processed( 44 | actor_type: &'static str, 45 | msg_type: &'static str, 46 | wait_time: Duration, 47 | processing_time: Duration, 48 | ) { 49 | #[cfg(feature = "metrics")] 50 | { 51 | increment_counter!(METRIC_ACTOR_MESSAGES_PROCESSED_TOTAL, 52 | LABEL_ACTOR_TYPE => actor_type, 53 | LABEL_MESSAGE_TYPE => msg_type 54 | ); 55 | 56 | histogram!(METRIC_ACTOR_MESSAGE_WAIT_TIME, 57 | wait_time, 58 | LABEL_ACTOR_TYPE => actor_type, 59 | LABEL_MESSAGE_TYPE => msg_type); 60 | 61 | histogram!(METRIC_ACTOR_MESSAGE_PROCESSING_TIME, 62 | processing_time, 63 | LABEL_ACTOR_TYPE => actor_type, 64 | LABEL_MESSAGE_TYPE => msg_type) 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /coerce/src/actor/scheduler/timer.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::message::{Handler, Message}; 2 | use crate::actor::{Actor, LocalActorRef}; 3 | use std::ops::Add; 4 | use tracing::trace; 5 | 6 | use std::time::Duration; 7 | use tokio::sync::oneshot; 8 | use tokio::time; 9 | use tokio::time::Instant; 10 | use uuid::Uuid; 11 | 12 | pub trait TimerTick: Message {} 13 | 14 | enum TimerMode { 15 | Notify, 16 | Send, 17 | } 18 | 19 | pub struct Timer { 20 | stop: oneshot::Sender, 21 | } 22 | 23 | impl Timer { 24 | pub fn start_immediately( 25 | actor: LocalActorRef, 26 | tick: Duration, 27 | msg: T, 28 | ) -> Timer 29 | where 30 | A: 'static + Handler + Sync + Send, 31 | T: 'static + Clone + Sync + Send, 32 | T::Result: 'static + Sync + Send, 33 | { 34 | let (stop, stop_rx) = oneshot::channel(); 35 | tokio::spawn(timer_loop(tick, msg, actor, stop_rx, true, TimerMode::Send)); 36 | 37 | Timer { stop } 38 | } 39 | 40 | pub fn start(actor: LocalActorRef, tick: Duration, msg: T) -> Timer 41 | where 42 | A: 'static + Handler + Sync + Send, 43 | T: 'static + Clone + Sync + Send, 44 | T::Result: 'static + Sync + Send, 45 | { 46 | let (stop, stop_rx) = oneshot::channel(); 47 | tokio::spawn(timer_loop( 48 | tick, 49 | msg, 50 | actor, 51 | stop_rx, 52 | false, 53 | TimerMode::Send, 54 | )); 55 | 56 | Timer { stop } 57 | } 58 | 59 | pub fn stop(self) -> bool { 60 | if self.stop.send(true).is_ok() { 61 | true 62 | } else { 63 | false 64 | } 65 | } 66 | } 67 | 68 | async fn timer_loop( 69 | tick: Duration, 70 | msg: T, 71 | actor: LocalActorRef, 72 | mut stop_rx: oneshot::Receiver, 73 | tick_immediately: bool, 74 | mode: TimerMode, 75 | ) where 76 | A: Handler, 77 | T: 'static + Clone + Sync + Send, 78 | { 79 | let start = if tick_immediately { 80 | Instant::now() 81 | } else { 82 | Instant::now().add(tick) 83 | }; 84 | 85 | let mut interval = time::interval_at(start, tick); 86 | let timer_id = Uuid::new_v4(); 87 | 88 | interval.tick().await; 89 | 90 | trace!("{} - timer starting", &timer_id); 91 | 92 | loop { 93 | if stop_rx.try_recv().is_ok() { 94 | break; 95 | } 96 | 97 | trace!("{} - timer tick", &timer_id); 98 | 99 | let now = Instant::now(); 100 | 101 | match mode { 102 | TimerMode::Notify => { 103 | if actor.notify(msg.clone()).is_err() { 104 | break; 105 | } 106 | } 107 | TimerMode::Send => { 108 | if actor.send(msg.clone()).await.is_err() { 109 | break; 110 | } 111 | 112 | interval.reset(); 113 | } 114 | } 115 | 116 | trace!( 117 | "{} - tick res received in {}ms", 118 | &timer_id, 119 | now.elapsed().as_millis() 120 | ); 121 | interval.tick().await; 122 | } 123 | 124 | trace!("{} - timer finished", timer_id); 125 | } 126 | -------------------------------------------------------------------------------- /coerce/src/actor/system/builder.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::scheduler::ActorScheduler; 2 | use crate::actor::system::{ActorSystem, ActorSystemCore}; 3 | use std::sync::atomic::{AtomicBool, AtomicU64}; 4 | use std::sync::Arc; 5 | use uuid::Uuid; 6 | 7 | #[cfg(feature = "persistence")] 8 | use crate::persistent::{provider::StorageProvider, Persistence}; 9 | 10 | #[derive(Default)] 11 | pub struct ActorSystemBuilder { 12 | system_id: Option, 13 | system_name: Option, 14 | 15 | #[cfg(feature = "persistence")] 16 | persistence: Option>, 17 | } 18 | 19 | impl ActorSystemBuilder { 20 | pub fn system_name(mut self, name: impl ToString) -> Self { 21 | self.system_name = Some(name.to_string()); 22 | self 23 | } 24 | 25 | #[cfg(feature = "persistence")] 26 | pub fn with_persistence(mut self, provider: S) -> Self { 27 | self.persistence = Some(Persistence::from(provider).into()); 28 | self 29 | } 30 | 31 | pub fn build(self) -> ActorSystem { 32 | let system_id = self.system_id.unwrap_or_else(|| Uuid::new_v4()); 33 | let system_name: Arc = self.system_name.map_or_else( 34 | || { 35 | std::env::var("COERCE_ACTOR_SYSTEM").map_or_else( 36 | |_e| format!("{}", system_id).into(), 37 | |v| v.to_string().into(), 38 | ) 39 | }, 40 | |s| s.into(), 41 | ); 42 | 43 | let scheduler = ActorScheduler::new(system_id, system_name.clone()); 44 | ActorSystem { 45 | core: Arc::new(ActorSystemCore { 46 | system_id, 47 | system_name, 48 | scheduler, 49 | is_terminated: Arc::new(AtomicBool::new(false)), 50 | context_counter: Arc::new(AtomicU64::new(1)), 51 | 52 | #[cfg(feature = "persistence")] 53 | persistence: self.persistence, 54 | 55 | #[cfg(feature = "remote")] 56 | remote: None, 57 | }), 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /coerce/src/actor/watch/mod.rs: -------------------------------------------------------------------------------- 1 | //! Actor watching allows one actor to watch another, ensuring that when the subject stops, the 2 | //! the watching actor will be notified immediately. 3 | //! 4 | 5 | use crate::actor::context::ActorContext; 6 | use crate::actor::message::{Handler, Message}; 7 | use crate::actor::{Actor, ActorId, BoxedActorRef, LocalActorRef, Receiver}; 8 | 9 | pub mod watchers; 10 | 11 | #[async_trait] 12 | pub trait ActorWatch { 13 | fn watch(&self, actor: &LocalActorRef, ctx: &ActorContext); 14 | 15 | fn unwatch(&self, actor: &LocalActorRef, ctx: &ActorContext); 16 | } 17 | 18 | impl ActorWatch for A 19 | where 20 | A: Handler, 21 | { 22 | fn watch(&self, actor: &LocalActorRef, ctx: &ActorContext) { 23 | let terminated_receiver = Receiver::::from(self.actor_ref(ctx)); 24 | let _ = actor.notify(Watch::from(terminated_receiver)); 25 | } 26 | 27 | fn unwatch(&self, actor: &LocalActorRef, ctx: &ActorContext) { 28 | let _ = actor.notify(Unwatch::from(ctx.id().clone())); 29 | } 30 | } 31 | 32 | #[async_trait] 33 | impl Handler for A { 34 | async fn handle(&mut self, message: Watch, ctx: &mut ActorContext) { 35 | ctx.watchers_mut().add_watcher(message.receiver); 36 | } 37 | } 38 | 39 | #[async_trait] 40 | impl Handler for A { 41 | async fn handle(&mut self, message: Unwatch, ctx: &mut ActorContext) { 42 | ctx.watchers_mut().remove_watcher(message.watcher_id); 43 | } 44 | } 45 | 46 | pub struct Watch { 47 | receiver: Receiver, 48 | } 49 | 50 | impl From> for Watch { 51 | fn from(value: Receiver) -> Self { 52 | Self { receiver: value } 53 | } 54 | } 55 | 56 | pub struct Unwatch { 57 | watcher_id: ActorId, 58 | } 59 | 60 | impl From for Unwatch { 61 | fn from(value: ActorId) -> Self { 62 | Self { watcher_id: value } 63 | } 64 | } 65 | 66 | impl Message for Watch { 67 | type Result = (); 68 | } 69 | 70 | impl Message for Unwatch { 71 | type Result = (); 72 | } 73 | 74 | #[derive(Clone)] 75 | pub struct ActorTerminated { 76 | actor_ref: BoxedActorRef, 77 | } 78 | 79 | impl ActorTerminated { 80 | pub fn actor_ref(&self) -> &BoxedActorRef { 81 | &self.actor_ref 82 | } 83 | } 84 | 85 | impl From for ActorTerminated { 86 | fn from(value: BoxedActorRef) -> Self { 87 | Self { actor_ref: value } 88 | } 89 | } 90 | 91 | impl Message for ActorTerminated { 92 | type Result = (); 93 | } 94 | -------------------------------------------------------------------------------- /coerce/src/actor/watch/watchers.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::watch::ActorTerminated; 2 | use crate::actor::{ActorId, BoxedActorRef, Receiver}; 3 | use std::collections::HashMap; 4 | 5 | #[derive(Default)] 6 | pub struct Watchers { 7 | watchers: HashMap>, 8 | } 9 | 10 | impl Watchers { 11 | pub fn iter(&self) -> impl Iterator> { 12 | self.watchers.values() 13 | } 14 | 15 | pub fn add_watcher(&mut self, watcher: Receiver) { 16 | self.watchers.insert(watcher.actor_id().to_owned(), watcher); 17 | } 18 | 19 | pub fn remove_watcher(&mut self, actor_id: ActorId) { 20 | self.watchers.remove(&actor_id); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /coerce/src/persistent/batch.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::actor::message::Message; 3 | use crate::actor::Actor; 4 | use crate::persistent::storage::JournalEntry; 5 | use crate::persistent::types::JournalTypes; 6 | use crate::persistent::{PersistentActor, Recover}; 7 | use std::sync::Arc; 8 | 9 | #[derive(Clone)] 10 | pub struct EventBatch { 11 | journal_types: Arc>, 12 | entries: Vec, 13 | } 14 | 15 | #[derive(Clone)] 16 | pub struct BatchedEntry { 17 | pub payload_type: Arc, 18 | pub bytes: Arc>, 19 | } 20 | 21 | impl EventBatch { 22 | pub fn create(ctx: &ActorContext) -> Self { 23 | let journal = ctx.persistence().journal::(); 24 | let journal_types = journal.get_types(); 25 | 26 | Self { 27 | journal_types, 28 | entries: vec![], 29 | } 30 | } 31 | 32 | pub fn message(&mut self, message: M) -> &mut Self 33 | where 34 | A: Recover, 35 | { 36 | let payload_type = self.journal_types.message_type_mapping::().unwrap(); 37 | let entry = BatchedEntry { 38 | payload_type, 39 | bytes: message.as_bytes().unwrap().into(), 40 | }; 41 | 42 | self.entries.push(entry); 43 | self 44 | } 45 | 46 | pub fn entries(&self) -> &Vec { 47 | &self.entries 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /coerce/src/persistent/context.rs: -------------------------------------------------------------------------------- 1 | use crate::persistent::journal::provider::StorageProviderRef; 2 | use crate::persistent::journal::Journal; 3 | use crate::persistent::PersistentActor; 4 | use std::any::Any; 5 | 6 | pub struct ActorPersistence { 7 | storage_provider: StorageProviderRef, 8 | journal: Option, 9 | } 10 | 11 | type BoxedJournal = Box; 12 | 13 | impl ActorPersistence { 14 | pub fn new(storage_provider: StorageProviderRef) -> Self { 15 | Self { 16 | storage_provider, 17 | journal: None, 18 | } 19 | } 20 | 21 | pub fn init_journal(&mut self, persistence_id: String) -> &mut Journal { 22 | let storage = self 23 | .storage_provider 24 | .journal_storage() 25 | .expect("journal storage not configured"); 26 | 27 | let journal = Journal::::new(persistence_id, storage); 28 | self.journal = Some(Box::new(journal)); 29 | self.journal.as_mut().unwrap().downcast_mut().unwrap() 30 | } 31 | 32 | pub fn journal(&self) -> &Journal { 33 | self.journal 34 | .as_ref() 35 | .expect("journal not initialised") 36 | .downcast_ref() 37 | .unwrap() 38 | } 39 | 40 | pub fn journal_mut(&mut self) -> &mut Journal { 41 | self.journal 42 | .as_mut() 43 | .expect("journal not initialised") 44 | .downcast_mut() 45 | .unwrap() 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /coerce/src/persistent/failure.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | 3 | use std::fmt; 4 | use std::fmt::{Display, Formatter}; 5 | use std::time::Duration; 6 | 7 | #[derive(Copy, Clone)] 8 | pub enum RecoveryFailurePolicy { 9 | Retry(Retry), 10 | StopActor, 11 | Panic, 12 | } 13 | 14 | #[derive(Copy, Clone)] 15 | pub enum PersistFailurePolicy { 16 | Retry(Retry), 17 | ReturnErr, 18 | StopActor, 19 | Panic, 20 | } 21 | 22 | #[derive(Copy, Clone)] 23 | pub enum Retry { 24 | UntilSuccess { 25 | delay: Option, 26 | }, 27 | MaxAttempts { 28 | max_attempts: usize, 29 | delay: Option, 30 | }, 31 | } 32 | 33 | pub(crate) async fn should_retry(ctx: &mut ActorContext, attempts: &usize, retry: Retry) -> bool { 34 | match retry { 35 | Retry::UntilSuccess { delay } => { 36 | if let Some(delay) = delay { 37 | tokio::time::sleep(delay).await; 38 | } 39 | } 40 | 41 | Retry::MaxAttempts { 42 | max_attempts, 43 | delay, 44 | } => { 45 | if attempts >= &max_attempts { 46 | ctx.stop(None); 47 | return false; 48 | } 49 | 50 | if let Some(delay) = delay { 51 | tokio::time::sleep(delay).await; 52 | } 53 | } 54 | } 55 | return true; 56 | } 57 | 58 | impl Display for RecoveryFailurePolicy { 59 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 60 | match &self { 61 | RecoveryFailurePolicy::StopActor => write!(f, "StopActor"), 62 | RecoveryFailurePolicy::Retry(r) => write!(f, "Retry({})", r), 63 | RecoveryFailurePolicy::Panic => write!(f, "Panic"), 64 | } 65 | } 66 | } 67 | 68 | impl Display for PersistFailurePolicy { 69 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 70 | match &self { 71 | PersistFailurePolicy::StopActor => write!(f, "StopActor"), 72 | PersistFailurePolicy::Retry(r) => write!(f, "Retry({})", r), 73 | PersistFailurePolicy::Panic => write!(f, "Panic"), 74 | PersistFailurePolicy::ReturnErr => write!(f, "ReturnErr"), 75 | } 76 | } 77 | } 78 | 79 | impl Display for Retry { 80 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 81 | match &self { 82 | Retry::UntilSuccess { delay } => { 83 | if let Some(delay) = delay.as_ref() { 84 | write!(f, "UntilSuccess(delay={} millis)", delay.as_millis()) 85 | } else { 86 | write!(f, "UntilSuccess(delay=None)") 87 | } 88 | } 89 | 90 | Retry::MaxAttempts { 91 | max_attempts, 92 | delay, 93 | } => { 94 | if let Some(delay) = delay.as_ref() { 95 | write!( 96 | f, 97 | "MaxAttempts(max_attempts={}, delay={} millis)", 98 | max_attempts, 99 | delay.as_millis() 100 | ) 101 | } else { 102 | write!(f, "MaxAttempts(max_attempts={}, delay=None)", max_attempts) 103 | } 104 | } 105 | } 106 | } 107 | } 108 | 109 | impl Default for RecoveryFailurePolicy { 110 | fn default() -> Self { 111 | RecoveryFailurePolicy::Retry(Retry::UntilSuccess { 112 | delay: Some(Duration::from_millis(500)), 113 | }) 114 | } 115 | } 116 | 117 | impl Default for PersistFailurePolicy { 118 | fn default() -> Self { 119 | PersistFailurePolicy::Retry(Retry::UntilSuccess { 120 | delay: Some(Duration::from_millis(500)), 121 | }) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /coerce/src/persistent/inspect/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /coerce/src/persistent/journal/proto/mod.rs: -------------------------------------------------------------------------------- 1 | // @generated 2 | 3 | pub mod journal; 4 | -------------------------------------------------------------------------------- /coerce/src/persistent/journal/snapshot.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::message::{Envelope, EnvelopeType, MessageUnwrapErr, MessageWrapErr}; 2 | 3 | pub struct JournalPayload { 4 | pub message_type: String, 5 | pub sequence: i64, 6 | pub bytes: Vec, 7 | } 8 | 9 | pub trait Snapshot: 'static + Sync + Send + Sized { 10 | fn into_envelope(self, envelope_type: EnvelopeType) -> Result, MessageWrapErr> { 11 | match envelope_type { 12 | EnvelopeType::Local => Ok(Envelope::Local(self)), 13 | EnvelopeType::Remote => self.into_remote_envelope(), 14 | } 15 | } 16 | 17 | fn into_remote_envelope(self) -> Result, MessageWrapErr> { 18 | Err(MessageWrapErr::NotTransmittable) 19 | } 20 | 21 | fn from_envelope(envelope: Envelope) -> Result { 22 | match envelope { 23 | Envelope::Local(msg) => Ok(msg), 24 | Envelope::Remote(bytes) => Self::from_remote_envelope(bytes), 25 | } 26 | } 27 | 28 | fn from_remote_envelope(_: Vec) -> Result { 29 | Err(MessageUnwrapErr::NotTransmittable) 30 | } 31 | 32 | fn type_name() -> &'static str 33 | where 34 | Self: Sized, 35 | { 36 | std::any::type_name::() 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /coerce/src/persistent/journal/storage.rs: -------------------------------------------------------------------------------- 1 | use crate::persistent::journal::proto::journal::JournalEntry as ProtoJournalEntry; 2 | use anyhow::Result; 3 | use protobuf::Message; 4 | use std::sync::Arc; 5 | 6 | #[derive(Clone, Debug)] 7 | pub struct JournalEntry { 8 | pub sequence: i64, 9 | pub payload_type: Arc, 10 | pub bytes: Arc>, 11 | } 12 | 13 | #[async_trait] 14 | pub trait JournalStorage: Send + Sync { 15 | // TODO: add the ability to limit the maximum size of snapshots, 16 | // loading these back could cause an unexpected OOM 17 | async fn write_snapshot(&self, persistence_id: &str, entry: JournalEntry) -> Result<()>; 18 | 19 | async fn write_message(&self, persistence_id: &str, entry: JournalEntry) -> Result<()>; 20 | 21 | async fn write_message_batch( 22 | &self, 23 | persistence_id: &str, 24 | entries: Vec, 25 | ) -> Result<()>; 26 | 27 | async fn read_latest_snapshot(&self, persistence_id: &str) -> Result>; 28 | 29 | // TODO: add the ability to stream the messages, rather than load all up front, 30 | // if the actor has a very large journal, this could cause an unexpected OOM. 31 | // payload size limits should also be applied. 32 | async fn read_latest_messages( 33 | &self, 34 | persistence_id: &str, 35 | from_sequence: i64, 36 | ) -> Result>>; 37 | 38 | async fn read_message( 39 | &self, 40 | persistence_id: &str, 41 | sequence_id: i64, 42 | ) -> Result>; 43 | 44 | async fn read_messages( 45 | &self, 46 | persistence_id: &str, 47 | from_sequence: i64, 48 | to_sequence: i64, 49 | ) -> Result>>; 50 | 51 | async fn delete_messages_to(&self, persistence_id: &str, to_sequence: i64) -> Result<()>; 52 | 53 | async fn delete_all(&self, persistence_id: &str) -> Result<()>; 54 | } 55 | 56 | pub type JournalStorageRef = Arc; 57 | 58 | impl JournalEntry { 59 | pub fn read_from_slice(data: &[u8]) -> Option { 60 | let journal_entry = ProtoJournalEntry::parse_from_bytes(data); 61 | if let Ok(journal_entry) = journal_entry { 62 | Some(JournalEntry { 63 | sequence: journal_entry.sequence, 64 | payload_type: journal_entry.payload_type.into(), 65 | bytes: Arc::new(journal_entry.bytes), 66 | }) 67 | } else { 68 | None 69 | } 70 | } 71 | 72 | pub fn read_from_bytes(data: Vec) -> Option { 73 | Self::read_from_slice(data.as_slice()) 74 | } 75 | 76 | pub fn write_to_bytes(&self) -> Option> { 77 | let journal_entry = self; 78 | let proto = ProtoJournalEntry { 79 | sequence: journal_entry.sequence, 80 | payload_type: journal_entry.payload_type.to_string(), 81 | bytes: journal_entry.bytes.as_ref().clone(), 82 | ..Default::default() 83 | }; 84 | 85 | let bytes = proto.write_to_bytes(); 86 | if let Ok(bytes) = bytes { 87 | Some(bytes) 88 | } else { 89 | None 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /coerce/src/persistent/journal/types.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::message::Message; 2 | use crate::persistent::journal::snapshot::Snapshot; 3 | use crate::persistent::journal::{ 4 | MessageRecoveryHandler, RecoveryHandlerRef, SnapshotRecoveryHandler, 5 | }; 6 | use crate::persistent::{PersistentActor, Recover, RecoverSnapshot}; 7 | use std::any::Any; 8 | use std::any::TypeId; 9 | use std::collections::HashMap; 10 | use std::sync::Arc; 11 | 12 | lazy_static! { 13 | static ref JOURNAL_TYPE_CACHE: parking_lot::Mutex>> = 14 | parking_lot::Mutex::new(HashMap::new()); 15 | } 16 | 17 | pub struct JournalTypes { 18 | message_type_map: HashMap>, 19 | snapshot_type_map: HashMap>, 20 | recoverable_messages: HashMap>, 21 | recoverable_snapshots: HashMap>, 22 | } 23 | 24 | impl Default for JournalTypes { 25 | fn default() -> Self { 26 | let message_type_map = HashMap::new(); 27 | let snapshot_type_map = HashMap::new(); 28 | let recoverable_messages = HashMap::new(); 29 | let recoverable_snapshots = HashMap::new(); 30 | JournalTypes { 31 | message_type_map, 32 | snapshot_type_map, 33 | recoverable_messages, 34 | recoverable_snapshots, 35 | } 36 | } 37 | } 38 | 39 | impl JournalTypes { 40 | pub fn message(&mut self, identifier: &str) -> &mut Self 41 | where 42 | A: Recover, 43 | { 44 | self.recoverable_messages.insert( 45 | identifier.to_string(), 46 | Arc::new(MessageRecoveryHandler::new()), 47 | ); 48 | 49 | self.message_type_map 50 | .insert(TypeId::of::(), identifier.into()); 51 | 52 | self 53 | } 54 | 55 | pub fn snapshot(&mut self, identifier: &str) -> &mut Self 56 | where 57 | A: RecoverSnapshot, 58 | { 59 | self.recoverable_snapshots.insert( 60 | identifier.to_string(), 61 | Arc::new(SnapshotRecoveryHandler::new()), 62 | ); 63 | 64 | self.snapshot_type_map 65 | .insert(TypeId::of::(), identifier.into()); 66 | 67 | self 68 | } 69 | 70 | pub fn snapshot_type_mapping(&self) -> Option> { 71 | self.snapshot_type_map.get(&TypeId::of::()).cloned() 72 | } 73 | 74 | pub fn message_type_mapping(&self) -> Option> { 75 | self.message_type_map.get(&TypeId::of::()).cloned() 76 | } 77 | 78 | pub fn recoverable_snapshots(&self) -> &HashMap> { 79 | &self.recoverable_snapshots 80 | } 81 | 82 | pub fn recoverable_messages(&self) -> &HashMap> { 83 | &self.recoverable_messages 84 | } 85 | } 86 | 87 | pub(crate) fn init_journal_types() -> Arc> { 88 | let actor_type_id = TypeId::of::(); 89 | if let Some(types) = get_cached_types(&actor_type_id) { 90 | return types; 91 | } 92 | 93 | let mut types = JournalTypes::default(); 94 | A::configure(&mut types); 95 | 96 | let types = Arc::new(types); 97 | JOURNAL_TYPE_CACHE 98 | .lock() 99 | .insert(actor_type_id, types.clone()); 100 | 101 | types 102 | } 103 | 104 | fn get_cached_types(actor_type_id: &TypeId) -> Option>> { 105 | if let Some(journal_types) = JOURNAL_TYPE_CACHE.lock().get(&actor_type_id) { 106 | Some(journal_types.clone().downcast().unwrap()) 107 | } else { 108 | None 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /coerce/src/persistent/mod.rs: -------------------------------------------------------------------------------- 1 | //! Coerce Actor Persistence 2 | 3 | pub mod actor; 4 | pub mod batch; 5 | pub mod context; 6 | pub mod failure; 7 | pub mod inspect; 8 | pub mod journal; 9 | pub mod recovery; 10 | 11 | pub use actor::*; 12 | pub use failure::*; 13 | pub use journal::*; 14 | pub use recovery::*; 15 | 16 | use std::any::TypeId; 17 | use std::collections::HashMap; 18 | 19 | use crate::persistent::journal::provider::{StorageProvider, StorageProviderRef}; 20 | use std::sync::Arc; 21 | 22 | #[derive(Clone)] 23 | pub struct Persistence { 24 | default_provider: StorageProviderRef, 25 | actor_type_specific_providers: HashMap, 26 | } 27 | 28 | impl From for Persistence { 29 | fn from(value: S) -> Self { 30 | Persistence::from(value) 31 | } 32 | } 33 | 34 | impl Persistence { 35 | pub fn from(provider: S) -> Persistence { 36 | let default_provider = Arc::new(provider); 37 | Persistence { 38 | default_provider, 39 | actor_type_specific_providers: HashMap::new(), 40 | } 41 | } 42 | 43 | pub fn actor_provider(mut self, provider: S) -> Self { 44 | let actor_type = TypeId::of::(); 45 | self.actor_type_specific_providers 46 | .insert(actor_type, Arc::new(provider)); 47 | self 48 | } 49 | 50 | pub fn provider(&self, actor_type_id: TypeId) -> StorageProviderRef { 51 | self.actor_type_specific_providers 52 | .get(&actor_type_id) 53 | .map_or_else(|| self.default_provider.clone(), |s| s.clone()) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /coerce/src/persistent/recovery.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::persistent::failure::{should_retry, RecoveryFailurePolicy}; 3 | use crate::persistent::journal::{RecoveredPayload, RecoveryErr}; 4 | use crate::persistent::PersistentActor; 5 | 6 | #[async_trait] 7 | pub trait ActorRecovery: PersistentActor { 8 | async fn recover_journal( 9 | &mut self, 10 | persistence_key: Option, 11 | ctx: &mut ActorContext, 12 | ) -> Recovery; 13 | } 14 | 15 | pub enum Recovery { 16 | Recovered(RecoveredJournal), 17 | Disabled, 18 | Failed, 19 | } 20 | 21 | #[async_trait] 22 | impl ActorRecovery for A { 23 | async fn recover_journal( 24 | &mut self, 25 | persistence_key: Option, 26 | ctx: &mut ActorContext, 27 | ) -> Recovery { 28 | let mut journal = None; 29 | let mut attempts = 1; 30 | 31 | let persistence_key = persistence_key.unwrap_or_else(|| self.persistence_key(ctx)); 32 | 33 | loop { 34 | match load_journal::(persistence_key.clone(), ctx).await { 35 | Ok(loaded_journal) => { 36 | journal = Some(loaded_journal); 37 | break; 38 | } 39 | 40 | Err(e) => { 41 | let policy = self.recovery_failure_policy(); 42 | 43 | error!( 44 | "persistent actor (actor_id={actor_id}, persistence_key={persistence_key}) failed to recover - {error}, attempt={attempt}, failure_policy={failure_policy}", 45 | actor_id = ctx.id(), 46 | persistence_key = &persistence_key, 47 | error = &e, 48 | attempt = attempts, 49 | failure_policy = &policy 50 | ); 51 | 52 | self.on_recovery_err(e, ctx).await; 53 | 54 | match policy { 55 | RecoveryFailurePolicy::StopActor => { 56 | ctx.stop(None); 57 | return Recovery::Failed; 58 | } 59 | 60 | RecoveryFailurePolicy::Retry(retry_policy) => { 61 | if !should_retry(ctx, &attempts, retry_policy).await { 62 | return Recovery::Failed; 63 | } 64 | } 65 | 66 | RecoveryFailurePolicy::Panic => panic!("Persistence failure"), 67 | } 68 | } 69 | } 70 | 71 | attempts += 1; 72 | } 73 | 74 | let journal = journal.expect("no journal loaded"); 75 | Recovery::Recovered(journal) 76 | } 77 | } 78 | 79 | pub struct RecoveredJournal { 80 | pub snapshot: Option>, 81 | pub messages: Option>>, 82 | } 83 | 84 | async fn load_journal( 85 | persistence_key: String, 86 | ctx: &mut ActorContext, 87 | ) -> Result, RecoveryErr> { 88 | let journal = ctx.persistence_mut().init_journal::(persistence_key); 89 | 90 | let snapshot = journal 91 | .recover_snapshot() 92 | .await 93 | .map_err(RecoveryErr::Snapshot)?; 94 | 95 | let messages = journal 96 | .recover_messages() 97 | .await 98 | .map_err(RecoveryErr::Messages)?; 99 | 100 | Ok(RecoveredJournal { snapshot, messages }) 101 | } 102 | -------------------------------------------------------------------------------- /coerce/src/protocol/persistent/journal.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package coerce.persistent.journal; 4 | 5 | message JournalEntry { 6 | int64 sequence = 1; 7 | 8 | string payload_type = 2; 9 | 10 | bytes bytes = 3; 11 | } 12 | -------------------------------------------------------------------------------- /coerce/src/protocol/sharding.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package coerce.sharding; 4 | 5 | message AllocateShard { 6 | uint32 shard_id = 1; 7 | 8 | bool rebalancing = 2; 9 | } 10 | 11 | message RemoteShard { 12 | uint32 shard_id = 1; 13 | 14 | uint64 node_id = 2; 15 | } 16 | 17 | message ShardAllocated { 18 | RemoteShard shard = 1; 19 | } 20 | 21 | message ShardReallocating { 22 | uint32 shard_id = 1; 23 | } 24 | 25 | message StopShard { 26 | uint32 shard_id = 1; 27 | 28 | uint64 origin_node_id = 2; 29 | 30 | string request_id = 3; 31 | } 32 | 33 | message ShardStopped { 34 | uint32 shard_id = 1; 35 | 36 | uint64 origin_node_id = 2; 37 | 38 | bool is_successful = 3; 39 | } 40 | 41 | message AllocateShardResult { 42 | enum Type { 43 | ALLOCATED = 0; 44 | ALREADY_ALLOCATED = 1; 45 | NOT_ALLOCATED = 2; 46 | ERR = 3; 47 | } 48 | 49 | enum AllocateShardErr { 50 | UNKNOWN = 0; 51 | PERSISTENCE = 1; 52 | } 53 | 54 | Type result_type = 1; 55 | 56 | RemoteShard allocation = 2; 57 | 58 | AllocateShardErr err = 3; 59 | } 60 | 61 | message RemoteEntityRequest { 62 | message Recipe { 63 | bytes recipe = 1; 64 | } 65 | 66 | string request_id = 1; 67 | 68 | string actor_id = 2; 69 | 70 | string message_type = 3; 71 | 72 | bytes message = 4; 73 | 74 | Recipe recipe = 5; 75 | 76 | uint64 origin_node = 6; 77 | } 78 | 79 | enum EntityState { 80 | IDLE = 0; 81 | ACTIVE = 1; 82 | PASSIVATED = 2; 83 | } 84 | 85 | message StartEntity { 86 | string actor_id = 1; 87 | 88 | bytes recipe = 2; 89 | } 90 | 91 | message PassivateEntity { 92 | string actor_id = 1; 93 | } 94 | 95 | message RemoveEntity { 96 | string actor_id = 1; 97 | } 98 | 99 | message ShardStateSnapshot { 100 | message Entity { 101 | string actor_id = 1; 102 | 103 | bytes recipe = 2; 104 | 105 | EntityState state = 3; 106 | } 107 | 108 | uint32 shard_id = 1; 109 | 110 | uint64 node_id = 2; 111 | 112 | repeated Entity entities = 3; 113 | } 114 | 115 | enum ShardHostStatus { 116 | UNKNOWN = 0; 117 | STARTING = 1; 118 | READY = 2; 119 | UNAVAILABLE = 3; 120 | } 121 | 122 | message GetShardingStats { 123 | 124 | } 125 | 126 | message NodeStats { 127 | uint64 node_id = 1; 128 | 129 | uint64 shard_count = 2; 130 | 131 | ShardHostStatus status = 3; 132 | } 133 | 134 | message ShardingStats { 135 | string entity_type = 1; 136 | 137 | uint64 total_shards = 2; 138 | 139 | repeated RemoteShard shards = 3; 140 | 141 | repeated NodeStats nodes = 4; 142 | } 143 | 144 | message GetShardStats { 145 | 146 | } 147 | 148 | message ShardStats { 149 | uint32 shard_id = 1; 150 | 151 | uint64 node_id = 2; 152 | 153 | repeated string entities = 3; 154 | } -------------------------------------------------------------------------------- /coerce/src/protocol/singleton.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package coerce.singleton; 4 | 5 | message GetStatus { 6 | uint64 source_node_id = 1; 7 | } 8 | 9 | message ManagerStatus { 10 | SingletonState singleton_state = 1; 11 | } 12 | 13 | enum SingletonState { 14 | JOINING = 0; 15 | IDLE = 1; 16 | STARTING = 2; 17 | RUNNING = 3; 18 | STOPPING = 4; 19 | } 20 | 21 | message RequestLease { 22 | uint64 source_node_id = 1; 23 | } 24 | 25 | message LeaseAck { 26 | uint64 source_node_id = 1; 27 | } 28 | 29 | message LeaseNack { 30 | uint64 source_node_id = 1; 31 | } 32 | 33 | message SingletonStarted { 34 | uint64 source_node_id = 1; 35 | } 36 | 37 | message SingletonStopping { 38 | uint64 source_node_id = 1; 39 | } 40 | 41 | message SingletonStopped { 42 | uint64 source_node_id = 1; 43 | } -------------------------------------------------------------------------------- /coerce/src/remote/actor/message.rs: -------------------------------------------------------------------------------- 1 | use crate::remote::actor::RemoteRequest; 2 | use crate::remote::cluster::node::{RemoteNode, RemoteNodeState}; 3 | use crate::remote::system::{NodeId, RemoteActorSystem}; 4 | 5 | use crate::actor::message::Message; 6 | use crate::remote::net::client::{ClientType, RemoteClient}; 7 | use crate::remote::net::message::SessionEvent; 8 | 9 | use crate::actor::{ActorId, LocalActorRef}; 10 | 11 | use uuid::Uuid; 12 | 13 | pub struct SetRemote(pub RemoteActorSystem); 14 | 15 | impl Message for SetRemote { 16 | type Result = (); 17 | } 18 | 19 | pub struct GetNodes; 20 | 21 | impl Message for GetNodes { 22 | type Result = Vec; 23 | } 24 | 25 | pub struct PushRequest(pub Uuid, pub RemoteRequest); 26 | 27 | impl Message for PushRequest { 28 | type Result = (); 29 | } 30 | 31 | pub struct PopRequest(pub Uuid); 32 | 33 | impl Message for PopRequest { 34 | type Result = Option; 35 | } 36 | 37 | pub struct NewClient { 38 | pub addr: String, 39 | pub client_type: ClientType, 40 | pub system: RemoteActorSystem, 41 | } 42 | 43 | impl Message for NewClient { 44 | type Result = Option>; 45 | } 46 | 47 | pub struct RemoveClient { 48 | pub addr: String, 49 | pub node_id: Option, 50 | } 51 | 52 | impl Message for RemoveClient { 53 | type Result = (); 54 | } 55 | 56 | pub struct ClientConnected { 57 | pub addr: String, 58 | pub remote_node_id: NodeId, 59 | pub client_actor_ref: LocalActorRef, 60 | } 61 | 62 | impl Message for ClientConnected { 63 | type Result = (); 64 | } 65 | 66 | pub struct RegisterNodes(pub Vec); 67 | 68 | impl Message for RegisterNodes { 69 | type Result = (); 70 | } 71 | 72 | pub struct UpdateNodes(pub Vec); 73 | 74 | impl Message for UpdateNodes { 75 | type Result = (); 76 | } 77 | 78 | pub struct RegisterNode(pub RemoteNode); 79 | 80 | impl Message for RegisterNode { 81 | type Result = (); 82 | } 83 | 84 | pub struct NodeTerminated(pub NodeId); 85 | 86 | impl Message for NodeTerminated { 87 | type Result = (); 88 | } 89 | 90 | pub struct ClientWrite(pub NodeId, pub SessionEvent); 91 | 92 | impl Message for ClientWrite { 93 | type Result = (); 94 | } 95 | 96 | #[derive(Debug)] 97 | pub struct RegisterActor { 98 | pub actor_id: ActorId, 99 | pub node_id: Option, 100 | } 101 | 102 | impl RegisterActor { 103 | pub fn new(actor_id: ActorId, node_id: Option) -> RegisterActor { 104 | RegisterActor { actor_id, node_id } 105 | } 106 | } 107 | 108 | impl Message for RegisterActor { 109 | type Result = (); 110 | } 111 | 112 | pub struct GetActorNode { 113 | pub actor_id: ActorId, 114 | pub sender: tokio::sync::oneshot::Sender>, 115 | } 116 | 117 | impl Message for GetActorNode { 118 | type Result = (); 119 | } 120 | -------------------------------------------------------------------------------- /coerce/src/remote/actor/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::ActorRefErr; 2 | 3 | use crate::remote::handler::{ActorHandler, ActorMessageHandler}; 4 | 5 | use std::collections::HashMap; 6 | use uuid::Uuid; 7 | 8 | pub mod clients; 9 | pub mod message; 10 | pub mod registry; 11 | 12 | pub(crate) type BoxedActorHandler = Box; 13 | 14 | pub(crate) type BoxedMessageHandler = Box; 15 | 16 | pub struct RemoteHandler { 17 | requests: HashMap, 18 | } 19 | 20 | impl RemoteHandler { 21 | pub fn push_request(&mut self, message_id: Uuid, request: RemoteRequest) { 22 | self.requests.insert(message_id, request); 23 | } 24 | 25 | pub fn pop_request(&mut self, message_id: Uuid) -> Option { 26 | self.requests.remove(&message_id) 27 | } 28 | 29 | pub fn inflight_request_count(&self) -> usize { 30 | self.requests.len() 31 | } 32 | } 33 | 34 | pub struct RemoteRequest { 35 | pub res_tx: tokio::sync::oneshot::Sender, 36 | } 37 | 38 | #[derive(Debug)] 39 | pub enum RemoteResponse { 40 | Ok(Vec), 41 | Err(ActorRefErr), 42 | } 43 | 44 | impl RemoteResponse { 45 | pub fn is_ok(&self) -> bool { 46 | match self { 47 | &RemoteResponse::Ok(..) => true, 48 | _ => false, 49 | } 50 | } 51 | 52 | pub fn is_err(&self) -> bool { 53 | match self { 54 | &RemoteResponse::Err(..) => true, 55 | _ => false, 56 | } 57 | } 58 | 59 | pub fn into_result(self) -> Result, ActorRefErr> { 60 | match self { 61 | RemoteResponse::Ok(buff) => Ok(buff), 62 | RemoteResponse::Err(buff) => Err(buff), 63 | } 64 | } 65 | } 66 | 67 | impl RemoteHandler { 68 | pub fn new() -> RemoteHandler { 69 | RemoteHandler { 70 | requests: HashMap::new(), 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /coerce/src/remote/api/builder.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::system::ActorSystem; 2 | use crate::actor::{IntoActor, LocalActorRef}; 3 | use crate::remote::api::{RemoteHttpApi, Routes}; 4 | 5 | use std::net::SocketAddr; 6 | 7 | pub struct HttpApiBuilder { 8 | listen_addr: Option, 9 | routes: Vec>, 10 | } 11 | 12 | impl HttpApiBuilder { 13 | pub fn new() -> Self { 14 | Self { 15 | listen_addr: None, 16 | routes: vec![], 17 | } 18 | } 19 | 20 | pub fn listen_addr(mut self, listen_addr: SocketAddr) -> Self { 21 | self.listen_addr = Some(listen_addr); 22 | self 23 | } 24 | 25 | pub fn routes(mut self, route: impl Routes) -> Self { 26 | self.routes.push(Box::new(route)); 27 | self 28 | } 29 | 30 | pub async fn start(self, system: &ActorSystem) -> LocalActorRef { 31 | let listen_addr = self 32 | .listen_addr 33 | .expect("listen_addr is required, no default is defined yet (TODO)"); 34 | 35 | RemoteHttpApi::new(listen_addr, self.routes) 36 | .into_actor(Some("http-api"), system) 37 | .await 38 | .unwrap() 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /coerce/src/remote/api/cluster/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::remote::api::Routes; 2 | use crate::remote::cluster::node::RemoteNodeState; 3 | use crate::remote::system::RemoteActorSystem; 4 | use axum::response::IntoResponse; 5 | use axum::routing::get; 6 | use axum::{Json, Router}; 7 | use std::collections::HashMap; 8 | use std::time::Duration; 9 | 10 | pub struct ClusterApi { 11 | system: RemoteActorSystem, 12 | } 13 | 14 | impl ClusterApi { 15 | pub fn new(system: RemoteActorSystem) -> Self { 16 | Self { system } 17 | } 18 | } 19 | 20 | impl Routes for ClusterApi { 21 | fn routes(&self, router: Router) -> Router { 22 | router.route("/cluster/nodes", { 23 | let system = self.system.clone(); 24 | get(move || get_nodes(system)) 25 | }) 26 | } 27 | } 28 | 29 | #[derive(Serialize, Deserialize, Debug, ToSchema)] 30 | pub struct ClusterNode { 31 | pub id: u64, 32 | pub addr: String, 33 | pub tag: String, 34 | pub ping_latency: Option, 35 | pub last_heartbeat: Option, 36 | pub node_started_at: Option>, 37 | pub status: NodeStatus, 38 | pub attributes: HashMap, 39 | } 40 | 41 | #[derive(Serialize, Deserialize, Debug, ToSchema)] 42 | pub struct ClusterNodes { 43 | pub node_id: u64, 44 | pub leader_node: Option, 45 | pub leader_node_tag: Option, 46 | pub nodes: Vec, 47 | } 48 | 49 | #[derive(Debug, Serialize, Deserialize, Clone, ToSchema)] 50 | pub enum NodeStatus { 51 | Joining, 52 | Healthy, 53 | Unhealthy, 54 | Terminated, 55 | } 56 | 57 | impl From for NodeStatus { 58 | fn from(value: crate::remote::cluster::node::NodeStatus) -> Self { 59 | match value { 60 | crate::remote::cluster::node::NodeStatus::Joining => Self::Joining, 61 | crate::remote::cluster::node::NodeStatus::Healthy => Self::Healthy, 62 | crate::remote::cluster::node::NodeStatus::Unhealthy => Self::Unhealthy, 63 | crate::remote::cluster::node::NodeStatus::Terminated => Self::Terminated, 64 | } 65 | } 66 | } 67 | 68 | #[utoipa::path( 69 | get, 70 | path = "/cluster/nodes", 71 | responses( 72 | ( 73 | status = 200, description = "All known Coerce cluster nodes", body = ClusterNodes), 74 | ) 75 | )] 76 | async fn get_nodes(system: RemoteActorSystem) -> impl IntoResponse { 77 | let node_id = system.node_id(); 78 | let leader_node = system.current_leader(); 79 | let mut nodes: Vec = system 80 | .get_nodes() 81 | .await 82 | .into_iter() 83 | .map(|node| node.into()) 84 | .collect(); 85 | 86 | nodes.sort_by(|a, b| a.id.partial_cmp(&b.id).unwrap()); 87 | let nodes = nodes; 88 | 89 | let leader_node_tag = if leader_node == Some(system.node_id()) { 90 | Some(system.node_tag().to_string()) 91 | } else { 92 | nodes 93 | .iter() 94 | .find(|n| Some(n.id) == leader_node) 95 | .map(|node| node.tag.clone()) 96 | }; 97 | 98 | Json(ClusterNodes { 99 | node_id, 100 | leader_node, 101 | leader_node_tag, 102 | nodes, 103 | }) 104 | } 105 | 106 | impl From for ClusterNode { 107 | fn from(node: RemoteNodeState) -> Self { 108 | ClusterNode { 109 | id: node.id, 110 | addr: node.addr, 111 | tag: node.tag, 112 | ping_latency: node.ping_latency, 113 | last_heartbeat: node.last_heartbeat.map(|h| format!("{:?}", h)), 114 | node_started_at: node.node_started_at.map(|p| p), 115 | status: node.status.into(), 116 | attributes: node 117 | .attributes 118 | .iter() 119 | .map(|(k, v)| (k.to_string(), v.to_string())) 120 | .collect(), 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /coerce/src/remote/api/metrics/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::remote::api::Routes; 2 | use crate::remote::system::{NodeId, RemoteActorSystem}; 3 | 4 | use axum::response::IntoResponse; 5 | use axum::routing::get; 6 | use axum::Router; 7 | use metrics_exporter_prometheus::{BuildError, PrometheusBuilder, PrometheusHandle}; 8 | use metrics_util::MetricKindMask; 9 | 10 | use std::time::Duration; 11 | 12 | pub struct MetricsApi { 13 | recorder_handle: PrometheusHandle, 14 | } 15 | 16 | impl From for MetricsApi { 17 | fn from(recorder_handle: PrometheusHandle) -> Self { 18 | Self { recorder_handle } 19 | } 20 | } 21 | 22 | impl MetricsApi { 23 | pub fn create( 24 | node_id: NodeId, 25 | node_tag: &str, 26 | cluster_name: Option<&str>, 27 | ) -> Result { 28 | let prometheus_builder = PrometheusBuilder::new(); 29 | 30 | let mut builder = prometheus_builder.idle_timeout( 31 | MetricKindMask::COUNTER | MetricKindMask::HISTOGRAM, 32 | Some(Duration::from_secs(60 * 30)), 33 | ); 34 | 35 | if let Some(cluster_name) = cluster_name { 36 | builder = builder.add_global_label("cluster", cluster_name.to_string()); 37 | } 38 | 39 | builder = builder 40 | .add_global_label("node_id", format!("{}", node_id)) 41 | .add_global_label("node_tag", node_tag); 42 | 43 | let recorder = builder.install_recorder()?; 44 | Ok(Self { 45 | recorder_handle: recorder, 46 | }) 47 | } 48 | 49 | pub fn new(system: RemoteActorSystem) -> Result { 50 | let node_id = system.node_id(); 51 | let node_tag = system.node_tag(); 52 | let cluster_name = system 53 | .config() 54 | .get_attributes() 55 | .get("cluster") 56 | .map(|s| s.as_ref()); 57 | 58 | Self::create(node_id, node_tag, cluster_name) 59 | } 60 | } 61 | 62 | impl Routes for MetricsApi { 63 | fn routes(&self, router: Router) -> Router { 64 | router.route("/metrics", { 65 | let handle = self.recorder_handle.clone(); 66 | 67 | get(|| async move { 68 | let handle = handle; 69 | handle.render().into_response() 70 | }) 71 | }) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /coerce/src/remote/api/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod builder; 2 | pub mod cluster; 3 | pub mod system; 4 | 5 | #[cfg(feature = "metrics")] 6 | pub mod metrics; 7 | 8 | pub mod openapi; 9 | 10 | #[cfg(feature = "sharding")] 11 | pub mod sharding; 12 | 13 | use crate::actor::context::ActorContext; 14 | 15 | use crate::actor::Actor; 16 | 17 | use axum::Router; 18 | use std::net::SocketAddr; 19 | use tokio::sync::oneshot; 20 | use tokio::sync::oneshot::Sender; 21 | 22 | pub struct RemoteHttpApi { 23 | listen_addr: SocketAddr, 24 | routes: Option>>, 25 | stop_tx: Option>, 26 | } 27 | 28 | #[async_trait] 29 | impl Actor for RemoteHttpApi { 30 | async fn started(&mut self, ctx: &mut ActorContext) { 31 | let node_id = ctx.system().remote().node_id(); 32 | let listen_addr = self.listen_addr; 33 | 34 | let routes = self.routes.take().unwrap(); 35 | 36 | let (stop_tx, stop_rx) = oneshot::channel(); 37 | let _ = tokio::spawn(async move { 38 | info!("[node={}] http api listening on {}", node_id, &listen_addr); 39 | 40 | let mut app = Router::new(); 41 | for route in routes { 42 | app = route.routes(app); 43 | } 44 | 45 | axum::Server::bind(&listen_addr) 46 | .serve(app.into_make_service()) 47 | .with_graceful_shutdown(async { stop_rx.await.unwrap() }) 48 | .await 49 | .unwrap() 50 | }); 51 | 52 | self.stop_tx = Some(stop_tx); 53 | } 54 | 55 | async fn stopped(&mut self, _ctx: &mut ActorContext) { 56 | if let Some(stop_tx) = self.stop_tx.take() { 57 | let _ = stop_tx.send(()); 58 | } 59 | } 60 | } 61 | 62 | impl RemoteHttpApi { 63 | pub fn new(listen_addr: SocketAddr, routes: Vec>) -> Self { 64 | RemoteHttpApi { 65 | listen_addr, 66 | routes: Some(routes), 67 | stop_tx: None, 68 | } 69 | } 70 | } 71 | 72 | pub trait Routes: 'static + Send + Sync { 73 | fn routes(&self, router: Router) -> Router; 74 | } 75 | -------------------------------------------------------------------------------- /coerce/src/remote/api/openapi.rs: -------------------------------------------------------------------------------- 1 | use crate::remote::api::cluster; 2 | use crate::remote::api::system; 3 | use crate::remote::api::system::actors; 4 | 5 | #[derive(OpenApi)] 6 | #[openapi( 7 | paths( 8 | system::health, 9 | system::get_stats, 10 | ), 11 | components( 12 | schemas( 13 | system::SystemHealth, 14 | system::HealthStatus, 15 | system::SystemStats, 16 | system::SystemStats, 17 | ) 18 | ), 19 | tags( 20 | (name = "system", description = "System API"), 21 | ) 22 | )] 23 | pub struct SystemApiDoc; 24 | 25 | #[derive(OpenApi)] 26 | #[openapi( 27 | paths( 28 | actors::get_all, 29 | ), 30 | components( 31 | schemas( 32 | 33 | system::actors::GetAll, 34 | system::actors::Actors, 35 | system::actors::ActorDescription, 36 | system::actors::Status, 37 | system::actors::ActorTags, 38 | system::actors::SupervisedDescription, 39 | ) 40 | ), 41 | tags( 42 | (name = "actors", description = "Actors API"), 43 | ) 44 | )] 45 | pub struct ActorsApiDoc; 46 | 47 | #[derive(OpenApi)] 48 | #[openapi( 49 | paths( 50 | cluster::get_nodes, 51 | ), 52 | components( 53 | schemas( 54 | cluster::ClusterNodes, 55 | cluster::ClusterNode, 56 | cluster::NodeStatus, 57 | ) 58 | ), 59 | tags( 60 | (name = "cluster", description = "Cluster API"), 61 | ) 62 | )] 63 | pub struct ClusterApiDoc; 64 | 65 | #[cfg(feature = "sharding")] 66 | pub mod sharding { 67 | #[derive(OpenApi)] 68 | #[openapi( 69 | paths( 70 | get_sharding_types, 71 | get_sharding_stats, 72 | get_node_stats, 73 | ), 74 | components( 75 | schemas( 76 | cluster::ShardingClusterStats, 77 | cluster::ShardingNode, 78 | cluster::ShardStats, 79 | cluster::ShardHostStatus, 80 | cluster::Entity, 81 | node::Stats, 82 | node::ShardStats, 83 | ShardTypes, 84 | ) 85 | ), 86 | tags( 87 | (name = "sharding", description = "Sharding API"), 88 | ) 89 | )] 90 | pub struct ShardingApiDoc; 91 | 92 | use crate::remote::api::sharding as sharding_api; 93 | pub use sharding_api::ShardTypes; 94 | 95 | pub use sharding_api::cluster; 96 | pub use sharding_api::node; 97 | 98 | pub use sharding_api::cluster::__path_get_sharding_stats; 99 | pub use sharding_api::cluster::get_sharding_stats; 100 | 101 | pub use sharding_api::node::__path_get_node_stats; 102 | pub use sharding_api::node::get_node_stats; 103 | 104 | pub use sharding_api::__path_get_sharding_types; 105 | pub use sharding_api::get_sharding_types; 106 | } 107 | -------------------------------------------------------------------------------- /coerce/src/remote/api/sharding/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod cluster; 2 | pub mod node; 3 | pub mod routes; 4 | 5 | use crate::actor::context::ActorContext; 6 | use crate::actor::message::{Handler, Message}; 7 | use crate::actor::system::ActorSystem; 8 | use crate::actor::{Actor, ActorFactory, IntoActor, IntoActorId, LocalActorRef}; 9 | use crate::sharding::host::ShardHost; 10 | use crate::sharding::Sharding; 11 | use axum::response::IntoResponse; 12 | use axum::Json; 13 | use std::collections::HashMap; 14 | 15 | #[derive(Default)] 16 | pub struct ShardingApi { 17 | shard_hosts: HashMap>, 18 | } 19 | 20 | impl Actor for ShardingApi {} 21 | 22 | impl ShardingApi { 23 | pub fn attach(mut self, sharding: &Sharding) -> Self { 24 | let shard_entity = sharding.shard_entity().clone(); 25 | let shard_host = sharding.shard_host().clone(); 26 | 27 | self.shard_hosts.insert(shard_entity, shard_host); 28 | self 29 | } 30 | 31 | pub async fn start(self, actor_system: &ActorSystem) -> LocalActorRef { 32 | self.into_actor(Some("sharding-api".into_actor_id()), actor_system) 33 | .await 34 | .expect("unable to start ShardingApi actor") 35 | } 36 | } 37 | 38 | pub struct GetShardTypes; 39 | 40 | impl Message for GetShardTypes { 41 | type Result = ShardTypes; 42 | } 43 | 44 | #[async_trait] 45 | impl Handler for ShardingApi { 46 | async fn handle(&mut self, _message: GetShardTypes, _ctx: &mut ActorContext) -> ShardTypes { 47 | ShardTypes { 48 | entities: self.shard_hosts.keys().cloned().collect(), 49 | } 50 | } 51 | } 52 | 53 | #[derive(Serialize, ToSchema)] 54 | pub struct ShardTypes { 55 | entities: Vec, 56 | } 57 | 58 | #[utoipa::path( 59 | get, 60 | path = "/sharding/types", 61 | responses( 62 | (status = 200, description = "All sharded entity types", body = ShardTypes), 63 | ), 64 | )] 65 | pub async fn get_sharding_types(sharding_api: LocalActorRef) -> impl IntoResponse { 66 | Json( 67 | sharding_api 68 | .send(GetShardTypes) 69 | .await 70 | .expect("unable to get shard types"), 71 | ) 72 | } 73 | -------------------------------------------------------------------------------- /coerce/src/remote/api/sharding/routes.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::LocalActorRef; 2 | use crate::remote::api::sharding::cluster::{get_shard_host_stats, get_sharding_stats}; 3 | use crate::remote::api::sharding::node::{get_node_stats, GetAllStats}; 4 | use crate::remote::api::sharding::{get_sharding_types, ShardingApi}; 5 | use crate::remote::api::Routes; 6 | 7 | use axum::routing::get; 8 | use axum::{Json, Router}; 9 | 10 | impl Routes for LocalActorRef { 11 | fn routes(&self, router: Router) -> Router { 12 | router 13 | .route("/sharding/types", { 14 | let actor_ref = self.clone(); 15 | get(move || get_sharding_types(actor_ref)) 16 | }) 17 | .route("/sharding/stats/cluster/:entity", { 18 | let actor_ref = self.clone(); 19 | get(move |path| get_sharding_stats(actor_ref, path)) 20 | }) 21 | .route("/sharding/host/stats/:entity", { 22 | let actor_ref = self.clone(); 23 | get(move |path| get_shard_host_stats(actor_ref, path)) 24 | }) 25 | .route("/sharding/stats/node/:entity", { 26 | let actor_ref = self.clone(); 27 | get(move |path| get_node_stats(actor_ref, path)) 28 | }) 29 | .route("/sharding/node/stats/all", { 30 | let actor_ref = self.clone(); 31 | get(|| async move { 32 | let actor_ref = actor_ref; 33 | Json( 34 | actor_ref 35 | .send(GetAllStats) 36 | .await 37 | .expect("unable to get shard types"), 38 | ) 39 | }) 40 | }) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /coerce/src/remote/cluster/builder/client.rs: -------------------------------------------------------------------------------- 1 | use crate::remote::cluster::client::RemoteClusterClient; 2 | use crate::remote::system::RemoteActorSystem; 3 | 4 | pub struct ClusterClientBuilder { 5 | system: RemoteActorSystem, 6 | seed_addr: Option, 7 | } 8 | 9 | impl ClusterClientBuilder { 10 | pub fn new(system: RemoteActorSystem) -> ClusterClientBuilder { 11 | ClusterClientBuilder { 12 | system, 13 | seed_addr: None, 14 | } 15 | } 16 | 17 | pub fn with_seed_addr(mut self, seed_addr: T) -> Self { 18 | self.seed_addr = Some(seed_addr.to_string()); 19 | 20 | self 21 | } 22 | 23 | pub async fn start(self) -> RemoteClusterClient { 24 | // let seed_addr = self.seed_addr.expect("no seed addr"); 25 | // let system = self.system.clone(); 26 | // let client = RemoteClient::connect(seed_addr.clone(), None, system, None, Client) 27 | // .await 28 | // .expect("failed to connect to seed server"); 29 | // 30 | // self.system 31 | // .register_node(RemoteNode::new( 32 | // client.node_id, 33 | // seed_addr, 34 | // client.node_tag.clone(), 35 | // Some(Utc::now()), 36 | // )) 37 | // .await; 38 | 39 | // self.system.register_client(client.node_id, client).await; 40 | RemoteClusterClient::new(self.system) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /coerce/src/remote/cluster/builder/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod client; 2 | pub mod worker; 3 | -------------------------------------------------------------------------------- /coerce/src/remote/cluster/client/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::remote::system::RemoteActorSystem; 2 | 3 | #[derive(Clone)] 4 | pub struct RemoteClusterClient { 5 | system: RemoteActorSystem, 6 | } 7 | 8 | impl RemoteClusterClient { 9 | pub fn new(system: RemoteActorSystem) -> RemoteClusterClient { 10 | RemoteClusterClient { system } 11 | } 12 | } 13 | 14 | impl RemoteClusterClient {} 15 | -------------------------------------------------------------------------------- /coerce/src/remote/cluster/discovery/dns.rs: -------------------------------------------------------------------------------- 1 | // use crate::remote::cluster::discovery::{ClusterSeed, ClusterSeedErr, DiscoveredWorker}; 2 | // 3 | // use std::net::{IpAddr, SocketAddr}; 4 | // 5 | // use std::str::FromStr; 6 | // use trust_dns_client::client::{AsyncClient, ClientHandle}; 7 | // 8 | // use trust_dns_client::rr::{DNSClass, Name, RecordType}; 9 | // use trust_dns_client::udp::UdpClientStream; 10 | // use trust_dns_proto::udp::UdpResponse; 11 | // 12 | // use tokio::net::UdpSocket; 13 | // 14 | // pub struct DnsClusterSeed { 15 | // client: AsyncClient, 16 | // seed_host: String, 17 | // } 18 | // 19 | // impl DnsClusterSeed { 20 | // pub async fn new(upstream: SocketAddr, seed_host: String) -> DnsClusterSeed { 21 | // let stream = UdpClientStream::::new(upstream); 22 | // let (client, bg) = AsyncClient::connect(stream).await.unwrap(); 23 | // 24 | // tokio::spawn(bg); 25 | // 26 | // DnsClusterSeed { client, seed_host } 27 | // } 28 | // 29 | // pub async fn all_a_records(&mut self, host: &str) -> Result, ClusterSeedErr> { 30 | // match self 31 | // .client 32 | // .query(Name::from_str(host).unwrap(), DNSClass::IN, RecordType::A) 33 | // .await 34 | // { 35 | // Ok(res) => Ok(res 36 | // .answers() 37 | // .into_iter() 38 | // .filter_map(|a| a.rdata().to_ip_addr()) 39 | // .collect()), 40 | // 41 | // Err(e) => Err(ClusterSeedErr::Err(format!("{:?}", e))), 42 | // } 43 | // } 44 | // } 45 | // 46 | // #[async_trait] 47 | // impl ClusterSeed for DnsClusterSeed { 48 | // async fn initial_workers(&mut self) -> Result, ClusterSeedErr> { 49 | // let seed_host = self.seed_host.clone(); 50 | // let records = self.all_a_records(&seed_host).await?; 51 | // 52 | // Ok(records 53 | // .into_iter() 54 | // .map(DiscoveredWorker::from_host) 55 | // .collect()) 56 | // } 57 | // } 58 | -------------------------------------------------------------------------------- /coerce/src/remote/cluster/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod builder; 2 | pub mod client; 3 | pub mod discovery; 4 | pub mod node; 5 | -------------------------------------------------------------------------------- /coerce/src/remote/mod.rs: -------------------------------------------------------------------------------- 1 | //! Coerce Remoting 2 | //! 3 | //! Coerce clusters are identified by a [`NodeId`] and a string-based node tag. 4 | //! 5 | //! The easiest way to create a full, batteries-included clustered actor system, is by 6 | //! using the [`RemoteActorSystemBuilder`]. From here, you can customise and 7 | //! create a [`RemoteActorSystem`]. 8 | //! 9 | //! ## Remote-enabled messages 10 | //! Messages are not available to be handled remotely by default, and do require being registered 11 | //! with the [`RemoteActorSystem`] during the creation process. 12 | //! 13 | //! All message handlers must have a unique identifier assigned. 14 | //! 15 | //! ## Builder Example 16 | //! ```rust 17 | //! use coerce::actor::system::ActorSystem; 18 | //! use coerce::remote::system::RemoteActorSystem; 19 | //! 20 | //! #[tokio::main] 21 | //! async fn main() { 22 | //! let system = ActorSystem::new(); 23 | //! let remote = RemoteActorSystem::builder() 24 | //! .with_id(1) 25 | //! .with_tag("node-name") 26 | //! .with_actor_system(system) 27 | //! .build() 28 | //! .await; 29 | //! } 30 | //! ``` 31 | //! 32 | //! ## Registering Remote-enabled Messages 33 | //! Remote-enabled messages can be enabled when creating a [`RemoteActorSystem`], below is an example 34 | //! of a message of type `MyMessageType` registered, with the actor that processes the 35 | //! message as type `MyActorType`. 36 | //! 37 | //! Note that a prerequisite for messages to be able to be transmitted remotely is that as part 38 | //! of defining the message, it must define how to serialise and deserialise to and from a `Vec`. 39 | //! 40 | //! ```rust 41 | //! use coerce::actor::{Actor, system::ActorSystem}; 42 | //! use coerce::actor::context::ActorContext; 43 | //! use coerce::actor::message::{Handler, Message, MessageUnwrapErr, MessageWrapErr}; 44 | //! use coerce::remote::system::RemoteActorSystem; 45 | //! use async_trait::async_trait; 46 | //! 47 | //! #[tokio::main] 48 | //! pub async fn main() { 49 | //! let system = ActorSystem::new(); 50 | //! let remote_system = RemoteActorSystem::builder() 51 | //! .with_id(1) 52 | //! .with_tag("node_name") 53 | //! .with_actor_system(system) 54 | //! .with_handlers(|handlers| { 55 | //! handlers 56 | //! .with_handler::("MyActor.MyMessage") 57 | //! }); 58 | //! 59 | //! // start servers, create actors etc 60 | //! } 61 | //! 62 | //! struct MyActor; 63 | //! 64 | //! impl Actor for MyActor { } 65 | //! 66 | //! struct MyMessage; 67 | //! 68 | //! #[async_trait] 69 | //! impl Handler for MyActor { 70 | //! async fn handle(&mut self, message: MyMessage, ctx: &mut ActorContext) { 71 | //! // handle the msg 72 | //! } 73 | //! } 74 | //! 75 | //! impl Message for MyMessage { 76 | //! type Result = (); 77 | //! 78 | //! fn as_bytes(&self) -> Result, MessageWrapErr> { 79 | //! Ok(vec![]) 80 | //! } 81 | //! 82 | //! fn from_bytes(_: Vec) -> Result { 83 | //! Ok(Self) 84 | //! } 85 | //! } 86 | //! ``` 87 | //! 88 | //! [`NodeId`]: system::NodeId 89 | //! [`RemoteActorSystemBuilder`]: system::builder::RemoteActorSystemBuilder 90 | //! [`RemoteActorSystem`]: system::RemoteActorSystem 91 | 92 | pub mod actor; 93 | pub mod actor_ref; 94 | pub mod cluster; 95 | pub mod config; 96 | pub mod handler; 97 | pub mod heartbeat; 98 | pub mod net; 99 | pub mod stream; 100 | pub mod system; 101 | pub mod tracing; 102 | 103 | #[cfg(feature = "api")] 104 | pub mod api; 105 | 106 | pub use actor_ref::*; 107 | -------------------------------------------------------------------------------- /coerce/src/remote/net/metrics.rs: -------------------------------------------------------------------------------- 1 | pub const METRIC_NETWORK_BYTES_RECV: &str = "coerce_network_bytes_recv"; 2 | pub const METRIC_NETWORK_BYTES_SENT: &str = "coerce_network_bytes_sent"; 3 | 4 | pub const LABEL_SRC_ADDR: &str = "src_addr"; 5 | pub const LABEL_DEST_ADDR: &str = "dest_addr"; 6 | 7 | pub struct NetworkMetrics; 8 | 9 | impl NetworkMetrics { 10 | #[inline] 11 | pub fn incr_bytes_received(len: u64, src_addr: &str) { 12 | #[cfg(feature = "metrics")] 13 | counter!( 14 | METRIC_NETWORK_BYTES_RECV, 15 | len, 16 | LABEL_SRC_ADDR => src_addr.to_owned() 17 | ); 18 | } 19 | 20 | #[inline] 21 | pub fn incr_bytes_sent(len: u64, dest_addr: &str) { 22 | #[cfg(feature = "metrics")] 23 | counter!( 24 | METRIC_NETWORK_BYTES_SENT, 25 | len, 26 | LABEL_DEST_ADDR => dest_addr.to_owned() 27 | ); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /coerce/src/remote/net/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::remote::system::RemoteActorSystem; 2 | 3 | use std::future::Future; 4 | use std::io::Error; 5 | 6 | use bytes::BytesMut; 7 | use std::pin::Pin; 8 | use std::task::{Context, Poll}; 9 | 10 | use futures::StreamExt; 11 | use tokio_util::codec::{FramedRead, LengthDelimitedCodec}; 12 | 13 | pub mod client; 14 | pub mod message; 15 | pub mod metrics; 16 | pub mod proto; 17 | pub mod security; 18 | pub mod server; 19 | 20 | pub trait StreamData: 'static + Send + Sync + Sized { 21 | fn read_from_bytes(data: Vec) -> Option; 22 | 23 | fn write_to_bytes(&self) -> Option>; 24 | } 25 | 26 | #[async_trait] 27 | pub trait StreamReceiver { 28 | type Message: StreamData; 29 | 30 | async fn on_receive(&mut self, msg: Self::Message, sys: &RemoteActorSystem); 31 | 32 | async fn on_close(&mut self, sys: &RemoteActorSystem); 33 | 34 | fn on_deserialisation_failed(&mut self); 35 | 36 | fn on_stream_lost(&mut self, error: Error); 37 | 38 | async fn close(&mut self); 39 | 40 | fn should_close(&self) -> bool; 41 | } 42 | 43 | pub struct StreamReceiverFuture { 44 | stream: FramedRead, 45 | stop_rx: tokio::sync::oneshot::Receiver, 46 | } 47 | 48 | impl StreamReceiverFuture { 49 | pub fn new( 50 | stream: FramedRead, 51 | stop_rx: tokio::sync::oneshot::Receiver, 52 | ) -> StreamReceiverFuture { 53 | StreamReceiverFuture { stream, stop_rx } 54 | } 55 | } 56 | 57 | impl tokio_stream::Stream for StreamReceiverFuture 58 | where 59 | S: Unpin, 60 | { 61 | type Item = Option; 62 | 63 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll>> { 64 | if let Poll::Ready(Ok(true)) = Pin::new(&mut self.stop_rx).poll(cx) { 65 | return Poll::Ready(None); 66 | } 67 | 68 | let result: Option> = 69 | futures::ready!(Pin::new(&mut self.stream).poll_next(cx)); 70 | 71 | Poll::Ready(match result { 72 | Some(Ok(message)) => Some(Some(message)), 73 | Some(Err(e)) => { 74 | error!("{:?}", e); 75 | Some(None) 76 | } 77 | None => None, 78 | }) 79 | } 80 | } 81 | 82 | pub async fn receive_loop( 83 | mut system: RemoteActorSystem, 84 | read: FramedRead, 85 | mut receiver: R, 86 | ) where 87 | R: Send, 88 | { 89 | let mut reader = read; 90 | while let Some(res) = reader.next().await { 91 | match res { 92 | Ok(res) => match R::Message::read_from_bytes(res.to_vec()) { 93 | Some(msg) => { 94 | receiver.on_receive(msg, &system).await; 95 | if receiver.should_close() { 96 | break; 97 | } 98 | } 99 | None => { 100 | // TODO: either pass the buffer into here or more context, this is pretty useless at the moment.. 101 | receiver.on_deserialisation_failed(); 102 | } 103 | }, 104 | Err(e) => { 105 | receiver.on_stream_lost(e); 106 | break; 107 | } 108 | } 109 | } 110 | 111 | receiver.on_close(&mut system).await; 112 | } 113 | -------------------------------------------------------------------------------- /coerce/src/remote/net/proto/mod.rs: -------------------------------------------------------------------------------- 1 | // @generated 2 | 3 | pub mod network; 4 | -------------------------------------------------------------------------------- /coerce/src/remote/net/security/auth/jwt.rs: -------------------------------------------------------------------------------- 1 | use chrono::{DateTime, TimeZone, Utc}; 2 | use hmac::digest::KeyInit; 3 | use hmac::Hmac; 4 | use jwt::{Error, SignWithKey, VerifyWithKey}; 5 | use sha2::Sha256; 6 | 7 | use std::time::Duration; 8 | use uuid::Uuid; 9 | 10 | pub struct Jwt { 11 | secret: Hmac, 12 | token_ttl: Option, 13 | } 14 | 15 | #[derive(Serialize, Deserialize)] 16 | struct Payload { 17 | id: Uuid, 18 | timestamp: DateTime, 19 | } 20 | 21 | impl Jwt { 22 | pub fn from_secret>>(secret: S, token_ttl: Option) -> Jwt { 23 | let secret = secret.into(); 24 | let secret: Hmac = Hmac::new_from_slice(&secret) 25 | .map_err(|_e| "invalid client auth secret key") 26 | .unwrap(); 27 | 28 | Self { secret, token_ttl } 29 | } 30 | 31 | pub fn generate_token(&self) -> Option { 32 | let payload = Payload { 33 | id: Uuid::new_v4(), 34 | timestamp: Utc::now(), 35 | }; 36 | 37 | SignWithKey::sign_with_key(payload, &self.secret).ok() 38 | } 39 | 40 | pub fn validate_token(&self, token: &str) -> bool { 41 | let payload: Result = VerifyWithKey::verify_with_key(token, &self.secret); 42 | 43 | if let Ok(payload) = payload { 44 | if let Some(token_ttl) = &self.token_ttl { 45 | is_token_still_valid(&payload, token_ttl) 46 | } else { 47 | true 48 | } 49 | } else { 50 | false 51 | } 52 | } 53 | } 54 | 55 | fn is_token_still_valid(payload: &Payload, token_ttl: &Duration) -> bool { 56 | let token_ttl = chrono::Duration::from_std(*token_ttl).unwrap(); 57 | let token_expires_at = payload.timestamp + token_ttl; 58 | let now = Utc::now(); 59 | 60 | if now <= token_expires_at { 61 | true 62 | } else { 63 | false 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /coerce/src/remote/net/security/auth/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "client-auth-jwt")] 2 | pub mod jwt; 3 | 4 | pub enum ClientAuth { 5 | None, 6 | 7 | #[cfg(feature = "client-auth-jwt")] 8 | Jwt(jwt::Jwt), 9 | } 10 | 11 | impl Default for ClientAuth { 12 | fn default() -> Self { 13 | Self::None 14 | } 15 | } 16 | 17 | impl ClientAuth { 18 | pub fn generate_token(&self) -> String { 19 | match &self { 20 | ClientAuth::None => String::default(), 21 | 22 | #[cfg(feature = "client-auth-jwt")] 23 | ClientAuth::Jwt(jwt) => jwt.generate_token().unwrap(), 24 | } 25 | } 26 | 27 | pub fn validate_token(&self, token: &str) -> bool { 28 | match &self { 29 | ClientAuth::None => true, 30 | 31 | #[cfg(feature = "client-auth-jwt")] 32 | ClientAuth::Jwt(jwt) => jwt.validate_token(token), 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /coerce/src/remote/net/security/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod auth; 2 | 3 | pub use auth::*; 4 | -------------------------------------------------------------------------------- /coerce/src/remote/net/server/session/store.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::actor::message::{Handler, Message}; 3 | use crate::actor::{Actor, IntoActorId, LocalActorRef}; 4 | use crate::remote::net::message::ClientEvent; 5 | use crate::remote::net::server::session::RemoteSession; 6 | use std::collections::HashMap; 7 | use uuid::Uuid; 8 | 9 | pub struct RemoteSessionStore { 10 | sessions: HashMap>, 11 | } 12 | 13 | impl Actor for RemoteSessionStore {} 14 | 15 | impl RemoteSessionStore { 16 | pub fn new() -> RemoteSessionStore { 17 | RemoteSessionStore { 18 | sessions: HashMap::new(), 19 | } 20 | } 21 | } 22 | 23 | pub struct NewSession(pub RemoteSession); 24 | 25 | pub struct SessionWrite(pub i64, pub ClientEvent); 26 | 27 | pub struct SessionClosed(pub i64); 28 | 29 | impl Message for NewSession { 30 | type Result = LocalActorRef; 31 | } 32 | 33 | impl Message for SessionClosed { 34 | type Result = (); 35 | } 36 | 37 | impl Message for SessionWrite { 38 | type Result = (); 39 | } 40 | 41 | #[async_trait] 42 | impl Handler for RemoteSessionStore { 43 | async fn handle( 44 | &mut self, 45 | message: NewSession, 46 | ctx: &mut ActorContext, 47 | ) -> LocalActorRef { 48 | let session_id = message.0.id; 49 | let session = message.0; 50 | 51 | let session_actor = ctx 52 | .spawn( 53 | format!("session-{}", session_id.to_string()).into_actor_id(), 54 | session, 55 | ) 56 | .await 57 | .expect("unable to create session actor"); 58 | 59 | let node_id = ctx.system().remote().node_id(); 60 | debug!(node_id = node_id, "new session {}", &session_id); 61 | self.sessions.insert(session_id, session_actor.clone()); 62 | session_actor 63 | } 64 | } 65 | 66 | #[async_trait] 67 | impl Handler for RemoteSessionStore { 68 | async fn handle(&mut self, message: SessionClosed, ctx: &mut ActorContext) { 69 | self.sessions.remove(&message.0); 70 | let node_id = ctx.system().remote().node_id(); 71 | debug!(node_id = node_id, "disposed session {}", &message.0); 72 | } 73 | } 74 | 75 | #[async_trait] 76 | impl Handler for RemoteSessionStore { 77 | async fn handle(&mut self, message: SessionWrite, _ctx: &mut ActorContext) { 78 | match self.sessions.get(&message.0) { 79 | Some(session) => { 80 | trace!("writing to session {}", &message.0); 81 | if let Err(e) = session.notify(message) { 82 | error!("error while notifying session of write operation: {}", e); 83 | } 84 | } 85 | None => { 86 | warn!("attempted to write to session that couldn't be found"); 87 | } 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /coerce/src/remote/stream/alerts.rs: -------------------------------------------------------------------------------- 1 | // use std::net::SocketAddr; 2 | // use std::sync::Arc; 3 | // 4 | // pub struct AlertTopic; 5 | // 6 | // #[derive(Debug)] 7 | // pub enum SystemAlert { 8 | // // Fired when a client has attempted to connect to this system as a cluster peer, 9 | // // sent a valid identify payload, but the token failed validation and/or authentication. 10 | // AuthenticationFailure(Arc), 11 | // } 12 | // 13 | // pub struct AuthenticationFailure { 14 | // client_addr: SocketAddr, 15 | // token: Vec, 16 | // } 17 | // 18 | // impl AuthenticationFailure { 19 | // pub fn new(client_addr: SocketAddr, token: Vec) -> AuthenticationFailure { 20 | // Self { client_addr, token } 21 | // } 22 | // } 23 | -------------------------------------------------------------------------------- /coerce/src/remote/stream/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod alerts; 2 | pub mod mediator; 3 | pub mod pubsub; 4 | pub mod system; 5 | -------------------------------------------------------------------------------- /coerce/src/remote/system/cluster.rs: -------------------------------------------------------------------------------- 1 | use crate::remote::actor::message::{ClientWrite, GetNodes, NewClient, RegisterNode, UpdateNodes}; 2 | use crate::remote::cluster::node::{RemoteNode, RemoteNodeState}; 3 | use crate::remote::net::client::{ClientType, RemoteClientRef}; 4 | use crate::remote::net::message::SessionEvent; 5 | use crate::remote::system::{NodeId, RemoteActorSystem}; 6 | use std::sync::atomic::Ordering; 7 | 8 | impl RemoteActorSystem { 9 | pub async fn register_node(&self, node: RemoteNode) { 10 | self.inner 11 | .registry_ref 12 | .send(RegisterNode(node)) 13 | .await 14 | .unwrap() 15 | } 16 | 17 | pub fn notify_register_node(&self, node: RemoteNode) { 18 | let _ = self.inner.registry_ref.notify(RegisterNode(node)); 19 | } 20 | 21 | pub async fn get_nodes(&self) -> Vec { 22 | self.inner.registry_ref.send(GetNodes).await.unwrap() 23 | } 24 | 25 | pub async fn update_nodes(&self, nodes: Vec) { 26 | self.inner 27 | .registry_ref 28 | .send(UpdateNodes(nodes)) 29 | .await 30 | .unwrap() 31 | } 32 | 33 | pub async fn notify_node(&self, node_id: NodeId, message: SessionEvent) { 34 | self.inner 35 | .clients_ref 36 | .send(ClientWrite(node_id, message)) 37 | .await 38 | .unwrap() 39 | } 40 | 41 | pub fn current_leader(&self) -> Option { 42 | let n = self.inner.current_leader.load(Ordering::SeqCst); 43 | if n >= 0 { 44 | Some(n as NodeId) 45 | } else { 46 | None 47 | } 48 | } 49 | 50 | pub fn update_leader(&self, new_leader: NodeId) -> Option { 51 | let n = self 52 | .inner 53 | .current_leader 54 | .swap(new_leader as i64, Ordering::SeqCst); 55 | if n >= 0 { 56 | Some(n as NodeId) 57 | } else { 58 | None 59 | } 60 | } 61 | 62 | pub async fn get_remote_client(&self, addr: String) -> Option { 63 | self.client_registry() 64 | .send(NewClient { 65 | addr, 66 | client_type: ClientType::Worker, 67 | system: self.clone(), 68 | }) 69 | .await 70 | .expect("get client from RemoteClientRegistry") 71 | .map(RemoteClientRef::from) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /coerce/src/remote/system/mod.rs: -------------------------------------------------------------------------------- 1 | use chrono::{DateTime, Utc}; 2 | use std::sync::atomic::AtomicI64; 3 | use std::sync::Arc; 4 | 5 | use crate::actor::system::ActorSystem; 6 | use crate::actor::LocalActorRef; 7 | use crate::remote::actor::{ 8 | clients::RemoteClientRegistry, registry::RemoteRegistry, RemoteHandler, 9 | }; 10 | use crate::remote::cluster::builder::client::ClusterClientBuilder; 11 | use crate::remote::cluster::builder::worker::ClusterWorkerBuilder; 12 | use crate::remote::cluster::discovery::NodeDiscovery; 13 | use crate::remote::heartbeat::Heartbeat; 14 | use crate::remote::stream::mediator::StreamMediator; 15 | use crate::remote::system::builder::RemoteActorSystemBuilder; 16 | 17 | pub mod actor; 18 | pub mod builder; 19 | pub mod cluster; 20 | pub mod rpc; 21 | 22 | use crate::remote::config::RemoteSystemConfig; 23 | pub use actor::*; 24 | pub use cluster::*; 25 | pub use rpc::*; 26 | 27 | #[derive(Clone)] 28 | pub struct RemoteActorSystem { 29 | inner: Arc, 30 | } 31 | 32 | pub type NodeId = u64; 33 | pub type AtomicNodeId = AtomicI64; 34 | 35 | #[derive(Clone)] 36 | pub struct RemoteSystemCore { 37 | node_id: NodeId, 38 | inner: ActorSystem, 39 | started_at: DateTime, 40 | handler_ref: Arc>, 41 | registry_ref: LocalActorRef, 42 | clients_ref: LocalActorRef, 43 | discovery_ref: LocalActorRef, 44 | heartbeat_ref: LocalActorRef, 45 | mediator_ref: Option>, 46 | config: Arc, 47 | current_leader: Arc, 48 | } 49 | 50 | impl RemoteActorSystem { 51 | pub async fn shutdown(&self) { 52 | self.inner.shutdown().await; 53 | } 54 | } 55 | 56 | impl RemoteSystemCore { 57 | pub async fn shutdown(&self) { 58 | let _ = self.heartbeat_ref.stop().await; 59 | let _ = self.clients_ref.stop().await; 60 | 61 | if let Some(mediator_ref) = self.mediator_ref.as_ref() { 62 | let _ = mediator_ref.stop().await; 63 | } 64 | 65 | let _ = self.discovery_ref.stop().await; 66 | let _ = self.registry_ref.stop().await; 67 | 68 | info!("shutdown complete"); 69 | } 70 | } 71 | 72 | impl RemoteActorSystem { 73 | pub fn builder() -> RemoteActorSystemBuilder { 74 | RemoteActorSystemBuilder::new() 75 | } 76 | 77 | pub fn cluster_worker(self) -> ClusterWorkerBuilder { 78 | ClusterWorkerBuilder::new(self) 79 | } 80 | 81 | pub fn cluster_client(self) -> ClusterClientBuilder { 82 | ClusterClientBuilder::new(self) 83 | } 84 | 85 | pub fn config(&self) -> &RemoteSystemConfig { 86 | &self.inner.config 87 | } 88 | 89 | pub fn node_tag(&self) -> &str { 90 | self.inner.config.node_tag() 91 | } 92 | 93 | pub fn node_version(&self) -> &str { 94 | self.inner.config.node_version() 95 | } 96 | 97 | pub fn node_id(&self) -> NodeId { 98 | self.inner.node_id 99 | } 100 | 101 | pub fn started_at(&self) -> &DateTime { 102 | &self.inner.started_at 103 | } 104 | 105 | pub fn heartbeat(&self) -> &LocalActorRef { 106 | &self.inner.heartbeat_ref 107 | } 108 | 109 | pub fn registry(&self) -> &LocalActorRef { 110 | &self.inner.registry_ref 111 | } 112 | 113 | pub fn client_registry(&self) -> &LocalActorRef { 114 | &self.inner.clients_ref 115 | } 116 | 117 | pub fn node_discovery(&self) -> &LocalActorRef { 118 | &self.inner.discovery_ref 119 | } 120 | 121 | pub fn stream_mediator(&self) -> Option<&LocalActorRef> { 122 | self.inner.mediator_ref.as_ref() 123 | } 124 | 125 | pub fn actor_system(&self) -> &ActorSystem { 126 | &self.inner.actor_system() 127 | } 128 | } 129 | 130 | impl RemoteSystemCore { 131 | pub fn actor_system(&self) -> &ActorSystem { 132 | &self.inner 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /coerce/src/remote/tracing/mod.rs: -------------------------------------------------------------------------------- 1 | // use opentelemetry::trace::{TraceContextExt, TraceFlags}; 2 | // use tracing::Span; 3 | // use tracing_opentelemetry::OpenTelemetrySpanExt; 4 | // 5 | // #[inline] 6 | // pub fn extract_trace_identifier(span: &Span) -> String { 7 | // let context = span.context(); 8 | // let span = context.span(); 9 | // let span_context = span.span_context(); 10 | // format!( 11 | // "{:02x}-{:032x}-{:016x}-{:02x}", 12 | // 0, 13 | // span_context.trace_id(), 14 | // span_context.span_id(), 15 | // span_context.trace_flags() & TraceFlags::SAMPLED 16 | // ) 17 | // } 18 | -------------------------------------------------------------------------------- /coerce/src/sharding/builder.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::{Actor, ActorFactory}; 2 | use crate::remote::system::RemoteActorSystem; 3 | use crate::sharding::host::ShardAllocator; 4 | use crate::sharding::Sharding; 5 | use std::marker::PhantomData; 6 | 7 | pub struct ShardingBuilder { 8 | shard_allocator: Option>, 9 | shard_entity: Option, 10 | system: Option, 11 | _a: PhantomData, 12 | } 13 | 14 | impl Sharding { 15 | pub fn builder(system: RemoteActorSystem) -> ShardingBuilder { 16 | ShardingBuilder { 17 | shard_allocator: None, 18 | shard_entity: None, 19 | system: Some(system), 20 | _a: PhantomData, 21 | } 22 | } 23 | } 24 | 25 | impl ShardingBuilder { 26 | pub fn with_entity_type(&mut self, entity_type: S) -> &mut Self { 27 | self.shard_entity = Some(entity_type.to_string()); 28 | self 29 | } 30 | 31 | pub fn with_allocator(&mut self, allocator: S) -> &mut Self { 32 | self.shard_allocator = Some(Box::new(allocator)); 33 | self 34 | } 35 | 36 | pub async fn build(&mut self) -> Sharding { 37 | Sharding::start( 38 | self.shard_entity 39 | .take() 40 | .unwrap_or_else(|| A::Actor::type_name().to_string()), 41 | self.system.take().unwrap(), 42 | self.shard_allocator.take(), 43 | ) 44 | .await 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /coerce/src/sharding/coordinator/discovery.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::actor::message::{Handler, Message}; 3 | use crate::remote::cluster::node::RemoteNode; 4 | use crate::sharding::coordinator::balancing::Rebalance; 5 | use crate::sharding::coordinator::{ShardCoordinator, ShardHostState, ShardHostStatus}; 6 | use crate::sharding::host::ShardHost; 7 | 8 | use crate::remote::stream::pubsub::Receive; 9 | use crate::remote::stream::system::{ClusterEvent, SystemEvent, SystemTopic}; 10 | use crate::remote::system::NodeId; 11 | use std::collections::hash_map::Entry; 12 | use std::sync::Arc; 13 | 14 | #[async_trait] 15 | impl Handler> for ShardCoordinator { 16 | async fn handle(&mut self, message: Receive, ctx: &mut ActorContext) { 17 | match message.0.as_ref() { 18 | SystemEvent::Cluster(event) => match event { 19 | ClusterEvent::NodeAdded(node) => { 20 | self.on_node_discovered(node.as_ref(), ctx); 21 | } 22 | 23 | ClusterEvent::NodeRemoved(node) => { 24 | self.on_node_removed(node.id, ctx).await; 25 | } 26 | _ => {} 27 | }, 28 | _ => {} 29 | } 30 | } 31 | } 32 | 33 | impl ShardCoordinator { 34 | pub fn on_node_discovered(&mut self, new_node: &RemoteNode, ctx: &ActorContext) { 35 | match self.hosts.entry(new_node.id) { 36 | Entry::Occupied(mut node) => { 37 | let node = node.get_mut(); 38 | if node.status != ShardHostStatus::Ready { 39 | node.status = ShardHostStatus::Ready; 40 | self.schedule_full_rebalance(ctx); 41 | } 42 | } 43 | 44 | Entry::Vacant(vacant_entry) => { 45 | let remote = ctx.system().remote_owned(); 46 | 47 | debug!( 48 | "new shard host (node_id={}, tag={}, addr={})", 49 | new_node.id, &new_node.tag, &new_node.addr 50 | ); 51 | 52 | vacant_entry.insert(ShardHostState { 53 | node_id: new_node.id, 54 | node_tag: new_node.tag.clone(), 55 | shards: Default::default(), 56 | actor: ShardHost::remote_ref(&self.shard_entity, new_node.id, &remote), 57 | status: ShardHostStatus::Ready/*TODO: shard hosts may not be immediately ready*/, 58 | }); 59 | 60 | self.schedule_full_rebalance(ctx); 61 | } 62 | } 63 | } 64 | 65 | pub async fn on_node_removed(&mut self, node_id: NodeId, ctx: &mut ActorContext) { 66 | match self.hosts.entry(node_id) { 67 | Entry::Occupied(_) => self.handle(Rebalance::NodeUnavailable(node_id), ctx).await, 68 | Entry::Vacant(_) => {} 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /coerce/src/sharding/coordinator/factory.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::LocalActorRef; 2 | use crate::sharding::coordinator::ShardCoordinator; 3 | use crate::sharding::host::ShardHost; 4 | use crate::singleton::factory::SingletonFactory; 5 | 6 | pub struct CoordinatorFactory { 7 | shard_entity: String, 8 | local_shard_host: LocalActorRef, 9 | } 10 | 11 | impl CoordinatorFactory { 12 | pub fn new(shard_entity: String, local_shard_host: LocalActorRef) -> Self { 13 | CoordinatorFactory { 14 | shard_entity, 15 | local_shard_host, 16 | } 17 | } 18 | } 19 | 20 | impl SingletonFactory for CoordinatorFactory { 21 | type Actor = ShardCoordinator; 22 | 23 | fn create(&self) -> Self::Actor { 24 | ShardCoordinator::new(self.shard_entity.clone(), self.local_shard_host.clone()) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /coerce/src/sharding/coordinator/stream.rs: -------------------------------------------------------------------------------- 1 | use crate::remote::net::StreamData; 2 | use crate::remote::stream::pubsub::Topic; 3 | use std::sync::Arc; 4 | 5 | pub struct ShardingTopic { 6 | sharded_entity: Arc, 7 | } 8 | 9 | pub enum ShardingEvent { 10 | Rebalance, 11 | } 12 | 13 | impl Topic for ShardingTopic { 14 | type Message = ShardingEvent; 15 | 16 | fn topic_name() -> &'static str { 17 | "sharding" 18 | } 19 | 20 | fn key(&self) -> String { 21 | format!("sharding-events-{}", &self.sharded_entity) 22 | } 23 | } 24 | 25 | impl StreamData for ShardingEvent { 26 | fn read_from_bytes(_data: Vec) -> Option { 27 | todo!() 28 | } 29 | 30 | fn write_to_bytes(&self) -> Option> { 31 | todo!() 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /coerce/src/sharding/host/stats.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::actor::message::{Handler, Message}; 3 | use crate::actor::{ActorRefErr, LocalActorRef}; 4 | use crate::remote::system::NodeId; 5 | use crate::sharding::coordinator::ShardId; 6 | use crate::sharding::host::ShardHost; 7 | use crate::sharding::shard::stats::{GetShardStats, ShardStats}; 8 | use crate::sharding::shard::Shard; 9 | use futures::future::join_all; 10 | use std::collections::HashMap; 11 | use tokio::sync::oneshot; 12 | 13 | pub struct GetStats; 14 | 15 | #[derive(Serialize, Deserialize)] 16 | pub struct RemoteShard { 17 | pub shard_id: ShardId, 18 | pub node_id: NodeId, 19 | } 20 | 21 | #[derive(Serialize, Deserialize)] 22 | pub struct HostStats { 23 | pub requests_pending_shard_allocation_count: usize, 24 | pub hosted_shard_count: usize, 25 | pub remote_shard_count: usize, 26 | pub hosted_shards: HashMap, 27 | pub remote_shards: Vec, 28 | } 29 | 30 | impl Message for GetStats { 31 | type Result = oneshot::Receiver; 32 | } 33 | 34 | #[async_trait] 35 | impl Handler for ShardHost { 36 | async fn handle( 37 | &mut self, 38 | _message: GetStats, 39 | _ctx: &mut ActorContext, 40 | ) -> oneshot::Receiver { 41 | let (tx, rx) = oneshot::channel(); 42 | let actors: Vec> = self 43 | .hosted_shards 44 | .iter() 45 | .filter_map(|s| s.1.actor_ref().cloned()) 46 | .collect(); 47 | 48 | let stats = HostStats { 49 | requests_pending_shard_allocation_count: self.requests_pending_shard_allocation.len(), 50 | hosted_shard_count: self.hosted_shards.len(), 51 | remote_shard_count: self.remote_shards.len(), 52 | hosted_shards: Default::default(), 53 | remote_shards: self 54 | .remote_shards 55 | .iter() 56 | .map(|s| RemoteShard { 57 | node_id: s.1.node_id().unwrap(), 58 | shard_id: *s.0, 59 | }) 60 | .collect(), 61 | }; 62 | 63 | tokio::spawn(async move { 64 | let mut stats = stats; 65 | let shard_stats: Vec> = 66 | join_all(actors.iter().map(|a| a.send(GetShardStats))).await; 67 | 68 | stats.hosted_shards = shard_stats 69 | .into_iter() 70 | .filter(|s| s.is_ok()) 71 | .map(|s| { 72 | let s = s.unwrap(); 73 | (s.shard_id, s) 74 | }) 75 | .collect(); 76 | 77 | let _ = tx.send(stats); 78 | }); 79 | 80 | rx 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /coerce/src/sharding/proto/mod.rs: -------------------------------------------------------------------------------- 1 | // @generated 2 | 3 | pub mod sharding; 4 | -------------------------------------------------------------------------------- /coerce/src/sharding/shard/message.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::message::{Message, MessageUnwrapErr, MessageWrapErr}; 2 | use crate::actor::{ActorId, ActorRefErr, BoxedActorRef, IntoActorId}; 3 | use crate::sharding::proto::sharding as proto; 4 | use crate::sharding::shard::RecipeRef; 5 | use protobuf::Message as ProtoMessage; 6 | use std::sync::Arc; 7 | 8 | pub struct StartEntity { 9 | pub actor_id: ActorId, 10 | pub recipe: RecipeRef, 11 | } 12 | 13 | pub struct RemoveEntity { 14 | pub actor_id: ActorId, 15 | } 16 | 17 | pub struct PassivateEntity { 18 | pub actor_id: ActorId, 19 | } 20 | 21 | pub struct EntityStartResult { 22 | pub actor_id: ActorId, 23 | pub result: Result, 24 | pub is_shard_recovery: bool, 25 | } 26 | 27 | impl Message for EntityStartResult { 28 | type Result = (); 29 | } 30 | 31 | impl Message for StartEntity { 32 | type Result = (); 33 | 34 | fn as_bytes(&self) -> Result, MessageWrapErr> { 35 | proto::StartEntity { 36 | actor_id: self.actor_id.to_string(), 37 | recipe: self.recipe.as_ref().clone(), 38 | ..Default::default() 39 | } 40 | .write_to_bytes() 41 | .map_err(|_| MessageWrapErr::SerializationErr) 42 | } 43 | 44 | fn from_bytes(b: Vec) -> Result { 45 | proto::StartEntity::parse_from_bytes(&b) 46 | .map(|r| Self { 47 | actor_id: r.actor_id.into_actor_id(), 48 | recipe: Arc::new(r.recipe), 49 | }) 50 | .map_err(|_e| MessageUnwrapErr::DeserializationErr) 51 | } 52 | 53 | fn read_remote_result(_: Vec) -> Result { 54 | Ok(()) 55 | } 56 | 57 | fn write_remote_result(_res: Self::Result) -> Result, MessageWrapErr> { 58 | Ok(vec![]) 59 | } 60 | } 61 | 62 | impl Message for RemoveEntity { 63 | type Result = (); 64 | 65 | fn as_bytes(&self) -> Result, MessageWrapErr> { 66 | proto::RemoveEntity { 67 | actor_id: self.actor_id.to_string(), 68 | ..Default::default() 69 | } 70 | .write_to_bytes() 71 | .map_err(|_| MessageWrapErr::SerializationErr) 72 | } 73 | 74 | fn from_bytes(b: Vec) -> Result { 75 | proto::RemoveEntity::parse_from_bytes(&b) 76 | .map(|r| Self { 77 | actor_id: r.actor_id.into_actor_id(), 78 | }) 79 | .map_err(|_e| MessageUnwrapErr::DeserializationErr) 80 | } 81 | 82 | fn read_remote_result(_: Vec) -> Result { 83 | Ok(()) 84 | } 85 | 86 | fn write_remote_result(_res: Self::Result) -> Result, MessageWrapErr> { 87 | Ok(vec![]) 88 | } 89 | } 90 | 91 | impl Message for PassivateEntity { 92 | type Result = (); 93 | 94 | fn as_bytes(&self) -> Result, MessageWrapErr> { 95 | proto::PassivateEntity { 96 | actor_id: self.actor_id.to_string(), 97 | ..Default::default() 98 | } 99 | .write_to_bytes() 100 | .map_err(|_| MessageWrapErr::SerializationErr) 101 | } 102 | 103 | fn from_bytes(b: Vec) -> Result { 104 | proto::PassivateEntity::parse_from_bytes(&b) 105 | .map(|r| Self { 106 | actor_id: r.actor_id.into_actor_id(), 107 | }) 108 | .map_err(|_e| MessageUnwrapErr::DeserializationErr) 109 | } 110 | 111 | fn read_remote_result(_: Vec) -> Result { 112 | Ok(()) 113 | } 114 | 115 | fn write_remote_result(_res: Self::Result) -> Result, MessageWrapErr> { 116 | Ok(vec![]) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /coerce/src/sharding/shard/passivation/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::actor::message::{Handler, Message}; 3 | use crate::actor::scheduler::timer::{Timer, TimerTick}; 4 | use crate::actor::{Actor, LocalActorRef}; 5 | use crate::sharding::shard::Shard; 6 | 7 | use std::time::Duration; 8 | 9 | pub struct PassivationConfig { 10 | entity_passivation_tick: Duration, 11 | entity_passivation_timeout: Duration, 12 | entity_deletion_timeout: Option, 13 | } 14 | 15 | pub struct PassivationWorker { 16 | shard: LocalActorRef, 17 | config: PassivationConfig, 18 | timer: Option, 19 | } 20 | 21 | impl PassivationWorker { 22 | pub fn new(shard: LocalActorRef, config: PassivationConfig) -> Self { 23 | let timer = None; 24 | PassivationWorker { 25 | shard, 26 | config, 27 | timer, 28 | } 29 | } 30 | } 31 | 32 | #[derive(Clone)] 33 | struct PassivationTimerTick; 34 | 35 | impl Message for PassivationTimerTick { 36 | type Result = (); 37 | } 38 | 39 | impl TimerTick for PassivationTimerTick {} 40 | 41 | #[async_trait] 42 | impl Actor for PassivationWorker { 43 | async fn started(&mut self, ctx: &mut ActorContext) { 44 | self.timer = Some(Timer::start( 45 | self.actor_ref(ctx), 46 | self.config.entity_passivation_tick, 47 | PassivationTimerTick, 48 | )); 49 | } 50 | 51 | async fn stopped(&mut self, _ctx: &mut ActorContext) { 52 | if let Some(entity_passivation_timer) = self.timer.take() { 53 | let _ = entity_passivation_timer.stop(); 54 | } 55 | } 56 | } 57 | #[async_trait] 58 | impl Handler for PassivationWorker { 59 | async fn handle(&mut self, _message: PassivationTimerTick, _ctx: &mut ActorContext) { 60 | // Still not sure if we should let entities themselves handle their own passivation 61 | // or if we have some sort of shard-level passivation timer here.. 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /coerce/src/sharding/shard/stats.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::actor::message::{Handler, Message, MessageUnwrapErr, MessageWrapErr, ToBytes}; 3 | 4 | use crate::sharding::coordinator::ShardId; 5 | 6 | use crate::remote::system::NodeId; 7 | use crate::sharding::proto::sharding as proto; 8 | use crate::sharding::shard::Shard; 9 | use protobuf::Message as ProtoMessage; 10 | use std::collections::HashSet; 11 | 12 | #[derive(Serialize, Deserialize)] 13 | pub struct GetShardStats; 14 | 15 | #[derive(Serialize, Deserialize)] 16 | pub struct ShardStats { 17 | pub shard_id: ShardId, 18 | pub node_id: NodeId, 19 | pub entities: HashSet, 20 | } 21 | 22 | #[async_trait] 23 | impl Handler for Shard { 24 | async fn handle(&mut self, _message: GetShardStats, ctx: &mut ActorContext) -> ShardStats { 25 | let node_id = ctx.system().remote().node_id(); 26 | let shard_id = self.shard_id; 27 | 28 | ShardStats { 29 | shard_id, 30 | node_id, 31 | entities: self.entities.keys().map(|e| e.to_string()).collect(), 32 | } 33 | } 34 | } 35 | 36 | impl Message for GetShardStats { 37 | type Result = ShardStats; 38 | 39 | fn as_bytes(&self) -> Result, MessageWrapErr> { 40 | proto::GetShardStats { 41 | ..Default::default() 42 | } 43 | .to_bytes() 44 | } 45 | 46 | fn from_bytes(_: Vec) -> Result { 47 | Ok(Self) 48 | } 49 | 50 | fn read_remote_result(res: Vec) -> Result { 51 | proto::ShardStats::parse_from_bytes(&res).map_or_else( 52 | |_| Err(MessageUnwrapErr::DeserializationErr), 53 | |s| { 54 | Ok(ShardStats { 55 | shard_id: s.shard_id, 56 | node_id: s.node_id, 57 | entities: s.entities.into_iter().map(|e| e.to_string()).collect(), 58 | }) 59 | }, 60 | ) 61 | } 62 | 63 | fn write_remote_result(res: ShardStats) -> Result, MessageWrapErr> { 64 | proto::ShardStats { 65 | shard_id: res.shard_id, 66 | node_id: res.node_id, 67 | entities: res.entities.into_iter().map(|e| e.to_string()).collect(), 68 | ..Default::default() 69 | } 70 | .to_bytes() 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /coerce/src/singleton/factory.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::Actor; 2 | 3 | pub trait SingletonFactory: 'static + Sync + Send { 4 | type Actor: Actor; 5 | 6 | fn create(&self) -> Self::Actor; 7 | } 8 | 9 | impl SingletonFactory for F 10 | where 11 | F: Fn() -> A + 'static + Sync + Send, 12 | { 13 | type Actor = A; 14 | 15 | fn create(&self) -> A { 16 | self() 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /coerce/src/singleton/manager/start.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::actor::message::{Handler, Message}; 3 | use crate::actor::{Actor, ActorRefErr, IntoActor, LocalActorRef, PipeTo}; 4 | use crate::singleton::factory::SingletonFactory; 5 | use crate::singleton::manager::{Manager, SingletonStarted, State}; 6 | use crate::singleton::proxy; 7 | 8 | pub enum ActorStartResult { 9 | Started(LocalActorRef), 10 | Failed(ActorRefErr), 11 | } 12 | 13 | impl Message for ActorStartResult { 14 | type Result = (); 15 | } 16 | 17 | impl Manager { 18 | pub async fn start_actor(&mut self, ctx: &ActorContext) { 19 | let state = self.factory.create(); 20 | let sys = self.sys.actor_system().clone(); 21 | let manager_ref = self.actor_ref(ctx); 22 | let actor_id = self.singleton_actor_id.clone(); 23 | 24 | async move { 25 | match state.into_actor(Some(actor_id), &sys).await { 26 | Ok(actor_ref) => ActorStartResult::Started(actor_ref), 27 | Err(e) => ActorStartResult::Failed(e), 28 | } 29 | } 30 | .pipe_to(manager_ref.into()); 31 | } 32 | } 33 | 34 | #[async_trait] 35 | impl Handler> for Manager { 36 | async fn handle(&mut self, message: ActorStartResult, ctx: &mut ActorContext) { 37 | match message { 38 | ActorStartResult::Started(actor_ref) => { 39 | match &self.state { 40 | State::Starting { .. } => { 41 | debug!("singleton actor started"); 42 | } 43 | _ => { 44 | warn!("Invalid state, expected `Starting`"); 45 | } 46 | } 47 | 48 | self.state = State::Running { 49 | actor_ref: actor_ref.clone(), 50 | }; 51 | 52 | let _ = self 53 | .proxy 54 | .notify(proxy::SingletonStarted::new(actor_ref.into())); 55 | 56 | self.notify_managers( 57 | SingletonStarted { 58 | source_node_id: self.node_id, 59 | }, 60 | ctx, 61 | ) 62 | .await; 63 | } 64 | ActorStartResult::Failed(e) => { 65 | error!(error = format!("{}", e), "singleton actor failed to start") 66 | } 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /coerce/src/singleton/manager/status.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::actor::message::{ 3 | FromBytes, Handler, Message, MessageUnwrapErr, MessageWrapErr, ToBytes, 4 | }; 5 | use crate::remote::system::NodeId; 6 | use crate::singleton::factory::SingletonFactory; 7 | use crate::singleton::manager::{Manager, State}; 8 | use crate::singleton::proto::singleton as proto; 9 | use protobuf::EnumOrUnknown; 10 | 11 | pub struct GetStatus { 12 | source_node_id: NodeId, 13 | } 14 | 15 | pub struct ManagerStatus { 16 | singleton_state: SingletonState, 17 | } 18 | 19 | pub enum SingletonState { 20 | Joining, 21 | Idle, 22 | Starting, 23 | Running, 24 | Stopping, 25 | } 26 | 27 | #[async_trait] 28 | impl Handler for Manager { 29 | async fn handle(&mut self, message: GetStatus, _ctx: &mut ActorContext) -> ManagerStatus { 30 | debug!( 31 | "Singleton manager status request from node_id={}", 32 | message.source_node_id 33 | ); 34 | 35 | let singleton_state = match &self.state { 36 | State::Joining { .. } => SingletonState::Joining, 37 | State::Idle => SingletonState::Idle, 38 | State::Starting { .. } => SingletonState::Starting, 39 | State::Running { .. } => SingletonState::Running, 40 | State::Stopping { .. } => SingletonState::Stopping, 41 | }; 42 | 43 | ManagerStatus { singleton_state } 44 | } 45 | } 46 | 47 | impl Message for GetStatus { 48 | type Result = ManagerStatus; 49 | 50 | fn as_bytes(&self) -> Result, MessageWrapErr> { 51 | proto::GetStatus { 52 | source_node_id: self.source_node_id, 53 | ..Default::default() 54 | } 55 | .to_bytes() 56 | } 57 | 58 | fn read_remote_result(buf: Vec) -> Result { 59 | proto::ManagerStatus::from_bytes(buf).map(|s| ManagerStatus { 60 | singleton_state: s.singleton_state.unwrap().into(), 61 | }) 62 | } 63 | 64 | fn write_remote_result(res: Self::Result) -> Result, MessageWrapErr> { 65 | proto::ManagerStatus { 66 | singleton_state: EnumOrUnknown::new(res.singleton_state.into()), 67 | ..Default::default() 68 | } 69 | .to_bytes() 70 | } 71 | } 72 | 73 | impl Into for SingletonState { 74 | fn into(self) -> proto::SingletonState { 75 | match self { 76 | SingletonState::Joining => proto::SingletonState::JOINING, 77 | SingletonState::Idle => proto::SingletonState::IDLE, 78 | SingletonState::Starting => proto::SingletonState::STARTING, 79 | SingletonState::Running => proto::SingletonState::RUNNING, 80 | SingletonState::Stopping => proto::SingletonState::STOPPING, 81 | } 82 | } 83 | } 84 | 85 | impl From for SingletonState { 86 | fn from(value: proto::SingletonState) -> Self { 87 | match value { 88 | proto::SingletonState::JOINING => Self::Joining, 89 | proto::SingletonState::IDLE => Self::Idle, 90 | proto::SingletonState::STARTING => Self::Starting, 91 | proto::SingletonState::RUNNING => Self::Running, 92 | proto::SingletonState::STOPPING => Self::Stopping, 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /coerce/src/singleton/proto/mod.rs: -------------------------------------------------------------------------------- 1 | // @generated 2 | 3 | pub mod singleton; 4 | -------------------------------------------------------------------------------- /coerce/src/singleton/proxy/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::actor::message::{Handler, Message}; 3 | use crate::actor::{Actor, ActorRef, ToActorId}; 4 | use crate::singleton::proxy::send::Buffered; 5 | use std::collections::VecDeque; 6 | 7 | pub mod send; 8 | 9 | pub enum ProxyState { 10 | Buffered { 11 | request_queue: VecDeque>>, 12 | }, 13 | 14 | Active { 15 | actor_ref: ActorRef, 16 | }, 17 | } 18 | 19 | pub struct Proxy { 20 | state: ProxyState, 21 | } 22 | 23 | impl Proxy { 24 | pub fn new() -> Self { 25 | Self { 26 | state: ProxyState::Buffered { 27 | request_queue: VecDeque::new(), 28 | }, 29 | } 30 | } 31 | } 32 | 33 | impl ProxyState { 34 | pub fn is_active(&self) -> bool { 35 | matches!(&self, &ProxyState::Active { .. }) 36 | } 37 | } 38 | 39 | #[async_trait] 40 | impl Actor for Proxy {} 41 | 42 | pub struct SingletonStarted { 43 | actor_ref: ActorRef, 44 | } 45 | 46 | impl SingletonStarted { 47 | pub fn new(actor_ref: ActorRef) -> Self { 48 | Self { actor_ref } 49 | } 50 | } 51 | 52 | pub struct SingletonStopping; 53 | 54 | impl Message for SingletonStarted { 55 | type Result = (); 56 | } 57 | 58 | impl Message for SingletonStopping { 59 | type Result = (); 60 | } 61 | 62 | #[async_trait] 63 | impl Handler> for Proxy { 64 | async fn handle(&mut self, message: SingletonStarted, ctx: &mut ActorContext) { 65 | let actor_ref = message.actor_ref; 66 | 67 | match &mut self.state { 68 | ProxyState::Buffered { request_queue } => { 69 | if request_queue.len() > 0 { 70 | debug!( 71 | buffered_msgs = request_queue.len(), 72 | actor_ref = format!("{}", &actor_ref), 73 | "emitting buffered messages", 74 | ); 75 | 76 | while let Some(mut buffered) = request_queue.pop_front() { 77 | buffered.send(actor_ref.clone()); 78 | } 79 | } 80 | } 81 | _ => {} 82 | } 83 | 84 | debug!( 85 | singleton_actor = format!("{}", actor_ref), 86 | "singleton proxy active - singleton started" 87 | ); 88 | self.state = ProxyState::Active { actor_ref }; 89 | } 90 | } 91 | 92 | #[async_trait] 93 | impl Handler for Proxy { 94 | async fn handle(&mut self, _: SingletonStopping, ctx: &mut ActorContext) { 95 | debug!("singleton actor stopped, buffering messages"); 96 | 97 | if self.state.is_active() { 98 | self.state = ProxyState::Buffered { 99 | request_queue: VecDeque::new(), 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /coerce/src/singleton/proxy/send.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::context::ActorContext; 2 | use crate::actor::message::{Handler, Message}; 3 | use crate::actor::{Actor, ActorRef, ActorRefErr}; 4 | use crate::singleton::factory::SingletonFactory; 5 | use crate::singleton::proxy::{Proxy, ProxyState}; 6 | use tokio::sync::oneshot::Sender; 7 | 8 | pub struct Deliver { 9 | message: Option, 10 | result_channel: Option>>, 11 | } 12 | 13 | impl Deliver { 14 | pub fn new(message: M, result_channel: Option>>) -> Self { 15 | Self { 16 | message: Some(message), 17 | result_channel, 18 | } 19 | } 20 | } 21 | 22 | impl Deliver { 23 | pub fn deliver(&mut self, actor: ActorRef) 24 | where 25 | A: Handler, 26 | { 27 | let message = self.message.take().unwrap(); 28 | let result_channel = self.result_channel.take().unwrap(); 29 | tokio::spawn(async move { 30 | debug!(msg_type = M::type_name(), "delivering message to singleton"); 31 | 32 | let res = actor.send(message).await; 33 | result_channel.send(res) 34 | }); 35 | } 36 | } 37 | 38 | pub trait Buffered: 'static + Sync + Send { 39 | fn send(&mut self, actor_ref: ActorRef); 40 | } 41 | 42 | impl Buffered for Deliver 43 | where 44 | A: Handler, 45 | { 46 | fn send(&mut self, actor_ref: ActorRef) { 47 | self.deliver(actor_ref) 48 | } 49 | } 50 | 51 | impl Message for Deliver { 52 | type Result = (); 53 | } 54 | 55 | #[async_trait] 56 | impl Handler> for Proxy 57 | where 58 | A: Handler, 59 | { 60 | async fn handle(&mut self, mut message: Deliver, ctx: &mut ActorContext) { 61 | match &mut self.state { 62 | ProxyState::Buffered { request_queue } => { 63 | request_queue.push_back(Box::new(message)); 64 | debug!( 65 | msg_type = M::type_name(), 66 | buffered_msgs = request_queue.len(), 67 | "singleton proxy buffered message", 68 | ); 69 | } 70 | 71 | ProxyState::Active { actor_ref } => { 72 | message.deliver(actor_ref.clone()); 73 | debug!(msg_type = M::type_name(), "singleton proxy sent message"); 74 | } 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /coerce/tests/sharding/mod.rs: -------------------------------------------------------------------------------- 1 | use coerce::actor::context::ActorContext; 2 | 3 | use coerce::actor::{Actor, ActorCreationErr, ActorFactory, ActorRecipe}; 4 | 5 | use std::collections::HashMap; 6 | use std::sync::atomic::AtomicBool; 7 | use std::sync::atomic::Ordering::Relaxed; 8 | use std::sync::Arc; 9 | 10 | #[derive(Clone)] 11 | pub struct ShardedActor { 12 | id: String, 13 | actor_started: Arc, 14 | actor_stopped: Arc, 15 | actor_dropped: Arc, 16 | } 17 | 18 | #[async_trait] 19 | impl Actor for ShardedActor { 20 | async fn stopped(&mut self, _ctx: &mut ActorContext) { 21 | self.actor_stopped.store(true, Relaxed); 22 | } 23 | } 24 | 25 | pub struct ShardedActorRecipe { 26 | id: String, 27 | } 28 | 29 | impl ActorRecipe for ShardedActorRecipe { 30 | fn read_from_bytes(bytes: &Vec) -> Option { 31 | Some(Self { 32 | id: String::from_utf8(bytes.clone()).unwrap(), 33 | }) 34 | } 35 | 36 | fn write_to_bytes(&self) -> Option> { 37 | Some(self.id.clone().into_bytes()) 38 | } 39 | } 40 | 41 | pub struct ShardedActorFactory { 42 | state_store: Arc>, 43 | } 44 | 45 | impl Clone for ShardedActorFactory { 46 | fn clone(&self) -> Self { 47 | Self { 48 | state_store: self.state_store.clone(), 49 | } 50 | } 51 | } 52 | 53 | impl ShardedActorFactory { 54 | pub fn new(actors: Vec) -> Self { 55 | Self { 56 | state_store: Arc::new(actors.into_iter().map(|a| (a.id.clone(), a)).collect()), 57 | } 58 | } 59 | 60 | pub fn assert_actor_started(&self, id: &str) { 61 | let actor_started = self 62 | .state_store 63 | .get(id) 64 | .unwrap() 65 | .actor_started 66 | .load(Relaxed); 67 | assert!(actor_started) 68 | } 69 | pub fn assert_actor_stopped(&self, id: &str) { 70 | let actor_stopped = self 71 | .state_store 72 | .get(id) 73 | .unwrap() 74 | .actor_stopped 75 | .load(Relaxed); 76 | assert!(actor_stopped) 77 | } 78 | 79 | pub fn assert_actor_dropped(&self, id: &str) { 80 | let actor_dropped = self 81 | .state_store 82 | .get(id) 83 | .unwrap() 84 | .actor_dropped 85 | .load(Relaxed); 86 | assert!(actor_dropped) 87 | } 88 | } 89 | 90 | #[async_trait] 91 | impl ActorFactory for ShardedActorFactory { 92 | type Actor = ShardedActor; 93 | type Recipe = ShardedActorRecipe; 94 | 95 | async fn create(&self, recipe: ShardedActorRecipe) -> Result { 96 | self.state_store.get(&recipe.id).cloned().ok_or_else(|| { 97 | ActorCreationErr::InvalidRecipe(format!( 98 | "no actor with id `{}` in state_store", 99 | &recipe.id 100 | )) 101 | }) 102 | } 103 | } 104 | 105 | impl Drop for ShardedActor { 106 | fn drop(&mut self) { 107 | self.actor_dropped.store(true, Relaxed); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /coerce/tests/system/mod.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LeonHartley/Coerce-rs/a2d3f19cd92ce1776b4dc0be6616784d53ff4b89/coerce/tests/system/mod.rs -------------------------------------------------------------------------------- /coerce/tests/test_actor_describe.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /coerce/tests/test_actor_lifecycle.rs: -------------------------------------------------------------------------------- 1 | use coerce::actor::context::ActorStatus; 2 | use coerce::actor::system::ActorSystem; 3 | use coerce::actor::ActorRefErr; 4 | 5 | use util::*; 6 | 7 | pub mod util; 8 | 9 | #[macro_use] 10 | extern crate serde; 11 | 12 | #[macro_use] 13 | extern crate async_trait; 14 | 15 | #[tokio::test] 16 | pub async fn test_actor_lifecycle_started() { 17 | let actor_ref = ActorSystem::new() 18 | .new_anon_actor(TestActor::new()) 19 | .await 20 | .unwrap(); 21 | 22 | let status = actor_ref.status().await; 23 | 24 | actor_ref.stop().await.expect("actor stop"); 25 | assert_eq!(status, Ok(ActorStatus::Started)) 26 | } 27 | 28 | #[tokio::test] 29 | pub async fn test_actor_lifecycle_stopping() { 30 | let actor_ref = ActorSystem::new() 31 | .new_anon_actor(TestActor::new()) 32 | .await 33 | .unwrap(); 34 | 35 | let status = actor_ref.status().await; 36 | let stopping = actor_ref.stop().await; 37 | let msg_send = actor_ref.status().await; 38 | 39 | assert_eq!(status, Ok(ActorStatus::Started)); 40 | assert_eq!(stopping, Ok(())); 41 | assert_eq!(msg_send, Err(ActorRefErr::InvalidRef)); 42 | } 43 | -------------------------------------------------------------------------------- /coerce/tests/test_actor_system.rs: -------------------------------------------------------------------------------- 1 | use coerce::actor::system::ActorSystem; 2 | use coerce::actor::{get_actor, new_actor, new_actor_id}; 3 | 4 | use util::*; 5 | 6 | pub mod util; 7 | 8 | #[macro_use] 9 | extern crate serde; 10 | 11 | #[macro_use] 12 | extern crate async_trait; 13 | 14 | #[tokio::test] 15 | pub async fn test_system_global_get_actor() { 16 | create_trace_logger(); 17 | 18 | let actor_ref = new_actor(TestActor::new()).await.unwrap(); 19 | 20 | let _ = actor_ref 21 | .exec(|mut actor| { 22 | actor.counter = 1337; 23 | }) 24 | .await; 25 | 26 | let actor = get_actor::(actor_ref.actor_id().clone()) 27 | .await 28 | .unwrap(); 29 | 30 | let counter = actor.exec(|actor| actor.counter).await; 31 | 32 | assert_eq!(counter, Ok(1337)); 33 | } 34 | 35 | #[tokio::test] 36 | pub async fn test_system_get_tracked_actor() { 37 | create_trace_logger(); 38 | 39 | let ctx = ActorSystem::new(); 40 | 41 | let actor_ref = ctx.new_tracked_actor(TestActor::new()).await.unwrap(); 42 | 43 | let _ = actor_ref 44 | .exec(|mut actor| { 45 | actor.counter = 1337; 46 | }) 47 | .await; 48 | 49 | let actor = ctx 50 | .get_tracked_actor::(actor_ref.id.clone()) 51 | .await 52 | .unwrap(); 53 | let counter = actor.exec(|actor| actor.counter).await; 54 | 55 | assert_eq!(counter, Ok(1337)); 56 | } 57 | 58 | #[tokio::test] 59 | pub async fn test_system_get_actor_not_found() { 60 | create_trace_logger(); 61 | 62 | let ctx = ActorSystem::new(); 63 | let actor = ctx.get_tracked_actor::(new_actor_id()).await; 64 | 65 | assert_eq!(actor.is_none(), true); 66 | } 67 | 68 | #[tokio::test] 69 | pub async fn test_system_stop_tracked_actor_get_not_found() { 70 | create_trace_logger(); 71 | 72 | let ctx = ActorSystem::new(); 73 | 74 | let actor_ref = ctx.new_tracked_actor(TestActor::new()).await.unwrap(); 75 | 76 | let _ = actor_ref 77 | .exec(|mut actor| { 78 | actor.counter = 1337; 79 | }) 80 | .await; 81 | 82 | let _stop = actor_ref.stop().await; 83 | 84 | let actor = ctx 85 | .get_tracked_actor::(actor_ref.actor_id().clone()) 86 | .await; 87 | 88 | assert_eq!(actor.is_none(), true); 89 | } 90 | -------------------------------------------------------------------------------- /coerce/tests/test_actor_system_blocking.rs: -------------------------------------------------------------------------------- 1 | // use crate::util::TestActor; 2 | // use coerce::actor::scheduler::ActorType; 3 | // use coerce::actor::system::ActorSystem; 4 | // use coerce::actor::ToActorId; 5 | // use tokio::runtime::Runtime; 6 | // 7 | // mod util; 8 | // 9 | // #[macro_use] 10 | // extern crate serde; 11 | // 12 | // #[macro_use] 13 | // extern crate async_trait; 14 | // 15 | // #[ignore] 16 | // #[test] 17 | // pub fn test_actor_system_blocking() { 18 | // let runtime = Runtime::new().unwrap(); 19 | // let system = runtime.block_on(async { 20 | // ActorSystem::new(); 21 | // }); 22 | // 23 | // let actor = system.new_actor_blocking( 24 | // "test".to_actor_id(), 25 | // TestActor { 26 | // status: None, 27 | // counter: 0, 28 | // }, 29 | // ActorType::Tracked, 30 | // ); 31 | // 32 | // assert!(actor.is_ok()); 33 | // } 34 | -------------------------------------------------------------------------------- /coerce/tests/test_actor_watching.rs: -------------------------------------------------------------------------------- 1 | use crate::util::TestActor; 2 | use async_trait::async_trait; 3 | use coerce::actor::context::ActorContext; 4 | use coerce::actor::message::Handler; 5 | use coerce::actor::system::ActorSystem; 6 | use coerce::actor::watch::{ActorTerminated, ActorWatch}; 7 | use coerce::actor::{Actor, ActorId, CoreActorRef, IntoActor, LocalActorRef}; 8 | use tokio::sync::oneshot; 9 | use tokio::sync::oneshot::Sender; 10 | 11 | pub mod util; 12 | 13 | pub struct Watchdog { 14 | target: LocalActorRef, 15 | on_actor_terminated: Option>, 16 | } 17 | 18 | #[async_trait] 19 | impl Actor for Watchdog { 20 | async fn started(&mut self, ctx: &mut ActorContext) { 21 | self.watch(&self.target, ctx); 22 | } 23 | } 24 | 25 | #[async_trait] 26 | impl Handler for Watchdog { 27 | async fn handle(&mut self, message: ActorTerminated, _ctx: &mut ActorContext) { 28 | if let Some(on_actor_terminated) = self.on_actor_terminated.take() { 29 | let _ = on_actor_terminated.send(message); 30 | } 31 | } 32 | } 33 | 34 | #[tokio::test] 35 | pub async fn test_actor_watch_notifications() { 36 | let system = ActorSystem::new(); 37 | let actor = TestActor::new() 38 | .into_actor(Option::::None, &system) 39 | .await 40 | .unwrap(); 41 | let (tx, rx) = oneshot::channel(); 42 | let _watchdog = Watchdog { 43 | target: actor.clone(), 44 | on_actor_terminated: Some(tx), 45 | } 46 | .into_actor(Option::::None, &system) 47 | .await 48 | .unwrap(); 49 | 50 | let _ = actor.notify_stop(); 51 | let actor_terminated = rx.await.unwrap(); 52 | let terminated_actor = actor_terminated.actor_ref(); 53 | 54 | assert_eq!(terminated_actor.actor_id(), actor.actor_id()); 55 | assert_eq!(terminated_actor.is_valid(), false); 56 | } 57 | -------------------------------------------------------------------------------- /coerce/tests/test_cluster_singleton.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use coerce::actor::context::ActorContext; 3 | use coerce::actor::message::{Handler, Message, MessageUnwrapErr, MessageWrapErr}; 4 | use coerce::actor::system::builder::ActorSystemBuilder; 5 | use coerce::actor::system::ActorSystem; 6 | use coerce::actor::Actor; 7 | use coerce::remote::system::RemoteActorSystem; 8 | use coerce::singleton::factory::SingletonFactory; 9 | use coerce::singleton::{singleton, SingletonBuilder}; 10 | use std::time::Duration; 11 | use tokio::time::sleep; 12 | use tracing::Level; 13 | 14 | mod util; 15 | 16 | struct SingletonActor {} 17 | 18 | impl Actor for SingletonActor {} 19 | 20 | struct Factory {} 21 | 22 | impl SingletonFactory for Factory { 23 | type Actor = SingletonActor; 24 | 25 | fn create(&self) -> Self::Actor { 26 | SingletonActor {} 27 | } 28 | } 29 | 30 | struct Echo { 31 | string: String, 32 | } 33 | 34 | #[async_trait] 35 | impl Handler for SingletonActor { 36 | async fn handle(&mut self, message: Echo, _ctx: &mut ActorContext) -> String { 37 | // Simple message to echo back the message property `string` as the result 38 | message.string 39 | } 40 | } 41 | 42 | #[tokio::test] 43 | pub async fn test_cluster_singleton_start_and_communicate() { 44 | util::create_logger(Some(Level::DEBUG)); 45 | 46 | let system = ActorSystem::builder().system_name("node-1").build(); 47 | let system2 = ActorSystem::builder().system_name("node-2").build(); 48 | 49 | let remote = RemoteActorSystem::builder() 50 | .with_tag("remote-1") 51 | .with_id(1) 52 | .with_actor_system(system) 53 | .configure(singleton::) 54 | .configure(|h| h.with_handler::("SingletonActor.Echo")) 55 | .build() 56 | .await; 57 | 58 | let remote2 = RemoteActorSystem::builder() 59 | .with_tag("remote-2") 60 | .with_id(2) 61 | .with_actor_system(system2) 62 | .configure(singleton::) 63 | .configure(|h| h.with_handler::("SingletonActor.Echo")) 64 | .build() 65 | .await; 66 | 67 | remote 68 | .clone() 69 | .cluster_worker() 70 | .listen_addr("localhost:30101") 71 | .start() 72 | .await; 73 | 74 | remote2 75 | .clone() 76 | .cluster_worker() 77 | .listen_addr("localhost:30102") 78 | .with_seed_addr("localhost:30101") 79 | .start() 80 | .await; 81 | 82 | let singleton = SingletonBuilder::new(remote) 83 | .factory(Factory {}) 84 | .build() 85 | .await; 86 | 87 | let singleton2 = SingletonBuilder::new(remote2) 88 | .factory(Factory {}) 89 | .build() 90 | .await; 91 | 92 | assert_eq!( 93 | singleton 94 | .send(Echo { 95 | string: "hello world".to_string() 96 | }) 97 | .await, 98 | Ok("hello world".to_string()) 99 | ); 100 | 101 | assert_eq!( 102 | singleton2 103 | .send(Echo { 104 | string: "hello world".to_string() 105 | }) 106 | .await, 107 | Ok("hello world".to_string()) 108 | ); 109 | } 110 | 111 | impl Message for Echo { 112 | type Result = String; 113 | 114 | fn as_bytes(&self) -> Result, MessageWrapErr> { 115 | Ok(self.string.as_bytes().to_vec()) 116 | } 117 | 118 | fn from_bytes(s: Vec) -> Result { 119 | Ok(Self { 120 | string: String::from_utf8(s).unwrap(), 121 | }) 122 | } 123 | 124 | fn read_remote_result(res: Vec) -> Result { 125 | Ok(String::from_utf8(res).unwrap()) 126 | } 127 | 128 | fn write_remote_result(res: Self::Result) -> Result, MessageWrapErr> { 129 | Ok(res.into_bytes()) 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /coerce/tests/test_remote_actor_err.rs: -------------------------------------------------------------------------------- 1 | use crate::util::{GetStatusRequest, SetStatusRequest, TestActor, TestActorStatus}; 2 | use coerce::actor::context::ActorContext; 3 | 4 | use coerce::actor::message::{Handler, Message, MessageWrapErr}; 5 | use coerce::actor::scheduler::ActorType::Tracked; 6 | 7 | use coerce::actor::{ActorRef, ActorRefErr, ToActorId}; 8 | use coerce::remote::system::builder::RemoteSystemConfigBuilder; 9 | 10 | use coerce::remote::RemoteActorRef; 11 | 12 | #[macro_use] 13 | extern crate async_trait; 14 | 15 | #[macro_use] 16 | extern crate serde; 17 | 18 | mod util; 19 | 20 | struct NotSerialisable; 21 | 22 | impl Message for NotSerialisable { 23 | type Result = (); 24 | } 25 | 26 | #[async_trait] 27 | impl Handler for TestActor { 28 | async fn handle(&mut self, _: NotSerialisable, _: &mut ActorContext) {} 29 | } 30 | 31 | fn remote_handlers(builder: &mut RemoteSystemConfigBuilder) -> &mut RemoteSystemConfigBuilder { 32 | builder.with_handler::("TestActor.GetStatusRequest") 33 | } 34 | 35 | #[tokio::test] 36 | pub async fn test_remote_actor_err() { 37 | util::create_trace_logger(); 38 | let node_1 = util::create_cluster_node(1, "localhost:30101", None, |handlers| { 39 | remote_handlers(handlers) 40 | }) 41 | .await; 42 | 43 | let node_2 = 44 | util::create_cluster_node(2, "localhost:30201", Some("localhost:30101"), |handlers| { 45 | remote_handlers(handlers) 46 | }) 47 | .await; 48 | 49 | let actor_id = "test_actor".to_actor_id(); 50 | let local_ref = node_1 51 | .actor_system() 52 | .new_actor(actor_id.clone(), TestActor::new(), Tracked) 53 | .await 54 | .expect("create TestActor on node=1"); 55 | 56 | let _ = local_ref.notify(SetStatusRequest { 57 | status: TestActorStatus::Active, 58 | }); 59 | 60 | let actor_ref = ActorRef::from(RemoteActorRef::::new( 61 | actor_id.clone(), 62 | node_1.node_id(), 63 | node_2.clone(), 64 | )); 65 | 66 | let status = actor_ref.send(GetStatusRequest).await; 67 | assert!(status.is_ok()); 68 | 69 | let _ = local_ref.stop().await; 70 | 71 | let status = actor_ref.send(GetStatusRequest).await; 72 | assert_eq!(status.unwrap_err(), ActorRefErr::NotFound(actor_id.clone())); 73 | 74 | let status = actor_ref.send(NotSerialisable).await; 75 | assert_eq!( 76 | status.unwrap_err(), 77 | ActorRefErr::Serialisation(MessageWrapErr::NotTransmittable) 78 | ); 79 | 80 | let status = actor_ref 81 | .send(SetStatusRequest { 82 | status: TestActorStatus::Inactive, 83 | }) 84 | .await; 85 | 86 | assert_eq!( 87 | status.unwrap_err(), 88 | ActorRefErr::NotSupported { 89 | actor_id, 90 | message_type: "test_remote_actor_err::util::SetStatusRequest".to_string(), 91 | actor_type: "test_remote_actor_err::util::TestActor".to_string() 92 | } 93 | ); 94 | } 95 | -------------------------------------------------------------------------------- /coerce/tests/test_remote_actor_locate.rs: -------------------------------------------------------------------------------- 1 | pub mod util; 2 | 3 | #[macro_use] 4 | extern crate serde; 5 | 6 | #[macro_use] 7 | extern crate async_trait; 8 | 9 | #[macro_use] 10 | extern crate coerce_macros; 11 | 12 | use coerce::actor::scheduler::ActorType::Tracked; 13 | use coerce::actor::system::ActorSystem; 14 | use coerce::actor::IntoActorId; 15 | 16 | use coerce::remote::system::RemoteActorSystem; 17 | 18 | #[coerce_test] 19 | pub async fn test_remote_actor_locate_node_locally() { 20 | util::create_trace_logger(); 21 | 22 | let system = ActorSystem::new(); 23 | let remote = RemoteActorSystem::builder() 24 | .with_actor_system(system.clone()) 25 | .build() 26 | .await; 27 | 28 | let locate_before_creation = remote.locate_actor_node("leon".into_actor_id()).await; 29 | 30 | let _ = system 31 | .new_actor( 32 | "leon".to_string(), 33 | util::TestActor { 34 | status: None, 35 | counter: 0, 36 | }, 37 | Tracked, 38 | ) 39 | .await; 40 | 41 | let locate_after_creation = remote.locate_actor_node("leon".into_actor_id()).await; 42 | 43 | assert!(locate_before_creation.is_none()); 44 | assert!(locate_after_creation.is_some()); 45 | } 46 | 47 | #[tokio::test] 48 | pub async fn test_remote_actor_locate_remotely() { 49 | util::create_trace_logger(); 50 | 51 | let system_a = ActorSystem::new(); 52 | let system_b = ActorSystem::new(); 53 | 54 | let remote_a = RemoteActorSystem::builder() 55 | .with_actor_system(system_a.clone()) 56 | .with_id(1) 57 | .with_tag("remote-a") 58 | .build() 59 | .await; 60 | 61 | let remote_b = RemoteActorSystem::builder() 62 | .with_actor_system(system_b.clone()) 63 | .with_id(2) 64 | .with_tag("remote-b") 65 | .build() 66 | .await; 67 | 68 | remote_a 69 | .clone() 70 | .cluster_worker() 71 | .listen_addr("localhost:30101") 72 | .start() 73 | .await; 74 | 75 | remote_b 76 | .clone() 77 | .cluster_worker() 78 | .listen_addr("localhost:30102") 79 | .with_seed_addr("localhost:30101") 80 | .start() 81 | .await; 82 | 83 | let locate_before_creation_a = remote_a.locate_actor_node("leon".into_actor_id()).await; 84 | let locate_before_creation_b = remote_b.locate_actor_node("leon".into_actor_id()).await; 85 | 86 | assert_eq!(locate_before_creation_a, None); 87 | assert_eq!(locate_before_creation_b, None); 88 | 89 | let _ = system_a 90 | .new_actor( 91 | "leon", 92 | util::TestActor { 93 | status: None, 94 | counter: 0, 95 | }, 96 | Tracked, 97 | ) 98 | .await; 99 | 100 | let local_ref = remote_a 101 | .actor_ref::("leon".into_actor_id()) 102 | .await 103 | .expect("unable to get local ref"); 104 | 105 | let remote_ref = remote_b 106 | .actor_ref::("leon".into_actor_id()) 107 | .await 108 | .expect("unable to get remote ref"); 109 | 110 | assert_eq!(remote_ref.is_remote(), true); 111 | assert_eq!(local_ref.is_local(), true); 112 | } 113 | -------------------------------------------------------------------------------- /coerce/tests/test_remote_api.rs: -------------------------------------------------------------------------------- 1 | use crate::util::{GetStatusRequest, SetStatusRequest, TestActor}; 2 | 3 | use coerce::actor::system::ActorSystem; 4 | use coerce::actor::{ActorCreationErr, ActorFactory, ActorRecipe}; 5 | use coerce::persistent::journal::provider::inmemory::InMemoryStorageProvider; 6 | use coerce::persistent::Persistence; 7 | use coerce::remote::api::builder::HttpApiBuilder; 8 | use coerce::remote::api::cluster::ClusterApi; 9 | use coerce::remote::api::sharding::ShardingApi; 10 | 11 | use coerce::remote::net::server::RemoteServer; 12 | use coerce::remote::system::RemoteActorSystem; 13 | use coerce::sharding::Sharding; 14 | use std::net::SocketAddr; 15 | use std::str::FromStr; 16 | 17 | mod util; 18 | 19 | #[macro_use] 20 | extern crate serde; 21 | 22 | #[macro_use] 23 | extern crate async_trait; 24 | 25 | #[macro_use] 26 | extern crate coerce_macros; 27 | 28 | #[derive(Clone)] 29 | struct TestActorFactory {} 30 | 31 | struct TestActorRecipe; 32 | 33 | impl ActorRecipe for TestActorRecipe { 34 | fn read_from_bytes(_bytes: &Vec) -> Option { 35 | Some(Self) 36 | } 37 | 38 | fn write_to_bytes(&self) -> Option> { 39 | Some(vec![]) 40 | } 41 | } 42 | #[async_trait] 43 | impl ActorFactory for TestActorFactory { 44 | type Actor = TestActor; 45 | type Recipe = TestActorRecipe; 46 | 47 | async fn create(&self, _recipe: Self::Recipe) -> Result { 48 | Ok(TestActor { 49 | status: None, 50 | counter: 0, 51 | }) 52 | } 53 | } 54 | 55 | #[tokio::test] 56 | pub async fn test_remote_api_routes() { 57 | util::create_trace_logger(); 58 | 59 | let persistence = Persistence::from(InMemoryStorageProvider::new()); 60 | async fn create_system(persistence: Persistence) -> (RemoteActorSystem, RemoteServer) { 61 | let sys = ActorSystem::new().to_persistent(persistence); 62 | let remote = RemoteActorSystem::builder() 63 | .with_actor_system(sys) 64 | .with_tag("system-one") 65 | .with_actors(|a| { 66 | a.with_actor(TestActorFactory {}) 67 | .with_handler::("GetStatusRequest") 68 | .with_handler::("SetStatusRequest") 69 | }) 70 | .with_id(1) 71 | .single_node() 72 | .build() 73 | .await; 74 | 75 | let server = remote 76 | .clone() 77 | .cluster_worker() 78 | .listen_addr("localhost:30101") 79 | .start() 80 | .await; 81 | 82 | (remote, server) 83 | } 84 | 85 | let (remote, _server) = create_system(persistence).await; 86 | let sharding = Sharding::::builder(remote.clone()) 87 | .with_entity_type("TestActor") 88 | .build() 89 | .await; 90 | 91 | let cluster_api = ClusterApi::new(remote.clone()); 92 | let sharding_api = ShardingApi::default() 93 | .attach(&sharding) 94 | .start(remote.actor_system()) 95 | .await; 96 | 97 | let _ = HttpApiBuilder::new() 98 | .listen_addr(SocketAddr::from_str("0.0.0.0:3000").unwrap()) 99 | .routes(cluster_api) 100 | .routes(sharding_api) 101 | .start(remote.actor_system()) 102 | .await; 103 | 104 | // TODO: actual api tests.. 105 | } 106 | -------------------------------------------------------------------------------- /coerce/tests/test_remote_cluster_client.rs: -------------------------------------------------------------------------------- 1 | pub mod util; 2 | 3 | #[macro_use] 4 | extern crate serde; 5 | 6 | #[macro_use] 7 | extern crate async_trait; 8 | 9 | #[macro_use] 10 | extern crate coerce_macros; 11 | 12 | use coerce::actor::system::ActorSystem; 13 | use coerce::actor::{ActorCreationErr, ActorFactory, ActorRecipe}; 14 | 15 | use coerce::remote::system::RemoteActorSystem; 16 | use util::*; 17 | 18 | #[derive(Serialize, Deserialize)] 19 | struct TestActorRecipe; 20 | 21 | #[derive(Clone)] 22 | struct TestActorFactory; 23 | 24 | impl ActorRecipe for TestActorRecipe { 25 | fn read_from_bytes(bytes: &Vec) -> Option { 26 | serde_json::from_slice(bytes).unwrap() 27 | } 28 | 29 | fn write_to_bytes(&self) -> Option> { 30 | serde_json::to_vec(&self).ok() 31 | } 32 | } 33 | 34 | #[async_trait] 35 | impl ActorFactory for TestActorFactory { 36 | type Actor = TestActor; 37 | type Recipe = TestActorRecipe; 38 | 39 | async fn create(&self, _recipe: Self::Recipe) -> Result { 40 | Ok(TestActor { 41 | status: None, 42 | counter: 0, 43 | }) 44 | } 45 | } 46 | 47 | #[coerce_test] 48 | pub async fn test_remote_get_actor() { 49 | util::create_trace_logger(); 50 | 51 | let system = ActorSystem::new(); 52 | let remote = RemoteActorSystem::builder() 53 | .with_actor_system(system) 54 | .with_handlers(|builder| builder.with_actor::(TestActorFactory {})) 55 | .build() 56 | .await; 57 | 58 | let created_actor = remote 59 | .actor_system() 60 | .new_tracked_actor(TestActor::new()) 61 | .await 62 | .unwrap(); 63 | 64 | let actor_id = created_actor.actor_id().clone(); 65 | let actor = remote.actor_ref::(actor_id).await; 66 | assert_eq!(actor.is_some(), true); 67 | } 68 | 69 | // 70 | // #[coerce_test] 71 | // pub async fn test_remote_cluster_client_create_actor() { 72 | // let mut system = ActorSystem::new(); 73 | // let _actor = system.new_tracked_actor(TestActor::new()).await.unwrap(); 74 | // let remote = RemoteActorSystem::builder() 75 | // .with_actor_system(system) 76 | // .with_handlers(|builder| { 77 | // builder.with_actor::("TestActor", TestActorFactory {}) 78 | // }) 79 | // .build() 80 | // .await; 81 | // 82 | // let mut client = remote.cluster_client().build(); 83 | // 84 | // let actor = client 85 | // .create_actor::(TestActorRecipe {}, Some(format!("TestActor"))) 86 | // .await; 87 | // 88 | // assert_eq!(actor.is_some(), false); 89 | // } 90 | -------------------------------------------------------------------------------- /coerce/tests/test_remote_cluster_heartbeat.rs: -------------------------------------------------------------------------------- 1 | pub mod util; 2 | 3 | #[macro_use] 4 | extern crate coerce_macros; 5 | 6 | #[macro_use] 7 | extern crate async_trait; 8 | 9 | #[macro_use] 10 | extern crate serde; 11 | 12 | use coerce::actor::system::ActorSystem; 13 | use coerce::remote::cluster::node::NodeStatus::{Healthy, Terminated}; 14 | use coerce::remote::heartbeat::HeartbeatConfig; 15 | use coerce::remote::system::builder::RemoteSystemConfigBuilder; 16 | use coerce::remote::system::RemoteActorSystem; 17 | use std::time::Duration; 18 | use tokio::time; 19 | 20 | #[coerce_test] 21 | pub async fn test_remote_cluster_heartbeat() { 22 | util::create_trace_logger(); 23 | fn configure_sys(c: &mut RemoteSystemConfigBuilder) -> &mut RemoteSystemConfigBuilder { 24 | c.heartbeat(HeartbeatConfig { 25 | interval: Duration::from_millis(250), 26 | ping_timeout: Duration::from_millis(10), 27 | unhealthy_node_heartbeat_timeout: Duration::from_millis(750), 28 | terminated_node_heartbeat_timeout: Duration::from_millis(1000), 29 | ..Default::default() 30 | }) 31 | } 32 | 33 | let remote = RemoteActorSystem::builder() 34 | .with_tag("remote-1") 35 | .with_id(1) 36 | .with_actor_system(ActorSystem::new()) 37 | .configure(configure_sys) 38 | .build() 39 | .await; 40 | 41 | let remote_2 = RemoteActorSystem::builder() 42 | .with_tag("remote-2") 43 | .with_id(2) 44 | .with_actor_system(ActorSystem::new()) 45 | .configure(configure_sys) 46 | .build() 47 | .await; 48 | 49 | let server = remote 50 | .clone() 51 | .cluster_worker() 52 | .listen_addr("localhost:30101") 53 | .start() 54 | .await; 55 | 56 | let _server_2 = remote_2 57 | .clone() 58 | .cluster_worker() 59 | .listen_addr("localhost:30102") 60 | .with_seed_addr("localhost:30101") 61 | .start() 62 | .await; 63 | 64 | // ensure both nodes have run a heartbeat atleast once. 65 | // TODO: We need the ability to hook into an on-cluster-joined event/future 66 | tokio::time::sleep(Duration::from_millis(750)).await; 67 | 68 | let nodes_a = remote.get_nodes().await; 69 | let nodes_b = remote_2.get_nodes().await; 70 | let node_a_1 = nodes_a.iter().find(|n| n.id == 1).cloned().unwrap(); 71 | let node_a_2 = nodes_a.iter().find(|n| n.id == 2).cloned().unwrap(); 72 | let node_b_1 = nodes_b.iter().find(|n| n.id == 1).cloned().unwrap(); 73 | let node_b_2 = nodes_b.iter().find(|n| n.id == 2).cloned().unwrap(); 74 | 75 | let leader_1_id = remote.current_leader(); 76 | let leader_2_id = remote_2.current_leader(); 77 | 78 | assert_eq!(leader_1_id, Some(remote.node_id())); 79 | assert_eq!(leader_2_id, Some(remote.node_id())); 80 | 81 | assert_eq!(node_a_1.status, Healthy); 82 | assert_eq!(node_a_2.status, Healthy); 83 | assert_eq!(node_b_1.status, Healthy); 84 | assert_eq!(node_b_2.status, Healthy); 85 | 86 | { 87 | let server = server; 88 | let remote = remote; 89 | server.stop(); 90 | remote.actor_system().shutdown().await; 91 | } 92 | 93 | time::sleep(Duration::from_secs(1)).await; 94 | 95 | let nodes_b = remote_2.get_nodes().await; 96 | let node_1 = nodes_b.iter().find(|n| n.id == 1).cloned().unwrap(); 97 | let node_2 = nodes_b.iter().find(|n| n.id == 2).cloned().unwrap(); 98 | 99 | let leader_id = remote_2.current_leader(); 100 | 101 | assert_eq!(leader_id, Some(remote_2.node_id())); 102 | assert_eq!(node_1.status, Terminated); 103 | assert_eq!(node_2.status, Healthy); 104 | } 105 | -------------------------------------------------------------------------------- /coerce/tests/test_remote_codec.rs: -------------------------------------------------------------------------------- 1 | use coerce::actor::system::ActorSystem; 2 | use coerce::remote::system::RemoteActorSystem; 3 | 4 | use util::*; 5 | 6 | pub mod util; 7 | 8 | #[macro_use] 9 | extern crate serde; 10 | 11 | #[macro_use] 12 | extern crate async_trait; 13 | 14 | #[tokio::test] 15 | pub async fn test_remote_create_message() { 16 | util::create_trace_logger(); 17 | 18 | let remote = RemoteActorSystem::builder() 19 | .with_actor_system(ActorSystem::new()) 20 | .with_handlers(move |handlers| { 21 | handlers.with_handler::("TestActor.SetStatusRequest") 22 | }) 23 | .build() 24 | .await; 25 | 26 | let _msg = SetStatusRequest { 27 | status: TestActorStatus::Active, 28 | }; 29 | 30 | let actor = remote 31 | .actor_system() 32 | .new_anon_actor(TestActor::new()) 33 | .await 34 | .unwrap(); 35 | 36 | let message = remote 37 | .create_header::(&actor.id) 38 | .unwrap(); 39 | 40 | assert_eq!(message.actor_id, actor.id); 41 | assert_eq!( 42 | message.handler_type, 43 | "TestActor.SetStatusRequest".to_string() 44 | ); 45 | } 46 | -------------------------------------------------------------------------------- /coerce/tests/test_remote_handle.rs: -------------------------------------------------------------------------------- 1 | use crate::util::create_trace_logger; 2 | use coerce::actor::system::ActorSystem; 3 | use coerce::remote::system::RemoteActorSystem; 4 | use util::*; 5 | 6 | pub mod util; 7 | 8 | #[macro_use] 9 | extern crate serde; 10 | 11 | #[macro_use] 12 | extern crate async_trait; 13 | 14 | #[tokio::test] 15 | pub async fn test_remote_handler_types() { 16 | let echo_get_counter = "EchoActor.GetCounterRequest".to_string(); 17 | let test_get_status = "TestActor.GetStatusRequest".to_string(); 18 | let test_set_status = "TestActor.SetStatusRequest".to_string(); 19 | 20 | let remote = RemoteActorSystem::builder() 21 | .with_actor_system(ActorSystem::new()) 22 | .with_handlers(|handlers| { 23 | handlers 24 | .with_handler::("TestActor.SetStatusRequest") 25 | .with_handler::("TestActor.GetStatusRequest") 26 | .with_handler::("EchoActor.GetCounterRequest") 27 | }) 28 | .build() 29 | .await; 30 | 31 | assert_eq!( 32 | remote.handler_name::(), 33 | Some(echo_get_counter) 34 | ); 35 | assert_eq!( 36 | remote.handler_name::(), 37 | Some(test_set_status) 38 | ); 39 | assert_eq!( 40 | remote.handler_name::(), 41 | Some(test_get_status) 42 | ); 43 | } 44 | 45 | #[tokio::test] 46 | pub async fn test_remote_handle_from_json() { 47 | create_trace_logger(); 48 | 49 | let ctx = ActorSystem::new(); 50 | let actor = ctx.new_tracked_actor(TestActor::new()).await.unwrap(); 51 | 52 | let remote = RemoteActorSystem::builder() 53 | .with_actor_system(ctx) 54 | .with_handlers(|handlers| { 55 | handlers 56 | .with_handler::("TestActor.SetStatusRequest") 57 | .with_handler::("TestActor.GetStatusRequest") 58 | .with_handler::("EchoActor.GetCounterRequest") 59 | }) 60 | .build() 61 | .await; 62 | 63 | let initial_status = actor.send(GetStatusRequest).await; 64 | 65 | let res = remote 66 | .handle_message( 67 | "TestActor.SetStatusRequest", 68 | actor.id.clone(), 69 | b"{\"status\": \"Active\"}", 70 | ) 71 | .await; 72 | 73 | let current_status = actor.send(GetStatusRequest).await; 74 | 75 | assert_eq!(res, Ok(b"\"Ok\"".to_vec())); 76 | 77 | assert_eq!(initial_status, Ok(GetStatusResponse::None)); 78 | assert_eq!( 79 | current_status, 80 | Ok(GetStatusResponse::Ok(TestActorStatus::Active)) 81 | ); 82 | } 83 | -------------------------------------------------------------------------------- /coerce/tests/test_remote_net.rs: -------------------------------------------------------------------------------- 1 | // use coerce::actor::system::ActorSystem; 2 | // use coerce::remote::net::client::{ClientType, RemoteClient}; 3 | // use coerce::remote::net::server::RemoteServer; 4 | // use coerce::remote::system::builder::RemoteActorHandlerBuilder; 5 | // use coerce::remote::system::RemoteActorSystem; 6 | // use coerce::remote::RemoteActorRef; 7 | // 8 | // use coerce::actor::ActorRef; 9 | // 10 | // use util::*; 11 | // 12 | // pub mod util; 13 | // 14 | // #[macro_use] 15 | // extern crate serde; 16 | // 17 | // #[macro_use] 18 | // extern crate async_trait; 19 | // 20 | // #[tokio::test] 21 | // pub async fn test_remote_server_client_connection() { 22 | // // let (tracer, _uninstall) = opentelemetry_jaeger::new_pipeline() 23 | // // .with_service_name("coerce") 24 | // // .install() 25 | // // .expect("jaeger"); 26 | // // let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer); 27 | // // tracing_subscriber::registry() 28 | // // .with(opentelemetry) 29 | // // .try_init() 30 | // // .expect("tracing init"); 31 | // 32 | // let system = ActorSystem::new(); 33 | // let actor = system.new_tracked_actor(TestActor::new()).await.unwrap(); 34 | // 35 | // let remote = RemoteActorSystem::builder() 36 | // .with_actor_system(system) 37 | // .with_handlers(build_handlers) 38 | // .build() 39 | // .await; 40 | // 41 | // let remote_2 = RemoteActorSystem::builder() 42 | // .with_actor_system(ActorSystem::new()) 43 | // .with_handlers(build_handlers) 44 | // .build() 45 | // .await; 46 | // 47 | // let mut server = RemoteServer::new(); 48 | // match server.start("localhost:30101".to_string(), remote).await { 49 | // Ok(_) => tracing::trace!("started!"), 50 | // Err(_e) => panic!("failed to start server"), 51 | // } 52 | // 53 | // let node_id = 1; 54 | // remote_2 55 | // .register_client( 56 | // node_id, 57 | // RemoteClient::connect( 58 | // "localhost:30101".to_string(), 59 | // Some(node_id), 60 | // remote_2.clone(), 61 | // None, 62 | // ClientType::Worker, 63 | // ) 64 | // .await 65 | // .unwrap(), 66 | // ) 67 | // .await; 68 | // 69 | // let remote_actor: ActorRef = 70 | // RemoteActorRef::::new(actor.id.clone(), node_id, remote_2).into(); 71 | // 72 | // for _i in 1..=10 as i32 { 73 | // let _initial_status = remote_actor.send(GetStatusRequest).await; 74 | // } 75 | // 76 | // let initial_status = remote_actor.send(GetStatusRequest).await; 77 | // let _ = remote_actor 78 | // .send(SetStatusRequest { 79 | // status: TestActorStatus::Active, 80 | // }) 81 | // .await; 82 | // 83 | // let current_status = remote_actor.send(GetStatusRequest).await; 84 | // 85 | // let _ = remote_actor 86 | // .send(SetStatusRequest { 87 | // status: TestActorStatus::Inactive, 88 | // }) 89 | // .await; 90 | // 91 | // let inactive_status = remote_actor.send(GetStatusRequest).await; 92 | // 93 | // assert_eq!(initial_status, Ok(GetStatusResponse::None)); 94 | // 95 | // assert_eq!( 96 | // inactive_status, 97 | // Ok(GetStatusResponse::Ok(TestActorStatus::Inactive)) 98 | // ); 99 | // 100 | // assert_eq!( 101 | // current_status, 102 | // Ok(GetStatusResponse::Ok(TestActorStatus::Active)) 103 | // ); 104 | // } 105 | // 106 | // fn build_handlers(handlers: &mut RemoteActorHandlerBuilder) -> &mut RemoteActorHandlerBuilder { 107 | // handlers 108 | // .with_handler::("TestActor.SetStatusRequest") 109 | // .with_handler::("TestActor.GetStatusRequest") 110 | // .with_handler::("EchoActor.GetCounterRequest") 111 | // } 112 | -------------------------------------------------------------------------------- /coerce/tests/test_remote_system_health.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use coerce::actor::context::ActorContext; 3 | use coerce::actor::message::{Handler, Message}; 4 | use coerce::actor::system::ActorSystem; 5 | use coerce::actor::{Actor, IntoActor}; 6 | use coerce::remote::heartbeat::health::HealthStatus; 7 | use coerce::remote::heartbeat::Heartbeat; 8 | use coerce::remote::system::RemoteActorSystem; 9 | 10 | use std::time::Duration; 11 | 12 | #[tokio::test] 13 | pub async fn test_heartbeat_actor_monitoring() { 14 | const VERSION: &str = "1.0.0"; 15 | let actor_system = ActorSystem::builder().system_name("heartbeat-test").build(); 16 | 17 | let remote = RemoteActorSystem::builder() 18 | .with_tag("remote-1") 19 | .with_version(VERSION) 20 | .with_id(1) 21 | .with_actor_system(actor_system) 22 | .configure(|c| c) 23 | .build() 24 | .await; 25 | 26 | let health_check = Heartbeat::get_system_health(&remote).await; 27 | assert_eq!(health_check.status, HealthStatus::Healthy); 28 | assert_eq!(&health_check.node_version, VERSION); 29 | 30 | let actor = SlowActor 31 | .into_actor(Some("slow-actor"), remote.actor_system()) 32 | .await 33 | .unwrap(); 34 | let _ = actor.notify(Delay(Duration::from_secs(2))); 35 | 36 | let health_check = Heartbeat::get_system_health(&remote).await; 37 | assert_eq!(health_check.status, HealthStatus::Degraded); 38 | 39 | let _ = actor.notify_stop(); 40 | let health_check = Heartbeat::get_system_health(&remote).await; 41 | assert_eq!(health_check.status, HealthStatus::Unhealthy); 42 | 43 | // De-register the actor from the health check 44 | Heartbeat::remove(actor.actor_id(), &remote); 45 | 46 | let health_check = Heartbeat::get_system_health(&remote).await; 47 | assert_eq!(health_check.status, HealthStatus::Healthy); 48 | } 49 | 50 | pub struct SlowActor; 51 | 52 | #[async_trait] 53 | impl Actor for SlowActor { 54 | async fn started(&mut self, ctx: &mut ActorContext) { 55 | Heartbeat::register(self.actor_ref(ctx), ctx.system().remote()); 56 | } 57 | } 58 | 59 | struct Delay(Duration); 60 | 61 | impl Message for Delay { 62 | type Result = (); 63 | } 64 | 65 | #[async_trait] 66 | impl Handler for SlowActor { 67 | async fn handle(&mut self, message: Delay, _ctx: &mut ActorContext) { 68 | tokio::time::sleep(message.0).await; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /coerce/tests/test_remote_system_security.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate serde; 3 | 4 | #[macro_use] 5 | extern crate tracing; 6 | 7 | #[macro_use] 8 | extern crate async_trait; 9 | 10 | use coerce::actor::system::ActorSystem; 11 | use coerce::remote::net::security::jwt::Jwt; 12 | use coerce::remote::system::RemoteActorSystem; 13 | use std::time::Duration; 14 | 15 | mod util; 16 | 17 | #[tokio::test] 18 | pub async fn test_jwt_validation() { 19 | let secret = b"secret"; 20 | let token_ttl = None; 21 | 22 | let jwt = Jwt::from_secret(secret.to_vec(), token_ttl); 23 | let token = jwt.generate_token().unwrap(); 24 | let token_validated = jwt.validate_token(&token); 25 | assert!(!token.is_empty()); 26 | assert!(token_validated); 27 | } 28 | 29 | #[tokio::test] 30 | pub async fn test_jwt_validation_with_expiry() { 31 | let secret = b"secret"; 32 | let token_ttl = Some(Duration::from_millis(100)); 33 | let jwt = Jwt::from_secret(secret.to_vec(), token_ttl); 34 | let token = jwt.generate_token().unwrap(); 35 | let token_validated = jwt.validate_token(&token); 36 | assert!(!token.is_empty()); 37 | assert!(token_validated); 38 | 39 | tokio::time::sleep(Duration::from_millis(200)).await; 40 | let token_validated = jwt.validate_token(&token); 41 | assert!(!token_validated); 42 | } 43 | 44 | #[tokio::test] 45 | pub async fn test_remote_cluster_inconsistent_secrets() { 46 | util::create_trace_logger(); 47 | 48 | let secrets = vec!["secret-1", "secret-2", "secret-3"]; 49 | 50 | let nodes = create_cluster_nodes("3102", secrets).await; 51 | 52 | let nodes_1 = nodes[0].get_nodes().await; 53 | let nodes_2 = nodes[1].get_nodes().await; 54 | let nodes_3 = nodes[2].get_nodes().await; 55 | 56 | assert_eq!(nodes_1.len(), 1); 57 | assert_eq!(nodes_2.len(), 1); 58 | assert_eq!(nodes_3.len(), 1); 59 | } 60 | 61 | #[tokio::test] 62 | pub async fn test_remote_cluster_consistent_secrets() { 63 | util::create_trace_logger(); 64 | 65 | let secrets = vec!["secret-1", "secret-1", "secret-1"]; 66 | 67 | let nodes = create_cluster_nodes("3101", secrets).await; 68 | 69 | let nodes_1 = nodes[0].get_nodes().await; 70 | let nodes_2 = nodes[1].get_nodes().await; 71 | let nodes_3 = nodes[2].get_nodes().await; 72 | 73 | assert_eq!(nodes_1.len(), 3); 74 | assert_eq!(nodes_2.len(), 3); 75 | assert_eq!(nodes_3.len(), 3); 76 | } 77 | 78 | async fn create_cluster_nodes( 79 | port_prefix: &'static str, 80 | secrets: Vec<&'static str>, 81 | ) -> Vec { 82 | let system = ActorSystem::new(); 83 | let remote = RemoteActorSystem::builder() 84 | .with_tag("remote-1") 85 | .with_id(1) 86 | .with_actor_system(system) 87 | .client_auth_jwt(secrets[0], None) 88 | .build() 89 | .await; 90 | 91 | let remote_2 = RemoteActorSystem::builder() 92 | .with_tag("remote-2") 93 | .with_id(2) 94 | .with_actor_system(ActorSystem::new()) 95 | .client_auth_jwt(secrets[1], None) 96 | .build() 97 | .await; 98 | 99 | let remote_3 = RemoteActorSystem::builder() 100 | .with_tag("remote-3") 101 | .with_id(3) 102 | .with_actor_system(ActorSystem::new()) 103 | .client_auth_jwt(secrets[2], None) 104 | .build() 105 | .await; 106 | 107 | remote 108 | .clone() 109 | .cluster_worker() 110 | .listen_addr(format!("localhost:{}1", port_prefix)) 111 | .start() 112 | .await; 113 | 114 | remote_2 115 | .clone() 116 | .cluster_worker() 117 | .listen_addr(format!("localhost:{}2", port_prefix)) 118 | .with_seed_addr(format!("localhost:{}1", port_prefix)) 119 | .start() 120 | .await; 121 | 122 | remote_3 123 | .clone() 124 | .cluster_worker() 125 | .listen_addr(format!("localhost:{}3", port_prefix)) 126 | .with_seed_addr(format!("localhost:{}2", port_prefix)) 127 | .start() 128 | .await; 129 | 130 | vec![remote, remote_2, remote_3] 131 | } 132 | -------------------------------------------------------------------------------- /coerce/tests/test_timer.rs: -------------------------------------------------------------------------------- 1 | use coerce::actor::context::ActorContext; 2 | use coerce::actor::message::{Handler, Message}; 3 | use coerce::actor::scheduler::timer::{Timer, TimerTick}; 4 | use coerce::actor::system::ActorSystem; 5 | use coerce::actor::Actor; 6 | use std::time::{Duration, Instant}; 7 | 8 | pub mod util; 9 | 10 | #[macro_use] 11 | extern crate serde; 12 | 13 | #[macro_use] 14 | extern crate async_trait; 15 | 16 | #[derive(Clone)] 17 | pub struct TestTimer {} 18 | 19 | struct TimerActor { 20 | ticks: Vec, 21 | } 22 | 23 | impl Message for TestTimer { 24 | type Result = (); 25 | } 26 | 27 | impl TimerTick for TestTimer {} 28 | 29 | impl Actor for TimerActor {} 30 | 31 | #[async_trait] 32 | impl Handler for TimerActor { 33 | async fn handle(&mut self, _message: TestTimer, _ctx: &mut ActorContext) { 34 | self.ticks.push(Instant::now()); 35 | } 36 | } 37 | 38 | #[ignore] 39 | #[tokio::test] 40 | pub async fn test_timer() { 41 | util::create_trace_logger(); 42 | let actor_ref = ActorSystem::new() 43 | .new_tracked_actor(TimerActor { ticks: vec![] }) 44 | .await 45 | .unwrap(); 46 | 47 | const TICK_DURATION: u64 = 1; 48 | let start = Instant::now(); 49 | let timer = Timer::start( 50 | actor_ref.clone(), 51 | Duration::from_secs(TICK_DURATION), 52 | TestTimer {}, 53 | ); 54 | 55 | tokio::time::sleep(Duration::from_secs(5)).await; 56 | 57 | timer.stop(); 58 | let ticks_after_stopping = actor_ref.exec(|a| a.ticks.clone()).await.unwrap(); 59 | tokio::time::sleep(Duration::from_secs(2)).await; 60 | let ticks_after_stopping_and_waiting = actor_ref.exec(|a| a.ticks.len()).await.unwrap(); 61 | 62 | let mut previous_tick = None; 63 | for tick in &ticks_after_stopping { 64 | if let Some(previous_tick) = previous_tick { 65 | tracing::info!("{}", tick.duration_since(previous_tick).as_secs()); 66 | assert!(tick.duration_since(previous_tick).as_secs() >= TICK_DURATION); 67 | } else { 68 | previous_tick = Some(*tick); 69 | tracing::info!("{}", tick.duration_since(start).as_secs()); 70 | assert!(tick.duration_since(start).as_secs() >= TICK_DURATION); 71 | } 72 | } 73 | assert_eq!(ticks_after_stopping.len(), ticks_after_stopping_and_waiting); 74 | } 75 | -------------------------------------------------------------------------------- /coerce/tests/test_tracing.rs: -------------------------------------------------------------------------------- 1 | // use coerce::actor::system::ActorSystem; 2 | // use coerce::actor::{Actor, IntoActor}; 3 | // use std::error::Error; 4 | // 5 | // use tracing_subscriber::prelude::*; 6 | // 7 | // struct TracingActor; 8 | // 9 | // impl Actor for TracingActor {} 10 | // 11 | // #[tracing::instrument] 12 | // async fn app() { 13 | // let sys = ActorSystem::new(); 14 | // 15 | // for i in 0..10 { 16 | // let actor_id = format!("actor-id-{}", i); 17 | // tracing::info!(actor_id = actor_id.as_str(), message = "starting actor",); 18 | // 19 | // TracingActor 20 | // .into_actor(Some(format!("actor-id-{}", i)), &sys) 21 | // .await; 22 | // } 23 | // } 24 | // 25 | // #[tokio::test] 26 | // pub async fn test_tracing() -> Result<(), Box> { 27 | // // tracing_subscriber::fmt() 28 | // // .with_max_level(tracing::Level::TRACE) 29 | // // // .with_span_events(FmtSpan::FULL) 30 | // // .try_init()?; 31 | // 32 | // let tracer = opentelemetry_jaeger::new_pipeline() 33 | // .with_service_name("coerce") 34 | // .install_simple()?; 35 | // 36 | // let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer); 37 | // tracing_subscriber::registry() 38 | // .with(opentelemetry) 39 | // .try_init()?; 40 | // 41 | // app().await; 42 | // 43 | // Ok(()) 44 | // } 45 | -------------------------------------------------------------------------------- /coerce/tests/test_worker.rs: -------------------------------------------------------------------------------- 1 | use coerce::actor::context::ActorContext; 2 | use coerce::actor::message::{Handler, Message}; 3 | use coerce::actor::system::ActorSystem; 4 | use coerce::actor::worker::{Worker, WorkerRefExt}; 5 | use coerce::actor::Actor; 6 | 7 | #[macro_use] 8 | extern crate async_trait; 9 | 10 | #[derive(Clone)] 11 | pub struct MyWorker {} 12 | 13 | pub struct HeavyTask; 14 | 15 | impl Actor for MyWorker {} 16 | 17 | impl Message for HeavyTask { 18 | type Result = &'static str; 19 | } 20 | 21 | #[async_trait] 22 | impl Handler for MyWorker { 23 | async fn handle(&mut self, _message: HeavyTask, _ctx: &mut ActorContext) -> &'static str { 24 | // do some IO with a connection pool attached to `MyWorker`? 25 | 26 | "my_result" 27 | } 28 | } 29 | 30 | #[tokio::test] 31 | pub async fn test_workers() { 32 | let mut system = ActorSystem::new(); 33 | 34 | let state = MyWorker {}; 35 | let mut worker = Worker::new(state, 4, "worker", &mut system).await.unwrap(); 36 | 37 | assert_eq!(worker.dispatch(HeavyTask).await, Ok("my_result")); 38 | } 39 | -------------------------------------------------------------------------------- /coerce/tools/coerce-proto-build/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "coerce-proto-build" 3 | version = "0.1.0" 4 | authors = ["Leon Hartley "] 5 | edition = "2021" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | protobuf = "3.2.0" 11 | protobuf-codegen = "3.2.0" -------------------------------------------------------------------------------- /coerce/tools/coerce-proto-build/src/main.rs: -------------------------------------------------------------------------------- 1 | use protobuf_codegen::Customize; 2 | use std::path::Path; 3 | 4 | struct ProtobufFile { 5 | pub proto_file: &'static str, 6 | pub output_dir: &'static str, 7 | } 8 | 9 | fn main() -> std::io::Result<()> { 10 | if !Path::new("coerce").exists() { 11 | panic!("could not find coerce root directory, please run from the coerce repository root"); 12 | } 13 | 14 | compile_proto( 15 | "coerce/src/protocol/", 16 | vec![ 17 | ( 18 | "coerce/src/protocol/network.proto", 19 | "coerce/src/remote/net/proto", 20 | ), 21 | ( 22 | "coerce/src/protocol/sharding.proto", 23 | "coerce/src/sharding/proto", 24 | ), 25 | ( 26 | "coerce/src/protocol/singleton.proto", 27 | "coerce/src/remote/cluster/singleton/proto", 28 | ), 29 | ( 30 | "coerce/src/protocol/persistent/journal.proto", 31 | "coerce/src/persistent/journal/proto", 32 | ), 33 | ] 34 | .into_iter(), 35 | ); 36 | 37 | compile_proto( 38 | "examples/coerce-sharded-chat-example/src/protocol/", 39 | vec![( 40 | "examples/coerce-sharded-chat-example/src/protocol/chat.proto", 41 | "examples/coerce-sharded-chat-example/src/protocol", 42 | )] 43 | .into_iter(), 44 | ); 45 | 46 | Ok(()) 47 | } 48 | 49 | fn compile_proto>( 50 | include_dir: &'static str, 51 | protobuf_files: I, 52 | ) { 53 | for file in protobuf_files { 54 | protobuf_codegen::Codegen::new() 55 | .customize(Customize::default().gen_mod_rs(true)) 56 | .out_dir(file.1) 57 | .input(file.0) 58 | .include(include_dir) 59 | .run() 60 | .unwrap_or_else(|e| panic!("protoc {}, error={}", file.0, e)); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /coerce/tools/coerce-test/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "coerce-test" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | coerce = { path = "../../" } -------------------------------------------------------------------------------- /coerce/tools/coerce-test/src/event/filter.rs: -------------------------------------------------------------------------------- 1 | pub struct EventFilter { 2 | 3 | } 4 | 5 | -------------------------------------------------------------------------------- /coerce/tools/coerce-test/src/event/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /coerce/tools/coerce-test/src/lib.rs: -------------------------------------------------------------------------------- 1 | use coerce::remote::system::{NodeId, RemoteActorSystem}; 2 | use std::collections::HashMap; 3 | 4 | pub struct TestRemoteActorSystem { 5 | system: RemoteActorSystem, 6 | node_id: NodeId, 7 | } 8 | 9 | pub struct TestCluster { 10 | systems: HashMap, 11 | } 12 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Coerce Examples 2 | The Coerce actor runtime can be used for many different types of applications. Here are just a few different examples of 3 | how you could use Coerce to create scalable, fault-tolerant, self-healing and high performance event-driven applications. 4 | 5 | ## Sharded Chat Example 6 | This example showcases Coerce's cluster sharding, distributed PubSub, easy integration with streams and more. 7 | Paired with [tokio_tungstenite](https://github.com/snapview/tokio-tungstenite), the chat example creates a cluster of 8 | websockets and a sharded cluster of `ChatStream` actors. 9 | Users connect via a WebSocket, join chats via communicating via a Sharded actor and receive broadcasted chat messages via 10 | a PubSub stream scoped to each `ChatStream`. 11 | 12 | More information can be found [here](http://todo-link). 13 | -------------------------------------------------------------------------------- /examples/coerce-cluster-example/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "coerce-cluster-example" 3 | version = "0.1.0" 4 | authors = ["Leon Hartley "] 5 | edition = "2021" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | coerce = { path = "../../coerce", features = ["remote"] } 11 | coerce-macros = { path = "../../coerce/macros" } 12 | tokio = { version = "1.28.1", features = ["full"] } 13 | serde = { version = "1.0", features = ["derive"] } 14 | serde_json = "1.0" 15 | async-trait = { version = "0.1" } 16 | tracing = "0.1" 17 | tracing-subscriber = "0.2" 18 | opentelemetry-jaeger = "0.10" 19 | tracing-opentelemetry = "0.10.0" 20 | opentelemetry = { version = "0.11", default-features = false, features = ["trace"] } 21 | 22 | [[bin]] 23 | name = "worker" 24 | path = "src/worker.rs" -------------------------------------------------------------------------------- /examples/coerce-cluster-example/src/actor.rs: -------------------------------------------------------------------------------- 1 | use coerce::actor::context::ActorContext; 2 | use coerce::actor::message::Handler; 3 | use coerce::actor::Actor; 4 | use coerce_macros::JsonMessage; 5 | 6 | pub struct EchoActor; 7 | 8 | impl Actor for EchoActor {} 9 | 10 | #[derive(JsonMessage, Serialize, Deserialize)] 11 | #[result("String")] 12 | pub struct Echo(pub String); 13 | 14 | #[async_trait] 15 | impl Handler for EchoActor { 16 | async fn handle(&mut self, message: Echo, _ctx: &mut ActorContext) -> String { 17 | message.0 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /examples/coerce-cluster-example/src/main.rs: -------------------------------------------------------------------------------- 1 | use actor::{Echo, EchoActor}; 2 | use coerce::actor::system::ActorSystem; 3 | use coerce::actor::IntoActor; 4 | use coerce::remote::system::RemoteActorSystem; 5 | use opentelemetry::global; 6 | use opentelemetry::sdk::propagation::TraceContextPropagator; 7 | use tracing_subscriber::layer::SubscriberExt; 8 | use tracing_subscriber::util::SubscriberInitExt; 9 | 10 | pub mod actor; 11 | 12 | #[macro_use] 13 | extern crate serde; 14 | 15 | #[macro_use] 16 | extern crate async_trait; 17 | 18 | #[tokio::main] 19 | pub async fn main() { 20 | global::set_text_map_propagator(TraceContextPropagator::new()); 21 | 22 | let (tracer, _uninstall) = opentelemetry_jaeger::new_pipeline() 23 | .with_service_name("example-main") 24 | .install() 25 | .unwrap(); 26 | let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer); 27 | tracing_subscriber::registry() 28 | .with(opentelemetry) 29 | .try_init() 30 | .unwrap(); 31 | 32 | let system = ActorSystem::new(); 33 | let remote = RemoteActorSystem::builder() 34 | .with_tag("example-main") 35 | .with_actor_system(system) 36 | .with_handlers(|handlers| handlers.with_handler::("EchoActor.Echo")) 37 | .build() 38 | .await; 39 | 40 | remote 41 | .clone() 42 | .cluster_worker() 43 | .listen_addr("localhost:30100") 44 | .start() 45 | .await; 46 | 47 | let _ = EchoActor 48 | .into_actor(Some("echo-actor".to_string()), remote.actor_system()) 49 | .await 50 | .expect("unable to start echo actor"); 51 | 52 | tokio::signal::ctrl_c() 53 | .await 54 | .expect("failed to listen for event"); 55 | } 56 | -------------------------------------------------------------------------------- /examples/coerce-cluster-example/src/worker.rs: -------------------------------------------------------------------------------- 1 | pub mod actor; 2 | 3 | use crate::actor::{Echo, EchoActor}; 4 | use coerce::actor::system::ActorSystem; 5 | use coerce::actor::ToActorId; 6 | use coerce::remote::system::RemoteActorSystem; 7 | use opentelemetry::global; 8 | use opentelemetry::sdk::propagation::TraceContextPropagator; 9 | use tracing_subscriber::layer::SubscriberExt; 10 | use tracing_subscriber::util::SubscriberInitExt; 11 | 12 | #[macro_use] 13 | extern crate serde; 14 | 15 | #[macro_use] 16 | extern crate async_trait; 17 | 18 | #[tokio::main] 19 | pub async fn main() { 20 | global::set_text_map_propagator(TraceContextPropagator::new()); 21 | 22 | let (tracer, _uninstall) = opentelemetry_jaeger::new_pipeline() 23 | .with_service_name("example-worker") 24 | .install() 25 | .unwrap(); 26 | 27 | let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer); 28 | tracing_subscriber::registry() 29 | .with(opentelemetry) 30 | .try_init() 31 | .unwrap(); 32 | 33 | let system = ActorSystem::new(); 34 | let remote = RemoteActorSystem::builder() 35 | .with_tag("example-worker") 36 | .with_actor_system(system) 37 | .with_handlers(|handlers| handlers.with_handler::("EchoActor.Echo")) 38 | .build() 39 | .await; 40 | 41 | remote 42 | .clone() 43 | .cluster_worker() 44 | .listen_addr("localhost:30101") 45 | .with_seed_addr("localhost:30100") 46 | .start() 47 | .await; 48 | 49 | { 50 | // let span = tracing::info_span!("CreateAndSend"); 51 | // let _enter = span.enter(); 52 | 53 | let actor = remote 54 | .actor_ref::("echo-actor".to_actor_id()) 55 | .await 56 | .expect("unable to get echo actor"); 57 | 58 | let result = actor.send(Echo("hello".to_string())).await; 59 | assert_eq!(result.unwrap(), "hello".to_string()); 60 | } 61 | 62 | tokio::signal::ctrl_c() 63 | .await 64 | .expect("failed to listen for event"); 65 | } 66 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "coerce-sharded-chat-example" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | coerce = { path = "../../coerce", features = ["full"] } 10 | coerce-macros = { version = "0.2.0" } 11 | coerce-redis = { path = "../../providers/persistence/coerce-redis" } 12 | coerce-k8s = { version = "0.1.7" } 13 | 14 | tokio = { version = "1.28.1", features = ["full"] } 15 | serde = { version = "1.0", features = ["derive", "rc"] } 16 | serde_json = "1.0" 17 | async-trait = { version = "0.1" } 18 | tungstenite = "0.17.3" 19 | tokio-tungstenite = { version = "0.17.2", features = ["stream", "connect"] } 20 | futures = { version = "0.3"} 21 | futures-io = { version = "0.3"} 22 | futures-util = { version = "0.3" } 23 | tracing = { version = "0.1", features = ["valuable"] } 24 | tracing-subscriber = { version = "0.3", features = ["json"] } 25 | chrono = { version = "0.4" } 26 | uuid = { version = "1.1.2", features = ["serde", "v4"] } 27 | clap = { version = "4.0", features = ["env", "derive"]} 28 | metrics-util= "0.14.0" 29 | metrics-exporter-prometheus = "0.11.0" 30 | env_logger = "0.9" 31 | protobuf = "3.2.0" 32 | metrics-process = "1.0.10" 33 | 34 | #jemallocator = { version = "0.3.0", features = ["profiling", "debug"] } 35 | #console-subscriber = "0.1.6" 36 | 37 | [[bin]] 38 | name = "sharded-chat-server" 39 | path = "bin/server.rs" 40 | 41 | [[bin]] 42 | name = "sharded-chat-client" 43 | path = "bin/client.rs" -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/Dockerfile-server: -------------------------------------------------------------------------------- 1 | FROM rust:1.64-bullseye as build 2 | 3 | WORKDIR /app 4 | 5 | COPY . ./ 6 | 7 | RUN cargo build --bin sharded-chat-server --release 8 | 9 | 10 | FROM debian:bullseye-slim 11 | 12 | #Coerce Remoting 13 | EXPOSE 31101 14 | 15 | #WebSocket Server 16 | EXPOSE 31102 17 | 18 | #Coerce HTTP API 19 | EXPOSE 31103 20 | 21 | #Metrics Exporter 22 | EXPOSE 31104 23 | 24 | ENV NODE_ID=1 25 | ENV REMOTE_LISTEN_ADDR="0.0.0.0:31101" 26 | ENV WEBSOCKET_LISTEN_ADDR="0.0.0.0:31102" 27 | ENV CLUSTER_API_LISTEN_ADDR="0.0.0.0:31103" 28 | ENV METRICS_EXPORTER_LISTEN_ADDR="0.0.0.0:31104" 29 | ENV LOG_LEVEL="INFO" 30 | 31 | WORKDIR /app 32 | COPY --from=build /app/target/release/sharded-chat-server /app/sharded-chat-server 33 | 34 | ENTRYPOINT ["./sharded-chat-server"] -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LeonHartley/Coerce-rs/a2d3f19cd92ce1776b4dc0be6616784d53ff4b89/examples/coerce-sharded-chat-example/README.md -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/bin/client.rs: -------------------------------------------------------------------------------- 1 | use clap::{arg, Command}; 2 | 3 | use coerce_sharded_chat_example::websocket::client::ChatClient; 4 | 5 | use std::str::FromStr; 6 | use tokio::io::{AsyncBufReadExt, BufReader}; 7 | use tokio::sync::mpsc::Receiver; 8 | use tokio_tungstenite::tungstenite::Message; 9 | 10 | #[macro_use] 11 | extern crate tracing; 12 | 13 | #[tokio::main] 14 | pub async fn main() { 15 | let mut matches = Command::new("coerce-sharded-chat-client") 16 | .arg(arg!(--name "The display name of the chat user tied to this connection")) 17 | .arg(arg!(--websocket_url "The host and port which the sharded chat websockets will listen from")) 18 | .arg(arg!(--log_level [LOG_LEVEL] "The minimum level at which the application log will be filtered (default=INFO)")) 19 | .get_matches(); 20 | 21 | let name = matches.remove_one::("name").unwrap(); 22 | let websocket_url = matches.remove_one::("websocket_url").unwrap(); 23 | 24 | setup_logging(matches.remove_one("log_level").unwrap_or("INFO")); 25 | 26 | let mut client = ChatClient::connect(&websocket_url, &name) 27 | .await 28 | .unwrap_or_else(|_| panic!("connect to websocket (url={})", &websocket_url)); 29 | 30 | let mut current_chat = None; 31 | 32 | info!("connected! Join a chat stream via /join [chat-stream-name]"); 33 | 34 | tokio::spawn(websocket_read(client.take_reader().unwrap())); 35 | 36 | let mut reader = BufReader::new(tokio::io::stdin()).lines(); 37 | while let Ok(line) = reader.next_line().await { 38 | if let Some(line) = line { 39 | if line.is_empty() { 40 | continue; 41 | } else if line.starts_with('/') { 42 | process_command(line, &mut client, &mut current_chat).await; 43 | } else if let Some(chat_stream) = ¤t_chat { 44 | client.chat(chat_stream.to_string(), line).await; 45 | } 46 | } 47 | } 48 | } 49 | 50 | async fn websocket_read(mut websocket_reader: Receiver) { 51 | while let Some(message) = websocket_reader.recv().await { 52 | match message { 53 | Message::Binary(bytes) => { 54 | info!("{}", String::from_utf8(bytes).unwrap()); 55 | } 56 | 57 | _ => {} 58 | } 59 | } 60 | } 61 | 62 | async fn process_command(line: String, client: &mut ChatClient, current_chat: &mut Option) { 63 | let mut words = line.split(' '); 64 | match words.next() { 65 | Some("/join") => { 66 | let stream_name = words.next().expect("chat stream name").to_string(); 67 | 68 | client.join_chat(stream_name.clone()).await; 69 | current_chat.replace(stream_name); 70 | } 71 | Some("/spam") => { 72 | if let Some(current_chat) = current_chat { 73 | let mut i = 0; 74 | loop { 75 | client.chat(current_chat.to_string(), format!("{i}")).await; 76 | 77 | i += 1; 78 | } 79 | } 80 | } 81 | 82 | _ => { 83 | error!("invalid command, commands available: /join "); 84 | } 85 | } 86 | } 87 | 88 | fn setup_logging(log_level: &str) { 89 | tracing_subscriber::fmt() 90 | .with_file(true) 91 | .with_line_number(true) 92 | .with_target(true) 93 | .with_thread_names(true) 94 | .with_ansi(false) 95 | .with_max_level(tracing::Level::from_str(log_level).unwrap()) 96 | .init(); 97 | } 98 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/bin/server.rs: -------------------------------------------------------------------------------- 1 | use clap::{arg, Command}; 2 | use coerce::remote::system::NodeId; 3 | use coerce_sharded_chat_example::app::{ShardedChat, ShardedChatConfig, ShardedChatPersistence}; 4 | 5 | use std::str::FromStr; 6 | 7 | use tracing_subscriber::fmt::format::FmtSpan; 8 | 9 | #[tokio::main] 10 | pub async fn main() { 11 | let config = configure_application(); 12 | let mut sharded_chat = ShardedChat::start(config).await; 13 | let _ = tokio::signal::ctrl_c().await; 14 | sharded_chat.stop().await; 15 | } 16 | 17 | fn configure_application() -> ShardedChatConfig { 18 | let mut matches = Command::new("coerce-sharded-chat-server") 19 | .arg(arg!(--node_id "The ID this node will identify itself as").env("NODE_ID")) 20 | .arg(arg!(--remote_listen_addr "The host and port which Coerce will listen to connections from").env("REMOTE_LISTEN_ADDR")) 21 | .arg(arg!(--websocket_listen_addr "The host and port which the sharded chat websockets will listen from").env("WEBSOCKET_LISTEN_ADDR")) 22 | .arg(arg!(--cluster_api_listen_addr "The host and port which the Coerce cluster HTTP API will listen from").env("CLUSTER_API_LISTEN_ADDR")) 23 | .arg(arg!(--log_level [LOG_LEVEL] "The minimum level at which the application log will be filtered (default=INFO)").env("LOG_LEVEL")) 24 | .arg(arg!(--remote_seed_addr [TCP_ADDR] "(optional) The host and port which Coerce will attempt to use as a seed node").env("REMOTE_SEED_ADDR")) 25 | .arg(arg!(--redis_addr [TCP_ADDR] "(optional) Redis TCP address. By default, in-memory persistence is enabled").env("REDIS_ADDR")) 26 | .get_matches(); 27 | 28 | let node_id: NodeId = matches 29 | .remove_one::("node_id") 30 | .unwrap() 31 | .parse() 32 | .expect("invalid node_id (unsigned 64bit integer)"); 33 | 34 | let remote_listen_addr = matches.remove_one::("remote_listen_addr").unwrap(); 35 | 36 | let websocket_listen_addr = matches 37 | .remove_one::("websocket_listen_addr") 38 | .unwrap(); 39 | 40 | let cluster_api_listen_addr = matches 41 | .remove_one::("cluster_api_listen_addr") 42 | .unwrap(); 43 | 44 | let remote_seed_addr = matches.remove_one::("remote_seed_addr"); 45 | 46 | let log_level = matches 47 | .remove_one::("log_level") 48 | .unwrap_or_else(|| "INFO".to_string()); 49 | 50 | tracing_subscriber::fmt() 51 | .compact() 52 | .with_max_level(tracing::Level::from_str(&log_level).unwrap()) 53 | .with_span_events(FmtSpan::NONE) 54 | .init(); 55 | 56 | let persistence = if let Some(redis_addr) = matches.remove_one::("redis_addr") { 57 | ShardedChatPersistence::Redis { 58 | host: Some(redis_addr), 59 | } 60 | } else { 61 | ShardedChatPersistence::InMemory 62 | }; 63 | 64 | ShardedChatConfig { 65 | node_id, 66 | remote_listen_addr, 67 | remote_seed_addr, 68 | websocket_listen_addr, 69 | cluster_api_listen_addr, 70 | persistence, 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/kustomize/sharded-chat-server/base/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - serviceaccount.yaml 3 | - service.yaml 4 | - statefulset.yaml -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/kustomize/sharded-chat-server/base/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: sharded-chat-server 5 | spec: 6 | clusterIP: None 7 | ports: 8 | - name: coerce 9 | port: 31101 10 | protocol: TCP 11 | - name: coerce-api 12 | port: 31103 13 | protocol: TCP 14 | selector: 15 | app: sharded-chat-server -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/kustomize/sharded-chat-server/base/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: sharded-chat-server 5 | --- 6 | kind: Role 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | metadata: 9 | name: pod-reader 10 | rules: 11 | - apiGroups: [ "" ] 12 | resources: [ "pods" ] 13 | verbs: [ "list" ] 14 | --- 15 | kind: RoleBinding 16 | apiVersion: rbac.authorization.k8s.io/v1 17 | metadata: 18 | name: read-pods 19 | subjects: 20 | - kind: ServiceAccount 21 | name: sharded-chat-server 22 | roleRef: 23 | kind: Role 24 | name: pod-reader 25 | apiGroup: rbac.authorization.k8s.io -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/kustomize/sharded-chat-server/base/statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: sharded-chat-server 5 | spec: 6 | replicas: 1 7 | serviceName: sharded-chat-server 8 | template: 9 | metadata: 10 | annotations: 11 | prometheus.io/scrape: 'true' 12 | prometheus.io/port: '31104' 13 | labels: 14 | app: "sharded-chat-server" 15 | spec: 16 | serviceAccountName: sharded-chat-server 17 | containers: 18 | - name: sharded-chat-server 19 | image: "sharded-chat-server:latest" 20 | imagePullPolicy: IfNotPresent 21 | livenessProbe: 22 | tcpSocket: 23 | port: coerce 24 | ports: 25 | - name: coerce 26 | containerPort: 31101 27 | protocol: TCP 28 | - name: websocket 29 | containerPort: 31102 30 | protocol: TCP 31 | - name: coerce-api 32 | containerPort: 31103 33 | protocol: TCP 34 | - name: metrics 35 | containerPort: 31104 36 | protocol: TCP 37 | resources: 38 | { } 39 | env: 40 | - name: LOG_LEVEL 41 | valueFrom: 42 | configMapKeyRef: 43 | name: server-config 44 | key: LOG_LEVEL 45 | - name: POD_NAME 46 | valueFrom: 47 | fieldRef: 48 | fieldPath: metadata.name 49 | - name: CLUSTER_IP 50 | valueFrom: 51 | fieldRef: 52 | fieldPath: status.podIP 53 | - name: COERCE_K8S_POD_SELECTOR 54 | value: "app=sharded-chat-server" 55 | - name: COERCE_OVERRIDE_INCOMING_NODE_ADDR 56 | value: "1" 57 | selector: 58 | matchLabels: 59 | app: sharded-chat-server -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/kustomize/sharded-chat-server/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ./base 5 | replicas: 6 | - name: sharded-chat-server 7 | count: 3 8 | #images: 9 | # - name: sharded-chat-server 10 | # newName: sharded-chat-server 11 | 12 | configMapGenerator: 13 | - name: server-config 14 | literals: 15 | - LOG_LEVEL=DEBUG -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/scripts/run-client-user-1.ps1: -------------------------------------------------------------------------------- 1 | cargo run --bin sharded-chat-client -- --websocket_url ws://localhost:31102 --name user1 -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/scripts/run-client-user-2.ps1: -------------------------------------------------------------------------------- 1 | cargo run --bin sharded-chat-client -- --websocket_url ws://localhost:32102 --name user2 -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/scripts/run-client-user-3.ps1: -------------------------------------------------------------------------------- 1 | cargo run --bin sharded-chat-client -- --websocket_url ws://localhost:33102 --name user3 -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/scripts/run-client-user-4.ps1: -------------------------------------------------------------------------------- 1 | cargo run --bin sharded-chat-client -- --websocket_url ws://localhost:34102 --name user4 -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/scripts/run-server-1.ps1: -------------------------------------------------------------------------------- 1 | cargo run --bin sharded-chat-server -- --node_id 0 --remote_seed_addr 127.0.0.1:31101 --remote_listen_addr 0.0.0.0:31101 --websocket_listen_addr localhost:31102 --cluster_api_listen_addr 127.0.0.1:31103 --log_level DEBUG -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/scripts/run-server-2.ps1: -------------------------------------------------------------------------------- 1 | cargo run --bin sharded-chat-server -- --node_id 2 --remote_listen_addr 0.0.0.0:32101 --websocket_listen_addr localhost:32102 --cluster_api_listen_addr 127.0.0.1:32103 --remote_seed_addr 127.0.0.1:33101 --log_level DEBUG 2 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/scripts/run-server-3.ps1: -------------------------------------------------------------------------------- 1 | cargo run --release --bin sharded-chat-server -- --node_id 3 --remote_listen_addr 0.0.0.0:33101 --websocket_listen_addr localhost:33102 --cluster_api_listen_addr 127.0.0.1:33103 --remote_seed_addr 127.0.0.1:32101 --log_level INFO 2 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/scripts/run-server-4.ps1: -------------------------------------------------------------------------------- 1 | cargo run --release --bin sharded-chat-server -- --node_id 4 --remote_listen_addr localhost:34101 --websocket_listen_addr localhost:34102 --cluster_api_listen_addr 0.0.0.0:34103 --remote_seed_addr localhost:31101 --log_level INFO --metrics_exporter_listen_addr 0.0.0.0:34104 -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/src/actor/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod peer; 2 | pub mod pubsub; 3 | pub mod stream; 4 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/src/actor/pubsub.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::stream::ChatMessage; 2 | use coerce::remote::net::StreamData; 3 | use coerce::remote::stream::pubsub::Topic; 4 | 5 | #[derive(Serialize, Deserialize)] 6 | pub enum ChatReceive { 7 | Message(ChatMessage), 8 | } 9 | 10 | #[derive(Clone)] 11 | pub struct ChatStreamTopic(pub String); 12 | 13 | impl Topic for ChatStreamTopic { 14 | type Message = ChatReceive; 15 | 16 | fn topic_name() -> &'static str { 17 | "ChatStream" 18 | } 19 | 20 | fn key(&self) -> String { 21 | self.0.clone() 22 | } 23 | } 24 | 25 | impl StreamData for ChatReceive { 26 | fn read_from_bytes(data: Vec) -> Option { 27 | serde_json::from_slice(&data).unwrap() 28 | } 29 | 30 | fn write_to_bytes(&self) -> Option> { 31 | Some(serde_json::to_vec(self).unwrap()) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate async_trait; 3 | 4 | #[macro_use] 5 | extern crate coerce_macros; 6 | 7 | #[macro_use] 8 | extern crate serde; 9 | 10 | #[macro_use] 11 | extern crate tracing; 12 | 13 | pub mod actor; 14 | pub mod app; 15 | pub mod protocol; 16 | pub mod websocket; 17 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/src/protocol/chat.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/wrappers.proto"; 4 | 5 | package coerce.example.sharding.chat; 6 | 7 | message Handshake { 8 | string name = 1; 9 | } 10 | 11 | message JoinChat { 12 | string chat_stream_id = 1; 13 | 14 | google.protobuf.StringValue join_token = 2; 15 | } 16 | 17 | message LeaveChat { 18 | string chat_stream_id = 1; 19 | } 20 | 21 | message SendChatMessage { 22 | ChatMessage message = 1; 23 | } 24 | 25 | message ChatMessage { 26 | string chat_stream_id = 1; 27 | 28 | google.protobuf.UInt64Value message_id = 2; 29 | 30 | google.protobuf.StringValue sender = 3; 31 | 32 | string message = 4; 33 | } 34 | 35 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/src/protocol/internal.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package coerce.example.sharding.internal; 4 | 5 | import "google/protobuf/wrappers.proto"; 6 | import "chat.proto"; 7 | 8 | message CreateChatStream { 9 | string chat_stream_id = 1; 10 | 11 | string creator = 2; 12 | } 13 | 14 | message Join { 15 | string chat_stream_id = 1; 16 | 17 | google.protobuf.StringValue join_token = 2; 18 | 19 | string name = 3; 20 | } 21 | 22 | message JoinResult { 23 | enum JoinErr { 24 | None = 0; 25 | NameCollision = 1; 26 | } 27 | 28 | message State { 29 | string chat_stream_id = 1; 30 | 31 | string creator = 2; 32 | 33 | repeated coerce.example.sharding.chat.ChatMessage chat_history = 3; 34 | 35 | string token = 4; 36 | } 37 | 38 | State stream_state = 1; 39 | 40 | JoinErr error = 2; 41 | } 42 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/src/protocol/mod.rs: -------------------------------------------------------------------------------- 1 | // @generated 2 | 3 | pub mod chat; 4 | -------------------------------------------------------------------------------- /examples/coerce-sharded-chat-example/src/websocket/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::actor::peer::Peer; 2 | use crate::actor::stream::{ChatStreamFactory, Handshake}; 3 | use coerce::actor::message::Message; 4 | use coerce::actor::system::ActorSystem; 5 | use coerce::actor::IntoActor; 6 | use coerce::sharding::Sharding; 7 | use futures_util::StreamExt; 8 | use std::net::SocketAddr; 9 | 10 | use tokio::net::{TcpListener, TcpStream, ToSocketAddrs}; 11 | use tokio_tungstenite::accept_async; 12 | use tungstenite::Error::{ConnectionClosed, Protocol, Utf8}; 13 | 14 | pub mod client; 15 | 16 | async fn handle_connection( 17 | peer_addr: SocketAddr, 18 | stream: TcpStream, 19 | actor_system: ActorSystem, 20 | sharding: Sharding, 21 | ) -> Result<(), tungstenite::Error> { 22 | let stream = accept_async(stream).await; 23 | if let Ok(stream) = stream { 24 | let (writer, mut reader) = stream.split(); 25 | let handshake = reader.next().await; 26 | if let Some(Ok(handshake)) = handshake { 27 | let handshake: Handshake = Handshake::from_bytes(handshake.into_data()).unwrap(); 28 | 29 | match Peer::new(handshake.name, peer_addr, reader, writer, sharding) 30 | .into_anon_actor(Some(format!("peer-{peer_addr}")), &actor_system) 31 | .await 32 | { 33 | Ok(actor_ref) => { 34 | debug!("peer actor created, ref: {:?}", actor_ref) 35 | } 36 | Err(e) => { 37 | warn!("error creating peer actor: {}", e) 38 | } 39 | } 40 | } else { 41 | warn!("failed to read handshake (connection={})", &peer_addr); 42 | } 43 | } 44 | 45 | Ok(()) 46 | } 47 | 48 | async fn accept_connection( 49 | peer: SocketAddr, 50 | stream: TcpStream, 51 | system: ActorSystem, 52 | sharding: Sharding, 53 | ) { 54 | if let Err(e) = handle_connection(peer, stream, system, sharding).await { 55 | match e { 56 | ConnectionClosed | Protocol(_) | Utf8 => (), 57 | _err => {} 58 | } 59 | } 60 | } 61 | 62 | pub async fn start( 63 | addr: S, 64 | system: ActorSystem, 65 | sharding: Sharding, 66 | ) { 67 | let listener = TcpListener::bind(addr).await.expect("websocket listen"); 68 | 69 | while let Ok((stream, _)) = listener.accept().await { 70 | let peer_address = stream.peer_addr().unwrap(); 71 | 72 | debug!("websocket connection from peer_addr: {}", peer_address); 73 | tokio::spawn(accept_connection( 74 | peer_address, 75 | stream, 76 | system.clone(), 77 | sharding.clone(), 78 | )); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /providers/discovery/coerce-k8s/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "coerce-k8s" 3 | version = "0.1.8" 4 | edition = "2021" 5 | description = "Kubernetes discovery provider, automatically discover cluster peers hosted in Kubernetes, based on a configurable pod-selection label" 6 | license = "Apache-2.0" 7 | authors = ["Leon Hartley "] 8 | repository = "https://github.com/leonhartley/coerce-rs" 9 | 10 | 11 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 12 | 13 | [dependencies] 14 | coerce = { version = "0.8.11", features = ["remote"] } 15 | kube = { version = "0.78.0", default-features = false, features = ["client", "rustls-tls"] } 16 | k8s-openapi = { version = "0.17.0", features = ["v1_24", "api"] } 17 | tracing = { version = "0.1" } -------------------------------------------------------------------------------- /providers/discovery/coerce-k8s/src/config.rs: -------------------------------------------------------------------------------- 1 | pub enum Address { 2 | /// Uses the pod's IP address for coerce cluster communication 3 | PodIp, 4 | 5 | /// Uses the pod's hostname and subdomain name (if set) 6 | Hostname, 7 | } 8 | 9 | pub struct KubernetesDiscoveryConfig { 10 | /// Pod label used to discover active Coerce cluster nodes (environment variable: COERCE_K8S_POD_SELECTOR) 11 | pub pod_selection_label: Option, 12 | 13 | /// Name of the port, as defined in the kubernetes pod spec ( 14 | pub coerce_remote_port_name: Option, 15 | 16 | pub cluster_node_address: Address, 17 | } 18 | 19 | impl Default for KubernetesDiscoveryConfig { 20 | fn default() -> Self { 21 | Self { 22 | pod_selection_label: Some( 23 | std::env::var("COERCE_K8S_POD_SELECTOR") 24 | .map_or_else(|_e| "app=coerce".to_string(), |s| s), 25 | ), 26 | coerce_remote_port_name: std::env::var("COERCE_K8S_PORT_NAME") 27 | .map_or_else(|_e| Some("coerce".to_string()), Some), 28 | cluster_node_address: std::env::var("COERCE_K8S_ADDR_MODE").map_or_else( 29 | |_e| Address::PodIp, 30 | |s| { 31 | if s.to_lowercase() == "hostname" { 32 | Address::Hostname 33 | } else { 34 | Address::PodIp 35 | } 36 | }, 37 | ), 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /providers/discovery/coerce-k8s/src/lib.rs: -------------------------------------------------------------------------------- 1 | use crate::config::{Address, KubernetesDiscoveryConfig}; 2 | use coerce::remote::system::RemoteActorSystem; 3 | use k8s_openapi::api::core::v1::Pod; 4 | use kube::api::ListParams; 5 | use kube::{Api, Client}; 6 | 7 | #[macro_use] 8 | extern crate tracing; 9 | pub mod config; 10 | 11 | pub struct KubernetesDiscovery { 12 | system: RemoteActorSystem, 13 | } 14 | 15 | impl KubernetesDiscovery { 16 | pub async fn discover(config: KubernetesDiscoveryConfig) -> Option> { 17 | let client = Client::try_default() 18 | .await 19 | .expect("failed to initialise k8s client"); 20 | let api = Api::::default_namespaced(client); 21 | 22 | let params = ListParams { 23 | label_selector: config.pod_selection_label.clone(), 24 | ..Default::default() 25 | }; 26 | 27 | let pods = api.list(¶ms).await; 28 | let mut cluster_nodes = vec![]; 29 | 30 | if let Ok(pods) = pods { 31 | debug!("pods={:#?}", &pods.items); 32 | 33 | let pods = pods.items; 34 | for pod in pods { 35 | let pod_spec = pod.spec.unwrap(); 36 | let pod_status = pod.status.unwrap(); 37 | 38 | debug!("pod_status={:?}", pod_status); 39 | 40 | // TODO: Check that the pod is available before using it as a 41 | // seed node. 42 | 43 | let addr = match &config.cluster_node_address { 44 | Address::PodIp => { 45 | let pod_ip = pod_status.pod_ip; 46 | if pod_ip.is_none() { 47 | continue; 48 | } 49 | 50 | pod_ip.unwrap() 51 | } 52 | 53 | Address::Hostname => { 54 | let hostname = pod_spec.hostname.unwrap(); 55 | let subdomain = pod_spec.subdomain.unwrap(); 56 | 57 | format!("{hostname}.{subdomain}") 58 | } 59 | }; 60 | 61 | for container in pod_spec.containers { 62 | let port = container 63 | .ports 64 | .unwrap() 65 | .into_iter() 66 | .find(|p| p.name == config.coerce_remote_port_name); 67 | if let Some(port) = port { 68 | cluster_nodes.push(format!("{}:{}", addr, port.container_port)); 69 | } 70 | } 71 | } 72 | } 73 | 74 | debug!("discovered nodes: {:#?}", &cluster_nodes); 75 | Some(cluster_nodes) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /providers/persistence/coerce-redis/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "coerce-redis" 3 | version = "0.4.4" 4 | authors = ["Leon Hartley "] 5 | edition = "2021" 6 | description = "Redis actor persistence provider for Coerce. Supports event sourcing and snapshots" 7 | license = "Apache-2.0" 8 | repository = "https://github.com/leonhartley/coerce-rs" 9 | 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | 12 | [features] 13 | default = [] 14 | cluster = [ 15 | "redis/cluster" 16 | ] 17 | 18 | [dependencies] 19 | coerce = { path = "../../../coerce", features = ["persistence"] } 20 | async-trait = { version = "0.1.64" } 21 | redis = { version = "0.23.0", features = ["tokio-comp"] } 22 | tokio = { version = "1.28.1", features = ["full"] } 23 | anyhow = "1" 24 | bytes = { version = "1.2.1" } -------------------------------------------------------------------------------- /providers/persistence/coerce-redis/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate async_trait; 3 | 4 | #[macro_use] 5 | extern crate redis; 6 | 7 | pub mod journal; 8 | --------------------------------------------------------------------------------