├── .gitignore ├── .editorconfig ├── .github ├── FUNDING.yml ├── semantic.yml └── workflows │ └── ci.yml ├── sdk ├── api │ ├── src │ │ ├── property │ │ │ ├── mod.rs │ │ │ └── storage.rs │ │ ├── lib.rs │ │ ├── config │ │ │ ├── meta.rs │ │ │ ├── mod.rs │ │ │ ├── broker.rs │ │ │ ├── server.rs │ │ │ ├── runtime.rs │ │ │ ├── telemetry.rs │ │ │ └── morax.rs │ │ └── request.rs │ └── Cargo.toml └── client │ ├── Cargo.toml │ └── src │ └── lib.rs ├── typos.toml ├── .config └── nextest.toml ├── crates ├── server │ ├── src │ │ ├── lib.rs │ │ ├── broker.rs │ │ └── server.rs │ └── Cargo.toml ├── broker │ ├── src │ │ ├── lib.rs │ │ ├── http.rs │ │ └── broker.rs │ └── Cargo.toml ├── meta │ ├── src │ │ ├── lib.rs │ │ ├── service │ │ │ ├── pull.rs │ │ │ ├── publish.rs │ │ │ ├── topic.rs │ │ │ ├── mod.rs │ │ │ └── subscription.rs │ │ ├── object.rs │ │ └── bootstrap.rs │ └── Cargo.toml ├── telemetry │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── storage │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── version │ ├── Cargo.toml │ ├── src │ │ └── lib.rs │ └── build.rs └── runtime │ ├── Cargo.toml │ └── src │ ├── lib.rs │ ├── global.rs │ └── runtime.rs ├── .cargo └── config.toml ├── rust-toolchain.toml ├── rustfmt.toml ├── clippy.toml ├── cmd └── morax │ ├── src │ ├── main.rs │ └── command.rs │ └── Cargo.toml ├── licenserc.toml ├── xtask ├── Cargo.toml └── src │ └── main.rs ├── tests ├── toolkit │ ├── src │ │ ├── lib.rs │ │ ├── state.rs │ │ └── container.rs │ └── Cargo.toml └── behavior │ ├── Cargo.toml │ ├── src │ └── lib.rs │ └── tests │ └── simple_pubsub.rs ├── dev ├── config.toml └── docker-compose.yml ├── README.md ├── taplo.toml ├── Cargo.toml └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /data 3 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | indent_style = space 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | 9 | [*.toml] 10 | indent_size = tab 11 | tab_width = 2 12 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | github: tisonkun 16 | 17 | -------------------------------------------------------------------------------- /sdk/api/src/property/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod storage; 16 | 17 | pub use storage::*; 18 | -------------------------------------------------------------------------------- /sdk/api/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod config; 16 | pub mod property; 17 | pub mod request; 18 | -------------------------------------------------------------------------------- /typos.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [default.extend-words] 16 | LSO = "LSO" 17 | 18 | [files] 19 | extend-exclude = [] 20 | -------------------------------------------------------------------------------- /.config/nextest.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [profile.default] 16 | slow-timeout = { period = "30s", terminate-after = 4 } 17 | -------------------------------------------------------------------------------- /crates/server/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #![feature(ip)] 16 | 17 | mod server; 18 | pub use server::*; 19 | 20 | mod broker; 21 | -------------------------------------------------------------------------------- /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [alias] 16 | x = "run --package x --" 17 | 18 | [env] 19 | CARGO_WORKSPACE_DIR = { value = "", relative = true } 20 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [toolchain] 16 | channel = "nightly-2025-04-19" 17 | components = ["cargo", "rustfmt", "clippy", "rust-analyzer"] 18 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | comment_width = 120 16 | format_code_in_doc_comments = true 17 | group_imports = "StdExternalCrate" 18 | imports_granularity = "Item" 19 | wrap_comments = true 20 | -------------------------------------------------------------------------------- /clippy.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | disallowed-types = [ 16 | { path = "tokio::runtime::Runtime", reason = "use runtime::Runtime" }, 17 | { path = "tokio::task::JoinHandle", reason = "use runtime::JoinHandle" }, 18 | ] 19 | -------------------------------------------------------------------------------- /crates/broker/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod broker; 16 | mod http; 17 | 18 | pub use http::make_broker_router; 19 | 20 | #[derive(Debug, thiserror::Error)] 21 | #[error("{0}")] 22 | pub struct BrokerError(String); 23 | -------------------------------------------------------------------------------- /crates/meta/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub use object::*; 16 | pub use service::PostgresMetaService; 17 | 18 | mod bootstrap; 19 | mod object; 20 | mod service; 21 | 22 | #[derive(Debug, thiserror::Error)] 23 | #[error("{0}")] 24 | pub struct MetaError(String); 25 | -------------------------------------------------------------------------------- /sdk/api/src/config/meta.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use serde::Deserialize; 16 | use serde::Serialize; 17 | 18 | #[derive(Debug, Clone, Serialize, Deserialize)] 19 | #[serde(deny_unknown_fields)] 20 | pub struct MetaServiceConfig { 21 | pub service_url: String, 22 | } 23 | -------------------------------------------------------------------------------- /sdk/api/src/config/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod broker; 16 | mod meta; 17 | mod morax; 18 | mod runtime; 19 | mod server; 20 | mod telemetry; 21 | 22 | pub use broker::*; 23 | pub use meta::*; 24 | pub use morax::*; 25 | pub use runtime::*; 26 | pub use server::*; 27 | pub use telemetry::*; 28 | -------------------------------------------------------------------------------- /cmd/morax/src/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use clap::Parser; 16 | 17 | mod command; 18 | 19 | #[derive(Debug, thiserror::Error)] 20 | #[error("{0}")] 21 | pub struct Error(pub String); 22 | 23 | fn main() -> error_stack::Result<(), Error> { 24 | let cmd = command::Command::parse(); 25 | cmd.run() 26 | } 27 | -------------------------------------------------------------------------------- /licenserc.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | headerPath = "Apache-2.0.txt" 16 | 17 | includes = ['**/*.proto', '**/*.rs', '**/*.yml', '**/*.yaml', '**/*.toml'] 18 | 19 | excludes = [ 20 | # Generated files 21 | ".github/workflows/release.yml", 22 | ] 23 | 24 | [properties] 25 | copyrightOwner = "tison " 26 | inceptionYear = 2024 27 | -------------------------------------------------------------------------------- /sdk/api/src/config/broker.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use serde::Deserialize; 16 | use serde::Serialize; 17 | 18 | #[derive(Debug, Clone, Serialize, Deserialize)] 19 | #[serde(deny_unknown_fields)] 20 | pub struct BrokerConfig { 21 | pub listen_addr: String, 22 | #[serde(skip_serializing_if = "Option::is_none")] 23 | pub advertise_addr: Option, 24 | } 25 | -------------------------------------------------------------------------------- /xtask/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "x" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [dependencies] 29 | clap = { workspace = true } 30 | which = { workspace = true } 31 | -------------------------------------------------------------------------------- /sdk/api/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "morax-api" 17 | publish = true 18 | 19 | description = "This crate defines public APIs for Morax." 20 | 21 | edition.workspace = true 22 | license.workspace = true 23 | readme.workspace = true 24 | repository.workspace = true 25 | version.workspace = true 26 | 27 | [dependencies] 28 | jiff = { workspace = true } 29 | serde = { workspace = true } 30 | 31 | [lints] 32 | workspace = true 33 | -------------------------------------------------------------------------------- /sdk/api/src/config/server.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use serde::Deserialize; 16 | use serde::Serialize; 17 | 18 | use crate::config::BrokerConfig; 19 | use crate::config::MetaServiceConfig; 20 | use crate::property::StorageProperty; 21 | 22 | #[derive(Debug, Clone, Serialize, Deserialize)] 23 | pub struct ServerConfig { 24 | pub meta: MetaServiceConfig, 25 | pub broker: BrokerConfig, 26 | pub default_storage: StorageProperty, 27 | } 28 | -------------------------------------------------------------------------------- /crates/telemetry/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "morax-telemetry" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [dependencies] 29 | logforth = { workspace = true } 30 | morax-api = { workspace = true } 31 | 32 | [lints] 33 | workspace = true 34 | -------------------------------------------------------------------------------- /sdk/api/src/config/runtime.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::num::NonZeroUsize; 16 | 17 | use serde::Deserialize; 18 | use serde::Serialize; 19 | 20 | #[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] 21 | #[serde(deny_unknown_fields)] 22 | pub struct RuntimeOptions { 23 | #[serde(skip_serializing_if = "Option::is_none")] 24 | pub server_runtime_threads: Option, 25 | #[serde(skip_serializing_if = "Option::is_none")] 26 | pub io_runtime_threads: Option, 27 | } 28 | -------------------------------------------------------------------------------- /crates/storage/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "morax-storage" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [dependencies] 29 | error-stack = { workspace = true } 30 | morax-api = { workspace = true } 31 | opendal = { workspace = true, features = ["services-s3"] } 32 | thiserror = { workspace = true } 33 | uuid = { workspace = true } 34 | 35 | [lints] 36 | workspace = true 37 | -------------------------------------------------------------------------------- /sdk/client/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "morax-client" 17 | publish = true 18 | 19 | description = "A client for Morax server" 20 | 21 | edition.workspace = true 22 | license.workspace = true 23 | readme.workspace = true 24 | repository.workspace = true 25 | version.workspace = true 26 | 27 | [dependencies] 28 | error-stack = { workspace = true } 29 | morax-api = { workspace = true } 30 | reqwest = { workspace = true } 31 | serde = { workspace = true } 32 | serde_json = { workspace = true } 33 | thiserror = { workspace = true } 34 | 35 | [lints] 36 | workspace = true 37 | -------------------------------------------------------------------------------- /tests/toolkit/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::any::Any; 16 | 17 | pub use state::make_test_env_state; 18 | pub use state::start_test_server; 19 | pub use state::TestEnvProps; 20 | pub use state::TestEnvState; 21 | 22 | mod container; 23 | mod state; 24 | 25 | type DropGuard = Box; 26 | 27 | pub fn make_test_name() -> String { 28 | let replacer = regex::Regex::new(r"[^a-zA-Z0-9]").unwrap(); 29 | let test_name = std::any::type_name::() 30 | .rsplit("::") 31 | .find(|part| *part != "{{closure}}") 32 | .unwrap(); 33 | replacer.replace_all(test_name, "_").to_string() 34 | } 35 | -------------------------------------------------------------------------------- /dev/config.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [server.meta] 16 | service_url = "postgres://morax:my_secret_password@127.0.0.1:5432/morax_meta" 17 | 18 | [server.broker] 19 | listen_addr = "0.0.0.0:8848" 20 | 21 | [server.default_storage] 22 | scheme = "s3" 23 | bucket = "test-bucket" 24 | region = "us-east-1" 25 | prefix = "/" 26 | endpoint = "http://127.0.0.1:9000" 27 | access_key_id = "minioadmin" 28 | secret_access_key = "minioadmin" 29 | virtual_host_style = false 30 | 31 | [telemetry.logs.stderr] 32 | filter = "DEBUG" 33 | 34 | [runtime] 35 | #server_runtime_threads = 2 36 | #exec_runtime_threads = 37 | #io_runtime_threads = 38 | -------------------------------------------------------------------------------- /crates/meta/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "morax-meta" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [dependencies] 29 | error-stack = { workspace = true } 30 | log = { workspace = true } 31 | morax-api = { workspace = true } 32 | morax-runtime = { workspace = true } 33 | serde = { workspace = true } 34 | sqlx = { workspace = true } 35 | thiserror = { workspace = true } 36 | uuid = { workspace = true } 37 | 38 | [lints] 39 | workspace = true 40 | -------------------------------------------------------------------------------- /crates/version/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "morax-version" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [dependencies] 29 | const_format = { workspace = true } 30 | serde = { workspace = true } 31 | shadow-rs = { workspace = true } 32 | 33 | [build-dependencies] 34 | build-data = { workspace = true } 35 | gix-discover = { workspace = true } 36 | shadow-rs = { workspace = true, features = ["build"] } 37 | 38 | [lints] 39 | workspace = true 40 | -------------------------------------------------------------------------------- /crates/runtime/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "morax-runtime" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [features] 29 | test = [] 30 | 31 | [dependencies] 32 | better-panic = { workspace = true } 33 | fastimer = { workspace = true } 34 | futures = { workspace = true } 35 | log = { workspace = true } 36 | morax-api = { workspace = true } 37 | pin-project = { workspace = true } 38 | tokio = { workspace = true } 39 | 40 | [lints] 41 | workspace = true 42 | -------------------------------------------------------------------------------- /.github/semantic.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # The pull request's title should be fulfilled the following pattern: 16 | # 17 | # [optional scope]: 18 | # 19 | # ... where valid types and scopes can be found below; for example: 20 | # 21 | # build(maven): One level down for native profile 22 | # 23 | # More about configurations on https://github.com/Ezard/semantic-prs#configuration 24 | 25 | enabled: true 26 | 27 | titleOnly: true 28 | 29 | types: 30 | - feat 31 | - fix 32 | - docs 33 | - style 34 | - refactor 35 | - perf 36 | - test 37 | - build 38 | - ci 39 | - chore 40 | - revert 41 | 42 | targetUrl: https://github.com/tisonkun/morax/blob/main/.github/semantic.yml 43 | -------------------------------------------------------------------------------- /crates/broker/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "morax-broker" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [dependencies] 29 | error-stack = { workspace = true } 30 | jiff = { workspace = true } 31 | log = { workspace = true } 32 | morax-api = { workspace = true } 33 | morax-meta = { workspace = true } 34 | morax-storage = { workspace = true } 35 | poem = { workspace = true } 36 | serde_json = { workspace = true } 37 | thiserror = { workspace = true } 38 | 39 | [lints] 40 | workspace = true 41 | -------------------------------------------------------------------------------- /crates/runtime/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod global; 16 | pub use global::*; 17 | 18 | mod runtime; 19 | pub use runtime::*; 20 | 21 | /// Returns the number of logical CPUs on the current machine. 22 | // This method fills the gap that `std::thread::available_parallelism()` 23 | // may return `Err` on some platforms, in which case we default to `1`. 24 | #[track_caller] 25 | pub fn num_cpus() -> std::num::NonZeroUsize { 26 | match std::thread::available_parallelism() { 27 | Ok(parallelism) => parallelism, 28 | Err(err) => { 29 | log::warn!(err: err; "failed to fetch the available parallelism; fallback to 1"); 30 | std::num::NonZeroUsize::new(1).unwrap() 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /sdk/api/src/config/telemetry.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use serde::Deserialize; 16 | use serde::Serialize; 17 | 18 | #[derive(Debug, Clone, Serialize, Deserialize)] 19 | pub struct TelemetryConfig { 20 | #[serde(default = "LogsConfig::disabled")] 21 | pub logs: LogsConfig, 22 | } 23 | 24 | #[derive(Debug, Clone, Serialize, Deserialize)] 25 | pub struct LogsConfig { 26 | #[serde(skip_serializing_if = "Option::is_none")] 27 | pub stderr: Option, 28 | } 29 | 30 | impl LogsConfig { 31 | pub fn disabled() -> Self { 32 | Self { stderr: None } 33 | } 34 | } 35 | 36 | #[derive(Debug, Clone, Serialize, Deserialize)] 37 | pub struct StderrAppenderConfig { 38 | pub filter: String, 39 | } 40 | -------------------------------------------------------------------------------- /tests/behavior/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "behavior-tests" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [dependencies] 29 | base64 = { workspace = true } 30 | insta = { workspace = true } 31 | morax-api = { workspace = true } 32 | morax-client = { workspace = true } 33 | morax-runtime = { workspace = true, features = ["test"] } 34 | morax-telemetry = { workspace = true } 35 | reqwest = { workspace = true } 36 | test-harness = { workspace = true } 37 | tests-toolkit = { workspace = true } 38 | 39 | [lints] 40 | workspace = true 41 | -------------------------------------------------------------------------------- /crates/server/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "morax-server" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [dependencies] 29 | error-stack = { workspace = true } 30 | futures = { workspace = true } 31 | local-ip-address = { workspace = true } 32 | log = { workspace = true } 33 | mea = { workspace = true } 34 | morax-api = { workspace = true } 35 | morax-broker = { workspace = true } 36 | morax-meta = { workspace = true } 37 | morax-runtime = { workspace = true } 38 | poem = { workspace = true } 39 | thiserror = { workspace = true } 40 | 41 | [lints] 42 | workspace = true 43 | -------------------------------------------------------------------------------- /cmd/morax/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "morax" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [[bin]] 29 | doc = false 30 | name = "morax" 31 | path = "src/main.rs" 32 | 33 | [dependencies] 34 | clap = { workspace = true } 35 | ctrlc = { workspace = true } 36 | error-stack = { workspace = true } 37 | morax-api = { workspace = true } 38 | morax-runtime = { workspace = true } 39 | morax-server = { workspace = true } 40 | morax-telemetry = { workspace = true } 41 | morax-version = { workspace = true } 42 | serde = { workspace = true } 43 | thiserror = { workspace = true } 44 | toml = { workspace = true } 45 | 46 | [lints] 47 | workspace = true 48 | -------------------------------------------------------------------------------- /tests/toolkit/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "tests-toolkit" 17 | publish = false 18 | 19 | edition.workspace = true 20 | license.workspace = true 21 | readme.workspace = true 22 | repository.workspace = true 23 | version.workspace = true 24 | 25 | [package.metadata.release] 26 | release = false 27 | 28 | [dependencies] 29 | local-ip-address = { workspace = true } 30 | morax-api = { workspace = true } 31 | morax-runtime = { workspace = true, features = ["test"] } 32 | morax-server = { workspace = true } 33 | morax-storage = { workspace = true } 34 | opendal = { workspace = true, features = ["services-s3"] } 35 | regex = { workspace = true } 36 | scopeguard = { workspace = true } 37 | serde = { workspace = true } 38 | sqlx = { workspace = true } 39 | testcontainers = { workspace = true } 40 | toml = { workspace = true } 41 | url = { workspace = true } 42 | 43 | [lints] 44 | workspace = true 45 | -------------------------------------------------------------------------------- /dev/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | services: 16 | postgres: 17 | image: postgres:16.3-bullseye 18 | ports: 19 | - "5432:5432" 20 | environment: 21 | POSTGRES_DB: morax_meta 22 | POSTGRES_USER: morax 23 | POSTGRES_PASSWORD: my_secret_password 24 | healthcheck: 25 | test: [ "CMD", "pg_isready", "-U", "morax", "-d", "morax_meta" ] 26 | interval: 5s 27 | timeout: 5s 28 | retries: 5 29 | 30 | minio-server: 31 | image: minio/minio:RELEASE.2024-07-16T23-46-41Z 32 | ports: 33 | - "9000:9000" 34 | - "9001:9001" 35 | entrypoint: bash 36 | command: -c 'mkdir -p /data/test-bucket && minio server /data --console-address ":9001"' 37 | environment: 38 | MINIO_ROOT_USER: minioadmin 39 | MINIO_ROOT_PASSWORD: minioadmin 40 | healthcheck: 41 | test: [ "CMD", "mc", "ready", "local" ] 42 | interval: 5s 43 | timeout: 5s 44 | retries: 5 45 | -------------------------------------------------------------------------------- /crates/telemetry/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use logforth::append; 16 | use logforth::filter::env_filter::EnvFilterBuilder; 17 | use logforth::filter::EnvFilter; 18 | use morax_api::config::TelemetryConfig; 19 | 20 | pub fn init(config: &TelemetryConfig) { 21 | let mut logger = logforth::builder(); 22 | 23 | // stderr logger 24 | if let Some(ref stderr) = config.logs.stderr { 25 | logger = logger.dispatch(|d| { 26 | d.filter(make_rust_log_filter_with_default_env(&stderr.filter)) 27 | .append(append::Stderr::default()) 28 | }); 29 | } 30 | 31 | logger.apply(); 32 | } 33 | 34 | fn make_rust_log_filter(filter: &str) -> EnvFilter { 35 | let builder = EnvFilterBuilder::new() 36 | .try_parse(filter) 37 | .unwrap_or_else(|_| panic!("failed to parse filter: {filter}")); 38 | EnvFilter::new(builder) 39 | } 40 | 41 | fn make_rust_log_filter_with_default_env(filter: &str) -> EnvFilter { 42 | if let Ok(filter) = std::env::var("RUST_LOG") { 43 | make_rust_log_filter(&filter) 44 | } else { 45 | make_rust_log_filter(filter) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Morax 2 | 3 | [![Discord][discord-badge]][discord-url] 4 | [![Apache 2.0 licensed][license-badge]][license-url] 5 | [![Build Status][actions-badge]][actions-url] 6 | 7 | [discord-badge]: https://img.shields.io/discord/1291345378246922363?logo=discord&label=discord 8 | [discord-url]: https://discord.gg/RRxbfYGqHM 9 | [license-badge]: https://img.shields.io/crates/l/morax 10 | [license-url]: LICENSE 11 | [actions-badge]: https://github.com/tisonkun/morax/workflows/CI/badge.svg 12 | [actions-url]:https://github.com/tisonkun/morax/actions?query=workflow%3ACI 13 | 14 | Morax is aimed at providing message queue and data streaming functionality based on cloud native services: 15 | 16 | * Meta service is backed by Postgres compatible relational database services (RDS, Aurora, etc.). 17 | * Data storage is backed by S3 compatible object storage services (S3, MinIO, etc.). 18 | 19 | ## Usage 20 | 21 | Currently, Morax supports basic PubSub APIs. You can try it out with the following steps. 22 | 23 | 1. Start the environment that provides a Postgres instance and a MinIO instance: 24 | 25 | ```shell 26 | docker compose -f ./dev/docker-compose.yml up 27 | ``` 28 | 29 | 2. Build the `morax` binary: 30 | 31 | ```shell 32 | cargo x build 33 | ``` 34 | 35 | 3. Start the broker: 36 | 37 | ```shell 38 | ./target/debug/morax start --config-file ./dev/config.toml 39 | ``` 40 | 41 | 42 | The broker is now running at `localhost:8848`. You can talk to it with the [`morax-client`](sdk/client). The wire protocol is HTTP so that all the HTTP ecosystem is ready for use. 43 | 44 | You can also get an impression of the interaction by reading the test cases in: 45 | 46 | * [behavior-tests](tests/behavior/tests) 47 | 48 | ## License 49 | 50 | This project is licensed under [Apache License, Version 2.0](https://github.com/tisonkun/logforth/blob/main/LICENSE). 51 | -------------------------------------------------------------------------------- /crates/meta/src/service/pull.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use error_stack::Result; 16 | use error_stack::ResultExt; 17 | 18 | use crate::FetchTopicSplitRequest; 19 | use crate::MetaError; 20 | use crate::PostgresMetaService; 21 | use crate::TopicSplit; 22 | 23 | impl PostgresMetaService { 24 | pub async fn fetch_topic_splits( 25 | &self, 26 | request: FetchTopicSplitRequest, 27 | ) -> Result, MetaError> { 28 | let make_error = || MetaError("failed to fetch topic splits".to_string()); 29 | let pool = self.pool.clone(); 30 | 31 | let mut txn = pool.begin().await.change_context_lazy(make_error)?; 32 | 33 | let topic_id = request.topic_id; 34 | let start_offset = request.start_offset; 35 | let end_offset = request.end_offset; 36 | 37 | let topic_splits: Vec = sqlx::query_as( 38 | r#" 39 | SELECT split_id, topic_id, start_offset, end_offset 40 | FROM topic_splits 41 | WHERE topic_id = $1 AND start_offset < $3 AND end_offset > $2 42 | "#, 43 | ) 44 | .bind(topic_id) 45 | .bind(start_offset) 46 | .bind(end_offset) 47 | .fetch_all(&mut *txn) 48 | .await 49 | .change_context_lazy(make_error)?; 50 | 51 | txn.commit().await.change_context_lazy(make_error)?; 52 | 53 | Ok(topic_splits) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /crates/meta/src/object.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use morax_api::property::TopicProperty; 16 | use sqlx::types::Json; 17 | use uuid::Uuid; 18 | 19 | #[derive(Debug, Clone)] 20 | pub struct CreateTopicRequest { 21 | pub name: String, 22 | pub properties: TopicProperty, 23 | } 24 | 25 | #[derive(Debug, Clone, sqlx::FromRow)] 26 | pub struct Topic { 27 | pub topic_id: i64, 28 | pub topic_name: String, 29 | pub properties: Json, 30 | } 31 | 32 | #[derive(Debug, Clone)] 33 | pub struct CreateSubscriptionRequest { 34 | pub name: String, 35 | pub topic: String, 36 | } 37 | 38 | #[derive(Debug, Clone, sqlx::FromRow)] 39 | pub struct Subscription { 40 | pub subscription_id: i64, 41 | pub subscription_name: String, 42 | pub topic_id: i64, 43 | pub topic_name: String, 44 | } 45 | 46 | #[derive(Debug, Clone)] 47 | pub struct CommitTopicSplitRequest { 48 | pub topic_id: i64, 49 | pub split_id: Uuid, 50 | pub count: i64, 51 | } 52 | 53 | #[derive(Debug, Clone)] 54 | pub struct AcknowledgeRequest { 55 | pub subscription_id: i64, 56 | pub ack_ids: Vec, 57 | } 58 | 59 | #[derive(Debug, Clone)] 60 | pub struct FetchTopicSplitRequest { 61 | pub topic_id: i64, 62 | pub start_offset: i64, 63 | pub end_offset: i64, 64 | } 65 | 66 | #[derive(Debug, Clone, sqlx::FromRow)] 67 | pub struct TopicSplit { 68 | pub topic_id: i64, 69 | pub start_offset: i64, 70 | pub end_offset: i64, 71 | pub split_id: Uuid, 72 | } 73 | -------------------------------------------------------------------------------- /crates/version/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use serde::Deserialize; 16 | use serde::Serialize; 17 | 18 | shadow_rs::shadow!(build); 19 | 20 | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] 21 | pub struct BuildInfo { 22 | pub branch: &'static str, 23 | pub commit: &'static str, 24 | pub commit_short: &'static str, 25 | pub clean: bool, 26 | pub source_time: &'static str, 27 | pub build_time: &'static str, 28 | pub rustc: &'static str, 29 | pub target: &'static str, 30 | pub version: &'static str, 31 | } 32 | 33 | pub const fn build_info() -> BuildInfo { 34 | BuildInfo { 35 | branch: build::BRANCH, 36 | commit: build::COMMIT_HASH, 37 | commit_short: build::SHORT_COMMIT, 38 | clean: build::GIT_CLEAN, 39 | source_time: env!("SOURCE_TIMESTAMP"), 40 | build_time: env!("BUILD_TIMESTAMP"), 41 | rustc: build::RUST_VERSION, 42 | target: build::BUILD_TARGET, 43 | version: build::PKG_VERSION, 44 | } 45 | } 46 | 47 | pub const fn version() -> &'static str { 48 | const BUILD_INFO: BuildInfo = build_info(); 49 | 50 | const_format::formatcp!( 51 | "\nversion: {}\nbranch: {}\ncommit: {}\nclean: {}\nsource_time: {}\nbuild_time: {}\nrustc: {}\ntarget: {}", 52 | BUILD_INFO.version, 53 | BUILD_INFO.branch, 54 | BUILD_INFO.commit, 55 | BUILD_INFO.clean, 56 | BUILD_INFO.source_time, 57 | BUILD_INFO.build_time, 58 | BUILD_INFO.rustc, 59 | BUILD_INFO.target, 60 | ) 61 | } 62 | -------------------------------------------------------------------------------- /taplo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | exclude = ["dev/config.toml"] 16 | include = ["Cargo.toml", "**/*.toml"] 17 | 18 | [formatting] 19 | # Align consecutive entries vertically. 20 | align_entries = false 21 | # Append trailing commas for multi-line arrays. 22 | array_trailing_comma = true 23 | # Expand arrays to multiple lines that exceed the maximum column width. 24 | array_auto_expand = true 25 | # Collapse arrays that don't exceed the maximum column width and don't contain comments. 26 | array_auto_collapse = true 27 | # Omit white space padding from single-line arrays 28 | compact_arrays = true 29 | # Omit white space padding from the start and end of inline tables. 30 | compact_inline_tables = false 31 | # Maximum column width in characters, affects array expansion and collapse, this doesn't take whitespace into account. 32 | # Note that this is not set in stone, and works on a best-effort basis. 33 | column_width = 80 34 | # Indent based on tables and arrays of tables and their subtables, subtables out of order are not indented. 35 | indent_tables = false 36 | # The substring that is used for indentation, should be tabs or spaces (but technically can be anything). 37 | indent_string = ' ' 38 | # Add trailing newline at the end of the file if not present. 39 | trailing_newline = true 40 | # Alphabetically reorder keys that are not separated by empty lines. 41 | reorder_keys = true 42 | # Maximum amount of allowed consecutive blank lines. This does not affect the whitespace at the end of the document, as it is always stripped. 43 | allowed_blank_lines = 1 44 | # Use CRLF for line endings. 45 | crlf = false 46 | -------------------------------------------------------------------------------- /tests/behavior/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::future::Future; 16 | use std::process::ExitCode; 17 | 18 | use morax_api::config::LogsConfig; 19 | use morax_api::config::StderrAppenderConfig; 20 | use morax_api::config::TelemetryConfig; 21 | use morax_api::property::TopicProperty; 22 | use tests_toolkit::make_test_name; 23 | 24 | pub struct Testkit { 25 | pub client: morax_client::HTTPClient, 26 | pub topic_props: TopicProperty, 27 | } 28 | 29 | pub fn harness(test: impl Send + FnOnce(Testkit) -> Fut) -> ExitCode 30 | where 31 | T: std::process::Termination, 32 | Fut: Send + Future, 33 | { 34 | morax_telemetry::init(&TelemetryConfig { 35 | logs: LogsConfig { 36 | stderr: Some(StderrAppenderConfig { 37 | filter: "DEBUG".to_string(), 38 | }), 39 | }, 40 | }); 41 | 42 | let test_name = make_test_name::(); 43 | let Some(state) = tests_toolkit::start_test_server(&test_name) else { 44 | return ExitCode::SUCCESS; 45 | }; 46 | 47 | morax_runtime::test_runtime().block_on(async move { 48 | let server_addr = format!("http://{}", state.server_state.broker_advertise_addr()); 49 | let builder = reqwest::ClientBuilder::new(); 50 | let client = morax_client::HTTPClient::new(server_addr, builder).unwrap(); 51 | 52 | let exit_code = test(Testkit { 53 | client, 54 | topic_props: TopicProperty { 55 | storage: state.env_props.storage, 56 | }, 57 | }) 58 | .await 59 | .report(); 60 | 61 | state.server_state.shutdown(); 62 | state.server_state.await_shutdown().await; 63 | exit_code 64 | }) 65 | } 66 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: CI 16 | on: 17 | pull_request: 18 | branches: [ main ] 19 | push: 20 | branches: [ main ] 21 | 22 | # Concurrency strategy: 23 | # github.workflow: distinguish this workflow from others 24 | # github.event_name: distinguish `push` event from `pull_request` event 25 | # github.event.number: set to the number of the pull request if `pull_request` event 26 | # github.run_id: otherwise, it's a `push` event, only cancel if we rerun the workflow 27 | # 28 | # Reference: 29 | # https://docs.github.com/en/actions/using-jobs/using-concurrency 30 | # https://docs.github.com/en/actions/learn-github-actions/contexts#github-context 31 | concurrency: 32 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }} 33 | cancel-in-progress: true 34 | 35 | jobs: 36 | check: 37 | name: Check lint 38 | runs-on: ubuntu-24.04 39 | steps: 40 | - uses: actions/checkout@v4 41 | - uses: Swatinem/rust-cache@v2 42 | - uses: taiki-e/install-action@v2 43 | with: 44 | tool: typos-cli,taplo-cli,hawkeye 45 | - run: cargo x lint 46 | 47 | test: 48 | name: Run tests with testcontainers 49 | runs-on: ubuntu-24.04 50 | steps: 51 | - uses: actions/checkout@v4 52 | - uses: Swatinem/rust-cache@v2 53 | - uses: taiki-e/install-action@nextest 54 | - run: cargo x test 55 | 56 | required: 57 | name: Required 58 | runs-on: ubuntu-24.04 59 | if: ${{ always() }} 60 | needs: 61 | - check 62 | - test 63 | steps: 64 | - name: Guardian 65 | run: | 66 | if [[ ! ( \ 67 | "${{ needs.check.result }}" == "success" \ 68 | && "${{ needs.test.result }}" == "success" \ 69 | ) ]]; then 70 | echo "Required jobs haven't been completed successfully." 71 | exit -1 72 | fi 73 | -------------------------------------------------------------------------------- /crates/meta/src/service/publish.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use error_stack::Result; 16 | use error_stack::ResultExt; 17 | 18 | use crate::CommitTopicSplitRequest; 19 | use crate::MetaError; 20 | use crate::PostgresMetaService; 21 | 22 | impl PostgresMetaService { 23 | pub async fn commit_topic_splits( 24 | &self, 25 | request: CommitTopicSplitRequest, 26 | ) -> Result<(i64, i64), MetaError> { 27 | let make_error = || MetaError("failed to commit topic splits".to_string()); 28 | let pool = self.pool.clone(); 29 | 30 | let mut txn = pool.begin().await.change_context_lazy(make_error)?; 31 | 32 | let topic_id = request.topic_id; 33 | let start_offset: i64 = sqlx::query_scalar( 34 | "SELECT last_offset FROM topic_offsets WHERE topic_id = $1 FOR UPDATE", 35 | ) 36 | .bind(topic_id) 37 | .fetch_one(&mut *txn) 38 | .await 39 | .change_context_lazy(make_error)?; 40 | 41 | let last_offset = start_offset + request.count; 42 | let end_offset = sqlx::query_scalar( 43 | "UPDATE topic_offsets SET last_offset = $1 WHERE topic_id = $2 RETURNING last_offset", 44 | ) 45 | .bind(last_offset) 46 | .bind(topic_id) 47 | .fetch_one(&mut *txn) 48 | .await 49 | .change_context_lazy(make_error)?; 50 | assert_eq!(last_offset, end_offset, "last offset mismatch"); 51 | 52 | sqlx::query("INSERT INTO topic_splits (topic_id, start_offset, end_offset, split_id) VALUES ($1, $2, $3, $4)") 53 | .bind(topic_id) 54 | .bind(start_offset) 55 | .bind(end_offset) 56 | .bind(request.split_id) 57 | .execute(&mut *txn) 58 | .await 59 | .change_context_lazy(make_error)?; 60 | 61 | txn.commit().await.change_context_lazy(make_error)?; 62 | Ok((start_offset, end_offset)) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /sdk/api/src/property/storage.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::fmt; 16 | 17 | use serde::Deserialize; 18 | use serde::Serialize; 19 | 20 | #[derive(Debug, Clone, Serialize, Deserialize)] 21 | #[serde(deny_unknown_fields)] 22 | pub struct TopicProperty { 23 | pub storage: StorageProperty, 24 | } 25 | 26 | #[derive(Debug, Clone, Serialize, Deserialize)] 27 | #[serde(tag = "scheme")] 28 | #[serde(deny_unknown_fields)] 29 | pub enum StorageProperty { 30 | #[serde(rename = "s3")] 31 | S3(S3StorageProperty), 32 | } 33 | 34 | #[derive(Clone, Serialize, Deserialize)] 35 | #[serde(deny_unknown_fields)] 36 | pub struct S3StorageProperty { 37 | /// Bucket name. 38 | pub bucket: String, 39 | /// Region name. 40 | pub region: String, 41 | /// URL prefix of table files, e.g. `/path/to/my/dir/`. 42 | /// 43 | /// Default to `/` 44 | #[serde(default = "default_prefix")] 45 | pub prefix: String, 46 | /// URL of S3 endpoint, e.g. `https://s3..amazonaws.com`. 47 | pub endpoint: String, 48 | /// Access key ID. 49 | pub access_key_id: String, 50 | /// Secret access key. 51 | pub secret_access_key: String, 52 | /// Whether to enable virtual host style, so that API requests will be sent 53 | /// in virtual host style instead of path style. 54 | /// 55 | /// Default to `true` 56 | #[serde(default = "default_virtual_host_style")] 57 | pub virtual_host_style: bool, 58 | } 59 | 60 | impl fmt::Debug for S3StorageProperty { 61 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 62 | f.debug_struct("S3Config") 63 | .field("prefix", &self.prefix) 64 | .field("bucket", &self.bucket) 65 | .field("endpoint", &self.endpoint) 66 | .field("region", &self.region) 67 | .finish_non_exhaustive() 68 | } 69 | } 70 | 71 | pub fn default_prefix() -> String { 72 | "/".to_string() 73 | } 74 | 75 | pub const fn default_virtual_host_style() -> bool { 76 | true 77 | } 78 | -------------------------------------------------------------------------------- /crates/meta/src/service/topic.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use error_stack::Result; 16 | use error_stack::ResultExt; 17 | use sqlx::types::Json; 18 | 19 | use crate::CreateTopicRequest; 20 | use crate::MetaError; 21 | use crate::PostgresMetaService; 22 | use crate::Topic; 23 | 24 | impl PostgresMetaService { 25 | pub async fn create_topic(&self, request: CreateTopicRequest) -> Result { 26 | let make_error = || MetaError("failed to create topic".to_string()); 27 | let pool = self.pool.clone(); 28 | 29 | let mut txn = pool.begin().await.change_context_lazy(make_error)?; 30 | 31 | let topic_name = request.name; 32 | let properties = request.properties; 33 | 34 | let topic: Topic = sqlx::query_as( 35 | r#" 36 | INSERT INTO topics (topic_id, topic_name, properties) 37 | VALUES (nextval('object_ids'), $1, $2) 38 | RETURNING topic_id, topic_name, properties 39 | "#, 40 | ) 41 | .bind(topic_name) 42 | .bind(Json(properties)) 43 | .fetch_one(&mut *txn) 44 | .await 45 | .change_context_lazy(make_error)?; 46 | 47 | sqlx::query("INSERT INTO topic_offsets (topic_id, last_offset) VALUES ($1, 0)") 48 | .bind(topic.topic_id) 49 | .execute(&mut *txn) 50 | .await 51 | .change_context_lazy(make_error)?; 52 | 53 | txn.commit().await.change_context_lazy(make_error)?; 54 | Ok(topic) 55 | } 56 | 57 | pub async fn get_topic_by_name(&self, topic_name: String) -> Result { 58 | let make_error = || MetaError("failed to get topic by name".to_string()); 59 | let pool = self.pool.clone(); 60 | 61 | sqlx::query_as("SELECT topic_id, topic_name, properties FROM topics WHERE topic_name = $1") 62 | .bind(&topic_name) 63 | .fetch_one(&pool) 64 | .await 65 | .change_context_lazy(make_error) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /sdk/api/src/config/morax.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use serde::Deserialize; 16 | use serde::Serialize; 17 | 18 | use crate::config::BrokerConfig; 19 | use crate::config::LogsConfig; 20 | use crate::config::MetaServiceConfig; 21 | use crate::config::RuntimeOptions; 22 | use crate::config::ServerConfig; 23 | use crate::config::StderrAppenderConfig; 24 | use crate::config::TelemetryConfig; 25 | use crate::property::S3StorageProperty; 26 | use crate::property::StorageProperty; 27 | 28 | #[derive(Debug, Clone, Serialize, Deserialize)] 29 | #[serde(deny_unknown_fields)] 30 | pub struct Config { 31 | pub server: ServerConfig, 32 | pub telemetry: TelemetryConfig, 33 | pub runtime: RuntimeOptions, 34 | } 35 | 36 | impl Default for Config { 37 | fn default() -> Self { 38 | Config { 39 | server: ServerConfig { 40 | broker: BrokerConfig { 41 | listen_addr: "0.0.0.0:8848".to_string(), 42 | advertise_addr: None, 43 | }, 44 | meta: MetaServiceConfig { 45 | service_url: "postgres://morax:my_secret_password@127.0.0.1:5432/morax_meta" 46 | .to_string(), 47 | }, 48 | default_storage: StorageProperty::S3(S3StorageProperty { 49 | bucket: "test-bucket".to_string(), 50 | region: "us-east-1".to_string(), 51 | prefix: "/".to_string(), 52 | endpoint: "http://127.0.0.1:9000".to_string(), 53 | access_key_id: "minioadmin".to_string(), 54 | secret_access_key: "minioadmin".to_string(), 55 | virtual_host_style: false, 56 | }), 57 | }, 58 | telemetry: TelemetryConfig { 59 | logs: LogsConfig { 60 | stderr: Some(StderrAppenderConfig { 61 | filter: "DEBUG".to_string(), 62 | }), 63 | }, 64 | }, 65 | runtime: RuntimeOptions::default(), 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /crates/storage/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use error_stack::Result; 16 | use morax_api::property::StorageProperty; 17 | use opendal::Operator; 18 | use uuid::Uuid; 19 | 20 | #[derive(Debug, thiserror::Error)] 21 | pub enum StorageError { 22 | #[error("{0}")] 23 | OpenDAL(opendal::Error), 24 | } 25 | 26 | #[derive(Debug)] 27 | pub struct TopicStorage { 28 | storage: StorageProperty, 29 | } 30 | 31 | impl TopicStorage { 32 | pub fn new(storage: StorageProperty) -> Self { 33 | Self { storage } 34 | } 35 | 36 | pub async fn read_split(&self, topic_id: i64, split_id: Uuid) -> Result, StorageError> { 37 | let op = make_op(self.storage.clone())?; 38 | let split_url = make_split_url(topic_id, split_id); 39 | let records = op.read(&split_url).await.map_err(StorageError::OpenDAL)?; 40 | Ok(records.to_vec()) 41 | } 42 | 43 | pub async fn write_split(&self, topic_id: i64, split: Vec) -> Result { 44 | let op = make_op(self.storage.clone())?; 45 | let split_id = Uuid::new_v4(); 46 | let split_url = make_split_url(topic_id, split_id); 47 | op.write(&split_url, split) 48 | .await 49 | .map_err(StorageError::OpenDAL)?; 50 | Ok(split_id) 51 | } 52 | } 53 | 54 | pub fn make_op(storage: StorageProperty) -> Result { 55 | match storage { 56 | StorageProperty::S3(config) => { 57 | let mut builder = opendal::services::S3::default() 58 | .bucket(&config.bucket) 59 | .region(&config.region) 60 | .root(&config.prefix) 61 | .endpoint(&config.endpoint) 62 | .access_key_id(&config.access_key_id) 63 | .secret_access_key(&config.secret_access_key); 64 | if config.virtual_host_style { 65 | builder = builder.enable_virtual_host_style(); 66 | } 67 | let builder = Operator::new(builder).map_err(StorageError::OpenDAL)?; 68 | Ok(builder.finish()) 69 | } 70 | } 71 | } 72 | 73 | fn make_split_url(topic_id: i64, split_id: Uuid) -> String { 74 | format!("topic_{topic_id}/{split_id}.split") 75 | } 76 | -------------------------------------------------------------------------------- /crates/version/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::collections::BTreeSet; 16 | use std::env; 17 | use std::path::Path; 18 | 19 | use build_data::format_timestamp; 20 | use build_data::get_source_time; 21 | use shadow_rs::ShadowBuilder; 22 | use shadow_rs::CARGO_METADATA; 23 | use shadow_rs::CARGO_TREE; 24 | 25 | fn configure_rerun_if_head_commit_changed() { 26 | let mut current = Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf(); 27 | 28 | // skip if no valid-looking git repository could be found 29 | while let Ok((dir, _)) = gix_discover::upwards(current.as_path()) { 30 | match dir { 31 | gix_discover::repository::Path::Repository(git_dir) => { 32 | unreachable!( 33 | "build.rs should never be placed in a git bare repository: {}", 34 | git_dir.display() 35 | ); 36 | } 37 | gix_discover::repository::Path::WorkTree(work_dir) => { 38 | let git_refs_heads = work_dir.join(".git/refs/heads"); 39 | println!("cargo::rerun-if-changed={}", git_refs_heads.display()); 40 | break; 41 | } 42 | gix_discover::repository::Path::LinkedWorkTree { work_dir, .. } => { 43 | current = work_dir 44 | .parent() 45 | .expect("submodule's work_dir must have parent") 46 | .to_path_buf(); 47 | continue; 48 | } 49 | }; 50 | } 51 | } 52 | 53 | fn main() -> shadow_rs::SdResult<()> { 54 | configure_rerun_if_head_commit_changed(); 55 | 56 | println!( 57 | "cargo::rustc-env=SOURCE_TIMESTAMP={}", 58 | if let Ok(t) = get_source_time() { 59 | format_timestamp(t) 60 | } else { 61 | "".to_string() 62 | } 63 | ); 64 | build_data::set_BUILD_TIMESTAMP(); 65 | 66 | // The "CARGO_WORKSPACE_DIR" is set manually (not by Rust itself) in Cargo config file, to 67 | // solve the problem where the "CARGO_MANIFEST_DIR" is not what we want when this repo is 68 | // made as a submodule in another repo. 69 | let src_path = env::var("CARGO_WORKSPACE_DIR").or_else(|_| env::var("CARGO_MANIFEST_DIR"))?; 70 | let out_path = env::var("OUT_DIR")?; 71 | let _ = ShadowBuilder::builder() 72 | .src_path(src_path) 73 | .out_path(out_path) 74 | // exclude these two large constants that we don't need 75 | .deny_const(BTreeSet::from([CARGO_METADATA, CARGO_TREE])) 76 | .build()?; 77 | Ok(()) 78 | } 79 | -------------------------------------------------------------------------------- /crates/meta/src/bootstrap.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use sqlx::Executor; 16 | use sqlx::PgPool; 17 | 18 | pub async fn bootstrap(pool: PgPool) -> error_stack::Result<(), sqlx::Error> { 19 | let mut txn = pool.begin().await?; 20 | 21 | // create the meta version table 22 | txn.execute("CREATE TABLE IF NOT EXISTS meta_version(version INT NOT NULL PRIMARY KEY);") 23 | .await?; 24 | txn.execute("INSERT INTO meta_version (version) VALUES (1) ON CONFLICT DO NOTHING;") 25 | .await?; 26 | 27 | // create a sequence for object ids 28 | txn.execute("CREATE SEQUENCE object_ids CYCLE").await?; 29 | 30 | // topics 31 | txn.execute( 32 | r#" 33 | CREATE TABLE IF NOT EXISTS topics ( 34 | topic_id BIGINT NOT NULL, 35 | topic_name TEXT NOT NULL, 36 | properties JSONB NOT NULL, 37 | UNIQUE (topic_id), 38 | UNIQUE (topic_name) 39 | ); 40 | "#, 41 | ) 42 | .await?; 43 | 44 | txn.execute( 45 | r#" 46 | CREATE TABLE IF NOT EXISTS topic_offsets ( 47 | topic_id BIGINT NOT NULL, 48 | last_offset BIGINT NOT NULL, 49 | UNIQUE (topic_id) 50 | ); 51 | "#, 52 | ) 53 | .await?; 54 | 55 | // topic splits: 56 | // * start_offset is inclusive 57 | // * end_offset is exclusive 58 | txn.execute( 59 | r#" 60 | CREATE TABLE IF NOT EXISTS topic_splits ( 61 | topic_id BIGINT NOT NULL, 62 | start_offset BIGINT NOT NULL, 63 | end_offset BIGINT NOT NULL, 64 | split_id UUID NOT NULL 65 | ); 66 | CREATE INDEX IF NOT EXISTS topic_splits_topic_id_idx ON topic_splits (topic_id); 67 | CREATE INDEX IF NOT EXISTS topic_splits_start_offset_idx ON topic_splits (start_offset); 68 | CREATE INDEX IF NOT EXISTS topic_splits_end_offset_idx ON topic_splits (end_offset); 69 | "#, 70 | ) 71 | .await?; 72 | 73 | // subscriptions 74 | txn.execute( 75 | r#" 76 | CREATE TABLE IF NOT EXISTS subscriptions ( 77 | subscription_id BIGINT NOT NULL, 78 | subscription_name TEXT NOT NULL, 79 | topic_id BIGINT NOT NULL, 80 | UNIQUE (subscription_id), 81 | UNIQUE (subscription_name) 82 | ); 83 | "#, 84 | ) 85 | .await?; 86 | 87 | // acknowledgements 88 | txn.execute( 89 | r#" 90 | CREATE TABLE IF NOT EXISTS acknowledgements ( 91 | subscription_id BIGINT NOT NULL, 92 | topic_id BIGINT NOT NULL, 93 | acks INT8RANGE[] NOT NULL DEFAULT '{}', 94 | UNIQUE (subscription_id) 95 | ); 96 | "#, 97 | ) 98 | .await?; 99 | 100 | txn.commit().await?; 101 | Ok(()) 102 | } 103 | -------------------------------------------------------------------------------- /crates/server/src/broker.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::net::SocketAddr; 16 | use std::sync::Arc; 17 | use std::time::Duration; 18 | 19 | use error_stack::Result; 20 | use error_stack::ResultExt; 21 | use mea::latch::Latch; 22 | use mea::waitgroup::WaitGroup; 23 | use morax_api::config::BrokerConfig; 24 | use morax_api::property::StorageProperty; 25 | use morax_meta::PostgresMetaService; 26 | use poem::listener::Acceptor; 27 | use poem::listener::Listener; 28 | 29 | use crate::server::resolve_advertise_addr; 30 | use crate::server::ServerFuture; 31 | use crate::ServerError; 32 | 33 | #[derive(Debug)] 34 | pub(crate) struct BrokerBootstrapContext { 35 | pub(crate) config: BrokerConfig, 36 | pub(crate) default_storage: StorageProperty, 37 | pub(crate) meta_service: Arc, 38 | pub(crate) wg: WaitGroup, 39 | pub(crate) shutdown: Arc, 40 | } 41 | 42 | pub(crate) async fn bootstrap_broker( 43 | context: BrokerBootstrapContext, 44 | ) -> Result<(SocketAddr, ServerFuture<()>), ServerError> { 45 | let BrokerBootstrapContext { 46 | config, 47 | default_storage, 48 | meta_service, 49 | wg, 50 | shutdown, 51 | } = context; 52 | 53 | let broker_addr = config.listen_addr.as_str(); 54 | let broker_acceptor = poem::listener::TcpListener::bind(broker_addr) 55 | .into_acceptor() 56 | .await 57 | .change_context_lazy(|| { 58 | ServerError(format!("failed to listen to broker: {broker_addr}")) 59 | })?; 60 | let broker_listen_addr = broker_acceptor.local_addr()[0] 61 | .as_socket_addr() 62 | .cloned() 63 | .ok_or_else(|| ServerError("failed to get local address of broker".to_string()))?; 64 | let broker_advertise_addr = 65 | resolve_advertise_addr(broker_listen_addr, config.advertise_addr.as_deref())?; 66 | 67 | let broker_fut = { 68 | let shutdown_clone = shutdown; 69 | let wg_clone = wg; 70 | 71 | let route = morax_broker::make_broker_router(meta_service, default_storage); 72 | let signal = async move { 73 | log::info!("Broker has started on [{broker_listen_addr}]"); 74 | drop(wg_clone); 75 | 76 | shutdown_clone.wait().await; 77 | log::info!("Broker is closing"); 78 | }; 79 | 80 | morax_runtime::server_runtime().spawn(async move { 81 | poem::Server::new_with_acceptor(broker_acceptor) 82 | .run_with_graceful_shutdown(route, signal, Some(Duration::from_secs(30))) 83 | .await 84 | .change_context_lazy(|| ServerError("failed to run the broker".to_string())) 85 | }) 86 | }; 87 | 88 | Ok((broker_advertise_addr, broker_fut)) 89 | } 90 | -------------------------------------------------------------------------------- /crates/meta/src/service/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use error_stack::bail; 16 | use error_stack::Result; 17 | use error_stack::ResultExt; 18 | use morax_api::config::MetaServiceConfig; 19 | use sqlx::migrate::MigrateDatabase; 20 | use sqlx::postgres::PgPoolOptions; 21 | use sqlx::Connection; 22 | use sqlx::PgConnection; 23 | use sqlx::PgPool; 24 | use sqlx::Postgres; 25 | 26 | use crate::bootstrap::bootstrap; 27 | use crate::MetaError; 28 | 29 | mod publish; 30 | mod pull; 31 | mod subscription; 32 | mod topic; 33 | 34 | async fn connect(url: &str) -> Result { 35 | Ok(PgPoolOptions::new().connect(url).await?) 36 | } 37 | 38 | async fn resolve_meta_version(url: &str) -> Result { 39 | let mut conn = PgConnection::connect(url).await?; 40 | 41 | let exists = sqlx::query_scalar( 42 | r#" 43 | SELECT EXISTS ( 44 | SELECT 1 45 | FROM information_schema.tables 46 | WHERE table_name = 'meta_version' 47 | )"#, 48 | ) 49 | .fetch_one(&mut conn) 50 | .await?; 51 | 52 | if exists { 53 | let version = sqlx::query_scalar("SELECT MAX(version) FROM meta_version") 54 | .fetch_one(&mut conn) 55 | .await?; 56 | Ok(version) 57 | } else { 58 | Ok(0) 59 | } 60 | } 61 | 62 | #[derive(Debug)] 63 | pub struct PostgresMetaService { 64 | pool: PgPool, 65 | } 66 | 67 | impl PostgresMetaService { 68 | pub async fn new(config: &MetaServiceConfig) -> Result { 69 | let make_error = || MetaError("failed to connect and bootstrap the database".to_string()); 70 | 71 | let url = config.service_url.as_str(); 72 | log::info!("connecting to meta service at {url}"); 73 | 74 | if !Postgres::database_exists(url) 75 | .await 76 | .change_context_lazy(make_error)? 77 | { 78 | log::info!("creating meta database at {url}"); 79 | Postgres::create_database(url) 80 | .await 81 | .change_context_lazy(make_error)?; 82 | } 83 | 84 | let meta_version = resolve_meta_version(url) 85 | .await 86 | .change_context_lazy(make_error)?; 87 | log::info!("resolved meta version: {meta_version}"); 88 | 89 | match meta_version { 90 | 0 => { 91 | log::info!("bootstrapping meta database at {url}"); 92 | let pool = connect(url).await.change_context_lazy(make_error)?; 93 | bootstrap(pool.clone()) 94 | .await 95 | .change_context_lazy(make_error)?; 96 | Ok(Self { pool }) 97 | } 98 | 1 => { 99 | log::info!("using existing meta database at {url}"); 100 | let pool = connect(url).await.change_context_lazy(make_error)?; 101 | Ok(Self { pool }) 102 | } 103 | version => bail!(MetaError(format!("unsupported meta version: {version}"))), 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /tests/behavior/tests/simple_pubsub.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use base64::prelude::BASE64_STANDARD; 16 | use base64::Engine; 17 | use behavior_tests::harness; 18 | use behavior_tests::Testkit; 19 | use insta::assert_compact_json_snapshot; 20 | use morax_api::request::AcknowledgeRequest; 21 | use morax_api::request::CreateSubscriptionRequest; 22 | use morax_api::request::CreateTopicRequest; 23 | use morax_api::request::PublishMessageRequest; 24 | use morax_api::request::PubsubMessage; 25 | use morax_api::request::PullMessageRequest; 26 | use test_harness::test; 27 | 28 | fn make_entry(payload: &str) -> PubsubMessage { 29 | PubsubMessage { 30 | message_id: None, 31 | publish_time: None, 32 | attributes: Default::default(), 33 | data: BASE64_STANDARD.encode(payload), 34 | } 35 | } 36 | 37 | #[test(harness)] 38 | async fn test_simple_pubsub(testkit: Testkit) { 39 | let topic_name = "wal".to_string(); 40 | let subscription_name = "wal_sub".to_string(); 41 | 42 | let r = testkit 43 | .client 44 | .create_topic(topic_name.clone(), CreateTopicRequest { storage: None }) 45 | .await 46 | .unwrap(); 47 | let resp = r.into_success().unwrap(); 48 | assert_compact_json_snapshot!(resp, @r#"{"name": "wal"}"#); 49 | 50 | let r = testkit 51 | .client 52 | .publish( 53 | topic_name.clone(), 54 | PublishMessageRequest { 55 | messages: vec![make_entry("0"), make_entry("1")], 56 | }, 57 | ) 58 | .await 59 | .unwrap(); 60 | let resp = r.into_success().unwrap(); 61 | assert_compact_json_snapshot!(resp, @r#"{"message_ids": ["0", "1"]}"#); 62 | 63 | let r = testkit 64 | .client 65 | .create_subscription( 66 | subscription_name.clone(), 67 | CreateSubscriptionRequest { topic_name }, 68 | ) 69 | .await 70 | .unwrap(); 71 | let resp = r.into_success().unwrap(); 72 | assert_compact_json_snapshot!(resp, @r#"{"topic": "wal", "name": "wal_sub"}"#); 73 | 74 | let r = testkit 75 | .client 76 | .pull( 77 | subscription_name.clone(), 78 | PullMessageRequest { max_messages: 64 }, 79 | ) 80 | .await 81 | .unwrap(); 82 | let resp = r.into_success().unwrap(); 83 | assert_compact_json_snapshot!(resp, { 84 | ".messages[].message.publish_time" => "2025-10-01T00:00:00Z", 85 | }, @r#" 86 | { 87 | "messages": [ 88 | { 89 | "ack_id": "0", 90 | "message": { 91 | "message_id": "0", 92 | "publish_time": "2025-10-01T00:00:00Z", 93 | "data": "MA==" 94 | } 95 | }, 96 | { 97 | "ack_id": "1", 98 | "message": { 99 | "message_id": "1", 100 | "publish_time": "2025-10-01T00:00:00Z", 101 | "data": "MQ==" 102 | } 103 | } 104 | ] 105 | } 106 | "#); 107 | 108 | let ack_ids = resp 109 | .messages 110 | .into_iter() 111 | .map(|m| m.ack_id) 112 | .collect::>(); 113 | let r = testkit 114 | .client 115 | .acknowledge(subscription_name.clone(), AcknowledgeRequest { ack_ids }) 116 | .await 117 | .unwrap(); 118 | let resp = r.into_success().unwrap(); 119 | assert_compact_json_snapshot!(resp, @"{}"); 120 | } 121 | -------------------------------------------------------------------------------- /crates/server/src/server.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::net::SocketAddr; 16 | use std::sync::Arc; 17 | 18 | use error_stack::Result; 19 | use error_stack::ResultExt; 20 | use mea::latch::Latch; 21 | use mea::waitgroup::WaitGroup; 22 | use morax_api::config::ServerConfig; 23 | use morax_meta::PostgresMetaService; 24 | 25 | use crate::broker::bootstrap_broker; 26 | use crate::broker::BrokerBootstrapContext; 27 | 28 | #[derive(Debug, thiserror::Error)] 29 | #[error("{0}")] 30 | pub struct ServerError(pub(crate) String); 31 | 32 | pub(crate) type ServerFuture = morax_runtime::JoinHandle>; 33 | 34 | #[derive(Debug)] 35 | pub struct ServerState { 36 | broker_advertise_addr: SocketAddr, 37 | broker_fut: ServerFuture<()>, 38 | shutdown: Arc, 39 | } 40 | 41 | impl ServerState { 42 | pub fn broker_advertise_addr(&self) -> SocketAddr { 43 | self.broker_advertise_addr 44 | } 45 | 46 | pub fn shutdown_handle(&self) -> impl Fn() { 47 | let shutdown = self.shutdown.clone(); 48 | move || shutdown.count_down() 49 | } 50 | 51 | pub fn shutdown(&self) { 52 | self.shutdown_handle()(); 53 | } 54 | 55 | pub async fn await_shutdown(self) { 56 | self.shutdown.wait().await; 57 | 58 | match futures::future::try_join_all(vec![self.broker_fut]).await { 59 | Ok(_) => log::info!("Morax server stopped."), 60 | Err(err) => log::error!(err:?; "Morax server failed."), 61 | } 62 | } 63 | } 64 | 65 | pub async fn start(config: ServerConfig) -> Result { 66 | let make_error = || ServerError("failed to start server".to_string()); 67 | let shutdown = Arc::new(Latch::new(1)); 68 | let wg = WaitGroup::new(); 69 | 70 | // initialize meta service 71 | let meta_service = PostgresMetaService::new(&config.meta) 72 | .await 73 | .map(Arc::new) 74 | .change_context_lazy(make_error)?; 75 | 76 | // initialize broker 77 | let (broker_advertise_addr, broker_fut) = bootstrap_broker(BrokerBootstrapContext { 78 | config: config.broker, 79 | default_storage: config.default_storage, 80 | meta_service: meta_service.clone(), 81 | wg: wg.clone(), 82 | shutdown: shutdown.clone(), 83 | }) 84 | .await?; 85 | 86 | // wait all servers to start and return 87 | wg.await; 88 | Ok(ServerState { 89 | broker_advertise_addr, 90 | broker_fut, 91 | shutdown, 92 | }) 93 | } 94 | 95 | pub(crate) fn resolve_advertise_addr( 96 | listen_addr: SocketAddr, 97 | advertise_addr: Option<&str>, 98 | ) -> Result { 99 | let make_error = || ServerError("failed to resolve advertise address".to_string()); 100 | 101 | match advertise_addr { 102 | None => { 103 | if listen_addr.ip().is_unspecified() { 104 | let ip = local_ip_address::local_ip().change_context_lazy(make_error)?; 105 | let port = listen_addr.port(); 106 | Ok(SocketAddr::new(ip, port)) 107 | } else { 108 | Ok(listen_addr) 109 | } 110 | } 111 | Some(advertise_addr) => { 112 | let advertise_addr = advertise_addr 113 | .parse::() 114 | .change_context_lazy(make_error)?; 115 | assert!( 116 | advertise_addr.ip().is_global(), 117 | "ip = {}", 118 | advertise_addr.ip() 119 | ); 120 | Ok(advertise_addr) 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 tison 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [workspace] 16 | members = [ 17 | "cmd/morax", 18 | "crates/broker", 19 | "crates/meta", 20 | "crates/runtime", 21 | "crates/server", 22 | "crates/storage", 23 | "crates/telemetry", 24 | "crates/version", 25 | "sdk/api", 26 | "sdk/client", 27 | "tests/behavior", 28 | "tests/toolkit", 29 | "xtask", 30 | ] 31 | 32 | resolver = "2" 33 | 34 | [workspace.package] 35 | edition = "2021" 36 | license = "Apache-2.0" 37 | readme = "README.md" 38 | repository = "https://github.com/tisonkun/morax/" 39 | version = "0.0.1" 40 | 41 | [workspace.dependencies] 42 | base64 = { version = "0.22" } 43 | better-panic = { version = "0.3" } 44 | build-data = { version = "0.2" } 45 | clap = { version = "4.5", features = ["derive"] } 46 | const_format = { version = "0.2" } 47 | ctrlc = { version = "3.4" } 48 | error-stack = { version = "0.5" } 49 | fastimer = { version = "0.9.0", features = ["logging"] } 50 | futures = { version = "0.3.31" } 51 | gix-discover = { version = "0.39.0" } 52 | insta = { version = "1.40", features = ["json", "redactions"] } 53 | jiff = { version = "0.2.9", features = ["serde"] } 54 | local-ip-address = { version = "0.6" } 55 | log = { version = "0.4", features = ["kv_unstable_serde", "serde"] } 56 | logforth = { version = "0.24.0" } 57 | mea = { version = "0.3.4" } 58 | opendal = { version = "0.53.1" } 59 | pin-project = { version = "1.1" } 60 | poem = { version = "3.1", features = ["compression", "rustls"] } 61 | regex = { version = "1.11" } 62 | reqwest = { version = "0.12", features = ["json", "rustls-tls"] } 63 | scopeguard = { version = "1.2" } 64 | serde = { version = "1.0", features = ["derive"] } 65 | serde_json = { version = "1.0" } 66 | shadow-rs = { version = "1.1.1", default-features = false } 67 | sqlx = { version = "0.8", features = [ 68 | "json", 69 | "postgres", 70 | "runtime-tokio-rustls", 71 | "uuid", 72 | ] } 73 | test-harness = { version = "0.3" } 74 | testcontainers = { version = "0.23", features = ["blocking"] } 75 | thiserror = { version = "2.0" } 76 | tokio = { version = "1.41", features = ["full"] } 77 | toml = { version = "0.8" } 78 | url = { version = "2.5" } 79 | uuid = { version = "1.11", features = ["v4"] } 80 | which = { version = "7.0" } 81 | 82 | # workspace dependencies 83 | morax-api = { version = "0.0.1", path = "sdk/api" } 84 | morax-broker = { version = "0.0.1", path = "crates/broker" } 85 | morax-client = { version = "0.0.1", path = "sdk/client" } 86 | morax-meta = { version = "0.0.1", path = "crates/meta" } 87 | morax-runtime = { version = "0.0.1", path = "crates/runtime" } 88 | morax-server = { version = "0.0.1", path = "crates/server" } 89 | morax-storage = { version = "0.0.1", path = "crates/storage" } 90 | morax-telemetry = { version = "0.0.1", path = "crates/telemetry" } 91 | morax-version = { version = "0.0.1", path = "crates/version" } 92 | tests-toolkit = { version = "0.0.1", path = "tests/toolkit" } 93 | 94 | [workspace.lints.rust] 95 | unknown_lints = "deny" 96 | unsafe_code = "deny" 97 | unused_must_use = "deny" 98 | 99 | [workspace.lints.clippy] 100 | dbg_macro = "deny" 101 | field_reassign_with_default = "allow" 102 | manual_range_contains = "allow" 103 | new_without_default = "allow" 104 | 105 | [workspace.metadata.release] 106 | pre-release-commit-message = "chore: release v{{version}}" 107 | shared-version = true 108 | sign-tag = true 109 | tag-name = "v{{version}}" 110 | 111 | [profile.release] 112 | debug = true 113 | lto = "thin" 114 | 115 | [profile.dist] 116 | codegen-units = 1 117 | debug = true 118 | inherits = "release" 119 | lto = "fat" 120 | 121 | [profile.dev.package] 122 | insta.opt-level = 3 123 | similar.opt-level = 3 124 | -------------------------------------------------------------------------------- /sdk/api/src/request.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::collections::BTreeMap; 16 | 17 | use serde::Deserialize; 18 | use serde::Serialize; 19 | 20 | use crate::property::StorageProperty; 21 | 22 | #[derive(Debug, Clone, Serialize, Deserialize)] 23 | pub struct PubsubMessage { 24 | /// The server-assigned ID of each published message. Guaranteed to be unique within the topic. 25 | /// It must not be populated by the publisher in a publish call. 26 | #[serde(skip_serializing_if = "Option::is_none")] 27 | pub message_id: Option, 28 | /// The time at which the message was published, populated by the server. It must not be 29 | /// populated by the publisher in a publish call. 30 | #[serde(skip_serializing_if = "Option::is_none")] 31 | pub publish_time: Option, 32 | /// Optional. Attributes for this message. If this field is empty, the message must contain 33 | /// non-empty data. This can be used to filter messages on the subscription. 34 | #[serde(skip_serializing_if = "BTreeMap::is_empty")] 35 | #[serde(default)] 36 | pub attributes: BTreeMap, 37 | /// A padded, base64-encoded string of bytes, encoded with a URL and filename safe alphabet 38 | /// (sometimes referred to as "web-safe" or "base64url"). Defined by [RFC4648]. 39 | /// 40 | /// [RFC4648]: https://datatracker.ietf.org/doc/html/rfc4648 41 | pub data: String, 42 | } 43 | 44 | #[derive(Debug, Clone, Serialize, Deserialize)] 45 | pub struct ReceivedMessage { 46 | /// This ID can be used to acknowledge the received message. 47 | pub ack_id: String, 48 | /// The message. 49 | pub message: PubsubMessage, 50 | } 51 | 52 | #[derive(Debug, Clone, Serialize, Deserialize)] 53 | pub struct CreateTopicRequest { 54 | /// Optional. The [`StorageProperty`] for the topic. If not specified, the default storage 55 | /// property will be used. 56 | #[serde(skip_serializing_if = "Option::is_none")] 57 | pub storage: Option, 58 | } 59 | 60 | #[derive(Debug, Clone, Serialize, Deserialize)] 61 | pub struct CreateTopicResponse { 62 | /// Name of the topic. 63 | pub name: String, 64 | } 65 | 66 | #[derive(Debug, Clone, Serialize, Deserialize)] 67 | pub struct PublishMessageRequest { 68 | /// Required. The messages to publish. 69 | pub messages: Vec, 70 | } 71 | 72 | #[derive(Debug, Clone, Serialize, Deserialize)] 73 | pub struct PublishMessageResponse { 74 | /// The server-assigned ID of each published message, in the same order as the messages in the 75 | /// request. IDs are guaranteed to be unique within the topic. 76 | pub message_ids: Vec, 77 | } 78 | 79 | #[derive(Debug, Clone, Serialize, Deserialize)] 80 | pub struct CreateSubscriptionRequest { 81 | /// Required. The name of the topic from which this subscription is receiving messages. 82 | pub topic_name: String, 83 | } 84 | 85 | #[derive(Debug, Clone, Serialize, Deserialize)] 86 | pub struct CreateSubscriptionResponse { 87 | /// The name of the topic from which this subscription is receiving messages. 88 | pub topic: String, 89 | /// Name of the subscription. 90 | pub name: String, 91 | } 92 | 93 | #[derive(Debug, Clone, Serialize, Deserialize)] 94 | pub struct PullMessageRequest { 95 | /// Required. The maximum number of messages to return for this request. Must be a positive 96 | /// integer. 97 | pub max_messages: i64, 98 | } 99 | 100 | #[derive(Debug, Clone, Serialize, Deserialize)] 101 | pub struct PullMessageResponse { 102 | /// Received Pub/Sub messages. 103 | pub messages: Vec, 104 | } 105 | 106 | #[derive(Debug, Clone, Serialize, Deserialize)] 107 | pub struct AcknowledgeRequest { 108 | /// Required. The acknowledgment ID for the messages being acknowledged. Must not be empty. 109 | pub ack_ids: Vec, 110 | } 111 | 112 | #[derive(Debug, Clone, Serialize, Deserialize)] 113 | pub struct AcknowledgeResponse {} 114 | -------------------------------------------------------------------------------- /cmd/morax/src/command.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::path::PathBuf; 16 | 17 | use clap::Parser; 18 | use clap::Subcommand; 19 | use error_stack::ResultExt; 20 | use morax_api::config::Config; 21 | use morax_version::version; 22 | 23 | use crate::Error; 24 | 25 | #[derive(Debug, Parser)] 26 | #[command(name = "morax", version, long_version = version())] 27 | pub struct Command { 28 | #[command(subcommand)] 29 | pub cmd: SubCommand, 30 | } 31 | 32 | impl Command { 33 | pub fn run(self) -> error_stack::Result<(), Error> { 34 | match self.cmd { 35 | SubCommand::Start(cmd) => cmd.run(), 36 | SubCommand::Generate(cmd) => cmd.run(), 37 | } 38 | } 39 | } 40 | 41 | #[derive(Debug, Subcommand)] 42 | pub enum SubCommand { 43 | /// Start a Morax broker node. 44 | #[command()] 45 | Start(CommandStart), 46 | /// Generate command-line interface utilities. 47 | #[command(name = "gen")] 48 | Generate(CommandGenerate), 49 | } 50 | 51 | #[derive(Debug, Parser)] 52 | pub struct CommandStart { 53 | /// Configure the server with the given file; if not specified, the 54 | /// [default configuration][crate::config::Config::default] is used. 55 | #[arg(short, long)] 56 | config_file: Option, 57 | } 58 | 59 | impl CommandStart { 60 | pub fn run(self) -> error_stack::Result<(), Error> { 61 | let config = if let Some(file) = self.config_file { 62 | let content = std::fs::read_to_string(&file).change_context_lazy(|| { 63 | Error(format!("failed to read config file: {}", file.display())) 64 | })?; 65 | toml::from_str(&content) 66 | .change_context_lazy(|| Error("failed to parse config content".to_string()))? 67 | } else { 68 | Config::default() 69 | }; 70 | 71 | morax_runtime::init(&config.runtime); 72 | morax_telemetry::init(&config.telemetry); 73 | 74 | error_stack::Report::set_color_mode(error_stack::fmt::ColorMode::None); 75 | error_stack::Report::set_charset(error_stack::fmt::Charset::Ascii); 76 | 77 | let rt = morax_runtime::make_runtime("morax-main", "morax-main", 1); 78 | rt.block_on(async move { 79 | let state = morax_server::start(config.server) 80 | .await 81 | .change_context_lazy(|| { 82 | Error("A fatal error has occurred in Morax server process.".to_string()) 83 | })?; 84 | 85 | let shutdown_handle = state.shutdown_handle(); 86 | ctrlc::set_handler(move || { 87 | shutdown_handle(); 88 | }) 89 | .change_context_lazy(|| Error("failed to setup ctrl-c signal handle".to_string()))?; 90 | 91 | state.await_shutdown().await; 92 | Ok(()) 93 | }) 94 | } 95 | } 96 | 97 | #[derive(Debug, Parser)] 98 | pub struct CommandGenerate { 99 | #[arg(short, long, global = true)] 100 | output: Option, 101 | 102 | #[command(subcommand)] 103 | cmd: GenerateTarget, 104 | } 105 | 106 | #[derive(Debug, Subcommand)] 107 | pub enum GenerateTarget { 108 | /// Generate the default server config. 109 | Config, 110 | } 111 | 112 | impl CommandGenerate { 113 | pub fn run(self) -> error_stack::Result<(), Error> { 114 | match self.cmd { 115 | GenerateTarget::Config => { 116 | let config = Config::default(); 117 | let content = toml::to_string(&config).change_context_lazy(|| { 118 | Error("default config must be always valid".to_string()) 119 | })?; 120 | if let Some(output) = self.output { 121 | std::fs::write(&output, content).change_context_lazy(|| { 122 | Error(format!("failed to write config to {}", output.display())) 123 | })?; 124 | } else { 125 | println!("{content}"); 126 | } 127 | } 128 | } 129 | 130 | Ok(()) 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /crates/runtime/src/global.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::num::NonZeroUsize; 16 | use std::sync::OnceLock; 17 | 18 | use morax_api::config::RuntimeOptions; 19 | 20 | use crate::num_cpus; 21 | use crate::Builder; 22 | use crate::Runtime; 23 | 24 | pub fn make_runtime(runtime_name: &str, thread_name: &str, worker_threads: usize) -> Runtime { 25 | log::info!( 26 | "creating runtime with runtime_name: {runtime_name}, thread_name: {thread_name}, work_threads: {worker_threads}." 27 | ); 28 | Builder::default() 29 | .runtime_name(runtime_name) 30 | .thread_name(thread_name) 31 | .worker_threads(worker_threads) 32 | .build() 33 | .expect("failed to create runtime") 34 | } 35 | 36 | pub fn telemetry_runtime() -> &'static Runtime { 37 | static RT: OnceLock = OnceLock::new(); 38 | RT.get_or_init(|| make_runtime("telemetry_runtime", "telemetry_thread", 1)) 39 | } 40 | 41 | #[cfg(any(test, feature = "test"))] 42 | pub fn test_runtime() -> &'static Runtime { 43 | static RT: OnceLock = OnceLock::new(); 44 | RT.get_or_init(|| make_runtime("test_runtime", "test_thread", 4)) 45 | } 46 | 47 | #[derive(Debug)] 48 | struct GlobalRuntimes { 49 | server_runtime: Runtime, 50 | io_runtime: Runtime, 51 | } 52 | 53 | static GLOBAL_RUNTIMES: OnceLock = OnceLock::new(); 54 | 55 | pub fn init(opts: &RuntimeOptions) { 56 | GLOBAL_RUNTIMES.get_or_init(|| do_initialize_runtimes(opts)); 57 | } 58 | 59 | fn do_initialize_runtimes(opts: &RuntimeOptions) -> GlobalRuntimes { 60 | log::info!("initializing global runtimes: {opts:?}"); 61 | 62 | set_panic_hook(); 63 | 64 | let RuntimeOptions { 65 | server_runtime_threads, 66 | io_runtime_threads, 67 | } = opts; 68 | 69 | let server_runtime = make_runtime( 70 | "server_runtime", 71 | "server_thread", 72 | server_runtime_threads 73 | .unwrap_or_else(default_server_threads) 74 | .get(), 75 | ); 76 | let io_runtime = make_runtime( 77 | "io_runtime", 78 | "io_thread", 79 | io_runtime_threads.unwrap_or_else(default_io_threads).get(), 80 | ); 81 | 82 | GlobalRuntimes { 83 | server_runtime, 84 | io_runtime, 85 | } 86 | } 87 | 88 | fn default_server_threads() -> NonZeroUsize { 89 | num_cpus() 90 | } 91 | 92 | fn default_io_threads() -> NonZeroUsize { 93 | num_cpus() 94 | } 95 | 96 | fn set_panic_hook() { 97 | std::panic::set_hook(Box::new(move |info| { 98 | let backtrace = std::backtrace::Backtrace::force_capture(); 99 | log::error!("panic occurred: {info}\nbacktrace:\n{backtrace}"); 100 | better_panic::Settings::auto().create_panic_handler()(info); 101 | log::info!("shutting down runtimes"); 102 | std::process::exit(1); 103 | })); 104 | } 105 | 106 | fn fetch_runtimes_or_default() -> &'static GlobalRuntimes { 107 | GLOBAL_RUNTIMES.get_or_init(|| do_initialize_runtimes(&RuntimeOptions::default())) 108 | } 109 | 110 | pub fn server_runtime() -> &'static Runtime { 111 | &fetch_runtimes_or_default().server_runtime 112 | } 113 | 114 | pub fn io_runtime() -> &'static Runtime { 115 | &fetch_runtimes_or_default().io_runtime 116 | } 117 | 118 | #[cfg(test)] 119 | mod tests { 120 | use super::*; 121 | 122 | #[test] 123 | fn test_spawn_block_on() { 124 | let handle = server_runtime().spawn(async { 1 + 1 }); 125 | assert_eq!(2, server_runtime().block_on(handle)); 126 | 127 | let handle = io_runtime().spawn(async { 4 + 4 }); 128 | assert_eq!(8, io_runtime().block_on(handle)); 129 | } 130 | 131 | #[test] 132 | fn test_spawn_from_blocking() { 133 | let runtimes = [server_runtime(), io_runtime()]; 134 | 135 | for runtime in runtimes { 136 | let out = runtime.block_on(async move { 137 | let inner = runtime 138 | .spawn_blocking(|| runtime.spawn(async move { "hello" })) 139 | .await; 140 | inner.await 141 | }); 142 | assert_eq!(out, "hello") 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /tests/toolkit/src/state.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::net::SocketAddr; 16 | 17 | use morax_api::config::BrokerConfig; 18 | use morax_api::config::MetaServiceConfig; 19 | use morax_api::config::ServerConfig; 20 | use morax_api::property::StorageProperty; 21 | use morax_server::ServerState; 22 | use morax_storage::make_op; 23 | use serde::Deserialize; 24 | use serde::Serialize; 25 | use sqlx::migrate::MigrateDatabase; 26 | use url::Url; 27 | 28 | use crate::container::make_testcontainers_env_props; 29 | use crate::DropGuard; 30 | 31 | #[derive(Debug)] 32 | pub struct TestServerState { 33 | pub server_state: ServerState, 34 | pub env_props: TestEnvProps, 35 | _drop_guards: Vec, 36 | } 37 | 38 | #[derive(Debug)] 39 | pub struct TestEnvState { 40 | pub env_props: TestEnvProps, 41 | _drop_guards: Vec, 42 | } 43 | 44 | #[derive(Debug, Clone, Serialize, Deserialize)] 45 | pub struct TestEnvProps { 46 | pub meta: MetaServiceConfig, 47 | pub storage: StorageProperty, 48 | } 49 | 50 | pub(crate) fn read_test_env_props() -> Option { 51 | let file = std::env::var("TEST_ENV_PROPS_FILE").ok()?; 52 | let path = std::path::Path::new(&file); 53 | let path = if path.is_absolute() { 54 | path.canonicalize().unwrap() 55 | } else { 56 | env!("CARGO_WORKSPACE_DIR") 57 | .parse::() 58 | .unwrap() 59 | .join(path) 60 | .canonicalize() 61 | .unwrap() 62 | }; 63 | let content = std::fs::read_to_string(&path).unwrap(); 64 | Some(toml::from_str(&content).unwrap()) 65 | } 66 | 67 | pub fn start_test_server(test_name: &str) -> Option { 68 | let TestEnvState { 69 | env_props, 70 | _drop_guards, 71 | } = make_test_env_state(test_name)?; 72 | let host = local_ip_address::local_ip().unwrap(); 73 | let broker = BrokerConfig { 74 | listen_addr: SocketAddr::new(host, 0).to_string(), 75 | advertise_addr: None, 76 | }; 77 | let server_state = morax_runtime::test_runtime() 78 | .block_on(morax_server::start(ServerConfig { 79 | broker, 80 | meta: env_props.meta.clone(), 81 | default_storage: env_props.storage.clone(), 82 | })) 83 | .unwrap(); 84 | Some(TestServerState { 85 | server_state, 86 | env_props, 87 | _drop_guards, 88 | }) 89 | } 90 | 91 | pub fn make_test_env_state(test_name: &str) -> Option { 92 | let mut _drop_guards = Vec::::new(); 93 | 94 | let mut props = if option_enabled("SKIP_INTEGRATION") { 95 | return None; 96 | } else if let Some(props) = read_test_env_props() { 97 | props 98 | } else { 99 | let (props, drop_guards) = make_testcontainers_env_props(); 100 | _drop_guards.extend(drop_guards); 101 | props 102 | }; 103 | 104 | props.meta.service_url = { 105 | let mut url = Url::parse(&props.meta.service_url).unwrap(); 106 | url.set_path(test_name); 107 | url.to_string() 108 | }; 109 | _drop_guards.push(Box::new(scopeguard::guard_on_success( 110 | props.meta.service_url.clone(), 111 | |url| { 112 | morax_runtime::test_runtime().block_on(async move { 113 | sqlx::Postgres::drop_database(&url).await.unwrap(); 114 | }); 115 | }, 116 | ))); 117 | 118 | match props.storage { 119 | StorageProperty::S3(ref mut config) => { 120 | config.prefix = format!("/{test_name}/"); 121 | } 122 | } 123 | 124 | let client = make_op(props.storage.clone()).unwrap(); 125 | morax_runtime::test_runtime().block_on(async { 126 | client.remove_all("/").await.unwrap(); 127 | }); 128 | _drop_guards.push(Box::new(scopeguard::guard_on_success((), move |()| { 129 | morax_runtime::test_runtime().block_on(async move { 130 | client.remove_all("/").await.unwrap(); 131 | }); 132 | }))); 133 | 134 | // ensure containers get dropped last 135 | _drop_guards.reverse(); 136 | Some(TestEnvState { 137 | env_props: props, 138 | _drop_guards, 139 | }) 140 | } 141 | 142 | fn option_enabled(name: &str) -> bool { 143 | std::env::var(name) 144 | .ok() 145 | .filter(|s| matches!(s.to_lowercase().as_str(), "1" | "true" | "yes" | "on")) 146 | .is_some() 147 | } 148 | -------------------------------------------------------------------------------- /tests/toolkit/src/container.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::borrow::Cow; 16 | 17 | use morax_api::config::MetaServiceConfig; 18 | use morax_api::property::default_prefix; 19 | use morax_api::property::S3StorageProperty; 20 | use morax_api::property::StorageProperty; 21 | use testcontainers::core::ContainerPort; 22 | use testcontainers::core::WaitFor; 23 | use testcontainers::runners::SyncRunner; 24 | use testcontainers::Container; 25 | use testcontainers::Image; 26 | use testcontainers::TestcontainersError; 27 | 28 | use crate::DropGuard; 29 | use crate::TestEnvProps; 30 | 31 | const USERNAME: &str = "morax"; 32 | const PASSWORD: &str = "my_secret_password"; 33 | 34 | #[derive(Default, Debug, Clone)] 35 | struct Postgres; 36 | 37 | impl Image for Postgres { 38 | fn name(&self) -> &str { 39 | "postgres" 40 | } 41 | 42 | fn tag(&self) -> &str { 43 | "16.3-bullseye" 44 | } 45 | 46 | fn ready_conditions(&self) -> Vec { 47 | vec![WaitFor::message_on_stderr( 48 | "database system is ready to accept connections", 49 | )] 50 | } 51 | 52 | fn env_vars( 53 | &self, 54 | ) -> impl IntoIterator>, impl Into>)> { 55 | vec![("POSTGRES_USER", USERNAME), ("POSTGRES_PASSWORD", PASSWORD)] 56 | } 57 | 58 | fn expose_ports(&self) -> &[ContainerPort] { 59 | &[ContainerPort::Tcp(5432)] 60 | } 61 | } 62 | 63 | fn make_meta_service_config(container: &Container) -> MetaServiceConfig { 64 | let host = local_ip_address::local_ip().unwrap(); 65 | let port = container.get_host_port_ipv4(5432).unwrap(); 66 | MetaServiceConfig { 67 | service_url: format!("postgres://{USERNAME}:{PASSWORD}@{host}:{port}"), 68 | } 69 | } 70 | 71 | const ACCESS_KEY_ID: &str = "morax_data_access_key"; 72 | const SECRET_ACCESS_KEY: &str = "morax_data_secret_access_key"; 73 | const BUCKET: &str = "test-bucket"; 74 | const REGION: &str = "us-east-1"; 75 | 76 | #[derive(Default, Debug, Clone)] 77 | struct MinIO; 78 | 79 | impl Image for MinIO { 80 | fn name(&self) -> &str { 81 | "minio/minio" 82 | } 83 | 84 | fn tag(&self) -> &str { 85 | "RELEASE.2024-07-16T23-46-41Z" 86 | } 87 | 88 | fn ready_conditions(&self) -> Vec { 89 | vec![WaitFor::message_on_stderr("API:")] 90 | } 91 | 92 | fn env_vars( 93 | &self, 94 | ) -> impl IntoIterator>, impl Into>)> { 95 | vec![ 96 | ("MINIO_ROOT_USER", ACCESS_KEY_ID), 97 | ("MINIO_ROOT_PASSWORD", SECRET_ACCESS_KEY), 98 | ("NO_COLOR", "true"), 99 | ] 100 | } 101 | 102 | fn entrypoint(&self) -> Option<&str> { 103 | Some("bash") 104 | } 105 | 106 | fn cmd(&self) -> impl IntoIterator>> { 107 | vec![ 108 | "-c".to_owned(), 109 | format!("mkdir -p /data/{BUCKET} && minio server /data"), 110 | ] 111 | } 112 | 113 | fn expose_ports(&self) -> &[ContainerPort] { 114 | &[ContainerPort::Tcp(9000)] 115 | } 116 | } 117 | 118 | fn make_s3_props(container: &Container) -> StorageProperty { 119 | let host = local_ip_address::local_ip().unwrap(); 120 | let port = container.get_host_port_ipv4(9000).unwrap(); 121 | 122 | StorageProperty::S3(S3StorageProperty { 123 | bucket: BUCKET.to_string(), 124 | region: REGION.to_string(), 125 | prefix: default_prefix(), 126 | endpoint: format!("http://{host}:{port}"), 127 | access_key_id: ACCESS_KEY_ID.to_string(), 128 | secret_access_key: SECRET_ACCESS_KEY.to_string(), 129 | virtual_host_style: false, 130 | }) 131 | } 132 | 133 | fn maybe_docker_error(err: TestcontainersError) -> TestcontainersError { 134 | if matches!(err, TestcontainersError::Client(_)) { 135 | eprintln!( 136 | "Error: Docker is not installed or running. Please install and start Docker, or set \ 137 | `SKIP_INTEGRATION=1` to skip these tests.\n\ 138 | Example: SKIP_INTEGRATION=1 cargo x test" 139 | ); 140 | } 141 | err 142 | } 143 | 144 | pub(crate) fn make_testcontainers_env_props() -> (TestEnvProps, Vec) { 145 | let postgres = Postgres.start().map_err(maybe_docker_error).unwrap(); 146 | let minio = MinIO.start().map_err(maybe_docker_error).unwrap(); 147 | let props = TestEnvProps { 148 | meta: make_meta_service_config(&postgres), 149 | storage: make_s3_props(&minio), 150 | }; 151 | let drop_guards: Vec = vec![Box::new(postgres), Box::new(minio)]; 152 | 153 | (props, drop_guards) 154 | } 155 | -------------------------------------------------------------------------------- /crates/broker/src/http.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::Arc; 16 | 17 | use morax_api::property::StorageProperty; 18 | use morax_api::request::AcknowledgeRequest; 19 | use morax_api::request::AcknowledgeResponse; 20 | use morax_api::request::CreateSubscriptionRequest; 21 | use morax_api::request::CreateSubscriptionResponse; 22 | use morax_api::request::CreateTopicRequest; 23 | use morax_api::request::CreateTopicResponse; 24 | use morax_api::request::PublishMessageRequest; 25 | use morax_api::request::PublishMessageResponse; 26 | use morax_api::request::PullMessageRequest; 27 | use morax_api::request::PullMessageResponse; 28 | use morax_meta::PostgresMetaService; 29 | use poem::http::StatusCode; 30 | use poem::middleware::AddData; 31 | use poem::middleware::Compression; 32 | use poem::web::Data; 33 | use poem::web::Json; 34 | use poem::web::Path; 35 | use poem::EndpointExt; 36 | use poem::Route; 37 | 38 | use crate::broker::Broker; 39 | 40 | #[poem::handler] 41 | pub async fn health_check() -> poem::Result { 42 | Ok("OK".to_string()) 43 | } 44 | 45 | #[poem::handler] 46 | pub async fn create_topic( 47 | Data(broker): Data<&Broker>, 48 | Path(topic_name): Path, 49 | Json(request): Json, 50 | ) -> poem::Result> { 51 | let response = broker 52 | .create_topic(topic_name, request) 53 | .await 54 | .inspect_err(|err| log::error!(err:?; "failed to create topic")) 55 | .map_err(|err| poem::Error::new(err.into_error(), StatusCode::UNPROCESSABLE_ENTITY))?; 56 | Ok(Json(response)) 57 | } 58 | 59 | #[poem::handler] 60 | pub async fn publish( 61 | Data(broker): Data<&Broker>, 62 | Path(topic_name): Path, 63 | Json(request): Json, 64 | ) -> poem::Result> { 65 | let response = broker 66 | .publish(topic_name, request) 67 | .await 68 | .inspect_err(|err| log::error!(err:?; "failed to publish messages")) 69 | .map_err(|err| poem::Error::new(err.into_error(), StatusCode::UNPROCESSABLE_ENTITY))?; 70 | Ok(Json(response)) 71 | } 72 | 73 | #[poem::handler] 74 | pub async fn create_subscription( 75 | Data(broker): Data<&Broker>, 76 | Path(subscription_name): Path, 77 | Json(request): Json, 78 | ) -> poem::Result> { 79 | let response = broker 80 | .create_subscription(subscription_name, request) 81 | .await 82 | .inspect_err(|err| log::error!(err:?; "failed to create subscription")) 83 | .map_err(|err| poem::Error::new(err.into_error(), StatusCode::UNPROCESSABLE_ENTITY))?; 84 | Ok(Json(response)) 85 | } 86 | 87 | #[poem::handler] 88 | pub async fn pull( 89 | Data(broker): Data<&Broker>, 90 | Path(subscription_name): Path, 91 | Json(request): Json, 92 | ) -> poem::Result> { 93 | let response = broker 94 | .pull(subscription_name, request) 95 | .await 96 | .inspect_err(|err| log::error!(err:?; "failed to pull messages")) 97 | .map_err(|err| poem::Error::new(err.into_error(), StatusCode::UNPROCESSABLE_ENTITY))?; 98 | Ok(Json(response)) 99 | } 100 | 101 | #[poem::handler] 102 | pub async fn acknowledge( 103 | Data(broker): Data<&Broker>, 104 | Path(subscription_name): Path, 105 | Json(request): Json, 106 | ) -> poem::Result> { 107 | let response = broker 108 | .acknowledge(subscription_name, request) 109 | .await 110 | .inspect_err(|err| log::error!(err:?; "failed to acknowledge")) 111 | .map_err(|err| poem::Error::new(err.into_error(), StatusCode::UNPROCESSABLE_ENTITY))?; 112 | Ok(Json(response)) 113 | } 114 | 115 | pub fn make_broker_router( 116 | meta: Arc, 117 | default_storage: StorageProperty, 118 | ) -> Route { 119 | let broker = Broker::new(meta, default_storage); 120 | 121 | let v1_route = Route::new() 122 | .at("/health", poem::get(health_check)) 123 | .at("/topics/:topic_name", poem::post(create_topic)) 124 | .at("/topics/:topic_name/publish", poem::post(publish)) 125 | .at( 126 | "/subscriptions/:subscription_name", 127 | poem::post(create_subscription), 128 | ) 129 | .at("/subscriptions/:subscription_name/pull", poem::post(pull)) 130 | .at( 131 | "/subscriptions/:subscription_name/acknowledge", 132 | poem::post(acknowledge), 133 | ) 134 | .with(Compression::new()) 135 | .with(AddData::new(broker)); 136 | 137 | Route::new().nest("v1", v1_route) 138 | } 139 | -------------------------------------------------------------------------------- /xtask/src/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::process::Command as StdCommand; 16 | 17 | use clap::Parser; 18 | use clap::Subcommand; 19 | 20 | #[derive(Parser)] 21 | struct Command { 22 | #[clap(subcommand)] 23 | sub: SubCommand, 24 | } 25 | 26 | impl Command { 27 | fn run(self) { 28 | match self.sub { 29 | SubCommand::Build(cmd) => cmd.run(), 30 | SubCommand::Lint(cmd) => cmd.run(), 31 | SubCommand::Test(cmd) => cmd.run(), 32 | } 33 | } 34 | } 35 | 36 | #[derive(Subcommand)] 37 | enum SubCommand { 38 | #[clap(about = "Compile workspace packages.")] 39 | Build(CommandBuild), 40 | #[clap(about = "Run format and clippy checks.")] 41 | Lint(CommandLint), 42 | #[clap(about = "Run unit tests.")] 43 | Test(CommandTest), 44 | } 45 | 46 | #[derive(Parser)] 47 | struct CommandBuild { 48 | #[arg(long, help = "Assert that `Cargo.lock` will remain unchanged.")] 49 | locked: bool, 50 | } 51 | 52 | impl CommandBuild { 53 | fn run(self) { 54 | run_command(make_build_cmd(self.locked)); 55 | } 56 | } 57 | 58 | #[derive(Parser)] 59 | struct CommandTest { 60 | #[arg(long, help = "Run tests serially and do not capture output.")] 61 | no_capture: bool, 62 | } 63 | 64 | impl CommandTest { 65 | fn run(self) { 66 | run_command(make_test_cmd(self.no_capture)); 67 | } 68 | } 69 | 70 | #[derive(Parser)] 71 | #[clap(name = "lint")] 72 | struct CommandLint { 73 | #[arg(long, help = "Automatically apply lint suggestions.")] 74 | fix: bool, 75 | } 76 | 77 | impl CommandLint { 78 | fn run(self) { 79 | run_command(make_clippy_cmd(self.fix)); 80 | run_command(make_format_cmd(self.fix)); 81 | run_command(make_taplo_cmd(self.fix)); 82 | run_command(make_typos_cmd()); 83 | run_command(make_hawkeye_cmd(self.fix)); 84 | } 85 | } 86 | 87 | fn find_command(cmd: &str) -> StdCommand { 88 | match which::which(cmd) { 89 | Ok(exe) => { 90 | let mut cmd = StdCommand::new(exe); 91 | cmd.current_dir(env!("CARGO_WORKSPACE_DIR")); 92 | cmd 93 | } 94 | Err(err) => { 95 | panic!("{cmd} not found: {err}"); 96 | } 97 | } 98 | } 99 | 100 | fn ensure_installed(bin: &str, crate_name: &str) { 101 | if which::which(bin).is_err() { 102 | let mut cmd = find_command("cargo"); 103 | cmd.args(["install", crate_name]); 104 | run_command(cmd); 105 | } 106 | } 107 | 108 | fn run_command(mut cmd: StdCommand) { 109 | println!("{cmd:?}"); 110 | let status = cmd.status().expect("failed to execute process"); 111 | assert!(status.success(), "command failed: {status}"); 112 | } 113 | 114 | fn make_build_cmd(locked: bool) -> StdCommand { 115 | let mut cmd = find_command("cargo"); 116 | cmd.args([ 117 | "build", 118 | "--workspace", 119 | "--all-features", 120 | "--tests", 121 | "--examples", 122 | "--benches", 123 | "--bins", 124 | ]); 125 | if locked { 126 | cmd.arg("--locked"); 127 | } 128 | cmd 129 | } 130 | 131 | fn make_test_cmd(no_capture: bool) -> StdCommand { 132 | ensure_installed("cargo-nextest", "cargo-nextest"); 133 | let mut cmd = find_command("cargo"); 134 | cmd.args(["nextest", "run", "--workspace"]); 135 | if no_capture { 136 | cmd.arg("--no-capture"); 137 | } 138 | cmd 139 | } 140 | 141 | fn make_format_cmd(fix: bool) -> StdCommand { 142 | let mut cmd = find_command("cargo"); 143 | cmd.args(["fmt", "--all"]); 144 | if !fix { 145 | cmd.arg("--check"); 146 | } 147 | cmd 148 | } 149 | 150 | fn make_clippy_cmd(fix: bool) -> StdCommand { 151 | let mut cmd = find_command("cargo"); 152 | cmd.args([ 153 | "clippy", 154 | "--tests", 155 | "--all-features", 156 | "--all-targets", 157 | "--workspace", 158 | ]); 159 | if fix { 160 | cmd.args(["--allow-staged", "--allow-dirty", "--fix"]); 161 | } else { 162 | cmd.args(["--", "-D", "warnings"]); 163 | } 164 | cmd 165 | } 166 | 167 | fn make_hawkeye_cmd(fix: bool) -> StdCommand { 168 | ensure_installed("hawkeye", "hawkeye"); 169 | let mut cmd = find_command("hawkeye"); 170 | if fix { 171 | cmd.args(["format", "--fail-if-updated=false"]); 172 | } else { 173 | cmd.args(["check"]); 174 | } 175 | cmd 176 | } 177 | 178 | fn make_typos_cmd() -> StdCommand { 179 | ensure_installed("typos", "typos-cli"); 180 | find_command("typos") 181 | } 182 | 183 | fn make_taplo_cmd(fix: bool) -> StdCommand { 184 | ensure_installed("taplo", "taplo-cli"); 185 | let mut cmd = find_command("taplo"); 186 | if fix { 187 | cmd.args(["format"]); 188 | } else { 189 | cmd.args(["format", "--check"]); 190 | } 191 | cmd 192 | } 193 | 194 | fn main() { 195 | let cmd = Command::parse(); 196 | cmd.run() 197 | } 198 | -------------------------------------------------------------------------------- /crates/meta/src/service/subscription.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::ops::Bound; 16 | 17 | use error_stack::Result; 18 | use error_stack::ResultExt; 19 | use sqlx::postgres::types::PgRange; 20 | 21 | use crate::AcknowledgeRequest; 22 | use crate::CreateSubscriptionRequest; 23 | use crate::MetaError; 24 | use crate::PostgresMetaService; 25 | use crate::Subscription; 26 | use crate::Topic; 27 | 28 | impl PostgresMetaService { 29 | pub async fn create_subscription( 30 | &self, 31 | request: CreateSubscriptionRequest, 32 | ) -> Result { 33 | let make_error = || MetaError("failed to create subscription".to_string()); 34 | let pool = self.pool.clone(); 35 | 36 | let mut txn = pool.begin().await.change_context_lazy(make_error)?; 37 | 38 | let subscription_name = request.name; 39 | let topic_name = request.topic; 40 | 41 | let topic: Topic = sqlx::query_as( 42 | r#" 43 | SELECT topic_id, topic_name, properties 44 | FROM topics 45 | WHERE topic_name = $1 46 | "#, 47 | ) 48 | .bind(&topic_name) 49 | .fetch_one(&mut *txn) 50 | .await 51 | .change_context_lazy(make_error)?; 52 | 53 | let subscription_id: i64 = sqlx::query_scalar( 54 | r#" 55 | INSERT INTO subscriptions (subscription_id, subscription_name, topic_id) 56 | VALUES (nextval('object_ids'), $1, $2) 57 | RETURNING subscription_id 58 | "#, 59 | ) 60 | .bind(&subscription_name) 61 | .bind(topic.topic_id) 62 | .fetch_one(&mut *txn) 63 | .await 64 | .change_context_lazy(make_error)?; 65 | 66 | sqlx::query( 67 | r#" 68 | INSERT INTO acknowledgements (subscription_id, topic_id, acks) VALUES ($1, $2, '{}') 69 | "#, 70 | ) 71 | .bind(subscription_id) 72 | .bind(topic.topic_id) 73 | .execute(&mut *txn) 74 | .await 75 | .change_context_lazy(make_error)?; 76 | 77 | txn.commit().await.change_context_lazy(make_error)?; 78 | Ok(Subscription { 79 | subscription_id, 80 | subscription_name, 81 | topic_id: topic.topic_id, 82 | topic_name, 83 | }) 84 | } 85 | 86 | pub async fn get_subscription_by_name( 87 | &self, 88 | subscription_name: String, 89 | ) -> Result { 90 | let make_error = || MetaError("failed to get subscription by name".to_string()); 91 | let pool = self.pool.clone(); 92 | 93 | sqlx::query_as( 94 | r#" 95 | SELECT subscription_id, subscription_name, topics.topic_id, topics.topic_name 96 | FROM subscriptions JOIN topics ON subscriptions.topic_id = topics.topic_id 97 | WHERE subscription_name = $1 98 | "#, 99 | ) 100 | .bind(&subscription_name) 101 | .fetch_one(&pool) 102 | .await 103 | .change_context_lazy(make_error) 104 | } 105 | 106 | pub async fn list_acks(&self, subscription_id: i64) -> Result, MetaError> { 107 | let make_error = || MetaError("failed to list acks".to_string()); 108 | let pool = self.pool.clone(); 109 | 110 | let ranges = sqlx::query_scalar( 111 | r#" 112 | SELECT acks FROM acknowledgements 113 | WHERE subscription_id = $1 114 | "#, 115 | ) 116 | .bind(subscription_id) 117 | .fetch_one(&pool) 118 | .await 119 | .change_context_lazy(make_error)?; 120 | 121 | Ok(pg_ranges_to_vec_ranges(ranges)) 122 | } 123 | 124 | pub async fn acknowledge(&self, request: AcknowledgeRequest) -> Result<(), MetaError> { 125 | let make_error = || MetaError("failed to acknowledge ids".to_string()); 126 | let pool = self.pool.clone(); 127 | 128 | let mut txn = pool.begin().await.change_context_lazy(make_error)?; 129 | 130 | let acks: Vec> = sqlx::query_scalar( 131 | r#" 132 | SELECT acks FROM acknowledgements 133 | WHERE subscription_id = $1 FOR UPDATE 134 | "#, 135 | ) 136 | .bind(request.subscription_id) 137 | .fetch_one(&mut *txn) 138 | .await 139 | .change_context_lazy(make_error)?; 140 | 141 | let mut acks = pg_ranges_to_vec_ranges(acks); 142 | for ack_id in request.ack_ids { 143 | acks.push((ack_id, ack_id + 1)) 144 | } 145 | acks = merge_ranges(acks); 146 | 147 | sqlx::query( 148 | r#" 149 | UPDATE acknowledgements 150 | SET acks = $1 151 | WHERE subscription_id = $2 152 | "#, 153 | ) 154 | .bind(vec_ranges_to_pg_ranges(acks)) 155 | .bind(request.subscription_id) 156 | .execute(&mut *txn) 157 | .await 158 | .change_context_lazy(make_error)?; 159 | 160 | txn.commit().await.change_context_lazy(make_error)?; 161 | Ok(()) 162 | } 163 | } 164 | 165 | fn pg_ranges_to_vec_ranges(ranges: Vec>) -> Vec<(i64, i64)> { 166 | ranges 167 | .into_iter() 168 | .map(|r| match (r.start, r.end) { 169 | (Bound::Included(start), Bound::Excluded(end)) => (start, end), 170 | _ => unreachable!("acks are always left-closed right-open"), 171 | }) 172 | .collect() 173 | } 174 | 175 | fn vec_ranges_to_pg_ranges(ranges: Vec<(i64, i64)>) -> Vec> { 176 | ranges 177 | .into_iter() 178 | .map(|(start, end)| PgRange::from([Bound::Included(start), Bound::Excluded(end)])) 179 | .collect() 180 | } 181 | 182 | // Merge a list of left-closed right-open ranges into a list of left-closed right-open ranges. 183 | fn merge_ranges(mut ranges: Vec<(i64, i64)>) -> Vec<(i64, i64)> { 184 | if ranges.len() <= 1 { 185 | return ranges; 186 | } 187 | 188 | ranges.sort(); 189 | let mut merged_ranges = vec![]; 190 | let mut current_range = ranges[0]; 191 | for range in ranges.iter().skip(1) { 192 | if range.0 <= current_range.1 { 193 | current_range.1 = current_range.1.max(range.1); 194 | } else { 195 | merged_ranges.push(current_range); 196 | current_range = *range; 197 | } 198 | } 199 | merged_ranges.push(current_range); 200 | merged_ranges 201 | } 202 | -------------------------------------------------------------------------------- /sdk/client/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use error_stack::ResultExt; 16 | use morax_api::request::AcknowledgeRequest; 17 | use morax_api::request::AcknowledgeResponse; 18 | use morax_api::request::CreateSubscriptionRequest; 19 | use morax_api::request::CreateSubscriptionResponse; 20 | use morax_api::request::CreateTopicRequest; 21 | use morax_api::request::CreateTopicResponse; 22 | use morax_api::request::PublishMessageRequest; 23 | use morax_api::request::PublishMessageResponse; 24 | use morax_api::request::PullMessageRequest; 25 | use morax_api::request::PullMessageResponse; 26 | use reqwest::Client; 27 | use reqwest::ClientBuilder; 28 | use reqwest::Response; 29 | use reqwest::StatusCode; 30 | use serde::de::DeserializeOwned; 31 | 32 | #[derive(Debug, thiserror::Error)] 33 | #[error("{0}")] 34 | pub struct ClientError(String); 35 | 36 | #[derive(Debug, Clone)] 37 | pub enum HTTPResponse { 38 | Success(T), 39 | Error(ErrorStatus), 40 | } 41 | 42 | impl HTTPResponse { 43 | pub fn into_success(self) -> Option { 44 | match self { 45 | HTTPResponse::Success(t) => Some(t), 46 | HTTPResponse::Error(_) => None, 47 | } 48 | } 49 | } 50 | 51 | #[derive(Debug, Clone)] 52 | pub struct ErrorStatus { 53 | code: StatusCode, 54 | payload: Vec, 55 | } 56 | 57 | impl std::fmt::Display for ErrorStatus { 58 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 59 | write!( 60 | f, 61 | "{:?} ({}): {}", 62 | self.code.canonical_reason(), 63 | self.code.as_u16(), 64 | if self.payload.is_empty() { 65 | "(no payload)".into() 66 | } else { 67 | String::from_utf8_lossy(&self.payload) 68 | } 69 | ) 70 | } 71 | } 72 | 73 | #[derive(Debug)] 74 | pub struct HTTPClient { 75 | endpoint: String, 76 | client: Client, 77 | } 78 | 79 | impl HTTPClient { 80 | pub fn new( 81 | endpoint: impl Into, 82 | builder: ClientBuilder, 83 | ) -> error_stack::Result { 84 | let endpoint = endpoint.into(); 85 | let make_error = || ClientError(format!("failed to create client: {endpoint:?}")); 86 | 87 | Ok(Self { 88 | endpoint: endpoint.clone(), 89 | client: builder.build().change_context_lazy(make_error)?, 90 | }) 91 | } 92 | 93 | pub async fn create_topic( 94 | &self, 95 | topic_name: String, 96 | request: CreateTopicRequest, 97 | ) -> error_stack::Result, ClientError> { 98 | let make_error = || ClientError(format!("failed to create topic: {request:?}")); 99 | 100 | let response = self 101 | .client 102 | .post(format!("{}/v1/topics/{topic_name}", self.endpoint)) 103 | .json(&request) 104 | .send() 105 | .await 106 | .change_context_lazy(make_error)?; 107 | 108 | make_response(response).await 109 | } 110 | 111 | pub async fn publish( 112 | &self, 113 | topic_name: String, 114 | request: PublishMessageRequest, 115 | ) -> error_stack::Result, ClientError> { 116 | let make_error = || ClientError(format!("failed to publish messages: {request:?}")); 117 | 118 | let response = self 119 | .client 120 | .post(format!("{}/v1/topics/{topic_name}/publish", self.endpoint)) 121 | .json(&request) 122 | .send() 123 | .await 124 | .change_context_lazy(make_error)?; 125 | 126 | make_response(response).await 127 | } 128 | 129 | pub async fn create_subscription( 130 | &self, 131 | subscription_name: String, 132 | request: CreateSubscriptionRequest, 133 | ) -> error_stack::Result, ClientError> { 134 | let make_error = || ClientError(format!("failed to create subscription: {request:?}")); 135 | 136 | let response = self 137 | .client 138 | .post(format!( 139 | "{}/v1/subscriptions/{subscription_name}", 140 | self.endpoint 141 | )) 142 | .json(&request) 143 | .send() 144 | .await 145 | .change_context_lazy(make_error)?; 146 | 147 | make_response(response).await 148 | } 149 | 150 | pub async fn pull( 151 | &self, 152 | subscription_name: String, 153 | request: PullMessageRequest, 154 | ) -> error_stack::Result, ClientError> { 155 | let make_error = || ClientError(format!("failed to pull messages: {request:?}")); 156 | 157 | let response = self 158 | .client 159 | .post(format!( 160 | "{}/v1/subscriptions/{subscription_name}/pull", 161 | self.endpoint 162 | )) 163 | .json(&request) 164 | .send() 165 | .await 166 | .change_context_lazy(make_error)?; 167 | 168 | make_response(response).await 169 | } 170 | 171 | pub async fn acknowledge( 172 | &self, 173 | subscription_name: String, 174 | request: AcknowledgeRequest, 175 | ) -> error_stack::Result, ClientError> { 176 | let make_error = || ClientError(format!("failed to acknowledge: {request:?}")); 177 | 178 | let response = self 179 | .client 180 | .post(format!( 181 | "{}/v1/subscriptions/{subscription_name}/acknowledge", 182 | self.endpoint 183 | )) 184 | .json(&request) 185 | .send() 186 | .await 187 | .change_context_lazy(make_error)?; 188 | 189 | make_response(response).await 190 | } 191 | } 192 | 193 | async fn make_response( 194 | r: Response, 195 | ) -> error_stack::Result, ClientError> { 196 | let make_error = || ClientError("failed to make response".to_string()); 197 | 198 | let status = r.status(); 199 | 200 | if status.is_success() { 201 | let result = r.json().await.change_context_lazy(make_error)?; 202 | return Ok(HTTPResponse::Success(result)); 203 | } 204 | 205 | let payload = r.bytes().await.change_context_lazy(make_error)?; 206 | Ok(HTTPResponse::Error(ErrorStatus { 207 | code: status, 208 | payload: payload.to_vec(), 209 | })) 210 | } 211 | -------------------------------------------------------------------------------- /crates/runtime/src/runtime.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #![allow(clippy::disallowed_types)] 16 | 17 | use std::future::Future; 18 | use std::panic::resume_unwind; 19 | use std::sync::atomic::AtomicUsize; 20 | use std::sync::atomic::Ordering; 21 | use std::sync::Arc; 22 | use std::time::Duration; 23 | 24 | use futures::ready; 25 | 26 | static RUNTIME_ID: AtomicUsize = AtomicUsize::new(0); 27 | 28 | /// A runtime to run future tasks 29 | #[derive(Debug, Clone)] 30 | pub struct Runtime { 31 | name: String, 32 | runtime: Arc, 33 | } 34 | 35 | impl Runtime { 36 | pub fn builder() -> Builder { 37 | Builder::default() 38 | } 39 | 40 | /// Spawn a future and execute it in this thread pool. 41 | /// 42 | /// Similar to `tokio::runtime::Runtime::spawn()`. 43 | pub fn spawn(&self, future: F) -> JoinHandle 44 | where 45 | F: Future + Send + 'static, 46 | F::Output: Send + 'static, 47 | { 48 | JoinHandle::new(self.runtime.spawn(future)) 49 | } 50 | 51 | /// Run the provided function on an executor dedicated to blocking 52 | /// operations. 53 | pub fn spawn_blocking(&self, func: F) -> JoinHandle 54 | where 55 | F: FnOnce() -> R + Send + 'static, 56 | R: Send + 'static, 57 | { 58 | JoinHandle::new(self.runtime.spawn_blocking(func)) 59 | } 60 | 61 | /// Run a future to complete, this is the runtime entry point 62 | pub fn block_on(&self, future: F) -> F::Output { 63 | self.runtime.block_on(future) 64 | } 65 | 66 | pub fn name(&self) -> &str { 67 | &self.name 68 | } 69 | } 70 | 71 | impl fastimer::Spawn for &'static Runtime { 72 | fn spawn + Send + 'static>(&self, future: F) { 73 | Runtime::spawn(self, future); 74 | } 75 | } 76 | 77 | #[pin_project::pin_project] 78 | #[derive(Debug)] 79 | pub struct JoinHandle { 80 | #[pin] 81 | inner: tokio::task::JoinHandle, 82 | } 83 | 84 | impl JoinHandle { 85 | fn new(inner: tokio::task::JoinHandle) -> Self { 86 | Self { inner } 87 | } 88 | } 89 | 90 | impl Future for JoinHandle { 91 | type Output = R; 92 | 93 | fn poll( 94 | self: std::pin::Pin<&mut Self>, 95 | cx: &mut std::task::Context<'_>, 96 | ) -> std::task::Poll { 97 | let this = self.project(); 98 | let val = ready!(this.inner.poll(cx)); 99 | match val { 100 | Ok(val) => std::task::Poll::Ready(val), 101 | Err(err) => { 102 | if err.is_panic() { 103 | resume_unwind(err.into_panic()) 104 | } else { 105 | unreachable!() 106 | } 107 | } 108 | } 109 | } 110 | } 111 | 112 | pub struct Builder { 113 | runtime_name: String, 114 | thread_name: String, 115 | builder: tokio::runtime::Builder, 116 | } 117 | 118 | impl Default for Builder { 119 | fn default() -> Self { 120 | Self { 121 | runtime_name: format!("runtime-{}", RUNTIME_ID.fetch_add(1, Ordering::Relaxed)), 122 | thread_name: "default-worker".to_string(), 123 | builder: tokio::runtime::Builder::new_multi_thread(), 124 | } 125 | } 126 | } 127 | 128 | impl Builder { 129 | /// Sets the number of worker threads the Runtime will use. 130 | /// 131 | /// This can be any number above 0. The default value is the number of cores available to the 132 | /// system. 133 | pub fn worker_threads(&mut self, val: usize) -> &mut Self { 134 | self.builder.worker_threads(val); 135 | self 136 | } 137 | 138 | /// Specifies the limit for additional threads spawned by the Runtime. 139 | /// 140 | /// These threads are used for blocking operations like tasks spawned through spawn_blocking, 141 | /// they are not always active and will exit if left idle for too long, You can change this 142 | /// timeout duration with thread_keep_alive. The default value is 512. 143 | pub fn max_blocking_threads(&mut self, val: usize) -> &mut Self { 144 | self.builder.max_blocking_threads(val); 145 | self 146 | } 147 | 148 | /// Sets a custom timeout for a thread in the blocking pool. 149 | /// 150 | /// By default, the timeout for a thread is set to 10 seconds. 151 | pub fn thread_keep_alive(&mut self, duration: Duration) -> &mut Self { 152 | self.builder.thread_keep_alive(duration); 153 | self 154 | } 155 | 156 | pub fn runtime_name(&mut self, val: impl Into) -> &mut Self { 157 | self.runtime_name = val.into(); 158 | self 159 | } 160 | 161 | /// Sets name of threads spawned by the Runtime thread pool 162 | pub fn thread_name(&mut self, val: impl Into) -> &mut Self { 163 | self.thread_name = val.into(); 164 | self 165 | } 166 | 167 | pub fn build(&mut self) -> std::io::Result { 168 | let name = self.runtime_name.clone(); 169 | let runtime = self 170 | .builder 171 | .enable_all() 172 | .thread_name(self.thread_name.clone()) 173 | .build() 174 | .map(Arc::new)?; 175 | Ok(Runtime { name, runtime }) 176 | } 177 | } 178 | 179 | #[cfg(test)] 180 | mod tests { 181 | use std::sync::Arc; 182 | use std::thread; 183 | use std::time::Duration; 184 | 185 | use tokio::sync::oneshot; 186 | 187 | use super::*; 188 | 189 | fn runtime() -> Arc { 190 | let rt = Builder::default() 191 | .worker_threads(2) 192 | .thread_name("test_spawn_join") 193 | .build(); 194 | Arc::new(rt.unwrap()) 195 | } 196 | 197 | #[test] 198 | fn test_block_on() { 199 | let rt = runtime(); 200 | let out = rt.block_on(async { 201 | let (tx, rx) = oneshot::channel(); 202 | 203 | let _ = thread::spawn(move || { 204 | thread::sleep(Duration::from_millis(50)); 205 | tx.send("ZONE").unwrap(); 206 | }); 207 | 208 | rx.await.unwrap() 209 | }); 210 | assert_eq!(out, "ZONE"); 211 | } 212 | 213 | #[test] 214 | fn test_spawn_blocking() { 215 | let rt = runtime(); 216 | let rt_clone = rt.clone(); 217 | let out = rt.block_on(async move { 218 | let rt = rt_clone; 219 | let rt_clone = rt.clone(); 220 | let inner = rt 221 | .spawn_blocking(move || rt_clone.spawn(async move { "hello" })) 222 | .await; 223 | inner.await 224 | }); 225 | assert_eq!(out, "hello") 226 | } 227 | 228 | #[test] 229 | fn test_spawn_join() { 230 | let rt = runtime(); 231 | let handle = rt.spawn(async { 1 + 1 }); 232 | assert_eq!(2, rt.block_on(handle)); 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /crates/broker/src/broker.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2024 tison 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::Arc; 16 | 17 | use error_stack::Result; 18 | use error_stack::ResultExt; 19 | use morax_api::property::StorageProperty; 20 | use morax_api::property::TopicProperty; 21 | use morax_api::request::AcknowledgeRequest; 22 | use morax_api::request::AcknowledgeResponse; 23 | use morax_api::request::CreateSubscriptionRequest; 24 | use morax_api::request::CreateSubscriptionResponse; 25 | use morax_api::request::CreateTopicRequest; 26 | use morax_api::request::CreateTopicResponse; 27 | use morax_api::request::PublishMessageRequest; 28 | use morax_api::request::PublishMessageResponse; 29 | use morax_api::request::PubsubMessage; 30 | use morax_api::request::PullMessageRequest; 31 | use morax_api::request::PullMessageResponse; 32 | use morax_api::request::ReceivedMessage; 33 | use morax_meta::PostgresMetaService; 34 | use morax_storage::TopicStorage; 35 | 36 | use crate::BrokerError; 37 | 38 | #[derive(Debug, Clone)] 39 | pub struct Broker { 40 | meta: Arc, 41 | default_storage: StorageProperty, 42 | } 43 | 44 | impl Broker { 45 | pub fn new(meta: Arc, default_storage: StorageProperty) -> Self { 46 | Broker { 47 | meta, 48 | default_storage, 49 | } 50 | } 51 | 52 | pub async fn create_topic( 53 | &self, 54 | topic_name: String, 55 | request: CreateTopicRequest, 56 | ) -> Result { 57 | let make_error = || BrokerError(format!("failed to create topic with name {topic_name}")); 58 | 59 | self.meta 60 | .create_topic(morax_meta::CreateTopicRequest { 61 | name: topic_name.clone(), 62 | properties: TopicProperty { 63 | storage: request 64 | .storage 65 | .unwrap_or_else(|| self.default_storage.clone()), 66 | }, 67 | }) 68 | .await 69 | .change_context_lazy(make_error)?; 70 | 71 | Ok(CreateTopicResponse { name: topic_name }) 72 | } 73 | 74 | pub async fn create_subscription( 75 | &self, 76 | subscription_name: String, 77 | request: CreateSubscriptionRequest, 78 | ) -> Result { 79 | let name = subscription_name; 80 | let topic = request.topic_name; 81 | 82 | let make_error = || { 83 | BrokerError(format!( 84 | "failed to create subscription {name} for topic {topic}" 85 | )) 86 | }; 87 | 88 | self.meta 89 | .create_subscription(morax_meta::CreateSubscriptionRequest { 90 | name: name.clone(), 91 | topic: topic.clone(), 92 | }) 93 | .await 94 | .change_context_lazy(make_error)?; 95 | 96 | Ok(CreateSubscriptionResponse { topic, name }) 97 | } 98 | 99 | pub async fn publish( 100 | &self, 101 | topic_name: String, 102 | request: PublishMessageRequest, 103 | ) -> Result { 104 | let make_error = || BrokerError(format!("failed to publish message to topic {topic_name}")); 105 | 106 | let topic = self 107 | .meta 108 | .get_topic_by_name(topic_name.clone()) 109 | .await 110 | .change_context_lazy(make_error)?; 111 | let topic_storage = TopicStorage::new(topic.properties.0.storage); 112 | 113 | let now = jiff::Timestamp::now(); 114 | let messages = request 115 | .messages 116 | .into_iter() 117 | .map(|msg| PubsubMessage { 118 | publish_time: Some(now), 119 | ..msg 120 | }) 121 | .collect::>(); 122 | let messages_count = messages.len(); 123 | let split = serialize_messages(messages)?; 124 | let split_id = topic_storage 125 | .write_split(topic.topic_id, split) 126 | .await 127 | .change_context_lazy(make_error)?; 128 | 129 | let (start, end) = self 130 | .meta 131 | .commit_topic_splits(morax_meta::CommitTopicSplitRequest { 132 | topic_id: topic.topic_id, 133 | split_id, 134 | count: messages_count as i64, 135 | }) 136 | .await 137 | .change_context_lazy(make_error)?; 138 | 139 | Ok(PublishMessageResponse { 140 | message_ids: (start..end).map(|id| id.to_string()).collect(), 141 | }) 142 | } 143 | 144 | pub async fn pull( 145 | &self, 146 | subscription_name: String, 147 | request: PullMessageRequest, 148 | ) -> Result { 149 | let make_error = || { 150 | BrokerError(format!( 151 | "failed to pull messages for subscription {subscription_name}" 152 | )) 153 | }; 154 | 155 | let subscription = self 156 | .meta 157 | .get_subscription_by_name(subscription_name.clone()) 158 | .await 159 | .change_context_lazy(make_error)?; 160 | 161 | let acks = self 162 | .meta 163 | .list_acks(subscription.subscription_id) 164 | .await 165 | .change_context_lazy(make_error)?; 166 | 167 | let mut ranges = vec![]; 168 | let mut current = (0, 0); 169 | for (start, end) in acks { 170 | if start > current.1 { 171 | ranges.push((current.1, start)); 172 | } 173 | current = (start, end); 174 | } 175 | ranges.push((current.1, current.1 + request.max_messages)); 176 | 177 | let topic = self 178 | .meta 179 | .get_topic_by_name(subscription.topic_name) 180 | .await 181 | .change_context_lazy(make_error)?; 182 | let topic_storage = TopicStorage::new(topic.properties.0.storage); 183 | 184 | let mut splits = vec![]; 185 | for (start, end) in ranges { 186 | splits.extend( 187 | self.meta 188 | .fetch_topic_splits(morax_meta::FetchTopicSplitRequest { 189 | topic_id: topic.topic_id, 190 | start_offset: start, 191 | end_offset: end, 192 | }) 193 | .await 194 | .change_context_lazy(make_error)?, 195 | ); 196 | } 197 | 198 | let mut messages = vec![]; 199 | for split in splits { 200 | let data = topic_storage 201 | .read_split(split.topic_id, split.split_id) 202 | .await 203 | .change_context_lazy(make_error)?; 204 | let msgs = deserialize_messages(&data)?; 205 | for (i, msg) in msgs.into_iter().enumerate() { 206 | let id = split.start_offset + i as i64; 207 | messages.push(ReceivedMessage { 208 | ack_id: id.to_string(), 209 | message: PubsubMessage { 210 | message_id: Some(id.to_string()), 211 | ..msg 212 | }, 213 | }) 214 | } 215 | } 216 | 217 | Ok(PullMessageResponse { messages }) 218 | } 219 | 220 | pub async fn acknowledge( 221 | &self, 222 | subscription_name: String, 223 | request: AcknowledgeRequest, 224 | ) -> Result { 225 | let make_error = || { 226 | BrokerError(format!( 227 | "failed to acknowledge messages for subscription {subscription_name}" 228 | )) 229 | }; 230 | 231 | let subscription = self 232 | .meta 233 | .get_subscription_by_name(subscription_name.clone()) 234 | .await 235 | .change_context_lazy(make_error)?; 236 | 237 | let mut ack_ids = vec![]; 238 | for id in request.ack_ids { 239 | let id = id.parse::().change_context_lazy(make_error)?; 240 | ack_ids.push(id); 241 | } 242 | 243 | self.meta 244 | .acknowledge(morax_meta::AcknowledgeRequest { 245 | subscription_id: subscription.subscription_id, 246 | ack_ids, 247 | }) 248 | .await 249 | .change_context_lazy(make_error)?; 250 | 251 | Ok(AcknowledgeResponse {}) 252 | } 253 | } 254 | 255 | fn serialize_messages(messages: Vec) -> Result, BrokerError> { 256 | serde_json::to_vec(&messages) 257 | .change_context_lazy(|| BrokerError("failed to serialize messages".to_string())) 258 | } 259 | 260 | fn deserialize_messages(data: &[u8]) -> Result, BrokerError> { 261 | serde_json::from_slice(data) 262 | .change_context_lazy(|| BrokerError("failed to deserialize messages".to_string())) 263 | } 264 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------