├── rust-toolchain.toml ├── .gitignore ├── .gitmodules ├── Cargo.toml ├── common ├── Cargo.toml └── src │ └── lib.rs ├── api ├── macros │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── Cargo.toml └── src │ ├── remote_result.rs │ ├── lib.rs │ └── c_api.rs ├── test-function ├── Cargo.toml ├── examples │ └── local.rs └── src │ ├── lib.rs │ └── main.rs ├── function-scheduler ├── Cargo.toml └── src │ └── bin │ ├── run-function.rs │ └── function-scheduler.rs ├── NOTICE.md ├── function-executor ├── Cargo.toml └── src │ ├── main.rs │ ├── wasmtime.rs │ └── wasmedge.rs ├── .github └── workflows │ └── ci.yml ├── README.md └── LICENSE /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | targets = ["wasm32-wasi"] 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | function-executor.log 3 | kvs.log 4 | route.log 5 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "anna-rs"] 2 | path = anna-rs 3 | url = https://github.com/essa-project/anna-rs.git 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "api", 4 | "api/macros", 5 | "common", 6 | "function-executor", 7 | "function-scheduler", 8 | "test-function", 9 | ] 10 | exclude = ["anna-rs"] 11 | -------------------------------------------------------------------------------- /common/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "essa-common" 3 | version = "0.1.0" 4 | edition = "2021" 5 | license = "Apache-2.0" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | # keep in sync with anna-rs 11 | zenoh = "0.7.0-rc" 12 | -------------------------------------------------------------------------------- /api/macros/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "essa-macros" 3 | version = "0.1.0" 4 | edition = "2021" 5 | license = "Apache-2.0" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [lib] 10 | proc-macro = true 11 | 12 | [dependencies] 13 | syn = { version = "1.0.81", features = ["full"] } 14 | quote = "1.0.10" 15 | proc-macro2 = "1.0.32" 16 | 17 | [dev-dependencies] 18 | essa-api = { path = ".." } 19 | -------------------------------------------------------------------------------- /test-function/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "essa-test-function" 3 | version = "0.1.0" 4 | edition = "2021" 5 | license = "Apache-2.0" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | anyhow = "1.0.45" 11 | serde = { version = "1.0.130", features = ["derive"] } 12 | bincode = "1.3.3" 13 | essa-api = { path = "../api" } 14 | anna-api = { git = "https://github.com/essa-project/anna-rs.git" } 15 | -------------------------------------------------------------------------------- /api/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "essa-api" 3 | version = "0.1.0" 4 | edition = "2021" 5 | license = "Apache-2.0" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | anyhow = "1.0.45" 11 | serde = { version = "1.0.130", features = ["derive"] } 12 | bincode = "1.3.3" 13 | num_enum = "0.5.4" 14 | essa-macros = { path = "macros" } 15 | anna-api = { git = "https://github.com/essa-project/anna-rs.git" } 16 | -------------------------------------------------------------------------------- /function-scheduler/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "essa-function-scheduler" 3 | version = "0.1.0" 4 | edition = "2021" 5 | license = "Apache-2.0" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | anyhow = "1.0.45" 11 | wasmtime = "9.0.3" 12 | wasmtime-wasi = "9.0.3" 13 | argh = "0.1.6" 14 | # keep in sync with anna-rs 15 | zenoh = "0.7.0-rc" 16 | essa-common = { path = "../common" } 17 | smol = "1.2.5" 18 | futures = "0.3.17" 19 | anna-api = { git = "https://github.com/essa-project/anna-rs.git" } 20 | -------------------------------------------------------------------------------- /NOTICE.md: -------------------------------------------------------------------------------- 1 | ## Copyright 2 | 3 | All content is the property of the respective authors or their employers. For more information regarding authorship of content, please consult the listed source code repository logs. 4 | 5 | ## License 6 | 7 | This project is licensed under the Apache License, Version 2.0 ([LICENSE](LICENSE) or ). Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be licensed as above, without any additional terms or conditions. 8 | 9 | ## Inspiration 10 | 11 | This project was heavily insprired by the **[`cloudburst`](https://github.com/hydro-project/cloudburst)** project, which provides a "low-latency, stateful serverless programming framework". Cloudburst is [licensed](https://github.com/hydro-project/cloudburst/blob/07905ae3f489fb2a9b920f701c3023c02ee8b877/LICENSE) under the Apache License, Version 2.0. 12 | -------------------------------------------------------------------------------- /function-executor/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | edition = "2021" 3 | license = "Apache-2.0" 4 | name = "essa-function-executor" 5 | version = "0.1.0" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | anyhow = "1.0.45" 11 | argh = "0.1.6" 12 | eyre = "0.6.5" 13 | wasmedge-sdk = { version = "0.8.1", optional = true } 14 | wasmtime = { version = "9.0.3", optional = true } 15 | wasmtime-wasi = { version = "9.0.3", optional = true } 16 | # keep in sync with anna-rs 17 | anna = { git = "https://github.com/essa-project/anna-rs.git" } 18 | bincode = "1.3.3" 19 | chrono = { version = "0.4.19", default-features = false } 20 | essa-common = { path = "../common" } 21 | fern = "0.6.0" 22 | log = "0.4.14" 23 | serde = { version = "1.0.130", features = ["derive"] } 24 | smol = "1.2.5" 25 | uuid = { version = "0.8.2", features = ["v4"] } 26 | zenoh = "0.7.0-rc" 27 | flume = "0.10.14" 28 | 29 | [features] 30 | default = ["wasmedge_executor"] 31 | wasmedge_executor = ["wasmedge-sdk"] 32 | wasmtime_executor = ["wasmtime", "wasmtime-wasi"] 33 | -------------------------------------------------------------------------------- /api/src/remote_result.rs: -------------------------------------------------------------------------------- 1 | use crate::{EssaResult, ResultHandle}; 2 | use std::marker::PhantomData; 3 | 4 | /// An external function result that might not be available yet. 5 | /// 6 | /// Used by the `essa-macros` crate. 7 | #[must_use] 8 | pub struct RemoteFunctionResult { 9 | result_handle: ResultHandle, 10 | data: PhantomData, 11 | } 12 | 13 | impl RemoteFunctionResult 14 | where 15 | T: for<'a> serde::Deserialize<'a>, 16 | { 17 | /// Creates a new `RemoteFunctionResult` from the given handle. 18 | /// 19 | /// The caller must make sure that the handle is of type `T`, otherwise 20 | /// a deserialization error might occur on [`get`][Self::get]. 21 | pub fn new(handle: ResultHandle) -> Self { 22 | Self { 23 | result_handle: handle, 24 | data: PhantomData, 25 | } 26 | } 27 | 28 | /// Waits until the result becomes available. 29 | pub fn get(self) -> Result { 30 | let value = self.result_handle.wait()?; 31 | let deserialized = bincode::deserialize(&value).map_err(|_| EssaResult::InvalidResult)?; 32 | Ok(deserialized) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /function-scheduler/src/bin/run-function.rs: -------------------------------------------------------------------------------- 1 | //! Triggers a run for the given WASM module. 2 | //! 3 | //! This executable sends a run request for the specified WASM file to the 4 | //! essa-rs scheduler and then exits. It does not wait for the results of the 5 | //! WASM module. 6 | 7 | use anyhow::Context; 8 | use essa_common::{essa_default_zenoh_prefix, scheduler_run_module_topic}; 9 | use std::{fs, path::PathBuf}; 10 | use zenoh::prelude::sync::SyncResolve; 11 | 12 | #[derive(argh::FromArgs)] 13 | /// Triggers a run of the given WASM file in essa-rs. 14 | struct Args { 15 | /// the WASM file that should be executed in essa-rs 16 | #[argh(positional)] 17 | wasm_file: PathBuf, 18 | } 19 | 20 | fn main() -> anyhow::Result<()> { 21 | // read the specified WASM file 22 | let args: Args = argh::from_env(); 23 | let wasm_bytes = fs::read(&args.wasm_file).context("failed to read given wasm file")?; 24 | 25 | let zenoh = zenoh::open(zenoh::config::Config::default()) 26 | .res() 27 | .map_err(|e| anyhow::anyhow!(e)) 28 | .context("failed to connect to zenoh")?; 29 | let zenoh_prefix = essa_default_zenoh_prefix(); 30 | 31 | // pass the WASM file to the scheduler 32 | zenoh 33 | .put(scheduler_run_module_topic(zenoh_prefix), wasm_bytes) 34 | .res() 35 | .map_err(|e| anyhow::anyhow!(e)) 36 | .context("failed to start module")?; 37 | 38 | zenoh 39 | .close() 40 | .res() 41 | .map_err(|e| anyhow::anyhow!(e)) 42 | .context("failed to close zenoh")?; 43 | 44 | Ok(()) 45 | } 46 | -------------------------------------------------------------------------------- /common/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Provides common functionality used by both the `essa-function-executor` 2 | //! and `essa-function-scheduler` crates. 3 | 4 | #![warn(missing_docs)] 5 | 6 | /// The default topic prefix for zenoh, used by all essa code. 7 | pub fn essa_default_zenoh_prefix() -> &'static str { 8 | "/essa" 9 | } 10 | 11 | /// The subtopic for instructing the essa function scheduler to start the given 12 | /// WASM module. 13 | pub fn scheduler_run_module_topic(zenoh_prefix: &str) -> String { 14 | format!("{}/run-module", zenoh_prefix) 15 | } 16 | 17 | /// The subtopic for instructing the essa function scheduler to invoke the 18 | /// WASM function with the given name on a remote node. 19 | pub fn scheduler_function_call_subscribe_topic(zenoh_prefix: &str) -> String { 20 | format!("{}/call/**", zenoh_prefix) 21 | } 22 | 23 | /// The subtopic for invoking a specific function of a specific module. 24 | pub fn scheduler_function_call_topic( 25 | zenoh_prefix: &str, 26 | module: &str, 27 | function: &str, 28 | args: &str, 29 | ) -> String { 30 | format!("{zenoh_prefix}/call/{module}/{function}/{args}",) 31 | } 32 | 33 | /// Subtopic for instructing a specific essa function executor to run the 34 | /// give WASM module. 35 | pub fn executor_run_module_topic(executor_id: u32, zenoh_prefix: &str) -> String { 36 | format!("{}/executor/{}/run-module", zenoh_prefix, executor_id) 37 | } 38 | 39 | /// Subtopic for instructing a specific essa function executor to run a 40 | /// function. 41 | pub fn executor_run_function_subscribe_topic(executor_id: u32, zenoh_prefix: &str) -> String { 42 | format!("{}/executor/{}/run-function/**", zenoh_prefix, executor_id) 43 | } 44 | 45 | /// Subtopic for instructing a specific essa function executor to run the 46 | /// WASM function with the given name. 47 | pub fn executor_run_function_topic( 48 | executor_id: u32, 49 | zenoh_prefix: &str, 50 | module: &str, 51 | function: &str, 52 | args: &str, 53 | ) -> String { 54 | format!("{zenoh_prefix}/executor/{executor_id}/run-function/{module}/{function}/{args}") 55 | } 56 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | pull_request: 6 | workflow_dispatch: 7 | 8 | jobs: 9 | test_wasmedge: 10 | name: "Test (wasmedge)" 11 | 12 | # The type of runner that the job will run on 13 | runs-on: ubuntu-latest 14 | 15 | # Steps represent a sequence of tasks that will be executed as part of the job 16 | steps: 17 | 18 | - name: "Build WasmEdge" 19 | run: curl -sSf https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh | bash 20 | 21 | - uses: actions/checkout@v3 22 | with: 23 | submodules: true 24 | 25 | - name: "Check Rust code" 26 | run: cargo check --workspace 27 | 28 | - name: "Run cargo clippy" 29 | run: cargo clippy --workspace 30 | 31 | - name: "Build Rust code" 32 | run: cargo build --workspace --exclude essa-test-function 33 | 34 | - name: "Build test-function" 35 | run: cargo build --release -p essa-test-function --target wasm32-wasi 36 | 37 | - name: "Test Rust code" 38 | run: source $HOME/.wasmedge/env && cargo test --workspace --exclude essa-test-function 39 | timeout-minutes: 10 40 | 41 | test_wasmtime: 42 | name: "Test (wasmtime)" 43 | 44 | # The type of runner that the job will run on 45 | runs-on: ubuntu-latest 46 | 47 | # Steps represent a sequence of tasks that will be executed as part of the job 48 | steps: 49 | - uses: actions/checkout@v3 50 | with: 51 | submodules: true 52 | 53 | - name: "Check Rust code" 54 | run: cargo check --workspace --no-default-features --features wasmtime_executor 55 | 56 | - name: "Run cargo clippy" 57 | run: cargo clippy --workspace --no-default-features --features wasmtime_executor 58 | 59 | - name: "Build Rust code" 60 | run: cargo build --workspace --exclude essa-test-function --no-default-features --features wasmtime_executor 61 | 62 | - name: "Build test-function" 63 | run: cargo build --release -p essa-test-function --target wasm32-wasi 64 | 65 | - name: "Test Rust code" 66 | run: cargo test --workspace --exclude essa-test-function --no-default-features --features wasmtime_executor 67 | timeout-minutes: 10 68 | -------------------------------------------------------------------------------- /test-function/examples/local.rs: -------------------------------------------------------------------------------- 1 | //! Starts up a minimal essa-rs system and runs the `essa-test-function` in it. 2 | 3 | use std::{ 4 | io::{self, stdin}, 5 | process::{Child, Command, Stdio}, 6 | thread, 7 | time::Duration, 8 | }; 9 | 10 | fn main() { 11 | // build essa-rs executables and the essa-test-function 12 | run_command( 13 | "cargo build --release --workspace --exclude essa-test-function", 14 | true, 15 | ) 16 | .unwrap() 17 | .wait() 18 | .unwrap(); 19 | run_command( 20 | "cargo build --release -p essa-test-function --target wasm32-wasi", 21 | true, 22 | ) 23 | .unwrap() 24 | .wait() 25 | .unwrap(); 26 | 27 | // start anna and essa nodes 28 | let mut routing = run_command("cargo run --release --manifest-path anna-rs/Cargo.toml --bin routing -- anna-rs/example-config.yml", false).unwrap(); 29 | let mut kvs = run_command("cargo run --release --manifest-path anna-rs/Cargo.toml --bin kvs -- anna-rs/example-config.yml", false).unwrap(); 30 | let mut scheduler = run_command( 31 | "cargo run --release -p essa-function-scheduler --bin function-scheduler", 32 | true, 33 | ) 34 | .unwrap(); 35 | let mut executor = 36 | run_command("cargo run --release -p essa-function-executor -- 0", true).unwrap(); 37 | 38 | // wait a bit, then trigger test function run 39 | thread::sleep(Duration::from_secs(2)); 40 | run_command("cargo run -p essa-function-scheduler --bin run-function -- target/wasm32-wasi/release/essa-test-function.wasm", true).unwrap().wait().unwrap(); 41 | 42 | // wait for user input 43 | println!("Press enter to exit"); 44 | let mut buffer = String::new(); 45 | stdin().read_line(&mut buffer).unwrap(); 46 | 47 | // kill all the essa-rs executables again 48 | executor.kill().unwrap(); 49 | scheduler.kill().unwrap(); 50 | kvs.kill().unwrap(); 51 | routing.kill().unwrap(); 52 | } 53 | 54 | /// Runs the given command as a subprocess. 55 | /// 56 | /// If `stdout` is true, the subprocess is started with an inherited stdout 57 | /// handle, i.e. the same stdout as this main process. If `stdout` is set 58 | /// to false, the `stdout` of the subprocess is set to `null`, i.e. all 59 | /// output is thrown away. 60 | fn run_command(command: &str, stdout: bool) -> io::Result { 61 | let mut split = command.split_ascii_whitespace(); 62 | let mut cmd = Command::new(split.next().unwrap()); 63 | cmd.args(split); 64 | if !stdout { 65 | cmd.stdout(Stdio::null()); 66 | } 67 | cmd.spawn() 68 | } 69 | -------------------------------------------------------------------------------- /test-function/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Contains some examples for essa-rs functions that can be called remotely. 2 | //! 3 | //! The `essa_wrap` attribute generates a wrapper function with the supplied 4 | //! name to call the attributed function remotely. All functions can still be 5 | //! called locally too, using their normal name. 6 | 7 | use essa_api::essa_wrap; 8 | 9 | /// Converts the given string to uppercase characters. 10 | /// 11 | /// The `essa_wrap` attribute creates a wrapper function called 12 | /// [`to_uppercase_extern`] that runs `to_uppercase` on a remote essa-rs node. 13 | /// The wrapper function returns a [`EssaFuture`](anna-api::EssaFuture), which 14 | /// can be used to retrieve the result asynchronously. 15 | #[essa_wrap(name = "to_uppercase_extern")] 16 | pub fn to_uppercase(val: String) -> String { 17 | val.to_ascii_uppercase() 18 | } 19 | 20 | /// Adds the string `"fooo"` to the given value. 21 | #[essa_wrap(name = "append_foo")] 22 | pub fn foo(val: String) -> String { 23 | val + " fooo" 24 | } 25 | 26 | /// Repeats the given string `count` times. 27 | /// 28 | /// Example for a function with multiple arguments. 29 | #[essa_wrap(name = "repeat_string_extern")] 30 | pub fn repeat_string(string: String, count: usize) -> String { 31 | let mut s = String::new(); 32 | for _ in 0..count { 33 | s += &string; 34 | } 35 | s 36 | } 37 | 38 | /// Stress-tests essa-rs and anna-rs through recursive calls and multiple KVS 39 | /// writes. 40 | /// 41 | /// The functions splits the given range into two halves and invokes itself 42 | /// recursively for both halves. The recursive calls are done as remote 43 | /// essa-rs calls. As soon as the range size becomes 1, the function writes 44 | /// the set `[x]` to the KVS under the given `key`, where `x` is the single 45 | /// number in that range. 46 | /// 47 | /// Set lattices are merged by anna-rs when there are multiple writes. Thus, 48 | /// the set value stored at the specified `key` should contain the full range 49 | /// when this function returns. 50 | #[essa_wrap(name = "concurrent_kvs_test_extern")] 51 | pub fn concurrent_kvs_test( 52 | key: anna_api::ClientKey, 53 | range_start: usize, 54 | range_end: usize, 55 | ) -> Result<(), String> { 56 | let range_size = range_end - range_start; 57 | if range_size == 1 { 58 | // write the set `[range_start]` to the KVS under `key` 59 | // 60 | // Note that `SetLattice`s are not overwritten on multiple writes to 61 | // the same key, but merged instead. 62 | essa_api::kvs_put( 63 | &key, 64 | &anna_api::lattice::SetLattice::new([range_start.to_string().into()].into()).into(), 65 | ) 66 | .unwrap(); 67 | } else { 68 | // split the range into two halves and do a recursive call on both 69 | // halves as a remote essa-rs call 70 | let mid = range_start + range_size / 2; 71 | let result_1 = 72 | concurrent_kvs_test_extern(key.clone(), range_start, mid).map_err(|e| e.to_string())?; 73 | let result_2 = 74 | concurrent_kvs_test_extern(key.clone(), mid, range_end).map_err(|e| e.to_string())?; 75 | 76 | // wait until the recursive calls are done 77 | result_1.get().map_err(|e| format!("{:?}", e))??; 78 | result_2.get().map_err(|e| format!("{:?}", e))??; 79 | } 80 | 81 | Ok(()) 82 | } 83 | -------------------------------------------------------------------------------- /test-function/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::collections::{BTreeSet, HashSet}; 2 | 3 | use anna_api::lattice::{Lattice, SetLattice}; 4 | use anyhow::Context; 5 | use essa_test_function::{ 6 | append_foo, concurrent_kvs_test_extern, repeat_string_extern, to_uppercase_extern, 7 | }; 8 | 9 | fn main() { 10 | println!("Hello world from test function!"); 11 | let result = to_uppercase_extern("foobar".into()).expect("extern function call failed"); 12 | println!("Waiting for result..."); 13 | let result = result.get().unwrap(); 14 | println!("Function result: {}", result); 15 | 16 | println!("Storing a set in the kvs"); 17 | let key = "some-test-key".into(); 18 | essa_api::kvs_put( 19 | &key, 20 | &SetLattice::new(["one".into(), "two".into(), "three".into()].into()).into(), 21 | ) 22 | .unwrap(); 23 | 24 | let result = append_foo(result).expect("extern function call failed"); 25 | println!("Waiting for result..."); 26 | let result = result.get().unwrap(); 27 | println!("Function result: {}", result); 28 | 29 | println!("Reading the set from the kvs"); 30 | let lattice = essa_api::kvs_get(&key) 31 | .unwrap() 32 | .into_set() 33 | .unwrap() 34 | .into_revealed(); 35 | println!( 36 | "Result: {:?}", 37 | lattice 38 | .iter() 39 | .map(|v| std::str::from_utf8(v)) 40 | .collect::, _>>() 41 | .unwrap() 42 | ); 43 | 44 | println!("Appending to the set in the kvs"); 45 | essa_api::kvs_put( 46 | &key, 47 | &SetLattice::new(["four".into(), "two".into(), "three".into()].into()).into(), 48 | ) 49 | .unwrap(); 50 | 51 | let result = repeat_string_extern(result, 15000).expect("extern function call failed"); 52 | println!("Waiting for result..."); 53 | let result = result.get().unwrap(); 54 | println!("Function result: {}", result.len()); 55 | 56 | println!("Reading the set from the kvs"); 57 | let lattice = essa_api::kvs_get(&key) 58 | .unwrap() 59 | .into_set() 60 | .unwrap() 61 | .into_revealed(); 62 | println!( 63 | "Result: {:?}", 64 | lattice 65 | .iter() 66 | .map(|v| std::str::from_utf8(v)) 67 | .collect::, _>>() 68 | .unwrap() 69 | ); 70 | 71 | println!("Running concurrent KVS test"); 72 | let key: anna_api::ClientKey = "concurrent-kvs_test-key".into(); 73 | let range_start = 48; 74 | let range_end = 113; 75 | let result = concurrent_kvs_test_extern(key.clone(), range_start, range_end) 76 | .expect("concurrent kvs test call failed"); 77 | result.get().unwrap().expect("function failed"); 78 | 79 | println!("Reading the concurrent KVS test result set from the kvs"); 80 | let lattice = essa_api::kvs_get(&key) 81 | .unwrap() 82 | .into_set() 83 | .unwrap() 84 | .into_revealed(); 85 | let result_set = lattice 86 | .iter() 87 | .map(|v| { 88 | let s = std::str::from_utf8(v).context("result entry not utf8")?; 89 | let i = s.parse().context("result entry not an usize")?; 90 | Result::::Ok(i) 91 | }) 92 | .collect::, _>>() 93 | .unwrap(); 94 | assert_eq!(result_set, (range_start..range_end).collect()); 95 | 96 | println!("DONE"); 97 | } 98 | -------------------------------------------------------------------------------- /function-scheduler/src/bin/function-scheduler.rs: -------------------------------------------------------------------------------- 1 | //! Schedules function call requests across available executor nodes. 2 | //! 3 | //! Handles both module run requests sent by the user (through `run-function`) 4 | //! and remote function call requests invoked by WASM functions. 5 | //! 6 | //! (Right now this is only a sample implementation that always chooses 7 | //! the executor node with ID 0 for all requests.) 8 | 9 | use std::sync::Arc; 10 | 11 | use anyhow::Context; 12 | use essa_common::{ 13 | essa_default_zenoh_prefix, executor_run_function_topic, executor_run_module_topic, 14 | scheduler_function_call_subscribe_topic, scheduler_run_module_topic, 15 | }; 16 | use futures::{select, StreamExt}; 17 | use zenoh::{ 18 | prelude::{r#async::AsyncResolve, SplitBuffer}, 19 | queryable::Query, 20 | }; 21 | 22 | fn main() -> anyhow::Result<()> { 23 | smol::block_on(run()) 24 | } 25 | 26 | async fn run() -> anyhow::Result<()> { 27 | let zenoh = zenoh::open(zenoh::config::Config::default()) 28 | .res() 29 | .await 30 | .map_err(|e| anyhow::anyhow!(e)) 31 | .context("failed to connect to zenoh")? 32 | .into_arc(); 33 | let zenoh_prefix = essa_default_zenoh_prefix(); 34 | 35 | // subscribe to module run requests issued through `run-function` 36 | let mut new_modules_sub = zenoh 37 | .declare_subscriber(scheduler_run_module_topic(zenoh_prefix)) 38 | .res() 39 | .await 40 | .map_err(|e| anyhow::anyhow!(e)) 41 | .context("failed to subscribe to new modules")?; 42 | let mut new_modules = new_modules_sub.receiver.into_stream(); 43 | 44 | // subscribe to remote function call requests issued by WASM functions 45 | let mut function_calls_sub = zenoh 46 | .declare_queryable(scheduler_function_call_subscribe_topic(zenoh_prefix)) 47 | .res() 48 | .await 49 | .map_err(|e| anyhow::anyhow!(e)) 50 | .context("failed to subscribe to function calls")?; 51 | let mut function_calls = function_calls_sub.receiver.into_stream(); 52 | 53 | loop { 54 | select! { 55 | change = new_modules.select_next_some() => { 56 | run_module(change.value.payload.contiguous().into_owned(), &zenoh, &zenoh_prefix) 57 | .await 58 | .context("failed to run module")? 59 | } 60 | query = function_calls.select_next_some() => { 61 | call_function(query, zenoh.clone(), zenoh_prefix) 62 | .await 63 | .context("failed to run module")? 64 | } 65 | complete => break, 66 | } 67 | } 68 | 69 | Ok(()) 70 | } 71 | 72 | async fn run_module( 73 | wasm_bytes: Vec, 74 | zenoh: &zenoh::Session, 75 | zenoh_prefix: &str, 76 | ) -> anyhow::Result<()> { 77 | // TODO: implement an actual scheduling policy instead of always choosing 78 | // executor node 0 79 | let executor_id = 0; 80 | 81 | // forward the request to the selected executor 82 | zenoh 83 | .put( 84 | executor_run_module_topic(executor_id, zenoh_prefix), 85 | wasm_bytes, 86 | ) 87 | .res() 88 | .await 89 | .map_err(|e| anyhow::anyhow!(e)) 90 | .context("failed to send module to executor")?; 91 | 92 | Ok(()) 93 | } 94 | 95 | async fn call_function( 96 | query: Query, 97 | zenoh: Arc, 98 | zenoh_prefix: &'static str, 99 | ) -> anyhow::Result<()> { 100 | // TODO: implement an actual scheduling policy instead of always choosing 101 | // executor node 0 102 | let executor_id = 0; 103 | 104 | let mut topic_split = query.key_expr().as_str().split('/'); 105 | let args = topic_split 106 | .next_back() 107 | .context("no args key in topic")? 108 | .to_owned(); 109 | let function = topic_split 110 | .next_back() 111 | .context("no function key in topic")? 112 | .to_owned(); 113 | let module = topic_split 114 | .next_back() 115 | .context("no module key in topic")? 116 | .to_owned(); 117 | 118 | // forward the request to the selected executor 119 | let task = async move { 120 | let topic = 121 | executor_run_function_topic(executor_id, &zenoh_prefix, &module, &function, &args); 122 | 123 | let reply = zenoh 124 | .get(topic) 125 | .res() 126 | .await 127 | .expect("failed to forward function call to executor") 128 | .recv_async() 129 | .await 130 | .expect("failed to receive reply"); 131 | 132 | query.reply(reply.sample).res().await; 133 | }; 134 | smol::spawn(task).detach(); 135 | 136 | Ok(()) 137 | } 138 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # essa-rs 2 | 3 | Essa is an experimental stateful serverless programming framework based on WebAssembly. 4 | 5 | This project was heavily insprired by **[`cloudburst`](https://github.com/hydro-project/cloudburst)**, which provides a "low-latency, stateful serverless programming framework". Essa differs from the original project in multiple ways: 6 | 7 | - **WebAssembly functions:** Instead of requiring that the user-provided functions are written in Python, we use the [_WebAssembly_](https://webassembly.org/) format. WebAssembly — or _"Wasm"_ for short — is a portable instruction format that aims to be as fast as native code, while being completely sandboxed. Multiple programming languages such as C/C++, Rust, Go, or C# can be compiled to it already. These properties make Wasm very suitable for running untrusted user functions in a serverless framework. 8 | - **Written in Rust:** While the original `cloudburst` is written in Python, we rely on the [Rust](https://www.rust-lang.org/) programming language. In addition to Rust's memory safety guarantees and high performance, this choice has the advantage that we can easily use the [`wasmedge-sdk`](https://github.com/WasmEdge/WasmEdge/tree/master/bindings/rust/wasmedge-sdk) or the [`wasmtime`](https://github.com/bytecodealliance/wasmtime) crate for running [_WASI_](https://wasi.dev/)-compatible WebAssembly functions. 9 | - **Based on our `anna-rs` project:** The original `cloudburst` project uses the [`anna` key-value store](https://github.com/hydro-project/anna) for storing function state. While we follow that design, we instead use [our `anna-rs` port](https://github.com/essa-project/anna-rs), which is written in Rust instead of C++ and communicates using [`zenoh`](https://zenoh.io/) instead of [`ZeroMQ`](https://zeromq.org/). 10 | 11 | **Note:** This project is still in a prototype state, so don't use it in production! 12 | 13 | ## Overview 14 | 15 | This project contains several executables: 16 | 17 | - **`essa-function-executor`**: Allows to compile and execute a WebAssembly module and its functions. We use [`WasmEdge`](https://github.com/WasmEdge/WasmEdge) as the default executor. By enabling the feature `wasmtime_executor`, we can use [`wasmtime`](https://github.com/bytecodealliance/wasmtime) as the executor. 18 | - **`essa-function-scheduler`**: Schedules function execution requests to function executors. 19 | - **`essa-test-function`:** A WebAssembly module that can be run in `essa`. It shows how to perform function calls across nodes and how to share state between them. 20 | - **`run-function`**: Helper executable to start a given WebAssembly module by passing it to a function scheduler. 21 | 22 | The following libraries are provided to interact with the Essa framework in WebAssembly crates: 23 | 24 | 25 | - **`essa-api`**: Provides an interface to `essa` functions, which can be used from WebAssembly modules written in Rust. 26 | - **`essa-macros`**: Provides macros to automatically make Rust functions in WebAssembly modules compatible with the `essa` function call ABI. Re-exported from `essa-api`. 27 | 28 | ## Build 29 | 30 | You need the latest version of [Rust](https://www.rust-lang.org/) for building. 31 | 32 | The build commands are: 33 | 34 | - `cargo build --workspace --exclude essa-test-function` for a debug build 35 | - `cargo build --workspace --exclude essa-test-function --release` for an optimized release build 36 | 37 | After building, you can find the resulting executables under `../target/debug` (for debug builds) or `../target/release` (for release builds). 38 | 39 | To run the **test suite**, execute `cargo test`. The **API documentation** can be generated through `cargo doc --open`. 40 | 41 | ## Run 42 | 43 | In order to run `essa-rs`, you need to first initialize the [`anna-rs`](https://github.com/essa-project/anna-rs) submodule through `git submodule update --init`. 44 | 45 | For a **quick demo**, run `cargo run -p essa-test-function --example local`. This command will compile and start all necessary runtime nodes, compile the `essa-test-function` to WASM, and then run the compiled WASM module on the started essa-rs nodes. It is worth noting that in this demo, [`WasmEdge`](https://github.com/WasmEdge/WasmEdge) will be used as the default executor. 46 | 47 | For a **manual run**, execute the following commands in different terminal windows: 48 | 49 | - Start an `anna-rs` routing node: 50 | 51 | ``` 52 | cargo run --release --manifest-path anna-rs/Cargo.toml --bin routing -- anna-rs/example-config.yml 53 | ``` 54 | - Start an `anna-rs` KVS node: 55 | 56 | ``` 57 | cargo run --release --manifest-path anna-rs/Cargo.toml --bin kvs -- anna-rs/example-config.yml 58 | ``` 59 | - Start the `essa-rs` function scheduler: 60 | 61 | ``` 62 | cargo run --release -p essa-function-scheduler --bin function-scheduler 63 | ``` 64 | - Start an `essa-rs` function executor node with ID `0`: 65 | 66 | - Use [`WasmEdge`](https://github.com/WasmEdge/WasmEdge) as the executor: 67 | 68 | ``` 69 | cargo run --release -p essa-function-executor -- 0 70 | ``` 71 | - Use [`wasmtime`](https://github.com/bytecodealliance/wasmtime) as the executor: 72 | 73 | ``` 74 | cargo run --no-default-features --features wasmtime_executor --release -p essa-function-executor -- 0 75 | ``` 76 | - Compile the `essa-test-function` to WASM and start it through the `run-function` executable: 77 | 78 | ``` 79 | cargo build --release -p essa-test-function --target wasm32-wasi 80 | cargo run -p essa-function-scheduler --bin run-function -- target/wasm32-wasi/release/essa-test-function.wasm 81 | ``` 82 | The output of the WebAssembly function is then shown in the console window of the `essa-function-executor` process. 83 | 84 | The above commands should result in the same output as the demo command mentioned at the beginning of this section. 85 | 86 | Instead of using `cargo run --release`, it is of course also possible to first compile the executables as described in the [_Build_](#build) section and then run them from the `target/release` folder. 87 | 88 | ## License 89 | 90 | Licensed under the Apache License, Version 2.0 ([LICENSE](LICENSE) or ). Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be licensed as above, without any additional terms or conditions. 91 | -------------------------------------------------------------------------------- /api/macros/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Provides the `essa_wrap` macro. 2 | //! 3 | //! This crate should not be used directly. Instead, the reexport at 4 | //! `essa_api::essa_wrap` should be used. 5 | 6 | #![warn(missing_docs)] 7 | 8 | use proc_macro::TokenStream; 9 | use proc_macro2::TokenStream as TokenStream2; 10 | use quote::quote; 11 | use std::fmt::Display; 12 | use syn::{Ident, ItemFn, MetaNameValue}; 13 | 14 | extern crate proc_macro; 15 | 16 | /// Makes the annotated function callable as an essa-rs function. 17 | /// 18 | /// ## Example 19 | /// 20 | /// Add the `essa_wrap` attribute to an arbitrary function to integrate it with essa-rs: 21 | /// 22 | /// ``` 23 | /// # use essa_macros::essa_wrap; 24 | /// 25 | /// #[essa_wrap(name = "to_uppercase_extern")] 26 | /// pub fn to_uppercase(val: String) -> String { 27 | /// val.to_ascii_uppercase() 28 | /// } 29 | /// ``` 30 | /// 31 | /// This creates a wrapper function named `to_uppercase_extern` that invokes 32 | /// the `to_uppercase` function on a different essa-rs node. The generated 33 | /// `to_uppercase_extern` function has the following signature: 34 | /// 35 | /// ```ignore 36 | /// fn to_uppercase_extern(String) -> Result>; 37 | /// ``` 38 | #[proc_macro_attribute] 39 | pub fn essa_wrap(attr: TokenStream, item: TokenStream) -> TokenStream { 40 | // convert from `TokenStream` to `TokenStream2`, which is used by the 41 | // `syn` crate 42 | let attr = TokenStream2::from(attr); 43 | let item = TokenStream2::from(item); 44 | // generate the additional wrapper functions 45 | let generated = essa_wrap_impl(attr, &item).unwrap_or_else(|err| err.to_compile_error()); 46 | // output the original item again, plus the generated functions 47 | let tokens = quote! { 48 | #item 49 | #generated 50 | }; 51 | // convert the type back from `TokenStream2` to `TokenStream` 52 | tokens.into() 53 | } 54 | 55 | /// Generates the wrapper functions for the annotated function. 56 | fn essa_wrap_impl(attr: TokenStream2, item: &TokenStream2) -> syn::Result { 57 | // parse the arguments given to the `essa_wrap` macro 58 | let attr_parsed: MetaNameValue = syn::parse2(attr.clone()) 59 | .map_err(|e| syn::Error::new(e.span(), "expected `name = \"...\"` argument"))?; 60 | // we require a `name` argument 61 | if attr_parsed 62 | .path 63 | .get_ident() 64 | .map(|i| i.to_string() != "name") 65 | .unwrap_or(true) 66 | { 67 | return Err(err(attr_parsed.path, "expected argument `name`")); 68 | } 69 | 70 | // parse the annotated item as a function declaration 71 | let function: ItemFn = syn::parse2(item.clone()).map_err(|_| { 72 | err( 73 | attr, 74 | "the #[essa_wrap] attribute is only supported on functions", 75 | ) 76 | })?; 77 | let function_name = function.sig.ident; 78 | 79 | // generate a new identifier for the wrapper function from the `name` 80 | // argument 81 | let remote_call_function_name: Ident = { 82 | let s = match &attr_parsed.lit { 83 | syn::Lit::Str(s) => s.value(), 84 | _ => return Err(err(attr_parsed.lit, "expected string")), 85 | }; 86 | syn::parse_str(&s).map_err(|e| syn::Error::new(attr_parsed.lit.span(), e))? 87 | }; 88 | // generate the API documentation for the new wrapper function 89 | let remote_call_function_name_doc = format!( 90 | "Calls the [`{}`] function asynchronously on a remote essa-rs node.", 91 | function_name 92 | ); 93 | 94 | // collect all arguments that the annotated function takes 95 | let mut args = Vec::new(); 96 | let mut args_call = Vec::new(); 97 | for arg in function.sig.inputs { 98 | let pattern = match arg { 99 | syn::FnArg::Receiver(r) => { 100 | return Err(err(r, "argument not supported by #[essa_wrap] attribute")) 101 | } 102 | syn::FnArg::Typed(a) => a, 103 | }; 104 | let ident = match *pattern.pat { 105 | syn::Pat::Ident(i) => i, 106 | other => { 107 | return Err(err( 108 | other, 109 | "pattern matching in arguments is not supported by #[essa_wrap] macro", 110 | )) 111 | } 112 | }; 113 | let ty = *pattern.ty; 114 | args.push(quote!(#ident: #ty)); 115 | args_call.push(quote!(#ident)); 116 | } 117 | 118 | // generate a struct to hold all the function arguments 119 | let arg_struct_name = Ident::new( 120 | &format!("__essaArgs__{}", function_name), 121 | function_name.span(), 122 | ); 123 | let arg_struct = quote! { 124 | #[allow(non_camel_case_types)] 125 | #[derive(essa_api::serde::Serialize, essa_api::serde::Deserialize)] 126 | #[serde(crate = "essa_api::serde")] 127 | struct #arg_struct_name { 128 | #(#args),* 129 | } 130 | }; 131 | 132 | // generate a C-compatible wrapper for the function that uses the 133 | // `essa_api` functions for getting the arguments and setting the return 134 | // value 135 | let wrapper_function_name_str = format!("__essa_wrapper__{}", function_name); 136 | let wrapper_function_name = Ident::new(&wrapper_function_name_str, function_name.span()); 137 | let wrapper_function = quote! { 138 | #[doc(hidden)] 139 | #[no_mangle] 140 | pub extern "C" fn #wrapper_function_name(args_raw_len: usize) { 141 | // get the serialized arguments from the essa-rs runtime 142 | let args_raw = essa_api::get_args_raw(args_raw_len).unwrap(); 143 | 144 | // try to deserialize the arguments 145 | let args: #arg_struct_name = essa_api::bincode::deserialize(&args_raw) 146 | .expect("failed to deserialize args"); 147 | 148 | // invoke the function annotated with the `essa_wrap` macro 149 | let result_val = #function_name( #(args.#args_call),* ); 150 | 151 | // serialize the result and register it with the essa-rs runtime 152 | let result_serialized = essa_api::bincode::serialize(&result_val) 153 | .expect("failed to serialize result"); 154 | essa_api::set_result(&result_serialized).unwrap(); 155 | } 156 | }; 157 | 158 | // generate a wrapper to call the annotated function on a remote node 159 | let func_ret = match function.sig.output { 160 | syn::ReturnType::Default => quote!(()), 161 | syn::ReturnType::Type(_, ty) => quote! { #ty }, 162 | }; 163 | let remote_call_wrapper = quote! { 164 | #[doc = #remote_call_function_name_doc] 165 | pub fn #remote_call_function_name(#(#args),*) 166 | -> Result, essa_api::EssaResult> 167 | { 168 | // initialize the argument struct and serialize it 169 | let args = #arg_struct_name { #(#args_call),* }; 170 | let args_serialized = essa_api::bincode::serialize(&args) 171 | .map_err(|_| essa_api::EssaResult::UnknownError)?; 172 | // call the function on a remote node 173 | let result_handle = 174 | essa_api::call_function(#wrapper_function_name_str, &args_serialized)?; 175 | Ok(essa_api::RemoteFunctionResult::new(result_handle)) 176 | } 177 | }; 178 | 179 | Ok(quote! { 180 | #arg_struct 181 | #wrapper_function 182 | #remote_call_wrapper 183 | }) 184 | } 185 | 186 | /// Generate a new `syn::Error` spanned to the given tokens. 187 | fn err(tokens: impl quote::ToTokens, message: impl Display) -> syn::Error { 188 | syn::Error::new_spanned(tokens, message) 189 | } 190 | -------------------------------------------------------------------------------- /api/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! The essa-rs API enables WebAssembly functions to perform external calls 2 | //! and provides access to a key-value store. 3 | 4 | #![warn(missing_docs)] 5 | 6 | use std::{thread, time::Duration}; 7 | 8 | use crate::c_api::{ 9 | essa_get_args, essa_get_lattice_data, essa_get_lattice_len, essa_put_lattice, essa_set_result, 10 | }; 11 | use anna_api::{ClientKey, LatticeValue}; 12 | use c_api::{essa_call, essa_get_result, essa_get_result_len}; 13 | 14 | /// Re-export the dependencies on serde and bincode to allow downstream 15 | /// crates to use the exact same version. 16 | /// 17 | /// These reexports are required for the generated code of the `essa_wrap` 18 | /// macro. 19 | pub use {bincode, serde}; 20 | pub use {essa_macros::essa_wrap, remote_result::RemoteFunctionResult}; 21 | 22 | pub mod c_api; 23 | #[doc(hidden)] 24 | pub mod remote_result; 25 | 26 | /// Requests the function arguments from the runtime as a serialized byte 27 | /// array. 28 | /// 29 | /// The runtime cannot write the arguments into the memory of the WASM 30 | /// module directly since this requires a prior memory allocation. For this 31 | /// reason, the runtime only passes the number of needed bytes as WASM 32 | /// argument when invoking a function. Through this function, the actual 33 | /// arguments can be requested from the runtime. 34 | /// 35 | /// This function is an abstraction over [`c_api::essa_get_args`]. 36 | pub fn get_args_raw(args_raw_len: usize) -> Result, EssaResult> { 37 | let mut args = vec![0; args_raw_len]; 38 | let result = unsafe { essa_get_args(args.as_mut_ptr(), args.len()) }; 39 | match EssaResult::try_from(result) { 40 | Ok(EssaResult::Ok) => Ok(args), 41 | Ok(other) => Err(other), 42 | Err(unknown) => panic!("unknown essaResult variant `{}`", unknown), 43 | } 44 | } 45 | 46 | /// Sets the given byte array as the function's result. 47 | /// 48 | /// We cannot use normal WASM return values for this since only primitive 49 | /// types are supported as return values. For this reason, functions 50 | /// should serialize their return value and then call `set_result` 51 | /// to register the return value with the runtime. Only a single return 52 | /// value must be set for each function. 53 | /// 54 | /// This function is an abstraction over [`c_api::essa_set_result`]. 55 | pub fn set_result(result_serialized: &[u8]) -> Result<(), EssaResult> { 56 | let result = unsafe { essa_set_result(result_serialized.as_ptr(), result_serialized.len()) }; 57 | match EssaResult::try_from(result) { 58 | Ok(EssaResult::Ok) => Ok(()), 59 | Ok(other) => Err(other), 60 | Err(unknown) => panic!("unknown essaResult variant `{}`", unknown), 61 | } 62 | } 63 | 64 | /// Invokes the specified function on a different node. 65 | /// 66 | /// The `args` argument specifies a byte array that should be passed to 67 | /// the external function as arguments. This is typically a serialized 68 | /// struct. 69 | /// 70 | /// This function is an abstraction over [`c_api::essa_call`]. 71 | pub fn call_function(function_name: &str, args: &[u8]) -> Result { 72 | let mut result_handle = 0; 73 | let result = unsafe { 74 | essa_call( 75 | function_name.as_ptr(), 76 | function_name.len(), 77 | args.as_ptr(), 78 | args.len(), 79 | &mut result_handle, 80 | ) 81 | }; 82 | 83 | match result { 84 | i if i == EssaResult::Ok as i32 => Ok(ResultHandle(result_handle)), 85 | other => return Err(other.try_into().unwrap()), 86 | } 87 | } 88 | 89 | /// Handle to retrieve an asynchronous result of a remote function call. 90 | /// 91 | /// To wait on the associated result, use [`wait`](Self::wait). 92 | pub struct ResultHandle(usize); 93 | 94 | impl ResultHandle { 95 | /// Tries to read the lattice value stored for given key from the key-value 96 | /// store. 97 | pub fn wait(self) -> Result, EssaResult> { 98 | let mut value_len = 0; 99 | let result = unsafe { essa_get_result_len(self.0, &mut value_len) }; 100 | let len = match EssaResult::try_from(result) { 101 | Ok(EssaResult::Ok) => value_len, 102 | Ok(other) => return Err(other), 103 | Err(unknown) => panic!("unknown EssaResult variant `{}`", unknown), 104 | }; 105 | 106 | let mut value = vec![0u8; len]; 107 | let result = 108 | unsafe { essa_get_result(self.0, value.as_mut_ptr(), value.len(), &mut value_len) }; 109 | match result { 110 | i if i == EssaResult::Ok as i32 => Ok(value), 111 | other => return Err(other.try_into().unwrap()), 112 | } 113 | } 114 | } 115 | 116 | /// Tries to read the lattice value stored for given key from the key-value 117 | /// store. 118 | pub fn kvs_try_get(key: &ClientKey) -> Result, EssaResult> { 119 | let mut value_len = 0; 120 | let result = unsafe { essa_get_lattice_len(key.as_ptr(), key.len(), &mut value_len) }; 121 | let len = match EssaResult::try_from(result) { 122 | Ok(EssaResult::Ok) => value_len, 123 | Ok(EssaResult::NotFound) => return Ok(None), 124 | Ok(other) => return Err(other), 125 | Err(unknown) => panic!("unknown essaResult variant `{}`", unknown), 126 | }; 127 | 128 | let mut value_serialized = vec![0u8; len]; 129 | let result = unsafe { 130 | essa_get_lattice_data( 131 | key.as_ptr(), 132 | key.len(), 133 | value_serialized.as_mut_ptr(), 134 | value_serialized.len(), 135 | &mut value_len, 136 | ) 137 | }; 138 | let value_serialized = match result { 139 | i if i == EssaResult::Ok as i32 => value_serialized, 140 | other => return Err(other.try_into().unwrap()), 141 | }; 142 | 143 | bincode::deserialize(&value_serialized) 144 | .map(Some) 145 | .map_err(|_| EssaResult::InvalidResult) 146 | } 147 | 148 | /// Read the lattice value stored for given key from the key-value store. 149 | /// 150 | /// Blocks until the requested value exists. 151 | pub fn kvs_get(key: &ClientKey) -> Result { 152 | loop { 153 | match kvs_try_get(key) { 154 | // TODO: do a proper wait instead of busy-looping 155 | Ok(None) | Err(EssaResult::NotFound) => { 156 | // wait a bit, then retry 157 | thread::sleep(Duration::from_millis(100)); 158 | } 159 | Ok(Some(value)) => break Ok(value), 160 | Err(err) => break Err(err), 161 | } 162 | } 163 | } 164 | 165 | /// Stores the given byte array under the given key in the key-value-store. 166 | /// 167 | /// This function is a safe abstraction over the [`c_api::essa_put_lattice`] function. 168 | pub fn kvs_put(key: &ClientKey, value: &LatticeValue) -> Result<(), EssaResult> { 169 | let serialized = bincode::serialize(&value).map_err(|_| EssaResult::UnknownError)?; 170 | 171 | let result = unsafe { 172 | essa_put_lattice( 173 | key.as_ptr(), 174 | key.len(), 175 | serialized.as_ptr(), 176 | serialized.len(), 177 | ) 178 | }; 179 | match EssaResult::try_from(result) { 180 | Ok(EssaResult::Ok) => Ok(()), 181 | Ok(other) => return Err(other), 182 | Err(unknown) => panic!("unknown essaResult variant `{}`", unknown), 183 | } 184 | } 185 | 186 | /// Errors that can occur when interacting with the essa runtime. 187 | #[derive(Debug, num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] 188 | #[repr(i32)] 189 | #[non_exhaustive] 190 | pub enum EssaResult { 191 | /// Finished successfully, equivalent to `Ok(())`. 192 | Ok = 0, 193 | /// Unspecified error. 194 | UnknownError = -1, 195 | /// The requested external function is not known to the runtime. 196 | NoSuchFunction = -2, 197 | /// A supplied memory buffer was not large enough. 198 | BufferTooSmall = -3, 199 | /// The requested value was not found, e.g. in the KVS. 200 | NotFound = -4, 201 | /// An invoked external function has an invalid signature. 202 | InvalidFunctionSignature = -5, 203 | /// A WASM function returned without a prior call to [`set_result`]. 204 | NoResult = -6, 205 | /// Failed to deserialize the result of a called WASM function. 206 | InvalidResult = -8, 207 | } 208 | 209 | impl std::fmt::Display for EssaResult { 210 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 211 | let s = match self { 212 | Self::Ok => "finished successfully", 213 | Self::UnknownError => "unspecified error", 214 | Self::NoSuchFunction => "the requested external function is not known to the runtime", 215 | Self::BufferTooSmall => "a supplied memory buffer was not large enough", 216 | Self::NotFound => "the requested value was not found, e.g. in the KVS", 217 | Self::InvalidFunctionSignature => { 218 | "an invoked external function has an invalid signature" 219 | } 220 | Self::NoResult => "a WASM function returned without a prior call to `set_result`", 221 | Self::InvalidResult => "failed to deserialize the result of a called WASM function", 222 | }; 223 | f.write_str(s) 224 | } 225 | } 226 | 227 | impl std::error::Error for EssaResult {} 228 | -------------------------------------------------------------------------------- /api/src/c_api.rs: -------------------------------------------------------------------------------- 1 | //! Specifies the API and ABI for communication between a WASM module and the runtime. 2 | //! 3 | //! Many different programming languages can be compiled to WebAssembly (WASM), 4 | //! so we need to define a stable ABI that is supported in all potential 5 | //! languages. For this reason, we define the interface as C-compatible 6 | //! functions. This makes the interface very low-level and cumbersome to use 7 | //! directly, so we create more high-level wrapper functions on top of it. 8 | //! 9 | //! ## Memory Management 10 | //! 11 | //! WASM and the wasmtime runtime have native support for passing primitive 12 | //! types as arguments and return values between the runtime and a WASM module 13 | //! (through host-defined functions). For passing larger data, pointers need to 14 | //! be used, which leads to some challenges: 15 | //! 16 | //! - WASM modules are sandboxed, so they cannot access any memory outside of 17 | //! their assigned memory region. So pointers to runtime memory are not 18 | //! possible. 19 | //! - The WASM runtime can read and write the sandboxed memory of any WASM 20 | //! module it executes without restrictions. However, it does not know 21 | //! which memory areas of the WASM module are unused and can be safely 22 | //! used for writing data. 23 | //! - The WASM modules manage their own memory and have their own custom 24 | //! memory allocator, which is not usable by the runtime. 25 | //! 26 | //! We decided on the following design around these challenges: 27 | //! 28 | //! - Pass data from WASM module to runtime: The WASM module passes the 29 | //! pointer (i.e. the offset) and the length of the data as two integer 30 | //! arguments to the host function defined by the runtime. Using these 31 | //! arguments, the runtime can read the data out of the sandbox. However, 32 | //! it must not access the data anymore after returning from the host 33 | //! function because the WASM module might deallocate that memory again. 34 | //! - Passing data from runtime to WASM module is a three-step process. First, 35 | //! the runtime tells the WASM module the size of the data it wants to pass 36 | //! (e.g. the length of a byte array). The WASM module then allocates a 37 | //! a suitable location in its sandboxed memory and passes the corresponding 38 | //! pointer to the runtime. The runtime then writes the data into the sandbox 39 | //! at the given location. 40 | //! 41 | //! An alternative approach for passing data from the runtime to WASM modules 42 | //! is to pre-allocate a larger amount of buffer memory that is then "owned" 43 | //! by the runtime. This way, the runtime can write data directly to WASM 44 | //! accessible-memory, which should be more performant. The drawbacks are 45 | //! the additional memory consumption of the buffer memory and that the buffer 46 | //! size limits the data size. 47 | //! 48 | //! ## Future Possibilities 49 | //! 50 | //! There is ongoing work on 51 | //! [WASM interface types](https://github.com/WebAssembly/interface-types/blob/main/proposals/interface-types/Explainer.md), 52 | //! which would allow us to define the API in a higher-level way. The most 53 | //! advanced project in this area seems to be 54 | //! [`wit-bindgen`](https://github.com/bytecodealliance/wit-bindgen), 55 | //! which generates bindings for different languages based on a `.wit` 56 | //! descriptor file. The ABI details are abstracted away this way. 57 | 58 | // Essa core functions: getting function arguments, setting the function 59 | // result, and calling a remote function. 60 | #[link(wasm_import_module = "host")] 61 | extern "C" { 62 | /// Requests the function arguments from the runtime as a serialized byte 63 | /// array. 64 | /// 65 | /// The runtime cannot write the arguments into the memory of the WASM 66 | /// module directly since this requires a prior memory allocation. For this 67 | /// reason, the runtime only passes the number of needed bytes as WASM 68 | /// argument when invoking a function. The function should then allocate 69 | /// the requested amount of memory and then call `essa_get_args` to copy 70 | /// the actual arguments into that allocated memory. 71 | /// 72 | /// The `args_len` must be identical to the raw argument byte length 73 | /// passed as function argument. 74 | /// 75 | /// The return value is an [`EssaResult`](crate::EssaResult). 76 | /// 77 | /// The [`crate::get_args_raw`] function provides a higher-level abstraction 78 | /// for this function. 79 | pub fn essa_get_args(args_ptr: *mut u8, args_len: usize) -> i32; 80 | 81 | /// Sets the given byte array as the function's result. 82 | /// 83 | /// We cannot use normal WASM return values for this since only primitive 84 | /// types are supported as return values. For this reason, functions 85 | /// should serialize their return value and then call `essa_set_result` 86 | /// to register the return value with the runtime. Only a single return 87 | /// value must be set for each function. 88 | /// 89 | /// The return value is an [`EssaResult`](crate::EssaResult). 90 | /// 91 | /// The [`crate::set_result`] function provides a higher-level abstraction 92 | /// for this function. 93 | pub fn essa_set_result(ptr: *const u8, len: usize) -> i32; 94 | 95 | /// Invokes the specified function on a different node. 96 | /// 97 | /// Since WASM has no native support for strings, we need to pass the 98 | /// function as a pointer and length pair. The function name must be valid 99 | /// UTF-8 and refer to a valid function registered with the runtime. 100 | /// 101 | /// The `args` argument specifies a byte array that should be passed to 102 | /// the external function as arguments. This is typically a serialized 103 | /// struct. 104 | /// 105 | /// The `return_handle` is filled with an unique handle that can be passed 106 | /// to `essa_get_result` to get the function's result value. 107 | /// 108 | /// The return value of this function is an [`EssaResult`](crate::EssaResult). 109 | /// 110 | /// The [`crate::call_function`] function provides a higher-level abstraction 111 | /// for this function. 112 | pub fn essa_call( 113 | function_name_ptr: *const u8, 114 | function_name_len: usize, 115 | args_ptr: *const u8, 116 | args_len: usize, 117 | result_handle: *mut usize, 118 | ) -> i32; 119 | 120 | /// Waits for the function result associated with the given handle and 121 | /// returns its length. 122 | /// 123 | /// The caller should use the returned length to allocate a large-enough 124 | /// memory region and then call [`essa_get_result`] to fill in the result. 125 | /// 126 | /// The return value is an [`EssaResult`](crate::EssaResult). 127 | pub fn essa_get_result_len(result_handle: usize, value_len_ptr: *mut usize) -> i32; 128 | 129 | /// Waits for the function result associated with the given handle and 130 | /// writes it to the given memory location. 131 | /// 132 | /// This function can only be used once for each handle. 133 | /// 134 | /// The `value_capacity` must be at least 135 | /// as large as the value length returned by [`essa_get_result_len`]. The 136 | /// `value_ptr` must point to a memory allocation of that size. The 137 | /// runtime will write the value to that pointer and return the actual 138 | /// value length through `value_len_ptr`. 139 | /// 140 | /// The return value is an [`EssaResult`](crate::EssaResult). 141 | pub fn essa_get_result( 142 | result_handle: usize, 143 | value_ptr: *mut u8, 144 | value_capacity: usize, 145 | value_len_ptr: *mut usize, 146 | ) -> i32; 147 | } 148 | 149 | // Functions for reading and writing data to a key-value store. 150 | // 151 | // (Right now this is based on `anna-rs` and lattice values, but other KVS 152 | // stores should work in a similar way.) 153 | #[link(wasm_import_module = "host")] 154 | extern "C" { 155 | /// Stores the given byte array under the given key in the key-value-store. 156 | /// 157 | /// The key must be valid UTF-8. The value must be a serialized _lattice_ 158 | /// type. 159 | /// 160 | /// The return value is an [`EssaResult`](crate::EssaResult). 161 | /// 162 | /// The [`crate::kvs_put`] function provides a safe, higher-level 163 | /// abstraction for this function. 164 | pub fn essa_put_lattice( 165 | key_ptr: *const u8, 166 | key_len: usize, 167 | value_ptr: *const u8, 168 | value_len: usize, 169 | ) -> i32; 170 | 171 | /// Reads the given key from the KVS and returns the size of the 172 | /// corresponding value. 173 | /// 174 | /// The key must be valid UTF-8. The value size is written to the given 175 | /// `value_len_ptr`. To read the actual value, allocate a large-enough 176 | /// memory region and then call `essa_get_lattice_data`. 177 | /// 178 | /// The return value is an [`EssaResult`](crate::EssaResult). 179 | pub fn essa_get_lattice_len( 180 | key_ptr: *const u8, 181 | key_len: usize, 182 | value_len_ptr: *mut usize, 183 | ) -> i32; 184 | 185 | /// Reads the lattice value stored for given key from the KVS. 186 | /// 187 | /// The key must be valid UTF-8. The `value_capacity` must be at least 188 | /// as large as the value length returned by [`essa_get_lattice_len`]. The 189 | /// `value_ptr` must point to a memory allocation of that size. The 190 | /// runtime will write the value to that pointer and return the actual 191 | /// value length through `value_len_ptr`. 192 | /// 193 | /// The return value is an [`EssaResult`](crate::EssaResult). 194 | /// 195 | /// The [`crate::kvs_try_get`] and [`crate::kvs_get`] functions provide 196 | /// safe, higher-level abstractions for this function. 197 | pub fn essa_get_lattice_data( 198 | key_ptr: *const u8, 199 | key_len: usize, 200 | value_ptr: *mut u8, 201 | value_capacity: usize, 202 | value_len_ptr: *mut usize, 203 | ) -> i32; 204 | } 205 | -------------------------------------------------------------------------------- /function-executor/src/main.rs: -------------------------------------------------------------------------------- 1 | //! Essa function execution nodes are responsible for running given WASM 2 | //! modules/functions. 3 | //! 4 | //! In an essa-rs system, there are typically multiple nodes, each running the 5 | //! essa function executor binary. The essa scheduler dispatches incoming run 6 | //! requests evenly across the available function executor nodes. 7 | 8 | #![warn(missing_docs)] 9 | 10 | use anna::{ 11 | anna_default_zenoh_prefix, 12 | lattice::Lattice, 13 | nodes::{request_cluster_info, ClientNode}, 14 | store::LatticeValue, 15 | topics::RoutingThread, 16 | ClientKey, 17 | }; 18 | use anyhow::Context; 19 | use essa_common::{ 20 | essa_default_zenoh_prefix, executor_run_function_subscribe_topic, executor_run_module_topic, 21 | }; 22 | use flume::RecvError; 23 | use std::{ 24 | sync::Arc, 25 | thread, 26 | time::{Duration, Instant}, 27 | }; 28 | use uuid::Uuid; 29 | use zenoh::prelude::{sync::SyncResolve, SplitBuffer}; 30 | 31 | #[cfg(all(feature = "wasmedge_executor", feature = "wasmtime_executor"))] 32 | compile_error!( 33 | "Both `wasmedge_executor` and `wasmtime_executor` features are enabled, but \ 34 | only one WASM runtime is supported at a time" 35 | ); 36 | #[cfg(all(not(feature = "wasmedge_executor"), not(feature = "wasmtime_executor")))] 37 | compile_error!("Either the `wasmedge_executor` or the `wasmtime_executor` feature must be enabled"); 38 | #[cfg(feature = "wasmedge_executor")] 39 | mod wasmedge; 40 | #[cfg(feature = "wasmtime_executor")] 41 | mod wasmtime; 42 | 43 | /// Starts a essa function executor. 44 | /// 45 | /// The given ID must be an unique identifier for this executor instance, i.e. 46 | /// there must not be another active executor instance with the same id. 47 | #[derive(argh::FromArgs, Debug, Clone)] 48 | struct Args { 49 | #[argh(positional)] 50 | id: u32, 51 | } 52 | 53 | fn main() -> anyhow::Result<()> { 54 | if let Err(err) = set_up_logger() { 55 | eprintln!( 56 | "ERROR: {:?}", 57 | anyhow::anyhow!(err).context("Failed to set up logger") 58 | ); 59 | } 60 | 61 | let args: Args = argh::from_env(); 62 | 63 | let zenoh = Arc::new( 64 | zenoh::open(zenoh::config::Config::default()) 65 | .res() 66 | .map_err(|e| anyhow::anyhow!(e)) 67 | .context("failed to connect to zenoh")?, 68 | ); 69 | let zenoh_prefix = essa_default_zenoh_prefix().to_owned(); 70 | 71 | // listen for function call requests in a separate thread 72 | { 73 | let zenoh = zenoh.clone(); 74 | let zenoh_prefix = zenoh_prefix.clone(); 75 | let args = args.clone(); 76 | 77 | thread::spawn(move || { 78 | if let Err(err) = function_call_receive_loop(args, zenoh, zenoh_prefix) { 79 | log::error!("{:?}", err) 80 | } 81 | }); 82 | } 83 | 84 | // listen for module run requests 85 | module_receive_loop(args, zenoh, zenoh_prefix)?; 86 | 87 | Ok(()) 88 | } 89 | 90 | /// Creates a new client connected to the `anna-rs` key-value store. 91 | fn new_anna_client(zenoh: Arc) -> Result { 92 | let anna_zenoh_prefix = anna_default_zenoh_prefix().to_owned(); 93 | 94 | // request the available routing nodes from an anna seed node 95 | let cluster_info = smol::block_on(request_cluster_info(&zenoh, &anna_zenoh_prefix)) 96 | .map_err(|e| anyhow::anyhow!(e)) 97 | .context("failed to request cluster info from seed node")?; 98 | 99 | let routing_threads: Vec<_> = cluster_info 100 | .routing_node_ids 101 | .into_iter() 102 | .map(|node_id| RoutingThread { 103 | node_id, 104 | // TODO: use anna config file to get number of threads per 105 | // routing node 106 | thread_id: 0, 107 | }) 108 | .collect(); 109 | 110 | // connect to anna as a new client node 111 | let mut anna = ClientNode::new( 112 | Uuid::new_v4().to_string(), 113 | 0, 114 | routing_threads, 115 | Duration::from_secs(10), 116 | zenoh, 117 | anna_zenoh_prefix, 118 | ) 119 | .map_err(eyre_to_anyhow) 120 | .context("failed to connect to anna")?; 121 | smol::block_on(anna.init_tcp_connections()) 122 | .map_err(eyre_to_anyhow) 123 | .context("failed to init TCP connections in anna client")?; 124 | Ok(anna) 125 | } 126 | 127 | /// Listens for incoming module run requests from the scheduler. 128 | fn module_receive_loop( 129 | args: Args, 130 | zenoh: Arc, 131 | zenoh_prefix: String, 132 | ) -> anyhow::Result<()> { 133 | let mut new_modules = zenoh 134 | .declare_subscriber(executor_run_module_topic(args.id, &zenoh_prefix)) 135 | .res() 136 | .map_err(|e| anyhow::anyhow!(e)) 137 | .context("failed to subscribe to new modules")?; 138 | 139 | loop { 140 | match new_modules.receiver.recv() { 141 | Ok(change) => { 142 | let wasm_bytes = change.value.payload.contiguous().into_owned(); 143 | 144 | // start a new function executor instance to compile the module 145 | // and run its main function 146 | let executor = FunctionExecutor { 147 | zenoh: zenoh.clone(), 148 | zenoh_prefix: zenoh_prefix.clone(), 149 | anna: new_anna_client(zenoh.clone())?, 150 | }; 151 | std::thread::spawn(move || { 152 | let start = Instant::now(); 153 | if let Err(err) = executor 154 | .run_module(wasm_bytes) 155 | .context("failed to run module") 156 | { 157 | log::error!("{:?}", err); 158 | } 159 | log::info!("Module run finished in {:?}", Instant::now() - start); 160 | }); 161 | } 162 | Err(RecvError::Disconnected) => break, 163 | } 164 | } 165 | Ok(()) 166 | } 167 | 168 | /// Listens for incoming function run requests. 169 | fn function_call_receive_loop( 170 | args: Args, 171 | zenoh: Arc, 172 | zenoh_prefix: String, 173 | ) -> anyhow::Result<()> { 174 | let mut function_calls = zenoh 175 | .declare_queryable(executor_run_function_subscribe_topic( 176 | args.id, 177 | &zenoh_prefix, 178 | )) 179 | .res() 180 | .map_err(|e| anyhow::anyhow!(e)) 181 | .context("failed to subscribe to new modules")?; 182 | 183 | loop { 184 | match function_calls.receiver.recv() { 185 | Ok(query) => { 186 | // start a new function executor instance to run the requested 187 | // function 188 | let executor = FunctionExecutor { 189 | zenoh: zenoh.clone(), 190 | zenoh_prefix: zenoh_prefix.clone(), 191 | anna: new_anna_client(zenoh.clone())?, 192 | }; 193 | std::thread::spawn(move || { 194 | if let Err(err) = executor 195 | .handle_function_call(query) 196 | .context("failed to run function call") 197 | { 198 | log::error!("{:?}", err); 199 | } 200 | }); 201 | } 202 | Err(RecvError::Disconnected) => break, 203 | } 204 | } 205 | Ok(()) 206 | } 207 | 208 | /// Responsible for handling module/function run requests. 209 | struct FunctionExecutor { 210 | zenoh: Arc, 211 | zenoh_prefix: String, 212 | anna: ClientNode, 213 | } 214 | 215 | /// Store the given value in the key-value store. 216 | fn kvs_put(key: ClientKey, value: LatticeValue, anna: &mut ClientNode) -> anyhow::Result<()> { 217 | let start = Instant::now(); 218 | 219 | smol::block_on(anna.put(key, value)) 220 | .map_err(eyre_to_anyhow) 221 | .context("put failed")?; 222 | 223 | let put_latency = (Instant::now() - start).as_millis(); 224 | if put_latency >= 100 { 225 | log::trace!("high kvs_put latency: {}ms", put_latency); 226 | } 227 | 228 | Ok(()) 229 | } 230 | 231 | /// Get the given value in the key-value store. 232 | /// 233 | /// Returns an error if the requested key does not exist in the KVS. 234 | fn kvs_get(key: ClientKey, anna: &mut ClientNode) -> anyhow::Result { 235 | smol::block_on(anna.get(key)) 236 | .map_err(eyre_to_anyhow) 237 | .context("get failed") 238 | } 239 | 240 | // Get the function arguments from the key-value store 241 | // 242 | // TODO: we only need them once, so it might make more sense to pass 243 | // them directly in the message 244 | fn get_args(args_key: ClientKey, anna: &mut ClientNode) -> Result, anyhow::Error> { 245 | let mut retries = 0; 246 | let value = loop { 247 | match kvs_get(args_key.clone(), anna).context("failed to get args from kvs") { 248 | Ok(value) => break value, 249 | Err(_) if retries < 5 => { 250 | retries += 1; 251 | } 252 | Err(err) => return Err(err), 253 | } 254 | }; 255 | 256 | Ok(value 257 | .into_lww() 258 | .map_err(eyre_to_anyhow) 259 | .context("args is not a LWW lattice")? 260 | .into_revealed() 261 | .into_value()) 262 | } 263 | 264 | /// Get the compiled WASM module from the key-value store 265 | fn get_module(module_key: ClientKey, anna: &mut ClientNode) -> Result, anyhow::Error> { 266 | Ok(kvs_get(module_key, anna) 267 | .context("failed to get module from kvs")? 268 | .into_lww() 269 | .map_err(eyre_to_anyhow) 270 | .context("module is not a LWW lattice")? 271 | .into_revealed() 272 | .into_value()) 273 | } 274 | 275 | /// Copy of `essa-api::EssaResult`, must be kept in sync. 276 | /// 277 | /// TODO: move to a separate crate to remove the duplication 278 | #[derive(Debug)] 279 | #[repr(i32)] 280 | #[allow(missing_docs)] 281 | pub enum EssaResult { 282 | Ok = 0, 283 | UnknownError = -1, 284 | NoSuchFunction = -2, 285 | BufferTooSmall = -3, 286 | NotFound = -4, 287 | InvalidFunctionSignature = -5, 288 | NoResult = -6, 289 | InvalidResult = -8, 290 | } 291 | 292 | /// Transforms an [`eyre::Report`] to an [`anyhow::Error`]. 293 | fn eyre_to_anyhow(err: eyre::Report) -> anyhow::Error { 294 | let err = Box::::from(err); 295 | anyhow::anyhow!(err) 296 | } 297 | 298 | /// Set up the `log` crate. 299 | fn set_up_logger() -> Result<(), fern::InitError> { 300 | fern::Dispatch::new() 301 | .format(|out, message, record| { 302 | out.finish(format_args!( 303 | "{}[{}][{}] {}", 304 | chrono::Local::now().format("[%Y-%m-%d][%H:%M:%S]"), 305 | record.target(), 306 | record.level(), 307 | message 308 | )) 309 | }) 310 | .level(log::LevelFilter::Info) 311 | .level_for("zenoh", log::LevelFilter::Warn) 312 | .level_for("essa_function_executor", log::LevelFilter::Trace) 313 | .chain(std::io::stdout()) 314 | .chain(fern::log_file("function-executor.log")?) 315 | .apply()?; 316 | Ok(()) 317 | } 318 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [2022] [The Essa-rs Authors] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /function-executor/src/wasmtime.rs: -------------------------------------------------------------------------------- 1 | //! Use wasmtime as executor's runtime. 2 | 3 | #![warn(missing_docs)] 4 | 5 | use crate::{get_args, get_module, kvs_get, kvs_put, EssaResult, FunctionExecutor}; 6 | use anna::{lattice::LastWriterWinsLattice, nodes::ClientNode, ClientKey}; 7 | use anyhow::{bail, Context}; 8 | use essa_common::scheduler_function_call_topic; 9 | use flume::Receiver; 10 | use std::{collections::HashMap, sync::Arc}; 11 | use uuid::Uuid; 12 | use wasmtime::{Caller, Engine, Extern, Linker, Module, Store, ValType}; 13 | use wasmtime_wasi::{WasiCtx, WasiCtxBuilder}; 14 | use zenoh::{ 15 | prelude::{sync::SyncResolve, Sample, SplitBuffer}, 16 | query::Reply, 17 | queryable::Query, 18 | }; 19 | 20 | impl FunctionExecutor { 21 | /// Runs the given WASM module. 22 | pub fn run_module(mut self, wasm_bytes: Vec) -> anyhow::Result<()> { 23 | log::info!("Start running WASM module"); 24 | 25 | // compile the WASM module 26 | let engine = Engine::default(); 27 | let module = Module::new(&engine, &wasm_bytes).context("failed to load wasm module")?; 28 | 29 | // store the compiled module in the key-value store under an 30 | // unique key (to avoid recompiling the module on future function 31 | // calls) 32 | log::info!("Storing compiled wasm module in anna KVS"); 33 | let module_key: ClientKey = Uuid::new_v4().to_string().into(); 34 | let value = LastWriterWinsLattice::new_now( 35 | module.serialize().context("failed to serialize module")?, 36 | ) 37 | .into(); 38 | kvs_put(module_key.clone(), value, &mut self.anna)?; 39 | 40 | let (mut store, linker) = set_up_module( 41 | engine, 42 | &module, 43 | module_key, 44 | vec![], 45 | self.zenoh.clone(), 46 | self.zenoh_prefix.clone(), 47 | self.anna, 48 | )?; 49 | 50 | // get the default function (i.e. the `main` function) 51 | let default_func = linker 52 | .get_default(&mut store, "") 53 | .context("module has no default function")? 54 | .typed::<(), ()>(&store) 55 | .context("default function has invalid type")?; 56 | 57 | log::info!("Starting default function of wasm module"); 58 | 59 | // call the function 60 | // 61 | // TODO: wasmtime allows fine-grained control of the execution, e.g. 62 | // periodic interrupts. We might want to use these features to guard 63 | // against malicious or buggy user programs. For example, an infinite 64 | // loop should not waste CPU time forever. 65 | default_func 66 | .call(&mut store, ()) 67 | .context("default function failed")?; 68 | 69 | Ok(()) 70 | } 71 | 72 | /// Runs the given function of a already compiled WASM module. 73 | pub fn handle_function_call(mut self, query: Query) -> anyhow::Result<()> { 74 | let mut topic_split = query.key_expr().as_str().split('/'); 75 | let args_key = ClientKey::from(topic_split.next_back().context("no args key in topic")?); 76 | let function_name = topic_split 77 | .next_back() 78 | .context("no function key in topic")?; 79 | let module_key = 80 | ClientKey::from(topic_split.next_back().context("no module key in topic")?); 81 | 82 | let module = get_module(module_key.clone(), &mut self.anna)?; 83 | 84 | let args = get_args(args_key, &mut self.anna)?; 85 | let args_len = u32::try_from(args.len()).unwrap(); 86 | 87 | // deserialize and set up WASM module 88 | let engine = Engine::default(); 89 | let module = 90 | unsafe { Module::deserialize(&engine, module).expect("failed to deserialize module") }; 91 | let (mut store, linker) = set_up_module( 92 | engine, 93 | &module, 94 | module_key, 95 | args, 96 | self.zenoh, 97 | self.zenoh_prefix.clone(), 98 | self.anna, 99 | )?; 100 | 101 | // get the function that we've been requested to call 102 | let func = linker 103 | .get(&mut store, "", function_name) 104 | .and_then(|e| e.into_func()) 105 | .with_context(|| format!("module has no function `{}`", function_name))? 106 | .typed::<(i32,), ()>(&store) 107 | .context("default function has invalid type")?; 108 | 109 | func.call(&mut store, (args_len as i32,)) 110 | .context("function trapped")?; 111 | 112 | // store the function's result into the key value store under 113 | // the requested key 114 | // 115 | // TODO: The result is needed only once, so it might make more sense to 116 | // send it as a message instead of storing it in the KVS. This would 117 | // also improve performance since the receiver would no longer need to 118 | // busy-wait on the result key in the KVS anymore. 119 | let mut host_state = store.into_data(); 120 | if let Some(result_value) = host_state.function_result.take() { 121 | let selector = query.key_expr().clone(); 122 | query 123 | .reply(Ok(Sample::new(selector, result_value))) 124 | .res() 125 | .map_err(|err| anyhow::anyhow!(err))?; 126 | 127 | Ok(()) 128 | } else { 129 | Err(anyhow::anyhow!("no result")) 130 | } 131 | } 132 | } 133 | 134 | /// Link the host functions and WASI abstractions into the WASM module. 135 | fn set_up_module( 136 | engine: Engine, 137 | module: &Module, 138 | module_key: ClientKey, 139 | args: Vec, 140 | zenoh: Arc, 141 | zenoh_prefix: String, 142 | anna: ClientNode, 143 | ) -> Result<(Store, Linker), anyhow::Error> { 144 | let wasi = WasiCtxBuilder::new() 145 | // TODO: we probably want to write to a log file or an anna key instead 146 | .inherit_stdout() 147 | .inherit_stderr() 148 | .build(); 149 | let mut store = Store::new( 150 | &engine, 151 | HostState { 152 | wasi, 153 | module: module.clone(), 154 | module_key, 155 | function_result: None, 156 | next_result_handle: 1, 157 | results: HashMap::new(), 158 | result_receivers: HashMap::new(), 159 | zenoh, 160 | zenoh_prefix, 161 | anna, 162 | }, 163 | ); 164 | let mut linker = Linker::new(&engine); 165 | 166 | // link in the essa host functions 167 | linker 168 | .func_wrap( 169 | "host", 170 | "essa_get_args", 171 | move |mut caller: Caller<'_, HostState>, buf_ptr: u32, buf_len: u32| { 172 | // the given buffer must be large enough to hold `args` 173 | if buf_len < u32::try_from(args.len()).unwrap() { 174 | return Ok(EssaResult::BufferTooSmall as i32); 175 | } 176 | 177 | // write `args` to the given memory region in the sandbox 178 | let mem = match caller.get_export("memory") { 179 | Some(Extern::Memory(mem)) => mem, 180 | _ => bail!("failed to find host memory"), 181 | }; 182 | mem.write(&mut caller, buf_ptr as usize, args.as_slice()) 183 | .context("val ptr/len out of bounds")?; 184 | 185 | Ok(EssaResult::Ok as i32) 186 | }, 187 | ) 188 | .context("failed to create essa_get_args")?; 189 | linker 190 | .func_wrap( 191 | "host", 192 | "essa_set_result", 193 | |mut caller: Caller<'_, HostState>, buf_ptr: u32, buf_len: u32| { 194 | // copy the given memory region out of the sandbox 195 | let mem = match caller.get_export("memory") { 196 | Some(Extern::Memory(mem)) => mem, 197 | _ => bail!("failed to find host memory"), 198 | }; 199 | let mut buf = vec![0; buf_len as usize]; 200 | mem.read(&mut caller, buf_ptr as usize, &mut buf) 201 | .context("ptr/len out of bounds")?; 202 | 203 | // set the function result in the host state 204 | caller.data_mut().function_result = Some(buf); 205 | 206 | Ok(EssaResult::Ok as i32) 207 | }, 208 | ) 209 | .context("failed to create essa_set_result")?; 210 | linker 211 | .func_wrap( 212 | "host", 213 | "essa_call", 214 | |caller: Caller<'_, HostState>, 215 | function_name_ptr: u32, 216 | function_name_len: u32, 217 | serialized_args_ptr: u32, 218 | serialized_arg_len: u32, 219 | result_handle_ptr: u32| { 220 | essa_call_wrapper( 221 | caller, 222 | function_name_ptr, 223 | function_name_len, 224 | serialized_args_ptr, 225 | serialized_arg_len, 226 | result_handle_ptr, 227 | ) 228 | .map(|r| r as i32) 229 | }, 230 | ) 231 | .context("failed to create essa_call host function")?; 232 | linker 233 | .func_wrap( 234 | "host", 235 | "essa_get_result_len", 236 | |caller: Caller<'_, HostState>, handle: u32, value_len_ptr: u32| { 237 | essa_get_result_len_wrapper(caller, handle, value_len_ptr).map(|r| r as i32) 238 | }, 239 | ) 240 | .context("failed to create essa_get_result_len host function")?; 241 | linker 242 | .func_wrap( 243 | "host", 244 | "essa_get_result", 245 | |caller: Caller<'_, HostState>, 246 | handle: u32, 247 | value_ptr: u32, 248 | value_capacity: u32, 249 | value_len_ptr: u32| { 250 | essa_get_result_wrapper(caller, handle, value_ptr, value_capacity, value_len_ptr) 251 | .map(|r| r as i32) 252 | }, 253 | ) 254 | .context("failed to create essa_get_result host function")?; 255 | linker 256 | .func_wrap( 257 | "host", 258 | "essa_put_lattice", 259 | |caller: Caller<'_, HostState>, 260 | key_ptr: u32, 261 | key_len: u32, 262 | value_ptr: u32, 263 | value_len: u32| { 264 | essa_put_lattice_wrapper(caller, key_ptr, key_len, value_ptr, value_len) 265 | .map(|r| r as i32) 266 | }, 267 | ) 268 | .context("failed to create essa_put_lattice host function")?; 269 | linker 270 | .func_wrap( 271 | "host", 272 | "essa_get_lattice_len", 273 | |caller: Caller<'_, HostState>, key_ptr: u32, key_len: u32, value_len_ptr: u32| { 274 | essa_get_lattice_len_wrapper(caller, key_ptr, key_len, value_len_ptr) 275 | .map(|r| r as i32) 276 | }, 277 | ) 278 | .context("failed to create essa_get_lattice host function")?; 279 | linker 280 | .func_wrap( 281 | "host", 282 | "essa_get_lattice_data", 283 | |caller: Caller<'_, HostState>, 284 | key_ptr: u32, 285 | key_len: u32, 286 | value_ptr: u32, 287 | value_capacity: u32, 288 | value_len_ptr: u32| { 289 | essa_get_lattice_data_wrapper( 290 | caller, 291 | key_ptr, 292 | key_len, 293 | value_ptr, 294 | value_capacity, 295 | value_len_ptr, 296 | ) 297 | .map(|r| r as i32) 298 | }, 299 | ) 300 | .context("failed to create essa_get_lattice_data host function")?; 301 | 302 | // add WASI functionality (e.g. stdin/stdout access) 303 | wasmtime_wasi::add_to_linker(&mut linker, |state: &mut HostState| &mut state.wasi) 304 | .context("failed to add wasi functionality to linker")?; 305 | 306 | // link the host and WASI functions with the user-supplied WASM module 307 | linker 308 | .module(&mut store, "", module) 309 | .context("failed to add module to linker")?; 310 | 311 | Ok((store, linker)) 312 | } 313 | 314 | /// Host function for calling the specified function on a remote node. 315 | fn essa_call_wrapper( 316 | mut caller: Caller, 317 | function_name_ptr: u32, 318 | function_name_len: u32, 319 | serialized_args_ptr: u32, 320 | serialized_args_len: u32, 321 | result_handle_ptr: u32, 322 | ) -> anyhow::Result { 323 | // Use our `caller` context to get the memory export of the 324 | // module which called this host function. 325 | let mem = match caller.get_export("memory") { 326 | Some(Extern::Memory(mem)) => mem, 327 | _ => bail!("failed to find host memory"), 328 | }; 329 | 330 | // read the function name from the WASM sandbox 331 | let function_name = { 332 | let mut data = vec![0u8; function_name_len as usize]; 333 | mem.read(&caller, function_name_ptr as usize, &mut data) 334 | .context("function name ptr/len out of bounds")?; 335 | String::from_utf8(data).context("function name not valid utf8")? 336 | }; 337 | // read the serialized function arguments from the WASM sandbox 338 | let args = { 339 | let mut data = vec![0u8; serialized_args_len as usize]; 340 | mem.read(&caller, serialized_args_ptr as usize, &mut data) 341 | .context("function name ptr/len out of bounds")?; 342 | data 343 | }; 344 | 345 | // trigger the external function call 346 | match caller.data_mut().essa_call(function_name, args) { 347 | Ok(reply) => { 348 | let host_state = caller.data_mut(); 349 | let handle = host_state.next_result_handle; 350 | host_state.next_result_handle += 1; 351 | host_state.result_receivers.insert(handle, reply); 352 | 353 | // write handle 354 | mem.write( 355 | &mut caller, 356 | result_handle_ptr as usize, 357 | &handle.to_le_bytes(), 358 | ) 359 | .context("result_handle_ptr out of bounds")?; 360 | 361 | Ok(EssaResult::Ok) 362 | } 363 | Err(err) => Ok(err), 364 | } 365 | } 366 | 367 | fn essa_get_result_len_wrapper( 368 | mut caller: Caller, 369 | handle: u32, 370 | val_len_ptr: u32, 371 | ) -> anyhow::Result { 372 | // Use our `caller` context to learn about the memory export of the 373 | // module which called this host function. 374 | let mem = match caller.get_export("memory") { 375 | Some(Extern::Memory(mem)) => mem, 376 | _ => bail!("failed to find host memory"), 377 | }; 378 | 379 | // get the corresponding value from the KVS 380 | match caller.data_mut().get_result(handle) { 381 | Ok(value) => { 382 | let len = value.len(); 383 | // write the length of the value into the sandbox 384 | // 385 | // We cannot write the value directly because the WASM module 386 | // needs to allocate some space for the (dynamically-sized) value 387 | // first. 388 | mem.write( 389 | &mut caller, 390 | val_len_ptr as usize, 391 | &u32::try_from(len).unwrap().to_le_bytes(), 392 | ) 393 | .context("val_len_ptr out of bounds")?; 394 | 395 | Ok(EssaResult::Ok) 396 | } 397 | Err(err) => Ok(err), 398 | } 399 | } 400 | 401 | fn essa_get_result_wrapper( 402 | mut caller: Caller, 403 | handle: u32, 404 | val_ptr: u32, 405 | val_capacity: u32, 406 | val_len_ptr: u32, 407 | ) -> anyhow::Result { 408 | // Use our `caller` context to learn about the memory export of the 409 | // module which called this host function. 410 | let mem = match caller.get_export("memory") { 411 | Some(Extern::Memory(mem)) => mem, 412 | _ => bail!("failed to find host memory"), 413 | }; 414 | 415 | // get the corresponding value from the KVS 416 | match caller.data_mut().get_result(handle) { 417 | Ok(value) => { 418 | if value.len() > val_capacity as usize { 419 | Ok(EssaResult::BufferTooSmall) 420 | } else { 421 | // write the value into the sandbox 422 | mem.write(&mut caller, val_ptr as usize, &value) 423 | .context("val ptr/len out of bounds")?; 424 | // write the length of the value 425 | mem.write( 426 | &mut caller, 427 | val_len_ptr as usize, 428 | &u32::try_from(value.len()).unwrap().to_le_bytes(), 429 | ) 430 | .context("val_len_ptr out of bounds")?; 431 | 432 | caller.data_mut().remove_result(handle); 433 | 434 | Ok(EssaResult::Ok) 435 | } 436 | } 437 | Err(err) => Ok(err), 438 | } 439 | } 440 | 441 | /// Host function for storing a given lattice value into the KVS. 442 | fn essa_put_lattice_wrapper( 443 | mut caller: Caller, 444 | key_ptr: u32, 445 | key_len: u32, 446 | value_ptr: u32, 447 | value_len: u32, 448 | ) -> anyhow::Result { 449 | // Use our `caller` context to learn about the memory export of the 450 | // module which called this host function. 451 | let mem = match caller.get_export("memory") { 452 | Some(Extern::Memory(mem)) => mem, 453 | _ => bail!("failed to find host memory"), 454 | }; 455 | // read out and parse the KVS key 456 | let key = { 457 | let mut data = vec![0u8; key_len as usize]; 458 | mem.read(&caller, key_ptr as usize, &mut data) 459 | .context("key ptr/len out of bounds")?; 460 | String::from_utf8(data) 461 | .context("key is not valid utf8")? 462 | .into() 463 | }; 464 | // read out the value that should be stored 465 | let value = { 466 | let mut data = vec![0u8; value_len as usize]; 467 | mem.read(&caller, value_ptr as usize, &mut data) 468 | .context("value ptr/len out of bounds")?; 469 | data 470 | }; 471 | 472 | match caller.data_mut().put_lattice(&key, &value) { 473 | Ok(()) => Ok(EssaResult::Ok), 474 | Err(other) => Ok(other), 475 | } 476 | } 477 | 478 | /// Host function for reading the length of value stored under a specific key 479 | /// in the KVS. 480 | fn essa_get_lattice_len_wrapper( 481 | mut caller: Caller, 482 | key_ptr: u32, 483 | key_len: u32, 484 | val_len_ptr: u32, 485 | ) -> anyhow::Result { 486 | // Use our `caller` context to learn about the memory export of the 487 | // module which called this host function. 488 | let mem = match caller.get_export("memory") { 489 | Some(Extern::Memory(mem)) => mem, 490 | _ => bail!("failed to find host memory"), 491 | }; 492 | // read out and parse the KVS key 493 | let key = { 494 | let mut data = vec![0u8; key_len as usize]; 495 | mem.read(&caller, key_ptr as usize, &mut data) 496 | .context("key ptr/len out of bounds")?; 497 | String::from_utf8(data) 498 | .context("key is not valid utf8")? 499 | .into() 500 | }; 501 | // get the corresponding value from the KVS 502 | match caller.data_mut().get_lattice(&key) { 503 | Ok(value) => { 504 | // write the length of the value into the sandbox 505 | // 506 | // We cannot write the value directly because the WASM module 507 | // needs to allocate some space for the (dynamically-sized) value 508 | // first. 509 | mem.write( 510 | &mut caller, 511 | val_len_ptr as usize, 512 | &u32::try_from(value.len()).unwrap().to_le_bytes(), 513 | ) 514 | .context("val_len_ptr out of bounds")?; 515 | 516 | Ok(EssaResult::Ok) 517 | } 518 | Err(err) => Ok(err), 519 | } 520 | } 521 | 522 | /// Host function for reading a specific value from the KVS. 523 | fn essa_get_lattice_data_wrapper( 524 | mut caller: Caller, 525 | key_ptr: u32, 526 | key_len: u32, 527 | val_ptr: u32, 528 | val_capacity: u32, 529 | val_len_ptr: u32, 530 | ) -> anyhow::Result { 531 | // Use our `caller` context to learn about the memory export of the 532 | // module which called this host function. 533 | let mem = match caller.get_export("memory") { 534 | Some(Extern::Memory(mem)) => mem, 535 | _ => bail!("failed to find host memory"), 536 | }; 537 | // read out and parse the KVS key 538 | let key = { 539 | let mut data = vec![0u8; key_len as usize]; 540 | mem.read(&caller, key_ptr as usize, &mut data) 541 | .context("key ptr/len out of bounds")?; 542 | String::from_utf8(data) 543 | .context("key is not valid utf8")? 544 | .into() 545 | }; 546 | // get the corresponding value from the KVS 547 | match caller.data_mut().get_lattice(&key) { 548 | Ok(value) => { 549 | if value.len() > val_capacity as usize { 550 | Ok(EssaResult::BufferTooSmall) 551 | } else { 552 | // write the value into the sandbox 553 | mem.write(&mut caller, val_ptr as usize, value.as_slice()) 554 | .context("val ptr/len out of bounds")?; 555 | // write the length of the value 556 | mem.write( 557 | &mut caller, 558 | val_len_ptr as usize, 559 | &u32::try_from(value.len()).unwrap().to_le_bytes(), 560 | ) 561 | .context("val_len_ptr out of bounds")?; 562 | 563 | Ok(EssaResult::Ok) 564 | } 565 | } 566 | Err(err) => Ok(err), 567 | } 568 | } 569 | 570 | /// Stores all the information needed during execution. 571 | /// 572 | /// The `wasmtime` crate gives this struct as an additional argument to all 573 | /// host functions, which makes it possible to keep state between across 574 | /// host function invocations. 575 | struct HostState { 576 | wasi: WasiCtx, 577 | /// The compiled WASM module. 578 | module: Module, 579 | /// The KVS key under which a serialized version of the compiled WASM 580 | /// module is stored. 581 | module_key: ClientKey, 582 | /// The result value of this function, set through the `essa_set_result` 583 | /// host function. 584 | function_result: Option>, 585 | 586 | next_result_handle: u32, 587 | result_receivers: HashMap>, 588 | results: HashMap>>, 589 | 590 | zenoh: Arc, 591 | zenoh_prefix: String, 592 | anna: ClientNode, 593 | } 594 | 595 | impl HostState { 596 | /// Calls the given function on a node and returns the reply receiver for 597 | /// the corresponding result. 598 | fn essa_call( 599 | &mut self, 600 | function_name: String, 601 | args: Vec, 602 | ) -> Result, EssaResult> { 603 | // get the requested function and check its signature 604 | let func = self 605 | .module 606 | .get_export(&function_name) 607 | .and_then(|e| e.func().cloned()) 608 | .ok_or(EssaResult::NoSuchFunction)?; 609 | if func.params().collect::>().as_slice() != [ValType::I32] 610 | || !func.results().collect::>().as_slice().is_empty() 611 | { 612 | return Err(EssaResult::InvalidFunctionSignature); 613 | } 614 | 615 | // store args in kvs 616 | let args_key: ClientKey = Uuid::new_v4().to_string().into(); 617 | kvs_put( 618 | args_key.clone(), 619 | LastWriterWinsLattice::new_now(args).into(), 620 | &mut self.anna, 621 | ) 622 | .map_err(|_| EssaResult::UnknownError)?; 623 | 624 | // trigger the function call on a remote node 625 | let reply = call_function_extern( 626 | self.module_key.clone(), 627 | function_name, 628 | args_key, 629 | self.zenoh.clone(), 630 | &self.zenoh_prefix, 631 | ) 632 | .unwrap(); 633 | 634 | Ok(reply) 635 | } 636 | 637 | /// Stores the given serialized `LattiveValue` in the KVS. 638 | fn put_lattice(&mut self, key: &ClientKey, value: &[u8]) -> Result<(), EssaResult> { 639 | let value = bincode::deserialize(value).map_err(|_| EssaResult::UnknownError)?; 640 | kvs_put(self.with_prefix(key), value, &mut self.anna).map_err(|_| EssaResult::UnknownError) 641 | } 642 | 643 | /// Reads the `LattiveValue` at the specified key from the KVS serializes it. 644 | fn get_lattice(&mut self, key: &ClientKey) -> Result, EssaResult> { 645 | kvs_get(self.with_prefix(key), &mut self.anna) 646 | .map_err(|_| EssaResult::NotFound) 647 | .and_then(|v| bincode::serialize(&v).map_err(|_| EssaResult::UnknownError)) 648 | } 649 | 650 | fn with_prefix(&self, key: &ClientKey) -> ClientKey { 651 | format!("{}/data/{}", self.module_key, key).into() 652 | } 653 | 654 | fn get_result(&mut self, handle: u32) -> Result>, EssaResult> { 655 | match self.results.entry(handle) { 656 | std::collections::hash_map::Entry::Occupied(entry) => Ok(entry.get().clone()), 657 | std::collections::hash_map::Entry::Vacant(entry) => { 658 | if let Some(result) = self.result_receivers.remove(&handle) { 659 | let reply = result.recv().map_err(|_| EssaResult::UnknownError)?; 660 | let value = reply 661 | .sample 662 | .map_err(|_| EssaResult::UnknownError)? 663 | .value 664 | .payload 665 | .contiguous() 666 | .into_owned(); 667 | let value = entry.insert(Arc::new(value)); 668 | Ok(value.clone()) 669 | } else { 670 | Err(EssaResult::NotFound) 671 | } 672 | } 673 | } 674 | } 675 | 676 | fn remove_result(&mut self, handle: u32) { 677 | self.results.remove(&handle); 678 | } 679 | } 680 | 681 | /// Call the specfied function on a remote node. 682 | fn call_function_extern( 683 | module_key: ClientKey, 684 | function_name: String, 685 | args_key: ClientKey, 686 | zenoh: Arc, 687 | zenoh_prefix: &str, 688 | ) -> anyhow::Result> { 689 | let topic = scheduler_function_call_topic(zenoh_prefix, &module_key, &function_name, &args_key); 690 | 691 | // send the request to the scheduler node 692 | let reply = zenoh 693 | .get(topic) 694 | .res() 695 | .map_err(|e| anyhow::anyhow!(e)) 696 | .context("failed to send function call request to scheduler")?; 697 | 698 | Ok(reply) 699 | } 700 | -------------------------------------------------------------------------------- /function-executor/src/wasmedge.rs: -------------------------------------------------------------------------------- 1 | //! Use WasmEdge as executor's runtime. 2 | 3 | #![warn(missing_docs)] 4 | 5 | use crate::{get_args, get_module, kvs_get, kvs_put, EssaResult, FunctionExecutor}; 6 | use anna::{lattice::LastWriterWinsLattice, nodes::ClientNode, ClientKey}; 7 | use anyhow::Context; 8 | use essa_common::scheduler_function_call_topic; 9 | use flume::Receiver; 10 | use std::{ 11 | collections::HashMap, 12 | sync::{Arc, Mutex}, 13 | }; 14 | use uuid::Uuid; 15 | use wasmedge_sdk::{ 16 | config::{CommonConfigOptions, ConfigBuilder, HostRegistrationConfigOptions}, 17 | error::HostFuncError, 18 | CallingFrame, ExternalInstanceType, Func, ImportObjectBuilder, Instance, Memory, Module, 19 | ValType, Vm, VmBuilder, WasmValue, 20 | }; 21 | use zenoh::{ 22 | prelude::{sync::SyncResolve, Sample, SplitBuffer}, 23 | query::Reply, 24 | queryable::Query, 25 | }; 26 | 27 | impl FunctionExecutor { 28 | /// Runs the given WASM module. 29 | pub fn run_module(mut self, wasm_bytes: Vec) -> anyhow::Result<()> { 30 | log::info!("Start running WASM module"); 31 | 32 | // compile the WASM module 33 | let module = Module::from_bytes(None, &wasm_bytes).context("failed to load wasm module")?; 34 | 35 | // store the compiled module in the key-value store under an 36 | // unique key (to avoid recompiling the module on future function 37 | // calls) 38 | log::info!("Storing compiled wasm module in anna KVS"); 39 | let module_key: ClientKey = Uuid::new_v4().to_string().into(); 40 | let value = LastWriterWinsLattice::new_now( 41 | // TODO: serialize the WasmEdge module into bytes, require WasmEdge core support. 42 | wasm_bytes, 43 | ) 44 | .into(); 45 | kvs_put(module_key.clone(), value, &mut self.anna)?; 46 | 47 | let mut host_state = HostState { 48 | module: module.clone(), 49 | module_key, 50 | function_result: None, 51 | next_result_handle: 1, 52 | results: HashMap::new(), 53 | result_receivers: HashMap::new(), 54 | zenoh: self.zenoh.clone(), 55 | zenoh_prefix: self.zenoh_prefix.clone(), 56 | anna: self.anna, 57 | }; 58 | 59 | let mut instance_wrapper = 60 | InstanceWrapper::new()?.register(module, &mut host_state, vec![])?; 61 | 62 | instance_wrapper.call_default()?; 63 | Ok(()) 64 | } 65 | 66 | /// Runs the given function of a already compiled WASM module. 67 | pub fn handle_function_call(mut self, query: Query) -> anyhow::Result<()> { 68 | let mut topic_split = query.key_expr().as_str().split('/'); 69 | let args_key = ClientKey::from(topic_split.next_back().context("no args key in topic")?); 70 | let function_name = topic_split 71 | .next_back() 72 | .context("no function key in topic")?; 73 | let module_key = 74 | ClientKey::from(topic_split.next_back().context("no module key in topic")?); 75 | 76 | let module = get_module(module_key.clone(), &mut self.anna)?; 77 | 78 | let args = get_args(args_key, &mut self.anna)?; 79 | let args_len = u32::try_from(args.len()).unwrap(); 80 | 81 | // deserialize and set up WASM module 82 | // TODO: deserialize the bytes into WasmEdge module, require WasmEdge core support. 83 | let module = Module::from_bytes(None, &module).context("failed to load wasm module")?; 84 | 85 | let mut host_state = HostState { 86 | module: module.clone(), 87 | module_key, 88 | function_result: None, 89 | next_result_handle: 1, 90 | results: HashMap::new(), 91 | result_receivers: HashMap::new(), 92 | zenoh: self.zenoh.clone(), 93 | zenoh_prefix: self.zenoh_prefix.clone(), 94 | anna: self.anna, 95 | }; 96 | 97 | let mut instance_wrapper = 98 | InstanceWrapper::new()?.register(module, &mut host_state, args)?; 99 | 100 | instance_wrapper.call(function_name, args_len as i32)?; 101 | 102 | // store the function's result into the key value store under 103 | // the requested key 104 | // 105 | // TODO: The result is needed only once, so it might make more sense to 106 | // send it as a message instead of storing it in the KVS. This would 107 | // also improve performance since the receiver would no longer need to 108 | // busy-wait on the result key in the KVS anymore. 109 | if let Some(result_value) = host_state.function_result.take() { 110 | let selector = query.key_expr().clone(); 111 | query.reply(Ok(Sample::new(selector, result_value))); 112 | 113 | Ok(()) 114 | } else { 115 | Err(anyhow::anyhow!("no result")) 116 | } 117 | } 118 | } 119 | 120 | // TODO: in order to use external variables in closure, it is currently necessary to use 121 | // TODO: some tricks (in an unsafe way) to avoid the current shortcomings of WasmEdge. 122 | struct HostContext { 123 | memory: *mut Option, 124 | host_state: *mut HostState, 125 | } 126 | unsafe impl Send for HostContext {} 127 | 128 | /// A wrapper that stores some important data structures of WasmEdge runtime. 129 | struct InstanceWrapper { 130 | vm: Vm, 131 | memory: Option, 132 | } 133 | 134 | impl InstanceWrapper { 135 | fn new() -> Result { 136 | // create a config 137 | let config = ConfigBuilder::new(CommonConfigOptions::default()) 138 | .with_host_registration_config(HostRegistrationConfigOptions::default().wasi(true)) 139 | .build() 140 | .context("failed to create a wasmedge config")?; 141 | // create a vm 142 | let vm = VmBuilder::default() 143 | .with_config(config) 144 | .build() 145 | .context("failed to create a wasmedge vm")?; 146 | 147 | Ok(InstanceWrapper { vm, memory: None }) 148 | } 149 | 150 | /// Register the import module, wasi module, active module to the WasmEdge Store. 151 | fn register( 152 | mut self, 153 | module: Module, 154 | host_state: &mut HostState, 155 | args: Vec, 156 | ) -> Result { 157 | // essa_get_args 158 | let host_context = Arc::new(Mutex::new(HostContext { 159 | memory: &mut self.memory as *mut Option, 160 | host_state: host_state as *mut HostState, 161 | })); 162 | let essa_get_args = move |_: CallingFrame, 163 | inputs: Vec| 164 | -> Result, HostFuncError> { 165 | let buf_ptr = inputs[0].to_i32() as u32; 166 | let buf_len = inputs[1].to_i32() as u32; 167 | 168 | // the given buffer must be large enough to hold `args` 169 | if buf_len < u32::try_from(args.len()).unwrap() { 170 | return Ok(vec![WasmValue::from_i32(EssaResult::BufferTooSmall as i32)]); 171 | } 172 | 173 | // write `args` to the given memory region in the sandbox 174 | let mem = unsafe { &mut *(host_context.lock().unwrap().memory) } 175 | .as_mut() 176 | .unwrap(); 177 | mem.write(args.clone(), buf_ptr).unwrap(); 178 | 179 | Ok(vec![WasmValue::from_i32(EssaResult::Ok as i32)]) 180 | }; 181 | 182 | // essa_set_result 183 | let host_context = Arc::new(Mutex::new(HostContext { 184 | memory: &mut self.memory as *mut Option, 185 | host_state: host_state as *mut HostState, 186 | })); 187 | let essa_set_result = move |_: CallingFrame, 188 | inputs: Vec| 189 | -> Result, HostFuncError> { 190 | let buf_ptr = inputs[0].to_i32() as u32; 191 | let buf_len = inputs[1].to_i32() as u32; 192 | 193 | // copy the given memory region out of the sandbox 194 | let mem = unsafe { &mut *(host_context.lock().unwrap().memory) } 195 | .as_mut() 196 | .unwrap(); 197 | let buf = mem.read(buf_ptr, buf_len).unwrap(); 198 | 199 | let host_state = unsafe { &mut *(host_context.lock().unwrap().host_state) }; 200 | host_state.function_result = Some(buf); 201 | 202 | Ok(vec![WasmValue::from_i32(EssaResult::Ok as i32)]) 203 | }; 204 | 205 | // essa_call 206 | let host_context = Arc::new(Mutex::new(HostContext { 207 | memory: &mut self.memory as *mut Option, 208 | host_state: host_state as *mut HostState, 209 | })); 210 | let essa_call = move |_: CallingFrame, 211 | inputs: Vec| 212 | -> Result, HostFuncError> { 213 | let function_name_ptr = inputs[0].to_i32() as u32; 214 | let function_name_len = inputs[1].to_i32() as u32; 215 | let serialized_args_ptr = inputs[2].to_i32() as u32; 216 | let serialized_arg_len = inputs[3].to_i32() as u32; 217 | let result_handle_ptr = inputs[4].to_i32() as u32; 218 | 219 | let memory = unsafe { &mut *(host_context.lock().unwrap().memory) } 220 | .as_mut() 221 | .unwrap(); 222 | let host_state = unsafe { &mut *(host_context.lock().unwrap().host_state) }; 223 | 224 | let res = essa_call_wrapper( 225 | memory, 226 | host_state, 227 | function_name_ptr, 228 | function_name_len, 229 | serialized_args_ptr, 230 | serialized_arg_len, 231 | result_handle_ptr, 232 | ) 233 | .unwrap(); 234 | 235 | Ok(vec![WasmValue::from_i32(res as i32)]) 236 | }; 237 | 238 | // essa_get_result_len 239 | let host_context = Arc::new(Mutex::new(HostContext { 240 | memory: &mut self.memory as *mut Option, 241 | host_state: host_state as *mut HostState, 242 | })); 243 | let essa_get_result_len = move |_: CallingFrame, 244 | inputs: Vec| 245 | -> Result, HostFuncError> { 246 | let handle = inputs[0].to_i32() as u32; 247 | let value_len_ptr = inputs[1].to_i32() as u32; 248 | 249 | let memory = unsafe { &mut *(host_context.lock().unwrap().memory) } 250 | .as_mut() 251 | .unwrap(); 252 | let host_state = unsafe { &mut *(host_context.lock().unwrap().host_state) }; 253 | 254 | let res = 255 | essa_get_result_len_wrapper(memory, host_state, handle, value_len_ptr).unwrap(); 256 | 257 | Ok(vec![WasmValue::from_i32(res as i32)]) 258 | }; 259 | 260 | // essa_get_result 261 | let host_context = Arc::new(Mutex::new(HostContext { 262 | memory: &mut self.memory as *mut Option, 263 | host_state: host_state as *mut HostState, 264 | })); 265 | let essa_get_result = move |_: CallingFrame, 266 | inputs: Vec| 267 | -> Result, HostFuncError> { 268 | let handle = inputs[0].to_i32() as u32; 269 | let value_ptr = inputs[1].to_i32() as u32; 270 | let value_capacity = inputs[2].to_i32() as u32; 271 | let value_len_ptr = inputs[3].to_i32() as u32; 272 | 273 | let memory = unsafe { &mut *(host_context.lock().unwrap().memory) } 274 | .as_mut() 275 | .unwrap(); 276 | let host_state = unsafe { &mut *(host_context.lock().unwrap().host_state) }; 277 | 278 | let res = essa_get_result_wrapper( 279 | memory, 280 | host_state, 281 | handle, 282 | value_ptr, 283 | value_capacity, 284 | value_len_ptr, 285 | ) 286 | .unwrap(); 287 | 288 | Ok(vec![WasmValue::from_i32(res as i32)]) 289 | }; 290 | 291 | // essa_put_lattice 292 | let host_context = Arc::new(Mutex::new(HostContext { 293 | memory: &mut self.memory as *mut Option, 294 | host_state: host_state as *mut HostState, 295 | })); 296 | let essa_put_lattice = move |_: CallingFrame, 297 | inputs: Vec| 298 | -> Result, HostFuncError> { 299 | let key_ptr = inputs[0].to_i32() as u32; 300 | let key_len = inputs[1].to_i32() as u32; 301 | let value_ptr = inputs[2].to_i32() as u32; 302 | let value_len = inputs[3].to_i32() as u32; 303 | 304 | let memory = unsafe { &mut *(host_context.lock().unwrap().memory) } 305 | .as_mut() 306 | .unwrap(); 307 | let host_state = unsafe { &mut *(host_context.lock().unwrap().host_state) }; 308 | 309 | let res = essa_put_lattice_wrapper( 310 | memory, host_state, key_ptr, key_len, value_ptr, value_len, 311 | ) 312 | .unwrap(); 313 | 314 | Ok(vec![WasmValue::from_i32(res as i32)]) 315 | }; 316 | 317 | // essa_get_lattice_len 318 | let host_context = Arc::new(Mutex::new(HostContext { 319 | memory: &mut self.memory as *mut Option, 320 | host_state: host_state as *mut HostState, 321 | })); 322 | let essa_get_lattice_len = move |_: CallingFrame, 323 | inputs: Vec| 324 | -> Result, HostFuncError> { 325 | let key_ptr = inputs[0].to_i32() as u32; 326 | let key_len = inputs[1].to_i32() as u32; 327 | let value_len_ptr = inputs[2].to_i32() as u32; 328 | 329 | let memory = unsafe { &mut *(host_context.lock().unwrap().memory) } 330 | .as_mut() 331 | .unwrap(); 332 | let host_state = unsafe { &mut *(host_context.lock().unwrap().host_state) }; 333 | 334 | let res = 335 | essa_get_lattice_len_wrapper(memory, host_state, key_ptr, key_len, value_len_ptr) 336 | .unwrap(); 337 | 338 | Ok(vec![WasmValue::from_i32(res as i32)]) 339 | }; 340 | 341 | // essa_get_lattice_data 342 | let host_context = Arc::new(Mutex::new(HostContext { 343 | memory: &mut self.memory as *mut Option, 344 | host_state: host_state as *mut HostState, 345 | })); 346 | let essa_get_lattice_data = move |_: CallingFrame, 347 | inputs: Vec| 348 | -> Result, HostFuncError> { 349 | let key_ptr = inputs[0].to_i32() as u32; 350 | let key_len = inputs[1].to_i32() as u32; 351 | let value_ptr = inputs[2].to_i32() as u32; 352 | let value_capacity = inputs[3].to_i32() as u32; 353 | let value_len_ptr = inputs[4].to_i32() as u32; 354 | 355 | let memory = unsafe { &mut *(host_context.lock().unwrap().memory) } 356 | .as_mut() 357 | .unwrap(); 358 | let host_state = unsafe { &mut *(host_context.lock().unwrap().host_state) }; 359 | 360 | let res = essa_get_lattice_data_wrapper( 361 | memory, 362 | host_state, 363 | key_ptr, 364 | key_len, 365 | value_ptr, 366 | value_capacity, 367 | value_len_ptr, 368 | ) 369 | .unwrap(); 370 | 371 | Ok(vec![WasmValue::from_i32(res as i32)]) 372 | }; 373 | 374 | // Register import module. 375 | let import = ImportObjectBuilder::new() 376 | .with_func::<(i32, i32), i32>("essa_get_args", essa_get_args)? 377 | .with_func::<(i32, i32), i32>("essa_set_result", essa_set_result)? 378 | .with_func::<(i32, i32, i32, i32, i32), i32>("essa_call", essa_call)? 379 | .with_func::<(i32, i32), i32>("essa_get_result_len", essa_get_result_len)? 380 | .with_func::<(i32, i32, i32, i32), i32>("essa_get_result", essa_get_result)? 381 | .with_func::<(i32, i32, i32, i32), i32>("essa_put_lattice", essa_put_lattice)? 382 | .with_func::<(i32, i32, i32), i32>("essa_get_lattice_len", essa_get_lattice_len)? 383 | .with_func::<(i32, i32, i32, i32, i32), i32>( 384 | "essa_get_lattice_data", 385 | essa_get_lattice_data, 386 | )? 387 | .build("host") 388 | .context("failed to create a ImportObject")?; 389 | self.vm = self 390 | .vm 391 | .register_import_module(import) 392 | .context("failed to register an import object into vm")?; 393 | 394 | // Register active module and get the instance. 395 | self.vm = self 396 | .vm 397 | .register_module(None, module) 398 | .context("failed to register an active module into vm")?; 399 | 400 | self.memory = Some( 401 | self.get_instance()? 402 | .memory("memory") 403 | .context("failed to find host memory")?, 404 | ); 405 | 406 | Ok(self) 407 | } 408 | 409 | // Call the “default function” of a module. 410 | fn call_default(&mut self) -> Result<(), anyhow::Error> { 411 | let func = get_default(self.get_instance()?).context("module has no default function")?; 412 | 413 | let func_ty = func.ty(); //.context("failed to get the function type")?; 414 | 415 | // Check the signature of the default function. 416 | if func_ty.args_len() != 0 || func_ty.returns_len() != 0 { 417 | return Err(anyhow::anyhow!("the default function has invalid type")); 418 | } 419 | 420 | log::info!("Starting default function of wasm module"); 421 | 422 | func.run(&mut self.vm.executor_mut(), []) 423 | .context("default function failed")?; 424 | 425 | Ok(()) 426 | } 427 | 428 | // Call the function with the `name`. 429 | fn call(&mut self, name: &str, args: i32) -> Result<(), anyhow::Error> { 430 | // get the function that we've been requested to call 431 | let func = self 432 | .get_instance()? 433 | .func(name) 434 | .with_context(|| format!("module has no function `{}`", name))?; 435 | 436 | let func_ty = func.ty(); 437 | 438 | // Check the signature of the function 439 | if func_ty 440 | .args() 441 | .context("failed to return the type of the arguments")? 442 | != [ValType::I32] 443 | || func_ty.returns_len() != 0 444 | { 445 | return Err(anyhow::anyhow!(format!( 446 | "the function `{}` has invalid type", 447 | name 448 | ))); 449 | } 450 | 451 | func.run(&mut self.vm.executor_mut(), vec![WasmValue::from_i32(args)]) 452 | .context("function trapped")?; 453 | 454 | Ok(()) 455 | } 456 | 457 | fn get_instance(&self) -> Result<&Instance, anyhow::Error> { 458 | self.vm 459 | .active_module() 460 | .context("failed to get the active module") 461 | } 462 | } 463 | 464 | /// Returns the "default export" of a WASM instance. 465 | fn get_default(instance: &Instance) -> Result { 466 | if let Ok(func) = instance.func("") { 467 | return Ok(func); 468 | } 469 | 470 | // For compatibility, also recognize "_start". 471 | if let Ok(func) = instance.func("_start") { 472 | return Ok(func); 473 | } 474 | 475 | // Otherwise return a no-op function. 476 | let func = |_: CallingFrame, _: Vec| -> Result, HostFuncError> { 477 | Ok(vec![]) 478 | }; 479 | Func::wrap::<(), ()>(func).context("failed to create wasmedge function") 480 | } 481 | 482 | /// Host function for calling the specified function on a remote node. 483 | fn essa_call_wrapper( 484 | memory: &mut Memory, 485 | host_state: &mut HostState, 486 | function_name_ptr: u32, 487 | function_name_len: u32, 488 | serialized_args_ptr: u32, 489 | serialized_args_len: u32, 490 | result_handle_ptr: u32, 491 | ) -> Result { 492 | // read the function name from the WASM sandbox 493 | let function_name = { 494 | let data = memory 495 | .read(function_name_ptr, function_name_len) 496 | .context("function name ptr/len out of bounds")?; 497 | String::from_utf8(data).context("function name not valid utf8")? 498 | }; 499 | // read the serialized function arguments from the WASM sandbox 500 | let args = memory 501 | .read(serialized_args_ptr, serialized_args_len) 502 | .context("function name ptr/len out of bounds")?; 503 | 504 | // trigger the external function call 505 | match host_state.essa_call(function_name, args) { 506 | Ok(reply) => { 507 | let handle = host_state.next_result_handle; 508 | host_state.next_result_handle += 1; 509 | host_state.result_receivers.insert(handle, reply); 510 | 511 | // write handle 512 | memory 513 | .write(handle.to_le_bytes(), result_handle_ptr) 514 | .context("result_handle_ptr out of bounds")?; 515 | 516 | Ok(EssaResult::Ok) 517 | } 518 | Err(err) => Ok(err), 519 | } 520 | } 521 | 522 | fn essa_get_result_len_wrapper( 523 | memory: &mut Memory, 524 | host_state: &mut HostState, 525 | handle: u32, 526 | val_len_ptr: u32, 527 | ) -> Result { 528 | // get the corresponding value from the KVS 529 | match host_state.get_result(handle) { 530 | Ok(value) => { 531 | let len = value.len(); 532 | // write the length of the value into the sandbox 533 | // 534 | // We cannot write the value directly because the WASM module 535 | // needs to allocate some space for the (dynamically-sized) value 536 | // first. 537 | memory 538 | .write(u32::try_from(len).unwrap().to_le_bytes(), val_len_ptr) 539 | .context("val_len_ptr out of bounds")?; 540 | 541 | Ok(EssaResult::Ok) 542 | } 543 | Err(err) => Ok(err), 544 | } 545 | } 546 | 547 | fn essa_get_result_wrapper( 548 | memory: &mut Memory, 549 | host_state: &mut HostState, 550 | handle: u32, 551 | val_ptr: u32, 552 | val_capacity: u32, 553 | val_len_ptr: u32, 554 | ) -> Result { 555 | // get the corresponding value from the KVS 556 | match host_state.get_result(handle) { 557 | Ok(value) => { 558 | if value.len() > val_capacity as usize { 559 | Ok(EssaResult::BufferTooSmall) 560 | } else { 561 | // write the value into the sandbox 562 | memory 563 | .write(unsafe { &*Arc::into_raw(value.clone()) }.clone(), val_ptr) 564 | .context("val ptr/len out of bounds")?; 565 | // write the length of the value 566 | memory 567 | .write( 568 | u32::try_from(value.len()).unwrap().to_le_bytes(), 569 | val_len_ptr, 570 | ) 571 | .context("val_len_ptr out of bounds")?; 572 | 573 | host_state.remove_result(handle); 574 | 575 | Ok(EssaResult::Ok) 576 | } 577 | } 578 | Err(err) => Ok(err), 579 | } 580 | } 581 | 582 | /// Host function for storing a given lattice value into the KVS. 583 | fn essa_put_lattice_wrapper( 584 | memory: &mut Memory, 585 | host_state: &mut HostState, 586 | key_ptr: u32, 587 | key_len: u32, 588 | value_ptr: u32, 589 | value_len: u32, 590 | ) -> Result { 591 | // read out and parse the KVS key 592 | let key = { 593 | let data = memory 594 | .read(key_ptr, key_len) 595 | .context("key ptr/len out of bounds")?; 596 | String::from_utf8(data) 597 | .context("key is not valid utf8")? 598 | .into() 599 | }; 600 | // read out the value that should be stored 601 | let value = memory 602 | .read(value_ptr, value_len) 603 | .context("value ptr/len out of bounds")?; 604 | 605 | match host_state.put_lattice(&key, &value) { 606 | Ok(()) => Ok(EssaResult::Ok), 607 | Err(other) => Ok(other), 608 | } 609 | } 610 | 611 | /// Host function for reading the length of value stored under a specific key 612 | /// in the KVS. 613 | fn essa_get_lattice_len_wrapper( 614 | memory: &mut Memory, 615 | host_state: &mut HostState, 616 | key_ptr: u32, 617 | key_len: u32, 618 | val_len_ptr: u32, 619 | ) -> Result { 620 | // read out and parse the KVS key 621 | let key = { 622 | let data = memory 623 | .read(key_ptr, key_len) 624 | .context("key ptr/len out of bounds")?; 625 | String::from_utf8(data) 626 | .context("key is not valid utf8")? 627 | .into() 628 | }; 629 | // get the corresponding value from the KVS 630 | match host_state.get_lattice(&key) { 631 | Ok(value) => { 632 | // write the length of the value into the sandbox 633 | // 634 | // We cannot write the value directly because the WASM module 635 | // needs to allocate some space for the (dynamically-sized) value 636 | // first. 637 | memory 638 | .write( 639 | u32::try_from(value.len()).unwrap().to_le_bytes(), 640 | val_len_ptr, 641 | ) 642 | .context("val_len_ptr out of bounds")?; 643 | 644 | Ok(EssaResult::Ok) 645 | } 646 | Err(err) => Ok(err), 647 | } 648 | } 649 | 650 | /// Host function for reading a specific value from the KVS. 651 | fn essa_get_lattice_data_wrapper( 652 | memory: &mut Memory, 653 | host_state: &mut HostState, 654 | key_ptr: u32, 655 | key_len: u32, 656 | val_ptr: u32, 657 | val_capacity: u32, 658 | val_len_ptr: u32, 659 | ) -> Result { 660 | // read out and parse the KVS key 661 | let key = { 662 | let data = memory 663 | .read(key_ptr, key_len) 664 | .context("key ptr/len out of bounds")?; 665 | String::from_utf8(data) 666 | .context("key is not valid utf8")? 667 | .into() 668 | }; 669 | // get the corresponding value from the KVS 670 | match host_state.get_lattice(&key) { 671 | Ok(value) => { 672 | if value.len() > val_capacity as usize { 673 | Ok(EssaResult::BufferTooSmall) 674 | } else { 675 | // write the value into the sandbox 676 | memory 677 | .write(value.clone(), val_ptr) 678 | .context("val ptr/len out of bounds")?; 679 | // write the length of the value 680 | memory 681 | .write( 682 | u32::try_from(value.len()).unwrap().to_le_bytes(), 683 | val_len_ptr, 684 | ) 685 | .context("val_len_ptr out of bounds")?; 686 | 687 | Ok(EssaResult::Ok) 688 | } 689 | } 690 | Err(err) => Ok(err), 691 | } 692 | } 693 | 694 | /// Stores all the information needed during execution. 695 | struct HostState { 696 | /// The compiled WASM module. 697 | module: Module, 698 | /// The KVS key under which a serialized version of the compiled WASM 699 | /// module is stored. 700 | module_key: ClientKey, 701 | /// The result value of this function, set through the `essa_set_result` 702 | /// host function. 703 | function_result: Option>, 704 | 705 | next_result_handle: u32, 706 | result_receivers: HashMap>, 707 | results: HashMap>>, 708 | 709 | zenoh: Arc, 710 | zenoh_prefix: String, 711 | anna: ClientNode, 712 | } 713 | 714 | impl HostState { 715 | /// Calls the given function on a node and returns the reply receiver for 716 | /// the corresponding result. 717 | fn essa_call( 718 | &mut self, 719 | function_name: String, 720 | args: Vec, 721 | ) -> Result, EssaResult> { 722 | // get the requested function and check its signature 723 | let func = self 724 | .module 725 | .exports() 726 | .into_iter() 727 | .find(|x| x.name() == function_name) 728 | .ok_or(EssaResult::NoSuchFunction)? 729 | .ty() 730 | .map_err(|_| EssaResult::NoSuchFunction)?; 731 | if let ExternalInstanceType::Func(func_type) = func { 732 | if func_type.args() != Some(&[ValType::I32]) || func_type.returns_len() != 0 { 733 | return Err(EssaResult::InvalidFunctionSignature); 734 | }; 735 | } else { 736 | return Err(EssaResult::NoSuchFunction); 737 | }; 738 | 739 | // store args in kvs 740 | let args_key: ClientKey = Uuid::new_v4().to_string().into(); 741 | kvs_put( 742 | args_key.clone(), 743 | LastWriterWinsLattice::new_now(args).into(), 744 | &mut self.anna, 745 | ) 746 | .map_err(|_| EssaResult::UnknownError)?; 747 | 748 | // trigger the function call on a remote node 749 | let reply = call_function_extern( 750 | self.module_key.clone(), 751 | function_name, 752 | args_key, 753 | self.zenoh.clone(), 754 | &self.zenoh_prefix, 755 | ) 756 | .unwrap(); 757 | 758 | Ok(reply) 759 | } 760 | 761 | /// Stores the given serialized `LattiveValue` in the KVS. 762 | fn put_lattice(&mut self, key: &ClientKey, value: &[u8]) -> Result<(), EssaResult> { 763 | let value = bincode::deserialize(value).map_err(|_| EssaResult::UnknownError)?; 764 | kvs_put(self.with_prefix(key), value, &mut self.anna).map_err(|_| EssaResult::UnknownError) 765 | } 766 | 767 | /// Reads the `LattiveValue` at the specified key from the KVS serializes it. 768 | fn get_lattice(&mut self, key: &ClientKey) -> Result, EssaResult> { 769 | kvs_get(self.with_prefix(key), &mut self.anna) 770 | .map_err(|_| EssaResult::NotFound) 771 | .and_then(|v| bincode::serialize(&v).map_err(|_| EssaResult::UnknownError)) 772 | } 773 | 774 | fn with_prefix(&self, key: &ClientKey) -> ClientKey { 775 | format!("{}/data/{}", self.module_key, key).into() 776 | } 777 | 778 | fn get_result(&mut self, handle: u32) -> Result>, EssaResult> { 779 | match self.results.entry(handle) { 780 | std::collections::hash_map::Entry::Occupied(entry) => Ok(entry.get().clone()), 781 | std::collections::hash_map::Entry::Vacant(entry) => { 782 | if let Some(result) = self.result_receivers.remove(&handle) { 783 | let reply = result.recv().map_err(|_| EssaResult::UnknownError)?; 784 | let value = reply 785 | .sample 786 | .map_err(|_| EssaResult::UnknownError)? 787 | .value 788 | .payload 789 | .contiguous() 790 | .into_owned(); 791 | let value = entry.insert(Arc::new(value)); 792 | Ok(value.clone()) 793 | } else { 794 | Err(EssaResult::NotFound) 795 | } 796 | } 797 | } 798 | } 799 | 800 | fn remove_result(&mut self, handle: u32) { 801 | self.results.remove(&handle); 802 | } 803 | } 804 | 805 | /// Call the specfied function on a remote node. 806 | fn call_function_extern( 807 | module_key: ClientKey, 808 | function_name: String, 809 | args_key: ClientKey, 810 | zenoh: Arc, 811 | zenoh_prefix: &str, 812 | ) -> anyhow::Result> { 813 | let topic = scheduler_function_call_topic(zenoh_prefix, &module_key, &function_name, &args_key); 814 | 815 | // send the request to the scheduler node 816 | let reply = zenoh 817 | .get(topic) 818 | .res() 819 | .map_err(|e| anyhow::anyhow!(e)) 820 | .context("failed to send function call request to scheduler")?; 821 | 822 | Ok(reply) 823 | } 824 | --------------------------------------------------------------------------------