├── .github └── workflows │ └── base.yml ├── .gitignore ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── LICENSE-THIRD-PARTY ├── README.md ├── archived ├── benches │ ├── compute_dag_bench.rs │ └── profile.bash ├── derive │ ├── Cargo.toml │ └── src │ │ ├── lib.rs │ │ ├── relay.rs │ │ └── task.rs ├── examples │ ├── actions.rs │ ├── compute_dag.rs │ ├── custom_log.rs │ ├── custom_parser_and_task.rs │ ├── dependencies.rs │ ├── derive_task.rs │ ├── engine.rs │ └── yaml_dag.rs ├── src │ ├── bin │ │ └── dagrs.rs │ ├── engine │ │ ├── dag.rs │ │ ├── graph.rs │ │ └── mod.rs │ ├── lib.rs │ ├── task │ │ ├── action.rs │ │ ├── cmd.rs │ │ ├── default_task.rs │ │ ├── mod.rs │ │ └── state.rs │ ├── utils │ │ ├── env.rs │ │ ├── file.rs │ │ ├── mod.rs │ │ └── parser.rs │ └── yaml │ │ ├── mod.rs │ │ ├── yaml_parser.rs │ │ └── yaml_task.rs └── tests │ ├── config │ ├── correct.yaml │ ├── custom_file_task.txt │ ├── empty_file.yaml │ ├── illegal_content.yaml │ ├── loop_error.yaml │ ├── no_run.yaml │ ├── no_script.yaml │ ├── no_start_with_dagrs.yaml │ ├── no_task_name.yaml │ ├── no_type.yaml │ ├── precursor_not_found.yaml │ ├── script_run_failed.yaml │ ├── self_loop_error.yaml │ ├── test.js │ ├── test.py │ └── test.sh │ ├── dag_job_test.rs │ ├── env_test.rs │ └── yaml_parser_test.rs ├── dagrs-derive ├── Cargo.toml ├── README.md └── src │ ├── auto_node.rs │ ├── lib.rs │ └── relay.rs ├── docs └── upgrade.md ├── examples ├── auto_node.rs ├── auto_relay.rs ├── compute_dag.rs ├── conditional_node.rs ├── custom_node.rs ├── dagrs-sklearn │ ├── Cargo.toml │ ├── examples │ │ ├── config.yml │ │ ├── ex3data1.mat │ │ ├── lr_i.py │ │ ├── lr_root.py │ │ ├── notebook.ipynb │ │ ├── raw.py │ │ ├── requirements.txt │ │ └── sklearn.rs │ ├── src │ │ ├── command_action.rs │ │ ├── lib.rs │ │ ├── parser │ │ │ ├── mod.rs │ │ │ ├── yaml_parser.rs │ │ │ └── yaml_task.rs │ │ └── utils │ │ │ ├── file.rs │ │ │ ├── mod.rs │ │ │ └── parser.rs │ └── tests │ │ ├── config │ │ ├── correct.yaml │ │ ├── custom_file_task.txt │ │ ├── empty_file.yaml │ │ ├── illegal_content.yaml │ │ ├── loop_error.yaml │ │ ├── no_run.yaml │ │ ├── no_script.yaml │ │ ├── no_start_with_dagrs.yaml │ │ ├── no_task_name.yaml │ │ ├── no_type.yaml │ │ ├── precursor_not_found.yaml │ │ ├── script_run_failed.yaml │ │ ├── self_loop_error.yaml │ │ ├── test.js │ │ ├── test.py │ │ └── test.sh │ │ ├── dag_job_test.rs │ │ └── yaml_parser_test.rs ├── hello_dagrs.rs └── loop_dag.rs └── src ├── connection ├── in_channel.rs ├── information_packet.rs ├── mod.rs └── out_channel.rs ├── graph ├── abstract_graph.rs ├── error.rs ├── graph.rs ├── loop_subgraph.rs └── mod.rs ├── lib.rs ├── node ├── action.rs ├── conditional_node.rs ├── default_node.rs ├── id_allocate.rs ├── mod.rs └── node.rs └── utils ├── env.rs ├── execstate.rs ├── mod.rs └── output.rs /.github/workflows/base.yml: -------------------------------------------------------------------------------- 1 | # Based on https://github.com/actions-rs/meta/blob/master/recipes/quickstart.md 2 | 3 | 4 | 5 | 6 | 7 | on: [ push, pull_request ] 8 | 9 | name: Check, Test and Lints 10 | 11 | jobs: 12 | check: 13 | name: Check 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: actions-rs/toolchain@v1 18 | with: 19 | profile: minimal 20 | toolchain: stable 21 | override: true 22 | - uses: actions-rs/cargo@v1 23 | with: 24 | command: check 25 | - uses: actions/setup-node@v3 26 | with: 27 | node-version: 16 28 | 29 | check_fmt: 30 | name: Fmt Check 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v2 34 | - uses: actions-rs/toolchain@v1 35 | with: 36 | profile: minimal 37 | toolchain: stable 38 | override: true 39 | - uses: actions-rs/cargo@v1 40 | with: 41 | command: fmt 42 | args: --check 43 | 44 | test-unix: 45 | name: Unit test & Doc test for unix 46 | runs-on: ubuntu-latest 47 | steps: 48 | - uses: actions/checkout@v2 49 | - uses: actions-rs/toolchain@v1 50 | with: 51 | profile: minimal 52 | toolchain: stable 53 | override: true 54 | - uses: actions-rs/cargo@v1 55 | with: 56 | command: test 57 | args: --all 58 | 59 | test-win: 60 | name: Unit test & Doc test for windows 61 | runs-on: windows-latest 62 | steps: 63 | - uses: actions/checkout@v2 64 | - uses: actions-rs/toolchain@v1 65 | with: 66 | profile: minimal 67 | toolchain: stable 68 | override: true 69 | - uses: actions-rs/cargo@v1 70 | with: 71 | command: test 72 | args: --all 73 | 74 | example-sklearn-win: 75 | name: example-sklearn for windows 76 | runs-on: windows-latest 77 | strategy: 78 | matrix: 79 | python-version: ["3.12"] 80 | steps: 81 | - uses: actions/checkout@v4 82 | - uses: actions-rs/toolchain@v1 83 | with: 84 | profile: minimal 85 | toolchain: stable 86 | override: true 87 | - name: Set up Python ${{ matrix.python-version }} 88 | uses: actions/setup-python@v5 89 | with: 90 | python-version: ${{ matrix.python-version }} 91 | - name: install packages 92 | run: pip install -r examples/dagrs-sklearn/examples/requirements.txt 93 | - name: run 94 | run: cd examples/dagrs-sklearn && cargo run --example sklearn --release 95 | 96 | example-sklearn-unix: 97 | name: example-sklearn for unix 98 | runs-on: ubuntu-latest 99 | strategy: 100 | matrix: 101 | python-version: ["3.12"] 102 | steps: 103 | - uses: actions/checkout@v4 104 | - uses: actions-rs/toolchain@v1 105 | with: 106 | profile: minimal 107 | toolchain: stable 108 | override: true 109 | - name: Set up Python ${{ matrix.python-version }} 110 | uses: actions/setup-python@v5 111 | with: 112 | python-version: ${{ matrix.python-version }} 113 | - name: install packages 114 | run: pip install -r examples/dagrs-sklearn/examples/requirements.txt 115 | - name: run 116 | run: cd examples/dagrs-sklearn && cargo run --example sklearn --release -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | 12 | # IDE 13 | .idea 14 | /.vscode/* 15 | !/.vscode/settings.json 16 | 17 | # Exclude execute log 18 | /*.log 19 | 20 | .vscode 21 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dagrs" 3 | authors = [ 4 | "Quanyi Ma ", 5 | "Xiaolong Fu ", 6 | "Zhilei Qiu ", 7 | ] 8 | version = "0.4.4" 9 | edition = "2021" 10 | license = "MIT OR Apache-2.0" 11 | description = "Dagrs follows the concept of Flow-based Programming and is suitable for the execution of multiple tasks with graph-like dependencies. Dagrs has the characteristics of high performance and asynchronous execution. It provides users with a convenient programming interface." 12 | readme = "README.md" 13 | repository = "https://github.com/dagrs-dev/dagrs" 14 | keywords = ["DAG", "task", "async", "fbp", "tokio"] 15 | 16 | [workspace] 17 | members = [".", "dagrs-derive", "examples/dagrs-sklearn"] 18 | default-members = [".", "dagrs-derive"] 19 | 20 | [dependencies] 21 | dagrs-derive = { path = "dagrs-derive", optional = true, version = "0.4.3" } 22 | tokio = { version = "1.28", features = ["rt", "sync", "rt-multi-thread"] } 23 | log = "0.4" 24 | async-trait = "0.1.83" 25 | futures = "0.3.31" 26 | 27 | [dev-dependencies] 28 | criterion = { version = "0.5.1", features = ["html_reports"] } 29 | env_logger = "0.11.6" 30 | 31 | [target.'cfg(unix)'.dev-dependencies] 32 | 33 | [features] 34 | default = ["derive"] 35 | derive = ["dagrs-derive/derive"] 36 | 37 | [[example]] 38 | name = "auto_node" 39 | required-features = ["derive"] 40 | 41 | [[example]] 42 | name = "auto_relay" 43 | required-features = ["derive"] 44 | 45 | [[example]] 46 | name = "compute_dag" 47 | 48 | [[example]] 49 | name = "custom_node" 50 | 51 | [[example]] 52 | name = "hello_dagrs" 53 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Open Rust Initiative 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LICENSE-THIRD-PARTY: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagrs-dev/dagrs/7c059dda74e2b906a9cc37d413bb2cfc9140d5f6/LICENSE-THIRD-PARTY -------------------------------------------------------------------------------- /archived/benches/compute_dag_bench.rs: -------------------------------------------------------------------------------- 1 | use criterion::{criterion_group, criterion_main, Criterion}; 2 | 3 | use dagrs::{Dag, DefaultTask, EnvVar, Input, Output, Task}; 4 | use std::sync::Arc; 5 | 6 | fn calc(input: Input, env: Arc) -> Output { 7 | let base = env.get::("base").unwrap(); 8 | let mut sum = 2; 9 | input.get_iter().for_each(|i| { 10 | let mut i_usize = *i.get::().unwrap(); 11 | // avoid overflow 12 | if i_usize > 1e6 as usize { 13 | i_usize = 2; 14 | } 15 | 16 | sum += i_usize * base 17 | }); 18 | Output::new(sum) 19 | } 20 | 21 | fn compute_dag(tasks: Vec) { 22 | let mut dag = Dag::with_tasks(tasks); 23 | let mut env = EnvVar::new(); 24 | env.set("base", 2usize); 25 | dag.set_env(env); 26 | 27 | assert!(dag.start().is_ok()); 28 | // Get execution result. 29 | let _res = dag.get_result::().unwrap(); 30 | } 31 | 32 | fn compute_dag_bench(bencher: &mut Criterion) { 33 | env_logger::init(); 34 | 35 | let mut tasks = (0..50usize) 36 | .map(|i_task| DefaultTask::with_closure(&i_task.to_string(), calc)) 37 | .collect::>(); 38 | 39 | // consider 8 dependency for each task (except first 20 tasks) 40 | for i_task in 20..tasks.len() { 41 | let predecessors_id = ((i_task - 8)..i_task) 42 | .map(|i_dep| tasks[i_dep].id()) 43 | .collect::>(); 44 | 45 | tasks[i_task].set_predecessors_by_id(predecessors_id); 46 | } 47 | 48 | bencher.bench_function("compute dag", |b| b.iter(|| compute_dag(tasks.clone()))); 49 | } 50 | 51 | criterion_group!( 52 | name = benches; 53 | config = { 54 | 55 | #[allow(unused_mut)] 56 | let mut criterion = Criterion::default().sample_size(4000).noise_threshold(0.05); 57 | 58 | #[cfg(feature = "bench-prost-codec")] 59 | { 60 | use pprof::criterion::{PProfProfiler, Output::Protobuf}; 61 | 62 | criterion = criterion.with_profiler(PProfProfiler::new(4000, Protobuf)); 63 | } 64 | 65 | criterion 66 | }; 67 | targets = compute_dag_bench 68 | ); 69 | 70 | criterion_main!(benches); 71 | -------------------------------------------------------------------------------- /archived/benches/profile.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash -e 2 | 3 | bench_script="RUSTFLAGS=-g cargo bench \ 4 | --features=bench-prost-codec \ 5 | --bench compute_dag_bench" 6 | 7 | # Profile the executable 8 | bash -c "$bench_script -- --profile-time 10" 9 | 10 | # Get the path to the executable from cargo 11 | exe_path_string=$(bash -c "$bench_script --no-run 2>&1") 12 | exe_path=$(echo $exe_path_string | sed -n 's/.*(\(.*\)).*/\1/p') 13 | 14 | # Display the profile 15 | /usr/local/go/pkg/tool/linux_amd64/pprof \ 16 | -call_tree -lines \ 17 | -output "./target/criterion/report/compute_dag_bench-profile.svg" -svg \ 18 | "$exe_path" \ 19 | "./target/criterion/compute dag/profile/profile.pb" 20 | -------------------------------------------------------------------------------- /archived/derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "derive" 3 | version = "0.3.0" 4 | edition = "2021" 5 | license = "MIT OR Apache-2.0" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | syn = { version = "2.0", features = ["full"] } 11 | quote = "1.0" 12 | proc-macro2= "1.0" 13 | 14 | [lib] 15 | proc-macro = true 16 | 17 | [features] 18 | default = ["derive"] 19 | derive = [] 20 | -------------------------------------------------------------------------------- /archived/derive/src/lib.rs: -------------------------------------------------------------------------------- 1 | use proc_macro::TokenStream; 2 | 3 | extern crate proc_macro2; 4 | extern crate quote; 5 | extern crate syn; 6 | 7 | #[cfg(feature = "derive")] 8 | mod relay; 9 | #[cfg(feature = "derive")] 10 | mod task; 11 | 12 | /// [`CustomTask`] is a derived macro that may be used when customizing tasks. It can only be 13 | /// marked on the structure, and the user needs to specify four attributes of the custom task 14 | /// type, which are task(attr="id"), task(attr = "name"), task(attr = "precursors ") and 15 | /// task(attr = "action"), which are used in the `derive_task` example. 16 | #[cfg(feature = "derive")] 17 | #[proc_macro_derive(CustomTask, attributes(task))] 18 | pub fn derive_task(input: TokenStream) -> TokenStream { 19 | use crate::task::parse_task; 20 | use syn::{parse_macro_input, DeriveInput}; 21 | let input = parse_macro_input!(input as DeriveInput); 22 | parse_task(&input).into() 23 | } 24 | 25 | /// The [`dependencies!`] macro allows users to specify all task dependencies in an easy-to-understand 26 | /// way. It will return to the user a series of `DefaultTask` in the order of tasks given by the user. 27 | #[cfg(feature = "derive")] 28 | #[proc_macro] 29 | pub fn dependencies(input: TokenStream) -> TokenStream { 30 | use crate::relay::generate_task; 31 | use relay::Tasks; 32 | let tasks = syn::parse_macro_input!(input as Tasks); 33 | let relies = tasks.resolve_dependencies(); 34 | if let Err(err) = relies { 35 | return err.into_compile_error().into(); 36 | } 37 | let token = generate_task(relies.unwrap()); 38 | token.into() 39 | } 40 | -------------------------------------------------------------------------------- /archived/derive/src/relay.rs: -------------------------------------------------------------------------------- 1 | use proc_macro2::Ident; 2 | use std::collections::{HashMap, HashSet}; 3 | use syn::{parse::Parse, Token}; 4 | 5 | pub(crate) struct Relay { 6 | pub(crate) task: Ident, 7 | pub(crate) successors: Vec, 8 | } 9 | 10 | pub(crate) struct Task { 11 | pub(crate) task: Ident, 12 | pub(crate) precursors: Vec, 13 | } 14 | 15 | pub(crate) struct Tasks(pub(crate) Vec); 16 | 17 | impl Parse for Tasks { 18 | fn parse(input: syn::parse::ParseStream) -> syn::Result { 19 | let mut relies = Vec::new(); 20 | loop { 21 | let mut successors = Vec::new(); 22 | let task = input.parse::()?; 23 | input.parse::)>()?; 24 | while !input.peek(Token!(,)) && !input.is_empty() { 25 | successors.push(input.parse::()?); 26 | } 27 | let relay = Relay { task, successors }; 28 | relies.push(relay); 29 | let _ = input.parse::(); 30 | if input.is_empty() { 31 | break; 32 | } 33 | } 34 | Ok(Self(relies)) 35 | } 36 | } 37 | 38 | impl Tasks { 39 | fn check_duplicate(&self) -> syn::Result<()> { 40 | let mut set = HashSet::new(); 41 | for relay in self.0.iter() { 42 | let task = &relay.task; 43 | if !set.contains(task) { 44 | set.insert(task); 45 | } else { 46 | let err_msg = format!("Duplicate task definition! [{}]", task); 47 | return Err(syn::Error::new_spanned(task, err_msg)); 48 | } 49 | } 50 | Ok(()) 51 | } 52 | 53 | pub(crate) fn resolve_dependencies(self) -> syn::Result> { 54 | self.check_duplicate()?; 55 | let mut seq = Vec::new(); 56 | let tasks: HashMap> = self 57 | .0 58 | .into_iter() 59 | .map(|item| { 60 | seq.push(item.task.clone()); 61 | (item.task, item.successors) 62 | }) 63 | .collect(); 64 | let res = seq 65 | .into_iter() 66 | .map(|item| { 67 | let mut pre = Vec::new(); 68 | tasks.iter().for_each(|(k, v)| { 69 | if v.iter().any(|ele| ele.eq(&item)) { 70 | pre.push(k.clone()); 71 | } 72 | }); 73 | Task { 74 | task: item.clone(), 75 | precursors: pre, 76 | } 77 | }) 78 | .collect(); 79 | Ok(res) 80 | } 81 | } 82 | 83 | fn init_tasks(tasks: &[Task]) -> proc_macro2::TokenStream { 84 | let mut token = proc_macro2::TokenStream::new(); 85 | for task in tasks.iter() { 86 | let ident = &task.task; 87 | let name = ident.to_string(); 88 | token.extend(quote::quote!( 89 | let mut #ident=dagrs::DefaultTask::new(#name); 90 | )); 91 | } 92 | token 93 | } 94 | 95 | fn init_precursors(tasks: &[Task]) -> proc_macro2::TokenStream { 96 | let mut token = proc_macro2::TokenStream::new(); 97 | for task in tasks.iter() { 98 | let ident = &task.task; 99 | let mut pres_token = proc_macro2::TokenStream::new(); 100 | task.precursors.iter().for_each(|item| { 101 | pres_token.extend(quote::quote!( 102 | &#item, 103 | )); 104 | }); 105 | token.extend(quote::quote!( 106 | #ident.set_predecessors(&[#pres_token]); 107 | )); 108 | } 109 | token 110 | } 111 | 112 | pub(crate) fn generate_task(tasks: Vec) -> proc_macro2::TokenStream { 113 | let tasks_defined_token: proc_macro2::TokenStream = init_tasks(&tasks); 114 | let init_pres_token: proc_macro2::TokenStream = init_precursors(&tasks); 115 | let tasks_ident: Vec = tasks.into_iter().map(|item| item.task).collect(); 116 | quote::quote!({ 117 | #tasks_defined_token 118 | #init_pres_token 119 | vec![#(#tasks_ident,)*] 120 | }) 121 | } 122 | -------------------------------------------------------------------------------- /archived/derive/src/task.rs: -------------------------------------------------------------------------------- 1 | use proc_macro2::TokenStream; 2 | use syn::{ 3 | Data, DeriveInput, Expr, ExprLit, Field, Fields, GenericArgument, Ident, Lit, MetaNameValue, 4 | PathArguments, Type, 5 | }; 6 | 7 | const ID: &str = "id"; 8 | const NAME: &str = "name"; 9 | const PRECURSORS: &str = "precursors"; 10 | const ACTION: &str = "action"; 11 | 12 | #[allow(unused)] 13 | pub(crate) fn parse_task(input: &DeriveInput) -> TokenStream { 14 | let struct_ident = &input.ident; 15 | let fields = match input.data { 16 | Data::Struct(ref str) => &str.fields, 17 | _ => { 18 | return syn::Error::new_spanned( 19 | struct_ident, 20 | "Task macros can only be annotated on struct.", 21 | ) 22 | .into_compile_error(); 23 | } 24 | }; 25 | let attr_token = generate_field_function(fields); 26 | if let Err(e) = attr_token { 27 | return e.into_compile_error(); 28 | } 29 | generate_impl(struct_ident, attr_token.unwrap()) 30 | } 31 | 32 | fn generate_field_function(fields: &Fields) -> syn::Result { 33 | let mut token = proc_macro2::TokenStream::new(); 34 | for field in fields.iter() { 35 | for attr in field.attrs.iter() { 36 | if attr.path().is_ident("task") { 37 | let kv: MetaNameValue = attr.parse_args()?; 38 | if kv.path.is_ident("attr") { 39 | let err_msg = format!( 40 | "The optional value of attr is [{},{},{},{}]", 41 | ID, NAME, PRECURSORS, ACTION 42 | ); 43 | let err = Err(syn::Error::new_spanned(&kv.value, err_msg)); 44 | if let Expr::Lit(ExprLit { 45 | lit: Lit::Str(lit), .. 46 | }) = kv.value 47 | { 48 | let tk = match lit.value().as_str() { 49 | ID => validate_id(field), 50 | NAME => validate_name(field), 51 | PRECURSORS => validate_precursors(field), 52 | ACTION => validate_action(field), 53 | _ => return err, 54 | }?; 55 | token.extend(tk); 56 | } else { 57 | return err; 58 | } 59 | } else { 60 | return Err(syn::Error::new_spanned(kv, "expect `task(attr = \"...\")`")); 61 | } 62 | } 63 | } 64 | } 65 | Ok(token) 66 | } 67 | 68 | fn validate_id(field: &Field) -> syn::Result { 69 | let ident = &field.ident; 70 | let err = Err(syn::Error::new_spanned( 71 | &field.ty, 72 | "The type of `id` should be `usize`", 73 | )); 74 | if let Type::Path(ref str) = field.ty { 75 | let ty = str.path.segments.last().unwrap(); 76 | if ty.ident.eq("usize") { 77 | Ok(quote::quote!( 78 | fn id(&self) -> usize { 79 | self.#ident 80 | } 81 | )) 82 | } else { 83 | err 84 | } 85 | } else { 86 | err 87 | } 88 | } 89 | 90 | fn validate_name(field: &Field) -> syn::Result { 91 | let ident = &field.ident; 92 | let err = Err(syn::Error::new_spanned( 93 | &field.ty, 94 | "The type of `name` should be `String`", 95 | )); 96 | if let Type::Path(ref str) = field.ty { 97 | let ty = str.path.segments.last().unwrap(); 98 | if ty.ident.eq("String") { 99 | Ok(quote::quote!( 100 | fn name(&self) -> &str { 101 | &self.#ident 102 | } 103 | )) 104 | } else { 105 | err 106 | } 107 | } else { 108 | err 109 | } 110 | } 111 | 112 | fn validate_precursors(field: &Field) -> syn::Result { 113 | let ident = &field.ident; 114 | let err = Err(syn::Error::new_spanned( 115 | &field.ty, 116 | "The type of `id` should be `Vec`", 117 | )); 118 | if let Type::Path(ref str) = field.ty { 119 | let ty = str.path.segments.last().unwrap(); 120 | if ty.ident.eq("Vec") { 121 | match ty.arguments { 122 | PathArguments::AngleBracketed(ref inner) => { 123 | if let GenericArgument::Type(Type::Path(inner_ty)) = inner.args.last().unwrap() 124 | { 125 | if inner_ty.path.is_ident("usize") { 126 | Ok(quote::quote!( 127 | fn precursors(&self) -> &[usize] { 128 | &self.#ident 129 | } 130 | )) 131 | } else { 132 | err 133 | } 134 | } else { 135 | err 136 | } 137 | } 138 | _ => err, 139 | } 140 | } else { 141 | err 142 | } 143 | } else { 144 | err 145 | } 146 | } 147 | 148 | fn validate_action(field: &Field) -> syn::Result { 149 | let ident = &field.ident; 150 | let err = Err(syn::Error::new_spanned( 151 | &field.ty, 152 | "The type of `id` should be `Action`", 153 | )); 154 | if let Type::Path(ref str) = field.ty { 155 | let ty = str.path.segments.last().unwrap(); 156 | if ty.ident.eq("Action") { 157 | Ok(quote::quote!( 158 | fn action(&self) -> Action { 159 | self.#ident.clone() 160 | } 161 | )) 162 | } else { 163 | err 164 | } 165 | } else { 166 | err 167 | } 168 | } 169 | 170 | fn generate_impl(struct_ident: &Ident, fields_function: proc_macro2::TokenStream) -> TokenStream { 171 | quote::quote!( 172 | impl dagrs::Task for #struct_ident{ 173 | #fields_function 174 | } 175 | unsafe impl Send for #struct_ident{} 176 | unsafe impl Sync for #struct_ident{} 177 | ) 178 | } 179 | -------------------------------------------------------------------------------- /archived/examples/actions.rs: -------------------------------------------------------------------------------- 1 | //! Construct two different actions. 2 | //! - Create a closure directly 3 | //! - Implementing Complex, the type can have some additional information. 4 | use dagrs::{Complex, DefaultTask, Output}; 5 | 6 | struct Act(usize); 7 | 8 | impl Complex for Act { 9 | fn run(&self, _input: dagrs::Input, _env: std::sync::Arc) -> Output { 10 | Output::new(self.0 + 10) 11 | } 12 | } 13 | fn main() { 14 | let simple = |_input, _env| Output::new("simple"); 15 | let _simple_task = DefaultTask::with_closure("simple task", simple); 16 | 17 | let complex = Act(20); 18 | let _complex_task = DefaultTask::with_action("complex action", complex); 19 | } 20 | -------------------------------------------------------------------------------- /archived/examples/compute_dag.rs: -------------------------------------------------------------------------------- 1 | //! Only use Dag, execute a job. The graph is as follows: 2 | //! 3 | //! ↱----------↴ 4 | //! B -→ E --→ G 5 | //! ↗ ↗ ↗ 6 | //! A --→ C / 7 | //! ↘ ↘ / 8 | //! D -→ F 9 | //! 10 | //! The final execution result is 272. 11 | 12 | extern crate dagrs; 13 | 14 | use dagrs::{Complex, Dag, DefaultTask, EnvVar, Input, Output}; 15 | use std::sync::Arc; 16 | 17 | struct Compute(usize); 18 | 19 | impl Complex for Compute { 20 | fn run(&self, input: Input, env: Arc) -> Output { 21 | let base = env.get::("base").unwrap(); 22 | let mut sum = self.0; 23 | input 24 | .get_iter() 25 | .for_each(|i| sum += i.get::().unwrap() * base); 26 | Output::new(sum) 27 | } 28 | } 29 | 30 | fn main() { 31 | // initialization log. 32 | env_logger::init(); 33 | 34 | // generate some tasks. 35 | let a = DefaultTask::with_action("Compute A", Compute(1)); 36 | 37 | let mut b = DefaultTask::with_action("Compute B", Compute(2)); 38 | 39 | let mut c = DefaultTask::new("Compute C"); 40 | c.set_action(Compute(4)); 41 | 42 | let mut d = DefaultTask::new("Compute D"); 43 | d.set_action(Compute(8)); 44 | 45 | let mut e = DefaultTask::with_closure("Compute E", |input, env| { 46 | let base = env.get::("base").unwrap(); 47 | let mut sum = 16; 48 | input 49 | .get_iter() 50 | .for_each(|i| sum += i.get::().unwrap() * base); 51 | Output::new(sum) 52 | }); 53 | let mut f = DefaultTask::with_closure("Compute F", |input, env| { 54 | let base = env.get::("base").unwrap(); 55 | let mut sum = 32; 56 | input 57 | .get_iter() 58 | .for_each(|i| sum += i.get::().unwrap() * base); 59 | Output::new(sum) 60 | }); 61 | 62 | let mut g = DefaultTask::new("Compute G"); 63 | g.set_closure(|input, env| { 64 | let base = env.get::("base").unwrap(); 65 | let mut sum = 64; 66 | input 67 | .get_iter() 68 | .for_each(|i| sum += i.get::().unwrap() * base); 69 | Output::new(sum) 70 | }); 71 | 72 | // Set up task dependencies. 73 | b.set_predecessors(&[&a]); 74 | c.set_predecessors(&[&a]); 75 | d.set_predecessors(&[&a]); 76 | e.set_predecessors(&[&b, &c]); 77 | f.set_predecessors(&[&c, &d]); 78 | g.set_predecessors(&[&b, &e, &f]); 79 | // Create a new Dag. 80 | let mut dag = Dag::with_tasks(vec![a, b, c, d, e, f, g]); 81 | // Set a global environment variable for this dag. 82 | let mut env = EnvVar::new(); 83 | env.set("base", 2usize); 84 | dag.set_env(env); 85 | // Start executing this dag 86 | assert!(dag.start().is_ok()); 87 | // Get execution result. 88 | let res = dag.get_result::().unwrap(); 89 | println!("The result is {}.", res); 90 | } 91 | -------------------------------------------------------------------------------- /archived/examples/custom_log.rs: -------------------------------------------------------------------------------- 1 | //! Use the simplelog for logging. 2 | 3 | extern crate dagrs; 4 | extern crate log; 5 | extern crate simplelog; 6 | 7 | use std::collections::HashMap; 8 | 9 | use dagrs::Dag; 10 | use simplelog::*; 11 | 12 | fn main() { 13 | // Initialize the global logger with a simplelogger as the logging backend. 14 | let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); 15 | 16 | let mut dag = Dag::with_yaml("tests/config/correct.yaml", HashMap::new()).unwrap(); 17 | assert!(dag.start().is_ok()); 18 | } 19 | -------------------------------------------------------------------------------- /archived/examples/custom_parser_and_task.rs: -------------------------------------------------------------------------------- 1 | //! Implement the Task trait to customize task properties. 2 | //! MyTask is basically the same as DefaultTask provided by dagrs. 3 | //! Implement the Parser interface to customize the task configuration file parser. 4 | //! The content of the configuration file is as follows: 5 | //! 6 | //! ``` 7 | //! a,Task a,b c,echo a 8 | //! b,Task b,c f g,echo b 9 | //! c,Task c,e g,echo c 10 | //! d,Task d,c e,echo d 11 | //! e,Task e,h,echo e 12 | //! f,Task f,g,python3 tests/config/test.py 13 | //! g,Task g,h,node tests/config/test.js 14 | //! h,Task h,,echo h 15 | //! ``` 16 | 17 | extern crate dagrs; 18 | 19 | use std::collections::HashMap; 20 | use std::fmt::{Display, Formatter}; 21 | use std::{fs, sync::Arc}; 22 | 23 | use dagrs::{Action, CommandAction, Dag, DagError, Parser, Task}; 24 | 25 | struct MyTask { 26 | tid: (String, usize), 27 | name: String, 28 | precursors: Vec, 29 | precursors_id: Vec, 30 | action: Action, 31 | } 32 | 33 | impl MyTask { 34 | pub fn new(txt_id: &str, precursors: Vec, name: String, action: CommandAction) -> Self { 35 | Self { 36 | tid: (txt_id.to_owned(), dagrs::alloc_id()), 37 | name, 38 | precursors, 39 | precursors_id: Vec::new(), 40 | action: Action::Structure(Arc::new(action)), 41 | } 42 | } 43 | 44 | pub fn init_precursors(&mut self, pres_id: Vec) { 45 | self.precursors_id = pres_id; 46 | } 47 | 48 | pub fn str_precursors(&self) -> Vec { 49 | self.precursors.clone() 50 | } 51 | 52 | pub fn str_id(&self) -> String { 53 | self.tid.0.clone() 54 | } 55 | } 56 | 57 | impl Task for MyTask { 58 | fn action(&self) -> Action { 59 | self.action.clone() 60 | } 61 | fn precursors(&self) -> &[usize] { 62 | &self.precursors_id 63 | } 64 | fn id(&self) -> usize { 65 | self.tid.1 66 | } 67 | fn name(&self) -> &str { 68 | &self.name 69 | } 70 | } 71 | 72 | impl Display for MyTask { 73 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 74 | write!( 75 | f, 76 | "{},{},{},{:?}", 77 | self.name, self.tid.0, self.tid.1, self.precursors 78 | ) 79 | } 80 | } 81 | 82 | struct ConfigParser; 83 | 84 | impl ConfigParser { 85 | fn load_file(&self, file: &str) -> Result { 86 | let contents = 87 | fs::read_to_string(file).map_err(|e| DagError::ParserError(e.to_string()))?; 88 | Ok(contents) 89 | } 90 | 91 | fn parse_one(&self, item: String) -> MyTask { 92 | let attr: Vec<&str> = item.split(',').collect(); 93 | 94 | let pres_item = *attr.get(2).unwrap(); 95 | let pres = if pres_item.eq("") { 96 | Vec::new() 97 | } else { 98 | pres_item.split(' ').map(|pre| pre.to_string()).collect() 99 | }; 100 | 101 | let id = *attr.first().unwrap(); 102 | let name = attr.get(1).unwrap().to_string(); 103 | let cmd = *attr.get(3).unwrap(); 104 | MyTask::new(id, pres, name, CommandAction::new(cmd)) 105 | } 106 | } 107 | 108 | impl Parser for ConfigParser { 109 | fn parse_tasks( 110 | &self, 111 | file: &str, 112 | specific_actions: HashMap, 113 | ) -> Result>, DagError> { 114 | let content = self.load_file(file)?; 115 | self.parse_tasks_from_str(&content, specific_actions) 116 | } 117 | fn parse_tasks_from_str( 118 | &self, 119 | content: &str, 120 | _specific_actions: HashMap, 121 | ) -> Result>, DagError> { 122 | let lines: Vec = content.lines().map(|line| line.to_string()).collect(); 123 | let mut map = HashMap::new(); 124 | let mut tasks = Vec::new(); 125 | lines.into_iter().for_each(|line| { 126 | let task = self.parse_one(line); 127 | map.insert(task.str_id(), task.id()); 128 | tasks.push(task); 129 | }); 130 | 131 | for task in tasks.iter_mut() { 132 | let mut pres = Vec::new(); 133 | let str_pre = task.str_precursors(); 134 | if !str_pre.is_empty() { 135 | for pre in str_pre { 136 | pres.push(map[&pre[..]]); 137 | } 138 | task.init_precursors(pres); 139 | } 140 | } 141 | Ok(tasks 142 | .into_iter() 143 | .map(|task| Box::new(task) as Box) 144 | .collect()) 145 | } 146 | } 147 | 148 | fn main() { 149 | env_logger::init(); 150 | let file = "tests/config/custom_file_task.txt"; 151 | let mut dag = 152 | Dag::with_config_file_and_parser(file, Box::new(ConfigParser), HashMap::new()).unwrap(); 153 | assert!(dag.start().is_ok()); 154 | } 155 | -------------------------------------------------------------------------------- /archived/examples/dependencies.rs: -------------------------------------------------------------------------------- 1 | use dagrs::{dependencies, Complex, EnvVar, Input, Output}; 2 | use std::sync::Arc; 3 | 4 | /// The `dependencies` macro allows users to specify all task dependencies in an easy-to-understand 5 | /// way. It will return to the user a series of `DefaultTask` in the order of tasks given by the user. 6 | /// 7 | /// # Example 8 | /// 9 | /// ↱----------↴ 10 | /// B -→ E --→ G 11 | /// ↗ ↗ ↗ 12 | /// A --→ C / 13 | /// ↘ ↘ / 14 | /// D -→ F 15 | /// 16 | /// If you want to define a task graph with such dependencies, the code is as follows: 17 | /// 18 | /// let mut tasks=dependencies!( 19 | /// a -> b c d, 20 | /// b -> e g, 21 | /// c -> e f, 22 | /// d -> f, 23 | /// e -> g, 24 | /// f -> g, 25 | /// g -> 26 | /// ); 27 | /// 28 | /// Note that although task g has no successor tasks, "g->" must also be written. The return 29 | /// value type tasks is a Vec. The name of each task is the same as the given 30 | /// identifier, which can be expressed as an array as [ "a","b","c","d","e","f","g"]. 31 | 32 | struct Compute(usize); 33 | 34 | impl Complex for Compute { 35 | fn run(&self, input: Input, env: Arc) -> Output { 36 | let base = env.get::("base").unwrap(); 37 | let mut sum = self.0; 38 | input 39 | .get_iter() 40 | .for_each(|i| sum += i.get::().unwrap() * base); 41 | Output::new(sum) 42 | } 43 | } 44 | 45 | fn main() { 46 | env_logger::init(); 47 | let mut tasks = dependencies!( 48 | a -> b c d, 49 | b -> e g, 50 | c -> e f, 51 | d -> f, 52 | e -> g, 53 | f -> g, 54 | g -> 55 | ); 56 | let mut x = 1; 57 | for task in tasks.iter_mut().take(4) { 58 | task.set_action(Compute(x * 2)); 59 | x *= 2; 60 | } 61 | 62 | for task in tasks.iter_mut().skip(4) { 63 | task.set_closure(|input, env| { 64 | let base = env.get::("base").unwrap(); 65 | let mut sum = 0; 66 | input 67 | .get_iter() 68 | .for_each(|i| sum += i.get::().unwrap() * base); 69 | Output::new(sum) 70 | }); 71 | } 72 | 73 | let mut dag = dagrs::Dag::with_tasks(tasks); 74 | let mut env = EnvVar::new(); 75 | env.set("base", 2usize); 76 | dag.set_env(env); 77 | assert!(dag.start().is_ok()); 78 | } 79 | -------------------------------------------------------------------------------- /archived/examples/derive_task.rs: -------------------------------------------------------------------------------- 1 | use dagrs::{Action, CustomTask, Output, Task}; 2 | use std::sync::Arc; 3 | 4 | /// `CustomTask` is a derived macro that may be used when customizing tasks. It can only be 5 | /// marked on the structure, and the user needs to specify four attributes of the custom task 6 | /// type, which are task(attr="id"), task(attr = "name"), task(attr = "precursors ") and 7 | /// task(attr = "action"), which are used in the `derive_task` example. 8 | /// 9 | /// # Example 10 | /// 11 | /// ```rust 12 | /// #[derive(CustomTask)] 13 | /// struct MyTask { 14 | /// #[task(attr = "id")] 15 | /// id: usize, 16 | /// #[task(attr = "name")] 17 | /// name: String, 18 | /// #[task(attr = "precursors")] 19 | /// pre: Vec, 20 | /// #[task(attr = "action")] 21 | /// action: Action, 22 | /// } 23 | /// ``` 24 | #[derive(CustomTask)] 25 | struct MyTask { 26 | #[task(attr = "id")] 27 | id: usize, 28 | #[task(attr = "name")] 29 | name: String, 30 | #[task(attr = "precursors")] 31 | pre: Vec, 32 | #[task(attr = "action")] 33 | action: Action, 34 | } 35 | 36 | fn main() { 37 | let action = Action::Closure(Arc::new(|_, _| Output::empty())); 38 | let task = MyTask { 39 | id: 10, 40 | name: "mytask".to_owned(), 41 | pre: vec![1, 2], 42 | action, 43 | }; 44 | println!("{}\t{}\t{:?}", task.id(), task.name(), task.precursors()); 45 | } 46 | -------------------------------------------------------------------------------- /archived/examples/engine.rs: -------------------------------------------------------------------------------- 1 | //! Use Engine to manage multiple Dag jobs. 2 | 3 | extern crate dagrs; 4 | 5 | use std::collections::HashMap; 6 | 7 | use dagrs::{Dag, DefaultTask, Engine, Output}; 8 | fn main() { 9 | // initialization log. 10 | env_logger::init(); 11 | // Create an Engine. 12 | let mut engine = Engine::default(); 13 | 14 | // Create some task for dag1. 15 | let t1_a = DefaultTask::with_closure("Compute A1", |_, _| Output::new(20usize)); 16 | let mut t1_b = DefaultTask::with_closure("Compute B1", |input, _| { 17 | let mut sum = 10; 18 | input.get_iter().for_each(|input| { 19 | sum += input.get::().unwrap(); 20 | }); 21 | Output::new(sum) 22 | }); 23 | let mut t1_c = DefaultTask::with_closure("Compute C1", |input, _| { 24 | let mut sum = 20; 25 | input.get_iter().for_each(|input| { 26 | sum += input.get::().unwrap(); 27 | }); 28 | Output::new(sum) 29 | }); 30 | 31 | let mut t1_d = DefaultTask::with_closure("Compute D1", |input, _| { 32 | let mut sum = 30; 33 | input.get_iter().for_each(|input| { 34 | sum += input.get::().unwrap(); 35 | }); 36 | Output::new(sum) 37 | }); 38 | t1_b.set_predecessors(&[&t1_a]); 39 | t1_c.set_predecessors(&[&t1_a]); 40 | t1_d.set_predecessors(&[&t1_b, &t1_c]); 41 | let dag1 = Dag::with_tasks(vec![t1_a, t1_b, t1_c, t1_d]); 42 | // Add dag1 to engine. 43 | engine.append_dag("graph1", dag1); 44 | 45 | // Create some task for dag2. 46 | let t2_a = DefaultTask::with_closure("Compute A2", |_, _| Output::new(2usize)); 47 | let mut t2_b = DefaultTask::with_closure("Compute B2", |input, _| { 48 | let mut sum = 4; 49 | input.get_iter().for_each(|input| { 50 | sum *= input.get::().unwrap(); 51 | }); 52 | Output::new(sum) 53 | }); 54 | let mut t2_c = DefaultTask::with_closure("Compute C2", |input, _| { 55 | let mut sum = 8; 56 | input.get_iter().for_each(|input| { 57 | sum *= input.get::().unwrap(); 58 | }); 59 | Output::new(sum) 60 | }); 61 | let mut t2_d = DefaultTask::with_closure("Compute D2", |input, _| { 62 | let mut sum = 16; 63 | input.get_iter().for_each(|input| { 64 | sum *= input.get::().unwrap(); 65 | }); 66 | Output::new(sum) 67 | }); 68 | t2_b.set_predecessors(&[&t2_a]); 69 | t2_c.set_predecessors(&[&t2_b]); 70 | t2_d.set_predecessors(&[&t2_c]); 71 | let dag2 = Dag::with_tasks(vec![t2_a, t2_b, t2_c, t2_d]); 72 | // Add dag2 to engine. 73 | engine.append_dag("graph2", dag2); 74 | // Read tasks from configuration files and resolve to dag3. 75 | let dag3 = Dag::with_yaml("tests/config/correct.yaml", HashMap::new()).unwrap(); 76 | // Add dag3 to engine. 77 | engine.append_dag("graph3", dag3); 78 | // Execute dag in order, the order should be dag1, dag2, dag3. 79 | assert!(engine.run_sequential().is_ok()); 80 | // Get the execution results of dag1 and dag2. 81 | assert_eq!( 82 | engine.get_dag_result::("graph1").unwrap().as_ref(), 83 | &100 84 | ); 85 | assert_eq!( 86 | engine.get_dag_result::("graph2").unwrap().as_ref(), 87 | &1024 88 | ); 89 | } 90 | -------------------------------------------------------------------------------- /archived/examples/yaml_dag.rs: -------------------------------------------------------------------------------- 1 | //! Read the task information configured in the yaml file. 2 | 3 | extern crate dagrs; 4 | 5 | use dagrs::task::Content; 6 | use dagrs::utils::file::load_file; 7 | use dagrs::Dag; 8 | use std::collections::HashMap; 9 | 10 | fn main() { 11 | env_logger::init(); 12 | let mut job = Dag::with_yaml("tests/config/correct.yaml", HashMap::new()).unwrap(); 13 | assert!(job.start().is_ok()); 14 | 15 | let content = load_file("tests/config/correct.yaml").unwrap(); 16 | let mut job = Dag::with_yaml_str(&content, HashMap::new()).unwrap(); 17 | assert!(job.start().is_ok()); 18 | let out = job.get_results::(); 19 | for (k, v) in out { 20 | println!("{k} {:#?}", v.unwrap().get::<(Vec, Vec)>()); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /archived/src/bin/dagrs.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, fs::File, str::FromStr}; 2 | 3 | use clap::Parser; 4 | use dagrs::Dag; 5 | 6 | #[derive(Parser, Debug)] 7 | #[command(name = "dagrs", version = "0.2.0")] 8 | struct Args { 9 | /// Log output file, the default is to print to the terminal. 10 | #[arg(long)] 11 | log_path: Option, 12 | /// yaml configuration file path. 13 | #[arg(long)] 14 | yaml: String, 15 | /// Log level, the default is 'info'. 16 | #[arg(long)] 17 | log_level: Option, 18 | } 19 | 20 | fn main() { 21 | let args = Args::parse(); 22 | 23 | init_logger(&args); 24 | 25 | let yaml_path = args.yaml; 26 | let mut dag = Dag::with_yaml(yaml_path.as_str(), HashMap::new()).unwrap(); 27 | assert!(dag.start().is_ok()); 28 | } 29 | 30 | fn init_logger(args: &Args) { 31 | let log_level = match &args.log_level { 32 | Some(level_str) => log::LevelFilter::from_str(level_str).unwrap(), 33 | None => log::LevelFilter::Info, 34 | }; 35 | let mut logger_builder = env_logger::Builder::new(); 36 | logger_builder.filter_level(log_level); 37 | 38 | // initialize the env_logger with the given log_path 39 | if let Some(log_path) = &args.log_path { 40 | logger_builder.target(env_logger::Target::Pipe(Box::new( 41 | File::create(log_path).unwrap(), 42 | ))); 43 | }; 44 | 45 | logger_builder.init(); 46 | } 47 | -------------------------------------------------------------------------------- /archived/src/engine/graph.rs: -------------------------------------------------------------------------------- 1 | /*! 2 | Task Graph 3 | 4 | # Graph stores dependency relations. 5 | 6 | [`Graph`] represents a series of tasks with dependencies, and stored in an adjacency 7 | list. It must be a directed acyclic graph, that is, the dependencies of the task 8 | cannot form a loop, otherwise the engine will not be able to execute the task successfully. 9 | It has some useful methods for building graphs, such as: adding edges, nodes, etc. 10 | And the most important of which is the `topo_sort` function, which uses topological 11 | sorting to generate the execution sequence of tasks. 12 | 13 | # An example of a directed acyclic graph 14 | 15 | task1 -→ task3 ---→ task6 ---- 16 | | ↗ ↓ ↓ ↘ 17 | | / task5 ---→ task7 ---→ task9 18 | ↓ / ↑ ↓ ↗ 19 | task2 -→ task4 ---→ task8 ---- 20 | 21 | The task execution sequence can be as follows: 22 | task1->task2->task3->task4->task5->task6->task7->task8->task9 23 | 24 | */ 25 | 26 | use bimap::BiMap; 27 | 28 | #[derive(Debug, Clone)] 29 | /// Graph Struct 30 | pub(crate) struct Graph { 31 | size: usize, 32 | /// Record node id and it's index 33 | nodes: BiMap, 34 | /// Adjacency list of graph (stored as a vector of vector of indices) 35 | adj: Vec>, 36 | /// Node's in_degree, used for topological sort 37 | in_degree: Vec, 38 | } 39 | 40 | impl Graph { 41 | /// Allocate an empty graph 42 | pub(crate) fn new() -> Graph { 43 | Graph { 44 | size: 0, 45 | nodes: BiMap::new(), 46 | adj: Vec::new(), 47 | in_degree: Vec::new(), 48 | } 49 | } 50 | 51 | /// Set graph size, size is the number of tasks 52 | pub(crate) fn set_graph_size(&mut self, size: usize) { 53 | self.size = size; 54 | self.adj.resize(size, Vec::new()); 55 | self.in_degree.resize(size, 0); 56 | self.nodes.reserve(size); 57 | } 58 | 59 | /// Add a node into the graph 60 | /// This operation will create a mapping between ID and its index. 61 | /// **Note:** `id` won't get repeated in dagrs, 62 | /// since yaml parser will overwrite its info if a task's ID repeats. 63 | pub(crate) fn add_node(&mut self, id: usize) { 64 | let index = self.nodes.len(); 65 | self.nodes.insert(id, index); 66 | } 67 | 68 | /// Add an edge into the graph. 69 | /// Above operation adds a arrow from node 0 to node 1, 70 | /// which means task 0 shall be executed before task 1. 71 | pub(crate) fn add_edge(&mut self, v: usize, w: usize) { 72 | self.adj[v].push(w); 73 | self.in_degree[w] += 1; 74 | } 75 | 76 | /// Find a task's index by its ID 77 | pub(crate) fn find_index_by_id(&self, id: &usize) -> Option { 78 | self.nodes.get_by_left(id).copied() 79 | } 80 | 81 | /// Find a task's ID by its index 82 | pub(crate) fn find_id_by_index(&self, index: usize) -> Option { 83 | self.nodes.get_by_right(&index).copied() 84 | } 85 | 86 | /// Do topo sort in graph, returns a possible execution sequence if DAG. 87 | /// This operation will judge whether graph is a DAG or not, 88 | /// returns Some(Possible Sequence) if yes, and None if no. 89 | /// 90 | /// 91 | /// **Note**: this function can only be called after graph's initialization (add nodes and edges, etc.) is done. 92 | /// 93 | /// # Principle 94 | /// Reference: [Topological Sorting](https://www.jianshu.com/p/b59db381561a) 95 | /// 96 | /// 1. For a graph g, we record the in-degree of every node. 97 | /// 98 | /// 2. Each time we start from a node with zero in-degree, name it N0, and N0 can be executed since it has no dependency. 99 | /// 100 | /// 3. And then we decrease the in-degree of N0's children (those tasks depend on N0), this would create some new zero in-degree nodes. 101 | /// 102 | /// 4. Just repeat step 2, 3 until no more zero degree nodes can be generated. 103 | /// If all tasks have been executed, then it's a DAG, or there must be a loop in the graph. 104 | pub(crate) fn topo_sort(&self) -> Option> { 105 | let mut queue = self 106 | .in_degree 107 | .iter() 108 | .enumerate() 109 | .filter_map(|(index, °ree)| if degree == 0 { Some(index) } else { None }) 110 | .collect::>(); 111 | 112 | let mut in_degree = self.in_degree.clone(); 113 | 114 | let mut sequence = Vec::with_capacity(self.size); 115 | 116 | while let Some(v) = queue.pop() { 117 | sequence.push(v); 118 | 119 | for &index in self.adj[v].iter() { 120 | in_degree[index] -= 1; 121 | if in_degree[index] == 0 { 122 | queue.push(index) 123 | } 124 | } 125 | } 126 | 127 | if sequence.len() < self.size { 128 | None 129 | } else { 130 | Some(sequence) 131 | } 132 | } 133 | 134 | /// Get the out degree of a node. 135 | pub(crate) fn get_node_out_degree(&self, id: &usize) -> usize { 136 | match self.nodes.get_by_left(id) { 137 | Some(index) => self.adj[*index].len(), 138 | None => 0, 139 | } 140 | } 141 | 142 | /// Get all the successors of a node (direct or indirect). 143 | /// This function will return a vector of indices of successors (including itself). 144 | pub(crate) fn get_node_successors(&self, id: &usize) -> Vec { 145 | match self.nodes.get_by_left(id) { 146 | Some(index) => { 147 | // initialize a vector to store successors with max possible size 148 | let mut successors = Vec::with_capacity(self.adj[*index].len()); 149 | 150 | // create a visited array to avoid visiting a node more than once 151 | let mut visited = vec![false; self.size]; 152 | 153 | // do BFS traversal starting from current node 154 | 155 | // mark the current node as visited and enqueue it 156 | visited[*index] = true; 157 | successors.push(*index); 158 | 159 | // the index of the queue 160 | let mut i_queue = 0; 161 | 162 | // while the queue is not empty 163 | while i_queue < successors.len() { 164 | let v = successors[i_queue]; 165 | 166 | for &index in self.adj[v].iter() { 167 | // if not visited, mark it as visited and collect it 168 | if !visited[index] { 169 | visited[index] = true; 170 | successors.push(index); 171 | } 172 | } 173 | i_queue += 1; 174 | } 175 | successors 176 | } 177 | // If node not found, return empty vector 178 | None => Vec::new(), 179 | } 180 | } 181 | } 182 | 183 | impl Default for Graph { 184 | fn default() -> Self { 185 | Graph { 186 | size: 0, 187 | nodes: BiMap::new(), 188 | adj: Vec::new(), 189 | in_degree: Vec::new(), 190 | } 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /archived/src/engine/mod.rs: -------------------------------------------------------------------------------- 1 | //! The Engine 2 | //! 3 | //! [`Dag`] consists of a series of executable tasks with dependencies. A Dag can be executed 4 | //! alone as a job. We can get the execution result and execution status of dag. 5 | //! [`Engine`] can manage multiple [`Dag`]. An Engine can consist of multiple Dags of different 6 | //! types of tasks. For example, you can give a Dag in the form of a yaml configuration file, 7 | //! then give a Dag in the form of a custom configuration file, and finally give it in a programmatic way. 8 | //! [`Engine`] stores each Dag in the form of a key-value pair (), and the user 9 | //! can specify which task to execute by giving the name of the Dag, or follow the order in which 10 | //! the Dags are added to the Engine , executing each Dag in turn. 11 | 12 | pub use dag::Dag; 13 | use log::error; 14 | use thiserror::Error; 15 | 16 | mod dag; 17 | mod graph; 18 | 19 | use std::{collections::HashMap, sync::Arc}; 20 | use tokio::runtime::Runtime; 21 | 22 | /// The Engine. Manage multiple Dags. 23 | pub struct Engine { 24 | dags: HashMap, 25 | /// According to the order in which Dags are added to the Engine, assign a sequence number to each Dag. 26 | /// Sequence numbers can be used to execute Dags sequentially. 27 | sequence: HashMap, 28 | /// A tokio runtime. 29 | /// In order to save computer resources, multiple Dags share one runtime. 30 | runtime: Runtime, 31 | } 32 | 33 | /// Errors that may be raised by building and running dag jobs. 34 | #[derive(Debug, Error)] 35 | /// A synthesis of all possible errors. 36 | pub enum DagError { 37 | /// Yaml file parsing error. 38 | #[error("Parsing error: {0}")] 39 | ParserError(String), 40 | /// Task dependency error. 41 | #[error("Task[{0}] dependency task not exist.")] 42 | RelyTaskIllegal(String), 43 | /// There are loops in task dependencies. 44 | #[error("Illegal directed a cyclic graph, loop Detect!")] 45 | LoopGraph, 46 | /// There are no tasks in the job. 47 | #[error("There are no tasks in the job.")] 48 | EmptyJob, 49 | /// Task error 50 | #[error("Task error: {0}")] 51 | TaskError(String), 52 | } 53 | 54 | impl Engine { 55 | /// Add a Dag to the Engine and assign a sequence number to the Dag. 56 | /// It should be noted that different Dags should specify different names. 57 | pub fn append_dag(&mut self, name: &str, mut dag: Dag) { 58 | if !self.dags.contains_key(name) { 59 | match dag.init() { 60 | Ok(()) => { 61 | self.dags.insert(name.to_string(), dag); 62 | let len = self.sequence.len(); 63 | self.sequence.insert(len + 1, name.to_string()); 64 | } 65 | Err(err) => { 66 | error!("Some error occur: {}", err); 67 | } 68 | } 69 | } 70 | } 71 | 72 | /// Given a Dag name, execute this Dag. 73 | /// Returns true if the given Dag executes successfully, otherwise false. 74 | pub fn run_dag(&mut self, name: &str) -> Result<(), DagError> { 75 | if let Some(dag) = self.dags.get(name) { 76 | self.runtime.block_on(dag.run()) 77 | } else { 78 | error!("No job named '{}'", name); 79 | Err(DagError::EmptyJob) 80 | } 81 | } 82 | 83 | /// Execute all the Dags in the Engine in sequence according to the order numbers of the Dags in 84 | pub fn run_sequential(&mut self) -> Result<(), DagError> { 85 | for seq in 1..self.sequence.len() + 1 { 86 | let name = self.sequence.get(&seq).unwrap().clone(); 87 | self.run_dag(name.as_str())?; 88 | } 89 | Ok(()) 90 | } 91 | 92 | /// Given the name of the Dag, get the execution result of the specified Dag. 93 | pub fn get_dag_result(&self, name: &str) -> Option> { 94 | self.dags.get(name).and_then(|dag| dag.get_result()) 95 | } 96 | } 97 | 98 | impl Default for Engine { 99 | fn default() -> Self { 100 | Self { 101 | dags: HashMap::new(), 102 | runtime: Runtime::new().unwrap(), 103 | sequence: HashMap::new(), 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /archived/src/lib.rs: -------------------------------------------------------------------------------- 1 | extern crate bimap; 2 | extern crate clap; 3 | #[cfg(feature = "derive")] 4 | extern crate derive; 5 | extern crate tokio; 6 | #[cfg(feature = "yaml")] 7 | extern crate yaml_rust; 8 | 9 | #[cfg(feature = "derive")] 10 | pub use derive::*; 11 | pub use engine::{Dag, DagError, Engine}; 12 | pub use task::{ 13 | alloc_id, Action, CommandAction, Complex, DefaultTask, Input, Output, Simple, Task, 14 | }; 15 | pub use utils::{EnvVar, Parser}; 16 | #[cfg(feature = "yaml")] 17 | pub use yaml::{FileContentError, FileNotFound, YamlParser, YamlTask, YamlTaskError}; 18 | 19 | pub mod engine; 20 | pub mod task; 21 | pub mod utils; 22 | #[cfg(feature = "yaml")] 23 | pub mod yaml; 24 | -------------------------------------------------------------------------------- /archived/src/task/action.rs: -------------------------------------------------------------------------------- 1 | use crate::{EnvVar, Input, Output}; 2 | use std::sync::Arc; 3 | 4 | /// The type of closure that performs logic. 5 | /// # [`Simple`] 6 | /// 7 | /// The specific type of [`Simple`] is `dyn Fn(Input, Arc) -> Output + Send + Sync`, 8 | /// which represents a closure. 9 | /// 10 | /// # Example 11 | /// 12 | /// ```rust 13 | /// use std::sync::Arc; 14 | /// use dagrs::{Action,Input,EnvVar,Output}; 15 | /// 16 | /// let closure=|_input,_env|Output::new(10); 17 | /// let action=Action::Closure(Arc::new(closure)); 18 | /// ``` 19 | pub type Simple = dyn Fn(Input, Arc) -> Output + Send + Sync; 20 | 21 | /// More complex types of execution logic. 22 | /// # [`Complex`] 23 | /// 24 | /// The usage of closures is suitable for simple cases. If the user wants to store some private 25 | /// properties when defining execution logic, the [`Complex`] trait can meet the needs. 26 | /// 27 | /// # Example 28 | /// 29 | /// ```rust 30 | /// use std::sync::Arc; 31 | /// use dagrs::{Action,Input,EnvVar,Output,Complex}; 32 | /// 33 | /// struct HelloAction{ 34 | /// statement: String, 35 | /// repeat: usize, 36 | /// } 37 | /// 38 | /// impl Complex for HelloAction{ 39 | /// fn run(&self, input: Input, env: Arc) -> Output{ 40 | /// for i in 0..self.repeat { 41 | /// println!("{}",self.statement); 42 | /// } 43 | /// Output::empty() 44 | /// } 45 | /// } 46 | /// 47 | /// let hello=HelloAction { 48 | /// statement: "hello world!".to_string(), 49 | /// repeat: 10 50 | /// }; 51 | /// let action = Action::Structure(Arc::new(hello)); 52 | /// ``` 53 | pub trait Complex { 54 | fn run(&self, input: Input, env: Arc) -> Output; 55 | } 56 | 57 | /// Task specific behavior 58 | /// 59 | /// [`Action`] stores the specific execution logic of a task. Action::Closure(Arc<[`Simple`]>) represents a 60 | /// closure, and Action::Structure(Arc) represents a specific type that 61 | /// implements the [`Complex`] trait. 62 | /// Attributes that must exist in each task are used to store specific execution logic. Specific 63 | /// execution logic can be given in two forms: given a closure or a specific type that implements 64 | /// a Complex trait. 65 | #[derive(Clone)] 66 | pub enum Action { 67 | Closure(Arc), 68 | Structure(Arc), 69 | } 70 | 71 | impl Action { 72 | pub fn run(&self, input: Input, env: Arc) -> Output { 73 | match self { 74 | Self::Closure(closure) => closure(input, env), 75 | Self::Structure(structure) => structure.run(input, env), 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /archived/src/task/cmd.rs: -------------------------------------------------------------------------------- 1 | use crate::{Complex, EnvVar, Input, Output}; 2 | use std::process::Command; 3 | use std::sync::Arc; 4 | 5 | use crate::task::Content; 6 | 7 | /// [`CommandAction`] is a specific implementation of [`Complex`], used to execute operating system commands. 8 | pub struct CommandAction { 9 | command: String, 10 | } 11 | 12 | impl CommandAction { 13 | #[allow(unused)] 14 | pub fn new(cmd: &str) -> Self { 15 | Self { 16 | command: cmd.to_owned(), 17 | } 18 | } 19 | } 20 | 21 | impl Complex for CommandAction { 22 | fn run(&self, input: Input, _env: Arc) -> Output { 23 | let mut args = Vec::new(); 24 | let mut cmd = if cfg!(target_os = "windows") { 25 | args.push("-Command"); 26 | Command::new("powershell") 27 | } else { 28 | args.push("-c"); 29 | Command::new("sh") 30 | }; 31 | args.push(&self.command); 32 | 33 | input.get_iter().for_each(|input| { 34 | if let Some(inp) = input.get::() { 35 | args.push(inp) 36 | } 37 | }); 38 | 39 | log::info!("cmd: {:?}, args: {:?}", cmd.get_program(), args); 40 | let out = match cmd.args(args).output() { 41 | Ok(o) => o, 42 | Err(e) => { 43 | return Output::error_with_exit_code( 44 | e.raw_os_error(), 45 | Some(Content::new(e.to_string())), 46 | ) 47 | } 48 | }; 49 | let code = out.status.code().unwrap_or(0); 50 | let stdout: Vec = { 51 | let out = String::from_utf8(out.stdout).unwrap_or("".to_string()); 52 | if cfg!(target_os = "windows") { 53 | out.rsplit_terminator("\r\n").map(str::to_string).collect() 54 | } else { 55 | out.split_terminator('\n').map(str::to_string).collect() 56 | } 57 | }; 58 | let stderr: Vec = { 59 | let out = String::from_utf8(out.stderr).unwrap_or("".to_string()); 60 | if cfg!(target_os = "windows") { 61 | out.rsplit_terminator("\r\n").map(str::to_string).collect() 62 | } else { 63 | out.split_terminator('\n').map(str::to_string).collect() 64 | } 65 | }; 66 | if out.status.success() { 67 | Output::new((stdout,stderr)) 68 | } else { 69 | let output = Content::new((stdout, stderr)); 70 | Output::error_with_exit_code(Some(code), Some(output)) 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /archived/src/task/default_task.rs: -------------------------------------------------------------------------------- 1 | use super::{Action, Complex, Task, ID_ALLOCATOR}; 2 | use crate::{EnvVar, Input, Output}; 3 | use std::sync::Arc; 4 | 5 | /// Common task types 6 | /// 7 | /// [`DefaultTask`] is a default implementation of the [`Task`] trait. Users can use this task 8 | /// type to build tasks to meet most needs. 9 | /// 10 | /// There are four ways to create a DefaultTask: 11 | /// 12 | /// #Example 13 | /// 14 | /// Using the `default` function to create a task is the simplest way. The name of the task defaults 15 | /// to "Task $id", and a closure without output is created by default as the execution logic of 16 | /// the task. Subsequently, users can specify the task name through the `set_name` function, and 17 | /// use the `set_action` or `set_closure` function to specify execution logic for the task. 18 | /// 19 | /// ```rust 20 | /// use dagrs::DefaultTask; 21 | /// let mut task=DefaultTask::default(); 22 | /// ``` 23 | /// Use the `new` function to create a task. The only difference between this function and the `default` 24 | /// function is that the task name is also given when creating the task. The subsequent work is the 25 | /// same as mentioned in the `default` function. 26 | /// 27 | /// ```rust 28 | /// use dagrs::DefaultTask; 29 | /// let mut task=DefaultTask::new("task"); 30 | /// ``` 31 | /// To build task execution logic, please see `action` module. 32 | /// 33 | /// Use the `with_closure` function to create a task and give it a task name and execution logic. 34 | /// The execution logic is given in the form of a closure. 35 | /// 36 | /// ```rust 37 | /// use dagrs::{ DefaultTask, Output }; 38 | /// let mut task = DefaultTask::with_closure("task",|_input,_env|Output::empty()); 39 | /// ``` 40 | /// Use the `with_action` function to create a task and give it a name and execution logic. 41 | /// The execution logic is given in the form of a concrete type that implements the [`Complex`] trait. 42 | /// For an explanation of the [`Complex`] feature, please see the `action` module. 43 | /// 44 | /// ```rust 45 | /// use dagrs::{ DefaultTask, Complex, Output, Input, EnvVar }; 46 | /// use std::sync::Arc; 47 | /// 48 | /// struct Act(u32); 49 | /// 50 | /// impl Complex for Act{ 51 | /// fn run(&self, input: Input, env: Arc) -> Output{ 52 | /// Output::new(self.0+10) 53 | /// } 54 | /// } 55 | /// 56 | /// let mut task = DefaultTask::with_action("task",Act(20)); 57 | /// ``` 58 | /// 59 | /// A default implementation of the Task trait. In general, use it to define the tasks of dagrs. 60 | #[derive(Clone)] 61 | pub struct DefaultTask { 62 | /// id is the unique identifier of each task, it will be assigned by the global [`IDAllocator`] 63 | /// when creating a new task, you can find this task through this identifier. 64 | id: usize, 65 | /// The task's name. 66 | name: String, 67 | /// Id of the predecessor tasks. 68 | precursors: Vec, 69 | /// Perform specific actions. 70 | action: Action, 71 | } 72 | 73 | impl DefaultTask { 74 | /// Create a task and specify the task name. You may need to call the `set_action` or `set_closure` function later. 75 | pub fn new(name: &str) -> Self { 76 | let action = |_, _| Output::empty(); 77 | DefaultTask { 78 | id: ID_ALLOCATOR.alloc(), 79 | action: Action::Closure(Arc::new(action)), 80 | name: name.to_owned(), 81 | precursors: Vec::new(), 82 | } 83 | } 84 | /// Create a task, give the task name, and provide a specific type that implements the [`Complex`] trait as the specific 85 | /// execution logic of the task. 86 | pub fn with_action(name: &str, action: impl Complex + Send + Sync + 'static) -> Self { 87 | Self::with_action_dyn(name, Arc::new(action)) 88 | } 89 | 90 | /// Create a task, give the task name, and provide a dynamic task that implements the [`Complex`] trait as the specific 91 | /// execution logic of the task. 92 | pub fn with_action_dyn(name: &str, action: Arc) -> Self { 93 | DefaultTask { 94 | id: ID_ALLOCATOR.alloc(), 95 | action: Action::Structure(action), 96 | name: name.to_owned(), 97 | precursors: Vec::new(), 98 | } 99 | } 100 | 101 | /// Create a task, give the task name, and provide a closure as the specific execution logic of the task. 102 | pub fn with_closure( 103 | name: &str, 104 | action: impl Fn(Input, Arc) -> Output + Send + Sync + 'static, 105 | ) -> Self { 106 | Self::with_closure_dyn(name, Arc::new(action)) 107 | } 108 | 109 | /// Create a task, give the task name, and provide a closure as the specific execution logic of the task. 110 | pub fn with_closure_dyn( 111 | name: &str, 112 | action: Arc) -> Output + Send + Sync>, 113 | ) -> Self { 114 | DefaultTask { 115 | id: ID_ALLOCATOR.alloc(), 116 | action: Action::Closure(action), 117 | name: name.to_owned(), 118 | precursors: Vec::new(), 119 | } 120 | } 121 | 122 | /// Give the task a name. 123 | pub fn set_name(&mut self, name: &str) { 124 | self.name = name.to_string(); 125 | } 126 | 127 | /// Tasks that shall be executed before this one. 128 | /// 129 | /// # Example 130 | /// ```rust 131 | /// use dagrs::{DefaultTask,Output}; 132 | /// let t1 = DefaultTask::with_closure("Task 1", |_input,_env|Output::empty()); 133 | /// let mut t2 = DefaultTask::with_closure("Task 2",|_input,_env|Output::empty()); 134 | /// t2.set_predecessors(&[&t1]); 135 | /// ``` 136 | /// In above code, `t1` will be executed before `t2`. 137 | pub fn set_predecessors<'a>( 138 | &mut self, 139 | predecessors: impl IntoIterator, 140 | ) { 141 | self.precursors 142 | .extend(predecessors.into_iter().map(|t| t.id())) 143 | } 144 | 145 | /// The same as `exec_after`, but input are tasks' ids 146 | /// rather than reference to [`DefaultTask`]. 147 | pub fn set_predecessors_by_id(&mut self, predecessors_id: impl IntoIterator) { 148 | self.precursors.extend(predecessors_id) 149 | } 150 | 151 | /// Provide a closure to specify execution logic for the task. 152 | pub fn set_closure( 153 | &mut self, 154 | action: impl Fn(Input, Arc) -> Output + Send + Sync + 'static, 155 | ) { 156 | self.action = Action::Closure(Arc::new(action)); 157 | } 158 | 159 | /// Provide a concrete type that implements the [`Complex`] trait to specify execution logic for the task. 160 | pub fn set_action(&mut self, action: impl Complex + Send + Sync + 'static) { 161 | self.action = Action::Structure(Arc::new(action)) 162 | } 163 | } 164 | 165 | impl Task for DefaultTask { 166 | fn action(&self) -> Action { 167 | self.action.clone() 168 | } 169 | 170 | fn precursors(&self) -> &[usize] { 171 | &self.precursors 172 | } 173 | 174 | fn id(&self) -> usize { 175 | self.id 176 | } 177 | 178 | fn name(&self) -> &str { 179 | &self.name 180 | } 181 | } 182 | 183 | impl Default for DefaultTask { 184 | fn default() -> Self { 185 | let id = ID_ALLOCATOR.alloc(); 186 | let name = format!("Task {}", id); 187 | let action = |_, _| Output::empty(); 188 | Self { 189 | id, 190 | name, 191 | precursors: Vec::new(), 192 | action: Action::Closure(Arc::new(action)), 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /archived/src/task/mod.rs: -------------------------------------------------------------------------------- 1 | //! Relevant definitions of tasks. 2 | //! 3 | //! # [`Task`]: the basic unit of scheduling 4 | //! 5 | //! A [`Task`] is the basic unit for scheduling execution of a dagrs. [`Task`] itself is a trait and users 6 | //! should use its concrete implementation [`DefaultTask`]. Of course, users can also customize [`Task`], 7 | //! but it should be noted that whether it is the default [`DefaultTask`] or a user-defined task type, they 8 | //! all need to have the following fields: 9 | //! - `id`: type is `usize`. When allocating tasks, there is a global task `id` allocator. 10 | //! Users can call the `alloc_id()` function to assign ids to tasks, and the obtained `id` type is `usize`. 11 | //! - `name`: type is `String`. This field represents the task name. 12 | //! - `action`: type is [`Action`]. This field is used to store the specific execution logic of the task. 13 | //! - `precursors`: type is `Vec`. This field is used to store the predecessor task `id` of this task. 14 | //! 15 | //! # [`Action`]: specific logical behavior 16 | //! 17 | //! Each task has an [`Action`] field inside, which stores the specific execution logic of the task. 18 | //! [`Action`] is an enumeration type. For [`Simple`] execution logic, you only need to provide a closure for [`Action`]. 19 | //! For slightly more complex execution logic, you can implement the [`Complex`] trait. For detailed analysis, 20 | //! please see the `action` module. 21 | //! 22 | //! # [`Input`] and [`Output`] 23 | //! 24 | //! Each task may produce output and may require the output of its predecessor task as its input. 25 | //! [`Output`] is used to construct and store the output obtained by task execution. [`Input`] is used as a tool 26 | //! to provide users with the output of the predecessor task. 27 | use std::fmt::Debug; 28 | use std::sync::atomic::AtomicUsize; 29 | 30 | pub use self::action::{Action, Complex, Simple}; 31 | pub use self::cmd::CommandAction; 32 | pub use self::default_task::DefaultTask; 33 | pub use self::state::Content; 34 | pub(crate) use self::state::ExecState; 35 | pub use self::state::{Input, Output}; 36 | 37 | mod action; 38 | mod cmd; 39 | mod default_task; 40 | mod state; 41 | /// The Task trait 42 | /// 43 | /// Tasks can have many attributes, among which `id`, `name`, `predecessor_tasks`, and 44 | /// `action` attributes are required, and users can also customize some other attributes. 45 | /// [`DefaultTask`] in this module is a [`Task`], the DAG engine uses it as the basic 46 | /// task by default. 47 | /// 48 | /// A task must provide methods to obtain precursors and required attributes, just as 49 | /// the methods defined below, users who want to customize tasks must implement these methods. 50 | pub trait Task: Send + Sync { 51 | /// Get a reference to an executable action. 52 | fn action(&self) -> Action; 53 | /// Get the id of all predecessor tasks of this task. 54 | fn precursors(&self) -> &[usize]; 55 | /// Get the id of this task. 56 | fn id(&self) -> usize; 57 | /// Get the name of this task. 58 | fn name(&self) -> &str; 59 | } 60 | 61 | /// IDAllocator for DefaultTask 62 | struct IDAllocator { 63 | id: AtomicUsize, 64 | } 65 | 66 | impl Debug for dyn Task { 67 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 68 | writeln!( 69 | f, 70 | "{},\t{},\t{:?}", 71 | self.id(), 72 | self.name(), 73 | self.precursors() 74 | ) 75 | } 76 | } 77 | 78 | impl IDAllocator { 79 | fn alloc(&self) -> usize { 80 | let origin = self.id.fetch_add(1, std::sync::atomic::Ordering::SeqCst); 81 | if origin > self.id.load(std::sync::atomic::Ordering::Relaxed) { 82 | panic!("Too many tasks.") 83 | } else { 84 | origin 85 | } 86 | } 87 | } 88 | 89 | /// The global task uniquely identifies an instance of the allocator. 90 | static ID_ALLOCATOR: IDAllocator = IDAllocator { 91 | id: AtomicUsize::new(1), 92 | }; 93 | 94 | /// public function to assign task's id. 95 | pub fn alloc_id() -> usize { 96 | ID_ALLOCATOR.alloc() 97 | } 98 | -------------------------------------------------------------------------------- /archived/src/utils/env.rs: -------------------------------------------------------------------------------- 1 | use crate::task::Content; 2 | 3 | use std::collections::HashMap; 4 | 5 | pub type Variable = Content; 6 | 7 | /// # Environment variable. 8 | /// 9 | /// When multiple tasks are running, they may need to share the same data or read 10 | /// the same configuration information. Environment variables can meet this requirement. 11 | /// Before all tasks run, the user builds a [`EnvVar`] and sets all the environment 12 | /// variables. One [`EnvVar`] corresponds to one dag. All tasks in a job can 13 | /// be shared and immutable at runtime. environment variables. 14 | #[derive(Debug, Default)] 15 | pub struct EnvVar { 16 | variables: HashMap, 17 | } 18 | 19 | impl EnvVar { 20 | /// Allocate a new [`EnvVar`]. 21 | pub fn new() -> Self { 22 | Self { 23 | variables: HashMap::new(), 24 | } 25 | } 26 | 27 | #[allow(unused)] 28 | /// Set a global variables. 29 | /// 30 | /// # Example 31 | /// ```rust 32 | /// # let mut env = dagrs::EnvVar::new(); 33 | /// env.set("Hello", "World".to_string()); 34 | /// ``` 35 | pub fn set(&mut self, name: &str, var: H) { 36 | let mut v = Variable::new(var); 37 | self.variables.insert(name.to_owned(), v); 38 | } 39 | 40 | /// Get environment variables through keys of type &str. 41 | /// 42 | /// Note: This method will clone the value. To avoid cloning, use [`get_ref`]. 43 | pub fn get(&self, name: &str) -> Option { 44 | self.get_ref(name).cloned() 45 | } 46 | 47 | /// Get environment variables through keys of type &str. 48 | pub fn get_ref(&self, name: &str) -> Option<&H> { 49 | if let Some(content) = self.variables.get(name) { 50 | content.get() 51 | } else { 52 | None 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /archived/src/utils/file.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::{Error, Read}; 3 | 4 | /// Given file path, and load configuration file. 5 | pub fn load_file(file: &str) -> Result { 6 | let mut content = String::new(); 7 | let mut fh = File::open(file)?; 8 | fh.read_to_string(&mut content)?; 9 | Ok(content) 10 | } 11 | -------------------------------------------------------------------------------- /archived/src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | //! general tool. 2 | //! 3 | //! This module contains common tools for the program, such as: environment 4 | //! variables, task generation macros. 5 | 6 | mod env; 7 | pub mod file; 8 | mod parser; 9 | 10 | pub use self::env::EnvVar; 11 | pub use self::parser::Parser; 12 | -------------------------------------------------------------------------------- /archived/src/utils/parser.rs: -------------------------------------------------------------------------------- 1 | //! Task configuration file parser interface 2 | 3 | use crate::{task::Task, Action, DagError}; 4 | use std::collections::HashMap; 5 | 6 | /// Generic parser traits. If users want to customize the configuration file parser, they must implement this trait. 7 | /// The yaml module's `YamlParser` is an example. 8 | pub trait Parser { 9 | /// Parses the contents of a configuration file into a series of tasks with dependencies. 10 | /// Parameter Description: 11 | /// - file: path information of the configuration file 12 | /// - specific_actions: When parsing the configuration file, the specific execution logic 13 | /// of some tasks does not need to be specified in the configuration file, but is given 14 | /// through this map. In the map's key-value pair, the key represents the unique identifier 15 | /// of the task in the task's configuration file, and the value represents the execution 16 | /// logic given by the user. 17 | /// 18 | /// Return value description: 19 | /// If an error is encountered during the parsing process, the return result is ParserError. 20 | /// Instead, return a series of concrete types that implement the [`Task`] trait. 21 | /// This may involve user-defined [`Task`], you can refer to `YamlTask` under the yaml module. 22 | fn parse_tasks( 23 | &self, 24 | file: &str, 25 | specific_actions: HashMap, 26 | ) -> Result>, DagError>; 27 | 28 | fn parse_tasks_from_str( 29 | &self, 30 | content: &str, 31 | specific_actions: HashMap, 32 | ) -> Result>, DagError>; 33 | } 34 | -------------------------------------------------------------------------------- /archived/src/yaml/mod.rs: -------------------------------------------------------------------------------- 1 | //! yaml configuration file type parser 2 | //! 3 | //! # Config file parser 4 | //! 5 | //! Use yaml configuration files to define a series of tasks, which eliminates the need for users to write code. 6 | //! [`YamlParser`] is responsible for parsing the yaml configuration file into a series of [`YamlTask`]. 7 | //! The program specifies the properties of the yaml task configuration file. The basic format of the yaml 8 | //! configuration file is as follows: 9 | //! 10 | //! ```yaml 11 | //! dagrs: 12 | //! a: 13 | //! name: "Task 1" 14 | //! after: [ b, c ] 15 | //! cmd: echo a 16 | //! b: 17 | //! name: "Task 2" 18 | //! after: [ c, f, g ] 19 | //! cmd: echo b 20 | //! c: 21 | //! name: "Task 3" 22 | //! after: [ e, g ] 23 | //! cmd: echo c 24 | //! d: 25 | //! name: "Task 4" 26 | //! after: [ c, e ] 27 | //! cmd: echo d 28 | //! e: 29 | //! name: "Task 5" 30 | //! after: [ h ] 31 | //! cmd: echo e 32 | //! f: 33 | //! name: "Task 6" 34 | //! after: [ g ] 35 | //! cmd: python3 ./tests/config/test.py 36 | //! g: 37 | //! name: "Task 7" 38 | //! after: [ h ] 39 | //! cmd: node ./tests/config/test.js 40 | //! h: 41 | //! name: "Task 8" 42 | //! cmd: echo h 43 | //! ``` 44 | //! 45 | //! Users can read the yaml configuration file programmatically or by using the compiled `dagrs` 46 | //! command line tool. Either way, you need to enable the `yaml` feature. 47 | //! 48 | //! # Example 49 | //! 50 | //! ```rust 51 | //! use dagrs::Dag; 52 | //! let dag = Dag::with_yaml("some_path",std::collections::HashMap::new()); 53 | //! ``` 54 | 55 | mod yaml_parser; 56 | mod yaml_task; 57 | 58 | use crate::DagError; 59 | use thiserror::Error; 60 | 61 | pub use self::yaml_parser::YamlParser; 62 | pub use self::yaml_task::YamlTask; 63 | 64 | /// Errors about task configuration items. 65 | #[derive(Debug, Error)] 66 | pub enum YamlTaskError { 67 | /// The configuration file should start with `dagrs:`. 68 | #[error("File content is not start with 'dagrs'.")] 69 | StartWordError, 70 | /// No task name configured. 71 | #[error("Task has no name field. [{0}]")] 72 | NoNameAttr(String), 73 | /// The specified task predecessor was not found. 74 | #[error("Task cannot find the specified predecessor. [{0}]")] 75 | NotFoundPrecursor(String), 76 | /// `script` is not defined. 77 | #[error("The 'script' attribute is not defined. [{0}]")] 78 | NoScriptAttr(String), 79 | } 80 | 81 | /// Error about file information. 82 | #[derive(Debug, Error)] 83 | pub enum FileContentError { 84 | /// The format of the yaml configuration file is not standardized. 85 | #[error("Illegal yaml content: {0}")] 86 | IllegalYamlContent(yaml_rust::ScanError), 87 | /// The file is empty. 88 | #[error("File is empty! [{0}]")] 89 | Empty(String), 90 | } 91 | 92 | /// Configuration file not found. 93 | #[derive(Debug, Error)] 94 | #[error("File not found. [{0}]")] 95 | pub struct FileNotFound(pub std::io::Error); 96 | 97 | impl From for DagError { 98 | fn from(value: YamlTaskError) -> Self { 99 | DagError::ParserError(value.to_string()) 100 | } 101 | } 102 | 103 | impl From for DagError { 104 | fn from(value: FileContentError) -> Self { 105 | DagError::ParserError(value.to_string()) 106 | } 107 | } 108 | 109 | impl From for DagError { 110 | fn from(value: FileNotFound) -> Self { 111 | DagError::ParserError(value.to_string().into()) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /archived/src/yaml/yaml_parser.rs: -------------------------------------------------------------------------------- 1 | //! Default yaml configuration file parser. 2 | 3 | use super::{FileContentError, YamlTask, YamlTaskError}; 4 | use crate::{utils::file::load_file, Action, CommandAction, DagError, Parser, Task}; 5 | use std::{collections::HashMap, sync::Arc}; 6 | use yaml_rust::{Yaml, YamlLoader}; 7 | 8 | /// An implementation of [`Parser`]. It is the default yaml configuration file parser. 9 | pub struct YamlParser; 10 | 11 | impl YamlParser { 12 | /// Parses an item in the configuration file into a task. 13 | /// An item refers to: 14 | /// 15 | /// ```yaml 16 | /// name: "Task 1" 17 | /// after: [b, c] 18 | /// cmd: echo a 19 | /// ``` 20 | fn parse_one( 21 | &self, 22 | id: &str, 23 | item: &Yaml, 24 | specific_action: Option, 25 | ) -> Result { 26 | // Get name first 27 | let name = item["name"] 28 | .as_str() 29 | .ok_or(YamlTaskError::NoNameAttr(id.to_owned()))? 30 | .to_owned(); 31 | // precursors can be empty 32 | let mut precursors = Vec::new(); 33 | if let Some(after_tasks) = item["after"].as_vec() { 34 | after_tasks 35 | .iter() 36 | .for_each(|task_id| precursors.push(task_id.as_str().unwrap().to_owned())); 37 | } 38 | 39 | if let Some(action) = specific_action { 40 | Ok(YamlTask::new(id, precursors, name, action)) 41 | } else { 42 | let cmd = item["cmd"] 43 | .as_str() 44 | .ok_or(YamlTaskError::NoScriptAttr(name.clone()))?; 45 | Ok(YamlTask::new( 46 | id, 47 | precursors, 48 | name, 49 | Action::Structure(Arc::new(CommandAction::new(cmd))), 50 | )) 51 | } 52 | } 53 | } 54 | 55 | impl Parser for YamlParser { 56 | fn parse_tasks( 57 | &self, 58 | file: &str, 59 | specific_actions: HashMap, 60 | ) -> Result>, DagError> { 61 | let content = load_file(file).map_err(|e| DagError::ParserError(e.to_string()))?; 62 | self.parse_tasks_from_str(&content, specific_actions) 63 | } 64 | 65 | fn parse_tasks_from_str( 66 | &self, 67 | content: &str, 68 | mut specific_actions: HashMap, 69 | ) -> Result>, DagError> { 70 | // Parse Yaml 71 | let yaml_tasks = 72 | YamlLoader::load_from_str(content).map_err(FileContentError::IllegalYamlContent)?; 73 | if yaml_tasks.is_empty() { 74 | return Err(DagError::ParserError("No Tasks found".to_string())); 75 | } 76 | let yaml_tasks = yaml_tasks[0]["dagrs"] 77 | .as_hash() 78 | .ok_or(YamlTaskError::StartWordError)?; 79 | 80 | let mut tasks = Vec::with_capacity(yaml_tasks.len()); 81 | let mut map = HashMap::with_capacity(yaml_tasks.len()); 82 | // Read tasks 83 | for (v, w) in yaml_tasks { 84 | let id = v 85 | .as_str() 86 | .ok_or(DagError::ParserError("Invalid YAML Node Type".to_string()))?; 87 | let task = specific_actions.remove(id).map_or_else( 88 | || self.parse_one(id, w, None), 89 | |action| self.parse_one(id, w, Some(action)), 90 | )?; 91 | map.insert(id, task.id()); 92 | tasks.push(task); 93 | } 94 | 95 | for task in tasks.iter_mut() { 96 | let mut pres = Vec::new(); 97 | for pre in task.str_precursors() { 98 | if map.contains_key(&pre[..]) { 99 | pres.push(map[&pre[..]]); 100 | } else { 101 | return Err(YamlTaskError::NotFoundPrecursor(task.name().to_string()).into()); 102 | } 103 | } 104 | task.init_precursors(pres); 105 | } 106 | 107 | Ok(tasks 108 | .into_iter() 109 | .map(|task| Box::new(task) as Box) 110 | .collect()) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /archived/src/yaml/yaml_task.rs: -------------------------------------------------------------------------------- 1 | //! Task definition of type Yaml. 2 | //! 3 | //! # The task type corresponding to the configuration file: [`YamlTask`] 4 | //! 5 | //! [`YamlTask`] implements the [`Task`] trait, which represents the tasks in the yaml 6 | //! configuration file, and a yaml configuration file will be parsed into a series of [`YamlTask`]. 7 | //! It is different from `DefaultTask`, in addition to the four mandatory attributes of the 8 | //! task type, he has several additional attributes. 9 | 10 | use crate::{alloc_id, Action, Task}; 11 | 12 | /// Task struct for yaml file. 13 | pub struct YamlTask { 14 | /// `yid` is the unique identifier defined in yaml, and `id` is the id assigned by the global id assigner. 15 | yid: String, 16 | id: usize, 17 | name: String, 18 | /// Precursor identifier defined in yaml. 19 | precursors: Vec, 20 | precursors_id: Vec, 21 | action: Action, 22 | } 23 | 24 | impl YamlTask { 25 | #[allow(unused)] 26 | pub fn new(yaml_id: &str, precursors: Vec, name: String, action: Action) -> Self { 27 | Self { 28 | yid: yaml_id.to_owned(), 29 | id: alloc_id(), 30 | name, 31 | precursors, 32 | precursors_id: Vec::new(), 33 | action, 34 | } 35 | } 36 | /// After the configuration file is parsed, the id of each task has been assigned. 37 | /// At this time, the `precursors_id` of this task will be initialized according to 38 | /// the id of the predecessor task of each task. 39 | #[allow(unused)] 40 | pub fn init_precursors(&mut self, pres_id: Vec) { 41 | self.precursors_id = pres_id; 42 | } 43 | 44 | /// Get the precursor identifier defined in yaml. 45 | #[allow(unused)] 46 | pub fn str_precursors(&self) -> Vec { 47 | self.precursors.clone() 48 | } 49 | /// Get the unique ID of the task defined in yaml. 50 | #[allow(unused)] 51 | pub fn str_id(&self) -> &str { 52 | &self.yid 53 | } 54 | } 55 | 56 | impl Task for YamlTask { 57 | fn action(&self) -> Action { 58 | self.action.clone() 59 | } 60 | fn precursors(&self) -> &[usize] { 61 | &self.precursors_id 62 | } 63 | fn id(&self) -> usize { 64 | self.id 65 | } 66 | fn name(&self) -> &str { 67 | &self.name 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /archived/tests/config/correct.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | after: [ b, c ] 5 | cmd: echo a 6 | b: 7 | name: "Task 2" 8 | after: [ c, f, g ] 9 | cmd: echo b 10 | c: 11 | name: "Task 3" 12 | after: [ e, g ] 13 | cmd: echo c 14 | d: 15 | name: "Task 4" 16 | after: [ c, e ] 17 | cmd: echo d 18 | e: 19 | name: "Task 5" 20 | after: [ h ] 21 | cmd: echo e 22 | f: 23 | name: "Task 6" 24 | after: [ g ] 25 | cmd: python3 ./tests/config/test.py 26 | g: 27 | name: "Task 7" 28 | after: [ h ] 29 | cmd: node ./tests/config/test.js 30 | h: 31 | name: "Task 8" 32 | cmd: echo h -------------------------------------------------------------------------------- /archived/tests/config/custom_file_task.txt: -------------------------------------------------------------------------------- 1 | a,Task a,b c,echo a 2 | b,Task b,c f g,echo b 3 | c,Task c,e g,echo c 4 | d,Task d,c e,echo d 5 | e,Task e,h,echo e 6 | f,Task f,g,python3 tests/config/test.py 7 | g,Task g,h,node tests/config/test.js 8 | h,Task h,,echo h -------------------------------------------------------------------------------- /archived/tests/config/empty_file.yaml: -------------------------------------------------------------------------------- 1 | # no tasks defined -------------------------------------------------------------------------------- /archived/tests/config/illegal_content.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name:"Task a" 4 | cmd: sh_script.sh -------------------------------------------------------------------------------- /archived/tests/config/loop_error.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | after: [ b, c ] 5 | cmd: echo x 6 | b: 7 | name: "Task 2" 8 | after: [ c ] 9 | cmd: echo x 10 | c: 11 | name: "Task 3" 12 | after: [ d ] 13 | cmd: echo x 14 | d: 15 | name: "Task 4" 16 | after: [ e ] 17 | cmd: echo x 18 | e: 19 | name: "Task 5" 20 | after: [ c ] 21 | cmd: echo x -------------------------------------------------------------------------------- /archived/tests/config/no_run.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" -------------------------------------------------------------------------------- /archived/tests/config/no_script.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | -------------------------------------------------------------------------------- /archived/tests/config/no_start_with_dagrs.yaml: -------------------------------------------------------------------------------- 1 | a: 2 | name: "Task 1" 3 | after: [ b, c ] 4 | cmd: echo a 5 | b: 6 | name: "Task 2" 7 | after: [ c, f, g ] 8 | cmd: echo b 9 | c: 10 | name: "Task 3" 11 | after: [ e, g ] 12 | cmd: echo c 13 | d: 14 | name: "Task 4" 15 | after: [ c, e ] 16 | cmd: echo d 17 | e: 18 | name: "Task 5" 19 | after: [ h ] 20 | cmd: echo e 21 | f: 22 | name: "Task 6" 23 | after: [ g ] 24 | cmd: python3 ./test.py 25 | g: 26 | name: "Task 7" 27 | after: [ h ] 28 | cmd: node ./test.js 29 | h: 30 | name: "Task 8" 31 | cmd: sh_script.sh -------------------------------------------------------------------------------- /archived/tests/config/no_task_name.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | cmd: echo "Task 1" -------------------------------------------------------------------------------- /archived/tests/config/no_type.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | run: -------------------------------------------------------------------------------- /archived/tests/config/precursor_not_found.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | after: [ b ] 5 | cmd: echo x -------------------------------------------------------------------------------- /archived/tests/config/script_run_failed.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | after: [ b, c ] 5 | cmd: error_cmd 6 | b: 7 | name: "Task 2" 8 | after: [ c, f, g ] 9 | cmd: error_cmd 10 | c: 11 | name: "Task 3" 12 | after: [ e, g ] 13 | cmd: echo c 14 | d: 15 | name: "Task 4" 16 | after: [ c, e ] 17 | cmd: echo d 18 | e: 19 | name: "Task 5" 20 | after: [ h ] 21 | cmd: echo e 22 | f: 23 | name: "Task 6" 24 | after: [ g ] 25 | cmd: err_cmd 26 | g: 27 | name: "Task 7" 28 | after: [ h ] 29 | cmd: err_cmd 30 | h: 31 | name: "Task 8" 32 | cmd: echo h -------------------------------------------------------------------------------- /archived/tests/config/self_loop_error.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | after: [ a ] 5 | cmd: echo x -------------------------------------------------------------------------------- /archived/tests/config/test.js: -------------------------------------------------------------------------------- 1 | console.log("hello js") -------------------------------------------------------------------------------- /archived/tests/config/test.py: -------------------------------------------------------------------------------- 1 | print("hello python") -------------------------------------------------------------------------------- /archived/tests/config/test.sh: -------------------------------------------------------------------------------- 1 | echo "hello sh" -------------------------------------------------------------------------------- /archived/tests/dag_job_test.rs: -------------------------------------------------------------------------------- 1 | //! Some tests of the dag engine. 2 | 3 | use std::{collections::HashMap, env::set_var, sync::Arc}; 4 | 5 | use dagrs::{Complex, Dag, DagError, DefaultTask, EnvVar, Input, Output}; 6 | 7 | #[test] 8 | fn yaml_task_correct_execute() { 9 | let mut job = Dag::with_yaml("tests/config/correct.yaml", HashMap::new()).unwrap(); 10 | assert!(job.start().is_ok()); 11 | } 12 | 13 | #[test] 14 | fn yaml_task_loop_graph() { 15 | let res = Dag::with_yaml("tests/config/loop_error.yaml", HashMap::new()) 16 | .unwrap() 17 | .start(); 18 | assert!(matches!(res, Err(DagError::LoopGraph))) 19 | } 20 | 21 | #[test] 22 | fn yaml_task_self_loop_graph() { 23 | let res = Dag::with_yaml("tests/config/self_loop_error.yaml", HashMap::new()) 24 | .unwrap() 25 | .start(); 26 | assert!(matches!(res, Err(DagError::LoopGraph))) 27 | } 28 | 29 | #[test] 30 | fn yaml_task_failed_execute() { 31 | let res = Dag::with_yaml("tests/config/script_run_failed.yaml", HashMap::new()) 32 | .unwrap() 33 | .start(); 34 | assert!(!res.is_ok()); 35 | } 36 | 37 | #[test] 38 | fn task_loop_graph() { 39 | let mut a = DefaultTask::with_closure("a", |_, _| Output::empty()); 40 | let mut b = DefaultTask::with_closure("b", |_, _| Output::empty()); 41 | let mut c = DefaultTask::with_closure("c", |_, _| Output::empty()); 42 | a.set_predecessors(&[&b]); 43 | b.set_predecessors(&[&c]); 44 | c.set_predecessors(&[&a]); 45 | 46 | let mut env = EnvVar::new(); 47 | env.set("base", 2usize); 48 | 49 | let mut job = Dag::with_tasks(vec![a, b, c]); 50 | job.set_env(env); 51 | let res = job.start(); 52 | assert!(matches!(res, Err(DagError::LoopGraph))); 53 | } 54 | 55 | #[test] 56 | fn non_job() { 57 | let tasks: Vec = Vec::new(); 58 | let res = Dag::with_tasks(tasks).start(); 59 | assert!(res.is_err()); 60 | } 61 | 62 | struct FailedActionC(usize); 63 | 64 | impl Complex for FailedActionC { 65 | fn run(&self, _input: Input, env: Arc) -> Output { 66 | let base = env.get::("base").unwrap(); 67 | Output::new(base / self.0) 68 | } 69 | } 70 | 71 | struct FailedActionD(usize); 72 | 73 | impl Complex for FailedActionD { 74 | fn run(&self, _input: Input, _env: Arc) -> Output { 75 | Output::Err("error".to_string()) 76 | } 77 | } 78 | 79 | macro_rules! generate_task { 80 | ($task:ident($val:expr),$name:literal) => {{ 81 | pub struct $task(usize); 82 | impl Complex for $task { 83 | fn run(&self, input: Input, env: Arc) -> Output { 84 | let base = env.get::("base").unwrap(); 85 | let mut sum = self.0; 86 | std::thread::sleep(std::time::Duration::from_millis(100)); 87 | input 88 | .get_iter() 89 | .for_each(|i| sum += i.get::().unwrap() * base); 90 | Output::new(sum) 91 | } 92 | } 93 | DefaultTask::with_action($name, $task($val)) 94 | }}; 95 | } 96 | 97 | fn test_dag(keep_going: bool, num_some_output: Option) { 98 | let a = generate_task!(A(1), "Compute A"); 99 | let mut b = generate_task!(B(2), "Compute B"); 100 | let mut c = DefaultTask::with_action("Compute C", FailedActionC(0)); 101 | let mut d = DefaultTask::with_action("Compute D", FailedActionD(1)); 102 | let mut e: DefaultTask = generate_task!(E(16), "Compute E"); 103 | let mut f = generate_task!(F(32), "Compute F"); 104 | let mut g = generate_task!(G(64), "Compute G"); 105 | let h = generate_task!(H(64), "Compute H"); 106 | let i = generate_task!(I(64), "Compute I"); 107 | let j = generate_task!(J(64), "Compute J"); 108 | let k = generate_task!(K(64), "Compute K"); 109 | let l = generate_task!(L(64), "Compute L"); 110 | let m = generate_task!(M(64), "Compute M"); 111 | 112 | b.set_predecessors(&[&a]); 113 | c.set_predecessors(&[&a]); 114 | d.set_predecessors(&[&a]); 115 | e.set_predecessors(&[&b, &c]); 116 | f.set_predecessors(&[&c, &d]); 117 | g.set_predecessors(&[&b, &e, &f]); 118 | 119 | set_var("TOKIO_WORKER_THREADS", "2"); 120 | 121 | let mut env = EnvVar::new(); 122 | env.set("base", 2usize); 123 | 124 | let mut job = Dag::with_tasks(vec![a, b, c, d, e, f, g, h, i, j, k, l, m]); 125 | 126 | if keep_going { 127 | job = job.keep_going(); 128 | } 129 | 130 | job.set_env(env); 131 | assert!(!job.start().is_ok()); // reports a failure 132 | 133 | // but the results for independent tasks are still available 134 | let output = job.get_results::(); 135 | 136 | assert_eq!(output.len(), 13); 137 | 138 | if let Some(num_some_output) = num_some_output { 139 | assert_eq!( 140 | output.values().filter(|o| o.is_some()).count(), 141 | num_some_output 142 | ); 143 | } 144 | } 145 | 146 | #[test] 147 | fn task_failed_execute() { 148 | test_dag(false, None); 149 | } 150 | 151 | #[test] 152 | fn task_keep_going() { 153 | test_dag(true, Some(8)); 154 | } 155 | 156 | #[test] 157 | fn error_with_exitcode() { 158 | let mut job = Dag::with_yaml("tests/config/error_with_exitcode.yaml", HashMap::new()).unwrap(); 159 | _ = job.start(); 160 | // hacky as ID_ALLOCATOR is static, so I don't know which id to use 161 | // to get the output of this single task 162 | match &job.get_outputs()[job.get_outputs().keys().next().unwrap()] { 163 | dagrs::Output::ErrWithExitCode(code, content) => { 164 | if let Some(output) = content { 165 | let (stdout, _stderr) = output.get::<(Vec, Vec)>().unwrap(); 166 | assert_eq!("testing 123", stdout[0]); 167 | assert_eq!(1, code.unwrap()); 168 | return assert!(true); 169 | } 170 | } 171 | _ => {} 172 | } 173 | panic!("Should not be here"); 174 | } 175 | -------------------------------------------------------------------------------- /archived/tests/env_test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use dagrs::EnvVar; 4 | 5 | #[test] 6 | fn env_set_get_test() { 7 | let env = init_env(); 8 | assert_eq!(env.get::("test1"), Some(1usize)); 9 | assert_eq!(env.get::("test2"), None); 10 | assert_eq!(env.get_ref::("test3"), Some(&"3".to_string())) 11 | } 12 | 13 | #[test] 14 | fn multi_thread_immutable_env_test() { 15 | let env = Arc::new(init_env()); 16 | let mut handles = Vec::new(); 17 | 18 | let env1 = env.clone(); 19 | handles.push(std::thread::spawn(move || { 20 | assert_eq!(env1.get::("test1"), Some(1usize)); 21 | })); 22 | 23 | let env2 = env.clone(); 24 | handles.push(std::thread::spawn(move || { 25 | assert_eq!(env2.get::("test1"), Some(1usize)); 26 | })); 27 | 28 | let env3 = env.clone(); 29 | handles.push(std::thread::spawn(move || { 30 | assert_eq!(env3.get::("test2"), None); 31 | })); 32 | handles 33 | .into_iter() 34 | .for_each(|handle| handle.join().unwrap()); 35 | } 36 | 37 | fn init_env() -> EnvVar { 38 | let mut env = EnvVar::new(); 39 | 40 | env.set("test1", 1usize); 41 | env.set("test2", 2i32); 42 | env.set("test3", "3".to_string()); 43 | env 44 | } 45 | -------------------------------------------------------------------------------- /archived/tests/yaml_parser_test.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use dagrs::{DagError, Parser, Task, YamlParser}; 4 | 5 | #[test] 6 | fn file_not_found_test() { 7 | let no_such_file: Result>, DagError> = 8 | YamlParser.parse_tasks("./no_such_file.yaml", HashMap::new()); 9 | // let err = no_such_file.unwrap_err().to_string(); 10 | // println!("{err}"); 11 | assert!(no_such_file.is_err()) 12 | } 13 | 14 | #[test] 15 | fn illegal_yaml_content() { 16 | let illegal_content: Result>, DagError> = 17 | YamlParser.parse_tasks("tests/config/illegal_content.yaml", HashMap::new()); 18 | // let err = illegal_content.unwrap_err().to_string(); 19 | // println!("{err}"); 20 | assert!(illegal_content.is_err()) 21 | } 22 | 23 | #[test] 24 | fn empty_content() { 25 | let empty_content: Result>, DagError> = 26 | YamlParser.parse_tasks("tests/config/empty_file.yaml", HashMap::new()); 27 | // let err = empty_content.unwrap_err().to_string(); 28 | // println!("{err}"); 29 | assert!(empty_content.is_err()) 30 | } 31 | 32 | #[test] 33 | fn yaml_no_start_with_dagrs() { 34 | let forget_dagrs: Result>, DagError> = 35 | YamlParser.parse_tasks("tests/config/no_start_with_dagrs.yaml", HashMap::new()); 36 | // let err = forget_dagrs.unwrap_err().to_string(); 37 | // println!("{err}"); 38 | assert!(forget_dagrs.is_err()) 39 | } 40 | 41 | #[test] 42 | fn yaml_task_no_name() { 43 | let no_task_name: Result>, DagError> = 44 | YamlParser.parse_tasks("tests/config/no_task_name.yaml", HashMap::new()); 45 | // let err = no_task_name.unwrap_err().to_string(); 46 | // println!("{err}"); 47 | assert!(no_task_name.is_err()) 48 | } 49 | 50 | #[test] 51 | fn yaml_task_not_found_precursor() { 52 | let not_found_pre: Result>, DagError> = 53 | YamlParser.parse_tasks("tests/config/precursor_not_found.yaml", HashMap::new()); 54 | // let err = not_found_pre.unwrap_err().to_string(); 55 | // println!("{err}"); 56 | assert!(not_found_pre.is_err()) 57 | } 58 | 59 | #[test] 60 | fn yaml_task_no_script_config() { 61 | let script: Result>, DagError> = 62 | YamlParser.parse_tasks("tests/config/no_script.yaml", HashMap::new()); 63 | // let err = script.unwrap_err().to_string(); 64 | // println!("{err}"); 65 | assert!(script.is_err()) 66 | } 67 | 68 | #[test] 69 | fn correct_parse() { 70 | let tasks: Result>, DagError> = 71 | YamlParser.parse_tasks("tests/config/correct.yaml", HashMap::new()); 72 | assert!(tasks.is_ok()); 73 | } 74 | -------------------------------------------------------------------------------- /dagrs-derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dagrs-derive" 3 | authors = [ 4 | "Quanyi Ma ", 5 | "Xiaolong Fu ", 6 | "Zhilei Qiu ", 7 | ] 8 | version = "0.4.3" 9 | edition = "2021" 10 | license = "MIT OR Apache-2.0" 11 | description = "Dagrs follows the concept of Flow-based Programming and is suitable for the execution of multiple tasks with graph-like dependencies. Dagrs has the characteristics of high performance and asynchronous execution. It provides users with a convenient programming interface." 12 | readme = "README.md" 13 | repository = "https://github.com/dagrs-dev/dagrs" 14 | keywords = ["DAG", "task", "async", "fbp", "tokio"] 15 | 16 | [dependencies] 17 | syn = { version = "2.0", features = ["full"] } 18 | quote = "1.0" 19 | proc-macro2 = "1.0" 20 | 21 | [lib] 22 | proc-macro = true 23 | 24 | [features] 25 | default = ["derive"] 26 | derive = [] 27 | -------------------------------------------------------------------------------- /dagrs-derive/src/auto_node.rs: -------------------------------------------------------------------------------- 1 | use proc_macro::TokenStream; 2 | use quote::quote; 3 | use syn::parse::Parser; 4 | use syn::{parse, parse_macro_input, Field, Generics, Ident, ItemStruct}; 5 | 6 | /// Generate fields & implements of `Node` trait. 7 | /// 8 | /// Step 1: generate fields (`id`, `name`, `input_channel`, `output_channel`, `action`) 9 | /// 10 | /// Step 2: generates methods for `Node` implementation. 11 | /// 12 | /// Step 3: append the generated fields to the input struct. 13 | /// 14 | /// Step 4: return tokens of the input struct & the generated methods. 15 | pub(crate) fn auto_node(args: TokenStream, input: TokenStream) -> TokenStream { 16 | let mut item_struct = parse_macro_input!(input as ItemStruct); 17 | let _ = parse_macro_input!(args as parse::Nothing); 18 | 19 | let generics = &item_struct.generics; 20 | 21 | let field_id = syn::Field::parse_named 22 | .parse2(quote! { 23 | id: dagrs::NodeId 24 | }) 25 | .unwrap(); 26 | 27 | let field_name = syn::Field::parse_named 28 | .parse2(quote! { 29 | name: String 30 | }) 31 | .unwrap(); 32 | 33 | let field_in_channels = syn::Field::parse_named 34 | .parse2(quote! { 35 | input_channels: dagrs::InChannels 36 | }) 37 | .unwrap(); 38 | 39 | let field_out_channels = syn::Field::parse_named 40 | .parse2(quote! { 41 | output_channels: dagrs::OutChannels 42 | }) 43 | .unwrap(); 44 | 45 | let field_action = syn::Field::parse_named 46 | .parse2(quote! { 47 | action: Box 48 | }) 49 | .unwrap(); 50 | 51 | let auto_impl = auto_impl_node( 52 | &item_struct.ident, 53 | generics, 54 | &field_id, 55 | &field_name, 56 | &field_in_channels, 57 | &field_out_channels, 58 | &field_action, 59 | ); 60 | 61 | match item_struct.fields { 62 | syn::Fields::Named(ref mut fields) => { 63 | fields.named.push(field_id); 64 | fields.named.push(field_name); 65 | fields.named.push(field_in_channels); 66 | fields.named.push(field_out_channels); 67 | fields.named.push(field_action); 68 | } 69 | syn::Fields::Unit => { 70 | item_struct.fields = syn::Fields::Named(syn::FieldsNamed { 71 | named: [ 72 | field_id, 73 | field_name, 74 | field_in_channels, 75 | field_out_channels, 76 | field_action, 77 | ] 78 | .into_iter() 79 | .collect(), 80 | brace_token: Default::default(), 81 | }); 82 | } 83 | _ => { 84 | return syn::Error::new_spanned( 85 | item_struct.ident, 86 | "`auto_node` macro can only be annotated on named struct or unit struct.", 87 | ) 88 | .into_compile_error() 89 | .into() 90 | } 91 | }; 92 | 93 | return quote! { 94 | #item_struct 95 | #auto_impl 96 | } 97 | .into(); 98 | } 99 | 100 | fn auto_impl_node( 101 | struct_ident: &Ident, 102 | generics: &Generics, 103 | field_id: &Field, 104 | field_name: &Field, 105 | field_in_channels: &Field, 106 | field_out_channels: &Field, 107 | field_action: &Field, 108 | ) -> proc_macro2::TokenStream { 109 | let mut impl_tokens = proc_macro2::TokenStream::new(); 110 | impl_tokens.extend([ 111 | impl_id(field_id), 112 | impl_name(field_name), 113 | impl_in_channels(field_in_channels), 114 | impl_out_channels(field_out_channels), 115 | impl_run(field_action, field_in_channels, field_out_channels), 116 | ]); 117 | 118 | quote::quote!( 119 | #[dagrs::async_trait::async_trait] 120 | impl #generics dagrs::Node for #struct_ident #generics { 121 | #impl_tokens 122 | } 123 | unsafe impl #generics Send for #struct_ident #generics{} 124 | unsafe impl #generics Sync for #struct_ident #generics{} 125 | ) 126 | } 127 | 128 | fn impl_id(field: &Field) -> proc_macro2::TokenStream { 129 | let ident = &field.ident; 130 | quote::quote!( 131 | fn id(&self) -> dagrs::NodeId { 132 | self.#ident 133 | } 134 | ) 135 | } 136 | 137 | fn impl_name(field: &Field) -> proc_macro2::TokenStream { 138 | let ident = &field.ident; 139 | quote::quote!( 140 | fn name(&self) -> dagrs::NodeName { 141 | self.#ident.clone() 142 | } 143 | ) 144 | } 145 | 146 | fn impl_in_channels(field: &Field) -> proc_macro2::TokenStream { 147 | let ident = &field.ident; 148 | quote::quote!( 149 | fn input_channels(&mut self) -> &mut dagrs::InChannels { 150 | &mut self.#ident 151 | } 152 | ) 153 | } 154 | 155 | fn impl_out_channels(field: &Field) -> proc_macro2::TokenStream { 156 | let ident = &field.ident; 157 | quote::quote!( 158 | fn output_channels(&mut self) -> &mut dagrs::OutChannels { 159 | &mut self.#ident 160 | } 161 | ) 162 | } 163 | 164 | fn impl_run( 165 | field: &Field, 166 | field_in_channels: &Field, 167 | field_out_channels: &Field, 168 | ) -> proc_macro2::TokenStream { 169 | let ident = &field.ident; 170 | let in_channels_ident = &field_in_channels.ident; 171 | let out_channels_ident = &field_out_channels.ident; 172 | quote::quote!( 173 | async fn run(&mut self, env: std::sync::Arc) -> dagrs::Output { 174 | self.#ident 175 | .run(&mut self.#in_channels_ident, &mut self.#out_channels_ident, env) 176 | .await 177 | } 178 | ) 179 | } 180 | -------------------------------------------------------------------------------- /dagrs-derive/src/lib.rs: -------------------------------------------------------------------------------- 1 | use proc_macro::TokenStream; 2 | #[cfg(feature = "derive")] 3 | mod auto_node; 4 | mod relay; 5 | 6 | /// [`auto_node`] is a macro that may be used when customizing nodes. It can only be 7 | /// marked on named struct or unit struct. 8 | /// 9 | /// The macro [`auto_node`] generates essential fields and implementation of traits for 10 | /// structs intended to represent `Node` in **Dagrs**. 11 | /// By applying this macro to a struct, it appends fields including `id: dagrs::NodeId`, 12 | /// `name: dagrs::NodeName`, `input_channels: dagrs::InChannels`, `output_channels: dagrs::OutChannels`, 13 | /// and `action: dagrs::Action`, and implements the required `dagrs::Node` trait. 14 | /// 15 | /// ## Example 16 | /// - Mark `auto_node` on a struct with customized fields. 17 | /// ```ignore 18 | /// use dagrs::auto_node; 19 | /// #[auto_node] 20 | /// struct MyNode {/*Put your customized fields here.*/} 21 | /// ``` 22 | /// 23 | /// - Mark `auto_node` on a struct with generic & lifetime params. 24 | /// ```ignore 25 | /// use dagrs::auto_node; 26 | /// #[auto_node] 27 | /// struct MyNode {/*Put your customized fields here.*/} 28 | /// ``` 29 | /// - Mark `auto_node` on a unit struct. 30 | /// ```ignore 31 | /// use dagrs::auto_node; 32 | /// #[auto_node] 33 | /// struct MyNode() 34 | /// ``` 35 | #[cfg(feature = "derive")] 36 | #[proc_macro_attribute] 37 | pub fn auto_node(args: TokenStream, input: TokenStream) -> TokenStream { 38 | use crate::auto_node::auto_node; 39 | auto_node(args, input).into() 40 | } 41 | 42 | /// The [`dependencies!`] macro allows users to specify all task dependencies in an easy-to-understand 43 | /// way. It will return the generated graph structure based on a set of defined dependencies 44 | #[cfg(feature = "derive")] 45 | #[proc_macro] 46 | pub fn dependencies(input: TokenStream) -> TokenStream { 47 | use relay::add_relay; 48 | use relay::Relaies; 49 | let relaies = syn::parse_macro_input!(input as Relaies); 50 | let token = add_relay(relaies); 51 | token.into() 52 | } 53 | -------------------------------------------------------------------------------- /dagrs-derive/src/relay.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashSet; 2 | 3 | use proc_macro2::Ident; 4 | use syn::{parse::Parse, Token}; 5 | 6 | /// Parses and processes a set of relay tasks and their successors, and generates a directed graph. 7 | /// 8 | /// Step 1: Define the `Relay` struct with a task and its associated successors (other tasks that depend on it). 9 | /// 10 | /// Step 2: Implement the `Parse` trait for `Relaies` to parse a sequence of task-successor pairs from input. This creates a vector of `Relay` objects. 11 | /// 12 | /// Step 3: In `add_relay`, initialize a directed graph structure using `Graph` and a hash map to store edges between nodes. 13 | /// 14 | /// Step 4: Iterate through each `Relay` and update the graph's edge list by adding nodes (tasks) and defining edges between tasks and their successors. 15 | /// 16 | /// Step 5: Ensure that each task is only added once to the graph using a cache (`HashSet`) to avoid duplicates. 17 | /// 18 | /// Step 6: Populate the edges of the graph with the previously processed data and return the graph. 19 | /// 20 | /// This code provides the logic to dynamically build a graph based on parsed task relationships, where each task is a node and the successors define directed edges between nodes. 21 | 22 | pub(crate) struct Relay { 23 | pub(crate) task: Ident, 24 | pub(crate) successors: Vec, 25 | } 26 | 27 | pub(crate) struct Relaies(pub(crate) Vec); 28 | 29 | impl Parse for Relaies { 30 | fn parse(input: syn::parse::ParseStream) -> syn::Result { 31 | let mut relies = Vec::new(); 32 | loop { 33 | let mut successors = Vec::new(); 34 | let task = input.parse::()?; 35 | input.parse::)>()?; 36 | while !input.peek(Token!(,)) && !input.is_empty() { 37 | successors.push(input.parse::()?); 38 | } 39 | let relay = Relay { task, successors }; 40 | relies.push(relay); 41 | let _ = input.parse::(); 42 | if input.is_empty() { 43 | break; 44 | } 45 | } 46 | Ok(Self(relies)) 47 | } 48 | } 49 | 50 | pub(crate) fn add_relay(relaies: Relaies) -> proc_macro2::TokenStream { 51 | let mut token = proc_macro2::TokenStream::new(); 52 | let mut cache: HashSet = HashSet::new(); 53 | token.extend(quote::quote!( 54 | use dagrs::Graph; 55 | use dagrs::NodeId; 56 | use std::collections::HashMap; 57 | use std::collections::HashSet; 58 | let mut edge: HashMap> = HashMap::new(); 59 | let mut graph = Graph::new(); 60 | )); 61 | for relay in relaies.0.iter() { 62 | let task = relay.task.clone(); 63 | token.extend(quote::quote!( 64 | let task_id = #task.id(); 65 | if(!edge.contains_key(&task_id)){ 66 | edge.insert(task_id, HashSet::new()); 67 | } 68 | )); 69 | for successor in relay.successors.iter() { 70 | token.extend(quote::quote!( 71 | let successor_id = #successor.id(); 72 | edge.entry(task_id) 73 | .or_insert_with(HashSet::new) 74 | .insert(successor_id); 75 | )); 76 | } 77 | } 78 | for relay in relaies.0.iter() { 79 | let task = relay.task.clone(); 80 | if !cache.contains(&task) { 81 | token.extend(quote::quote!( 82 | graph.add_node(#task); 83 | )); 84 | cache.insert(task); 85 | } 86 | for successor in relay.successors.iter() { 87 | if !cache.contains(successor) { 88 | token.extend(quote::quote!( 89 | graph.add_node(#successor); 90 | )); 91 | cache.insert(successor.clone()); 92 | } 93 | } 94 | } 95 | token.extend(quote::quote!(for (key, value) in &edge { 96 | let vec = value.iter().cloned().collect(); 97 | graph.add_edge(key.clone(), vec); 98 | })); 99 | 100 | quote::quote!( 101 | { 102 | #token; 103 | graph 104 | } 105 | ) 106 | } 107 | -------------------------------------------------------------------------------- /docs/upgrade.md: -------------------------------------------------------------------------------- 1 | # Upgrade Plan 2 | 3 | Our Goals: 4 | 5 | - Encapsulated processes and information packets.(Nov. 2024) 6 | - The external definition of connections.(Oct. 2024) 7 | - Asynchronous.(Nov. 2024) 8 | - Information packets with unique ownership and defined lifetimes.(Rust features) 9 | - Bounded connections with a finite capacity. (tokio) 10 | - Reserve pressure. (tokio) 11 | - Parser.(Nov. or Dec. 2024) 12 | 13 | 14 | 15 | ## Encapsulated processes and information packets 16 | 17 | - [ ] Provide a new trait `Node` that defines unique identifiers for nodes, input channels for receiving packets, output channels for sending packets, and an interface to start the workload, replacing the trait `Task` in the old version. 18 | - [ ] Enable asynchronous operations inside and between processes in trait `Action`. 19 | - [ ] Use `Content` as encapsulation of information packet. 20 | 21 | ## The external definition of connections 22 | 23 | - [ ] Provide asynchronous channels encapsulating the tokio channels and provide a unified interface. 24 | 25 | ## Asynchronous 26 | 27 | - [ ] Provide a struct `Graph`, replacing `Dag` in the old version. 28 | - [ ] Remove field `rely_graph`. 29 | - [ ] Automatically create channels and assign corresponding senders and receivers to the nodes when building the graph. 30 | - [ ] Modify the logic of error-handling: If one node fails, the graph will not stop running, and users can then handle exceptions or errors at a successor node. 31 | 32 | ## Bounded connections with a finite capacity 33 | 34 | [tokio]: https://tokio.rs/tokio/tutorial "tokio" 35 | 36 | provides four different channel implementations. 37 | 38 | - `one-shot` is a channel with a single producer and a single consumer. Only one value can be sent at the same time. 39 | 40 | - `mpsc` is a channel that supports multiple producers and a single consumer. It is different from one-shot in that it supports sending multiple messages. 41 | 42 | - `broadcast` is a channel that supports multiple producers and multiple consumers. Multiple values can be sent. 43 | 44 | - `watch` is a variant of broadcast. The receiver can only see whether the latest value has changed. 45 | 46 | We pick `mpsc` and `broadcast` as the channels used in Dagrs because they meet the requirements for connections in FBP: they are bounded, have finite capacity and support reserve pressure. 47 | 48 | ## Reserve pressure 49 | 50 | `tokio` provides congestion handling mechanisms for both `mpsc`and `broadcast` to deal with insufficient channel capacity or the consumer is too slow. 51 | 52 | - `mpsc` will block the sender when the capacity is full. 53 | - `broadcast` will drop the oldest value, and the slow consumers will get an exception the next time it calls the receive function. 54 | 55 | ## Parser 56 | 57 | - [ ] Support defining custom nodes via macros. 58 | - [ ] Modify the macro `dependencies!` to add dependencies to the already defined nodes (implemented with the `Node` trait) and return with a `Graph` ready to start. -------------------------------------------------------------------------------- /examples/auto_node.rs: -------------------------------------------------------------------------------- 1 | //! # Example: auto_node 2 | //! The procedural macro `auto_node` simplifies the implementation of `Node` trait for custom types. 3 | //! It works on structs except [tuple structs](https://doc.rust-lang.org/book/ch05-01-defining-structs.html#using-tuple-structs-without-named-fields-to-create-different-types). 4 | 5 | use std::sync::Arc; 6 | 7 | use dagrs::{auto_node, EmptyAction, EnvVar, InChannels, Node, NodeTable, OutChannels}; 8 | 9 | #[auto_node] 10 | struct MyNode {/*Put customized fields here.*/} 11 | 12 | #[auto_node] 13 | struct _MyNodeGeneric { 14 | /*Put customized fields here.*/ 15 | my_field: Vec, 16 | my_name: &'a str, 17 | } 18 | 19 | #[auto_node] 20 | struct _MyUnitNode; 21 | 22 | fn main() { 23 | let mut node_table = NodeTable::default(); 24 | 25 | let node_name = "auto_node".to_string(); 26 | 27 | let mut s = MyNode { 28 | id: node_table.alloc_id_for(&node_name), 29 | name: node_name.clone(), 30 | input_channels: InChannels::default(), 31 | output_channels: OutChannels::default(), 32 | action: Box::new(EmptyAction), 33 | }; 34 | 35 | assert_eq!(&s.id(), node_table.get(&node_name).unwrap()); 36 | assert_eq!(&s.name(), &node_name); 37 | 38 | let output = tokio::runtime::Runtime::new() 39 | .unwrap() 40 | .block_on(async { s.run(Arc::new(EnvVar::new(NodeTable::default()))).await }); 41 | match output { 42 | dagrs::Output::Out(content) => assert!(content.is_none()), 43 | _ => panic!(), 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /examples/auto_relay.rs: -------------------------------------------------------------------------------- 1 | //! # Example: auto_relay 2 | //! The macro `dependencies!` simplifies the construction of a `Graph`, 3 | //! including the addition of nodes and edges. 4 | 5 | use dagrs::{auto_node, dependencies, EmptyAction, InChannels, Node, NodeTable, OutChannels}; 6 | 7 | #[auto_node] 8 | struct MyNode {/*Put customized fields here.*/} 9 | 10 | impl MyNode { 11 | fn new(name: &str, node_table: &mut NodeTable) -> Self { 12 | Self { 13 | id: node_table.alloc_id_for(name), 14 | name: name.to_string(), 15 | input_channels: InChannels::default(), 16 | output_channels: OutChannels::default(), 17 | action: Box::new(EmptyAction), 18 | } 19 | } 20 | } 21 | 22 | fn main() { 23 | let mut node_table = NodeTable::default(); 24 | 25 | let node_name = "auto_node"; 26 | 27 | let s = MyNode::new(node_name, &mut node_table); 28 | let a = MyNode::new(node_name, &mut node_table); 29 | let b = MyNode::new(node_name, &mut node_table); 30 | 31 | let mut g = dependencies!( 32 | s -> a b, 33 | b -> a 34 | ); 35 | 36 | g.start().unwrap(); 37 | } 38 | -------------------------------------------------------------------------------- /examples/compute_dag.rs: -------------------------------------------------------------------------------- 1 | //! Only use Dag, execute a job. The graph is as follows: 2 | //! 3 | //! ↱----------↴ 4 | //! B -→ E --→ G 5 | //! ↗ ↗ ↗ 6 | //! A --→ C / 7 | //! ↘ ↘ / 8 | //! D -→ F 9 | //! 10 | //! The final execution result is 272. 11 | 12 | use std::sync::Arc; 13 | 14 | use async_trait::async_trait; 15 | use dagrs::{ 16 | Action, Content, DefaultNode, EnvVar, Graph, InChannels, Node, NodeTable, OutChannels, Output, 17 | }; 18 | 19 | const BASE: &str = "base"; 20 | 21 | struct Compute(usize); 22 | 23 | #[async_trait] 24 | impl Action for Compute { 25 | async fn run( 26 | &self, 27 | in_channels: &mut InChannels, 28 | out_channels: &mut OutChannels, 29 | env: Arc, 30 | ) -> Output { 31 | let base = env.get::(BASE).unwrap(); 32 | let mut sum = self.0; 33 | 34 | in_channels 35 | .map(|content| content.unwrap().into_inner::().unwrap()) 36 | .await 37 | .into_iter() 38 | .for_each(|x| sum += *x * base); 39 | 40 | out_channels.broadcast(Content::new(sum)).await; 41 | 42 | Output::Out(Some(Content::new(sum))) 43 | } 44 | } 45 | 46 | fn main() { 47 | // Initialization log. 48 | env_logger::init(); 49 | 50 | // Create a new `NodeTable`. 51 | let mut node_table = NodeTable::default(); 52 | 53 | // Generate some tasks. 54 | let a = DefaultNode::with_action("Compute A".to_string(), Compute(1), &mut node_table); 55 | let a_id = a.id(); 56 | 57 | let b = DefaultNode::with_action("Compute B".to_string(), Compute(2), &mut node_table); 58 | let b_id = b.id(); 59 | 60 | let mut c = DefaultNode::new("Compute C".to_string(), &mut node_table); 61 | c.set_action(Compute(4)); 62 | let c_id = c.id(); 63 | 64 | let mut d = DefaultNode::new("Compute D".to_string(), &mut node_table); 65 | d.set_action(Compute(8)); 66 | let d_id = d.id(); 67 | 68 | let e = DefaultNode::with_action("Compute E".to_string(), Compute(16), &mut node_table); 69 | let e_id = e.id(); 70 | let f = DefaultNode::with_action("Compute F".to_string(), Compute(32), &mut node_table); 71 | let f_id = f.id(); 72 | 73 | let g = DefaultNode::with_action("Compute G".to_string(), Compute(64), &mut node_table); 74 | let g_id = g.id(); 75 | 76 | // Create a graph. 77 | let mut graph = Graph::new(); 78 | vec![a, b, c, d, e, f, g] 79 | .into_iter() 80 | .for_each(|node| graph.add_node(node)); 81 | 82 | // Set up task dependencies. 83 | graph.add_edge(a_id, vec![b_id, c_id, d_id]); 84 | graph.add_edge(b_id, vec![e_id, g_id]); 85 | graph.add_edge(c_id, vec![e_id, f_id]); 86 | graph.add_edge(d_id, vec![f_id]); 87 | graph.add_edge(e_id, vec![g_id]); 88 | graph.add_edge(f_id, vec![g_id]); 89 | 90 | // Set a global environment variable for this dag. 91 | let mut env = EnvVar::new(node_table); 92 | env.set("base", 2usize); 93 | graph.set_env(env); 94 | 95 | // Start executing this dag. 96 | match graph.start() { 97 | Ok(_) => { 98 | let res = graph 99 | .get_results::() 100 | .get(&g_id) 101 | .unwrap() 102 | .clone() 103 | .unwrap(); 104 | // Verify execution result. 105 | assert_eq!(*res, 272) 106 | } 107 | Err(e) => { 108 | panic!("Graph execution failed: {:?}", e); 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /examples/conditional_node.rs: -------------------------------------------------------------------------------- 1 | //! Variant of the example `compute_dag`. 2 | //! Only use Dag, execute a job. The graph is as follows: 3 | //! 4 | //! ┌───────────↴ 5 | //! B ──→ E ──→ X ──→ G 6 | //! ↗ ↗ ↗ 7 | //! A ───→ C ╱ 8 | //! ↘ ↘ ╱ 9 | //! D ───→ F ───╱ 10 | //! 11 | //! The node `X` is a conditional node which will return false. 12 | //! 13 | //! Dagrs will split the graph into two blocks: 14 | //! - A, B, C, D, E, F, X 15 | //! - G 16 | //! 17 | //! Since node X's condition (VerifyGT(128)) is not met, 18 | //! execution stops at node X and never reaches node G. 19 | //! Therefore G's result should be None. 20 | 21 | use std::{env, sync::Arc}; 22 | 23 | use async_trait::async_trait; 24 | use dagrs::{ 25 | node::conditional_node::{Condition, ConditionalNode}, 26 | Action, Content, DefaultNode, EnvVar, Graph, InChannels, Node, NodeTable, OutChannels, Output, 27 | }; 28 | 29 | const BASE: &str = "base"; 30 | 31 | struct Compute(usize); 32 | 33 | #[async_trait] 34 | impl Action for Compute { 35 | async fn run( 36 | &self, 37 | in_channels: &mut InChannels, 38 | out_channels: &mut OutChannels, 39 | env: Arc, 40 | ) -> Output { 41 | let base = env.get::(BASE).unwrap(); 42 | let mut sum = self.0; 43 | 44 | in_channels 45 | .map(|content| content.unwrap().into_inner::().unwrap()) 46 | .await 47 | .into_iter() 48 | .for_each(|x| sum += *x * base); 49 | 50 | out_channels.broadcast(Content::new(sum)).await; 51 | 52 | Output::Out(Some(Content::new(sum))) 53 | } 54 | } 55 | 56 | struct VerifyGT(usize); 57 | #[async_trait] 58 | impl Condition for VerifyGT { 59 | async fn run( 60 | &self, 61 | in_channels: &mut InChannels, 62 | out_channels: &OutChannels, 63 | _: Arc, 64 | ) -> bool { 65 | let mut sum = 0; 66 | in_channels 67 | .map(|content| content.unwrap().into_inner::().unwrap()) 68 | .await 69 | .into_iter() 70 | .for_each(|x| sum += *x); 71 | 72 | let verify = sum > self.0; 73 | if verify { 74 | out_channels.broadcast(Content::new(sum)).await; 75 | } 76 | 77 | verify 78 | } 79 | } 80 | 81 | fn main() { 82 | // Initialization log. 83 | env::set_var("RUST_LOG", "debug"); 84 | env_logger::init(); 85 | 86 | // Create a new `NodeTable`. 87 | let mut node_table = NodeTable::default(); 88 | 89 | // Generate some tasks. 90 | let a = DefaultNode::with_action("Compute A".to_string(), Compute(1), &mut node_table); 91 | let a_id = a.id(); 92 | 93 | let b = DefaultNode::with_action("Compute B".to_string(), Compute(2), &mut node_table); 94 | let b_id = b.id(); 95 | 96 | let mut c = DefaultNode::new("Compute C".to_string(), &mut node_table); 97 | c.set_action(Compute(4)); 98 | let c_id = c.id(); 99 | 100 | let mut d = DefaultNode::new("Compute D".to_string(), &mut node_table); 101 | d.set_action(Compute(8)); 102 | let d_id = d.id(); 103 | 104 | let e = DefaultNode::with_action("Compute E".to_string(), Compute(16), &mut node_table); 105 | let e_id = e.id(); 106 | let f = DefaultNode::with_action("Compute F".to_string(), Compute(32), &mut node_table); 107 | let f_id = f.id(); 108 | 109 | let x = 110 | ConditionalNode::with_condition("Condition X".to_string(), VerifyGT(128), &mut node_table); 111 | let x_id = x.id(); 112 | 113 | let g = DefaultNode::with_action("Compute G".to_string(), Compute(64), &mut node_table); 114 | let g_id = g.id(); 115 | 116 | // Create a graph. 117 | let mut graph = Graph::new(); 118 | vec![a, b, c, d, e, f, g] 119 | .into_iter() 120 | .for_each(|node| graph.add_node(node)); 121 | graph.add_node(x); 122 | 123 | // Set up task dependencies. 124 | graph.add_edge(a_id, vec![b_id, c_id, d_id]); 125 | graph.add_edge(b_id, vec![e_id, x_id]); 126 | graph.add_edge(c_id, vec![e_id, f_id]); 127 | graph.add_edge(d_id, vec![f_id]); 128 | graph.add_edge(e_id, vec![x_id]); 129 | graph.add_edge(x_id, vec![g_id]); 130 | graph.add_edge(f_id, vec![g_id]); 131 | 132 | // Set a global environment variable for this dag. 133 | let mut env = EnvVar::new(node_table); 134 | env.set("base", 2usize); 135 | graph.set_env(env); 136 | 137 | // Start executing this dag. 138 | match graph.start() { 139 | Ok(_) => { 140 | // Since node X's condition (VerifyGT(128)) is not met, 141 | // execution stops at node X and never reaches node G. 142 | // Therefore G's result should be None. 143 | let res = graph.get_results::().get(&g_id).unwrap().clone(); 144 | assert!(res.is_none()); 145 | } 146 | Err(e) => { 147 | panic!("Graph execution failed: {:?}", e); 148 | } 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /examples/custom_node.rs: -------------------------------------------------------------------------------- 1 | //! # Example: custom_node 2 | //! Creates a custom implementation of [`Node`] that returns a [`String`], 3 | //! then create a new [`Graph`] with this node and run. 4 | 5 | use std::sync::Arc; 6 | 7 | use async_trait::async_trait; 8 | use dagrs::{ 9 | Content, EnvVar, Graph, InChannels, Node, NodeId, NodeName, NodeTable, OutChannels, Output, 10 | }; 11 | 12 | struct MessageNode { 13 | id: NodeId, 14 | name: NodeName, 15 | in_channels: InChannels, 16 | out_channels: OutChannels, 17 | /*Put your custom fields here.*/ 18 | message: String, 19 | } 20 | 21 | #[async_trait] 22 | impl Node for MessageNode { 23 | fn id(&self) -> NodeId { 24 | self.id 25 | } 26 | 27 | fn name(&self) -> NodeName { 28 | self.name.clone() 29 | } 30 | 31 | fn input_channels(&mut self) -> &mut InChannels { 32 | &mut self.in_channels 33 | } 34 | 35 | fn output_channels(&mut self) -> &mut OutChannels { 36 | &mut self.out_channels 37 | } 38 | 39 | async fn run(&mut self, _: Arc) -> Output { 40 | Output::Out(Some(Content::new(self.message.clone()))) 41 | } 42 | } 43 | 44 | impl MessageNode { 45 | fn new(name: String, node_table: &mut NodeTable) -> Self { 46 | Self { 47 | id: node_table.alloc_id_for(&name), 48 | name, 49 | in_channels: InChannels::default(), 50 | out_channels: OutChannels::default(), 51 | message: "hello dagrs".to_string(), 52 | } 53 | } 54 | } 55 | 56 | fn main() { 57 | // create an empty `NodeTable` 58 | let mut node_table = NodeTable::new(); 59 | // create a `MessageNode` 60 | let node = MessageNode::new("message node".to_string(), &mut node_table); 61 | let id: &dagrs::NodeId = &node.id(); 62 | 63 | // create a graph with this node and run 64 | let mut graph = Graph::new(); 65 | graph.add_node(node); 66 | match graph.start() { 67 | Ok(_) => { 68 | // verify the output of this node 69 | let outputs = graph.get_outputs(); 70 | assert_eq!(outputs.len(), 1); 71 | 72 | let content = outputs.get(id).unwrap().get_out().unwrap(); 73 | let node_output = content.get::().unwrap(); 74 | assert_eq!(node_output, "hello dagrs") 75 | } 76 | Err(e) => { 77 | eprintln!("Graph execution failed: {:?}", e); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dagrs-sklearn" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | dagrs = {path = "../../", version = "0.4.2"} 8 | thiserror = "2" 9 | yaml-rust = "0.4" 10 | log = "0.4.22" 11 | env_logger = "0.11.6" -------------------------------------------------------------------------------- /examples/dagrs-sklearn/examples/config.yml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | node0: 3 | name: "node_0" 4 | after: [] 5 | cmd: "" 6 | node1: 7 | name: "node_1" 8 | after: [] 9 | cmd: "" 10 | node2: 11 | name: "node_2" 12 | after: [] 13 | cmd: "" 14 | node3: 15 | name: "node_3" 16 | after: [] 17 | cmd: "" 18 | node4: 19 | name: "node_4" 20 | after: [] 21 | cmd: "" 22 | node5: 23 | name: "node_5" 24 | after: [] 25 | cmd: "" 26 | node6: 27 | name: "node_6" 28 | after: [] 29 | cmd: "" 30 | node7: 31 | name: "node_7" 32 | after: [] 33 | cmd: "" 34 | node8: 35 | name: "node_8" 36 | after: [] 37 | cmd: "" 38 | node9: 39 | name: "node_9" 40 | after: [] 41 | cmd: "" 42 | root: 43 | name: "root" 44 | after: [ 45 | "node0", 46 | "node1", 47 | "node2", 48 | "node3", 49 | "node4", 50 | "node5", 51 | "node6", 52 | "node7", 53 | "node8", 54 | "node9", 55 | ] 56 | cmd: "" -------------------------------------------------------------------------------- /examples/dagrs-sklearn/examples/ex3data1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dagrs-dev/dagrs/7c059dda74e2b906a9cc37d413bb2cfc9140d5f6/examples/dagrs-sklearn/examples/ex3data1.mat -------------------------------------------------------------------------------- /examples/dagrs-sklearn/examples/lr_i.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.io import loadmat 3 | from scipy.optimize import minimize 4 | import sys 5 | 6 | 7 | def load_data(path): 8 | data=loadmat(path) 9 | X=data["X"] 10 | y=data["y"] 11 | return X,y 12 | 13 | def sigmoid(z): 14 | return 1/(1+np.exp(-z)) 15 | 16 | def regularized_cost(Theta,X,y,l): 17 | ThetaReg=Theta[1:] 18 | cost=(-y*np.log(sigmoid(X@Theta)))-(1-y)*np.log(1-sigmoid((X@Theta))) 19 | reg=(ThetaReg@ThetaReg)*l/(2*len(X)) 20 | return np.mean(cost)+reg 21 | 22 | 23 | def regularized_gradient(Theta,X,y,l): 24 | ThetaReg=Theta[1:] 25 | cost=(X.T@(sigmoid(X@Theta)-y))*(1/len(X)) 26 | reg=np.concatenate([np.array([0]),(l/len(X))*ThetaReg]) 27 | return cost+reg 28 | 29 | def one_vs_all(X,y,l,i): 30 | Theta=np.zeros(X.shape[1]) 31 | y_i=np.array([1 if labal==(i + 1) else 0 for labal in y]) 32 | ret=minimize(fun=regularized_cost, x0=Theta,args=(X,y_i,l),method='TNC',jac=regularized_gradient) 33 | return ret.x 34 | 35 | argc = len(sys.argv) 36 | argv = sys.argv 37 | 38 | X, y = load_data(argv[1]) 39 | X=np.insert(X,0,1,axis=1) 40 | y=y.flatten() 41 | 42 | i = int(argv[2]) 43 | theta = (one_vs_all(X, y, 1, i)) 44 | 45 | s = np.array2string(theta, max_line_width=1 << 31) 46 | with open(f"theta_{i}.txt", "w") as f: 47 | f.write(s) 48 | print(f"theta_{i}.txt") -------------------------------------------------------------------------------- /examples/dagrs-sklearn/examples/lr_root.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.io import loadmat 3 | import sys 4 | import os 5 | 6 | def load_data(path): 7 | data=loadmat(path) 8 | X=data["X"] 9 | y=data["y"] 10 | return X,y 11 | 12 | def sigmoid(z): 13 | return 1 / (1 + np.exp(-z)) 14 | 15 | def predict(X,All_Theta): 16 | h=sigmoid(X@All_Theta.T) 17 | h_argmax=np.argmax(h,axis=1) 18 | h_argmax=h_argmax+1 19 | return h_argmax 20 | 21 | def main(): 22 | np.set_printoptions(threshold=sys.maxsize) 23 | 24 | argv = sys.argv 25 | X, y = load_data(argv[1]) 26 | X=np.insert(X,0,1,axis=1) 27 | y=y.flatten() 28 | 29 | # Load theta values for each class from the provided files 30 | All_Theta = np.zeros((10, X.shape[1])) # 10 classes, with the same number of features 31 | 32 | for i in range(0, 10): 33 | # Load theta for class (i+1) from the file 34 | file = argv[2+i] 35 | j = int(file[6]) 36 | with open(file, "r") as f: 37 | s = f.read().strip("[] ") 38 | All_Theta[j] = np.fromstring(s, sep=" ", dtype=float) 39 | os.remove(file) 40 | 41 | # Predict the class using the provided theta values 42 | y_predict = predict(X, All_Theta) 43 | 44 | # Calculate accuracy 45 | y_predict=predict(X, All_Theta) 46 | accuracy=np.mean(y_predict==y) 47 | 48 | print(f"Accuracy: {accuracy * 100:.2f}%\n") 49 | 50 | if __name__ == "__main__": 51 | main() -------------------------------------------------------------------------------- /examples/dagrs-sklearn/examples/raw.py: -------------------------------------------------------------------------------- 1 | # https://juejin.cn/post/7083799151162425357 2 | # author: Phoenix_Zenghao 3 | 4 | from scipy.io import loadmat 5 | import numpy as np 6 | from scipy.optimize import minimize 7 | 8 | def load_data(path): 9 | data=loadmat(path) 10 | X=data["X"] 11 | y=data["y"] 12 | return X,y 13 | 14 | def sigmoid(z): 15 | return 1/(1+np.exp(-z)) 16 | 17 | def regularized_cost(Theta,X,y,l): 18 | ThetaReg=Theta[1:] 19 | cost=(-y*np.log(sigmoid(X@Theta)))-(1-y)*np.log(1-sigmoid((X@Theta))) 20 | reg=(ThetaReg@ThetaReg)*l/(2*len(X)) 21 | return np.mean(cost)+reg 22 | 23 | def regularized_gradient(Theta,X,y,l): 24 | ThetaReg=Theta[1:] 25 | cost=(X.T@(sigmoid(X@Theta)-y))*(1/len(X)) 26 | reg=np.concatenate([np.array([0]),(l/len(X))*ThetaReg]) 27 | return cost+reg 28 | 29 | def one_vs_all(X,y,l,K): 30 | All_Theta=np.zeros((K,X.shape[1])) 31 | for i in range(1,K+1): 32 | Theta=np.zeros(X.shape[1]) 33 | y_i=np.array([1 if labal==i else 0 for labal in y]) 34 | ret=minimize(fun=regularized_cost, x0=Theta,args=(X,y_i,l),method='TNC',jac=regularized_gradient) 35 | All_Theta[i-1:]=ret.x 36 | return All_Theta 37 | 38 | def predict(X,All_Theta): 39 | h=sigmoid(X@All_Theta.T) 40 | h_argmax=np.argmax(h,axis=1) 41 | h_argmax=h_argmax+1 42 | return h_argmax 43 | 44 | 45 | # np.set_printoptions(threshold=sys.maxsize) 46 | X, y = load_data('ex3data1.mat') 47 | X=np.insert(X,0,1,axis=1) 48 | y=y.flatten() 49 | All_Theta=one_vs_all(X, y, 1, 10) 50 | y_predict=predict(X, All_Theta) 51 | accuracy=np.mean(y_predict==y) 52 | print("accuracy=%.2f%%"%(accuracy*100)) -------------------------------------------------------------------------------- /examples/dagrs-sklearn/examples/requirements.txt: -------------------------------------------------------------------------------- 1 | SciPy 2 | numpy -------------------------------------------------------------------------------- /examples/dagrs-sklearn/examples/sklearn.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, sync::Arc}; 2 | 3 | use dagrs::{async_trait::async_trait, Action, Content, EnvVar, InChannels, OutChannels, Output}; 4 | use dagrs_sklearn::{yaml_parser::YamlParser, CommandAction, Parser}; 5 | 6 | const ENV_DATA_SRC: &str = "data_src"; 7 | 8 | struct NodeAction { 9 | class: usize, 10 | } 11 | 12 | #[async_trait] 13 | impl Action for NodeAction { 14 | async fn run( 15 | &self, 16 | _: &mut InChannels, 17 | out_channels: &mut OutChannels, 18 | env: Arc, 19 | ) -> Output { 20 | let data_src: &&str = env.get_ref(ENV_DATA_SRC).unwrap(); 21 | 22 | let cmd_act = CommandAction::new( 23 | "python", 24 | vec![ 25 | "examples/lr_i.py".to_string(), 26 | format!("{}", data_src), 27 | format!("{}", self.class), 28 | ], 29 | ); 30 | let result = cmd_act 31 | .run(&mut InChannels::default(), &mut OutChannels::default(), env) 32 | .await; 33 | match result { 34 | Output::Out(content) => { 35 | let content: Arc<(Vec, Vec)> = 36 | content.unwrap().into_inner().unwrap(); 37 | let (stdout, _) = (&content.0, &content.1); 38 | let theta = stdout.get(0).unwrap().clone(); 39 | out_channels.broadcast(Content::new(theta)).await 40 | } 41 | Output::Err(e) => panic!("{}", e), 42 | Output::ErrWithExitCode(code, content) => panic!( 43 | "Exit with {:?}, {:?}", 44 | code, 45 | content.unwrap().get::<(Vec, Vec)>() 46 | ), 47 | _ => panic!(), 48 | }; 49 | Output::empty() 50 | } 51 | } 52 | 53 | impl NodeAction { 54 | fn new(class: usize) -> Self { 55 | Self { class } 56 | } 57 | } 58 | 59 | struct RootAction; 60 | #[async_trait] 61 | impl Action for RootAction { 62 | async fn run( 63 | &self, 64 | in_channels: &mut InChannels, 65 | _: &mut OutChannels, 66 | env: Arc, 67 | ) -> Output { 68 | let data_src: &&str = env.get_ref(ENV_DATA_SRC).unwrap(); 69 | let mut thetas: Vec = in_channels 70 | .map(|theta| { 71 | let theta = theta.unwrap(); 72 | let theta = theta.get::().unwrap(); 73 | theta.clone() 74 | }) 75 | .await; 76 | 77 | let mut args = vec!["examples/lr_root.py".to_string(), format!("{}", data_src)]; 78 | 79 | args.append(&mut thetas); 80 | 81 | let cmd_action = CommandAction::new("python", args); 82 | cmd_action 83 | .run(&mut InChannels::default(), &mut OutChannels::default(), env) 84 | .await 85 | } 86 | } 87 | 88 | fn main() { 89 | env_logger::init(); 90 | 91 | let specific_actions: HashMap> = HashMap::from([ 92 | ( 93 | "node0".to_string(), 94 | Box::new(NodeAction::new(0)) as Box, 95 | ), 96 | ( 97 | "node1".to_string(), 98 | Box::new(NodeAction::new(1)) as Box, 99 | ), 100 | ( 101 | "node2".to_string(), 102 | Box::new(NodeAction::new(2)) as Box, 103 | ), 104 | ( 105 | "node3".to_string(), 106 | Box::new(NodeAction::new(3)) as Box, 107 | ), 108 | ( 109 | "node4".to_string(), 110 | Box::new(NodeAction::new(4)) as Box, 111 | ), 112 | ( 113 | "node5".to_string(), 114 | Box::new(NodeAction::new(5)) as Box, 115 | ), 116 | ( 117 | "node6".to_string(), 118 | Box::new(NodeAction::new(6)) as Box, 119 | ), 120 | ( 121 | "node7".to_string(), 122 | Box::new(NodeAction::new(7)) as Box, 123 | ), 124 | ( 125 | "node8".to_string(), 126 | Box::new(NodeAction::new(8)) as Box, 127 | ), 128 | ( 129 | "node9".to_string(), 130 | Box::new(NodeAction::new(9)) as Box, 131 | ), 132 | ("root".to_string(), Box::new(RootAction) as Box), 133 | ]); 134 | 135 | let (mut dag, mut env_var) = YamlParser 136 | .parse_tasks("examples/config.yml", specific_actions) 137 | .unwrap(); 138 | env_var.set(ENV_DATA_SRC, "examples/ex3data1.mat"); 139 | 140 | let root_id = *env_var.get_node_id("root").unwrap(); 141 | dag.set_env(env_var); 142 | dag.start().unwrap(); 143 | 144 | let outputs = dag.get_outputs(); 145 | let result = outputs.get(&root_id).unwrap().get_out().unwrap(); 146 | let (stdout, _) = result.get::<(Vec, Vec)>().unwrap(); 147 | 148 | let acc = if cfg!(target_os = "windows") { 149 | stdout.get(1).unwrap() 150 | } else { 151 | stdout.get(0).unwrap() 152 | }; 153 | assert_eq!("Accuracy: 94.46%", acc); 154 | 155 | println!("{}", acc) 156 | } 157 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/src/command_action.rs: -------------------------------------------------------------------------------- 1 | use std::process::Command; 2 | 3 | use dagrs::async_trait::async_trait; 4 | use dagrs::{Action, Content, Output}; 5 | 6 | /// [`CommandAction`] is a specific implementation of [`Complex`], used to execute operating system commands. 7 | pub struct CommandAction { 8 | command: String, 9 | args: Vec, 10 | } 11 | 12 | impl CommandAction { 13 | #[allow(unused)] 14 | pub fn new(cmd: &str, args: Vec) -> Self { 15 | Self { 16 | command: cmd.to_owned(), 17 | args, 18 | } 19 | } 20 | } 21 | 22 | #[async_trait] 23 | impl Action for CommandAction { 24 | async fn run( 25 | &self, 26 | in_channels: &mut dagrs::InChannels, 27 | out_channels: &mut dagrs::OutChannels, 28 | _: std::sync::Arc, 29 | ) -> dagrs::Output { 30 | let mut args = Vec::new(); 31 | let mut cmd = if cfg!(target_os = "windows") { 32 | args.push("-Command"); 33 | args.push(&self.command); 34 | Command::new("powershell") 35 | } else { 36 | Command::new(&self.command) 37 | }; 38 | 39 | let mut inputs = vec![]; 40 | in_channels 41 | .map(|input| { 42 | if let Ok(inp) = input { 43 | if let Some(inp) = inp.get::() { 44 | inputs.push(inp.to_owned()); 45 | } 46 | } 47 | }) 48 | .await; 49 | args.append(&mut self.args.iter().map(|s| s.as_str()).collect()); 50 | args.append(&mut inputs.iter().map(|x| x.as_str()).collect()); 51 | 52 | log::info!("cmd: {:?}, args: {:?}", cmd.get_program(), args); 53 | 54 | let out = match cmd.args(args).output() { 55 | Ok(o) => o, 56 | Err(e) => { 57 | out_channels.broadcast(Content::new(e.raw_os_error())).await; 58 | return Output::error_with_exit_code( 59 | e.raw_os_error(), 60 | Some(Content::new(e.to_string())), 61 | ); 62 | } 63 | }; 64 | let code = out.status.code().unwrap_or(0); 65 | let stdout: Vec = { 66 | let out = String::from_utf8(out.stdout).unwrap_or("".to_string()); 67 | if cfg!(target_os = "windows") { 68 | out.rsplit_terminator("\r\n").map(str::to_string).collect() 69 | } else { 70 | out.split_terminator('\n').map(str::to_string).collect() 71 | } 72 | }; 73 | let stderr: Vec = { 74 | let out = String::from_utf8(out.stderr).unwrap_or("".to_string()); 75 | if cfg!(target_os = "windows") { 76 | out.rsplit_terminator("\r\n").map(str::to_string).collect() 77 | } else { 78 | out.split_terminator('\n').map(str::to_string).collect() 79 | } 80 | }; 81 | if out.status.success() { 82 | out_channels 83 | .broadcast(Content::new((stdout.clone(), stderr.clone()))) 84 | .await; 85 | Output::new((stdout.clone(), stderr.clone())) 86 | } else { 87 | out_channels 88 | .broadcast(Content::new((stdout.clone(), stderr.clone()))) 89 | .await; 90 | let output = Content::new((stdout, stderr)); 91 | Output::error_with_exit_code(Some(code), Some(output)) 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod command_action; 2 | mod parser; 3 | pub mod utils; 4 | 5 | pub use command_action::*; 6 | pub use dagrs::*; 7 | pub use parser::*; 8 | pub use utils::parser::*; 9 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/src/parser/mod.rs: -------------------------------------------------------------------------------- 1 | //! yaml configuration file type parser 2 | //! 3 | //! # Config file parser 4 | //! 5 | //! Use yaml configuration files to define a series of tasks, which eliminates the need for users to write code. 6 | //! [`YamlParser`] is responsible for parsing the yaml configuration file into a series of [`YamlTask`]. 7 | //! The program specifies the properties of the yaml task configuration file. The basic format of the yaml 8 | //! configuration file is as follows: 9 | //! 10 | //! ```yaml 11 | //! dagrs: 12 | //! a: 13 | //! name: "Task 1" 14 | //! after: [ b, c ] 15 | //! cmd: echo a 16 | //! b: 17 | //! name: "Task 2" 18 | //! after: [ c, f, g ] 19 | //! cmd: echo b 20 | //! c: 21 | //! name: "Task 3" 22 | //! after: [ e, g ] 23 | //! cmd: echo c 24 | //! d: 25 | //! name: "Task 4" 26 | //! after: [ c, e ] 27 | //! cmd: echo d 28 | //! e: 29 | //! name: "Task 5" 30 | //! after: [ h ] 31 | //! cmd: echo e 32 | //! f: 33 | //! name: "Task 6" 34 | //! after: [ g ] 35 | //! cmd: python3 ./tests/config/test.py 36 | //! g: 37 | //! name: "Task 7" 38 | //! after: [ h ] 39 | //! cmd: node ./tests/config/test.js 40 | //! h: 41 | //! name: "Task 8" 42 | //! cmd: echo h 43 | //! ``` 44 | //! 45 | //! Users can read the yaml configuration file programmatically or by using the compiled `dagrs` 46 | //! command line tool. Either way, you need to enable the `yaml` feature. 47 | //! 48 | //! # Example 49 | //! 50 | //! ```rust 51 | //! #[ignore] 52 | //! use dagrs_sklearn::{yaml_parser::YamlParser, Parser}; 53 | //! let dag = YamlParser.parse_tasks("some_path",std::collections::HashMap::new()); 54 | //! ``` 55 | 56 | pub mod yaml_parser; 57 | pub mod yaml_task; 58 | 59 | use thiserror::Error; 60 | use yaml_rust; 61 | 62 | use crate::utils::parser::ParseError; 63 | 64 | pub use self::yaml_task::YamlTask; 65 | 66 | /// Errors about task configuration items. 67 | #[derive(Debug, Error)] 68 | pub enum YamlTaskError { 69 | /// The configuration file should start with `dagrs:`. 70 | #[error("File content is not start with 'dagrs'.")] 71 | StartWordError, 72 | /// No task name configured. 73 | #[error("Task has no name field. [{0}]")] 74 | NoNameAttr(String), 75 | /// The specified task predecessor was not found. 76 | #[error("Task cannot find the specified predecessor. [{0}]")] 77 | NotFoundPrecursor(String), 78 | /// `script` is not defined. 79 | #[error("The 'script' attribute is not defined. [{0}]")] 80 | NoScriptAttr(String), 81 | } 82 | 83 | /// Error about file information. 84 | #[derive(Debug, Error)] 85 | pub enum FileContentError { 86 | /// The format of the yaml configuration file is not standardized. 87 | #[error("Illegal yaml content: {0}")] 88 | IllegalYamlContent(yaml_rust::ScanError), 89 | /// The file is empty. 90 | #[allow(unused)] 91 | #[error("File is empty! [{0}]")] 92 | Empty(String), 93 | } 94 | 95 | /// Configuration file not found. 96 | #[derive(Debug, Error)] 97 | #[error("File not found. [{0}]")] 98 | pub struct FileNotFound(pub std::io::Error); 99 | 100 | impl From for ParseError { 101 | fn from(value: YamlTaskError) -> Self { 102 | ParseError(value.to_string()) 103 | } 104 | } 105 | 106 | impl From for ParseError { 107 | fn from(value: FileContentError) -> Self { 108 | ParseError(value.to_string()) 109 | } 110 | } 111 | 112 | impl From for ParseError { 113 | fn from(value: FileNotFound) -> Self { 114 | ParseError(value.to_string().into()) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/src/parser/yaml_parser.rs: -------------------------------------------------------------------------------- 1 | //! Default yaml configuration file parser. 2 | 3 | use super::{FileContentError, YamlTask, YamlTaskError}; 4 | use crate::{ 5 | command_action::CommandAction, 6 | parser::ParseError, 7 | utils::{file::load_file, parser::Parser}, 8 | }; 9 | use dagrs::{Action, EnvVar, Graph, Node, NodeId, NodeTable}; 10 | use std::collections::HashMap; 11 | use yaml_rust::{Yaml, YamlLoader}; 12 | 13 | /// An implementation of [`Parser`]. It is the default yaml configuration file parser. 14 | pub struct YamlParser; 15 | 16 | impl YamlParser { 17 | /// Parses an item in the configuration file into a task. 18 | /// An item refers to: 19 | /// 20 | /// ```yaml 21 | /// name: "Task 1" 22 | /// after: [b, c] 23 | /// cmd: echo a 24 | /// ``` 25 | fn parse_one( 26 | &self, 27 | id: &str, 28 | item: &Yaml, 29 | specific_action: Option>, 30 | node_table: &mut NodeTable, 31 | ) -> Result { 32 | // Get name first 33 | let name = item["name"] 34 | .as_str() 35 | .ok_or(YamlTaskError::NoNameAttr(id.to_owned()))? 36 | .to_owned(); 37 | // precursors can be empty 38 | let mut precursors = Vec::new(); 39 | if let Some(after_tasks) = item["after"].as_vec() { 40 | after_tasks 41 | .iter() 42 | .for_each(|task_id| precursors.push(task_id.as_str().unwrap().to_owned())); 43 | } 44 | 45 | if let Some(action) = specific_action { 46 | Ok(YamlTask::new(id, precursors, name, action, node_table)) 47 | } else { 48 | let cmd_args = item["cmd"] 49 | .as_str() 50 | .ok_or(YamlTaskError::NoScriptAttr(name.clone()))? 51 | .split(' ') 52 | .collect::>(); 53 | 54 | let cmd = cmd_args.get(0).unwrap_or(&""); 55 | let args = cmd_args[1..].iter().map(|s| s.to_string()).collect(); 56 | 57 | Ok(YamlTask::new( 58 | id, 59 | precursors, 60 | name, 61 | Box::new(CommandAction::new(cmd, args)), 62 | node_table, 63 | )) 64 | } 65 | } 66 | } 67 | 68 | impl Parser for YamlParser { 69 | fn parse_tasks( 70 | &self, 71 | file: &str, 72 | specific_actions: HashMap>, 73 | ) -> Result<(Graph, EnvVar), ParseError> { 74 | let content = load_file(file).map_err(|e| ParseError(e.to_string()))?; 75 | self.parse_tasks_from_str(&content, specific_actions) 76 | } 77 | 78 | fn parse_tasks_from_str( 79 | &self, 80 | content: &str, 81 | mut specific_actions: HashMap>, 82 | ) -> Result<(Graph, EnvVar), ParseError> { 83 | let mut node_table = NodeTable::default(); 84 | // Parse Yaml 85 | let yaml_tasks = 86 | YamlLoader::load_from_str(content).map_err(FileContentError::IllegalYamlContent)?; 87 | if yaml_tasks.is_empty() { 88 | return Err(ParseError("No Tasks found".to_string())); 89 | } 90 | let yaml_tasks = yaml_tasks[0]["dagrs"] 91 | .as_hash() 92 | .ok_or(YamlTaskError::StartWordError)?; 93 | 94 | let mut tasks = Vec::with_capacity(yaml_tasks.len()); 95 | let mut map = HashMap::with_capacity(yaml_tasks.len()); 96 | // Read tasks 97 | for (v, w) in yaml_tasks { 98 | let id = v 99 | .as_str() 100 | .ok_or(ParseError("Invalid YAML Node Type".to_string()))?; 101 | let task = self.parse_one(id, w, specific_actions.remove(id), &mut node_table)?; 102 | map.insert(id, task.id()); 103 | tasks.push(task); 104 | } 105 | 106 | let mut dag = Graph::new(); 107 | let mut edges: HashMap> = HashMap::new(); 108 | 109 | for task in tasks.iter_mut() { 110 | let mut pres = Vec::new(); 111 | for pre in task.str_precursors() { 112 | if map.contains_key(&pre[..]) { 113 | pres.push(map[&pre[..]]); 114 | } else { 115 | return Err(YamlTaskError::NotFoundPrecursor(task.name().to_string()).into()); 116 | } 117 | } 118 | 119 | let succ_id = task.id(); 120 | pres.iter().for_each(|p| { 121 | if let Some(p) = edges.get_mut(&p) { 122 | p.push(succ_id); 123 | } else { 124 | edges.insert(*p, vec![succ_id]); 125 | } 126 | }); 127 | 128 | task.init_precursors(pres.clone()); 129 | } 130 | 131 | tasks.into_iter().for_each(|task| dag.add_node(task)); 132 | edges.into_iter().for_each(|(x, ys)| { 133 | dag.add_edge(x, ys); 134 | }); 135 | 136 | let env_var = EnvVar::new(node_table); 137 | dag.set_env(env_var.clone()); 138 | 139 | Ok((dag, env_var)) 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/src/parser/yaml_task.rs: -------------------------------------------------------------------------------- 1 | //! Task definition of type Yaml. 2 | //! 3 | //! # The task type corresponding to the configuration file: [`YamlTask`] 4 | //! 5 | //! [`YamlTask`] implements the [`Task`] trait, which represents the tasks in the yaml 6 | //! configuration file, and a yaml configuration file will be parsed into a series of [`YamlTask`]. 7 | //! It is different from `DefaultTask`, in addition to the four mandatory attributes of the 8 | //! task type, he has several additional attributes. 9 | 10 | use dagrs::{auto_node, Action, InChannels, NodeId, NodeTable, OutChannels}; 11 | 12 | /// Task struct for yaml file. 13 | #[auto_node] 14 | pub struct YamlTask { 15 | /// `yid` is the unique identifier defined in yaml, and `id` is the id assigned by the global id assigner. 16 | yid: String, 17 | /// Precursor identifier defined in yaml. 18 | precursors: Vec, 19 | precursors_id: Vec, 20 | } 21 | 22 | impl YamlTask { 23 | #[allow(unused)] 24 | pub fn new( 25 | yaml_id: &str, 26 | precursors: Vec, 27 | name: String, 28 | action: Box, 29 | node_table: &mut NodeTable, 30 | ) -> Self { 31 | Self { 32 | yid: yaml_id.to_owned(), 33 | id: node_table.alloc_id_for(&name), 34 | name, 35 | precursors, 36 | precursors_id: Vec::new(), 37 | action, 38 | input_channels: InChannels::default(), 39 | output_channels: OutChannels::default(), 40 | } 41 | } 42 | /// After the configuration file is parsed, the id of each task has been assigned. 43 | /// At this time, the `precursors_id` of this task will be initialized according to 44 | /// the id of the predecessor task of each task. 45 | #[allow(unused)] 46 | pub fn init_precursors(&mut self, pres_id: Vec) { 47 | self.precursors_id = pres_id; 48 | } 49 | 50 | /// Get the precursor identifier defined in yaml. 51 | #[allow(unused)] 52 | pub fn str_precursors(&self) -> Vec { 53 | self.precursors.clone() 54 | } 55 | /// Get the unique ID of the task defined in yaml. 56 | #[allow(unused)] 57 | pub fn str_id(&self) -> &str { 58 | &self.yid 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/src/utils/file.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::{Error, Read}; 3 | 4 | /// Given file path, and load configuration file. 5 | pub(crate) fn load_file(file: &str) -> Result { 6 | let mut content = String::new(); 7 | let mut fh = File::open(file)?; 8 | fh.read_to_string(&mut content)?; 9 | Ok(content) 10 | } 11 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod file; 2 | pub mod parser; 3 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/src/utils/parser.rs: -------------------------------------------------------------------------------- 1 | //! Task configuration file parser interface 2 | use std::collections::HashMap; 3 | 4 | use dagrs::{Action, EnvVar, Graph}; 5 | 6 | /// Generic parser traits. If users want to customize the configuration file parser, they must implement this trait. 7 | /// The yaml module's `YamlParser` is an example. 8 | pub trait Parser { 9 | /// Parses the contents of a configuration file into a series of tasks with dependencies. 10 | /// Parameter Description: 11 | /// - file: path information of the configuration file 12 | /// - specific_actions: When parsing the configuration file, the specific execution logic 13 | /// of some tasks does not need to be specified in the configuration file, but is given 14 | /// through this map. In the map's key-value pair, the key represents the unique identifier 15 | /// of the task in the task's configuration file, and the value represents the execution 16 | /// logic given by the user. 17 | /// 18 | /// Return value description: 19 | /// If an error is encountered during the parsing process, the return result is ParserError. 20 | /// Instead, return a series of concrete types that implement the [`Task`] trait. 21 | /// This may involve user-defined [`Task`], you can refer to `YamlTask` under the yaml module. 22 | #[allow(unused)] 23 | fn parse_tasks( 24 | &self, 25 | file: &str, 26 | specific_actions: HashMap>, 27 | ) -> Result<(Graph, EnvVar), ParseError>; 28 | 29 | fn parse_tasks_from_str( 30 | &self, 31 | content: &str, 32 | specific_actions: HashMap>, 33 | ) -> Result<(Graph, EnvVar), ParseError>; 34 | } 35 | 36 | #[allow(unused)] 37 | #[derive(Debug)] 38 | pub struct ParseError(pub String); 39 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/correct.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | after: [ b, c ] 5 | cmd: echo a 6 | b: 7 | name: "Task 2" 8 | after: [ c, f ] 9 | cmd: echo b 10 | c: 11 | name: "Task 3" 12 | after: [ e ] 13 | cmd: echo c 14 | d: 15 | name: "Task 4" 16 | after: [ c, e ] 17 | cmd: echo d 18 | e: 19 | name: "Task 5" 20 | after: [ h ] 21 | cmd: echo e 22 | f: 23 | name: "Task 6" 24 | cmd: python3 ./tests/config/test.py 25 | h: 26 | name: "Task 8" 27 | cmd: echo h -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/custom_file_task.txt: -------------------------------------------------------------------------------- 1 | a,Task a,b c,echo a 2 | b,Task b,c f g,echo b 3 | c,Task c,e g,echo c 4 | d,Task d,c e,echo d 5 | e,Task e,h,echo e 6 | f,Task f,g,python3 tests/config/test.py 7 | g,Task g,h,node tests/config/test.js 8 | h,Task h,,echo h -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/empty_file.yaml: -------------------------------------------------------------------------------- 1 | # no tasks defined -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/illegal_content.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name:"Task a" 4 | cmd: sh sh_script.sh -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/loop_error.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | after: [ b, c ] 5 | cmd: sh -c echo x 6 | b: 7 | name: "Task 2" 8 | after: [ c ] 9 | cmd: sh -c echo x 10 | c: 11 | name: "Task 3" 12 | after: [ d ] 13 | cmd: sh -c echo x 14 | d: 15 | name: "Task 4" 16 | after: [ e ] 17 | cmd: sh -c echo x 18 | e: 19 | name: "Task 5" 20 | after: [ c ] 21 | cmd: sh -c echo x -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/no_run.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/no_script.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/no_start_with_dagrs.yaml: -------------------------------------------------------------------------------- 1 | a: 2 | name: "Task 1" 3 | after: [ b, c ] 4 | cmd: sh -c echo a 5 | b: 6 | name: "Task 2" 7 | after: [ c, f, g ] 8 | cmd: sh -c echo b 9 | c: 10 | name: "Task 3" 11 | after: [ e, g ] 12 | cmd: sh -c echo c 13 | d: 14 | name: "Task 4" 15 | after: [ c, e ] 16 | cmd: sh -c echo d 17 | e: 18 | name: "Task 5" 19 | after: [ h ] 20 | cmd: sh -c echo e 21 | f: 22 | name: "Task 6" 23 | after: [ g ] 24 | cmd: python3 ./test.py 25 | g: 26 | name: "Task 7" 27 | after: [ h ] 28 | cmd: node ./test.js 29 | h: 30 | name: "Task 8" 31 | cmd: sh -c sh_script.sh -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/no_task_name.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | cmd: sh -c echo "Task 1" -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/no_type.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | run: -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/precursor_not_found.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | after: [ b ] 5 | cmd: sh -c echo x -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/script_run_failed.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | after: [ b, c ] 5 | cmd: error_cmd 6 | b: 7 | name: "Task 2" 8 | after: [ c, f, g ] 9 | cmd: error_cmd 10 | c: 11 | name: "Task 3" 12 | after: [ e, g ] 13 | cmd: sh -c echo c 14 | d: 15 | name: "Task 4" 16 | after: [ c, e ] 17 | cmd: sh -c echo d 18 | e: 19 | name: "Task 5" 20 | after: [ h ] 21 | cmd: sh -c echo e 22 | f: 23 | name: "Task 6" 24 | after: [ g ] 25 | cmd: err_cmd 26 | g: 27 | name: "Task 7" 28 | after: [ h ] 29 | cmd: err_cmd 30 | h: 31 | name: "Task 8" 32 | cmd: sh -c echo h -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/self_loop_error.yaml: -------------------------------------------------------------------------------- 1 | dagrs: 2 | a: 3 | name: "Task 1" 4 | after: [ a ] 5 | cmd: sh -c echo x -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/test.js: -------------------------------------------------------------------------------- 1 | console.log("hello js") -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/test.py: -------------------------------------------------------------------------------- 1 | print("hello python") -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/config/test.sh: -------------------------------------------------------------------------------- 1 | echo "hello sh" -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/dag_job_test.rs: -------------------------------------------------------------------------------- 1 | //! Some tests of the dag engine. 2 | 3 | use std::collections::HashMap; 4 | 5 | use dagrs::graph::error::GraphError; 6 | use dagrs_sklearn::{yaml_parser::YamlParser, Parser}; 7 | 8 | #[test] 9 | fn yaml_task_correct_execute() { 10 | let (mut job, _) = YamlParser 11 | .parse_tasks("tests/config/correct.yaml", HashMap::new()) 12 | .unwrap(); 13 | job.start().unwrap(); 14 | } 15 | 16 | #[test] 17 | fn yaml_task_loop_graph() { 18 | let (mut res, _) = YamlParser 19 | .parse_tasks("tests/config/loop_error.yaml", HashMap::new()) 20 | .unwrap(); 21 | 22 | let res = res.start(); 23 | assert!(matches!(res, Err(GraphError::GraphLoopDetected))) 24 | } 25 | 26 | #[test] 27 | fn yaml_task_self_loop_graph() { 28 | let (mut res, _) = YamlParser 29 | .parse_tasks("tests/config/self_loop_error.yaml", HashMap::new()) 30 | .unwrap(); 31 | let res = res.start(); 32 | assert!(matches!(res, Err(GraphError::GraphLoopDetected))) 33 | } 34 | 35 | #[test] 36 | fn yaml_task_failed_execute() { 37 | let (mut res, _) = YamlParser 38 | .parse_tasks("tests/config/script_run_failed.yaml", HashMap::new()) 39 | .unwrap(); 40 | let res = res.start(); 41 | assert!(!res.is_ok()) 42 | } 43 | -------------------------------------------------------------------------------- /examples/dagrs-sklearn/tests/yaml_parser_test.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use dagrs_sklearn::{yaml_parser::YamlParser, Parser}; 4 | 5 | #[test] 6 | fn file_not_found_test() { 7 | let no_such_file = YamlParser.parse_tasks("./no_such_file.yaml", HashMap::new()); 8 | assert!(no_such_file.is_err()) 9 | } 10 | 11 | #[test] 12 | fn illegal_yaml_content() { 13 | let illegal_content = 14 | YamlParser.parse_tasks("tests/config/illegal_content.yaml", HashMap::new()); 15 | assert!(illegal_content.is_err()) 16 | } 17 | 18 | #[test] 19 | fn empty_content() { 20 | let empty_content = YamlParser.parse_tasks("tests/config/empty_file.yaml", HashMap::new()); 21 | 22 | assert!(empty_content.is_err()) 23 | } 24 | 25 | #[test] 26 | fn yaml_no_start_with_dagrs() { 27 | let forget_dagrs = 28 | YamlParser.parse_tasks("tests/config/no_start_with_dagrs.yaml", HashMap::new()); 29 | assert!(forget_dagrs.is_err()) 30 | } 31 | 32 | #[test] 33 | fn yaml_task_no_name() { 34 | let no_task_name = YamlParser.parse_tasks("tests/config/no_task_name.yaml", HashMap::new()); 35 | assert!(no_task_name.is_err()) 36 | } 37 | 38 | #[test] 39 | fn yaml_task_not_found_precursor() { 40 | let not_found_pre = 41 | YamlParser.parse_tasks("tests/config/precursor_not_found.yaml", HashMap::new()); 42 | assert!(not_found_pre.is_err()) 43 | } 44 | 45 | #[test] 46 | fn yaml_task_no_script_config() { 47 | let script = YamlParser.parse_tasks("tests/config/no_script.yaml", HashMap::new()); 48 | assert!(script.is_err()) 49 | } 50 | 51 | #[test] 52 | fn correct_parse() { 53 | let tasks = YamlParser.parse_tasks("tests/config/correct.yaml", HashMap::new()); 54 | assert!(tasks.is_ok()); 55 | } 56 | -------------------------------------------------------------------------------- /examples/hello_dagrs.rs: -------------------------------------------------------------------------------- 1 | //! # Example: hello_dagrs 2 | //! Creates a `DefaultNode` that returns with "Hello Dagrs", 3 | //! then create a new `Graph` with this node and run. 4 | 5 | use std::sync::Arc; 6 | 7 | use async_trait::async_trait; 8 | use dagrs::{ 9 | Action, Content, DefaultNode, EnvVar, Graph, InChannels, Node, NodeTable, OutChannels, Output, 10 | }; 11 | 12 | /// An implementation of [`Action`] that returns [`Output::Out`] containing a String "Hello world". 13 | #[derive(Default)] 14 | pub struct HelloAction; 15 | #[async_trait] 16 | impl Action for HelloAction { 17 | async fn run(&self, _: &mut InChannels, _: &mut OutChannels, _: Arc) -> Output { 18 | Output::Out(Some(Content::new("Hello Dagrs".to_string()))) 19 | } 20 | } 21 | 22 | fn main() { 23 | // create an empty `NodeTable` 24 | let mut node_table = NodeTable::new(); 25 | // create a `DefaultNode` with action `HelloAction` 26 | let hello_node = DefaultNode::with_action( 27 | "Hello Dagrs".to_string(), 28 | HelloAction::default(), 29 | &mut node_table, 30 | ); 31 | let id: &dagrs::NodeId = &hello_node.id(); 32 | 33 | // create a graph with this node and run 34 | let mut graph = Graph::new(); 35 | graph.add_node(hello_node); 36 | 37 | match graph.start() { 38 | Ok(_) => { 39 | // verify the output of this node 40 | let outputs = graph.get_outputs(); 41 | assert_eq!(outputs.len(), 1); 42 | 43 | let content = outputs.get(id).unwrap().get_out().unwrap(); 44 | let node_output = content.get::().unwrap(); 45 | assert_eq!(node_output, "Hello Dagrs") 46 | } 47 | Err(e) => { 48 | eprintln!("Graph execution failed: {:?}", e); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /examples/loop_dag.rs: -------------------------------------------------------------------------------- 1 | //! IN 2 | //! | +----------+ 3 | //! | ↓ | 4 | //! +-----> INTER ----> PROC 5 | //！ 6 | //！ A loop dag where K is the starter ("kicker") process, that emits a single packet containing a blank, 7 | //! (imagine) INTER controls a screen. In this figure, INTER receives some data from PROC, displays it on 8 | //! the screen, and then sends information back to PROC. 9 | //! If the software infrastructure allows it, waiting for input need only suspend INTER, and other processes 10 | //! could be working on their input while INTER is suspended. 11 | 12 | use std::{env, fmt::Display, sync::Arc}; 13 | 14 | use async_trait::async_trait; 15 | use dagrs::{ 16 | graph::loop_subgraph::LoopSubgraph, Action, Content, DefaultNode, EnvVar, Graph, InChannels, 17 | Node, NodeId, NodeTable, OutChannels, Output, 18 | }; 19 | 20 | struct InAction; 21 | 22 | /// Send a unit to the INTER node. 23 | #[async_trait] 24 | impl Action for InAction { 25 | async fn run( 26 | &self, 27 | _in_channels: &mut InChannels, 28 | out_channels: &mut OutChannels, 29 | _env: Arc, 30 | ) -> Output { 31 | log::info!("`In` send start signal to INTER node"); 32 | out_channels.broadcast(Content::new(())).await; 33 | Output::Out(None) 34 | } 35 | } 36 | 37 | struct InterAction { 38 | in_id: NodeId, 39 | proc_id: NodeId, 40 | limit: usize, 41 | } 42 | 43 | #[async_trait] 44 | impl Action for InterAction { 45 | async fn run( 46 | &self, 47 | in_channels: &mut InChannels, 48 | out_channels: &mut OutChannels, 49 | _env: Arc, 50 | ) -> Output { 51 | // Recv a start signal from the IN node. 52 | let content = in_channels.recv_from(&self.in_id).await.unwrap(); 53 | in_channels.close_async(&self.in_id).await; 54 | log::info!("`Inter` Received start signal from IN node"); 55 | 56 | let mut times = 0usize; 57 | 58 | out_channels.send_to(&self.proc_id, content).await.unwrap(); 59 | log::info!("`Inter` send start signal to PROC node"); 60 | 61 | while let Ok(content) = in_channels.recv_from(&self.proc_id).await { 62 | // Simulate screen display and user input 63 | log::info!( 64 | "`Inter` Displaying input: [{}]", 65 | content.get::>().unwrap() 66 | ); 67 | out_channels.send_to(&self.proc_id, content).await.unwrap(); 68 | log::info!("`Inter` send output to PROC node"); 69 | 70 | times += 1; 71 | if times >= self.limit { 72 | log::info!("`Inter` reached iter limit {}, exit", times); 73 | out_channels.close(&self.proc_id); 74 | break; 75 | } 76 | } 77 | 78 | log::info!("`Inter` exit"); 79 | Output::empty() 80 | } 81 | } 82 | 83 | struct ProcAction { 84 | inter_node: NodeId, 85 | } 86 | 87 | #[async_trait] 88 | impl Action for ProcAction { 89 | async fn run( 90 | &self, 91 | in_channels: &mut InChannels, 92 | out_channels: &mut OutChannels, 93 | _env: Arc, 94 | ) -> Output { 95 | let mut times = 0usize; 96 | while let Ok(_) = in_channels.recv_from(&self.inter_node).await { 97 | log::info!("`Proc` send {} to INTER node", times); 98 | out_channels 99 | .send_to( 100 | &self.inter_node, 101 | Content::new(Arc::new(times) as Arc), 102 | ) 103 | .await 104 | .unwrap(); 105 | times += 1; 106 | } 107 | 108 | log::info!("`Proc` exit"); 109 | Output::empty() 110 | } 111 | } 112 | 113 | fn main() { 114 | env::set_var("RUST_LOG", "info"); 115 | env_logger::init(); 116 | 117 | let mut node_table = NodeTable::default(); 118 | 119 | // Create nodes 120 | let in_node = DefaultNode::with_action("IN".to_string(), InAction, &mut node_table); 121 | let in_id = in_node.id(); 122 | 123 | let mut inter = DefaultNode::new("Inter".to_string(), &mut node_table); 124 | let inter_id = inter.id(); 125 | 126 | let mut proc = DefaultNode::new("Proc".to_string(), &mut node_table); 127 | let proc_id = proc.id(); 128 | 129 | inter.set_action(InterAction { 130 | in_id, 131 | proc_id, 132 | limit: 10, 133 | }); 134 | proc.set_action(ProcAction { 135 | inter_node: inter_id, 136 | }); 137 | 138 | let mut inter_proc = LoopSubgraph::new("inter_proc".to_string(), &mut node_table); 139 | inter_proc.add_node(inter); 140 | inter_proc.add_node(proc); 141 | 142 | // Create graph and add nodes 143 | let mut graph = Graph::new(); 144 | graph.add_node(in_node); 145 | graph.add_node(inter_proc); 146 | 147 | // Set up dependencies to create the loop 148 | graph.add_edge(in_id, vec![inter_id]); 149 | graph.add_edge(inter_id, vec![proc_id]); 150 | graph.add_edge(proc_id, vec![inter_id]); 151 | 152 | // Execute graph 153 | match graph.start() { 154 | Ok(_) => println!("Graph executed successfully"), 155 | Err(e) => panic!("Graph execution failed: {:?}", e), 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/connection/in_channel.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, sync::Arc}; 2 | 3 | use futures::future::join_all; 4 | use tokio::sync::{broadcast, mpsc, Mutex}; 5 | 6 | use crate::node::node::NodeId; 7 | 8 | use super::information_packet::Content; 9 | 10 | /// # Input Channels 11 | /// A hash-table mapping `NodeId` to `InChannel`. In **Dagrs**, each `Node` stores input 12 | /// channels in this map, enabling `Node` to receive information packets from other `Node`s. 13 | #[derive(Default)] 14 | pub struct InChannels(pub(crate) HashMap>>); 15 | 16 | impl InChannels { 17 | /// Perform a blocking receive on the incoming channel from `NodeId`. 18 | pub fn blocking_recv_from(&mut self, id: &NodeId) -> Result { 19 | match self.get(id) { 20 | Some(channel) => channel.blocking_lock().blocking_recv(), 21 | None => Err(RecvErr::NoSuchChannel), 22 | } 23 | } 24 | /// Perform a asynchronous receive on the incoming channel from `NodeId`. 25 | pub async fn recv_from(&mut self, id: &NodeId) -> Result { 26 | match self.get(id) { 27 | Some(channel) => channel.lock().await.recv().await, 28 | None => Err(RecvErr::NoSuchChannel), 29 | } 30 | } 31 | 32 | /// Calls `blocking_recv` for all the [`InChannel`]s, and applies transformation `f` to 33 | /// the return values of the call. 34 | pub fn blocking_map(&mut self, mut f: F) -> Vec 35 | where 36 | F: FnMut(Result) -> T, 37 | { 38 | self.keys() 39 | .into_iter() 40 | .map(|id| f(self.blocking_recv_from(&id))) 41 | .collect() 42 | } 43 | 44 | /// Calls `recv` for all the [`InChannel`]s, and applies transformation `f` to 45 | /// the return values of the call asynchronously. 46 | pub async fn map(&mut self, mut f: F) -> Vec 47 | where 48 | F: FnMut(Result) -> T, 49 | { 50 | let futures = self 51 | .0 52 | .iter_mut() 53 | .map(|(_, c)| async { c.lock().await.recv().await }); 54 | join_all(futures).await.into_iter().map(|x| f(x)).collect() 55 | } 56 | 57 | /// Close the channel by the given `NodeId` asynchronously, and remove the channel in this map. 58 | pub async fn close_async(&mut self, id: &NodeId) { 59 | if let Some(c) = self.get(id) { 60 | c.lock().await.close(); 61 | self.0.remove(id); 62 | } 63 | } 64 | 65 | /// Close the channel by the given `NodeId`, and remove the channel in this map. 66 | pub fn close(&mut self, id: &NodeId) { 67 | if let Some(c) = self.get(id) { 68 | c.blocking_lock().close(); 69 | self.0.remove(id); 70 | } 71 | } 72 | 73 | pub(crate) fn insert(&mut self, node_id: NodeId, channel: Arc>) { 74 | self.0.insert(node_id, channel); 75 | } 76 | 77 | pub(crate) fn close_all(&mut self) { 78 | self.0.values_mut().for_each(|c| c.blocking_lock().close()); 79 | } 80 | 81 | fn get(&self, id: &NodeId) -> Option>> { 82 | match self.0.get(id) { 83 | Some(c) => Some(c.clone()), 84 | None => None, 85 | } 86 | } 87 | 88 | fn keys(&self) -> Vec { 89 | self.0.keys().map(|x| *x).collect() 90 | } 91 | } 92 | 93 | /// # Input Channel 94 | /// Wrapper of receivers of `tokio::sync::mpsc` and `tokio::sync::broadcast`. **Dagrs** will 95 | /// decide the inner type of channel when building the graph. 96 | /// Learn more about [Tokio Channels](https://tokio.rs/tokio/tutorial/channels). 97 | pub enum InChannel { 98 | /// Receiver of a `tokio::sync::mpsc` channel. 99 | Mpsc(mpsc::Receiver), 100 | /// Receiver of a `tokio::sync::broadcast` channel. 101 | Bcst(broadcast::Receiver), 102 | } 103 | 104 | impl InChannel { 105 | /// Perform a blocking receive on this channel. 106 | fn blocking_recv(&mut self) -> Result { 107 | match self { 108 | InChannel::Mpsc(receiver) => { 109 | if let Some(content) = receiver.blocking_recv() { 110 | Ok(content) 111 | } else { 112 | Err(RecvErr::Closed) 113 | } 114 | } 115 | InChannel::Bcst(receiver) => match receiver.blocking_recv() { 116 | Ok(v) => Ok(v), 117 | Err(e) => match e { 118 | broadcast::error::RecvError::Closed => Err(RecvErr::Closed), 119 | broadcast::error::RecvError::Lagged(x) => Err(RecvErr::Lagged(x)), 120 | }, 121 | }, 122 | } 123 | } 124 | 125 | /// Perform a asynchronous receive on this channel. 126 | async fn recv(&mut self) -> Result { 127 | match self { 128 | InChannel::Mpsc(receiver) => { 129 | if let Some(content) = receiver.recv().await { 130 | Ok(content) 131 | } else { 132 | Err(RecvErr::Closed) 133 | } 134 | } 135 | InChannel::Bcst(receiver) => match receiver.recv().await { 136 | Ok(v) => Ok(v), 137 | Err(e) => match e { 138 | broadcast::error::RecvError::Closed => Err(RecvErr::Closed), 139 | broadcast::error::RecvError::Lagged(x) => Err(RecvErr::Lagged(x)), 140 | }, 141 | }, 142 | } 143 | } 144 | 145 | /// Close the channel and drop the messages inside. 146 | fn close(&mut self) { 147 | match self { 148 | InChannel::Mpsc(receiver) => receiver.close(), 149 | // Broadcast channel will be closed after `self` is dropped. 150 | InChannel::Bcst(_) => (), 151 | } 152 | } 153 | } 154 | 155 | /// # Input Channel Error Types 156 | /// - NoSuchChannel: try to get a channel with an invalid `NodeId`. 157 | /// - Closed: the channel to receive messages from is closed and empty already. 158 | /// - Lagged(x): the channel encounters a cache overflow and `x` information 159 | /// pakages are dropped on this receiver's side. 160 | #[derive(Debug)] 161 | pub enum RecvErr { 162 | NoSuchChannel, 163 | Closed, 164 | Lagged(u64), 165 | } 166 | -------------------------------------------------------------------------------- /src/connection/information_packet.rs: -------------------------------------------------------------------------------- 1 | use std::{any::Any, sync::Arc}; 2 | 3 | /// Container type to store task output. 4 | #[derive(Debug, Clone)] 5 | pub struct Content { 6 | inner: Arc, 7 | } 8 | 9 | impl Content { 10 | /// Construct a new [`Content`]. 11 | pub fn new(val: H) -> Self { 12 | Self { 13 | inner: Arc::new(val), 14 | } 15 | } 16 | 17 | pub fn from_arc(val: Arc) -> Self { 18 | Self { inner: val } 19 | } 20 | 21 | pub fn get(&self) -> Option<&H> { 22 | self.inner.downcast_ref::() 23 | } 24 | 25 | pub fn into_inner(self) -> Option> { 26 | self.inner.downcast::().ok() 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/connection/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod in_channel; 2 | pub mod information_packet; 3 | pub mod out_channel; 4 | -------------------------------------------------------------------------------- /src/connection/out_channel.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, sync::Arc}; 2 | 3 | use futures::future::join_all; 4 | use tokio::sync::{broadcast, mpsc, Mutex}; 5 | 6 | use crate::node::node::NodeId; 7 | 8 | use super::information_packet::Content; 9 | 10 | /// # Output Channels 11 | /// A hash-table mapping `NodeId` to `OutChannel`. In **Dagrs**, each `Node` stores output 12 | /// channels in this map, enabling `Node` to send information packets to other `Node`s. 13 | #[derive(Default)] 14 | pub struct OutChannels(pub(crate) HashMap>>); 15 | 16 | impl OutChannels { 17 | /// Perform a blocking send on the outcoming channel from `NodeId`. 18 | pub fn blocking_send_to(&self, id: &NodeId, content: Content) -> Result<(), SendErr> { 19 | match self.get(id) { 20 | Some(channel) => channel.blocking_lock().blocking_send(content), 21 | None => Err(SendErr::NoSuchChannel), 22 | } 23 | } 24 | 25 | /// Perform a asynchronous send on the outcoming channel from `NodeId`. 26 | pub async fn send_to(&self, id: &NodeId, content: Content) -> Result<(), SendErr> { 27 | match self.get(id) { 28 | Some(channel) => channel.lock().await.send(content).await, 29 | None => Err(SendErr::NoSuchChannel), 30 | } 31 | } 32 | 33 | /// Broadcasts the `content` to all the [`OutChannel`]s asynchronously. 34 | pub async fn broadcast(&self, content: Content) -> Vec> { 35 | let futures = self 36 | .0 37 | .iter() 38 | .map(|(_, c)| async { c.lock().await.send(content.clone()).await }); 39 | 40 | join_all(futures).await 41 | } 42 | 43 | /// Blocking broadcasts the `content` to all the [`OutChannel`]s. 44 | pub fn blocking_broadcast(&self, content: Content) -> Vec> { 45 | self.0 46 | .iter() 47 | .map(|(_, c)| c.blocking_lock().blocking_send(content.clone())) 48 | .collect() 49 | } 50 | 51 | /// Close the channel by the given `NodeId`, and remove the channel in this map. 52 | pub fn close(&mut self, id: &NodeId) { 53 | if let Some(_) = self.get(id) { 54 | self.0.remove(id); 55 | } 56 | } 57 | 58 | pub(crate) fn close_all(&mut self) { 59 | self.0.clear(); 60 | } 61 | 62 | fn get(&self, id: &NodeId) -> Option>> { 63 | match self.0.get(id) { 64 | Some(c) => Some(c.clone()), 65 | None => None, 66 | } 67 | } 68 | 69 | pub(crate) fn insert(&mut self, node_id: NodeId, channel: Arc>) { 70 | self.0.insert(node_id, channel); 71 | } 72 | } 73 | 74 | /// # Output Channel 75 | /// Wrapper of senderrs of `tokio::sync::mpsc` and `tokio::sync::broadcast`. **Dagrs** will 76 | /// decide the inner type of channel when building the graph. 77 | /// Learn more about [Tokio Channels](https://tokio.rs/tokio/tutorial/channels). 78 | pub enum OutChannel { 79 | /// Sender of a `tokio::sync::mpsc` channel. 80 | Mpsc(mpsc::Sender), 81 | /// Sender of a `tokio::sync::broadcast` channel. 82 | Bcst(broadcast::Sender), 83 | } 84 | 85 | impl OutChannel { 86 | /// Perform a blocking send on this channel. 87 | fn blocking_send(&self, value: Content) -> Result<(), SendErr> { 88 | match self { 89 | OutChannel::Mpsc(sender) => match sender.blocking_send(value) { 90 | Ok(_) => Ok(()), 91 | Err(e) => Err(SendErr::ClosedChannel(e.0)), 92 | }, 93 | OutChannel::Bcst(sender) => match sender.send(value) { 94 | Ok(_) => Ok(()), 95 | Err(e) => Err(SendErr::ClosedChannel(e.0)), 96 | }, 97 | } 98 | } 99 | 100 | /// Perform a asynchronous send on this channel. 101 | async fn send(&self, value: Content) -> Result<(), SendErr> { 102 | match self { 103 | OutChannel::Mpsc(sender) => match sender.send(value).await { 104 | Ok(_) => Ok(()), 105 | Err(e) => Err(SendErr::ClosedChannel(e.0)), 106 | }, 107 | OutChannel::Bcst(sender) => match sender.send(value) { 108 | Ok(_) => Ok(()), 109 | Err(e) => Err(SendErr::ClosedChannel(e.0)), 110 | }, 111 | } 112 | } 113 | } 114 | 115 | /// # Output Channel Error Types 116 | /// - NoSuchChannel: try to get a channel with an invalid `NodeId`. 117 | /// - ClosedChannel: the channel is closed alredy. 118 | /// 119 | /// In cases of getting errs of type `MpscError` and `BcstError`, the sender 120 | /// will find there are no active receivers left, so try to send messages is 121 | /// meaningless for now. 122 | #[derive(Debug)] 123 | pub enum SendErr { 124 | NoSuchChannel, 125 | ClosedChannel(Content), 126 | } 127 | -------------------------------------------------------------------------------- /src/graph/abstract_graph.rs: -------------------------------------------------------------------------------- 1 | use crate::node::node::NodeId; 2 | use std::collections::{HashMap, HashSet, VecDeque}; 3 | 4 | /// A simplified graph structure used for cycle detection 5 | pub(crate) struct AbstractGraph { 6 | /// Maps node IDs to their in-degrees 7 | pub in_degree: HashMap, 8 | /// Maps node IDs to their outgoing edges (destination node IDs) 9 | pub edges: HashMap>, 10 | /// Maps folded node IDs to their abstract node IDs 11 | pub folded_nodes: HashMap, 12 | /// Maps abstract node IDs to concrete node IDs 13 | pub unfold_abstract_nodes: HashMap>, 14 | } 15 | 16 | impl AbstractGraph { 17 | /// Creates a new empty abstract graph 18 | pub fn new() -> Self { 19 | Self { 20 | in_degree: HashMap::new(), 21 | edges: HashMap::new(), 22 | folded_nodes: HashMap::new(), 23 | unfold_abstract_nodes: HashMap::new(), 24 | } 25 | } 26 | 27 | /// Adds a node to the abstract graph 28 | pub fn add_node(&mut self, node_id: NodeId) { 29 | if !self.in_degree.contains_key(&node_id) { 30 | self.in_degree.insert(node_id, 0); 31 | self.edges.insert(node_id, HashSet::new()); 32 | } 33 | } 34 | 35 | /// Adds an edge between two nodes in the abstract graph 36 | pub fn add_edge(&mut self, from: NodeId, to: NodeId) { 37 | // Look up the abstract node ID that a concrete node ID has been folded into. 38 | let mut abstract_flag_from = false; 39 | let mut abstract_flag_to = false; 40 | let from = if let Some(abstract_id) = self.get_abstract_node_id(&from) { 41 | abstract_flag_from = true; 42 | *abstract_id 43 | } else { 44 | from 45 | }; 46 | let to = if let Some(abstract_id) = self.get_abstract_node_id(&to) { 47 | abstract_flag_to = true; 48 | *abstract_id 49 | } else { 50 | to 51 | }; 52 | 53 | // If both `from` and `to` are abstract node IDs and `from` == `to`, skip the edge addition 54 | if abstract_flag_from && abstract_flag_to && from == to { 55 | return; 56 | } 57 | 58 | log::debug!("Adding edge from {:?} to {:?}", from, to); 59 | 60 | self.edges.get_mut(&from).unwrap().insert(to); 61 | *self.in_degree.get_mut(&to).unwrap() += 1; 62 | } 63 | 64 | /// Adds a folded node to the abstract graph 65 | pub fn add_folded_node(&mut self, abstract_node_id: NodeId, concrete_node_id: Vec) { 66 | self.add_node(abstract_node_id); 67 | 68 | for concrete_id in &concrete_node_id { 69 | self.folded_nodes.insert(*concrete_id, abstract_node_id); 70 | } 71 | 72 | self.unfold_abstract_nodes 73 | .insert(abstract_node_id, concrete_node_id); 74 | } 75 | 76 | /// Look up the concrete node IDs that an abstract node ID has been unfolded into. 77 | pub fn unfold_node(&self, abstract_node_id: NodeId) -> Option<&Vec> { 78 | self.unfold_abstract_nodes.get(&abstract_node_id) 79 | } 80 | 81 | /// Look up the abstract node ID that a concrete node ID has been folded into. 82 | /// Returns None if the node ID has not been folded into an abstract node. 83 | pub fn get_abstract_node_id(&self, node_id: &NodeId) -> Option<&NodeId> { 84 | self.folded_nodes.get(node_id) 85 | } 86 | 87 | /// Returns the total number of nodes in the abstract graph 88 | pub fn size(&self) -> usize { 89 | self.in_degree.len() 90 | } 91 | 92 | /// Check if the graph contains any cycles/loops using a topological sorting approach. 93 | /// Returns true if the graph contains a cycle, false otherwise. 94 | pub fn check_loop(&self) -> bool { 95 | let mut in_degree = self.in_degree.clone(); 96 | let mut visited_count = 0; 97 | 98 | // Start with nodes that have 0 in-degree 99 | let mut queue: VecDeque = in_degree 100 | .iter() 101 | .filter_map(|(&node, °ree)| if degree == 0 { Some(node) } else { None }) 102 | .collect(); 103 | 104 | while let Some(node) = queue.pop_front() { 105 | log::debug!("Visiting node: {:?}", node); 106 | visited_count += 1; 107 | 108 | // For each outgoing edge 109 | if let Some(nexts) = self.edges.get(&node) { 110 | // Decrease in-degree of the target node 111 | for next in nexts { 112 | let degree = in_degree.get_mut(next).unwrap(); 113 | *degree -= 1; 114 | // If in-degree becomes 0, add to queue 115 | if *degree == 0 { 116 | queue.push_back(*next); 117 | } 118 | } 119 | } 120 | } 121 | 122 | // If we haven't visited all nodes, there must be a cycle (visited_count != size) 123 | log::debug!("Visited count: {}, Size: {}", visited_count, self.size()); 124 | visited_count != self.size() 125 | } 126 | } 127 | 128 | #[cfg(test)] 129 | mod abstract_graph_test { 130 | use super::*; 131 | 132 | /// Tests the cycle detection functionality of the graph. 133 | /// Creates a graph with two nodes (1 and 2) and adds edges to form a cycle: 134 | /// Node 1 <-> Node 2 135 | /// check_loop() returns true. 136 | #[test] 137 | fn test_check_loop() { 138 | let mut graph = AbstractGraph::new(); 139 | graph.add_node(NodeId(1)); 140 | graph.add_node(NodeId(2)); 141 | graph.add_edge(NodeId(1), NodeId(2)); 142 | graph.add_edge(NodeId(2), NodeId(1)); 143 | assert!(graph.check_loop()); 144 | } 145 | 146 | /// Tests the cycle detection functionality of the graph. 147 | /// Creates a graph with two nodes (1 and 2) and adds a single directed edge: 148 | /// Node 1 -> Node 2 149 | /// Since there is no cycle in this graph, check_loop() returns false. 150 | #[test] 151 | fn test_check_no_loop() { 152 | let mut graph = AbstractGraph::new(); 153 | graph.add_node(NodeId(1)); 154 | graph.add_node(NodeId(2)); 155 | graph.add_edge(NodeId(1), NodeId(2)); 156 | assert!(!graph.check_loop()); 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/graph/error.rs: -------------------------------------------------------------------------------- 1 | #[derive(Clone, Debug)] 2 | pub enum GraphError { 3 | GraphLoopDetected, 4 | GraphNotActive, 5 | ExecutionFailed { 6 | node_name: String, 7 | node_id: usize, 8 | error: String, 9 | }, 10 | PanicOccurred { 11 | node_name: String, 12 | node_id: usize, 13 | }, 14 | MultipleErrors(Vec), 15 | } 16 | 17 | impl std::fmt::Display for GraphError { 18 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 19 | write!(f, "{:?}", self) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/graph/loop_subgraph.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use std::sync::Arc; 3 | use tokio::sync::Mutex; 4 | 5 | use crate::{EnvVar, InChannels, Node, NodeId, NodeName, NodeTable, OutChannels, Output}; 6 | 7 | /// A special node type that represents a subgraph of nodes in a loop structure. 8 | /// 9 | /// The LoopSubgraph is included in the main graph as a single node, but internally contains 10 | /// multiple nodes that will be executed repeatedly. The connection and execution of the loop is controlled 11 | /// by the parent graph rather than the LoopSubgraph itself. 12 | pub struct LoopSubgraph { 13 | id: NodeId, 14 | name: NodeName, 15 | in_channels: InChannels, 16 | out_channels: OutChannels, 17 | // Inner nodes, contains the nodes that need to be executed in a loop 18 | inner_nodes: Vec>>, 19 | } 20 | 21 | impl LoopSubgraph { 22 | pub fn new(name: NodeName, node_table: &mut NodeTable) -> Self { 23 | Self { 24 | id: node_table.alloc_id_for(&name), 25 | name, 26 | in_channels: InChannels::default(), 27 | out_channels: OutChannels::default(), 28 | inner_nodes: Vec::new(), 29 | } 30 | } 31 | 32 | /// Add a node to the subgraph 33 | pub fn add_node(&mut self, node: impl Node + 'static) { 34 | self.inner_nodes.push(Arc::new(Mutex::new(node))); 35 | } 36 | } 37 | 38 | #[async_trait] 39 | impl Node for LoopSubgraph { 40 | fn id(&self) -> NodeId { 41 | self.id 42 | } 43 | 44 | fn name(&self) -> NodeName { 45 | self.name.clone() 46 | } 47 | 48 | fn input_channels(&mut self) -> &mut InChannels { 49 | &mut self.in_channels 50 | } 51 | 52 | fn output_channels(&mut self) -> &mut OutChannels { 53 | &mut self.out_channels 54 | } 55 | 56 | fn loop_structure(&self) -> Option>>> { 57 | Some(self.inner_nodes.clone()) 58 | } 59 | 60 | async fn run(&mut self, _: Arc) -> Output { 61 | panic!("Loop subgraph is not executed directly, it will be executed by the parent graph."); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/graph/mod.rs: -------------------------------------------------------------------------------- 1 | mod abstract_graph; 2 | pub mod error; 3 | pub mod graph; 4 | pub mod loop_subgraph; 5 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod connection; 2 | pub mod graph; 3 | pub mod node; 4 | pub mod utils; 5 | 6 | pub use connection::{ 7 | in_channel::{InChannels, RecvErr}, 8 | information_packet::Content, 9 | out_channel::{OutChannels, SendErr}, 10 | }; 11 | pub use node::{ 12 | action::{Action, EmptyAction}, 13 | default_node::DefaultNode, 14 | node::*, 15 | }; 16 | 17 | pub use async_trait; 18 | pub use graph::graph::*; 19 | pub use tokio; 20 | pub use utils::{env::EnvVar, output::Output}; 21 | 22 | #[cfg(feature = "derive")] 23 | pub use dagrs_derive::*; 24 | -------------------------------------------------------------------------------- /src/node/action.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use async_trait::async_trait; 4 | 5 | use crate::{ 6 | connection::{in_channel::InChannels, out_channel::OutChannels}, 7 | utils::{env::EnvVar, output::Output}, 8 | }; 9 | 10 | /// Node specific behavior 11 | /// 12 | /// [`Action`] stores the specific execution logic of a task. 13 | /// 14 | /// # Example 15 | /// An implementation of [`Action`]: `HelloAction`, having private 16 | /// fields `statement` and `repeat`. 17 | /// 18 | /// ```rust 19 | /// use std::sync::Arc; 20 | /// use dagrs::{Action, EnvVar, Output, InChannels, OutChannels}; 21 | /// use async_trait::async_trait; 22 | /// 23 | /// struct HelloAction{ 24 | /// statement: String, 25 | /// repeat: usize, 26 | /// } 27 | /// 28 | /// #[async_trait] 29 | /// impl Action for HelloAction{ 30 | /// async fn run(&self, _: &mut InChannels, _: &mut OutChannels, _: Arc) -> Output{ 31 | /// for i in 0..self.repeat { 32 | /// println!("{}",self.statement); 33 | /// } 34 | /// Output::empty() 35 | /// } 36 | /// } 37 | /// 38 | /// let hello=HelloAction { 39 | /// statement: "hello world!".to_string(), 40 | /// repeat: 10 41 | /// }; 42 | /// 43 | /// ``` 44 | #[async_trait] 45 | pub trait Action: Send + Sync { 46 | async fn run( 47 | &self, 48 | in_channels: &mut InChannels, 49 | out_channels: &mut OutChannels, 50 | env: Arc, 51 | ) -> Output; 52 | } 53 | 54 | /// An empty implementaion of [`Action`]. 55 | /// 56 | /// Used as a placeholder when creating a `Node` without `Action`. 57 | pub struct EmptyAction; 58 | #[async_trait] 59 | impl Action for EmptyAction { 60 | async fn run(&self, _: &mut InChannels, _: &mut OutChannels, _: Arc) -> Output { 61 | Output::Out(None) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/node/conditional_node.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use async_trait::async_trait; 4 | 5 | use crate::{EnvVar, InChannels, Node, NodeId, NodeName, NodeTable, OutChannels, Output}; 6 | 7 | /// # `Condition` trait 8 | /// 9 | /// The [`Condition`] trait trait defines the condition logic for a [`ConditionalNode`]. 10 | /// The `run` method evaluates the condition based on input channels, output channels and environment variables. 11 | /// Returns a boolean indicating whether the condition is met. 12 | #[async_trait] 13 | pub trait Condition: Send + Sync { 14 | async fn run( 15 | &self, 16 | in_channels: &mut InChannels, 17 | out_channels: &OutChannels, 18 | env: Arc, 19 | ) -> bool; 20 | } 21 | 22 | /// # Conditional node type 23 | /// 24 | /// [`ConditionalNode`] is a node type that executes based on a condition. 25 | /// It can evaluate the condition using input channels, output channels and environment variables. 26 | /// The condition logic is defined by implementing the [`Condition`] trait. 27 | /// 28 | /// When the condition is met during execution, Dagrs will continue running. 29 | /// If the condition is not met, Dagrs will stop on this node. 30 | pub struct ConditionalNode { 31 | id: NodeId, 32 | name: NodeName, 33 | condition: Box, 34 | in_channels: InChannels, 35 | out_channels: OutChannels, 36 | } 37 | 38 | impl ConditionalNode { 39 | /// Creates a new [`ConditionalNode`] with the given name and condition. 40 | /// The [`Condition`] determines whether execution should continue or stop at this node. 41 | /// The [`NodeTable`] is used to allocate a unique ID for this node. 42 | pub fn with_condition( 43 | name: NodeName, 44 | condition: impl Condition + 'static, 45 | node_table: &mut NodeTable, 46 | ) -> Self { 47 | Self { 48 | id: node_table.alloc_id_for(&name), 49 | name, 50 | condition: Box::new(condition), 51 | in_channels: InChannels::default(), 52 | out_channels: OutChannels::default(), 53 | } 54 | } 55 | } 56 | 57 | #[async_trait] 58 | impl Node for ConditionalNode { 59 | /// Returns the unique identifier of this conditional node 60 | fn id(&self) -> NodeId { 61 | self.id 62 | } 63 | 64 | /// Returns the name of this conditional node 65 | fn name(&self) -> NodeName { 66 | self.name.clone() 67 | } 68 | 69 | /// Returns a mutable reference to the input channels of this node 70 | fn input_channels(&mut self) -> &mut InChannels { 71 | &mut self.in_channels 72 | } 73 | 74 | /// Returns a mutable reference to the output channels of this node 75 | fn output_channels(&mut self) -> &mut OutChannels { 76 | &mut self.out_channels 77 | } 78 | 79 | /// Executes the condition logic of this node 80 | async fn run(&mut self, env: Arc) -> Output { 81 | Output::ConditionResult( 82 | self.condition 83 | .run(&mut self.in_channels, &self.out_channels, env) 84 | .await, 85 | ) 86 | } 87 | 88 | /// Returns true since this is a conditional node 89 | fn is_condition(&self) -> bool { 90 | true 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/node/default_node.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use async_trait::async_trait; 4 | 5 | use crate::{ 6 | connection::{in_channel::InChannels, out_channel::OutChannels}, 7 | utils::{env::EnvVar, output::Output}, 8 | }; 9 | 10 | use super::{ 11 | action::{Action, EmptyAction}, 12 | node::{Node, NodeId, NodeName, NodeTable}, 13 | }; 14 | 15 | /// # Default node type 16 | /// 17 | /// [`DefaultNode`] is a default implementation of the [`Node`] trait. Users can use this node 18 | /// type to build tasks to meet most needs. 19 | /// 20 | /// ## Create a `DefaultNode`: 21 | /// - use the method `new`. Required attributes: node's name; [`NodeTable`](for id allocation). 22 | /// 23 | /// ```rust 24 | /// use dagrs::{NodeName, NodeTable, DefaultNode}; 25 | /// 26 | /// let node_name = "Node X"; 27 | /// let mut node_table = NodeTable::new(); 28 | /// let mut node = DefaultNode::new( 29 | /// NodeName::from(node_name), 30 | /// &mut node_table, 31 | /// ); 32 | /// ``` 33 | /// 34 | /// - use the method `with_action`. Required attributes: node's name; [`NodeTable`](for id allocation); 35 | /// execution logic [`Action`]. 36 | /// 37 | /// ```rust 38 | /// use dagrs::{NodeName, NodeTable, DefaultNode, EmptyAction}; 39 | /// 40 | /// let node_name = "Node X"; 41 | /// let mut node_table = NodeTable::new(); 42 | /// let mut node = DefaultNode::with_action( 43 | /// NodeName::from(node_name), 44 | /// EmptyAction, 45 | /// &mut node_table, 46 | /// ); 47 | /// ``` 48 | pub struct DefaultNode { 49 | id: NodeId, 50 | name: NodeName, 51 | action: Box, 52 | in_channels: InChannels, 53 | out_channels: OutChannels, 54 | } 55 | #[async_trait] 56 | impl Node for DefaultNode { 57 | fn id(&self) -> NodeId { 58 | self.id 59 | } 60 | 61 | fn name(&self) -> NodeName { 62 | self.name.clone() 63 | } 64 | 65 | fn input_channels(&mut self) -> &mut InChannels { 66 | &mut self.in_channels 67 | } 68 | 69 | fn output_channels(&mut self) -> &mut OutChannels { 70 | &mut self.out_channels 71 | } 72 | 73 | async fn run(&mut self, env: Arc) -> Output { 74 | self.action 75 | .run(&mut self.in_channels, &mut self.out_channels, env) 76 | .await 77 | } 78 | } 79 | 80 | impl DefaultNode { 81 | pub fn new(name: NodeName, node_table: &mut NodeTable) -> Self { 82 | Self { 83 | id: node_table.alloc_id_for(&name), 84 | name, 85 | action: Box::new(EmptyAction), 86 | in_channels: InChannels::default(), 87 | out_channels: OutChannels::default(), 88 | } 89 | } 90 | 91 | pub fn with_action( 92 | name: NodeName, 93 | action: impl Action + 'static, 94 | node_table: &mut NodeTable, 95 | ) -> Self { 96 | Self { 97 | id: node_table.alloc_id_for(&name), 98 | name, 99 | action: Box::new(action), 100 | in_channels: InChannels::default(), 101 | out_channels: OutChannels::default(), 102 | } 103 | } 104 | 105 | pub fn set_action(&mut self, action: impl Action + 'static) { 106 | self.action = Box::new(action) 107 | } 108 | } 109 | 110 | #[cfg(test)] 111 | mod test_default_node { 112 | 113 | use std::sync::Arc; 114 | 115 | use crate::{Content, EnvVar, InChannels, Node, NodeName, NodeTable, OutChannels, Output}; 116 | 117 | use super::{Action, DefaultNode}; 118 | 119 | use async_trait::async_trait; 120 | 121 | /// An implementation of [`Action`] that returns [`Output::Out`] containing a String "Hello world". 122 | #[derive(Default)] 123 | pub struct HelloAction; 124 | #[async_trait] 125 | impl Action for HelloAction { 126 | async fn run(&self, _: &mut InChannels, _: &mut OutChannels, _: Arc) -> Output { 127 | Output::Out(Some(Content::new("Hello world".to_string()))) 128 | } 129 | } 130 | 131 | impl HelloAction { 132 | pub fn new() -> Self { 133 | Self::default() 134 | } 135 | } 136 | 137 | /// Test for create a default node. 138 | /// 139 | /// Step 1: create a [`DefaultNode`] with [`HelloAction`]. 140 | /// 141 | /// Step 2: run the node and verify its output. 142 | #[test] 143 | fn create_default_node() { 144 | let node_name = "Test Node"; 145 | 146 | let mut node_table = NodeTable::new(); 147 | let mut node = DefaultNode::with_action( 148 | NodeName::from(node_name), 149 | HelloAction::new(), 150 | &mut node_table, 151 | ); 152 | 153 | // Check if node table has key-value pair (node.name, node.id) 154 | assert_eq!(node_table.get(node_name).unwrap(), &node.id()); 155 | 156 | let env = Arc::new(EnvVar::new(node_table)); 157 | let out = tokio::runtime::Runtime::new() 158 | .unwrap() 159 | .block_on(async { node.run(env).await.get_out().unwrap() }); 160 | let out: &String = out.get().unwrap(); 161 | assert_eq!(out, "Hello world"); 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /src/node/id_allocate.rs: -------------------------------------------------------------------------------- 1 | use std::sync::atomic::AtomicUsize; 2 | 3 | use super::node::NodeId; 4 | 5 | /// IDAllocator for Node. 6 | struct IDAllocator { 7 | id: AtomicUsize, 8 | } 9 | 10 | impl IDAllocator { 11 | fn alloc(&self) -> NodeId { 12 | let origin = self.id.fetch_add(1, std::sync::atomic::Ordering::SeqCst); 13 | if origin > self.id.load(std::sync::atomic::Ordering::Relaxed) { 14 | panic!("Too many tasks.") 15 | } else { 16 | NodeId(origin) 17 | } 18 | } 19 | } 20 | 21 | /// The global task uniquely identifies an instance of the allocator. 22 | static ID_ALLOCATOR: IDAllocator = IDAllocator { 23 | id: AtomicUsize::new(1), 24 | }; 25 | 26 | /// Assign node's id. 27 | pub(crate) fn alloc_id() -> NodeId { 28 | ID_ALLOCATOR.alloc() 29 | } 30 | -------------------------------------------------------------------------------- /src/node/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod action; 2 | pub mod conditional_node; 3 | pub mod default_node; 4 | pub mod id_allocate; 5 | pub mod node; 6 | -------------------------------------------------------------------------------- /src/node/node.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, sync::Arc}; 2 | 3 | use async_trait::async_trait; 4 | use tokio::sync::Mutex; 5 | 6 | use crate::{ 7 | connection::{in_channel::InChannels, out_channel::OutChannels}, 8 | utils::{env::EnvVar, output::Output}, 9 | }; 10 | 11 | use super::id_allocate::alloc_id; 12 | 13 | ///# The [`Node`] trait 14 | /// 15 | /// Nodes are the basic scheduling units of Graph. They can be identified by 16 | /// a globally assigned [`NodeId`] and a user-provided name. 17 | /// 18 | /// Nodes can communicate with others asynchronously through [`InChannels`] and [`OutChannels`]. 19 | /// 20 | /// In addition to the above properties, users can also customize some other attributes. 21 | #[async_trait] 22 | pub trait Node: Send + Sync { 23 | /// id is the unique identifier of each node, it will be assigned by the [`NodeTable`] 24 | /// when creating a new node, you can find this node through this identifier. 25 | fn id(&self) -> NodeId; 26 | /// The node's name. 27 | fn name(&self) -> NodeName; 28 | /// Input Channels of this node. 29 | fn input_channels(&mut self) -> &mut InChannels; 30 | /// Output Channels of this node. 31 | fn output_channels(&mut self) -> &mut OutChannels; 32 | /// Execute a run of this node. 33 | async fn run(&mut self, env: Arc) -> Output; 34 | /// Return true if this node is conditional node. By default, it returns false. 35 | fn is_condition(&self) -> bool { 36 | false 37 | } 38 | /// Returns the list of nodes that are part of this node's loop structure, if any. 39 | /// 40 | /// This method is used to identify nodes that are part of a loop-like structure, such as a loop subgraph. 41 | /// When this method returns Some(nodes), the loop detection check will skip checking these nodes for cycles. 42 | /// 43 | /// Returns None by default, indicating this is not a loop-containing node. 44 | fn loop_structure(&self) -> Option>>> { 45 | None 46 | } 47 | } 48 | 49 | #[derive(Debug, Hash, PartialEq, Eq, Clone, Copy, Ord, PartialOrd)] 50 | pub struct NodeId(pub(crate) usize); 51 | 52 | pub type NodeName = String; 53 | 54 | /// [NodeTable]: a mapping from [Node]'s name to [NodeId]. 55 | #[derive(Default)] 56 | pub struct NodeTable(pub(crate) HashMap); 57 | 58 | /// [NodeTable]'s name in [`EnvVar`]. 59 | pub const NODE_TABLE_STR: &str = "node_table"; 60 | 61 | impl NodeTable { 62 | /// Alloc a new [NodeId] for a [Node]. 63 | /// 64 | /// If there is a Node requesting for an ID with a duplicate name, 65 | /// the older one's info will be overwritten. 66 | pub fn alloc_id_for(&mut self, name: &str) -> NodeId { 67 | let id = alloc_id(); 68 | log::debug!("alloc id {:?} for {:?}", id, name); 69 | 70 | if let Some(v) = self.0.insert(name.to_string(), id.clone()) { 71 | log::warn!("Node {} is already allocated with id {:?}.", name, v); 72 | }; 73 | id 74 | } 75 | 76 | /// Get the [`NodeId`] of the node corresponding to its name. 77 | pub fn get(&self, name: &str) -> Option<&NodeId> { 78 | self.0.get(name) 79 | } 80 | 81 | /// Create an empty [`NodeTable`]. 82 | pub fn new() -> Self { 83 | Self::default() 84 | } 85 | } 86 | 87 | impl EnvVar { 88 | /// Get a [`Node`]'s [`NodeId`] by providing its name. 89 | pub fn get_node_id(&self, node_name: &str) -> Option<&NodeId> { 90 | let node_table: &NodeTable = self.get_ref(NODE_TABLE_STR).unwrap(); 91 | node_table.get(node_name) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/utils/env.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use crate::{ 4 | connection::information_packet::Content, 5 | node::node::{NodeTable, NODE_TABLE_STR}, 6 | }; 7 | 8 | pub type Variable = Content; 9 | 10 | /// # Environment variable. 11 | /// 12 | /// When multiple nodes are running, they may need to share the same data or read 13 | /// the same configuration information. Environment variables can meet this requirement. 14 | /// Before all nodes run, the user builds a [`EnvVar`] and sets all the environment 15 | /// variables. One [`EnvVar`] corresponds to one dag. All nodes in a job can 16 | /// be shared and immutable at runtime. environment variables. 17 | /// 18 | /// Variables that [`EnvVar`] should have: 19 | /// - [NodeTable] : a mapping from node's name to `NodeId`. 20 | /// During the runtime of a `Graph`, [`NodeTable`] allows 21 | /// each `Node` to look up the id of a specific node by its name. 22 | #[derive(Debug, Clone)] 23 | pub struct EnvVar { 24 | variables: HashMap, 25 | } 26 | 27 | impl EnvVar { 28 | /// Allocate a new [`EnvVar`]. 29 | pub fn new(node_table: NodeTable) -> Self { 30 | let mut env = Self { 31 | variables: HashMap::default(), 32 | }; 33 | env.set(NODE_TABLE_STR, node_table); 34 | env 35 | } 36 | 37 | #[allow(unused)] 38 | /// Set a global variables. 39 | /// 40 | /// # Example 41 | /// ```rust 42 | /// use dagrs::{EnvVar, NodeTable}; 43 | /// 44 | /// # let mut env = EnvVar::new(NodeTable::default()); 45 | /// env.set("Hello", "World".to_string()); 46 | /// ``` 47 | pub fn set(&mut self, name: &str, var: H) { 48 | let mut v = Variable::new(var); 49 | self.variables.insert(name.to_owned(), v); 50 | } 51 | 52 | /// Get environment variables through keys of type &str. 53 | /// 54 | /// Note: This method will clone the value. To avoid cloning, use `get_ref`. 55 | pub fn get(&self, name: &str) -> Option { 56 | self.get_ref(name).cloned() 57 | } 58 | 59 | /// Get environment variables through keys of type &str. 60 | pub fn get_ref(&self, name: &str) -> Option<&H> { 61 | if let Some(content) = self.variables.get(name) { 62 | content.get() 63 | } else { 64 | None 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/utils/execstate.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{ 2 | atomic::{AtomicBool, Ordering}, 3 | Arc, Mutex, 4 | }; 5 | 6 | use super::output::Output; 7 | use crate::connection::information_packet::Content; 8 | 9 | #[derive(Debug)] 10 | pub(crate) struct ExecState { 11 | /// The execution succeed or not. 12 | success: AtomicBool, 13 | /// Output produced by a task. 14 | output: Arc>, 15 | /*/// The semaphore is used to control the synchronous blocking of subsequent tasks to obtain the 16 | /// execution results of this task. 17 | /// When a task is successfully executed, the permits inside the semaphore will be increased to 18 | /// n (n represents the number of successor tasks of this task or can also be called the output 19 | /// of the node), which means that the output of the task is available, and then each successor 20 | /// The task will obtain a permits synchronously (the permit will not be returned), which means 21 | /// that the subsequent task has obtained the execution result of this task. 22 | semaphore: Semaphore,*/ 23 | } 24 | 25 | impl ExecState { 26 | /// Construct a new [`ExeState`]. 27 | pub(crate) fn new() -> Self { 28 | // initialize the task to failure without output. 29 | Self { 30 | success: AtomicBool::new(false), 31 | output: Arc::new(Mutex::new(Output::empty())), 32 | //semaphore: Semaphore::new(0), 33 | } 34 | } 35 | 36 | /// After the task is successfully executed, set the execution result. 37 | pub(crate) fn set_output(&self, output: Output) { 38 | self.success.store(true, Ordering::Relaxed); 39 | *self.output.lock().unwrap() = output; 40 | } 41 | 42 | /// [`Output`] for fetching internal storage. 43 | /// This function is generally not called directly, but first uses the semaphore for synchronization control. 44 | pub(crate) fn get_output(&self) -> Option { 45 | self.output.lock().unwrap().get_out() 46 | } 47 | pub(crate) fn get_full_output(&self) -> Output { 48 | self.output.lock().unwrap().clone() 49 | } 50 | 51 | pub(crate) fn exe_success(&self) { 52 | self.success.store(true, Ordering::Relaxed) 53 | } 54 | 55 | pub(crate) fn exe_fail(&self) { 56 | self.success.store(false, Ordering::Relaxed) 57 | } 58 | 59 | /*/// The semaphore is used to control the synchronous acquisition of task output results. 60 | /// Under normal circumstances, first use the semaphore to obtain a permit, and then call 61 | /// the `get_output` function to obtain the output. If the current task is not completed 62 | /// (no output is generated), the subsequent task will be blocked until the current task 63 | /// is completed and output is generated. 64 | pub(crate) fn semaphore(&self) -> &Semaphore { 65 | &self.semaphore 66 | }*/ 67 | } 68 | -------------------------------------------------------------------------------- /src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod env; 2 | pub mod execstate; 3 | pub mod output; 4 | -------------------------------------------------------------------------------- /src/utils/output.rs: -------------------------------------------------------------------------------- 1 | //! Node output 2 | //! 3 | //! [`Output`] represents the output of the Node respectively. 4 | //! 5 | //! Users should consider the output results of the Node when defining the specific 6 | //! behavior of the Node. The input results may be: normal output, no output, or Node 7 | //! execution error message. 8 | //! It should be noted that the content stored in [`Output`] must implement the [`Clone`] trait. 9 | //! 10 | //! # Example 11 | //! In general, a Node may produce output or no output: 12 | //! ```rust 13 | //! use dagrs::Output; 14 | //! let out=Output::new(10); 15 | //! let non_out=Output::empty(); 16 | //! ``` 17 | //! In some special cases, when a predictable error occurs in the execution of a Node's 18 | //! specific behavior, the user can choose to return the error message as the output of 19 | //! the Node. Of course, this will cause subsequent Nodes to abandon execution. 20 | //! 21 | //! ```rust 22 | //! use dagrs::Output; 23 | //! use dagrs::Content; 24 | //! let err_out = Output::Err("some error messages!".to_string()); 25 | 26 | use crate::connection::information_packet::Content; 27 | 28 | /// [`Output`] represents the output of a node. Different from information packet (`Content`, 29 | /// used to communicate with other Nodes), `Output` carries the information that `Node` 30 | /// needs to pass to the `Graph`. 31 | #[derive(Clone, Debug)] 32 | pub enum Output { 33 | Out(Option), 34 | Err(String), 35 | ErrWithExitCode(Option, Option), 36 | /// ... 37 | ConditionResult(bool), 38 | } 39 | 40 | impl Output { 41 | /// Construct a new [`Output`]. 42 | /// 43 | /// Since the return value may be transferred between threads, 44 | /// [`Send`], [`Sync`] is needed. 45 | pub fn new(val: H) -> Self { 46 | Self::Out(Some(Content::new(val))) 47 | } 48 | 49 | /// Construct an empty [`Output`]. 50 | pub fn empty() -> Self { 51 | Self::Out(None) 52 | } 53 | 54 | /// Construct an [`Output`]` with an error message. 55 | pub fn error(msg: String) -> Self { 56 | Self::Err(msg) 57 | } 58 | 59 | /// Construct an [`Output`]` with an exit code and an optional error message. 60 | pub fn error_with_exit_code(code: Option, msg: Option) -> Self { 61 | Self::ErrWithExitCode(code, msg) 62 | } 63 | 64 | /// Determine whether [`Output`] stores error information. 65 | pub(crate) fn is_err(&self) -> bool { 66 | match self { 67 | Self::Err(_) | Self::ErrWithExitCode(_, _) => true, 68 | Self::Out(_) | Self::ConditionResult(_) => false, 69 | } 70 | } 71 | 72 | /// Get the contents of [`Output`]. 73 | pub fn get_out(&self) -> Option { 74 | match self { 75 | Self::Out(ref out) => out.clone(), 76 | Self::Err(_) | Self::ErrWithExitCode(_, _) | Self::ConditionResult(_) => None, 77 | } 78 | } 79 | 80 | /// Get error information stored in [`Output`]. 81 | pub fn get_err(&self) -> Option { 82 | match self { 83 | Self::Out(_) | Self::ConditionResult(_) => None, 84 | Self::Err(err) => Some(err.to_string()), 85 | Self::ErrWithExitCode(code, _) => { 86 | let error_code = code.map_or("".to_string(), |v| v.to_string()); 87 | Some(format!("code: {error_code}")) 88 | } 89 | } 90 | } 91 | 92 | /// Get the condition result stored in [`Output`]. 93 | /// 94 | /// Returns `Some(bool)` if this is a `ConditionResult` variant, 95 | /// otherwise returns `None`. 96 | pub(crate) fn conditional_result(&self) -> Option { 97 | match self { 98 | Self::ConditionResult(b) => Some(*b), 99 | _ => None, 100 | } 101 | } 102 | } 103 | --------------------------------------------------------------------------------