├── demo ├── data │ ├── input │ │ ├── format_err │ │ │ ├── csv │ │ │ │ ├── .DS_Store │ │ │ │ ├── encrypt_err.csv │ │ │ │ ├── format_err.csv │ │ │ │ └── csv_processor_encrypt_error.csv │ │ │ └── json │ │ │ │ └── format_err.json │ │ ├── csv │ │ │ ├── level │ │ │ │ └── random_data.csv │ │ │ └── random_data.csv │ │ └── json │ │ │ ├── arr_in_arr.json │ │ │ ├── is_obj.json │ │ │ ├── generated.json │ │ │ └── level │ │ │ └── generated.json │ └── output │ │ ├── csv │ │ ├── mask │ │ │ └── random_data.csv │ │ ├── encrypt │ │ │ └── random_data.csv │ │ ├── format_err │ │ │ ├── decrypt_err.csv │ │ │ ├── processor_err │ │ │ │ └── random_data.csv │ │ │ ├── encrypt_err.csv │ │ │ └── demo │ │ │ │ └── data │ │ │ │ └── input │ │ │ │ └── format_err │ │ │ │ └── csv │ │ │ │ ├── format_err.csv │ │ │ │ ├── csv_processor_encrypt_error.csv │ │ │ │ └── encrypt_err.csv │ │ └── demo │ │ │ └── data │ │ │ └── input │ │ │ └── csv │ │ │ ├── random_data.csv │ │ │ └── level │ │ │ └── random_data.csv │ │ └── json │ │ ├── format_err │ │ └── generated.json │ │ ├── mask │ │ ├── arr_in_arr.json │ │ ├── is_obj.json │ │ └── generated.json │ │ ├── demo │ │ └── data │ │ │ └── input │ │ │ └── json │ │ │ ├── arr_in_arr.json │ │ │ ├── is_obj.json │ │ │ ├── generated.json │ │ │ └── level │ │ │ └── generated.json │ │ ├── decrypt │ │ └── generated.json │ │ └── encrypt │ │ └── generated.json └── conf │ ├── invalid_conf.yaml │ ├── conf_csv.yaml │ └── conf_json.yaml ├── med_core ├── src │ ├── audit │ │ ├── mod.rs │ │ ├── app.rs │ │ └── db.rs │ ├── models │ │ ├── mod.rs │ │ ├── metrics.rs │ │ ├── params.rs │ │ └── enums.rs │ ├── lib.rs │ ├── app │ │ ├── mod.rs │ │ ├── worker.rs │ │ ├── processor.rs │ │ ├── csv.rs │ │ ├── core.rs │ │ └── json.rs │ ├── utils │ │ ├── mod.rs │ │ ├── tests │ │ │ ├── logger_test.rs │ │ │ ├── config_test.rs │ │ │ ├── helpers_test.rs │ │ │ ├── crypto_test.rs │ │ │ └── error_test.rs │ │ ├── helpers.rs │ │ ├── progress_bar.rs │ │ ├── config.rs │ │ ├── logger.rs │ │ ├── crypto.rs │ │ └── error.rs │ └── tests │ │ ├── csv_test.rs │ │ ├── models_test.rs │ │ ├── core_test.rs │ │ └── json_test.rs ├── sqlx-data.json ├── Cargo.toml └── README.md ├── med_cli ├── src │ ├── cli │ │ ├── mod.rs │ │ ├── custom_validation.rs │ │ └── app.rs │ └── main.rs ├── Cargo.toml └── README.md ├── documents └── logo │ └── data-encryption.png ├── db └── migrations │ ├── 20230622150359_add_col_elasped_time.sql │ ├── 20230512195802_audit_sqlite_datastore.down.sql │ └── 20230512195802_audit_sqlite_datastore.up.sql ├── .vscode └── settings.json ├── CHANGELOG.md ├── Cargo.toml ├── sqlx-data.json ├── .gitignore ├── .github └── workflows │ ├── crate.io.yml │ ├── rpm.yml │ ├── PULL_REQUEST_TEMPLATE.md │ ├── ci.yml │ ├── brew.yml │ └── release.yml ├── Makefile ├── README.md └── LICENSE /demo/data/input/format_err/csv/.DS_Store: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/data/output/csv/mask/random_data.csv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/conf/invalid_conf.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | str -------------------------------------------------------------------------------- /demo/data/output/csv/encrypt/random_data.csv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/data/output/json/format_err/generated.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /med_core/src/audit/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod app; 2 | pub mod db; 3 | -------------------------------------------------------------------------------- /med_cli/src/cli/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod app; 2 | mod custom_validation; 3 | -------------------------------------------------------------------------------- /demo/data/output/csv/format_err/decrypt_err.csv: -------------------------------------------------------------------------------- 1 | job_type,name 2 | xadsfad, 3 | -------------------------------------------------------------------------------- /demo/data/input/format_err/csv/encrypt_err.csv: -------------------------------------------------------------------------------- 1 | job_type,name 2 | xadsfad,#### 3 | -------------------------------------------------------------------------------- /demo/data/input/format_err/csv/format_err.csv: -------------------------------------------------------------------------------- 1 | job_type,name, 2 | asdf,asdfa 3 | -------------------------------------------------------------------------------- /demo/data/output/csv/format_err/processor_err/random_data.csv: -------------------------------------------------------------------------------- 1 | job_type,name, 2 | -------------------------------------------------------------------------------- /demo/data/input/csv/level/random_data.csv: -------------------------------------------------------------------------------- 1 | job_type,name 2 | JuniorEngineer,Antony Brandt -------------------------------------------------------------------------------- /demo/data/input/csv/random_data.csv: -------------------------------------------------------------------------------- 1 | job_type,name 2 | JuniorEngineer,Antony Brandt 3 | -------------------------------------------------------------------------------- /demo/data/input/format_err/csv/csv_processor_encrypt_error.csv: -------------------------------------------------------------------------------- 1 | job_type,name 2 | x,#, -------------------------------------------------------------------------------- /med_core/src/models/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod enums; 2 | pub mod metrics; 3 | pub mod params; 4 | -------------------------------------------------------------------------------- /demo/data/output/csv/format_err/encrypt_err.csv: -------------------------------------------------------------------------------- 1 | job_type,name 2 | xadsfad,pQWoo7BNQXw= 3 | -------------------------------------------------------------------------------- /demo/data/output/csv/format_err/demo/data/input/format_err/csv/format_err.csv: -------------------------------------------------------------------------------- 1 | job_type,name, 2 | -------------------------------------------------------------------------------- /med_core/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod app; 2 | pub mod audit; 3 | pub mod models; 4 | pub mod utils; 5 | -------------------------------------------------------------------------------- /demo/data/input/json/arr_in_arr.json: -------------------------------------------------------------------------------- 1 | [{"data": [{"test": [[{"name":"test", "test": "hello"}], "phone"]}]}] -------------------------------------------------------------------------------- /demo/data/output/json/mask/arr_in_arr.json: -------------------------------------------------------------------------------- 1 | [{"data":[{"test":[[{"name":"#####","test":"hello"}],"phone"]}]}] -------------------------------------------------------------------------------- /demo/data/output/csv/format_err/demo/data/input/format_err/csv/csv_processor_encrypt_error.csv: -------------------------------------------------------------------------------- 1 | job_type,name 2 | -------------------------------------------------------------------------------- /demo/data/output/csv/demo/data/input/csv/random_data.csv: -------------------------------------------------------------------------------- 1 | job_type,name 2 | JuniorEngineer,Zbj4lGvuZDpc7pJCvMQEFg== 3 | -------------------------------------------------------------------------------- /demo/data/output/csv/format_err/demo/data/input/format_err/csv/encrypt_err.csv: -------------------------------------------------------------------------------- 1 | job_type,name 2 | xadsfad,##### 3 | -------------------------------------------------------------------------------- /demo/data/output/json/mask/is_obj.json: -------------------------------------------------------------------------------- 1 | {"data":"646ae336271a76d64e27c4db","name":"#####","test":[{"name":"#####"}]} -------------------------------------------------------------------------------- /demo/data/output/json/demo/data/input/json/arr_in_arr.json: -------------------------------------------------------------------------------- 1 | [{"data":[{"test":[[{"name":"#####","test":"hello"}],"phone"]}]}] -------------------------------------------------------------------------------- /med_core/src/app/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod core; 2 | pub mod csv; 3 | pub mod json; 4 | pub mod processor; 5 | pub mod worker; 6 | -------------------------------------------------------------------------------- /demo/conf/conf_csv.yaml: -------------------------------------------------------------------------------- 1 | mask_symbols: "#####" # mask symbols 2 | fields: # list of the cols/fields you want to mask 3 | - name -------------------------------------------------------------------------------- /demo/data/output/csv/demo/data/input/csv/level/random_data.csv: -------------------------------------------------------------------------------- 1 | job_type,name 2 | JuniorEngineer,Zbj4lGvuZDpc7pJCvMQEFg== 3 | -------------------------------------------------------------------------------- /documents/logo/data-encryption.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jayhuang75/rust-cli-med/HEAD/documents/logo/data-encryption.png -------------------------------------------------------------------------------- /demo/data/output/json/demo/data/input/json/is_obj.json: -------------------------------------------------------------------------------- 1 | {"data":"646ae336271a76d64e27c4db","name":"#####","test":[{"name":"#####"}]} -------------------------------------------------------------------------------- /db/migrations/20230622150359_add_col_elasped_time.sql: -------------------------------------------------------------------------------- 1 | -- Adding elapsed_time col 2 | ALTER TABLE audit ADD COLUMN elapsed_time TEXT; 3 | -------------------------------------------------------------------------------- /demo/data/input/json/is_obj.json: -------------------------------------------------------------------------------- 1 | { 2 | "data" : "646ae336271a76d64e27c4db", 3 | "name" : "hello world", 4 | "test" : [{"name":"code"}] 5 | } -------------------------------------------------------------------------------- /med_core/src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod config; 2 | pub mod crypto; 3 | pub mod error; 4 | pub mod helpers; 5 | pub mod logger; 6 | pub mod progress_bar; 7 | -------------------------------------------------------------------------------- /db/migrations/20230512195802_audit_sqlite_datastore.down.sql: -------------------------------------------------------------------------------- 1 | -- Add migration script here 2 | -- Add down migration script here 3 | DROP TABLE IF EXISTS audit; -------------------------------------------------------------------------------- /demo/conf/conf_json.yaml: -------------------------------------------------------------------------------- 1 | mask_symbols: "#####" # mask symbols 2 | fields: # list of the cols/fields you want to mask 3 | - name 4 | - email 5 | - phone -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rust-analyzer.linkedProjects": [ 3 | "./Cargo.toml", 4 | "./Cargo.toml" 5 | ], 6 | "rust-analyzer.showUnlinkedFileNotification": false 7 | } -------------------------------------------------------------------------------- /med_core/src/models/metrics.rs: -------------------------------------------------------------------------------- 1 | use crate::utils::error::MedError; 2 | 3 | #[derive(Debug, Default, Clone)] 4 | pub struct Metrics { 5 | pub total_files: usize, 6 | pub metadata: Metadata, 7 | } 8 | 9 | #[derive(Debug, Default, Clone)] 10 | pub struct Metadata { 11 | pub total_records: usize, 12 | pub failed_records: usize, 13 | pub record_failed_reason: Vec, 14 | } 15 | -------------------------------------------------------------------------------- /med_core/src/utils/tests/logger_test.rs: -------------------------------------------------------------------------------- 1 | use crate::{app::core::App, models::params::Params}; 2 | 3 | #[tokio::test] 4 | async fn test_logger() { 5 | let new_params = Params { 6 | conf_path: "../demo/conf/conf_csv.yaml".to_owned(), 7 | debug: true, 8 | ..Default::default() 9 | }; 10 | 11 | let new_app = App::new(new_params.clone()).await.unwrap(); 12 | assert!(new_app.params.debug); 13 | } 14 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # TBD 2 | 3 | === 4 | Unreleased changes. 5 | 6 | 1. JSON file support 7 | 2. File writer wroker parallel execution. 8 | 9 | ## 0.1.0 (2023-05-20) 10 | 11 | =================== 12 | First release for the M.E.D (Mask, Encrypt, and Decrypt). Key feature of this release: 13 | 14 | 1. Rust powered performance. 15 | 2. Provide Masking, and Encyption/Decryption capabilities. 16 | 3. Auditable with build-in SQLite powered Audit table. 17 | 4. CSV file supported. 18 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "med_cli", 4 | "med_core", 5 | ] 6 | 7 | [workspace.dependencies] 8 | tokio = { version = "^1.28.0", features = ["full"] } 9 | clap = { version = "^4.2.5", features = ["cargo"]} 10 | tracing = { version = "^0.1.37", features = ["log"] } 11 | tracing-subscriber = { version = "^0.3.17", default-features = false, features = ["fmt"]} 12 | colored = "^2.0.0" 13 | num_cpus = "^1.15.0" 14 | serde_json = "^1.0.96" 15 | openssl = { version = "^0.10.54", features = ["vendored"] } 16 | -------------------------------------------------------------------------------- /med_core/src/utils/tests/config_test.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use crate::utils::{config::JobConfig, error::MedErrorType}; 4 | 5 | #[tokio::test] 6 | async fn test_new_config_failed_load() { 7 | let test_config_path = Path::new(""); 8 | let test_config = JobConfig::new(test_config_path).await; 9 | match test_config { 10 | Ok(_) => { 11 | unimplemented!() 12 | } 13 | Err(err) => { 14 | assert_eq!(err.error_type, MedErrorType::ConfigError); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /sqlx-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "db": "SQLite", 3 | "5c26f600720adc375e14260181778d1b69f49fad676d6fcac1d85015c153fc15": { 4 | "describe": { 5 | "columns": [], 6 | "nullable": [], 7 | "parameters": { 8 | "Right": 10 9 | } 10 | }, 11 | "query": "\n INSERT INTO audit ( user, hostname, total_files, total_records, failed_records, record_failed_reason, runtime_conf, process_failure_reason, successed, elapsed_time )\n VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)\n " 12 | } 13 | } -------------------------------------------------------------------------------- /db/migrations/20230512195802_audit_sqlite_datastore.up.sql: -------------------------------------------------------------------------------- 1 | -- Add migration script here 2 | CREATE TABLE IF NOT EXISTS audit ( 3 | id INTEGER PRIMARY KEY, 4 | user TEXT NOT NULL, 5 | hostname TEXT NOT NULL, 6 | total_files INTEGER NOT NULL, 7 | total_records INTEGER NOT NULL, 8 | failed_records INTEGER NOT NULL, 9 | record_failed_reason TEXT, 10 | runtime_conf TEXT NOT NULL, 11 | process_failure_reason TEXT, 12 | successed BOOLEAN NOT NULL DEFAULT FALSE, 13 | created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL 14 | ); -------------------------------------------------------------------------------- /med_core/sqlx-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "db": "SQLite", 3 | "5c26f600720adc375e14260181778d1b69f49fad676d6fcac1d85015c153fc15": { 4 | "describe": { 5 | "columns": [], 6 | "nullable": [], 7 | "parameters": { 8 | "Right": 10 9 | } 10 | }, 11 | "query": "\n INSERT INTO audit ( user, hostname, total_files, total_records, failed_records, record_failed_reason, runtime_conf, process_failure_reason, successed, elapsed_time )\n VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)\n " 12 | } 13 | } -------------------------------------------------------------------------------- /med_core/src/utils/tests/helpers_test.rs: -------------------------------------------------------------------------------- 1 | use walkdir::WalkDir; 2 | 3 | use crate::utils::helpers::is_not_hidden; 4 | 5 | #[tokio::test] 6 | async fn test_is_not_hidden() { 7 | let path = "../demo/data/input/format_err/csv"; 8 | 9 | let is_ignored = WalkDir::new(path) 10 | .follow_links(false) 11 | .into_iter() 12 | .filter_entry(is_not_hidden) 13 | .count(); 14 | assert_eq!(is_ignored, 4); 15 | 16 | let is_not_ignored = WalkDir::new(path).follow_links(false).into_iter().count(); 17 | assert_eq!(is_not_ignored, 5); 18 | } 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug/ 4 | target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | 13 | # MSVC Windows builds of rustc generate these, which store debugging information 14 | *.pdb 15 | 16 | # Added by cargo 17 | 18 | /target 19 | /output/**/*.csv 20 | /output/**/*.json 21 | /demo/data/input/demo/** 22 | 23 | .env 24 | *.db 25 | *.db-* -------------------------------------------------------------------------------- /.github/workflows/crate.io.yml: -------------------------------------------------------------------------------- 1 | name: Crate.io build and deploy 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | types: [closed] 8 | 9 | jobs: 10 | crate_io_publish: 11 | name: Crate.io Publish 12 | if: ${{ github.event.pull_request.merged }} 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout sources 16 | uses: actions/checkout@v2 17 | - name: Install stable toolchain 18 | uses: actions-rs/toolchain@v1 19 | with: 20 | profile: minimal 21 | toolchain: stable 22 | override: true 23 | - name: publish med_core & med_cli 24 | run: | 25 | cargo publish -p med_core --token ${CRATE_TOKEN} 26 | cargo publish -p med_cli --token ${CRATE_TOKEN} 27 | env: 28 | CRATE_TOKEN: ${{ secrets.CRATE_TOKEN }} -------------------------------------------------------------------------------- /med_core/src/audit/app.rs: -------------------------------------------------------------------------------- 1 | use crate::{audit::db::Database, models::metrics::Metrics, utils::error::MedError}; 2 | 3 | #[derive(Debug, Default, Clone)] 4 | pub struct Summary { 5 | pub user: String, 6 | pub hostname: String, 7 | pub metrics: Metrics, 8 | pub runtime_conf: String, 9 | pub process_failure_reason: Option, 10 | pub successed: bool, 11 | pub elapsed_time: String, 12 | } 13 | 14 | pub struct Audit { 15 | pub db: Database, 16 | pub summary: Summary, 17 | } 18 | 19 | impl Audit { 20 | pub async fn new() -> Result { 21 | let db = Database::new().await?; 22 | let summary = Summary::default(); 23 | Ok(Audit { db, summary }) 24 | } 25 | 26 | #[cfg(not(tarpaulin_include))] 27 | pub async fn insert(&mut self) -> Result { 28 | let id = self.db.insert(&self.summary).await?; 29 | Ok(id) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /med_core/src/utils/helpers.rs: -------------------------------------------------------------------------------- 1 | use crate::utils::error::MedError; 2 | use std::fs; 3 | use walkdir::{DirEntry, WalkDir}; 4 | 5 | #[cfg(not(tarpaulin_include))] 6 | pub async fn create_output_dir(output_dir: &str, file_dir: &str) -> Result<(), MedError> { 7 | WalkDir::new(file_dir) 8 | .follow_links(true) 9 | .into_iter() 10 | .filter_map(|e| e.ok()) 11 | .filter(|e| e.path().is_dir()) 12 | .for_each(|e| { 13 | let output_path = format!("{}/{}", output_dir, e.path().display()); 14 | fs::create_dir_all(output_path).unwrap(); 15 | }); 16 | Ok(()) 17 | } 18 | 19 | pub fn is_not_hidden(entry: &DirEntry) -> bool { 20 | entry 21 | .file_name() 22 | .to_str() 23 | .map(|s| entry.depth() == 0 || !s.starts_with('.')) 24 | .unwrap_or(false) 25 | } 26 | 27 | #[cfg(test)] 28 | #[path = "./tests/helpers_test.rs"] 29 | mod helpers_test; 30 | -------------------------------------------------------------------------------- /med_core/src/utils/progress_bar.rs: -------------------------------------------------------------------------------- 1 | use indicatif::{ProgressBar, ProgressStyle}; 2 | use std::time::Duration; 3 | 4 | #[cfg(not(windows))] 5 | const TICK_SETTINGS: (&str, u64) = ("⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏ ", 80); 6 | 7 | #[cfg(windows)] 8 | const TICK_SETTINGS: (&str, u64) = (r"+-x| ", 200); 9 | 10 | /// Return a pre-configured progress bar 11 | pub fn get_progress_bar(length: u64, msg: &str) -> ProgressBar { 12 | let progressbar_style = ProgressStyle::default_spinner() 13 | .tick_chars(TICK_SETTINGS.0) 14 | .progress_chars("=> ") 15 | .template(" {spinner} {msg} {percent}% [{bar:30}] {pos}/{len} ETA {elapsed}") 16 | .expect("no template error"); 17 | 18 | let progress_bar = ProgressBar::new(length); 19 | progress_bar.set_style(progressbar_style); 20 | progress_bar.enable_steady_tick(Duration::from_millis(TICK_SETTINGS.1)); 21 | progress_bar.set_message(msg.to_owned()); 22 | 23 | progress_bar 24 | } 25 | -------------------------------------------------------------------------------- /med_core/src/utils/config.rs: -------------------------------------------------------------------------------- 1 | use serde::Deserialize; 2 | use std::path::Path; 3 | 4 | use crate::utils::error::{MedError, MedErrorType}; 5 | 6 | #[derive(Debug, Deserialize, Clone, PartialEq)] 7 | pub struct JobConfig { 8 | pub mask_symbols: String, 9 | pub fields: Vec, 10 | } 11 | 12 | impl JobConfig { 13 | pub async fn new(path: &Path) -> Result { 14 | let f = match std::fs::File::open(path) { 15 | Ok(f) => f, 16 | Err(e) => { 17 | return Err(MedError { 18 | message: Some(e.to_string()), 19 | cause: Some("load job configuration yaml file failed!".to_string()), 20 | error_type: MedErrorType::ConfigError, 21 | }) 22 | } 23 | }; 24 | let config: JobConfig = serde_yaml::from_reader(f)?; 25 | Ok(config) 26 | } 27 | } 28 | 29 | #[cfg(test)] 30 | #[path = "./tests/config_test.rs"] 31 | mod config_test; 32 | -------------------------------------------------------------------------------- /med_cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "med_cli" 3 | version = "0.6.4" 4 | edition = "2021" 5 | authors = ["Wei Huang "] 6 | license = "Apache-2.0" 7 | description = "A Rust Powered CLI tool for CSV/JSON Masking, Encryption, and Decryption." 8 | readme = "README.md" 9 | homepage = "https://github.com/jayhuang75/rust-cli-med" 10 | repository = "https://github.com/jayhuang75/rust-cli-med" 11 | keywords = ["cli", "encrypt", "decrypt", "masking"] 12 | categories = ["command-line-utilities"] 13 | 14 | 15 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 16 | [[bin]] 17 | name = "med" 18 | path = "src/main.rs" 19 | 20 | [dependencies] 21 | tokio = { workspace = true } 22 | clap = { workspace = true } 23 | tracing = { workspace = true } 24 | tracing-subscriber = { workspace = true } 25 | colored = { workspace = true } 26 | num_cpus = { workspace = true } 27 | serde_json = { workspace = true} 28 | med_core = { version = "0.6.4", path = "../med_core"} 29 | 30 | 31 | -------------------------------------------------------------------------------- /med_cli/src/main.rs: -------------------------------------------------------------------------------- 1 | use colored::Colorize; 2 | use tokio::time::Instant; 3 | use tracing::info; 4 | mod cli; 5 | 6 | use med_core::app::core::App; 7 | use med_core::utils::error::MedError; 8 | 9 | use cli::app::Cli; 10 | 11 | #[tokio::main] 12 | #[cfg(not(tarpaulin_include))] 13 | async fn main() -> Result<(), MedError> { 14 | let now = Instant::now(); 15 | let new_cli = Cli::new().await?; 16 | let params = new_cli.params; 17 | 18 | let mut new_app = App::new(params).await?; 19 | let metrics = new_app.process().await?; 20 | let audit_id = new_app.update_audit(format!("{:?}", now.elapsed())).await?; 21 | 22 | info!( 23 | "total processed {} files, {} records, with {} records failed, elapsed time {:?}, audit record_id {}", 24 | metrics.total_files.to_string().bold().green(), 25 | metrics.metadata.total_records.to_string().bold().green(), 26 | metrics.metadata.failed_records.to_string().bold().green(), 27 | now.elapsed(), 28 | audit_id 29 | ); 30 | 31 | Ok(()) 32 | } 33 | -------------------------------------------------------------------------------- /med_core/src/utils/logger.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Once; 2 | 3 | use tracing_subscriber::fmt::format; 4 | 5 | /// params: debug bool 6 | pub async fn logging(debug: bool) { 7 | static START: Once = Once::new(); 8 | 9 | START.call_once(|| { 10 | let subscriber = tracing_subscriber::fmt() // disabling time is handy because CloudWatch will add the ingestion time. 11 | .event_format(format().compact()); 12 | 13 | match debug { 14 | true => { 15 | subscriber 16 | .with_line_number(true) 17 | .with_target(true) 18 | .with_file(true) 19 | .with_max_level(tracing::Level::DEBUG) 20 | .init(); 21 | } 22 | false => { 23 | subscriber 24 | .with_target(false) 25 | .with_max_level(tracing::Level::INFO) 26 | .init(); 27 | } 28 | } 29 | }); 30 | } 31 | 32 | #[cfg(test)] 33 | #[path = "./tests/logger_test.rs"] 34 | mod logger_test; 35 | -------------------------------------------------------------------------------- /.github/workflows/rpm.yml: -------------------------------------------------------------------------------- 1 | name: Rpm build and deploy 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | types: [closed] 8 | 9 | jobs: 10 | rpm_build: 11 | name: rpm build and deploy 12 | if: ${{ github.event.pull_request.merged }} 13 | runs-on: ubuntu-latest 14 | container: docker.io/fedora:latest 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@master 18 | - name: build 19 | shell: bash 20 | run: | 21 | echo ">>> write to fedora_conf" 22 | printf "\nlogin = $FEDORA_LOGIN\n" >> ./build/rpm/fedora_conf 23 | printf "username = $FEDORA_USERNAME\n" >> ./build/rpm/fedora_conf 24 | printf "token = $FEDORA_TOKEN\n" >> ./build/rpm/fedora_conf 25 | cat ./build/rpm/fedora_conf 26 | echo ">>> running build script" 27 | ./build/rpm/build.sh 28 | env: 29 | FEDORA_LOGIN : ${{ secrets.FEDORA_LOGIN }} 30 | FEDORA_TOKEN : ${{ secrets.FEDORA_TOKEN }} 31 | FEDORA_USERNAME: ${{ secrets.FEDORA_USERNAME }} -------------------------------------------------------------------------------- /med_core/src/app/worker.rs: -------------------------------------------------------------------------------- 1 | use crate::utils::error::MedError; 2 | use std::sync::Once; 3 | use threadpool::ThreadPool; 4 | 5 | #[derive(Debug)] 6 | pub struct Worker { 7 | pub cpu_num: u16, 8 | pub pool: ThreadPool, 9 | } 10 | 11 | impl Worker { 12 | /// Returns a Worker instant 13 | /// 14 | /// # Arguments 15 | /// 16 | /// * `cpu_num` [u16] - cpu number 17 | /// 18 | /// # Examples 19 | /// 20 | /// ``` 21 | /// use med_core::utils::error::MedError; 22 | /// use med_core::app::worker::Worker; 23 | /// 24 | /// #[tokio::main] 25 | /// async fn main() -> Result<(), MedError> { 26 | /// let worker = Worker::new(4).await?; 27 | /// Ok(()) 28 | /// } 29 | /// 30 | /// ``` 31 | pub async fn new(cpu_num: u16) -> Result { 32 | let pool = ThreadPool::new(cpu_num as usize); 33 | static START: Once = Once::new(); 34 | 35 | START.call_once(|| { 36 | rayon::ThreadPoolBuilder::new() 37 | .num_threads(cpu_num as usize) 38 | .build_global() 39 | .unwrap(); 40 | }); 41 | 42 | Ok(Worker { cpu_num, pool }) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: 2 | update_fedora: 3 | sudo dnf update && sudo dnf upgrade && sudo dnf autoremove 4 | 5 | .PHONY: 6 | update_debian: 7 | sudo apt update -y && sudo apt upgrade -y && sudo apt autoremove -y && sudo apt autoclean -y 8 | 9 | .PHONY: 10 | json_encrypt: 11 | cargo run --bin med encrypt -t json -f demo/data/input/json -c demo/conf/conf_json.yaml -w 6 -k 1q2w3er -s des64 12 | 13 | .PHONY: 14 | json_mask: 15 | cargo run --bin med mask -t json -f demo/data/input/json -c demo/conf/conf_json.yaml -w 6 16 | 17 | .PHONY: 18 | csv_mask: 19 | cargo run --bin med mask -f demo/data/input/csv -c demo/conf/conf_csv.yaml -w 6 20 | 21 | .PHONY: 22 | csv_encrypt: 23 | cargo run --bin med encrypt -f demo/data/input/csv -c demo/conf/conf_csv.yaml -w 6 -k 1q2w3e4r -s des64 24 | 25 | .PHONY: 26 | csv_mask_performance: 27 | cargo run --bin med mask -f /Users/huangwh/rust/rust-design-pattern/demo -c demo/conf/conf_csv.yaml -w 6 28 | 29 | .PHONY: 30 | test_package: 31 | cargo build --release 32 | cp -R demo target/release 33 | mkdir target/release/med-0.6.0 34 | mv target/release/med target/release/med-0.6.0 35 | tar --directory=target/release -cf macos_x86_archive-test.tar.gz med-0.6.0 demo 36 | 37 | .PHONY: 38 | pre_release: 39 | cargo fmt 40 | cargo clippy 41 | cargo tarpaulin -------------------------------------------------------------------------------- /med_core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "med_core" 3 | version = "0.6.4" 4 | edition = "2021" 5 | authors = ["Wei Huang "] 6 | license = "Apache-2.0" 7 | description = "A Rust Powered Core Engine for M.E.D. Masking, Encryption, and Decryption CSV/JSON" 8 | readme = "README.md" 9 | homepage = "https://github.com/jayhuang75/rust-cli-med" 10 | repository = "https://github.com/jayhuang75/rust-cli-med" 11 | keywords = ["cli", "encrypt", "decrypt", "masking"] 12 | categories = ["command-line-utilities"] 13 | 14 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 15 | 16 | [dependencies] 17 | tokio = { workspace = true } 18 | clap = { workspace = true } 19 | tracing = { workspace = true } 20 | tracing-subscriber = { workspace = true } 21 | colored = { workspace=true } 22 | num_cpus = { workspace = true } 23 | serde_json = { workspace = true} 24 | 25 | async-trait = "0.1.68" 26 | rayon = "1.7.0" 27 | sqlx = { version = "0.6.3", features = ["runtime-tokio-native-tls", "sqlite", "offline"] } 28 | serde = { version = "1.0.163", features = ["derive"] } 29 | serde_yaml = "0.9.21" 30 | 31 | csv = "1.2.2" 32 | flume = "0.10.14" 33 | walkdir = "2.3.3" 34 | threadpool = "1.8.1" 35 | magic-crypt = "3.1.12" 36 | indicatif = {version = "0.17.4", features = ["rayon"]} 37 | whoami = "1.4.0" 38 | 39 | dirs = "5.0.1" 40 | 41 | [build-dependencies] 42 | openssl = { workspace = true } -------------------------------------------------------------------------------- /med_core/README.md: -------------------------------------------------------------------------------- 1 | [![Crates.io](https://img.shields.io/crates/v/med_core)](https://crates.io/crates/med_core) [![Actions Status](https://github.com/jayhuang75/rust-cli-med/workflows/ci/badge.svg)](https://github.com/jayhuang75/rust-cli-med/actions) [![codecov](https://codecov.io/gh/jayhuang75/rust-cli-med/branch/main/graph/badge.svg?token=Z1LMSs2tQC)](https://codecov.io/gh/jayhuang75/rust-cli-med) [![Crates.io](https://img.shields.io/crates/d/med_core)](https://crates.io/crates/med_core) 2 | 3 | ### M.E.D. (Mask, Encrypt, Decrypt) - The Core Engine for Masking, Encryption, and Decryption the CSV/JSON files 4 | 5 | The core engine design for the plugin by different use case and context. 6 | 7 | Currently its the [CLI interface](../med_cli/README.md). If you have different programming or integration need, you can interact with the Core by is APIs. 8 | 9 | ### Example 10 | 11 | ```Rust 12 | 13 | let now = Instant::now(); 14 | 15 | let mut new_params = Params::default(); 16 | new_params.conf_path = "../demo/conf/conf_json.yaml".to_owned(); 17 | new_params.file_path = "../demo/data/input/csv".to_owned(); 18 | new_params.mode = Mode::MASK; 19 | new_params.file_type = FileType::CSV; 20 | 21 | let mut new_app = App::new(new_params.clone()).await.unwrap(); 22 | let metrics = new_app.process().await.unwrap(); 23 | let audit_id = new_app.update_audit(format!("{:?}", now.elapsed())).await.unwrap(); 24 | 25 | ``` 26 | 27 | ### Roadmap 28 | 29 | - [X] csv processor 30 | - [X] json processor 31 | - [ ] SDK for the med_core engine 32 | -------------------------------------------------------------------------------- /.github/workflows/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # M.E.D Pull Request 2 | 3 | ## Description 4 | 5 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. 6 | 7 | Fixes # (issue) 8 | 9 | ## Type of change 10 | 11 | Please delete options that are not relevant. 12 | 13 | - [ ] Bug fix (non-breaking change which fixes an issue) 14 | - [ ] New feature (non-breaking change which adds functionality) 15 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 16 | - [ ] This change requires a documentation update 17 | 18 | ## How Has This Been Tested? 19 | 20 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration 21 | 22 | - [ ] Unit Test 23 | - [ ] Integration Test 24 | - [ ] AB Test 25 | 26 | ## Checklist 27 | 28 | - [ ] My code follows the style guidelines of this project 29 | - [ ] I have performed a self-review of my own code 30 | - [ ] I have commented the code, particularly in hard-to-understand areas 31 | - [ ] I have made corresponding changes to the documentation 32 | - [ ] My changes generate no new warnings 33 | - [ ] I have added tests that prove my fix is effective or that my feature works 34 | - [ ] New and existing unit tests pass locally with my changes 35 | - [ ] Any dependent changes have been merged and published in downstream modules 36 | - [ ] I have checked my code and corrected any misspellings -------------------------------------------------------------------------------- /med_core/src/models/params.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use serde::Serialize; 4 | 5 | use crate::models::enums::{AppMode, FileType, Mode, Standard}; 6 | 7 | #[derive(Debug, Clone, PartialEq, Serialize)] 8 | pub struct Params { 9 | pub app_mode: AppMode, 10 | pub file_path: String, 11 | pub file_type: FileType, 12 | pub conf_path: String, 13 | pub output_path: String, 14 | pub mode: Mode, 15 | pub worker: u16, 16 | pub key: Option, 17 | pub standard: Standard, 18 | pub debug: bool, 19 | } 20 | 21 | impl fmt::Display for Params { 22 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 23 | write!( 24 | f, 25 | "app_mode: {}, file_path: {}, file_type: {}, conf_path: {}, output_path: {}, mode: {}, key: {:?}, debug: {}, worker: {}", 26 | self.app_mode, self.file_path, self.file_type, self.conf_path, self.output_path, self.mode, self.key, self.debug, self.worker 27 | ) 28 | } 29 | } 30 | 31 | impl Default for Params { 32 | fn default() -> Self { 33 | let app_mode: AppMode = AppMode::default(); 34 | let file_path: String = String::default(); 35 | let file_type: FileType = FileType::default(); 36 | let conf_path: String = String::default(); 37 | let output_path: String = String::default(); 38 | let mode: Mode = Mode::default(); 39 | let key: String = String::default(); 40 | let debug: bool = false; 41 | let worker = 2; 42 | let standard = Standard::default(); 43 | 44 | Params { 45 | app_mode, 46 | file_path, 47 | file_type, 48 | conf_path, 49 | output_path, 50 | mode, 51 | key: Some(key), 52 | standard, 53 | debug, 54 | worker, 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /med_core/src/utils/crypto.rs: -------------------------------------------------------------------------------- 1 | use magic_crypt::{ 2 | new_magic_crypt, MagicCrypt128, MagicCrypt192, MagicCrypt256, MagicCrypt64, MagicCryptTrait, 3 | }; 4 | 5 | use crate::models::enums::Standard; 6 | 7 | use super::error::MedError; 8 | 9 | #[derive(Debug, Clone)] 10 | pub struct Cypher { 11 | key64: MagicCrypt64, 12 | key128: MagicCrypt128, 13 | key192: MagicCrypt192, 14 | key256: MagicCrypt256, 15 | } 16 | 17 | impl Cypher { 18 | pub fn new(key: &str) -> Self { 19 | Cypher { 20 | key64: new_magic_crypt!(key, 64), 21 | key128: new_magic_crypt!(key, 128), 22 | key192: new_magic_crypt!(key, 192), 23 | key256: new_magic_crypt!(key, 256), 24 | } 25 | } 26 | 27 | pub fn encrypt(&self, data: &str, standard: &Standard) -> Result { 28 | let encrypted_str: String = match standard { 29 | Standard::DES64 => self.key64.encrypt_str_to_base64(data), 30 | Standard::AES128 => self.key128.encrypt_str_to_base64(data), 31 | Standard::AES192 => self.key192.encrypt_str_to_base64(data), 32 | Standard::AES256 => self.key256.encrypt_str_to_base64(data), 33 | }; 34 | Ok(encrypted_str) 35 | } 36 | 37 | #[allow(dead_code)] 38 | pub fn decrypt(&self, data: &str, standard: &Standard) -> Result { 39 | let decrypted_str: String = match standard { 40 | Standard::DES64 => self.key64.decrypt_base64_to_string(data)?, 41 | Standard::AES128 => self.key128.decrypt_base64_to_string(data)?, 42 | Standard::AES192 => self.key192.decrypt_base64_to_string(data)?, 43 | Standard::AES256 => self.key256.decrypt_base64_to_string(data)?, 44 | }; 45 | Ok(decrypted_str) 46 | } 47 | } 48 | 49 | #[cfg(test)] 50 | #[path = "./tests/crypto_test.rs"] 51 | mod crypto_test; 52 | -------------------------------------------------------------------------------- /med_core/src/utils/tests/crypto_test.rs: -------------------------------------------------------------------------------- 1 | use crate::models::enums::Standard; 2 | use crate::utils::crypto::Cypher; 3 | 4 | #[tokio::test] 5 | async fn test_crypto_data() { 6 | let crypto = Cypher::new("magickey"); 7 | 8 | // test AES256 9 | let data = crypto 10 | .encrypt("http://magiclen.org", &Standard::AES256) 11 | .unwrap(); 12 | assert_eq!("DS/2U8royDnJDiNY2ps3f6ZoTbpZo8ZtUGYLGEjwLDQ=", data); 13 | 14 | let data = crypto 15 | .decrypt( 16 | "DS/2U8royDnJDiNY2ps3f6ZoTbpZo8ZtUGYLGEjwLDQ=", 17 | &Standard::AES256, 18 | ) 19 | .unwrap(); 20 | assert_eq!("http://magiclen.org", data); 21 | 22 | // test DES64 23 | let data = crypto 24 | .encrypt("http://magiclen.org", &Standard::DES64) 25 | .unwrap(); 26 | assert_eq!("nqIQCAbQ0TKs6x6eGRdwrouES803NhvC", data); 27 | 28 | let data = crypto 29 | .decrypt("nqIQCAbQ0TKs6x6eGRdwrouES803NhvC", &Standard::DES64) 30 | .unwrap(); 31 | assert_eq!("http://magiclen.org", data); 32 | 33 | // test AES128 34 | let data = crypto 35 | .encrypt("http://magiclen.org", &Standard::AES128) 36 | .unwrap(); 37 | assert_eq!("Pdpj9HqTAN7vY7Z9msMzlIXJcNQ5N+cIJsiQhLqyjVI=", data); 38 | 39 | let data = crypto 40 | .decrypt( 41 | "Pdpj9HqTAN7vY7Z9msMzlIXJcNQ5N+cIJsiQhLqyjVI=", 42 | &Standard::AES128, 43 | ) 44 | .unwrap(); 45 | assert_eq!("http://magiclen.org", data); 46 | 47 | // test AES192 48 | let data = crypto 49 | .encrypt("http://magiclen.org", &Standard::AES192) 50 | .unwrap(); 51 | assert_eq!("p0X9IHMqaxA78T0X8Y9DnNeEmVXIgUxrXmeyUEO1Muo=", data); 52 | 53 | let data = crypto 54 | .decrypt( 55 | "p0X9IHMqaxA78T0X8Y9DnNeEmVXIgUxrXmeyUEO1Muo=", 56 | &Standard::AES192, 57 | ) 58 | .unwrap(); 59 | assert_eq!("http://magiclen.org", data); 60 | } 61 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | pull_request: 5 | branches: [development] 6 | 7 | env: 8 | CARGO_TERM_COLOR: always 9 | 10 | jobs: 11 | check: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - uses: actions/cache@v1 16 | with: 17 | path: ~/.cargo/registry 18 | key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} 19 | - uses: actions/cache@v1 20 | with: 21 | path: ~/.cargo/git 22 | key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }} 23 | - uses: actions/cache@v1 24 | with: 25 | path: target 26 | key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }} 27 | - uses: actions-rs/toolchain@v1 28 | with: 29 | profile: minimal 30 | toolchain: stable 31 | override: true 32 | - uses: actions-rs/cargo@v1 33 | with: 34 | command: check 35 | lints: 36 | runs-on: ubuntu-latest 37 | steps: 38 | - uses: actions/checkout@v2 39 | - uses: actions/cache@v1 40 | with: 41 | path: ~/.cargo/registry 42 | key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} 43 | - uses: actions/cache@v1 44 | with: 45 | path: ~/.cargo/git 46 | key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }} 47 | - uses: actions/cache@v1 48 | with: 49 | path: target 50 | key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }} 51 | - uses: actions-rs/toolchain@v1 52 | with: 53 | profile: minimal 54 | toolchain: stable 55 | override: true 56 | components: "rustfmt, clippy" 57 | - uses: actions-rs/cargo@v1 58 | with: 59 | command: fmt 60 | args: "--all -- --check" 61 | - uses: actions-rs/cargo@v1 62 | with: 63 | command: clippy 64 | args: "-- -D warnings" 65 | 66 | unit_test: 67 | runs-on: ubuntu-latest 68 | steps: 69 | - name: Checkout repository 70 | uses: actions/checkout@v2 71 | 72 | - name: Install stable toolchain 73 | uses: actions-rs/toolchain@v1 74 | with: 75 | toolchain: stable 76 | override: true 77 | 78 | - name: Run cargo-tarpaulin 79 | uses: actions-rs/tarpaulin@v0.1 80 | with: 81 | version: '0.15.0' 82 | args: '-- --test-threads 2' 83 | 84 | - name: Upload coverage reports to Codecov 85 | uses: codecov/codecov-action@v3 86 | with: 87 | token: ${{secrets.CODECOV_TOKEN}} -------------------------------------------------------------------------------- /.github/workflows/brew.yml: -------------------------------------------------------------------------------- 1 | name: Bump up brew 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | types: [closed] 8 | 9 | jobs: 10 | crate_metadata: 11 | name: Extract crate metadata 12 | if: ${{ github.event.pull_request.merged }} 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@master 16 | - name: Extract crate information 17 | id: crate_metadata 18 | run: | 19 | cargo metadata --no-deps --format-version 1 | jq -r '"version=" + .packages[0].version' | tee -a $GITHUB_OUTPUT 20 | outputs: 21 | version: ${{ steps.crate_metadata.outputs.version }} 22 | 23 | bump_up_brew: 24 | name: Bump up brew 25 | if: ${{ github.event.pull_request.merged }} 26 | needs: [crate_metadata] 27 | runs-on: macos-latest 28 | steps: 29 | - name: download mac x86_64 package with checksum 30 | shell: bash 31 | run: | 32 | echo ${{ github.workspace }} 33 | curl -LO https://github.com/jayhuang75/rust-cli-med/releases/download/${{ needs.crate_metadata.outputs.version }}/med-macos_x86_64-archive-${{ needs.crate_metadata.outputs.version }}.tar.gz > ${{ github.workspace }}/med-macos_x86_64-archive-${{ needs.crate_metadata.outputs.version }}.tar.gz 34 | pwd 35 | ls -la 36 | 37 | - name: Set SHA 38 | id: shasum 39 | run: | 40 | pwd 41 | echo "sha=$(shasum -a 256 med-macos_x86_64-archive-${{ needs.crate_metadata.outputs.version }}.tar.gz | awk '{printf $1}')" >> $GITHUB_OUTPUT 42 | 43 | - name: upload binaries x86 to release 44 | uses: softprops/action-gh-release@v1 45 | with: 46 | tag_name: ${{ needs.crate_metadata.outputs.version }} 47 | files: 'med-macos_x86_64-archive-${{ needs.crate_metadata.outputs.version }}.tar.gz' 48 | env: 49 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 50 | 51 | - name: Bump brew 52 | env: 53 | HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.BREW_TOKEN }} 54 | run: | 55 | echo ">>> brew tap" 56 | brew tap jayhuang75/med 57 | 58 | echo ">>> brew bump formula pr" 59 | echo ${{ steps.shasum.outputs.sha }} 60 | brew bump-formula-pr -v --version=${{ needs.crate_metadata.outputs.version }} --no-browse --no-audit --sha256=${{ steps.shasum.outputs.sha }} --url="https://github.com/jayhuang75/rust-cli-med/releases/download/${{ needs.crate_metadata.outputs.version }}/med-macos_x86_64-archive-${{ needs.crate_metadata.outputs.version }}.tar.gz" jayhuang75/med/med 61 | -------------------------------------------------------------------------------- /med_core/src/utils/tests/error_test.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use crate::utils::{ 4 | config::JobConfig, 5 | crypto::Cypher, 6 | error::{MedError, MedErrorType}, 7 | }; 8 | 9 | #[tokio::test] 10 | async fn test_message() { 11 | let msg = "Io Error message".to_string(); 12 | 13 | let err = MedError { 14 | message: Some(msg.to_string()), 15 | cause: None, 16 | error_type: MedErrorType::IoError, 17 | }; 18 | assert_eq!(err.message(), msg, "Io Error message"); 19 | } 20 | 21 | #[tokio::test] 22 | async fn test_serde_yaml_error() { 23 | let test_config_path = Path::new("../demo/conf/invalid_conf.yaml"); 24 | let test_config = JobConfig::new(test_config_path).await; 25 | match test_config { 26 | Ok(_) => { 27 | unimplemented!() 28 | } 29 | Err(err) => { 30 | assert_eq!(err.error_type, MedErrorType::ConfigError); 31 | } 32 | } 33 | } 34 | 35 | #[tokio::test] 36 | async fn test_magic_crypt_error() { 37 | // let sparkle_heart: Vec = vec![0, 159, 146, 150]; 38 | 39 | let expect_magic_crypt_err = MedError { 40 | message: Some("Invalid byte 240, offset 0.".to_string()), 41 | cause: Some("magic_crypt error".to_string()), 42 | error_type: MedErrorType::CryptoError, 43 | }; 44 | 45 | let key = "key".to_string(); 46 | let new_cypto = Cypher::new(&key); 47 | 48 | // let sparkle_heart = vec![0, 159, 146, 150]; 49 | let sparkle_heart = vec![240, 159, 146, 150]; 50 | 51 | // We know these bytes are valid, so we'll use `unwrap()`. 52 | let sparkle_heart_str = String::from_utf8(sparkle_heart).unwrap(); 53 | 54 | // testing the decryption failed. 55 | match new_cypto.decrypt(&sparkle_heart_str, &crate::models::enums::Standard::DES64) { 56 | Ok(_) => { 57 | unimplemented!() 58 | } 59 | Err(err) => { 60 | assert_eq!(err, expect_magic_crypt_err); 61 | } 62 | } 63 | } 64 | 65 | // #[tokio::test] 66 | // async fn test_io_error() { 67 | // let data = r#" 68 | // { 69 | // "name": "John Doe", 70 | // "age": 43, 71 | // "phones": [ 72 | // "+44 1234567", 73 | // "+44 2345678" 74 | // ] 75 | // }"#; 76 | 77 | // // Parse the string of data into serde_json::Value. 78 | // let v: Value = serde_json::from_str(data).unwrap(); 79 | // match write_json(&v, "") { 80 | // Ok(_) => { 81 | // unimplemented!() 82 | // } 83 | // Err(e) => { 84 | // assert_eq!(e.error_type, MedErrorType::IoError); 85 | // } 86 | // } 87 | // } 88 | 89 | // #[tokio::test] 90 | // async fn test_rayon_thread_pool_error() { 91 | // match Worker::new(0).await { 92 | // Ok(_) => unimplemented!(), 93 | // Err(e) => { 94 | // assert_eq!(e.error_type, MedErrorType::WorkerError); 95 | // } 96 | // } 97 | // } 98 | -------------------------------------------------------------------------------- /med_core/src/tests/csv_test.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | app::{ 3 | csv::{csv_fields_exist, csv_processor}, 4 | processor::ProcessRuntime, 5 | }, 6 | models::enums::{Mode, Standard}, 7 | utils::{crypto::Cypher, error::MedErrorType}, 8 | }; 9 | use csv::StringRecord; 10 | 11 | #[test] 12 | fn test_csv_fields_exist() { 13 | let fields = vec!["name".to_string()]; 14 | let mut headers = StringRecord::new(); 15 | headers.push_field("job_type"); 16 | headers.push_field("name"); 17 | let index = csv_fields_exist(headers, &fields); 18 | assert_eq!(index[0], 1); 19 | } 20 | 21 | #[test] 22 | fn test_csv_processor_error() { 23 | // tx_metadata: flume::Sender, 24 | // files_path: &str, 25 | // output_path: &str, 26 | // process_runtime: ProcessRuntime, 27 | let (tx_metadata, _) = flume::unbounded(); 28 | let files_path = ""; 29 | let output_path = ""; 30 | let process_runtime = ProcessRuntime { 31 | fields: vec!["name".to_string()], 32 | cypher: None, 33 | standard: None, 34 | mask_symbols: Some("#####".to_string()), 35 | mode: Mode::MASK, 36 | }; 37 | 38 | match csv_processor( 39 | tx_metadata.clone(), 40 | files_path, 41 | output_path, 42 | process_runtime, 43 | ) { 44 | Ok(_) => {} 45 | Err(err) => { 46 | assert_eq!(err.error_type, MedErrorType::CsvError); 47 | } 48 | } 49 | 50 | // drop the channel once it done. 51 | drop(tx_metadata); 52 | } 53 | 54 | #[tokio::test] 55 | async fn test_csv_processor_format_err() { 56 | let (tx_metadata, rx_metadata) = flume::unbounded(); 57 | let process_runtime = ProcessRuntime { 58 | fields: vec!["name".to_string()], 59 | cypher: None, 60 | standard: None, 61 | mask_symbols: Some("#####".to_string()), 62 | mode: Mode::MASK, 63 | }; 64 | let files_path: &str = "../demo/data/input/format_err/csv/format_err.csv"; 65 | let output_path = "../demo/data/output/csv/format_err/processor_err/random_data.csv"; 66 | 67 | csv_processor( 68 | tx_metadata.clone(), 69 | files_path, 70 | output_path, 71 | process_runtime, 72 | ) 73 | .unwrap(); 74 | 75 | // drop the channel once it done. 76 | drop(tx_metadata); 77 | 78 | rx_metadata.iter().for_each(|item| { 79 | assert_eq!(item.failed_records, 1); 80 | }); 81 | } 82 | 83 | #[tokio::test] 84 | async fn test_csv_processor_decrypt_error() { 85 | let (tx_metadata, rx_metadata) = flume::unbounded(); 86 | let key = "123"; 87 | let process_runtime = ProcessRuntime { 88 | fields: vec!["name".to_string()], 89 | cypher: Some(Cypher::new(key)), 90 | standard: Some(Standard::DES64), 91 | mask_symbols: None, 92 | mode: Mode::DECRYPT, 93 | }; 94 | let files_path: &str = "../demo/data/input/format_err/csv/encrypt_err.csv"; 95 | let output_path = "../demo/data/output/csv/format_err/decrypt_err.csv"; 96 | 97 | csv_processor( 98 | tx_metadata.clone(), 99 | files_path, 100 | output_path, 101 | process_runtime, 102 | ) 103 | .unwrap(); 104 | 105 | // drop the channel once it done. 106 | drop(tx_metadata); 107 | 108 | rx_metadata.iter().for_each(|item| { 109 | assert_eq!(item.failed_records, 1); 110 | }); 111 | } 112 | -------------------------------------------------------------------------------- /med_cli/src/cli/custom_validation.rs: -------------------------------------------------------------------------------- 1 | use std::{ops::RangeInclusive, path::PathBuf}; 2 | 3 | /// Return the worker range of current runtime machine 4 | /// 5 | /// # Examples 6 | /// ``` 7 | /// match worker_in_range(&num_cpus::get().to_string()) { 8 | /// Ok(num) => { 9 | /// assert_eq!(num, num_cpus::get() as u16); 10 | /// } 11 | /// Err(e) => { 12 | /// assert_eq!(e,format!("worker is over your current max cores, consider lower the workerworker not in range 2-{:?} (max)", num_cpus::get())); 13 | /// } 14 | /// } 15 | /// ``` 16 | pub fn worker_in_range(s: &str) -> Result { 17 | let worker: usize = s 18 | .parse() 19 | .map_err(|_| format!("`{}` isn't a worker number", s))?; 20 | 21 | let worker_range: RangeInclusive = 2..=num_cpus::get(); 22 | 23 | if worker_range.contains(&worker) { 24 | Ok(worker as u16) 25 | } else { 26 | Err(format!( 27 | "worker is over your current max cores, consider lower the workerworker not in range {}-{} (max)", 28 | worker_range.start(), 29 | worker_range.end() 30 | )) 31 | } 32 | } 33 | 34 | /// Check and Return if the path exist 35 | /// 36 | /// # Examples 37 | /// ``` 38 | /// match dir_exist("./") { 39 | /// Ok(path) => { 40 | /// assert_eq!(path, PathBuf::from("./")); 41 | /// } 42 | /// Err(e) => { 43 | /// assert_eq!(e, ""); 44 | /// } 45 | /// } 46 | /// ``` 47 | pub fn dir_exist(s: &str) -> Result { 48 | let path = PathBuf::from(s); 49 | match path.is_dir() { 50 | true => Ok(path), 51 | false => Err(format!("{} not exist", s)), 52 | } 53 | } 54 | 55 | //************************************************************************************************ 56 | // Unit Test 57 | //////////////////////////////// */ 58 | #[cfg(test)] 59 | mod tests { 60 | use std::path::PathBuf; 61 | 62 | use super::{dir_exist, worker_in_range}; 63 | 64 | #[test] 65 | fn test_worker_in_range() { 66 | match worker_in_range(&num_cpus::get().to_string()) { 67 | Ok(num) => { 68 | assert_eq!(num, num_cpus::get() as u16); 69 | } 70 | Err(e) => { 71 | assert_eq!(e,format!("worker is over your current max cores, consider lower the workerworker not in range 2-{:?} (max)", num_cpus::get())); 72 | } 73 | } 74 | } 75 | 76 | #[test] 77 | fn test_worker_in_range_failed() { 78 | match worker_in_range("100") { 79 | Ok(_) => { 80 | unimplemented!() 81 | } 82 | Err(e) => { 83 | assert_eq!(e,format!("worker is over your current max cores, consider lower the workerworker not in range 2-{:?} (max)", num_cpus::get())); 84 | } 85 | } 86 | } 87 | 88 | #[test] 89 | fn test_dir_exist() { 90 | match dir_exist("./") { 91 | Ok(path) => { 92 | assert_eq!(path, PathBuf::from("./")); 93 | } 94 | Err(e) => { 95 | assert_eq!(e, ""); 96 | } 97 | } 98 | match dir_exist("./test") { 99 | Ok(path) => { 100 | assert_eq!(path, PathBuf::from("./")); 101 | } 102 | Err(e) => { 103 | assert_eq!(e, "./test not exist"); 104 | } 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Actions Status](https://github.com/jayhuang75/rust-cli-med/workflows/ci/badge.svg)](https://github.com/jayhuang75/rust-cli-med/actions) [![codecov](https://codecov.io/gh/jayhuang75/rust-cli-med/branch/main/graph/badge.svg?token=Z1LMSs2tQC)](https://codecov.io/gh/jayhuang75/rust-cli-med) 2 | 3 | # M.E.D. (Mask, Encrypt, Decrypt) - a RUST-powered CLI tool for CSV/JSON files 4 | 5 | ![picture](documents/logo/data-encryption.png) 6 | 7 | ## Background & Motivation 8 | 9 | This is a personal hobby project; based on the observation, sometimes we need a simple enough CLI tool with auditable capability for Data Masking/Encryption/Decryption for CSV/JSON files. 10 | 11 | ## Key Features 12 | 13 | 1. Rust powered performance. 14 | 2. Provide Masking and Encyption/Decryption capabilities. 15 | 3. Auditable with a built-in SQLite-powered Audit table. 16 | 17 | ## Extendability 18 | 19 | There are two central crates in this package. 20 | 21 | 1. [MED_CLI](med_cli/README.md) - the CLI interface for the med binary.[![Crates.io](https://img.shields.io/crates/v/med_cli)](https://crates.io/crates/med_cli) [![Crates.io](https://img.shields.io/crates/d/med_cli)](https://crates.io/crates/med_cli) 22 | 2. [MED_CORE](med_core/README.md) - the core engineer to execution the CSV/JSON files Masking, Encryption, and Decryption, which you can use in your use-case/project/context implementation. [![Crates.io](https://img.shields.io/crates/v/med_core)](https://crates.io/crates/med_core) [![Crates.io](https://img.shields.io/crates/d/med_core)](https://crates.io/crates/med_core) 23 | 24 | ## Publication 25 | 26 | - [Introduction M.E.D.](https://medium.com/dev-genius/introduction-m-e-d-e001cd83a39f) 27 | - [Build a CLI Tool for Data Masking, Encryption, and Decryption With Rust](https://medium.com/better-programming/build-a-cli-tool-for-data-masking-encryption-and-decryption-with-rust-ad36bea27559) 28 | - [Reduce memory footprint by about 600% for M.E.D. — Performance Matters](https://medium.com/gitconnected/reduce-memory-footprint-by-about-600-for-m-e-d-performance-matters-bec407833e7c) 29 | 30 | ## Installation 31 | 32 | ### Download from GitHub release (from sources) 33 | 34 | The binary name for M.E.D. is med; it depends on the [med_core](../med_core/README.md). 35 | 36 | Archives of precompiled binaries for med are available for [Windows, macOS, and Linux](https://github.com/jayhuang75/rust-cli-med/releases). Users of platforms not explicitly mentioned below are advised to download one of these archives. 37 | 38 | ### Fedora and Centos 39 | 40 | ```bash 41 | dnf install med 42 | ``` 43 | 44 | ### MacOS X86_64 45 | 46 | ```bash 47 | brew tap jayhuang75/med 48 | brew install med 49 | ``` 50 | 51 | ## Benchmark 52 | 53 | ```bash 54 | Model Name: MacBook Pro 55 | Processor Name: 6-Core Intel Core i7 56 | Processor Speed: 2.6 GHz 57 | Total Number of Cores: 6 58 | Memory: 16 GB 59 | ``` 60 | 61 | | File Type | Records | File Size | File Counts | Mode | Field num for mask/encrypt| Elapsed Time | Memory Consumption| 62 | | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |------------- | 63 | | CSV | 120,000,000 | 2.8G | 3 | mask | 1 | ~78 seconds (1.3 mins)| ~2 MB | 64 | | CSV | 120,000,000 | 2.8G | 3 | encrypt (DES64) | 1 | ~182 seconds (3 mins)| ~1.9 MB | 65 | | CSV | 120,000,000 | 2.8G | 3 | encrypt (AES128) | 1 | ~221 seconds (3.6 mins)| ~1.9 MB | 66 | | CSV | 20,000,000 | 471M | 2 | mask | 1 | ~7 seconds| ~1.8 MB | 67 | | CSV | 10,000,000 | 236M | 1 | mask | 1 | ~5 seconds| ~1.8 MB | 68 | | JSON | 129,220 | 10G | 129,220 | mask | 1 | ~2200 seconds(36 mins) | ~62 MB | 69 | | JSON | 64,641 | 5.1G | 64,641 | mask | 1 | ~792 seconds(13 mins) | ~30 MB | 70 | | JSON | 23,519 | 1.9G | 23,519 | mask | 1 | ~284 seconds(4.7 mins) | ~18 MB | 71 | | JSON | 23,519 | 1.9G | 23,519 | encrypt | 1 | ~282 seconds(4.4 mins) | ~18 MB | 72 | 73 | ## Contributions 74 | 75 | This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. 76 | 77 | Please complete the [template](.github/workflows/PULL_REQUEST_TEMPLATE.md) before the PR. 78 | 79 | Contributions of any kind are welcome! 80 | 81 | ## Show your support 82 | 83 | Give a ⭐️ if this project helped you! 84 | 85 | ## License 86 | 87 | Copyright © 2023 [Wei Huang](https://github.com/jayhuang75/) 88 | 89 | This project is Licensed under Apache. 90 | -------------------------------------------------------------------------------- /med_core/src/utils/error.rs: -------------------------------------------------------------------------------- 1 | use serde::Serialize; 2 | use serde_json::Error; 3 | use std::fmt; 4 | use tokio::io; 5 | 6 | #[derive(Debug, PartialEq, Eq, Clone, Serialize)] 7 | pub enum MedErrorType { 8 | ConfigError, 9 | IoError, 10 | CryptoError, 11 | WorkerError, 12 | SerdeJsonError, 13 | DatabaseError, 14 | CsvError, 15 | } 16 | 17 | #[derive(Debug, PartialEq, Eq, Clone, Serialize)] 18 | pub struct MedError { 19 | pub message: Option, 20 | pub cause: Option, 21 | pub error_type: MedErrorType, 22 | } 23 | 24 | impl MedError { 25 | #[allow(dead_code)] 26 | pub fn message(&self) -> String { 27 | match self { 28 | MedError { 29 | message: Some(message), 30 | .. 31 | } => message.clone(), 32 | _ => "An unexpected error has occurred".to_string(), 33 | } 34 | } 35 | } 36 | 37 | impl From for MedError { 38 | fn from(error: serde_yaml::Error) -> MedError { 39 | MedError { 40 | message: Some(error.to_string()), 41 | cause: Some("can not open the conf.yml".to_string()), 42 | error_type: MedErrorType::ConfigError, 43 | } 44 | } 45 | } 46 | 47 | // impl From for MedError { 48 | // fn from(error: VarError) -> MedError { 49 | // MedError { 50 | // message: Some(error.to_string()), 51 | // cause: Some("can not read env variable".to_string()), 52 | // error_type: MedErrorType::ConfigError, 53 | // } 54 | // } 55 | // } 56 | 57 | impl From for MedError { 58 | fn from(error: io::Error) -> MedError { 59 | MedError { 60 | message: Some(error.to_string()), 61 | cause: Some(error.to_string()), 62 | error_type: MedErrorType::IoError, 63 | } 64 | } 65 | } 66 | 67 | impl From for MedError { 68 | fn from(error: csv::Error) -> MedError { 69 | MedError { 70 | message: Some(error.to_string()), 71 | cause: Some(error.to_string()), 72 | error_type: MedErrorType::CsvError, 73 | } 74 | } 75 | } 76 | 77 | impl From for MedError { 78 | fn from(error: magic_crypt::MagicCryptError) -> MedError { 79 | MedError { 80 | message: Some(error.to_string()), 81 | cause: Some("magic_crypt error".to_string()), 82 | error_type: MedErrorType::CryptoError, 83 | } 84 | } 85 | } 86 | 87 | // impl From for MedError { 88 | // fn from(error: rayon::ThreadPoolBuildError) -> MedError { 89 | // MedError { 90 | // message: Some(error.to_string()), 91 | // cause: Some("rayon worker error".to_string()), 92 | // error_type: MedErrorType::WorkerError, 93 | // } 94 | // } 95 | // } 96 | #[cfg(not(tarpaulin_include))] 97 | impl From for MedError { 98 | fn from(error: Error) -> MedError { 99 | MedError { 100 | message: Some(error.to_string()), 101 | cause: Some("serde json error".to_string()), 102 | error_type: MedErrorType::SerdeJsonError, 103 | } 104 | } 105 | } 106 | 107 | #[cfg(not(tarpaulin_include))] 108 | impl From for MedError { 109 | fn from(error: sqlx::Error) -> MedError { 110 | MedError { 111 | message: Some(error.to_string()), 112 | cause: Some("database error".to_string()), 113 | error_type: MedErrorType::DatabaseError, 114 | } 115 | } 116 | } 117 | 118 | #[cfg(not(tarpaulin_include))] 119 | impl From for MedError { 120 | fn from(error: sqlx::migrate::MigrateError) -> MedError { 121 | MedError { 122 | message: Some(error.to_string()), 123 | cause: Some("database migration error".to_string()), 124 | error_type: MedErrorType::DatabaseError, 125 | } 126 | } 127 | } 128 | 129 | impl fmt::Display for MedError { 130 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { 131 | write!(f, "{:?}", self) 132 | } 133 | } 134 | 135 | #[cfg(test)] 136 | #[path = "./tests/error_test.rs"] 137 | mod error_test; 138 | -------------------------------------------------------------------------------- /med_core/src/tests/models_test.rs: -------------------------------------------------------------------------------- 1 | use clap::{builder::PossibleValue, ValueEnum}; 2 | 3 | use crate::models::{ 4 | enums::{AppMode, FileType, Mode, Standard}, 5 | params::Params, 6 | }; 7 | 8 | #[tokio::test] 9 | async fn test_mode_enum() { 10 | let mode = Mode::MASK; 11 | assert_eq!(mode.to_string(), "mask"); 12 | assert_eq!(format!("{mode:?}"), "MASK"); 13 | 14 | let mode = Mode::ENCRYPT; 15 | assert_eq!(mode.to_string(), "encrypt"); 16 | assert_eq!(format!("{mode:?}"), "ENCRYPT"); 17 | 18 | let mode = Mode::DECRYPT; 19 | assert_eq!(mode.to_string(), "decrypt"); 20 | assert_eq!(format!("{mode:?}"), "DECRYPT"); 21 | 22 | assert_eq!( 23 | Mode::value_variants(), 24 | &[Mode::MASK, Mode::ENCRYPT, Mode::DECRYPT] 25 | ); 26 | 27 | assert_eq!( 28 | Mode::to_possible_value(&Mode::MASK), 29 | Some(PossibleValue::new("mask").help("Masking the data")) 30 | ); 31 | assert_eq!( 32 | Mode::to_possible_value(&Mode::ENCRYPT), 33 | Some(PossibleValue::new("encrypt").help("Encrypt the data with provided KEY")) 34 | ); 35 | assert_eq!( 36 | Mode::to_possible_value(&Mode::DECRYPT), 37 | Some(PossibleValue::new("decrypt").help("Decrypt the data with provided KEY")) 38 | ); 39 | 40 | match Mode::from_str("test", true) { 41 | Ok(m) => { 42 | println!("m {:?}", m); 43 | } 44 | Err(err) => { 45 | assert_eq!(err, "invalid variant: test".to_owned()) 46 | } 47 | } 48 | 49 | match Mode::from_str("mask", true) { 50 | Ok(m) => { 51 | assert_eq!(m, Mode::MASK); 52 | } 53 | Err(err) => { 54 | assert_eq!(err, "invalid variant: test".to_owned()) 55 | } 56 | } 57 | } 58 | 59 | #[tokio::test] 60 | async fn test_app_mode_enum() { 61 | let app_mode = AppMode::CLI; 62 | assert_eq!(app_mode.to_string(), "CLI"); 63 | assert_eq!(format!("{app_mode:?}"), "CLI"); 64 | 65 | let app_mode = AppMode::SDK; 66 | assert_eq!(app_mode.to_string(), "SDK"); 67 | assert_eq!(format!("{app_mode:?}"), "SDK"); 68 | } 69 | 70 | #[tokio::test] 71 | async fn test_standard_enum() { 72 | let new_standard = Standard::DES64; 73 | assert_eq!(new_standard.to_string(), "des64"); 74 | assert_eq!(format!("{new_standard:?}"), "DES64"); 75 | 76 | let new_standard = Standard::AES128; 77 | assert_eq!(new_standard.to_string(), "aes128"); 78 | assert_eq!(format!("{new_standard:?}"), "AES128"); 79 | 80 | let new_standard = Standard::AES192; 81 | assert_eq!(new_standard.to_string(), "aes192"); 82 | assert_eq!(format!("{new_standard:?}"), "AES192"); 83 | 84 | let new_standard = Standard::AES256; 85 | assert_eq!(new_standard.to_string(), "aes256"); 86 | assert_eq!(format!("{new_standard:?}"), "AES256"); 87 | 88 | assert_eq!( 89 | Standard::value_variants(), 90 | &[ 91 | Standard::DES64, 92 | Standard::AES128, 93 | Standard::AES192, 94 | Standard::AES256 95 | ] 96 | ); 97 | assert_eq!( 98 | Standard::to_possible_value(&Standard::DES64), 99 | Some(PossibleValue::new("des64").help("DES standard 64")) 100 | ); 101 | assert_eq!( 102 | Standard::to_possible_value(&Standard::AES128), 103 | Some(PossibleValue::new("aes128").help("AES standard 128")) 104 | ); 105 | assert_eq!( 106 | Standard::to_possible_value(&Standard::AES192), 107 | Some(PossibleValue::new("aes192").help("AES standard 192")) 108 | ); 109 | assert_eq!( 110 | Standard::to_possible_value(&Standard::AES256), 111 | Some(PossibleValue::new("aes256").help("AES standard 256")) 112 | ); 113 | 114 | match Standard::from_str("des64", true) { 115 | Ok(s) => { 116 | assert_eq!(s, Standard::DES64); 117 | } 118 | Err(_) => { 119 | unimplemented!() 120 | } 121 | } 122 | 123 | match Standard::from_str("test", true) { 124 | Ok(_) => { 125 | unimplemented!() 126 | } 127 | Err(e) => { 128 | assert_eq!(e, "invalid variant: test"); 129 | } 130 | } 131 | } 132 | 133 | #[tokio::test] 134 | async fn test_params_init() { 135 | let new_params = Params::default(); 136 | assert_eq!(new_params.app_mode, AppMode::CLI); 137 | assert!(!new_params.debug); 138 | assert_eq!(new_params.file_type, FileType::CSV); 139 | assert_eq!(new_params.file_path, ""); 140 | assert_eq!(new_params.conf_path, ""); 141 | assert_eq!(new_params.mode, Mode::MASK); 142 | assert_eq!(new_params.standard, Standard::DES64); 143 | assert_eq!(new_params.key, Some("".to_owned())); 144 | assert_eq!(new_params.to_string(), "app_mode: CLI, file_path: , file_type: csv, conf_path: , output_path: , mode: mask, key: Some(\"\"), debug: false, worker: 2"); 145 | } 146 | -------------------------------------------------------------------------------- /med_cli/README.md: -------------------------------------------------------------------------------- 1 | [![Crates.io](https://img.shields.io/crates/v/med_cli)](https://crates.io/crates/med_cli) [![Actions Status](https://github.com/jayhuang75/rust-cli-med/workflows/ci/badge.svg)](https://github.com/jayhuang75/rust-cli-med/actions) [![codecov](https://codecov.io/gh/jayhuang75/rust-cli-med/branch/main/graph/badge.svg?token=Z1LMSs2tQC)](https://codecov.io/gh/jayhuang75/rust-cli-med) [![Crates.io](https://img.shields.io/crates/d/med_cli)](https://crates.io/crates/med_cli) 2 | 3 | # M.E.D. (Mask, Encrypt, Decrypt) - a RUST powered CLI tool for CSV/JSON files. 4 | 5 | ## Background & Motivation 6 | 7 | This is a personal hobby project; based on the observation, sometimes we need a simple enough CLI tool with auditable capability for Data Masking/Encyption/Decryption for CSV/JSON files. 8 | 9 | ## Key Features 10 | 11 | 1. Rust powered performance. 12 | 2. Provide Masking, and Encyption/Decryption capabilities. 13 | 3. Auditable with build-in SQLite powered Audit table. 14 | 15 | ### Installation 16 | 17 | #### Download from github release 18 | 19 | The binary name for M.E.D. is med, it depends on the [med_core](../med_core/README.md). 20 | 21 | Archives of precompiled binaries for med are available for [Windows, macOS and Linux](https://github.com/jayhuang75/rust-cli-med/releases). Users of platforms not explicitly mentioned below are advised to download one of these archives. 22 | 23 | #### Fedora and Centos 24 | 25 | ```bash 26 | dnf install med 27 | ``` 28 | 29 | ## Usage 30 | 31 | ```bash 32 | $ med --help 33 | A simple to use, enterprise ready, rust powered data masking/encryption/decription cli tool 34 | 35 | Usage: med --file [OPTIONS] 36 | 37 | Arguments: 38 | 39 | What mode to run the program in 40 | Possible values: 41 | - mask: Mask the data by * 42 | - encrypt: Encrypt the data with provided KEY 43 | - decrypt: Decrypt the data with provided KEY 44 | 45 | Options: 46 | -t, --type type of file we will process, available option [csv, json] [default: csv] 47 | -k, --key key for Encrypt and Decrypt the file. 48 | -s, --standard set the Encrypt and Decrypt standard 49 | Possible values: 50 | - des64: DES standard 64 51 | - aes128: AES standard 128 52 | - aes192: AES standard 192 53 | - aes256: AES standard 256 54 | -f, --file file path for the 55 | -c, --config Sets a custom config yml path [default: conf.yaml] 56 | -o, --output Sets a file/directory path for output [default: output] 57 | -d, --debug Sets debug flag [possible values: true, false] 58 | -w, --worker Sets work flag 59 | -h, --help Print help (see a summary with '-h') 60 | -V, --version Print version 61 | ``` 62 | 63 | ### User Guide 64 | 65 | #### Configuration 66 | 67 | The configuration file can be any given name of [yaml file](demo/conf/conf_json.yaml). 68 | 69 | ```bash 70 | // example of the conf.yaml 71 | mask_symbols: "#####" # mask symbols 72 | fields: # list of the cols/fields you want to mask 73 | - name 74 | - email 75 | - phone 76 | ``` 77 | 78 | #### Example of how to 79 | 80 | 1. All the demo data are available in the package when you download it. And it's all **RANDOMLY** generated. [csv](demo/data/csv/random_data.csv) [json](demo/data/json/generated.json) 81 | 2. You only need to point to the root dir for your files. M.E.D. will take care of the rest. 82 | 83 | ```bash 84 | // mask the csv files in folders 85 | med mask -f demo/data/csv -c demo/conf/conf_csv.yaml -w 3 86 | 87 | // mask the json files in folders 88 | med mask -t json -f demo/data/json -c demo/conf/conf_json.yaml -w 3 89 | 90 | // encrypt the csv files 91 | med encrypt -f demo/data/csv -c demo/conf/conf_csv.yaml -w 4 -k YOUR_SECRET -s des64 92 | 93 | // decrypt the json files 94 | med decrypt -t json -f output/demo/data/json -c demo/conf/conf_json.yaml -w 5 -k YOUR_SECRET -s des64 95 | 96 | ``` 97 | 98 | #### Audit database (Sqlite) 99 | 100 | M.E.D. uses SQLite for the audit capture, mainly ensuring following the Entreprise level Audit base standard, capture, Who, When, Where(which machine), do what, and status, etc. 101 | 102 | The metadata and migration are available [here](audit/migrations/20230512195802_audit_sqlite_datastore.up.sql). 103 | 104 | The audit db location will be different depending on your OS. 105 | 106 | ##### location 107 | 108 | | Platform | Value | Example | 109 | | ------------- | ------------- | ------------- | 110 | | Linux | $HOME/.config/med | /home/bob/.config/med | 111 | | MacOS | $HOME/Library/Application Support/med | /Users/Bob/Library/Application Support/med | 112 | | Windows | {FOLDERID_RoamingAppData}/med | C:\Users\Bob\AppData\Roaming\med | 113 | 114 | #### database migration 115 | 116 | We prepare the database migration capabilites and this migrations folder NEED to be in the same directoy of your binary. 117 | -------------------------------------------------------------------------------- /med_core/src/tests/core_test.rs: -------------------------------------------------------------------------------- 1 | use crate::app::core::App; 2 | use crate::models::enums::{FileType, Mode}; 3 | use crate::models::params::Params; 4 | use crate::utils::error::MedErrorType::ConfigError; 5 | use crate::utils::error::{MedError, MedErrorType}; 6 | 7 | #[tokio::test] 8 | async fn test_csv_mask_app() { 9 | let new_params = Params { 10 | conf_path: "../demo/conf/conf_csv.yaml".to_owned(), 11 | debug: true, 12 | ..Default::default() 13 | }; 14 | 15 | let mut new_app = App::new(new_params.clone()).await.unwrap(); 16 | assert_eq!(new_app.hostname, whoami::hostname()); 17 | assert_eq!(new_app.params, new_params); 18 | assert_eq!(new_app.user, whoami::username()); 19 | 20 | let metrics = new_app.process().await.unwrap(); 21 | assert_eq!(metrics.total_files, 0); 22 | } 23 | 24 | #[tokio::test] 25 | async fn test_load_job_config() { 26 | let new_params = Params { 27 | conf_path: "../demo/conf/conf_csv.yaml".to_owned(), 28 | debug: false, 29 | ..Default::default() 30 | }; 31 | 32 | let new_app = App::new(new_params).await.unwrap(); 33 | let conf = new_app.load_job_config().await.unwrap(); 34 | assert_eq!(conf.mask_symbols, "#####".to_string()); 35 | } 36 | 37 | #[tokio::test] 38 | async fn test_load_job_config_failed() { 39 | let new_params = Params { 40 | conf_path: "".to_owned(), 41 | debug: false, 42 | ..Default::default() 43 | }; 44 | 45 | let new_app = App::new(new_params).await.unwrap(); 46 | match new_app.load_job_config().await { 47 | Ok(_) => {} 48 | Err(e) => { 49 | assert_eq!( 50 | e, 51 | MedError { 52 | message: Some("No such file or directory (os error 2)".to_owned()), 53 | cause: Some("load job configuration yaml file failed!".to_owned()), 54 | error_type: ConfigError 55 | } 56 | ) 57 | } 58 | } 59 | } 60 | 61 | #[tokio::test] 62 | async fn test_file_processor_failed() { 63 | let new_params = Params { 64 | conf_path: "../demo/conf/conf_csv.yaml".to_owned(), 65 | file_path: "../demo/data/input/format_err/csv".to_owned(), 66 | output_path: "../demo/data/output/csv/format_err/processor_err".to_owned(), 67 | file_type: FileType::CSV, 68 | mode: Mode::MASK, 69 | ..Default::default() 70 | }; 71 | 72 | match App::new(new_params).await { 73 | Ok(mut new_app) => { 74 | let metrics = new_app.process().await.unwrap(); 75 | assert!(!metrics.metadata.record_failed_reason.is_empty()); 76 | } 77 | Err(err) => { 78 | assert_eq!(err.error_type, MedErrorType::DatabaseError); 79 | } 80 | } 81 | } 82 | 83 | #[tokio::test] 84 | async fn test_processor_run_encrypt() { 85 | let new_params = Params { 86 | conf_path: "../demo/conf/conf_csv.yaml".to_owned(), 87 | file_path: "../demo/data/input/csv".to_owned(), 88 | output_path: "../demo/data/output/csv/mask".to_owned(), 89 | file_type: FileType::CSV, 90 | mode: Mode::ENCRYPT, 91 | key: Some("123".to_owned()), 92 | ..Default::default() 93 | }; 94 | 95 | let mut new_app = App::new(new_params).await.unwrap(); 96 | let metrics = new_app.process().await.unwrap(); 97 | assert_eq!(metrics.metadata.failed_records, 0); 98 | 99 | let new_params = Params { 100 | conf_path: "../demo/conf/conf_csv.yaml".to_owned(), 101 | output_path: "../demo/data/input/csv".to_owned(), 102 | file_path: "../demo/data/output/csv/demo/data/input/csv".to_owned(), 103 | file_type: FileType::CSV, 104 | mode: Mode::DECRYPT, 105 | key: Some("123".to_owned()), 106 | ..Default::default() 107 | }; 108 | 109 | let mut new_app = App::new(new_params).await.unwrap(); 110 | let metrics = new_app.process().await.unwrap(); 111 | assert_eq!(metrics.metadata.failed_records, 0); 112 | } 113 | 114 | #[tokio::test] 115 | async fn test_processor_run_encrypt_without_key() { 116 | let new_params = Params { 117 | conf_path: "../demo/conf/conf_csv.yaml".to_owned(), 118 | file_path: "../demo/data/input/csv".to_owned(), 119 | output_path: "../demo/data/output/csv/mask".to_owned(), 120 | file_type: FileType::CSV, 121 | mode: Mode::ENCRYPT, 122 | key: None, 123 | ..Default::default() 124 | }; 125 | 126 | let mut new_app = App::new(new_params).await.unwrap(); 127 | let metrics = new_app.process().await.unwrap(); 128 | assert_eq!(metrics.metadata.total_records, 0); 129 | } 130 | 131 | #[tokio::test] 132 | async fn test_processor_run_json_mask() { 133 | let new_params = Params { 134 | conf_path: "../demo/conf/conf_json.yaml".to_owned(), 135 | file_path: "../demo/data/input/json".to_owned(), 136 | output_path: "../demo/data/output/json/mask".to_owned(), 137 | file_type: FileType::JSON, 138 | mode: Mode::MASK, 139 | ..Default::default() 140 | }; 141 | 142 | let mut new_app = App::new(new_params).await.unwrap(); 143 | let metrics = new_app.process().await.unwrap(); 144 | assert_eq!(metrics.metadata.failed_records, 0); 145 | } 146 | -------------------------------------------------------------------------------- /demo/data/output/json/demo/data/input/json/generated.json: -------------------------------------------------------------------------------- 1 | [{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"#####","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"#####","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"#####","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"#####","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"#####","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"#####","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"#####","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"#####","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"#####","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"#####","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}}] -------------------------------------------------------------------------------- /demo/data/output/json/demo/data/input/json/level/generated.json: -------------------------------------------------------------------------------- 1 | [{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"#####","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"#####","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"#####","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"#####","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"#####","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"#####","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"#####","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"#####","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"#####","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"#####","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}}] -------------------------------------------------------------------------------- /med_core/src/models/enums.rs: -------------------------------------------------------------------------------- 1 | use clap::{builder::PossibleValue, ValueEnum}; 2 | use serde::Serialize; 3 | use std::fmt; 4 | 5 | #[derive(Debug, Clone, Serialize, Default, PartialEq)] 6 | pub enum FileType { 7 | #[default] 8 | CSV, 9 | JSON, 10 | } 11 | 12 | impl fmt::Display for FileType { 13 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 14 | match self { 15 | FileType::CSV => write!(f, "csv"), 16 | FileType::JSON => write!(f, "json"), 17 | } 18 | } 19 | } 20 | 21 | #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Default)] 22 | pub enum Mode { 23 | #[default] 24 | MASK, 25 | ENCRYPT, 26 | DECRYPT, 27 | } 28 | 29 | // Can also be derived with feature flag `derive` 30 | impl ValueEnum for Mode { 31 | fn value_variants<'a>() -> &'a [Self] { 32 | &[Mode::MASK, Mode::ENCRYPT, Mode::DECRYPT] 33 | } 34 | 35 | fn to_possible_value<'a>(&self) -> Option { 36 | Some(match self { 37 | Mode::MASK => PossibleValue::new("mask").help("Masking the data"), 38 | Mode::ENCRYPT => { 39 | PossibleValue::new("encrypt").help("Encrypt the data with provided KEY") 40 | } 41 | Mode::DECRYPT => { 42 | PossibleValue::new("decrypt").help("Decrypt the data with provided KEY") 43 | } 44 | }) 45 | } 46 | } 47 | 48 | impl std::fmt::Display for Mode { 49 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 50 | self.to_possible_value() 51 | .expect("no values are skipped") 52 | .get_name() 53 | .fmt(f) 54 | } 55 | } 56 | 57 | impl std::str::FromStr for Mode { 58 | type Err = String; 59 | fn from_str(s: &str) -> Result { 60 | for variant in Self::value_variants() { 61 | if variant.to_possible_value().unwrap().matches(s, false) { 62 | return Ok(*variant); 63 | } 64 | } 65 | Err(format!("invalid variant: {}", s)) 66 | } 67 | } 68 | 69 | impl std::fmt::Debug for Mode { 70 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 71 | match self { 72 | Self::MASK => write!(f, "MASK"), 73 | Self::ENCRYPT => write!(f, "ENCRYPT"), 74 | Self::DECRYPT => write!(f, "DECRYPT"), 75 | } 76 | } 77 | } 78 | 79 | #[derive(Clone, Serialize, PartialEq)] 80 | #[allow(dead_code)] 81 | #[derive(Default)] 82 | pub enum AppMode { 83 | #[default] 84 | CLI, 85 | SDK, 86 | } 87 | 88 | impl std::fmt::Debug for AppMode { 89 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 90 | match self { 91 | Self::CLI => write!(f, "CLI"), 92 | Self::SDK => write!(f, "SDK"), 93 | } 94 | } 95 | } 96 | 97 | impl fmt::Display for AppMode { 98 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 99 | match self { 100 | AppMode::CLI => write!(f, "CLI"), 101 | AppMode::SDK => write!(f, "SDK"), 102 | } 103 | } 104 | } 105 | 106 | #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Default)] 107 | pub enum Standard { 108 | #[default] 109 | DES64 = 64, 110 | AES128 = 128, 111 | AES192 = 192, 112 | AES256 = 256, 113 | } 114 | 115 | // Can also be derived with feature flag `derive` 116 | impl ValueEnum for Standard { 117 | fn value_variants<'a>() -> &'a [Self] { 118 | &[ 119 | Standard::DES64, 120 | Standard::AES128, 121 | Standard::AES192, 122 | Standard::AES256, 123 | ] 124 | } 125 | 126 | fn to_possible_value<'a>(&self) -> Option { 127 | Some(match self { 128 | Standard::DES64 => PossibleValue::new("des64").help("DES standard 64"), 129 | Standard::AES128 => PossibleValue::new("aes128").help("AES standard 128"), 130 | Standard::AES192 => PossibleValue::new("aes192").help("AES standard 192"), 131 | Standard::AES256 => PossibleValue::new("aes256").help("AES standard 256"), 132 | }) 133 | } 134 | } 135 | 136 | impl std::fmt::Display for Standard { 137 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 138 | self.to_possible_value() 139 | .expect("no values are skipped") 140 | .get_name() 141 | .fmt(f) 142 | } 143 | } 144 | 145 | impl std::str::FromStr for Standard { 146 | type Err = String; 147 | fn from_str(s: &str) -> Result { 148 | for variant in Self::value_variants() { 149 | if variant.to_possible_value().unwrap().matches(s, false) { 150 | return Ok(*variant); 151 | } 152 | } 153 | Err(format!("invalid variant: {}", s)) 154 | } 155 | } 156 | 157 | impl std::fmt::Debug for Standard { 158 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 159 | match self { 160 | Self::DES64 => write!(f, "DES64"), 161 | Self::AES128 => write!(f, "AES128"), 162 | Self::AES192 => write!(f, "AES192"), 163 | Self::AES256 => write!(f, "AES256"), 164 | } 165 | } 166 | } 167 | 168 | #[cfg(test)] 169 | #[path = "../tests/models_test.rs"] 170 | mod models_test; 171 | -------------------------------------------------------------------------------- /demo/data/output/json/mask/generated.json: -------------------------------------------------------------------------------- 1 | [{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"#####"},{"id":1,"name":"#####"},{"id":2,"name":"#####"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"#####","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"#####"}}] -------------------------------------------------------------------------------- /demo/data/input/json/generated.json: -------------------------------------------------------------------------------- 1 | [{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}}] -------------------------------------------------------------------------------- /demo/data/input/format_err/json/format_err.json: -------------------------------------------------------------------------------- 1 | [{"_id":"646ae336271a76d64e27c4db""about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}}] -------------------------------------------------------------------------------- /demo/data/output/json/decrypt/generated.json: -------------------------------------------------------------------------------- 1 | [{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"Marci Pollard"},{"id":1,"name":"Mcconnell Knight"},{"id":2,"name":"Cummings Quinn"}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"Joyce Woods","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"rust king"}}] -------------------------------------------------------------------------------- /med_core/src/app/processor.rs: -------------------------------------------------------------------------------- 1 | use tracing::debug; 2 | use walkdir::WalkDir; 3 | 4 | use crate::app::csv::csv_processor; 5 | use crate::app::json::json_processor; 6 | use crate::app::worker::Worker; 7 | use crate::models::enums::{FileType, Mode, Standard}; 8 | use crate::models::metrics::Metrics; 9 | use crate::utils::config::JobConfig; 10 | use crate::utils::crypto::Cypher; 11 | use crate::utils::error::MedErrorType; 12 | use crate::utils::helpers::{create_output_dir, is_not_hidden}; 13 | use crate::utils::progress_bar::get_progress_bar; 14 | use crate::{models::params::Params, utils::error::MedError}; 15 | 16 | #[derive(Debug, Clone, Default)] 17 | pub struct FileProcessor { 18 | metrics: Metrics, 19 | runtime_params: Params, 20 | pub process_runtime: ProcessRuntime, 21 | } 22 | 23 | #[derive(Debug, Clone, Default)] 24 | pub struct ProcessRuntime { 25 | pub fields: Vec, 26 | pub mask_symbols: Option, 27 | pub cypher: Option, 28 | pub standard: Option, 29 | pub mode: Mode, 30 | } 31 | 32 | impl FileProcessor { 33 | pub async fn new(runtime_params: Params, job_conf: JobConfig) -> Self { 34 | let mode = runtime_params.mode; 35 | FileProcessor { 36 | metrics: Metrics::default(), 37 | runtime_params, 38 | process_runtime: ProcessRuntime { 39 | fields: job_conf.fields, 40 | mask_symbols: Some(job_conf.mask_symbols), 41 | cypher: None, 42 | standard: None, 43 | mode, 44 | }, 45 | } 46 | } 47 | pub async fn run(&mut self) -> Result { 48 | match self.runtime_params.mode { 49 | Mode::ENCRYPT | Mode::DECRYPT => match &self.runtime_params.key { 50 | Some(key) => { 51 | self.process_runtime.cypher = Some(Cypher::new(key)); 52 | self.process_runtime.standard = Some(self.runtime_params.standard); 53 | } 54 | None => { 55 | return Err(MedError { 56 | message: Some( 57 | "Missing key for Encyption and Decryption input!".to_string(), 58 | ), 59 | cause: Some("missing -k or --key".to_string()), 60 | error_type: MedErrorType::ConfigError, 61 | }) 62 | } 63 | }, 64 | Mode::MASK => (), 65 | } 66 | self.metrics = self.load().await?; 67 | 68 | Ok(self.metrics.clone()) 69 | } 70 | 71 | async fn load(&mut self) -> Result { 72 | // prepare the channel to send back the metrics 73 | let (tx_metadata, rx_metadata) = flume::unbounded(); 74 | 75 | // inital worker based on the input 76 | let new_worker = Worker::new(self.runtime_params.worker).await?; 77 | 78 | // init the files number as 0 79 | let mut files_number: u64 = 0; 80 | 81 | // create outpu dir 82 | create_output_dir( 83 | &self.runtime_params.output_path, 84 | &self.runtime_params.file_path, 85 | ) 86 | .await?; 87 | 88 | // loop over the files path 89 | for entry in WalkDir::new(&self.runtime_params.file_path) 90 | .follow_links(true) 91 | .into_iter() 92 | .filter_entry(is_not_hidden) 93 | .filter_map(|e| e.ok()) 94 | .filter(|e| !e.path().is_dir()) 95 | { 96 | // prepare the worker processing 97 | let tx_metadata = tx_metadata.clone(); 98 | let files_path = entry.path().display().to_string(); 99 | let output_dir = format!("{}/{}", self.runtime_params.output_path, files_path); 100 | let process_runtime = self.process_runtime.clone(); 101 | 102 | // debug ensure the files have been process 103 | debug!( 104 | "load {:?} files: {:?}", 105 | self.runtime_params.file_type, 106 | entry.path().display().to_string() 107 | ); 108 | 109 | // increase file number 110 | files_number += 1; 111 | 112 | match self.runtime_params.file_type { 113 | FileType::CSV => { 114 | // worker execution 115 | new_worker.pool.execute(move || { 116 | csv_processor(tx_metadata, &files_path, &output_dir, process_runtime) 117 | .unwrap(); 118 | }); 119 | } 120 | FileType::JSON => { 121 | new_worker.pool.execute(move || { 122 | json_processor(tx_metadata, &files_path, &output_dir, process_runtime) 123 | .unwrap(); 124 | }); 125 | } 126 | } 127 | } 128 | 129 | // drop the channel once it done. 130 | drop(tx_metadata); 131 | 132 | let bar = get_progress_bar( 133 | files_number, 134 | &format!("processing {:?} files", self.runtime_params.file_type), 135 | ); 136 | rx_metadata.iter().for_each(|item| { 137 | bar.inc(1); 138 | self.metrics.total_files = files_number as usize; 139 | self.metrics.metadata.total_records += item.total_records; 140 | self.metrics.metadata.failed_records += item.failed_records; 141 | self.metrics 142 | .metadata 143 | .record_failed_reason 144 | .extend(item.record_failed_reason); 145 | }); 146 | bar.finish_and_clear(); 147 | 148 | debug!("metrics {:?}", self.metrics); 149 | 150 | Ok(self.metrics.clone()) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /demo/data/output/json/encrypt/generated.json: -------------------------------------------------------------------------------- 1 | [{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"8RXHQBe6dl4DfYYCNJRT+A=="},{"id":1,"name":"ztilIm8PKBXqtOLAux0Al8/w+sbpQOcf"},{"id":2,"name":"hlxQ4g1fNFrEM6RhldCHTA=="}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"fso/8XWLnmOi4RmdUTU/Zg==","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"WV4uQFjDPCnwltBuhrdzSg=="}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"8RXHQBe6dl4DfYYCNJRT+A=="},{"id":1,"name":"ztilIm8PKBXqtOLAux0Al8/w+sbpQOcf"},{"id":2,"name":"hlxQ4g1fNFrEM6RhldCHTA=="}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"fso/8XWLnmOi4RmdUTU/Zg==","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"WV4uQFjDPCnwltBuhrdzSg=="}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"8RXHQBe6dl4DfYYCNJRT+A=="},{"id":1,"name":"ztilIm8PKBXqtOLAux0Al8/w+sbpQOcf"},{"id":2,"name":"hlxQ4g1fNFrEM6RhldCHTA=="}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"fso/8XWLnmOi4RmdUTU/Zg==","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"WV4uQFjDPCnwltBuhrdzSg=="}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"8RXHQBe6dl4DfYYCNJRT+A=="},{"id":1,"name":"ztilIm8PKBXqtOLAux0Al8/w+sbpQOcf"},{"id":2,"name":"hlxQ4g1fNFrEM6RhldCHTA=="}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"fso/8XWLnmOi4RmdUTU/Zg==","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"WV4uQFjDPCnwltBuhrdzSg=="}},{"_id":"646ae336271a76d64e27c4db","about":"Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n","address":"527 Commercial Street, Bowmansville, Missouri, 5038","age":23,"balance":"$1,429.90","company":"PARLEYNET","email":"joycewoods@parleynet.com","eyeColor":"green","favoriteFruit":"apple","friends":[{"id":0,"name":"8RXHQBe6dl4DfYYCNJRT+A=="},{"id":1,"name":"ztilIm8PKBXqtOLAux0Al8/w+sbpQOcf"},{"id":2,"name":"hlxQ4g1fNFrEM6RhldCHTA=="}],"gender":"female","greeting":"Hello, Joyce Woods! You have 7 unread messages.","guid":"6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e","index":0,"isActive":false,"latitude":-11.6502,"longitude":-113.585217,"name":"fso/8XWLnmOi4RmdUTU/Zg==","phone":"+1 (883) 513-3787","picture":"http://placehold.it/32x32","registered":"2022-05-23T03:46:20 +04:00","tags":["tempor","enim","quis","nulla","culpa","adipisicing","irure"],"test":{"item":"adfad","name":"WV4uQFjDPCnwltBuhrdzSg=="}}] -------------------------------------------------------------------------------- /med_core/src/audit/db.rs: -------------------------------------------------------------------------------- 1 | use std::{path::PathBuf, str::FromStr, time::Duration}; 2 | 3 | use colored::Colorize; 4 | use sqlx::{ 5 | migrate::MigrateDatabase, 6 | sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous}, 7 | Pool, Row, Sqlite, 8 | }; 9 | use tracing::{debug, info}; 10 | 11 | use crate::utils::error::MedError; 12 | 13 | use super::app::Summary; 14 | 15 | pub struct Database { 16 | pub pool: sqlx::Pool, 17 | } 18 | 19 | #[derive(Debug, Default, Clone)] 20 | pub struct AuditSummary { 21 | pub user: String, 22 | pub hostname: String, 23 | pub total_files: usize, 24 | pub total_records: usize, 25 | pub failed_records: usize, 26 | pub record_failed_reason: Vec, 27 | pub runtime_conf: String, 28 | pub process_failure_reason: Option, 29 | pub successed: bool, 30 | } 31 | 32 | #[cfg(not(tarpaulin_include))] 33 | impl Database { 34 | pub async fn new() -> Result { 35 | let database_url = Self::create_audit_db().await?; 36 | 37 | if !Sqlite::database_exists(database_url.to_str().unwrap()) 38 | .await 39 | .unwrap_or(false) 40 | { 41 | Sqlite::create_database(database_url.to_str().unwrap()).await?; 42 | info!( 43 | "audit database {} created", 44 | database_url.display().to_string().bold().green() 45 | ); 46 | } else { 47 | info!( 48 | "audit database {} exist", 49 | database_url.display().to_string().bold().green() 50 | ); 51 | } 52 | let pool_timeout = Duration::from_secs(30); 53 | 54 | let connection_options = 55 | SqliteConnectOptions::from_str(&database_url.display().to_string().bold().green())? 56 | .create_if_missing(true) 57 | .journal_mode(SqliteJournalMode::Wal) 58 | .synchronous(SqliteSynchronous::Normal) 59 | .busy_timeout(pool_timeout); 60 | 61 | let pool = SqlitePoolOptions::new() 62 | .max_connections(20) 63 | .connect_with(connection_options) 64 | .await?; 65 | 66 | // Self::create_table(&pool).await?; 67 | Self::migrate(&pool).await?; 68 | 69 | // Self::migrate(&pool).await?; 70 | Ok(Database { pool }) 71 | } 72 | 73 | async fn create_audit_db() -> Result { 74 | let path = dirs::config_dir().unwrap().join("med.db"); 75 | Ok(path) 76 | } 77 | 78 | async fn migrate(pool: &Pool) -> Result<(), MedError> { 79 | // this is the implementation for the migrations folder 80 | // let migrations = std::path::Path::new("./migrations"); 81 | // sqlx::migrate::Migrator::new(migrations) 82 | // .await? 83 | // .run(pool) 84 | // .await?; 85 | Self::create_table(pool).await?; 86 | Self::alter_table(pool).await?; 87 | Ok(()) 88 | } 89 | 90 | async fn create_table(pool: &Pool) -> Result<(), MedError> { 91 | let result = sqlx::query( 92 | " 93 | CREATE TABLE IF NOT EXISTS audit ( 94 | id INTEGER PRIMARY KEY, 95 | user TEXT NOT NULL, 96 | hostname TEXT NOT NULL, 97 | total_files INTEGER NOT NULL, 98 | total_records INTEGER NOT NULL, 99 | failed_records INTEGER NOT NULL, 100 | record_failed_reason TEXT, 101 | runtime_conf TEXT NOT NULL, 102 | process_failure_reason TEXT, 103 | successed BOOLEAN NOT NULL DEFAULT FALSE, 104 | created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL 105 | ); 106 | ", 107 | ) 108 | .execute(pool) 109 | .await?; 110 | debug!("audit database {:?} create table successed", result); 111 | Ok(()) 112 | } 113 | 114 | async fn alter_table(pool: &Pool) -> Result<(), MedError> { 115 | let res = sqlx::query( 116 | "select count(*) as count from pragma_table_info('audit') where name='elapsed_time';", 117 | ) 118 | .fetch_one(pool) 119 | .await?; 120 | 121 | if res.get::("count") == 0 { 122 | let result = sqlx::query( 123 | " 124 | ALTER TABLE audit ADD COLUMN elapsed_time TEXT; 125 | ", 126 | ) 127 | .execute(pool) 128 | .await?; 129 | debug!("audit database {:?} alter table successed", result); 130 | } else { 131 | debug!("audit database alter cols exist skip"); 132 | } 133 | 134 | Ok(()) 135 | } 136 | 137 | pub async fn insert(&mut self, summary: &Summary) -> Result { 138 | let total_files = summary.metrics.total_files as i64; 139 | let total_records = summary.metrics.metadata.total_records as i64; 140 | let failed_records: i64 = summary.metrics.metadata.failed_records as i64; 141 | let record_failed_reason = 142 | serde_json::to_string(&summary.metrics.metadata.record_failed_reason)?; 143 | let elapsed_time = summary.elapsed_time.to_owned(); 144 | 145 | let id = sqlx::query!( 146 | r#" 147 | INSERT INTO audit ( user, hostname, total_files, total_records, failed_records, record_failed_reason, runtime_conf, process_failure_reason, successed, elapsed_time ) 148 | VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) 149 | "#, 150 | summary.user, summary.hostname, total_files, total_records, failed_records, record_failed_reason, summary.runtime_conf, summary.process_failure_reason, summary.successed, elapsed_time 151 | ) 152 | .execute(&self.pool) 153 | .await? 154 | .last_insert_rowid(); 155 | Ok(id) 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /med_core/src/tests/json_test.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | app::{json::json_processor, processor::ProcessRuntime}, 3 | models::enums::Mode, 4 | utils::{crypto::Cypher, error::MedErrorType}, 5 | }; 6 | 7 | // const KEY: &str = "123"; 8 | 9 | #[tokio::test] 10 | async fn test_json_processor_error() { 11 | // tx_metadata: flume::Sender, 12 | // files_path: &str, 13 | // output_path: &str, 14 | // process_runtime: ProcessRuntime, 15 | let (tx_metadata, rx_metadata) = flume::unbounded(); 16 | let process_runtime = ProcessRuntime { 17 | fields: vec!["name".to_string()], 18 | cypher: None, 19 | standard: None, 20 | mask_symbols: Some("#####".to_string()), 21 | mode: Mode::MASK, 22 | }; 23 | 24 | let files_path: &str = ""; 25 | let output_path = ""; 26 | 27 | match json_processor( 28 | tx_metadata.clone(), 29 | files_path, 30 | output_path, 31 | process_runtime, 32 | ) { 33 | Ok(_) => {} 34 | Err(err) => { 35 | assert_eq!(err.error_type, MedErrorType::IoError); 36 | } 37 | } 38 | 39 | // drop the channel once it done. 40 | drop(tx_metadata); 41 | 42 | rx_metadata.iter().for_each(|item| { 43 | assert_eq!(item.total_records, 0); 44 | }); 45 | } 46 | 47 | #[tokio::test] 48 | async fn test_json_processor_format_err() { 49 | let (tx_metadata, rx_metadata) = flume::unbounded(); 50 | let process_runtime = ProcessRuntime { 51 | fields: vec!["name".to_string()], 52 | cypher: None, 53 | standard: None, 54 | mask_symbols: Some("#####".to_string()), 55 | mode: Mode::MASK, 56 | }; 57 | let files_path: &str = "../demo/data/input/format_err/json/format_err.json"; 58 | let output_path = "../demo/data/output/json/format_err/generated.json"; 59 | 60 | json_processor( 61 | tx_metadata.clone(), 62 | files_path, 63 | output_path, 64 | process_runtime, 65 | ) 66 | .unwrap(); 67 | 68 | // drop the channel once it done. 69 | drop(tx_metadata); 70 | 71 | rx_metadata.iter().for_each(|item| { 72 | assert_eq!(item.failed_records, 1); 73 | }); 74 | } 75 | 76 | #[tokio::test] 77 | async fn test_json_is_obj() { 78 | let (tx_metadata, rx_metadata) = flume::unbounded(); 79 | let process_runtime = ProcessRuntime { 80 | fields: vec!["name".to_string()], 81 | cypher: None, 82 | standard: None, 83 | mask_symbols: Some("#####".to_string()), 84 | mode: Mode::MASK, 85 | }; 86 | let files_path: &str = "../demo/data/input/json/is_obj.json"; 87 | let output_path = "../demo/data/output/json/mask/is_obj.json"; 88 | 89 | json_processor( 90 | tx_metadata.clone(), 91 | files_path, 92 | output_path, 93 | process_runtime, 94 | ) 95 | .unwrap(); 96 | 97 | // drop the channel once it done. 98 | drop(tx_metadata); 99 | 100 | rx_metadata.iter().for_each(|item| { 101 | // println!("item : {:?}", item ); 102 | assert_eq!(item.total_records, 1); 103 | }); 104 | } 105 | 106 | #[tokio::test] 107 | async fn test_json_arr_in_arr() { 108 | let (tx_metadata, rx_metadata) = flume::unbounded(); 109 | let process_runtime = ProcessRuntime { 110 | fields: vec!["name".to_string()], 111 | cypher: None, 112 | standard: None, 113 | mask_symbols: Some("#####".to_string()), 114 | mode: Mode::MASK, 115 | }; 116 | let files_path: &str = "../demo/data/input/json/arr_in_arr.json"; 117 | let output_path = "../demo/data/output/json/mask/arr_in_arr.json"; 118 | 119 | json_processor( 120 | tx_metadata.clone(), 121 | files_path, 122 | output_path, 123 | process_runtime, 124 | ) 125 | .unwrap(); 126 | 127 | // drop the channel once it done. 128 | drop(tx_metadata); 129 | 130 | rx_metadata.iter().for_each(|item| { 131 | // println!("item : {:?}", item ); 132 | assert_eq!(item.failed_records, 0); 133 | }); 134 | } 135 | 136 | #[tokio::test] 137 | async fn test_json_encrypt() { 138 | let (tx_metadata, rx_metadata) = flume::unbounded(); 139 | let process_runtime = ProcessRuntime { 140 | fields: vec!["name".to_string()], 141 | cypher: Some(Cypher::new("1234")), 142 | standard: Some(crate::models::enums::Standard::DES64), 143 | mask_symbols: None, 144 | mode: Mode::ENCRYPT, 145 | }; 146 | let files_path: &str = "../demo/data/input/json/level/generated.json"; 147 | let output_path: &str = "../demo/data/output/json/encrypt/generated.json"; 148 | 149 | json_processor( 150 | tx_metadata.clone(), 151 | files_path, 152 | output_path, 153 | process_runtime, 154 | ) 155 | .unwrap(); 156 | 157 | // drop the channel once it done. 158 | drop(tx_metadata); 159 | 160 | rx_metadata.iter().for_each(|item| { 161 | // info!("item : {:?}", item); 162 | assert_eq!(item.failed_records, 0); 163 | }); 164 | } 165 | 166 | #[tokio::test] 167 | async fn test_json_decrypt() { 168 | let (tx_metadata, rx_metadata) = flume::unbounded(); 169 | let process_runtime = ProcessRuntime { 170 | fields: vec!["name".to_string()], 171 | cypher: Some(Cypher::new("1234")), 172 | standard: Some(crate::models::enums::Standard::DES64), 173 | mask_symbols: None, 174 | mode: Mode::DECRYPT, 175 | }; 176 | let output_path: &str = "../demo/data/output/json/decrypt/generated.json"; 177 | let files_path: &str = "../demo/data/output/json/encrypt/generated.json"; 178 | 179 | json_processor( 180 | tx_metadata.clone(), 181 | files_path, 182 | output_path, 183 | process_runtime, 184 | ) 185 | .unwrap(); 186 | 187 | // drop the channel once it done. 188 | drop(tx_metadata); 189 | 190 | rx_metadata.iter().for_each(|item| { 191 | // info!("item : {:?}", item); 192 | assert_eq!(item.failed_records, 0); 193 | }); 194 | } 195 | -------------------------------------------------------------------------------- /med_core/src/app/csv.rs: -------------------------------------------------------------------------------- 1 | use colored::Colorize; 2 | use csv::{StringRecord, Writer}; 3 | use tracing::{debug, info, warn}; 4 | 5 | use crate::{ 6 | models::{enums::Mode, metrics::Metadata}, 7 | utils::error::{MedError, MedErrorType}, 8 | }; 9 | 10 | use crate::app::processor::ProcessRuntime; 11 | 12 | pub fn csv_processor( 13 | tx_metadata: flume::Sender, 14 | files_path: &str, 15 | output_path: &str, 16 | process_runtime: ProcessRuntime, 17 | ) -> Result<(), MedError> { 18 | // prepare the reader and read the file 19 | let mut reader = csv::Reader::from_path(files_path)?; 20 | 21 | // get the header of the file 22 | let headers = reader.headers()?.to_owned(); 23 | 24 | // prepare the metrics 25 | let mut failed_records: usize = 0; 26 | let mut record_failed_reason: Vec = Vec::new(); 27 | 28 | let indexs = csv_fields_exist(headers.clone(), &process_runtime.fields); 29 | debug!("write to location : {:?}", output_path); 30 | 31 | let mut total_records = 0; 32 | 33 | // prepare the writer 34 | let mut wtr = Writer::from_path(output_path)?; 35 | 36 | // write the header 37 | wtr.write_record(&headers)?; 38 | 39 | reader.into_records().for_each(|record| { 40 | total_records += 1; 41 | match record { 42 | Ok(records) => { 43 | let mut masked_record: StringRecord = StringRecord::new(); 44 | records.iter().enumerate().for_each(|(i, item)| { 45 | match indexs.contains(&i) { 46 | true => { 47 | let mut masked: String = String::new(); 48 | match process_runtime.mode { 49 | Mode::MASK => { 50 | if let Some(symbols) = process_runtime.mask_symbols.clone() { 51 | masked = symbols; 52 | } 53 | } 54 | Mode::ENCRYPT => { 55 | if let Some(cypher) = process_runtime.cypher.clone() { 56 | if let Some(standard) = process_runtime.standard { 57 | masked = cypher.encrypt(item, &standard).unwrap(); 58 | } 59 | } 60 | } 61 | Mode::DECRYPT => { 62 | if let Some(cypher) = process_runtime.cypher.clone() { 63 | if let Some(standard) = process_runtime.standard { 64 | match cypher.decrypt(item, &standard) { 65 | Ok(m) => masked = m, 66 | Err(err) => { 67 | let record_error = MedError { 68 | message: Some(format!( 69 | "please check {} {:?} format", 70 | files_path, process_runtime.mode 71 | )), 72 | cause: Some(err.to_string()), 73 | error_type: MedErrorType::CsvError, 74 | }; 75 | let error_str = 76 | serde_json::to_string(&record_error) 77 | .unwrap(); 78 | info!( 79 | "{}: {}", 80 | "warning".bold().yellow(), 81 | error_str 82 | ); 83 | record_failed_reason.push(record_error); 84 | failed_records += 1; 85 | } 86 | } 87 | } 88 | } 89 | } 90 | } 91 | masked_record.push_field(&masked); 92 | } 93 | false => masked_record.push_field(item), 94 | }; 95 | }); 96 | wtr.write_record(&masked_record).unwrap(); 97 | } 98 | Err(err) => { 99 | let record_error = MedError { 100 | message: Some(format!( 101 | "please check {} {:?} format", 102 | files_path, process_runtime.mode 103 | )), 104 | cause: Some(err.to_string()), 105 | error_type: MedErrorType::CsvError, 106 | }; 107 | let error_str = serde_json::to_string(&record_error).unwrap(); 108 | info!("{}: {}", "warning".bold().yellow(), error_str); 109 | record_failed_reason.push(record_error); 110 | failed_records += 1; 111 | } 112 | }; 113 | }); 114 | // clear the writer 115 | wtr.flush()?; 116 | 117 | tx_metadata 118 | .send(Metadata { 119 | total_records, 120 | failed_records, 121 | record_failed_reason, 122 | }) 123 | .unwrap(); 124 | 125 | Ok(()) 126 | } 127 | 128 | fn csv_fields_exist(headers: StringRecord, fields: &[String]) -> Vec { 129 | let indexs = headers 130 | .iter() 131 | .enumerate() 132 | .filter(|(_, item)| fields.contains(&item.to_string())) 133 | .map(|(i, _)| i) 134 | .collect::>(); 135 | 136 | if indexs.is_empty() { 137 | warn!("Please check your csv file, there is no marched header found in the csv files"); 138 | std::process::exit(1); 139 | } 140 | indexs 141 | } 142 | 143 | #[cfg(test)] 144 | #[path = "../tests/csv_test.rs"] 145 | mod csv_test; 146 | -------------------------------------------------------------------------------- /med_cli/src/cli/app.rs: -------------------------------------------------------------------------------- 1 | use crate::cli::custom_validation::{dir_exist, worker_in_range}; 2 | use clap::{arg, command, value_parser, ArgMatches}; 3 | use med_core::models::enums::{FileType, Mode, Standard}; 4 | use med_core::models::params::Params; 5 | use med_core::utils::error::MedError; 6 | use std::path::PathBuf; 7 | use tracing::log::info; 8 | 9 | pub struct Cli { 10 | pub params: Params, 11 | } 12 | 13 | #[cfg(not(tarpaulin_include))] 14 | impl Cli { 15 | /// Returns a Cli with the input config 16 | /// 17 | /// # Examples 18 | /// ``` 19 | /// 20 | /// let new_cli = Cli::new().await?; 21 | /// let params = new_cli.params; 22 | /// 23 | /// ``` 24 | pub async fn new() -> Result { 25 | // Initial Default CLI params 26 | let new_cli = Params::default(); 27 | 28 | // Get the cli input params 29 | let matches = Self::get_params().await; 30 | 31 | // replace the default cli params by the cli input from the prompt 32 | let params = Self::fulfill_cli(matches, new_cli).await?; 33 | 34 | // return the fulfilled CLI Params 35 | Ok(Cli { params }) 36 | } 37 | 38 | /// Privite function fulfill the Cli Struct 39 | async fn fulfill_cli(matches: ArgMatches, mut params: Params) -> Result { 40 | // Note, it's safe to call unwrap() because the arg is required 41 | match matches 42 | .get_one::("MODE") 43 | .expect("'MODE' is required and parsing will fail if its missing") 44 | { 45 | Mode::MASK => { 46 | params.mode = Mode::MASK; 47 | params.key = None; 48 | } 49 | Mode::ENCRYPT => { 50 | params.mode = Mode::ENCRYPT; 51 | if let Some(key) = matches.get_one::("key") { 52 | params.key = Some(key.to_owned()); 53 | } 54 | } 55 | Mode::DECRYPT => { 56 | params.mode = Mode::DECRYPT; 57 | if let Some(key) = matches.get_one::("key") { 58 | params.key = Some(key.to_owned()); 59 | } 60 | } 61 | } 62 | 63 | if let Some(path) = matches.get_one::("config") { 64 | info!("conf.yml location {:?} : ", path.display()); 65 | params.conf_path = path.display().to_string(); 66 | } 67 | 68 | if let Some(path) = matches.get_one::("file") { 69 | info!("file location {:?} : ", path.display()); 70 | params.file_path = path.display().to_string(); 71 | } 72 | 73 | if let Some(path) = matches.get_one::("output") { 74 | info!("output file location {:?} : ", path.display()); 75 | params.output_path = path.display().to_string(); 76 | } 77 | 78 | if let Some(f_type) = matches.get_one::("type") { 79 | if *f_type != FileType::CSV.to_string() { 80 | params.file_type = FileType::JSON; 81 | } 82 | } 83 | 84 | if let Some(debug) = matches.get_one::("debug") { 85 | params.debug = debug.to_owned(); 86 | } 87 | 88 | if let Some(worker) = matches.get_one::("worker") { 89 | params.worker = worker.to_owned(); 90 | } 91 | 92 | if let Some(standard) = matches.get_one::("standard") { 93 | match standard { 94 | Standard::AES128 => { 95 | params.standard = Standard::AES128; 96 | } 97 | Standard::AES192 => { 98 | params.standard = Standard::AES192; 99 | } 100 | Standard::AES256 => { 101 | params.standard = Standard::AES256; 102 | } 103 | Standard::DES64 => { 104 | params.standard = Standard::DES64; 105 | } 106 | } 107 | } 108 | 109 | Ok(params) 110 | } 111 | 112 | /// Privite function get the Clap parsed params. 113 | async fn get_params() -> ArgMatches { 114 | command!() 115 | .propagate_version(true) 116 | .arg_required_else_help(true) 117 | .arg( 118 | arg!() 119 | .required(true) 120 | .help("What mode to run the program in") 121 | .value_parser(value_parser!(Mode)), 122 | ) 123 | .arg( 124 | arg!( 125 | -t --type "Sets a process file type [csv, json]" 126 | ) 127 | .required(false) 128 | .help("Type of file we will process, available option [csv, json]") 129 | .default_value("csv"), 130 | ) 131 | .arg( 132 | arg!( 133 | -k --key "Sets a KEY to process file" 134 | ) 135 | .help("Key for Encrypt and Decrypt the file.") 136 | .required_if_eq_any([("MODE", "decrypt"), ("MODE", "encrypt")]) 137 | .requires("standard"), 138 | ) 139 | .arg( 140 | arg!( 141 | -s --standard "Sets a Encrypt or Decrypt Standard" 142 | ) 143 | .help("Set the Encrypt and Decrypt standard") 144 | .value_parser(value_parser!(Standard)), 145 | ) 146 | .arg( 147 | arg!( 148 | -f --file "Sets a file/directory path" 149 | ) 150 | .required(true) 151 | .help("Path for the process target files") 152 | .value_parser(dir_exist), 153 | ) 154 | .arg( 155 | arg!( 156 | -c --config "Sets a custom config yml path" 157 | ) 158 | .required(false) 159 | .default_value("conf.yaml") 160 | .value_parser(value_parser!(PathBuf)), 161 | ) 162 | .arg( 163 | arg!( 164 | -o --output "Sets a file/directory path for output" 165 | ) 166 | .required(false) 167 | .default_value("output") 168 | .value_parser(value_parser!(PathBuf)), 169 | ) 170 | .arg( 171 | arg!( 172 | -d --debug "Sets debug flag" 173 | ) 174 | .required(false) 175 | .value_parser(clap::value_parser!(bool)), 176 | ) 177 | .arg( 178 | arg!( 179 | -w --worker "Sets work flag" 180 | ) 181 | .required(false) 182 | .value_parser(worker_in_range), 183 | ) 184 | .get_matches() 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /med_core/src/app/core.rs: -------------------------------------------------------------------------------- 1 | use crate::app::processor::FileProcessor; 2 | use crate::audit::app::Audit; 3 | use crate::{utils::config::JobConfig, utils::error::MedError}; 4 | use colored::Colorize; 5 | use std::path::Path; 6 | use tokio::time::Instant; 7 | use tracing::{debug, info}; 8 | 9 | use crate::models::{metrics::Metrics, params::Params}; 10 | use crate::utils::logger::logging; 11 | 12 | pub struct App { 13 | pub params: Params, 14 | pub user: String, 15 | pub hostname: String, 16 | pub audit: Audit, 17 | pub metrics: Metrics, 18 | } 19 | 20 | impl App { 21 | /// Returns a App Struct for processing 22 | /// 23 | /// # Arguments 24 | /// 25 | /// * `params` [Params] - params passed by the CLI or other integration 26 | /// 27 | /// # Examples 28 | /// 29 | /// ``` 30 | /// use med_core::app::core::App; 31 | /// use med_core::utils::error::MedError; 32 | /// use med_core::models::params::Params; 33 | /// 34 | /// #[tokio::main] 35 | /// async fn main() -> Result<(), MedError> { 36 | /// let params = Params::default(); 37 | /// let app = App::new(params).await.unwrap(); 38 | /// Ok(()) 39 | /// } 40 | /// 41 | /// ``` 42 | pub async fn new(params: Params) -> Result { 43 | logging(params.debug).await; 44 | 45 | let user = whoami::username(); 46 | let hostname = whoami::hostname(); 47 | let audit = Audit::new().await?; 48 | let metrics = Metrics::default(); 49 | 50 | info!( 51 | "{} on {} run {} mode for {}", 52 | user.bold().green(), 53 | hostname.bold().green(), 54 | params.app_mode.to_string().bold().green(), 55 | params.mode.to_string().bold().green() 56 | ); 57 | 58 | debug!("app {} {:?}", "runtime params".bold().green(), params); 59 | 60 | Ok(App { 61 | params, 62 | user, 63 | hostname, 64 | audit, 65 | metrics, 66 | }) 67 | } 68 | 69 | /// Privite function Returns job config 70 | async fn load_job_config(&self) -> Result { 71 | let conf = JobConfig::new(Path::new(&self.params.conf_path)).await?; 72 | debug!("{} {:?}", "job config".bold().green(), conf); 73 | Ok(conf) 74 | } 75 | 76 | /// Returns metrics [Metrics] 77 | /// 78 | /// # Examples 79 | /// 80 | /// ``` 81 | /// use med_core::app::core::App; 82 | /// use med_core::utils::error::MedError; 83 | /// use med_core::models::params::Params; 84 | /// use med_core::models::enums::{FileType, Mode}; 85 | /// 86 | /// #[tokio::main] 87 | /// async fn main() -> Result<(), MedError> { 88 | /// let params = Params::default(); 89 | /// let params = Params { 90 | /// conf_path: "../demo/conf/conf_csv.yaml".to_owned(), 91 | /// file_path: "../demo/data/input/format_err/csv".to_owned(), 92 | /// output_path: "../demo/data/output/csv/format_err/processor_err".to_owned(), 93 | /// file_type: FileType::CSV, 94 | /// mode: Mode::MASK, 95 | /// ..Default::default() 96 | /// }; 97 | /// let mut app = App::new(params).await.unwrap(); 98 | /// let metrics = app.process().await.unwrap(); 99 | /// Ok(()) 100 | /// } 101 | /// 102 | /// ``` 103 | pub async fn process(&mut self) -> Result { 104 | info!( 105 | "processing '{}' files start", 106 | self.params.file_type.to_string().bold().green() 107 | ); 108 | info!("file directory {} ", self.params.file_path.bold().green()); 109 | info!( 110 | "number of workers {}", 111 | self.params.worker.to_string().bold().green() 112 | ); 113 | 114 | let now = Instant::now(); 115 | let job_conf = self.load_job_config().await?; 116 | info!( 117 | "load job conf from {} elapsed time {:?}", 118 | self.params.conf_path.bold().green(), 119 | now.elapsed() 120 | ); 121 | 122 | let now = Instant::now(); 123 | let mut processor = FileProcessor::new(self.params.clone(), job_conf).await; 124 | match processor.run().await { 125 | Ok(metrics) => { 126 | self.metrics = metrics.clone(); 127 | self.audit.summary.metrics = metrics.clone(); 128 | if !metrics.metadata.record_failed_reason.is_empty() { 129 | info!( 130 | "{}: {:?}", 131 | "warning".bold().yellow(), 132 | metrics.metadata.record_failed_reason 133 | ); 134 | } 135 | self.audit.summary.successed = true; 136 | } 137 | Err(err) => { 138 | self.audit.summary.process_failure_reason = Some(serde_json::to_string(&err)?); 139 | info!("{} {:?}", "error".bold().red(), err.to_string()); 140 | } 141 | } 142 | info!( 143 | "process {} completed elapsed time {:?}", 144 | self.params.output_path.bold().green(), 145 | now.elapsed() 146 | ); 147 | Ok(self.metrics.clone()) 148 | } 149 | 150 | /// Returns audit_id [i64] 151 | /// 152 | /// # Arguments 153 | /// 154 | /// * `elapsed_time` - A string slice for the elasped_time 155 | /// 156 | /// # Examples 157 | /// 158 | /// ``` 159 | /// use med_core::app::core::App; 160 | /// use med_core::utils::error::MedError; 161 | /// use med_core::models::params::Params; 162 | /// use med_core::models::enums::{FileType, Mode}; 163 | /// use tokio::time::Instant; 164 | /// 165 | /// #[tokio::main] 166 | /// async fn main() -> Result<(), MedError> { 167 | /// let now = Instant::now(); 168 | /// let params = Params { 169 | /// conf_path: "../demo/conf/conf_csv.yaml".to_owned(), 170 | /// file_path: "../demo/data/input/format_err/csv".to_owned(), 171 | /// output_path: "../demo/data/output/csv/format_err/processor_err".to_owned(), 172 | /// file_type: FileType::CSV, 173 | /// mode: Mode::MASK, 174 | /// ..Default::default() 175 | /// }; 176 | /// let mut app = App::new(params).await.unwrap(); 177 | /// let metrics = app.process().await.unwrap(); 178 | /// let audit_id = app.update_audit(format!("{:?}", now.elapsed())).await?; 179 | /// Ok(()) 180 | /// } 181 | /// 182 | /// ``` 183 | #[cfg(not(tarpaulin_include))] 184 | pub async fn update_audit(&mut self, elapsed_time: String) -> Result { 185 | // update the runtime params for the audit record. 186 | if self.params.key.is_some() { 187 | self.params.key = Some("****".to_owned()); 188 | } 189 | self.audit.summary.user = self.user.clone(); 190 | self.audit.summary.hostname = self.hostname.clone(); 191 | self.audit.summary.runtime_conf = serde_json::to_string(&self.params)?; 192 | self.audit.summary.elapsed_time = elapsed_time; 193 | debug!("audit summary : {:?}", self.audit.summary); 194 | 195 | // audit update 196 | let id = self.audit.insert().await?; 197 | Ok(id) 198 | } 199 | } 200 | 201 | #[cfg(test)] 202 | #[path = "../tests/core_test.rs"] 203 | mod core_test; 204 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | 7 | jobs: 8 | crate_metadata: 9 | name: Extract crate metadata 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@master 13 | - name: Extract crate information 14 | id: crate_metadata 15 | run: | 16 | cargo metadata --no-deps --format-version 1 | jq -r '"name=" + .packages[0].name' | tee -a $GITHUB_OUTPUT 17 | cargo metadata --no-deps --format-version 1 | jq -r '"version=" + .packages[0].version' | tee -a $GITHUB_OUTPUT 18 | cargo metadata --no-deps --format-version 1 | jq -r '"maintainer=" + .packages[0].authors[0]' | tee -a $GITHUB_OUTPUT 19 | cargo metadata --no-deps --format-version 1 | jq -r '"homepage=" + .packages[0].homepage' | tee -a $GITHUB_OUTPUT 20 | outputs: 21 | name: ${{ steps.crate_metadata.outputs.name }} 22 | version: ${{ steps.crate_metadata.outputs.version }} 23 | maintainer: ${{ steps.crate_metadata.outputs.maintainer }} 24 | homepage: ${{ steps.crate_metadata.outputs.homepage }} 25 | 26 | unit-test: 27 | runs-on: ubuntu-latest 28 | needs: crate_metadata 29 | steps: 30 | - name: Checkout repository 31 | uses: actions/checkout@v2 32 | - name: Install stable toolchain 33 | uses: actions-rs/toolchain@v1 34 | with: 35 | toolchain: stable 36 | override: true 37 | - name: Run cargo-tarpaulin 38 | uses: actions-rs/tarpaulin@v0.1 39 | with: 40 | version: '0.15.0' 41 | args: '-- --test-threads 2' 42 | - name: Upload coverage reports to Codecov 43 | uses: codecov/codecov-action@v3 44 | with: 45 | token: ${{secrets.CODECOV_TOKEN}} 46 | 47 | build-release: 48 | name: ${{ matrix.target }} 49 | runs-on: ubuntu-latest 50 | needs: [crate_metadata, unit-test] 51 | strategy: 52 | fail-fast: true 53 | matrix: 54 | target: [x86_64-unknown-linux-gnu,x86_64-unknown-linux-musl] 55 | steps: 56 | - name: Checkout 57 | uses: actions/checkout@master 58 | - name: Install rust 59 | uses: actions-rs/toolchain@v1 60 | with: 61 | toolchain: stable 62 | profile: minimal 63 | override: true 64 | target: ${{ matrix.target }} 65 | - name: Build target 66 | uses: actions-rs/cargo@v1 67 | with: 68 | use-cross: true 69 | command: build 70 | args: --release --target ${{ matrix.target }} 71 | - name: Package 72 | shell: bash 73 | run: | 74 | cp -R demo target/${{ matrix.target }}/release 75 | cd target/${{ matrix.target }}/release 76 | mkdir med-${{ needs.crate_metadata.outputs.version }} 77 | mv med med-${{ needs.crate_metadata.outputs.version }} 78 | tar czvf ../../../med-${{ matrix.target }}-${{ needs.crate_metadata.outputs.version }}.tar.gz med-${{ needs.crate_metadata.outputs.version }} demo/ 79 | cd - 80 | - name: Publish 81 | uses: softprops/action-gh-release@v1 82 | with: 83 | tag_name: ${{ needs.crate_metadata.outputs.version }} 84 | files: 'med*' 85 | env: 86 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 87 | 88 | windows-msvc-release: 89 | name: release windows msvc 90 | runs-on: windows-latest 91 | needs: [crate_metadata, unit-test] 92 | steps: 93 | - name: Check Out Code 94 | uses: actions/checkout@master 95 | - name: Build 96 | run: | 97 | cargo build --release 98 | - name: tar 99 | run: | 100 | # $pwd 101 | # New-Item -Path "D:\a\rust-cli-med\rust-cli-med\target\release" -Name "demo" -ItemType Directory 102 | # Copy-Item 'D:\a\rust-cli-med\rust-cli-med\demo\*' -Destination 'D:\a\rust-cli-med\rust-cli-med\target\release\demo\' -Recurse -Force 103 | # tar --directory=target/release -cf med-0.6.2-x86_64-windows-msvc.tar.gz med.exe 104 | Compress-Archive -Path D:\a\rust-cli-med\rust-cli-med\target\release\med.exe -DestinationPath D:\a\rust-cli-med\rust-cli-med\med-${{ needs.crate_metadata.outputs.version }}-x86_64-windows-msvc.zip 105 | # Get-ChildItem 106 | - name: Upload binaries to release 107 | uses: softprops/action-gh-release@v1 108 | with: 109 | tag_name: ${{ needs.crate_metadata.outputs.version }} 110 | files: 'med-${{ needs.crate_metadata.outputs.version }}-x86_64-windows-msvc.zip' 111 | env: 112 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 113 | 114 | macos-X86_64-release: 115 | name: release macos x86_64 release 116 | runs-on: macos-latest 117 | needs: [crate_metadata, unit-test] 118 | steps: 119 | - uses: actions/checkout@master 120 | - name: check toolchain 121 | run: rustup default 122 | - name: Build 123 | run: | 124 | cargo build --release 125 | - name: tar 126 | run: | 127 | cp -R demo target/release/ 128 | mkdir target/release/med-${{ needs.crate_metadata.outputs.version }} 129 | mv target/release/med target/release/med-${{ needs.crate_metadata.outputs.version }} 130 | tar --directory=target/release -cf med-macos_x86_64-archive-${{ needs.crate_metadata.outputs.version }}.tar.gz med-${{ needs.crate_metadata.outputs.version }} demo/ 131 | - name: Upload binaries x86_64 to release 132 | uses: softprops/action-gh-release@v1 133 | with: 134 | tag_name: ${{ needs.crate_metadata.outputs.version }} 135 | files: 'med-macos_x86_64-archive-${{ needs.crate_metadata.outputs.version }}.tar.gz' 136 | env: 137 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 138 | 139 | macos-arm-release: 140 | name: release macos arm release 141 | runs-on: macos-latest 142 | needs: [crate_metadata, unit-test] 143 | steps: 144 | - uses: actions/checkout@master 145 | - name: check toolchain 146 | run: rustup default 147 | - name: Build 148 | run: | 149 | rustup toolchain install stable-aarch64-apple-darwin 150 | rustup target add aarch64-apple-darwin 151 | cargo build --release --target aarch64-apple-darwin 152 | - name: tar 153 | run: | 154 | cp -R demo target/aarch64-apple-darwin/release 155 | mkdir target/aarch64-apple-darwin/release/med-${{ needs.crate_metadata.outputs.version }} 156 | mv target/aarch64-apple-darwin/release/med target/aarch64-apple-darwin/release/med-${{ needs.crate_metadata.outputs.version }} 157 | tar --directory=target/aarch64-apple-darwin/release -cf macos_arm_archive-${{ needs.crate_metadata.outputs.version }}.tar.gz med-${{ needs.crate_metadata.outputs.version }} demo/ 158 | - name: Upload binaries arm to release 159 | uses: softprops/action-gh-release@v1 160 | with: 161 | tag_name: ${{ needs.crate_metadata.outputs.version }} 162 | files: 'macos_arm_archive-${{ needs.crate_metadata.outputs.version }}.tar.gz' 163 | env: 164 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 165 | 166 | build_release_notes: 167 | name: build release notes 168 | runs-on: ubuntu-latest 169 | needs: [crate_metadata, unit-test, macos-arm-release, macos-X86_64-release, windows-msvc-release, build-release] 170 | steps: 171 | - name: Checkout 172 | uses: actions/checkout@v3 173 | - name: "Build Changelog" 174 | id: build_changelog 175 | uses: mikepenz/release-changelog-builder-action@v4 176 | with: 177 | configuration: "./build/changelog.config.json" 178 | - name: Upload binaries arm to release 179 | uses: softprops/action-gh-release@v1 180 | with: 181 | tag_name: ${{ needs.crate_metadata.outputs.version }} 182 | body: ${{ steps.build_changelog.outputs.changelog }} 183 | env: 184 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /demo/data/input/json/level/generated.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "646ae336271a76d64e27c4db", 4 | "index": 0, 5 | "guid": "6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e", 6 | "isActive": false, 7 | "balance": "$1,429.90", 8 | "picture": "http://placehold.it/32x32", 9 | "age": 23, 10 | "eyeColor": "green", 11 | "name": "Joyce Woods", 12 | "gender": "female", 13 | "company": "PARLEYNET", 14 | "email": "joycewoods@parleynet.com", 15 | "phone": "+1 (883) 513-3787", 16 | "address": "527 Commercial Street, Bowmansville, Missouri, 5038", 17 | "about": "Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n", 18 | "registered": "2022-05-23T03:46:20 +04:00", 19 | "latitude": -11.6502, 20 | "longitude": -113.585217, 21 | "tags": [ 22 | "tempor", 23 | "enim", 24 | "quis", 25 | "nulla", 26 | "culpa", 27 | "adipisicing", 28 | "irure" 29 | ], 30 | "test": { 31 | "name": "rust king", 32 | "item": "adfad" 33 | }, 34 | "friends": [ 35 | { 36 | "id": 0, 37 | "name": "Marci Pollard" 38 | }, 39 | { 40 | "id": 1, 41 | "name": "Mcconnell Knight" 42 | }, 43 | { 44 | "id": 2, 45 | "name": "Cummings Quinn" 46 | } 47 | ], 48 | "greeting": "Hello, Joyce Woods! You have 7 unread messages.", 49 | "favoriteFruit": "apple" 50 | }, 51 | { 52 | "_id": "646ae336271a76d64e27c4db", 53 | "index": 0, 54 | "guid": "6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e", 55 | "isActive": false, 56 | "balance": "$1,429.90", 57 | "picture": "http://placehold.it/32x32", 58 | "age": 23, 59 | "eyeColor": "green", 60 | "name": "Joyce Woods", 61 | "gender": "female", 62 | "company": "PARLEYNET", 63 | "email": "joycewoods@parleynet.com", 64 | "phone": "+1 (883) 513-3787", 65 | "address": "527 Commercial Street, Bowmansville, Missouri, 5038", 66 | "about": "Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n", 67 | "registered": "2022-05-23T03:46:20 +04:00", 68 | "latitude": -11.6502, 69 | "longitude": -113.585217, 70 | "tags": [ 71 | "tempor", 72 | "enim", 73 | "quis", 74 | "nulla", 75 | "culpa", 76 | "adipisicing", 77 | "irure" 78 | ], 79 | "test": { 80 | "name": "rust king", 81 | "item": "adfad" 82 | }, 83 | "friends": [ 84 | { 85 | "id": 0, 86 | "name": "Marci Pollard" 87 | }, 88 | { 89 | "id": 1, 90 | "name": "Mcconnell Knight" 91 | }, 92 | { 93 | "id": 2, 94 | "name": "Cummings Quinn" 95 | } 96 | ], 97 | "greeting": "Hello, Joyce Woods! You have 7 unread messages.", 98 | "favoriteFruit": "apple" 99 | }, 100 | { 101 | "_id": "646ae336271a76d64e27c4db", 102 | "index": 0, 103 | "guid": "6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e", 104 | "isActive": false, 105 | "balance": "$1,429.90", 106 | "picture": "http://placehold.it/32x32", 107 | "age": 23, 108 | "eyeColor": "green", 109 | "name": "Joyce Woods", 110 | "gender": "female", 111 | "company": "PARLEYNET", 112 | "email": "joycewoods@parleynet.com", 113 | "phone": "+1 (883) 513-3787", 114 | "address": "527 Commercial Street, Bowmansville, Missouri, 5038", 115 | "about": "Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n", 116 | "registered": "2022-05-23T03:46:20 +04:00", 117 | "latitude": -11.6502, 118 | "longitude": -113.585217, 119 | "tags": [ 120 | "tempor", 121 | "enim", 122 | "quis", 123 | "nulla", 124 | "culpa", 125 | "adipisicing", 126 | "irure" 127 | ], 128 | "test": { 129 | "name": "rust king", 130 | "item": "adfad" 131 | }, 132 | "friends": [ 133 | { 134 | "id": 0, 135 | "name": "Marci Pollard" 136 | }, 137 | { 138 | "id": 1, 139 | "name": "Mcconnell Knight" 140 | }, 141 | { 142 | "id": 2, 143 | "name": "Cummings Quinn" 144 | } 145 | ], 146 | "greeting": "Hello, Joyce Woods! You have 7 unread messages.", 147 | "favoriteFruit": "apple" 148 | }, 149 | { 150 | "_id": "646ae336271a76d64e27c4db", 151 | "index": 0, 152 | "guid": "6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e", 153 | "isActive": false, 154 | "balance": "$1,429.90", 155 | "picture": "http://placehold.it/32x32", 156 | "age": 23, 157 | "eyeColor": "green", 158 | "name": "Joyce Woods", 159 | "gender": "female", 160 | "company": "PARLEYNET", 161 | "email": "joycewoods@parleynet.com", 162 | "phone": "+1 (883) 513-3787", 163 | "address": "527 Commercial Street, Bowmansville, Missouri, 5038", 164 | "about": "Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n", 165 | "registered": "2022-05-23T03:46:20 +04:00", 166 | "latitude": -11.6502, 167 | "longitude": -113.585217, 168 | "tags": [ 169 | "tempor", 170 | "enim", 171 | "quis", 172 | "nulla", 173 | "culpa", 174 | "adipisicing", 175 | "irure" 176 | ], 177 | "test": { 178 | "name": "rust king", 179 | "item": "adfad" 180 | }, 181 | "friends": [ 182 | { 183 | "id": 0, 184 | "name": "Marci Pollard" 185 | }, 186 | { 187 | "id": 1, 188 | "name": "Mcconnell Knight" 189 | }, 190 | { 191 | "id": 2, 192 | "name": "Cummings Quinn" 193 | } 194 | ], 195 | "greeting": "Hello, Joyce Woods! You have 7 unread messages.", 196 | "favoriteFruit": "apple" 197 | }, 198 | { 199 | "_id": "646ae336271a76d64e27c4db", 200 | "index": 0, 201 | "guid": "6e7faf85-6416-4e5b-9511-d8e8d3f8aa5e", 202 | "isActive": false, 203 | "balance": "$1,429.90", 204 | "picture": "http://placehold.it/32x32", 205 | "age": 23, 206 | "eyeColor": "green", 207 | "name": "Joyce Woods", 208 | "gender": "female", 209 | "company": "PARLEYNET", 210 | "email": "joycewoods@parleynet.com", 211 | "phone": "+1 (883) 513-3787", 212 | "address": "527 Commercial Street, Bowmansville, Missouri, 5038", 213 | "about": "Magna amet anim officia incididunt ea culpa exercitation laboris ad mollit sit id qui. Dolor commodo excepteur sint ut in quis irure exercitation ullamco anim dolore consectetur dolor. Mollit incididunt anim culpa anim est id culpa sunt ad reprehenderit. Aliquip elit laboris magna consequat sunt et. Incididunt proident officia sint eiusmod minim mollit ea ex in laborum quis.\r\n", 214 | "registered": "2022-05-23T03:46:20 +04:00", 215 | "latitude": -11.6502, 216 | "longitude": -113.585217, 217 | "tags": [ 218 | "tempor", 219 | "enim", 220 | "quis", 221 | "nulla", 222 | "culpa", 223 | "adipisicing", 224 | "irure" 225 | ], 226 | "test": { 227 | "name": "rust king", 228 | "item": "adfad" 229 | }, 230 | "friends": [ 231 | { 232 | "id": 0, 233 | "name": "Marci Pollard" 234 | }, 235 | { 236 | "id": 1, 237 | "name": "Mcconnell Knight" 238 | }, 239 | { 240 | "id": 2, 241 | "name": "Cummings Quinn" 242 | } 243 | ], 244 | "greeting": "Hello, Joyce Woods! You have 7 unread messages.", 245 | "favoriteFruit": "apple" 246 | } 247 | ] 248 | -------------------------------------------------------------------------------- /med_core/src/app/json.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | use std::fs::File; 3 | use std::io::Write; 4 | 5 | use crate::{ 6 | models::{enums::Mode, metrics::Metadata}, 7 | utils::error::{MedError, MedErrorType}, 8 | }; 9 | 10 | use crate::app::processor::ProcessRuntime; 11 | 12 | pub fn json_processor( 13 | tx_metadata: flume::Sender, 14 | files_path: &str, 15 | output_path: &str, 16 | process_runtime: ProcessRuntime, 17 | ) -> Result<(), MedError> { 18 | // prepare the metrics 19 | let mut total_records: usize = 0; 20 | let mut failed_records: usize = 0; 21 | let mut record_failed_reason: Vec = Vec::new(); 22 | 23 | match std::fs::read_to_string(files_path) { 24 | Ok(text) => match serde_json::from_str::(&text) { 25 | Ok(data) => { 26 | if data.is_array() { 27 | total_records = data.as_array().unwrap().len(); 28 | } else { 29 | total_records = 1; 30 | } 31 | let mut json_data = data; 32 | let new_json_data = json_med_core(&mut json_data, &process_runtime); 33 | write_json(&new_json_data, output_path).unwrap(); 34 | } 35 | Err(err) => { 36 | let record_error = MedError { 37 | message: Some(format!( 38 | "please check {} {:?} format", 39 | files_path, process_runtime.mode 40 | )), 41 | cause: Some(err.to_string()), 42 | error_type: MedErrorType::CsvError, 43 | }; 44 | record_failed_reason.push(record_error); 45 | failed_records += 1; 46 | } 47 | }, 48 | Err(err) => { 49 | let record_error = MedError { 50 | message: Some(format!( 51 | "please check {} {:?} format", 52 | files_path, process_runtime.mode 53 | )), 54 | cause: Some(err.to_string()), 55 | error_type: MedErrorType::CsvError, 56 | }; 57 | record_failed_reason.push(record_error); 58 | failed_records += 1; 59 | } 60 | } 61 | 62 | tx_metadata 63 | .send(Metadata { 64 | total_records, 65 | failed_records, 66 | record_failed_reason, 67 | }) 68 | .unwrap(); 69 | 70 | Ok(()) 71 | } 72 | 73 | fn json_med_core(value: &mut Value, process_runtime: &ProcessRuntime) -> Value { 74 | match value { 75 | Value::Array(arr) => { 76 | // debug!("[arr] {:?}", arr); 77 | for item in arr { 78 | if item.is_array() { 79 | json_med_core(item, process_runtime); 80 | } 81 | 82 | if item.is_object() { 83 | // info!("is obj {:?} ", val); 84 | item.as_object_mut() 85 | .unwrap() 86 | .into_iter() 87 | .for_each(|(key, val)| { 88 | //debug!("key: {:?}, val: {:?} ", key, val); 89 | //mask parent lvl 90 | if process_runtime.fields.contains(key) { 91 | if let Value::String(mut masked_val) = val.to_owned() { 92 | match process_runtime.mode { 93 | Mode::MASK => { 94 | masked_val.clear(); 95 | let symbols = 96 | process_runtime.to_owned().mask_symbols.unwrap(); 97 | masked_val.push_str(&symbols); 98 | } 99 | Mode::ENCRYPT => { 100 | if let Some(cypher) = process_runtime.to_owned().cypher 101 | { 102 | if let Some(standard) = process_runtime.standard { 103 | let masked = cypher 104 | .encrypt(&masked_val, &standard) 105 | .unwrap(); 106 | masked_val.clear(); 107 | masked_val.push_str(&masked); 108 | } 109 | } 110 | } 111 | Mode::DECRYPT => { 112 | if let Some(cypher) = process_runtime.to_owned().cypher 113 | { 114 | if let Some(standard) = process_runtime.standard { 115 | let masked = cypher 116 | .decrypt(&masked_val, &standard) 117 | .unwrap(); 118 | masked_val.clear(); 119 | masked_val.push_str(&masked); 120 | } 121 | } 122 | } 123 | } 124 | *val = Value::String(masked_val); 125 | } 126 | } 127 | 128 | if val.is_array() { 129 | json_med_core(val, process_runtime); 130 | } 131 | 132 | if val.is_object() { 133 | json_med_core(val, process_runtime); 134 | } 135 | }); 136 | } 137 | } 138 | } 139 | Value::Object(obj) => { 140 | for (key, val) in obj { 141 | // debug!("key : {:?}, val: {:?}", key, val); 142 | if val.is_array() { 143 | json_med_core(val, process_runtime); 144 | } 145 | if process_runtime.fields.contains(key) { 146 | if let Value::String(mut masked_val) = val.to_owned() { 147 | match process_runtime.mode { 148 | Mode::MASK => { 149 | masked_val.clear(); 150 | masked_val 151 | .push_str(&process_runtime.to_owned().mask_symbols.unwrap()); 152 | } 153 | Mode::ENCRYPT => { 154 | if let Some(cypher) = process_runtime.to_owned().cypher { 155 | if let Some(standard) = process_runtime.standard { 156 | let masked = 157 | cypher.encrypt(&masked_val, &standard).unwrap(); 158 | masked_val.clear(); 159 | masked_val.push_str(&masked); 160 | } 161 | } 162 | } 163 | Mode::DECRYPT => { 164 | if let Some(cypher) = process_runtime.to_owned().cypher { 165 | if let Some(standard) = process_runtime.standard { 166 | let masked = 167 | cypher.decrypt(&masked_val, &standard).unwrap(); 168 | masked_val.clear(); 169 | masked_val.push_str(&masked); 170 | } 171 | } 172 | } 173 | } 174 | *val = Value::String(masked_val); 175 | } 176 | } 177 | } 178 | } 179 | _ => {} 180 | } 181 | value.clone() 182 | } 183 | 184 | pub fn write_json(masked_data: &Value, output_file: &str) -> Result<(), MedError> { 185 | let mut json_file = File::create(output_file)?; 186 | let data = serde_json::to_string(masked_data)?; 187 | json_file.write_all(data.as_bytes())?; 188 | json_file.sync_data()?; 189 | Ok(()) 190 | } 191 | 192 | #[cfg(test)] 193 | #[path = "../tests/json_test.rs"] 194 | mod json_test; 195 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------