├── .gitignore ├── fuzz ├── .gitignore ├── Cargo.toml └── fuzz_targets │ └── fuzz_employee_db.rs ├── .github ├── FUNDING.yml └── workflows │ ├── static.yml │ └── CI.yml ├── www ├── venndb_social_media.jpeg ├── style.css └── index.html ├── docs └── img │ └── plabayo_mech_store_venndb.png ├── venndb-usage ├── tests │ ├── fails │ │ ├── derive_tuple_struct.rs │ │ ├── derive_enum.rs │ │ ├── derive_enum.stderr │ │ ├── any_bool.rs │ │ ├── unknown_attr_struct.rs │ │ ├── any_key.rs │ │ ├── key_any.rs │ │ ├── lonely_any.rs │ │ ├── option_key.rs │ │ ├── unknown_attr_field.rs │ │ ├── any_filter_bool.rs │ │ ├── filter_any_bool.rs │ │ ├── derive_tuple_struct.stderr │ │ ├── any_bool.stderr │ │ ├── option_key.stderr │ │ ├── unknown_attr_struct.stderr │ │ ├── key_any.stderr │ │ ├── unknown_attr_field.stderr │ │ ├── lonely_any.stderr │ │ ├── any_filter_bool.stderr │ │ ├── filter_any_bool.stderr │ │ ├── filter_and_key.stderr │ │ ├── key_and_filter.stderr │ │ ├── filter_and_key.rs │ │ ├── key_and_filter.rs │ │ ├── derive_struct_with_validator_str.stderr │ │ ├── filter_skipped_field.stderr │ │ ├── any_key.stderr │ │ ├── filter_skipped_field.rs │ │ └── derive_struct_with_validator_str.rs │ ├── compiles │ │ ├── derive_struct_empty.rs │ │ ├── filter_and_key_and_skip.rs │ │ ├── derive_struct.rs │ │ ├── derive_struct_skip_all.rs │ │ ├── derive_struct_with_key.rs │ │ ├── derive_struct_custom_name.rs │ │ ├── derive_struct_with_filter_map.rs │ │ ├── derive_struct_with_explicit_filters.rs │ │ ├── derive_struct_with_filter_map_any.rs │ │ ├── derive_struct_with_validator.rs │ │ └── derive_struct_all_the_things.rs │ └── compilation_tests.rs └── Cargo.toml ├── venndb-macros ├── README.md ├── Cargo.toml └── src │ ├── lib.rs │ ├── field.rs │ ├── errors.rs │ └── parse_attrs.rs ├── deny.toml ├── SECURITY.md ├── LICENSE-MIT ├── justfile ├── src ├── lib.rs └── bitvec.rs ├── scripts ├── generate_proxies.py └── plot_bench_charts.py ├── CONTRIBUTING.md ├── Cargo.toml ├── benches ├── proxydb.rs └── proxies │ └── mod.rs ├── CODE_OF_CONDUCT.md ├── CHANGELOG.md ├── LICENSE-APACHE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | corpus 3 | artifacts 4 | coverage 5 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: plabayo 4 | -------------------------------------------------------------------------------- /www/venndb_social_media.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/plabayo/venndb/HEAD/www/venndb_social_media.jpeg -------------------------------------------------------------------------------- /docs/img/plabayo_mech_store_venndb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/plabayo/venndb/HEAD/docs/img/plabayo_mech_store_venndb.png -------------------------------------------------------------------------------- /venndb-usage/tests/fails/derive_tuple_struct.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(VennDB)] 4 | struct MyStruct(u32); 5 | 6 | fn main() {} 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/derive_struct_empty.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee {} 5 | 6 | fn main() {} 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/derive_enum.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(VennDB)] 4 | enum MyEnum { 5 | A, 6 | B, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/derive_enum.stderr: -------------------------------------------------------------------------------- 1 | error: `#[derive(VennDB)]` cannot be applied to enums 2 | --> tests/fails/derive_enum.rs:4:1 3 | | 4 | 4 | enum MyEnum { 5 | | ^^^^ 6 | -------------------------------------------------------------------------------- /venndb-macros/README.md: -------------------------------------------------------------------------------- 1 | ## venndb-macros 2 | 3 | Macros crate for [`venndb`](https://crates.io/crates/venndb), not to be used directly. 4 | 5 | Repository: https://github.com/plabayo/venndb 6 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/any_bool.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(any)] 6 | is_alive: bool, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/unknown_attr_struct.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | #[venndb(foo)] 5 | struct Employee { 6 | id: u32, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/any_key.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(any, key)] 6 | country: String, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/key_any.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(key, any)] 6 | country: String, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/lonely_any.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(any)] 6 | country: String, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/option_key.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(key)] 6 | id: Option, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/unknown_attr_field.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(foo)] 6 | id: u32, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/any_filter_bool.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(any, filter)] 6 | is_alive: bool, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/filter_any_bool.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(filter, any)] 6 | is_alive: bool, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/derive_tuple_struct.stderr: -------------------------------------------------------------------------------- 1 | error: `#![derive(VennDB)]` is not currently supported on tuple structs 2 | --> tests/fails/derive_tuple_struct.rs:4:1 3 | | 4 | 4 | struct MyStruct(u32); 5 | | ^^^^^^ 6 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/any_bool.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | `any` cannot be used with `bool` 3 | --> tests/fails/any_bool.rs:6:15 4 | | 5 | 6 | is_alive: bool, 6 | | ^^^^ 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/option_key.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | `key` fields cannot be `Option` 3 | --> tests/fails/option_key.rs:6:9 4 | | 5 | 6 | id: Option, 6 | | ^^^^^^ 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/unknown_attr_struct.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | Expected one of: `name` 3 | --> tests/fails/unknown_attr_struct.rs:4:10 4 | | 5 | 4 | #[venndb(foo)] 6 | | ^^^ 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/key_any.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | Cannot have both `key` and `any` 3 | --> tests/fails/key_any.rs:5:19 4 | | 5 | 5 | #[venndb(key, any)] 6 | | ^^^ 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/unknown_attr_field.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | Expected one of: `key` 3 | --> tests/fails/unknown_attr_field.rs:5:14 4 | | 5 | 5 | #[venndb(foo)] 6 | | ^^^ 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/lonely_any.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | `any` can only be used with `filter` 3 | --> tests/fails/lonely_any.rs:6:14 4 | | 5 | 6 | country: String, 6 | | ^^^^^^ 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/any_filter_bool.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | `any` cannot be used with `bool` 3 | --> tests/fails/any_filter_bool.rs:6:15 4 | | 5 | 6 | is_alive: bool, 6 | | ^^^^ 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/filter_any_bool.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | `any` cannot be used with `bool` 3 | --> tests/fails/filter_any_bool.rs:6:15 4 | | 5 | 6 | is_alive: bool, 6 | | ^^^^ 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/filter_and_key.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | Cannot have both `key` and `filter` 3 | --> tests/fails/filter_and_key.rs:8:22 4 | | 5 | 8 | #[venndb(filter, key)] 6 | | ^^^ 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/key_and_filter.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | Cannot have both `key` and `filter` 3 | --> tests/fails/key_and_filter.rs:8:19 4 | | 5 | 8 | #[venndb(key, filter)] 6 | | ^^^^^^ 7 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/filter_and_key.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | id: u32, 6 | is_manager: bool, 7 | is_active: bool, 8 | #[venndb(filter, key)] 9 | country: String, 10 | } 11 | 12 | fn main() {} 13 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/key_and_filter.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | id: u32, 6 | is_manager: bool, 7 | is_active: bool, 8 | #[venndb(key, filter)] 9 | country: String, 10 | } 11 | 12 | fn main() {} 13 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/filter_and_key_and_skip.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | id: u32, 6 | is_manager: bool, 7 | is_active: bool, 8 | #[venndb(filter, key, skip)] 9 | country: String, 10 | } 11 | 12 | fn main() {} 13 | -------------------------------------------------------------------------------- /venndb-usage/tests/compilation_tests.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn should_compile() { 3 | let t = trybuild::TestCases::new(); 4 | t.pass("tests/compiles/*.rs"); 5 | } 6 | 7 | #[test] 8 | fn should_not_compile() { 9 | let t = trybuild::TestCases::new(); 10 | t.compile_fail("tests/fails/*.rs"); 11 | } 12 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/derive_struct_with_validator_str.stderr: -------------------------------------------------------------------------------- 1 | error: Expected path attribute, found "employee_validator" attribute (literal) 2 | --> tests/fails/derive_struct_with_validator_str.rs:4:22 3 | | 4 | 4 | #[venndb(validator = "employee_validator")] 5 | | ^^^^^^^^^^^^^^^^^^^^ 6 | -------------------------------------------------------------------------------- /venndb-usage/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "venndb-usage" 3 | version = { workspace = true } 4 | rust-version = { workspace = true } 5 | edition = { workspace = true } 6 | publish = false 7 | 8 | [dependencies] 9 | venndb = { workspace = true } 10 | 11 | [dev-dependencies] 12 | trybuild = { workspace = true } 13 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/filter_skipped_field.stderr: -------------------------------------------------------------------------------- 1 | error[E0599]: no method named `is_active` found for struct `EmployeeDBQuery<'a>` in the current scope 2 | --> tests/fails/filter_skipped_field.rs:20:11 3 | | 4 | 3 | #[derive(Debug, VennDB)] 5 | | ------ method `is_active` not found for this struct 6 | ... 7 | 20 | query.is_active(true); 8 | | ^^^^^^^^^ method not found in `EmployeeDBQuery<'_>` 9 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/any_key.stderr: -------------------------------------------------------------------------------- 1 | error: Invalid field-level `venndb` attribute 2 | Cannot have both `key` and `any` 3 | --> tests/fails/any_key.rs:5:19 4 | | 5 | 5 | #[venndb(any, key)] 6 | | ^^^ 7 | 8 | error: Invalid field-level `venndb` attribute 9 | `any` can only be used with `filter` 10 | --> tests/fails/any_key.rs:6:14 11 | | 12 | 6 | country: String, 13 | | ^^^^^^ 14 | -------------------------------------------------------------------------------- /fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "venndb-fuzz" 3 | version = "0.0.0" 4 | publish = false 5 | edition = "2021" 6 | 7 | [package.metadata] 8 | cargo-fuzz = true 9 | 10 | [[bin]] 11 | name = "fuzz_employee_db" 12 | path = "fuzz_targets/fuzz_employee_db.rs" 13 | test = false 14 | doc = false 15 | bench = false 16 | 17 | [dependencies] 18 | libfuzzer-sys = { version = "0.4", features = ["arbitrary-derive"] } 19 | 20 | [dependencies.venndb] 21 | path = ".." 22 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/derive_struct.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | id: u32, 6 | name: String, 7 | is_manager: bool, 8 | is_admin: bool, 9 | is_active: bool, 10 | department: Department, 11 | } 12 | 13 | #[derive(Debug)] 14 | pub enum Department { 15 | Engineering, 16 | Sales, 17 | Marketing, 18 | HR, 19 | } 20 | 21 | fn main() { 22 | let _ = EmployeeDB::new(); 23 | } 24 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/derive_struct_skip_all.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(skip)] 6 | id: u32, 7 | #[venndb(skip)] 8 | name: String, 9 | #[venndb(skip)] 10 | is_manager: bool, 11 | #[venndb(skip)] 12 | department: Department, 13 | } 14 | 15 | #[derive(Debug)] 16 | pub enum Department { 17 | Engineering, 18 | Sales, 19 | Marketing, 20 | HR, 21 | } 22 | 23 | fn main() {} 24 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/derive_struct_with_key.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(key)] 6 | id: u32, 7 | name: String, 8 | is_manager: bool, 9 | is_admin: bool, 10 | is_active: bool, 11 | department: Department, 12 | } 13 | 14 | #[derive(Debug)] 15 | pub enum Department { 16 | Engineering, 17 | Sales, 18 | Marketing, 19 | HR, 20 | } 21 | 22 | fn main() { 23 | let _ = EmployeeDB::new(); 24 | } 25 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/derive_struct_custom_name.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | #[venndb(name = "Database")] 5 | struct Employee { 6 | id: u32, 7 | name: String, 8 | is_manager: bool, 9 | is_admin: bool, 10 | is_active: bool, 11 | department: Department, 12 | } 13 | 14 | #[derive(Debug)] 15 | pub enum Department { 16 | Engineering, 17 | Sales, 18 | Marketing, 19 | HR, 20 | } 21 | 22 | fn main() { 23 | let _ = Database::new(); 24 | } 25 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/filter_skipped_field.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | id: u32, 6 | is_manager: bool, 7 | #[venndb(skip)] 8 | is_active: bool, 9 | } 10 | 11 | fn main() { 12 | let mut db = EmployeeDB::new(); 13 | db.append(Employee { 14 | id: 1, 15 | is_manager: true, 16 | is_active: true, 17 | }); 18 | 19 | let mut query = db.query(); 20 | query.is_active(true); 21 | assert!(query.execute().is_some()); 22 | } 23 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/derive_struct_with_filter_map.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | id: u32, 6 | name: String, 7 | is_manager: bool, 8 | is_admin: bool, 9 | is_active: bool, 10 | #[venndb(filter)] 11 | department: Department, 12 | } 13 | 14 | #[derive(Debug, PartialEq, Eq, Hash, Clone)] 15 | pub enum Department { 16 | Engineering, 17 | Sales, 18 | Marketing, 19 | HR, 20 | } 21 | 22 | fn main() { 23 | let _ = EmployeeDB::new(); 24 | } 25 | -------------------------------------------------------------------------------- /deny.toml: -------------------------------------------------------------------------------- 1 | [advisories] 2 | version = 2 3 | ignore = [] 4 | 5 | [licenses] 6 | confidence-threshold = 0.93 7 | allow = [ 8 | "Apache-2.0", 9 | "Apache-2.0 WITH LLVM-exception", 10 | "MIT", 11 | "MPL-2.0", 12 | "BSD-3-Clause", 13 | "ISC", 14 | "Unicode-3.0", 15 | "OpenSSL", 16 | "Zlib", 17 | "CDLA-Permissive-2.0", 18 | ] 19 | 20 | [bans] 21 | multiple-versions = "warn" 22 | highlight = "all" 23 | skip-tree = [] 24 | 25 | [sources] 26 | unknown-registry = "warn" 27 | unknown-git = "warn" 28 | allow-git = [] 29 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/derive_struct_with_explicit_filters.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | #[venndb(key)] 6 | id: u32, 7 | name: String, 8 | #[venndb(filter)] 9 | is_manager: bool, 10 | #[venndb(filter)] 11 | is_admin: bool, 12 | #[venndb(filter)] 13 | is_active: bool, 14 | department: Department, 15 | } 16 | 17 | #[derive(Debug)] 18 | pub enum Department { 19 | Engineering, 20 | Sales, 21 | Marketing, 22 | HR, 23 | } 24 | 25 | fn main() { 26 | let _ = EmployeeDB::new(); 27 | } 28 | -------------------------------------------------------------------------------- /venndb-usage/tests/fails/derive_struct_with_validator_str.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | #[venndb(validator = "employee_validator")] 5 | struct Employee { 6 | pub id: u32, 7 | pub name: String, 8 | pub is_manager: bool, 9 | pub is_admin: bool, 10 | pub is_active: bool, 11 | pub department: Department, 12 | } 13 | 14 | #[derive(Debug)] 15 | pub enum Department { 16 | Engineering, 17 | Sales, 18 | Marketing, 19 | HR, 20 | } 21 | 22 | fn employee_validator(employee: &Employee) -> bool { 23 | employee.id > 0 && !employee.name.is_empty() 24 | } 25 | 26 | fn main() { 27 | let _ = EmployeeDB::new(); 28 | } 29 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/derive_struct_with_filter_map_any.rs: -------------------------------------------------------------------------------- 1 | use venndb::{Any, VennDB}; 2 | 3 | #[derive(Debug, VennDB)] 4 | struct Employee { 5 | id: u32, 6 | name: String, 7 | is_manager: bool, 8 | is_admin: bool, 9 | is_active: bool, 10 | #[venndb(filter, any)] 11 | department: Department, 12 | } 13 | 14 | #[derive(Debug, PartialEq, Eq, Hash, Clone)] 15 | pub enum Department { 16 | Any, 17 | Engineering, 18 | Sales, 19 | Marketing, 20 | HR, 21 | } 22 | 23 | impl Any for Department { 24 | fn is_any(&self) -> bool { 25 | self == &Department::Any 26 | } 27 | } 28 | 29 | fn main() { 30 | let _ = EmployeeDB::new(); 31 | } 32 | -------------------------------------------------------------------------------- /venndb-macros/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | description = "macros for venndb, cannot be used directly" 3 | edition = { workspace = true } 4 | homepage = { workspace = true } 5 | license = { workspace = true } 6 | name = "venndb-macros" 7 | readme = "README.md" 8 | repository = { workspace = true } 9 | keywords = ["database", "db", "memory", "bits"] 10 | categories = ["database"] 11 | authors = { workspace = true } 12 | version = { workspace = true } 13 | rust-version = { workspace = true } 14 | 15 | [package.metadata.docs.rs] 16 | all-features = true 17 | rustdoc-args = ["--cfg", "docsrs"] 18 | 19 | [lib] 20 | proc-macro = true 21 | 22 | [dependencies] 23 | proc-macro2 = { workspace = true } 24 | quote = { workspace = true } 25 | syn = { workspace = true } 26 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/derive_struct_with_validator.rs: -------------------------------------------------------------------------------- 1 | use venndb::VennDB; 2 | 3 | #[derive(Debug, VennDB)] 4 | #[venndb(validator = sealed::employee_validator)] 5 | struct Employee { 6 | pub id: u32, 7 | pub name: String, 8 | pub is_manager: bool, 9 | pub is_admin: bool, 10 | pub is_active: bool, 11 | pub department: Department, 12 | } 13 | 14 | #[derive(Debug)] 15 | pub enum Department { 16 | Engineering, 17 | Sales, 18 | Marketing, 19 | HR, 20 | } 21 | 22 | mod sealed { 23 | use super::Employee; 24 | 25 | pub(super) fn employee_validator(employee: &Employee) -> bool { 26 | employee.id > 0 && !employee.name.is_empty() 27 | } 28 | } 29 | 30 | fn main() { 31 | let _ = EmployeeDB::new(); 32 | } 33 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported | 6 | |----------|--------------------| 7 | | >= 0.0.0 | :white_check_mark: | 8 | 9 | ## Reporting a Vulnerability 10 | 11 | If you discover a vulnerability, please do the following: 12 | 13 | - E-mail your findings to security [at] plabayo [dot] tech. 14 | - Do not take advantage of the vulnerability or problem you have discovered, for example by downloading more data than necessary to demonstrate the vulnerability or deleting or modifying other people's data. 15 | - Do not reveal the problem to others until it has been resolved. 16 | - Do not use attacks on physical security, social engineering, distributed denial of service, spam or applications of third parties. 17 | - Do provide sufficient information to reproduce the problem, so we will be able to resolve it as quickly as possible. Complex vulnerabilities may require further explanation! 18 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/fuzz_employee_db.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use libfuzzer_sys::arbitrary::{self, Arbitrary}; 4 | use libfuzzer_sys::fuzz_target; 5 | use venndb::{Any, VennDB}; 6 | 7 | #[derive(Clone, Debug, Arbitrary, VennDB)] 8 | pub struct Employee { 9 | #[venndb(key)] 10 | id: u16, 11 | _name: String, 12 | earth: bool, 13 | alive: Option, 14 | #[venndb(filter)] 15 | faction: Faction, 16 | #[venndb(filter, any)] 17 | planet: Option, 18 | } 19 | 20 | #[derive(Clone, Debug, Arbitrary, PartialEq, Eq, Hash)] 21 | pub enum Faction { 22 | Rebel, 23 | Empire, 24 | } 25 | 26 | #[derive(Clone, Debug, Arbitrary, PartialEq, Eq, Hash)] 27 | pub enum Planet { 28 | Any, 29 | Earth, 30 | Mars, 31 | } 32 | 33 | impl Any for Planet { 34 | fn is_any(&self) -> bool { 35 | self == &Planet::Any 36 | } 37 | } 38 | 39 | fuzz_target!(|rows: Vec| { 40 | let _ = EmployeeDB::from_rows(rows); 41 | }); 42 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 - Glen Henri J. De Cauwsemaecker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | set windows-shell := ["powershell.exe", "-NoLogo", "-Command"] 2 | 3 | export RUSTFLAGS := "-D warnings" 4 | export RUSTDOCFLAGS := "-D rustdoc::broken-intra-doc-links" 5 | export RUST_LOG := "debug" 6 | 7 | fmt: 8 | cargo fmt --all 9 | 10 | sort: 11 | cargo sort --workspace --grouped 12 | 13 | lint: fmt sort 14 | 15 | check: 16 | cargo check --all --all-targets --all-features 17 | 18 | clippy: 19 | cargo clippy --all --all-targets --all-features 20 | 21 | clippy-fix: 22 | cargo clippy --fix 23 | 24 | typos: 25 | typos -w 26 | 27 | doc: 28 | cargo doc --all-features --no-deps 29 | 30 | doc-open: 31 | cargo doc --all-features --no-deps --open 32 | 33 | hack: 34 | cargo hack check --each-feature --no-dev-deps --workspace 35 | 36 | test: 37 | cargo test --all-features --workspace 38 | 39 | qa: lint check clippy doc hack test 40 | 41 | watch-docs: 42 | cargo watch -x doc 43 | 44 | watch-check: 45 | cargo watch -x check -x test 46 | 47 | fuzz: 48 | cargo +nightly fuzz run fuzz_employee_db -- -max_len=131072 49 | 50 | fuzz-30s: 51 | cargo +nightly fuzz run fuzz_employee_db -- -max_len=131072 -max_total_time=60 52 | 53 | bench: 54 | cargo bench 55 | 56 | detect-unused-deps: 57 | @cargo install cargo-machete 58 | cargo machete --skip-target-dir 59 | 60 | update-deps: 61 | cargo upgrades 62 | cargo update 63 | 64 | publish: 65 | cargo publish -p venndb-macros 66 | cargo publish -p venndb 67 | -------------------------------------------------------------------------------- /.github/workflows/static.yml: -------------------------------------------------------------------------------- 1 | # Simple workflow for deploying static content to GitHub Pages 2 | name: Deploy static content to Pages 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | push: 7 | branches: ["main"] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 13 | permissions: 14 | contents: read 15 | pages: write 16 | id-token: write 17 | 18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 20 | concurrency: 21 | group: "pages" 22 | cancel-in-progress: false 23 | 24 | jobs: 25 | # Single deploy job since we're just deploying 26 | deploy: 27 | environment: 28 | name: github-pages 29 | url: ${{ steps.deployment.outputs.page_url }} 30 | runs-on: ubuntu-latest 31 | steps: 32 | - name: Checkout 33 | uses: actions/checkout@v4 34 | - name: Setup Pages 35 | uses: actions/configure-pages@v5 36 | - name: Upload artifact 37 | uses: actions/upload-pages-artifact@v3 38 | with: 39 | # Upload entire repository 40 | path: 'www' 41 | - name: Deploy to GitHub Pages 42 | id: deployment 43 | uses: actions/deploy-pages@v4 44 | -------------------------------------------------------------------------------- /venndb-usage/tests/compiles/derive_struct_all_the_things.rs: -------------------------------------------------------------------------------- 1 | use venndb::{Any, VennDB}; 2 | 3 | #[derive(Debug, VennDB)] 4 | #[venndb(name = "EmployeeSheet", validator = employee_validator)] 5 | struct Employee { 6 | #[venndb(key)] 7 | id: u32, 8 | name: String, 9 | #[venndb(filter)] // explicit bool filter == regular bool 10 | is_manager: bool, 11 | is_admin: bool, 12 | is_something: Option, 13 | #[venndb(skip)] 14 | is_active: bool, 15 | #[venndb(filter, any)] 16 | department: Department, 17 | #[venndb(filter)] 18 | country: Option, 19 | } 20 | 21 | #[derive(Debug, PartialEq, Eq, Clone, Hash)] 22 | pub enum Department { 23 | Any, 24 | Engineering, 25 | Sales, 26 | Marketing, 27 | HR, 28 | } 29 | 30 | fn employee_validator(employee: &Employee) -> bool { 31 | employee.id > 0 && !employee.name.is_empty() 32 | } 33 | 34 | impl Any for Department { 35 | fn is_any(&self) -> bool { 36 | self == &Department::Any 37 | } 38 | } 39 | 40 | fn main() { 41 | let mut db = EmployeeSheet::new(); 42 | db.append(Employee { 43 | id: 1, 44 | name: "Alice".to_string(), 45 | is_manager: true, 46 | is_admin: false, 47 | is_something: None, 48 | is_active: true, 49 | department: Department::Engineering, 50 | country: None, 51 | }) 52 | .unwrap(); 53 | 54 | let employee_ref = db.get_by_id(&1).unwrap(); 55 | assert_eq!(employee_ref.id, 1); 56 | assert_eq!(employee_ref.name, "Alice"); 57 | assert_eq!(employee_ref.is_something, None); 58 | assert_eq!(employee_ref.country, None); 59 | 60 | let mut query = db.query(); 61 | query 62 | .is_manager(true) 63 | .is_admin(true) 64 | .department(Department::Engineering) 65 | .department(Department::Sales); 66 | assert!(query.execute().is_none()); 67 | } 68 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | 3 | pub use venndb_macros::VennDB; 4 | 5 | /// A trait that types can implement in order to support `#[venndb(any)]` attribute filters. 6 | pub trait Any { 7 | /// Returns true if the value is considered to be "any" within the context of the type. 8 | /// 9 | /// # Example 10 | /// 11 | /// ``` 12 | /// use venndb::Any; 13 | /// 14 | /// #[derive(Debug)] 15 | /// struct MyString(String); 16 | /// 17 | /// impl Any for MyString { 18 | /// fn is_any(&self) -> bool { 19 | /// self.0 == "*" 20 | /// } 21 | /// } 22 | /// 23 | /// let my_string = MyString("*".to_string()); 24 | /// assert!(my_string.is_any()); 25 | /// 26 | /// let my_string = MyString("hello".to_string()); 27 | /// assert!(!my_string.is_any()); 28 | /// ``` 29 | fn is_any(&self) -> bool; 30 | } 31 | 32 | impl Any for &T { 33 | fn is_any(&self) -> bool { 34 | T::is_any(*self) 35 | } 36 | } 37 | 38 | impl Any for Option { 39 | fn is_any(&self) -> bool { 40 | match self { 41 | Some(value) => value.is_any(), 42 | None => false, 43 | } 44 | } 45 | } 46 | 47 | impl Any for std::sync::Arc { 48 | fn is_any(&self) -> bool { 49 | T::is_any(&**self) 50 | } 51 | } 52 | 53 | impl Any for std::rc::Rc { 54 | fn is_any(&self) -> bool { 55 | T::is_any(&**self) 56 | } 57 | } 58 | 59 | impl Any for Box { 60 | fn is_any(&self) -> bool { 61 | T::is_any(&**self) 62 | } 63 | } 64 | 65 | mod bitvec; 66 | 67 | #[doc(hidden)] 68 | pub mod __internal { 69 | //! Hidden thirdparty dependencies for venndb, 70 | //! not to be relied upon directly, as they may change at any time. 71 | 72 | pub use crate::bitvec::{BitVec, IterOnes}; 73 | pub use hashbrown::HashMap; 74 | use rand::Rng; 75 | 76 | #[must_use] 77 | /// Generate a random `usize`. 78 | pub fn rand_range(limit: usize) -> usize { 79 | rand::rng().random_range(0..limit) 80 | } 81 | 82 | pub mod hash_map { 83 | //! Internal types related to hash map. 84 | 85 | pub use hashbrown::hash_map::Entry; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /scripts/generate_proxies.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | # Define constants 4 | POOLS = [ 5 | "poolA", 6 | "poolB", 7 | "poolC", 8 | "poolD", 9 | "poolE", 10 | "poolF", 11 | "poolG", 12 | "poolH", 13 | "poolI", 14 | "poolJ", 15 | "", 16 | ] 17 | COUNTRIES = ["US", "CA", "GB", "AU", "JP", "DE", "FR", "IT", "ES", "NL"] 18 | 19 | 20 | # Define function to generate random IPv4 address 21 | def random_ipv4(): 22 | return ".".join(str(random.randint(0, 255)) for _ in range(4)) 23 | 24 | 25 | known_proxies = set() 26 | 27 | 28 | # Define function to generate random row 29 | def random_row(): 30 | while "true": 31 | id = random.randint(1, 4294967295) 32 | if id not in known_proxies: 33 | known_proxies.add(id) 34 | break 35 | 36 | address = f"{random_ipv4()}:{random.randint(1000, 9999)}" 37 | username = f"user{random.randint(1, 100)}" 38 | password = f"pass{random.randint(1, 100)}" 39 | tcp = random.choice(["true", "false"]) 40 | udp = random.choice(["true", "false"]) 41 | http = random.choice(["true", "false"]) 42 | socks5 = random.choice(["true", "false"]) 43 | datacenter = random.choice(["true", "false"]) 44 | residential = random.choice(["true", "false"]) 45 | mobile = random.choice(["true", "false"]) 46 | pool = random.choice(POOLS) 47 | if pool in ["poolA", "poolB"]: 48 | country = "US" 49 | if pool == "poolJ": 50 | country = "*" 51 | elif pool == "poolB": 52 | country = random.choice(["US", "CA"]) 53 | elif pool == "poolC": 54 | country = random.choice(["US", "CA", "GB"]) 55 | elif pool == "poolD": 56 | country = random.choice(["US", "CA", "GB", "AU"]) 57 | elif pool == "poolE": 58 | country = random.choice(COUNTRIES[:5]) 59 | else: 60 | country = random.choice(COUNTRIES) 61 | return [ 62 | id, 63 | address, 64 | username, 65 | password, 66 | tcp, 67 | udp, 68 | http, 69 | socks5, 70 | datacenter, 71 | residential, 72 | mobile, 73 | pool, 74 | country, 75 | ] 76 | 77 | 78 | # Generate CSV rows 79 | for i in range(100000): 80 | row = random_row() 81 | print(",".join(str(col) for col in row)) 82 | -------------------------------------------------------------------------------- /www/style.css: -------------------------------------------------------------------------------- 1 | *, 2 | *::before, 3 | *::after { 4 | box-sizing: border-box; 5 | } 6 | 7 | * { 8 | margin: 0; 9 | } 10 | 11 | body { 12 | line-height: 1.5; 13 | -webkit-font-smoothing: antialiased; 14 | background-color: #fef9ef; 15 | } 16 | 17 | img, 18 | picture, 19 | video, 20 | canvas, 21 | svg { 22 | display: block; 23 | max-width: 100%; 24 | } 25 | 26 | input, 27 | button, 28 | textarea, 29 | select { 30 | font: inherit; 31 | } 32 | 33 | p, 34 | h1, 35 | h2, 36 | h3, 37 | h4, 38 | h5, 39 | h6 { 40 | overflow-wrap: break-word; 41 | } 42 | 43 | h1 a { 44 | text-decoration: none; 45 | font-weight: bold; 46 | color: darkblue; 47 | } 48 | 49 | #root, 50 | #__next { 51 | isolation: isolate; 52 | } 53 | 54 | main { 55 | margin: 0 auto; 56 | padding: 0 15px; 57 | max-width: 800px; 58 | height: 100vh; 59 | display: grid; 60 | /* grid spread even vertically */ 61 | align-items: center; 62 | justify-items: center; 63 | } 64 | 65 | #about p { 66 | margin: 0 auto; 67 | padding: 10px; 68 | } 69 | 70 | .copy-button { 71 | text-align: start; 72 | cursor: pointer; 73 | display: flex; 74 | align-items: center; 75 | justify-content: space-between; 76 | font-size: 14px; 77 | line-height: 1.5em; 78 | border: solid 1px grey; 79 | margin: 10px auto; 80 | } 81 | 82 | .copy-button>span { 83 | padding: 10px; 84 | } 85 | 86 | .copy-button:hover { 87 | background-color: #fff; 88 | } 89 | 90 | #menu { 91 | margin: 20px auto; 92 | padding: 10px; 93 | } 94 | 95 | #menu ul { 96 | list-style-type: none; 97 | overflow: hidden; 98 | text-align: center; 99 | padding: 0; 100 | } 101 | 102 | #menu li { 103 | display: inline-block; 104 | margin: 5px; 105 | } 106 | 107 | #menu img { 108 | height: 25px; 109 | } 110 | 111 | #code-block { 112 | padding: 10px; 113 | border: 1px solid grey; 114 | background-color: black; 115 | color: lightgreen; 116 | font-family: monospace; 117 | font-size: 0.9em; 118 | } 119 | 120 | pre { 121 | white-space: pre-wrap; 122 | } 123 | 124 | quote { 125 | display: block; 126 | margin: 10px auto; 127 | padding: 10px; 128 | font-style: italic; 129 | text-align: center; 130 | } -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | 1. [File an issue](https://github.com/plabayo/venndb/issues/new). 4 | The issue will be used to discuss the bug or feature and should be created before opening an MR. 5 | > Best to even wait on actually developing it as to make sure 6 | > that we're all aligned on what you're trying to contribute, 7 | > as to avoid having to reject your hard work and code. 8 | 9 | In case you also want to help resolve it by contributing to the code base you would continue as follows: 10 | 11 | 2. Install Rust and configure correctly (https://www.rust-lang.org/tools/install). 12 | 3. Clone the repo: `git clone https://github.com/plabayo/venndb` 13 | 4. Change into the checked out source: `cd news` 14 | 5. Fork the repo. 15 | 6. Set your fork as a remote: `git remote add fork git@github.com:GITHUB_USERNAME/venndb.git` 16 | 7. Make changes, commit to your fork. 17 | Please add a short summary and a detailed commit message for each commit. 18 | > Feel free to make as many commits as you want in your branch, 19 | > prior to making your MR ready for review, please clean up the git history 20 | > from your branch by rebasing and squashing relevant commits together, 21 | > with the final commits being minimal in number and detailed in their description. 22 | 8. To minimize friction, consider setting Allow edits from maintainers on the PR, 23 | which will enable project committers and automation to update your PR. 24 | 9. A maintainer will review the pull request and make comments. 25 | 26 | Prefer adding additional commits over amending and force-pushing 27 | since it can be difficult to follow code reviews when the commit history changes. 28 | 29 | Commits will be squashed when they're merged. 30 | 31 | ## Testing 32 | 33 | All tests can be run locally against the latest Rust version (or whichever supported Rust version you're using on your development machine). However for contributor/developer purposes it is already sufficient if you check that your tests run 34 | using the standard rust toolchain. 35 | 36 | This way you can for example run the regular tests using: 37 | 38 | ``` 39 | cargo test --all-features --workspace 40 | ``` 41 | 42 | In case you do want to run most tests for your `rustc` version and platform, 43 | you can do easily using the following command: 44 | 45 | ```bash 46 | just qa 47 | ``` 48 | 49 | Before you can do this you do require the following to be installed: 50 | 51 | * `Rust`, version 1.88 or beyond: 52 | * `just` (to run _just_ (config) files): 53 | * `cargo hack`: 54 | 55 | Once this is all done you should be able to run `just qa`. 56 | When all these pasts you can be pretty certain that all tests in the GitHub CI step 57 | will also succeed. The difference still though is that GitHub Action will also run some of these tests on the MSRV (`1.88`) and three platforms in total: MacOS, Linux and Windows. 58 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | description = "an in-memory database in Rust for rows queried using bit (flag) columns" 3 | edition = { workspace = true } 4 | homepage = { workspace = true } 5 | license = { workspace = true } 6 | name = "venndb" 7 | readme = "README.md" 8 | repository = { workspace = true } 9 | keywords = ["database", "db", "memory", "bits"] 10 | categories = ["database"] 11 | authors = ["Glen De Cauwsemaecker "] 12 | version = { workspace = true } 13 | rust-version = { workspace = true } 14 | 15 | [package.metadata.docs.rs] 16 | all-features = true 17 | rustdoc-args = ["--cfg", "docsrs"] 18 | [workspace] 19 | members = [".", "fuzz", "venndb-macros", "venndb-usage"] 20 | 21 | [workspace.dependencies] 22 | divan = "0.1.21" 23 | hashbrown = "0.16.0" 24 | itertools = "0.14.0" 25 | proc-macro2 = "1.0" 26 | quote = "1.0" 27 | rand = "0.9.2" 28 | sqlite = "0.37.0" 29 | syn = "2.0" 30 | trybuild = "1" 31 | venndb = { version = "0.6.1", path = "." } 32 | venndb-macros = { version = "0.6.1", path = "venndb-macros" } 33 | 34 | [workspace.package] 35 | edition = "2024" 36 | homepage = "https://venndb.plabayo.tech" 37 | repository = "https://github.com/plabayo/venndb" 38 | rust-version = "1.88.0" 39 | version = "0.6.1" 40 | authors = ["Glen De Cauwsemaecker "] 41 | license = "MIT OR Apache-2.0" 42 | 43 | [workspace.lints.rust] 44 | unreachable_pub = "deny" 45 | unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] } 46 | 47 | [workspace.lints.clippy] 48 | all = { level = "warn", priority = -1 } 49 | todo = "warn" 50 | empty_enum = "warn" 51 | enum_glob_use = "warn" 52 | equatable_if_let = "warn" 53 | mem_forget = "warn" 54 | unused_self = "warn" 55 | filter_map_next = "warn" 56 | needless_continue = "warn" 57 | needless_borrow = "warn" 58 | match_wildcard_for_single_variants = "warn" 59 | if_let_mutex = "warn" 60 | implicit_clone = "warn" 61 | await_holding_lock = "warn" 62 | imprecise_flops = "warn" 63 | suboptimal_flops = "warn" 64 | lossy_float_literal = "warn" 65 | rest_pat_in_fully_bound_structs = "warn" 66 | fn_params_excessive_bools = "warn" 67 | exit = "warn" 68 | inefficient_to_string = "warn" 69 | linkedlist = "warn" 70 | macro_use_imports = "warn" 71 | manual_let_else = "warn" 72 | match_same_arms = "warn" 73 | must_use_candidate = "warn" 74 | needless_pass_by_ref_mut = "warn" 75 | needless_pass_by_value = "warn" 76 | option_option = "warn" 77 | redundant_clone = "warn" 78 | ref_option = "warn" 79 | verbose_file_reads = "warn" 80 | unnested_or_patterns = "warn" 81 | str_to_string = "warn" 82 | type_complexity = "allow" 83 | return_self_not_must_use = "warn" 84 | single_match_else = "warn" 85 | trivially_copy_pass_by_ref = "warn" 86 | use_self = "warn" 87 | 88 | [dependencies] 89 | hashbrown = { workspace = true } 90 | rand = { workspace = true } 91 | venndb-macros = { workspace = true } 92 | 93 | [dev-dependencies] 94 | divan = { workspace = true } 95 | itertools = { workspace = true } 96 | sqlite = { workspace = true } 97 | 98 | [lints] 99 | workspace = true 100 | 101 | [[bench]] 102 | name = "proxydb" 103 | harness = false 104 | -------------------------------------------------------------------------------- /benches/proxydb.rs: -------------------------------------------------------------------------------- 1 | mod proxies; 2 | use divan::AllocProfiler; 3 | use proxies::{InMemProxyDB, NaiveProxyDB, ProxyDB, SqlLiteProxyDB}; 4 | use std::sync::atomic::AtomicUsize; 5 | 6 | #[global_allocator] 7 | static ALLOC: AllocProfiler = AllocProfiler::system(); 8 | 9 | fn main() { 10 | // Run registered benchmarks. 11 | divan::main(); 12 | } 13 | 14 | const POOLS: [&str; 14] = [ 15 | "poolA", "poolB", "poolC", "poolD", "poolE", "poolF", "poolG", "poolH", "poolI", "poolJ", 16 | "poolA", "poolB", "poolC", "poolD", 17 | ]; 18 | const COUNTRIES: [&str; 13] = [ 19 | "US", "CA", "GB", "DE", "FR", "IT", "ES", "AU", "JP", "CN", "FR", "IT", "ES", 20 | ]; 21 | 22 | #[allow(clippy::declare_interior_mutable_const)] 23 | const COUNTER: AtomicUsize = AtomicUsize::new(0); 24 | 25 | fn next_round() -> usize { 26 | #[allow(clippy::borrow_interior_mutable_const)] 27 | COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed) 28 | } 29 | 30 | fn test_db(db: &impl ProxyDB) { 31 | let i = next_round(); 32 | 33 | let pool = POOLS[i % POOLS.len()]; 34 | let country = COUNTRIES[i % COUNTRIES.len()]; 35 | 36 | let result = db.get(i as u64); 37 | divan::black_box(result); 38 | 39 | let result = db.any_tcp(pool, country); 40 | divan::black_box(result); 41 | 42 | let result = db.any_socks5_isp(pool, country); 43 | divan::black_box(result); 44 | } 45 | 46 | #[divan::bench] 47 | fn venn_proxy_db_100(bencher: divan::Bencher) { 48 | bencher 49 | .with_inputs(|| InMemProxyDB::create(100)) 50 | .bench_refs(|db| test_db(db)); 51 | } 52 | 53 | #[divan::bench] 54 | fn naive_proxy_db_100(bencher: divan::Bencher) { 55 | bencher 56 | .with_inputs(|| NaiveProxyDB::create(100)) 57 | .bench_refs(|db| test_db(db)); 58 | } 59 | 60 | #[divan::bench] 61 | fn sql_lite_proxy_db_100(bencher: divan::Bencher) { 62 | bencher 63 | .with_inputs(|| SqlLiteProxyDB::create(100)) 64 | .bench_refs(|db| test_db(db)); 65 | } 66 | 67 | #[divan::bench] 68 | fn venn_proxy_db_12_500(bencher: divan::Bencher) { 69 | bencher 70 | .with_inputs(|| InMemProxyDB::create(12_500)) 71 | .bench_refs(|db| test_db(db)); 72 | } 73 | 74 | #[divan::bench] 75 | fn naive_proxy_db_12_500(bencher: divan::Bencher) { 76 | bencher 77 | .with_inputs(|| NaiveProxyDB::create(12_500)) 78 | .bench_refs(|db| test_db(db)); 79 | } 80 | 81 | #[divan::bench] 82 | fn sql_lite_proxy_db_12_500(bencher: divan::Bencher) { 83 | bencher 84 | .with_inputs(|| SqlLiteProxyDB::create(12_500)) 85 | .bench_refs(|db| test_db(db)); 86 | } 87 | 88 | #[divan::bench] 89 | fn venn_proxy_db_100_000(bencher: divan::Bencher) { 90 | bencher 91 | .with_inputs(|| InMemProxyDB::create(100_000)) 92 | .bench_refs(|db| test_db(db)); 93 | } 94 | 95 | #[divan::bench] 96 | fn naive_proxy_db_100_000(bencher: divan::Bencher) { 97 | bencher 98 | .with_inputs(|| NaiveProxyDB::create(100_000)) 99 | .bench_refs(|db| test_db(db)); 100 | } 101 | 102 | #[divan::bench] 103 | fn sql_lite_proxy_db_100_000(bencher: divan::Bencher) { 104 | bencher 105 | .with_inputs(|| SqlLiteProxyDB::create(100_000)) 106 | .bench_refs(|db| test_db(db)); 107 | } 108 | -------------------------------------------------------------------------------- /venndb-macros/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![forbid(unsafe_code)] 2 | 3 | mod errors; 4 | mod field; 5 | mod generate_db; 6 | mod parse_attrs; 7 | 8 | use errors::Errors; 9 | use field::StructField; 10 | use parse_attrs::{FieldAttrs, TypeAttrs}; 11 | use proc_macro2::TokenStream; 12 | use quote::{ToTokens, format_ident, quote}; 13 | 14 | /// Derive macro generating VennDB functionality for this struct. 15 | /// 16 | /// See for more information on how to use it. 17 | /// Or check out the README and usage tests in [the repository][repo] of this macro. 18 | /// 19 | /// [repo]: https://github.com/plabayo/venndb 20 | #[proc_macro_derive(VennDB, attributes(venndb))] 21 | pub fn venndb(input: proc_macro::TokenStream) -> proc_macro::TokenStream { 22 | let ast = syn::parse_macro_input!(input as syn::DeriveInput); 23 | let ts: TokenStream = impl_from_args(&ast); 24 | ts.into() 25 | } 26 | 27 | /// Transform the input into a token stream containing any generated implementations, 28 | /// as well as all errors that occurred. 29 | fn impl_from_args(input: &syn::DeriveInput) -> TokenStream { 30 | let errors = &Errors::default(); 31 | let type_attrs = &TypeAttrs::parse(errors, input); 32 | let mut output_tokens = match &input.data { 33 | syn::Data::Struct(ds) => impl_from_args_struct( 34 | errors, 35 | &input.vis, 36 | &input.ident, 37 | type_attrs, 38 | &input.generics, 39 | ds, 40 | ), 41 | syn::Data::Enum(_) => { 42 | errors.err(input, "`#[derive(VennDB)]` cannot be applied to enums"); 43 | TokenStream::new() 44 | } 45 | syn::Data::Union(_) => { 46 | errors.err(input, "`#[derive(VennDB)]` cannot be applied to unions"); 47 | TokenStream::new() 48 | } 49 | }; 50 | errors.to_tokens(&mut output_tokens); 51 | output_tokens 52 | } 53 | 54 | /// Implements `VennDB` for a `#[derive(VennDB)]` struct. 55 | fn impl_from_args_struct( 56 | errors: &Errors, 57 | vis: &syn::Visibility, 58 | name: &syn::Ident, 59 | type_attrs: &TypeAttrs, 60 | _generic_args: &syn::Generics, 61 | ds: &syn::DataStruct, 62 | ) -> TokenStream { 63 | let fields = match &ds.fields { 64 | syn::Fields::Named(fields) => fields, 65 | syn::Fields::Unnamed(_) => { 66 | errors.err( 67 | &ds.struct_token, 68 | "`#![derive(VennDB)]` is not currently supported on tuple structs", 69 | ); 70 | return TokenStream::new(); 71 | } 72 | syn::Fields::Unit => { 73 | errors.err( 74 | &ds.struct_token, 75 | "#![derive(VennDB)]` cannot be applied to unit structs", 76 | ); 77 | return TokenStream::new(); 78 | } 79 | }; 80 | 81 | let fields: Vec<_> = fields 82 | .named 83 | .iter() 84 | .filter_map(|field| { 85 | let attrs = FieldAttrs::parse(errors, field); 86 | StructField::new(errors, field, attrs) 87 | }) 88 | .collect(); 89 | 90 | let name_db = match &type_attrs.name { 91 | Some(name) => format_ident!("{}", name.value()), 92 | None => format_ident!("{}DB", name), 93 | }; 94 | 95 | let db_code = generate_db::generate_db( 96 | name, 97 | &name_db, 98 | type_attrs.validator.as_ref(), 99 | vis, 100 | &fields[..], 101 | ); 102 | 103 | quote! { 104 | #db_code 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /venndb-macros/src/field.rs: -------------------------------------------------------------------------------- 1 | //! Struct Field Info 2 | 3 | use crate::{ 4 | errors::Errors, 5 | parse_attrs::{FieldAttrs, FieldKind}, 6 | }; 7 | use quote::format_ident; 8 | use syn::Ident; 9 | 10 | /// A field of a `#![derive(VennDB)]` struct with attributes and some other 11 | /// notable metadata appended. 12 | pub struct StructField<'a> { 13 | /// The original parsed field 14 | field: &'a syn::Field, 15 | /// The parsed attributes of the field 16 | attrs: FieldAttrs<'a>, 17 | /// The field name. This is contained optionally inside `field`, 18 | /// but is duplicated non-optionally here to indicate that all field that 19 | /// have reached this point must have a field name, and it no longer 20 | /// needs to be unwrapped. 21 | name: &'a syn::Ident, 22 | } 23 | 24 | pub enum FieldInfo<'a> { 25 | Key(KeyField<'a>), 26 | Filter(FilterField<'a>), 27 | FilterMap(FilterMapField<'a>), 28 | } 29 | 30 | pub struct KeyField<'a> { 31 | pub name: &'a Ident, 32 | pub ty: &'a syn::Type, 33 | } 34 | 35 | impl<'a> KeyField<'a> { 36 | pub fn name(&'a self) -> &'a Ident { 37 | self.name 38 | } 39 | 40 | pub fn ty(&'a self) -> &'a syn::Type { 41 | self.ty 42 | } 43 | 44 | pub fn method_name(&self) -> Ident { 45 | format_ident!("get_by_{}", self.name) 46 | } 47 | 48 | pub fn map_name(&self) -> Ident { 49 | format_ident!("map_{}", self.name) 50 | } 51 | } 52 | 53 | pub struct FilterField<'a> { 54 | pub name: &'a Ident, 55 | pub optional: bool, 56 | } 57 | 58 | impl<'a> FilterField<'a> { 59 | pub fn name(&'a self) -> &'a Ident { 60 | self.name 61 | } 62 | 63 | pub fn filter_name(&self) -> Ident { 64 | format_ident!("filter_{}", self.name) 65 | } 66 | 67 | pub fn filter_not_name(&self) -> Ident { 68 | format_ident!("filter_not_{}", self.name) 69 | } 70 | } 71 | 72 | impl<'a> StructField<'a> { 73 | /// Attempts to parse a field of a `#[derive(VennDB)]` struct, pulling out the 74 | /// fields required for code generation. 75 | pub fn new(_errors: &Errors, field: &'a syn::Field, attrs: FieldAttrs<'a>) -> Option { 76 | let name = field.ident.as_ref().expect("missing ident for named field"); 77 | Some(StructField { field, attrs, name }) 78 | } 79 | 80 | /// Return the method name for this struct field. 81 | pub fn info(&self) -> Option> { 82 | self.attrs.kind.as_ref().map(|kind| match kind { 83 | FieldKind::Key => FieldInfo::Key(KeyField { 84 | name: self.name, 85 | ty: self.attrs.option_ty.unwrap_or(&self.field.ty), 86 | }), 87 | FieldKind::Filter => FieldInfo::Filter(FilterField { 88 | name: self.name, 89 | optional: self.attrs.option_ty.is_some(), 90 | }), 91 | FieldKind::FilterMap { any } => FieldInfo::FilterMap(FilterMapField { 92 | name: self.name, 93 | ty: self.attrs.option_ty.unwrap_or(&self.field.ty), 94 | optional: self.attrs.option_ty.is_some(), 95 | any: *any, 96 | }), 97 | }) 98 | } 99 | } 100 | 101 | pub struct FilterMapField<'a> { 102 | pub name: &'a Ident, 103 | pub ty: &'a syn::Type, 104 | pub optional: bool, 105 | pub any: bool, 106 | } 107 | 108 | impl<'a> FilterMapField<'a> { 109 | pub fn name(&'a self) -> &'a Ident { 110 | self.name 111 | } 112 | 113 | pub fn ty(&'a self) -> &'a syn::Type { 114 | self.ty 115 | } 116 | 117 | pub fn filter_map_name(&self) -> Ident { 118 | format_ident!("filter_map_{}", self.name) 119 | } 120 | 121 | pub fn filter_vec_name(&self) -> Ident { 122 | format_ident!("filter_vec_{}", self.name) 123 | } 124 | 125 | pub fn filter_any_name(&self) -> Option { 126 | if self.any { 127 | Some(format_ident!("filter_any_{}", self.name)) 128 | } else { 129 | None 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | pn@plabayo.tech. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /venndb-macros/src/errors.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | use { 4 | proc_macro2::{Span, TokenStream}, 5 | quote::ToTokens, 6 | std::cell::RefCell, 7 | }; 8 | 9 | /// Produce functions to expect particular literals in `syn::Expr` 10 | macro_rules! expect_lit_fn { 11 | ($(($fn_name:ident, $syn_type:ident, $variant:ident, $lit_name:literal),)*) => { 12 | $( 13 | pub fn $fn_name<'a>(&self, e: &'a syn::Expr) -> Option<&'a syn::$syn_type> { 14 | if let syn::Expr::Lit(syn::ExprLit { lit: syn::Lit::$variant(inner), .. }) = e { 15 | Some(inner) 16 | } else { 17 | self.unexpected_lit($lit_name, e); 18 | None 19 | } 20 | } 21 | )* 22 | } 23 | } 24 | 25 | /// Produce functions to expect particular variants of `syn::Meta` 26 | macro_rules! expect_meta_fn { 27 | ($(($fn_name:ident, $syn_type:ident, $variant:ident, $meta_name:literal),)*) => { 28 | $( 29 | pub fn $fn_name<'a>(&self, meta: &'a syn::Meta) -> Option<&'a syn::$syn_type> { 30 | if let syn::Meta::$variant(inner) = meta { 31 | Some(inner) 32 | } else { 33 | self.unexpected_meta($meta_name, meta); 34 | None 35 | } 36 | } 37 | )* 38 | } 39 | } 40 | 41 | /// A type for collecting procedural macro errors. 42 | #[derive(Default)] 43 | pub struct Errors { 44 | errors: RefCell>, 45 | } 46 | 47 | impl Errors { 48 | expect_lit_fn![ 49 | (expect_lit_str, LitStr, Str, "string"), 50 | (expect_lit_char, LitChar, Char, "character"), 51 | (expect_lit_int, LitInt, Int, "integer"), 52 | ]; 53 | 54 | expect_meta_fn![ 55 | (expect_meta_word, Path, Path, "path"), 56 | (expect_meta_list, MetaList, List, "list"), 57 | ( 58 | expect_meta_name_value, 59 | MetaNameValue, 60 | NameValue, 61 | "name-value pair" 62 | ), 63 | ]; 64 | 65 | pub fn expect_path<'a>(&self, e: &'a syn::Expr) -> Option<&'a syn::Path> { 66 | if let syn::Expr::Path(path) = e { 67 | Some(&path.path) 68 | } else { 69 | self.unexpected_value("path", e); 70 | None 71 | } 72 | } 73 | 74 | fn unexpected_lit(&self, expected: &str, found: &syn::Expr) { 75 | fn lit_kind(lit: &syn::Lit) -> &'static str { 76 | use syn::Lit::{Bool, Byte, ByteStr, Char, Float, Int, Str, Verbatim}; 77 | match lit { 78 | Str(_) => "string", 79 | ByteStr(_) => "bytestring", 80 | Byte(_) => "byte", 81 | Char(_) => "character", 82 | Int(_) => "integer", 83 | Float(_) => "float", 84 | Bool(_) => "boolean", 85 | Verbatim(_) => "unknown (possibly extra-large integer)", 86 | _ => "unknown literal kind", 87 | } 88 | } 89 | 90 | if let syn::Expr::Lit(syn::ExprLit { lit, .. }) = found { 91 | self.err( 92 | found, 93 | &[ 94 | "Expected ", 95 | expected, 96 | " literal, found ", 97 | lit_kind(lit), 98 | " literal", 99 | ] 100 | .concat(), 101 | ) 102 | } else { 103 | self.err( 104 | found, 105 | &[ 106 | "Expected ", 107 | expected, 108 | " literal, found non-literal expression.", 109 | ] 110 | .concat(), 111 | ) 112 | } 113 | } 114 | 115 | fn unexpected_meta(&self, expected: &str, found: &syn::Meta) { 116 | fn meta_kind(meta: &syn::Meta) -> &'static str { 117 | use syn::Meta::{List, NameValue, Path}; 118 | match meta { 119 | Path(_) => "path", 120 | List(_) => "list", 121 | NameValue(_) => "name-value pair", 122 | } 123 | } 124 | 125 | self.err( 126 | found, 127 | &[ 128 | "Expected ", 129 | expected, 130 | " attribute, found ", 131 | meta_kind(found), 132 | " attribute", 133 | ] 134 | .concat(), 135 | ) 136 | } 137 | 138 | fn unexpected_value(&self, expected: &str, found: &syn::Expr) { 139 | fn expr_kind(expr: &syn::Expr) -> &'static str { 140 | use syn::Expr::{ 141 | Array, Assign, Async, Await, Binary, Block, Break, Call, Cast, Closure, Const, 142 | Continue, Field, ForLoop, Group, If, Index, Infer, Let, Lit, Loop, Macro, Match, 143 | MethodCall, Paren, Path, Range, Reference, Repeat, Return, Struct, Try, TryBlock, 144 | Tuple, Unary, Unsafe, Verbatim, While, Yield, 145 | }; 146 | match expr { 147 | Array(_) => "array", 148 | Assign(_) => "assignment", 149 | Async(_) => "async block", 150 | Await(_) => "await", 151 | Binary(_) => "binary operation", 152 | Block(_) => "block", 153 | Break(_) => "break", 154 | Call(_) => "function call", 155 | Cast(_) => "cast", 156 | Closure(_) => "closure", 157 | Const(_) => "const", 158 | Continue(_) => "continue", 159 | Field(_) => "field access", 160 | ForLoop(_) => "for loop", 161 | Group(_) => "group", 162 | If(_) => "if", 163 | Index(_) => "index", 164 | Infer(_) => "inferred type", 165 | Let(_) => "let", 166 | Lit(_) => "literal", 167 | Loop(_) => "loop", 168 | Macro(_) => "macro", 169 | Match(_) => "match", 170 | MethodCall(_) => "method call", 171 | Paren(_) => "parentheses", 172 | Path(_) => "path", 173 | Range(_) => "range", 174 | Reference(_) => "reference", 175 | Repeat(_) => "repeat", 176 | Return(_) => "return", 177 | Struct(_) => "struct", 178 | Try(_) => "try", 179 | TryBlock(_) => "try block", 180 | Tuple(_) => "tuple", 181 | Unary(_) => "unary operation", 182 | Unsafe(_) => "unsafe block", 183 | Verbatim(_) => "verbatim", 184 | While(_) => "while", 185 | Yield(_) => "yield", 186 | _ => "unknown expression kind", 187 | } 188 | } 189 | 190 | self.err( 191 | found, 192 | &[ 193 | "Expected ", 194 | expected, 195 | " attribute, found ", 196 | found.to_token_stream().to_string().as_str(), 197 | " attribute (", 198 | expr_kind(found), 199 | ")", 200 | ] 201 | .concat(), 202 | ) 203 | } 204 | 205 | /// Issue an error relating to a particular `Spanned` structure. 206 | pub fn err(&self, spanned: &impl syn::spanned::Spanned, msg: &str) { 207 | self.err_span(spanned.span(), msg); 208 | } 209 | 210 | /// Issue an error relating to a particular `Span`. 211 | pub fn err_span(&self, span: Span, msg: &str) { 212 | self.push(syn::Error::new(span, msg)); 213 | } 214 | 215 | /// Issue an error spanning over the given syntax tree node. 216 | pub fn err_span_tokens(&self, tokens: T, msg: &str) { 217 | self.push(syn::Error::new_spanned(tokens, msg)); 218 | } 219 | 220 | /// Push a `syn::Error` onto the list of errors to issue. 221 | pub fn push(&self, err: syn::Error) { 222 | self.errors.borrow_mut().push(err); 223 | } 224 | 225 | /// Convert a `syn::Result` to an `Option`, logging the error if present. 226 | pub fn ok(&self, r: syn::Result) -> Option { 227 | match r { 228 | Ok(v) => Some(v), 229 | Err(e) => { 230 | self.push(e); 231 | None 232 | } 233 | } 234 | } 235 | } 236 | 237 | impl ToTokens for Errors { 238 | /// Convert the errors into tokens that, when emit, will cause 239 | /// the user of the macro to receive compiler errors. 240 | fn to_tokens(&self, tokens: &mut TokenStream) { 241 | tokens.extend(self.errors.borrow().iter().map(|e| e.to_compile_error())); 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /.github/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | env: 4 | CARGO_TERM_COLOR: always 5 | RUST_TOOLCHAIN: stable 6 | RUST_TOOLCHAIN_NIGHTLY: nightly 7 | RUST_TOOLCHAIN_MSRV: 1.88.0 8 | RUST_TOOLCHAIN_BETA: beta 9 | 10 | on: 11 | push: 12 | branches: 13 | - main 14 | pull_request: {} 15 | 16 | jobs: 17 | check-msrv: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v4 21 | - uses: dtolnay/rust-toolchain@master 22 | with: 23 | toolchain: ${{env.RUST_TOOLCHAIN_MSRV}} 24 | components: clippy, rustfmt 25 | - uses: Swatinem/rust-cache@v2 26 | - name: check 27 | run: | 28 | cargo check --all --all-targets --all-features 29 | - name: clippy 30 | run: | 31 | cargo clippy --all --all-targets --all-features 32 | - name: rustfmt 33 | run: | 34 | cargo fmt --all -- --check 35 | 36 | check: 37 | runs-on: ubuntu-latest 38 | steps: 39 | - uses: actions/checkout@v4 40 | - uses: dtolnay/rust-toolchain@master 41 | with: 42 | toolchain: ${{env.RUST_TOOLCHAIN}} 43 | components: clippy, rustfmt 44 | - uses: Swatinem/rust-cache@v2 45 | - name: check 46 | run: | 47 | cargo check --all --all-targets --all-features 48 | - name: clippy 49 | run: | 50 | cargo clippy --all --all-targets --all-features 51 | - name: rustfmt 52 | run: | 53 | cargo fmt --all -- --check 54 | 55 | check-docs: 56 | runs-on: ubuntu-latest 57 | needs: [check, check-msrv] 58 | steps: 59 | - uses: actions/checkout@v4 60 | - uses: dtolnay/rust-toolchain@master 61 | with: 62 | toolchain: ${{env.RUST_TOOLCHAIN}} 63 | - uses: Swatinem/rust-cache@v2 64 | - name: cargo doc 65 | env: 66 | RUSTDOCFLAGS: "-D rustdoc::broken_intra_doc_links" 67 | run: cargo doc --all-features --no-deps --workspace 68 | 69 | test: 70 | needs: [check, check-msrv] 71 | runs-on: ubuntu-latest 72 | steps: 73 | - uses: actions/checkout@v4 74 | - uses: dtolnay/rust-toolchain@master 75 | with: 76 | toolchain: ${{env.RUST_TOOLCHAIN}} 77 | - uses: Swatinem/rust-cache@v2 78 | - name: Run tests 79 | run: cargo test --all-features --workspace 80 | 81 | test-windows: 82 | needs: [check, check-msrv] 83 | runs-on: windows-latest 84 | steps: 85 | - uses: actions/checkout@v4 86 | - uses: dtolnay/rust-toolchain@master 87 | with: 88 | toolchain: ${{env.RUST_TOOLCHAIN}} 89 | - uses: Swatinem/rust-cache@v2 90 | - name: Run tests 91 | run: cargo test --all-features --workspace 92 | 93 | test-macos: 94 | needs: [check, check-msrv] 95 | runs-on: macos-latest 96 | steps: 97 | - uses: actions/checkout@v4 98 | - uses: dtolnay/rust-toolchain@master 99 | with: 100 | toolchain: ${{env.RUST_TOOLCHAIN}} 101 | - uses: Swatinem/rust-cache@v2 102 | - name: Run tests 103 | run: cargo test --all-features --workspace 104 | 105 | test-docs: 106 | needs: [check, check-msrv] 107 | runs-on: ubuntu-latest 108 | steps: 109 | - uses: actions/checkout@v4 110 | - uses: dtolnay/rust-toolchain@master 111 | with: 112 | toolchain: ${{env.RUST_TOOLCHAIN}} 113 | - uses: Swatinem/rust-cache@v2 114 | - name: Run doc tests 115 | run: cargo test --doc --all-features --workspace 116 | 117 | test-examples-beta: 118 | needs: [check, check-msrv] 119 | runs-on: ubuntu-latest 120 | steps: 121 | - uses: actions/checkout@v4 122 | - uses: dtolnay/rust-toolchain@master 123 | with: 124 | toolchain: ${{env.RUST_TOOLCHAIN_BETA}} 125 | - uses: Swatinem/rust-cache@v2 126 | - name: Run doc tests 127 | run: cargo test --all-features --examples --workspace 128 | 129 | test-examples-msrv: 130 | needs: [check, check-msrv] 131 | runs-on: ubuntu-latest 132 | steps: 133 | - uses: actions/checkout@v4 134 | - uses: dtolnay/rust-toolchain@master 135 | with: 136 | toolchain: ${{env.RUST_TOOLCHAIN_MSRV}} 137 | - uses: Swatinem/rust-cache@v2 138 | - name: Run doc tests 139 | run: cargo test --all-features --examples --workspace 140 | 141 | test-examples: 142 | needs: [check, check-msrv] 143 | runs-on: ubuntu-latest 144 | steps: 145 | - uses: actions/checkout@v4 146 | - uses: dtolnay/rust-toolchain@master 147 | with: 148 | toolchain: ${{env.RUST_TOOLCHAIN}} 149 | - uses: Swatinem/rust-cache@v2 150 | - name: Run doc tests 151 | run: cargo test --all-features --examples --workspace 152 | 153 | test-examples-windows-beta: 154 | needs: [check, check-msrv] 155 | runs-on: windows-latest 156 | steps: 157 | - uses: actions/checkout@v4 158 | - uses: dtolnay/rust-toolchain@master 159 | with: 160 | toolchain: ${{env.RUST_TOOLCHAIN_BETA}} 161 | - uses: Swatinem/rust-cache@v2 162 | - name: Run doc tests 163 | run: cargo test --all-features --examples --workspace 164 | 165 | test-examples-windows-msrv: 166 | needs: [check, check-msrv] 167 | runs-on: windows-latest 168 | steps: 169 | - uses: actions/checkout@v4 170 | - uses: dtolnay/rust-toolchain@master 171 | with: 172 | toolchain: ${{env.RUST_TOOLCHAIN_MSRV}} 173 | - uses: Swatinem/rust-cache@v2 174 | - name: Run doc tests 175 | run: cargo test --all-features --examples --workspace 176 | 177 | test-examples-windows: 178 | needs: [check, check-msrv] 179 | runs-on: windows-latest 180 | steps: 181 | - uses: actions/checkout@v4 182 | - uses: dtolnay/rust-toolchain@master 183 | with: 184 | toolchain: ${{env.RUST_TOOLCHAIN}} 185 | - uses: Swatinem/rust-cache@v2 186 | - name: Run doc tests 187 | run: cargo test --all-features --examples --workspace 188 | 189 | test-examples-macos-beta: 190 | needs: [check, check-msrv] 191 | runs-on: macos-latest 192 | steps: 193 | - uses: actions/checkout@v4 194 | - uses: dtolnay/rust-toolchain@master 195 | with: 196 | toolchain: ${{env.RUST_TOOLCHAIN_BETA}} 197 | - uses: Swatinem/rust-cache@v2 198 | - name: Run doc tests 199 | run: cargo test --all-features --examples --workspace 200 | 201 | test-examples-macos-msrv: 202 | needs: [check, check-msrv] 203 | runs-on: macos-latest 204 | steps: 205 | - uses: actions/checkout@v4 206 | - uses: dtolnay/rust-toolchain@master 207 | with: 208 | toolchain: ${{env.RUST_TOOLCHAIN_MSRV}} 209 | - uses: Swatinem/rust-cache@v2 210 | - name: Run doc tests 211 | run: cargo test --all-features --examples --workspace 212 | 213 | test-examples-macos: 214 | needs: [check, check-msrv] 215 | runs-on: macos-latest 216 | steps: 217 | - uses: actions/checkout@v4 218 | - uses: dtolnay/rust-toolchain@master 219 | with: 220 | toolchain: ${{env.RUST_TOOLCHAIN}} 221 | - uses: Swatinem/rust-cache@v2 222 | - name: Run doc tests 223 | run: cargo test --all-features --examples --workspace 224 | 225 | cargo-hack: 226 | needs: [check, check-msrv] 227 | runs-on: ubuntu-latest 228 | steps: 229 | - uses: actions/checkout@v4 230 | - uses: dtolnay/rust-toolchain@master 231 | with: 232 | toolchain: ${{env.RUST_TOOLCHAIN}} 233 | - name: install cargo-hack 234 | uses: taiki-e/install-action@cargo-hack 235 | - name: cargo hack check 236 | run: cargo hack check --each-feature --no-dev-deps --workspace 237 | 238 | dependencies-are-sorted: 239 | needs: [check, check-msrv] 240 | runs-on: ubuntu-latest 241 | steps: 242 | - uses: actions/checkout@v4 243 | - uses: dtolnay/rust-toolchain@master 244 | with: 245 | toolchain: ${{env.RUST_TOOLCHAIN}} 246 | - uses: Swatinem/rust-cache@v2 247 | - name: Install cargo-sort 248 | run: | 249 | cargo install cargo-sort 250 | - name: Check dependency tables 251 | working-directory: . 252 | run: | 253 | cargo sort --workspace --grouped --check 254 | 255 | cargo-deny: 256 | needs: [check, check-msrv] 257 | runs-on: ubuntu-latest 258 | steps: 259 | - uses: actions/checkout@v4 260 | - uses: EmbarkStudios/cargo-deny-action@v2 261 | 262 | cargo-fuzz: 263 | needs: [check, check-msrv] 264 | runs-on: ubuntu-latest 265 | steps: 266 | - uses: actions/checkout@v4 267 | - uses: dtolnay/rust-toolchain@nightly 268 | with: 269 | toolchain: ${{env.RUST_TOOLCHAIN_NIGHTLY}} 270 | - name: Install cargo-fuzz 271 | run: | 272 | cargo install cargo-fuzz 273 | - name: cargo fuzz check 274 | run: | 275 | cargo fuzz run fuzz_employee_db -- -max_len=131072 -max_total_time=30 276 | 277 | semver-checks: 278 | needs: [check, check-msrv] 279 | runs-on: ubuntu-latest 280 | steps: 281 | - name: Checkout 282 | uses: actions/checkout@v4 283 | - name: Check semver 284 | uses: obi1kenobi/cargo-semver-checks-action@v2 285 | with: 286 | rust-toolchain: ${{env.RUST_TOOLCHAIN}} 287 | -------------------------------------------------------------------------------- /www/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | venndb 8 | 9 | 13 | 14 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 28 | 29 | 30 | 34 | 35 | 36 | 37 | 54 | 55 | 56 | 57 |
58 |
59 | venndb banner 60 |
61 |

62 | An append-only in-memory database in 63 | Rust for rows queried using bit (flag) columns. This 64 | database is designed for a very specific use case where 65 | you have mostly static data that you typically load at 66 | startup and have to query constantly using very simple 67 | filters. Datasets like these can be large and should be 68 | both fast and compact. 69 |

70 | 71 |

72 | For the limited usecases where venndb can be applied to, 73 | it has less dependencies and is faster then traditional 74 | choices, such as a naive implementation or a more heavy 75 | lifted dependency such as Sqlite. 76 |

77 | 78 | 79 | See 80 | the benchmarks 83 | for more information on this topic. 84 | 85 | 86 |

87 | This project was developed originally in function of 88 | rama, where you can 89 | see it being used for example to provide an in-memory 90 | (upstream) proxy database. Do 91 | let us know in 92 | case you use it as well in your project, such that we 93 | can assemble a showcase list. 94 |

95 |
96 | 119 | 161 |
162 |

Example

163 |
164 |
165 | 
use venndb::VennDB
166 | 
167 | #[derive(Debug, VennDB)]
168 | pub struct Employee {
169 |     #[venndb(key)]
170 |     id: u32,
171 |     name: String,
172 |     is_manager: Option<bool>,
173 |     is_admin: bool,
174 |     #[venndb(skip)]
175 |     foo: bool,
176 |     #[venndb(filter, any)]
177 |     department: Department,
178 |     #[venndb(filter)]
179 |     country: Option<String>,
180 | }
181 | 
182 | fn main() {
183 |     let db = EmployeeDB::from_iter(/* .. */);
184 | 
185 |     let mut query = db.query();
186 |     let employee = query
187 |         .is_admin(true)
188 |         .is_manager(false)
189 |         .department(Department::Engineering)
190 |         .execute()
191 |         .expect("to have found at least one")
192 |         .any();
193 | 
194 |     println!("non-manager admin engineer: {:?}", employee);
195 | }
196 |
197 |
198 | 199 |

200 | Learn more at 201 | https://github.com/plabayo/venndb. 204 |

206 |
207 |
208 |
209 | 210 | 211 | -------------------------------------------------------------------------------- /src/bitvec.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | //! Fork from 3 | //! Original License: 4 | 5 | #[must_use] 6 | #[derive(Debug, Clone, Default)] 7 | pub struct BitVec { 8 | len: usize, 9 | data: Vec, 10 | } 11 | 12 | #[derive(Debug)] 13 | pub struct IterOnes<'a> { 14 | index: usize, 15 | bv: &'a BitVec, 16 | } 17 | 18 | impl BitVec { 19 | #[inline(always)] 20 | pub fn new() -> Self { 21 | Self::default() 22 | } 23 | 24 | pub fn with_capacity(capacity: usize) -> Self { 25 | Self { 26 | len: 0, 27 | data: Vec::with_capacity(blocks_required(capacity)), 28 | } 29 | } 30 | 31 | pub fn repeat(value: bool, len: usize) -> Self { 32 | let mut v = Self { 33 | len, 34 | data: vec![if value { !0 } else { 0 }; blocks_required(len)], 35 | }; 36 | v.mask_last_block(); 37 | v 38 | } 39 | 40 | #[must_use] 41 | pub fn iter_ones(&self) -> IterOnes<'_> { 42 | IterOnes { index: 0, bv: self } 43 | } 44 | 45 | #[must_use] 46 | pub fn count_ones(&self) -> usize { 47 | self.iter_ones().count() 48 | } 49 | 50 | #[must_use] 51 | pub fn any(&self) -> bool { 52 | self.iter_ones().next().is_some() 53 | } 54 | 55 | pub fn push(&mut self, value: bool) { 56 | debug_assert_eq!(self.data.len(), blocks_required(self.len)); 57 | if self.len.is_multiple_of(BITS_PER_BLOCK) { 58 | self.data.push(0); 59 | } 60 | let i = self.len; 61 | self.len = i.checked_add(1).expect("Overflow detected"); 62 | self.set(i, value); 63 | } 64 | 65 | fn set(&mut self, index: usize, value: bool) -> bool { 66 | if index >= self.len { 67 | panic!( 68 | "Index out of bounds: the len is {} but the index is {}", 69 | self.len, index 70 | ); 71 | } 72 | let msk = 1 << (index % BITS_PER_BLOCK); 73 | let off = block_offset(index); 74 | let old_v = self.data[off]; 75 | let new_v = if value { old_v | msk } else { old_v & !msk }; 76 | if new_v != old_v { 77 | self.data[off] = new_v; 78 | true 79 | } else { 80 | false 81 | } 82 | } 83 | 84 | pub fn or(&mut self, other: &Self) -> bool { 85 | let mut chngd = false; 86 | for (self_blk, other_blk) in self 87 | .data 88 | .iter_mut() 89 | .zip(other.data.iter().chain(std::iter::repeat(&0))) 90 | { 91 | let old_v = *self_blk; 92 | let new_v = old_v | *other_blk; 93 | *self_blk = new_v; 94 | chngd |= old_v != new_v; 95 | } 96 | // We don't need to mask the last block per our assumptions 97 | chngd 98 | } 99 | 100 | pub fn and(&mut self, other: &Self) -> bool { 101 | let mut chngd = false; 102 | for (self_blk, other_blk) in self 103 | .data 104 | .iter_mut() 105 | .zip(other.data.iter().chain(std::iter::repeat(&0))) 106 | { 107 | let old_v = *self_blk; 108 | let new_v = old_v & *other_blk; 109 | *self_blk = new_v; 110 | chngd |= old_v != new_v; 111 | } 112 | // We don't need to mask the last block as those bits can't be set by "&" by definition. 113 | chngd 114 | } 115 | 116 | /// We guarantee that the last storage block has no bits set past the "last" bit: this function 117 | /// clears any such bits. 118 | fn mask_last_block(&mut self) { 119 | debug_assert_eq!(self.data.len(), blocks_required(self.len)); 120 | let ub = self.len % BITS_PER_BLOCK; 121 | // If there are no unused bits, there's no need to perform masking. 122 | if ub > 0 { 123 | let msk = (1 << ub) - 1; 124 | let off = block_offset(self.len); 125 | let old_v = self.data[off]; 126 | let new_v = old_v & msk; 127 | if new_v != old_v { 128 | self.data[off] = new_v; 129 | } 130 | } 131 | } 132 | } 133 | 134 | impl Iterator for IterOnes<'_> { 135 | type Item = usize; 136 | 137 | fn next(&mut self) -> Option { 138 | if self.index >= self.bv.len { 139 | return None; 140 | } 141 | 142 | // start at current index, mask off earlier bits in the starting block 143 | let mut b = self.index / BITS_PER_BLOCK; 144 | let off = self.index % BITS_PER_BLOCK; 145 | 146 | // guard against empty storage 147 | if b >= self.bv.data.len() { 148 | self.index = self.bv.len; 149 | return None; 150 | } 151 | 152 | let mut v = self.bv.data[b]; 153 | if off != 0 { 154 | v &= usize::MAX << off; 155 | } 156 | 157 | loop { 158 | if v != 0 { 159 | let tz = v.trailing_zeros() as usize; 160 | let bit = b * BITS_PER_BLOCK + tz; 161 | if bit < self.bv.len { 162 | self.index = bit + 1; 163 | return Some(bit); 164 | } else { 165 | self.index = self.bv.len; 166 | return None; 167 | } 168 | } 169 | 170 | b += 1; 171 | if b >= self.bv.data.len() { 172 | self.index = self.bv.len; 173 | return None; 174 | } 175 | v = self.bv.data[b]; 176 | } 177 | } 178 | 179 | fn size_hint(&self) -> (usize, Option) { 180 | // cannot know remaining ones cheaply, use a safe upper bound 181 | let remaining = self.bv.len.saturating_sub(self.index); 182 | (0, Some(remaining)) 183 | } 184 | } 185 | 186 | impl std::ops::BitOrAssign<&Self> for BitVec { 187 | #[inline(always)] 188 | fn bitor_assign(&mut self, other: &Self) { 189 | let _ = self.or(other); 190 | } 191 | } 192 | 193 | impl std::ops::BitOr<&BitVec> for &BitVec { 194 | type Output = BitVec; 195 | #[inline(always)] 196 | fn bitor(self, other: &BitVec) -> BitVec { 197 | let mut rv = self.clone(); 198 | let _ = rv.or(other); 199 | rv 200 | } 201 | } 202 | 203 | impl std::ops::BitOr<&Self> for BitVec { 204 | type Output = Self; 205 | 206 | #[inline(always)] 207 | fn bitor(mut self, other: &Self) -> Self { 208 | let _ = self.or(other); 209 | self 210 | } 211 | } 212 | 213 | impl std::ops::BitAndAssign<&Self> for BitVec { 214 | #[inline(always)] 215 | fn bitand_assign(&mut self, other: &Self) { 216 | let _ = self.and(other); 217 | } 218 | } 219 | 220 | impl std::ops::BitAnd<&BitVec> for &BitVec { 221 | type Output = BitVec; 222 | #[inline(always)] 223 | fn bitand(self, other: &BitVec) -> BitVec { 224 | let mut rv = self.clone(); 225 | let _ = rv.and(other); 226 | rv 227 | } 228 | } 229 | 230 | impl std::ops::BitAnd<&Self> for BitVec { 231 | type Output = Self; 232 | #[inline(always)] 233 | fn bitand(mut self, other: &Self) -> Self { 234 | let _ = self.and(other); 235 | self 236 | } 237 | } 238 | 239 | const BYTES_PER_BLOCK: usize = size_of::(); 240 | const BITS_PER_BLOCK: usize = BYTES_PER_BLOCK * 8; 241 | 242 | #[inline(always)] 243 | /// Takes as input a number of bits requiring storage; returns an aligned number of blocks needed 244 | /// to store those bits. 245 | const fn blocks_required(num_bits: usize) -> usize { 246 | let n = num_bits / BITS_PER_BLOCK; 247 | if !num_bits.is_multiple_of(BITS_PER_BLOCK) { 248 | n + 1 249 | } else { 250 | n 251 | } 252 | } 253 | 254 | #[inline(always)] 255 | /// Return the offset in the vector of the storage block storing the bit `off`. 256 | const fn block_offset(off: usize) -> usize { 257 | off / BITS_PER_BLOCK 258 | } 259 | 260 | #[cfg(test)] 261 | mod tests { 262 | use super::*; 263 | 264 | #[test] 265 | fn push_adjusts_vec_correctly_one() { 266 | let mut v = BitVec::new(); 267 | assert_eq!(v.data.len(), 0); 268 | v.push(false); 269 | assert_eq!(v.data.len(), 1); 270 | } 271 | 272 | fn random_bitvec(len: usize) -> BitVec { 273 | let mut vob = BitVec::with_capacity(len); 274 | for _ in 0..len { 275 | vob.push(rand::random()); 276 | } 277 | // these tests can later be dialed down, as they noticeable slow down every random vob test. 278 | assert_eq!( 279 | vob.iter_ones().count(), 280 | vob.iter_ones().filter(|_| true).count() 281 | ); 282 | vob 283 | } 284 | 285 | #[test] 286 | fn test_count() { 287 | for test_len in 1..128 { 288 | let _ = random_bitvec(test_len); 289 | } 290 | } 291 | 292 | #[test] 293 | fn test_iter_ones() { 294 | #[allow(clippy::needless_pass_by_value)] 295 | fn t(v: &BitVec, expected: Vec) { 296 | assert_eq!(v.iter_ones().collect::>(), expected); 297 | } 298 | 299 | t(&BitVec::repeat(true, 131), (0..131).collect::>()); 300 | 301 | let mut v1 = BitVec::new(); 302 | 303 | v1.push(false); 304 | v1.push(true); 305 | v1.push(false); 306 | v1.push(true); 307 | 308 | t(&v1, vec![1, 3]); 309 | } 310 | } 311 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | # 0.6.1 (2025-11-18) 9 | 10 | Other Changes: 11 | 12 | * Update dependencies (incl major update from Hashbrown from 0.15 to 0.16) 13 | * Update homepage from venndb.rs to venndb.plabayo.tech (save on registrar costs) 14 | 15 | # 0.6.0 (2025-08-23) 16 | 17 | Breaking Changes: 18 | 19 | * Bump MSRV to 1.88 and rust edition 2024; 20 | 21 | Other Changes: 22 | 23 | * Replace `bitvec` crate dep with internal custom impl; 24 | * `bitvec` was no longer maintained and starting 25 | to cause issues on nightly; 26 | 27 | # 0.5.0 (2024-05-23) 28 | 29 | New Features: 30 | 31 | - [[`#15`](https://github.com/plabayo/venndb/issues/15)]: support multi dimensional filter options to be filtered in group 32 | 33 | Example: 34 | 35 | ```rust 36 | use venndb::{Any, VennDB}; 37 | 38 | #[derive(Debug, VennDB)] 39 | pub struct Value { 40 | #[venndb(filter)] 41 | pub foo: String, 42 | pub bar: u32, 43 | } 44 | 45 | let db = ValueDB::from_iter([ 46 | Value { 47 | foo: "a".to_owned(), 48 | bar: 8, 49 | }, 50 | Value { 51 | foo: "b".to_owned(), 52 | bar: 12, 53 | }, 54 | Value { 55 | foo: "c".to_owned(), 56 | bar: 16, 57 | }, 58 | ].into_Iter()).unwrap(); 59 | 60 | let mut query = db.query(); 61 | query.foo(MyString("a".to_owned())); 62 | query.foo(MyString("c".to_owned())); 63 | let values: Vec<_> = query.execute().unwrap().iter().collect(); 64 | assert_eq!(values.len(), 2); 65 | assert_eq!(values[0].bar, 8); 66 | assert_eq!(values[0].bar, 16); 67 | ``` 68 | 69 | # 0.4.0 (2024-04-19) 70 | 71 | Breaking Changes: 72 | 73 | * [[`#7`](https://github.com/plabayo/venndb/issues/7)]: correct the behaviour of any filter map query values: 74 | - When using an any value as a query filter map value it will now only match rows 75 | which have an any value registered for the row; 76 | - Prior to this release it was matching on all rows, as if the filter wasn't defined. 77 | This seemed correct when deciding on it, but on hindsight is is incorrect behaviour. 78 | 79 | New Features: 80 | 81 | * [[`#8`](https://github.com/plabayo/venndb/issues/8)]: support custom validations of rows prior to appending them 82 | 83 | Example: 84 | 85 | ```rust 86 | #[derive(Debug, VennDB)] 87 | #[venndb(name = "MyDB", validator = my_validator_fn)] 88 | pub struct Value { 89 | pub foo: String, 90 | pub bar: u32, 91 | } 92 | 93 | fn my_validator_fn(value: &Value) -> bool { 94 | !value.foo.is_empty() && value.bar > 0 95 | } 96 | 97 | let mut db = MyDB::default(); 98 | assert!(db.append(Value { 99 | foo: "".to_owned(), 100 | bar: 42, 101 | }).is_err()); // fails because foo == empty 102 | ``` 103 | 104 | # 0.3.0 (2024-04-18) 105 | 106 | Breaking Changes: 107 | 108 | * [[`#6`](https://github.com/plabayo/venndb/issues/6)] query filter maps now accept arguments as `impl Into` instead of `T`, 109 | this can be a breaking change for users that were inserting them as `value.into()`, 110 | as the compiler will for these cases now give a compile time error due to the now introduced ambiguity; 111 | * Migration is as easy as removing the manual `.into()` (like) calls that you previously had to add yourself; 112 | 113 | Bug Fixes from [0.2.1](#021-2024-04-15): 114 | 115 | * [[`#5`](https://github.com/plabayo/venndb/issues/5)] any filters now also allow rows to match on unknown filter map variants. 116 | * e.g. if you have a `MyType` filter map and have not a single row that has `"foo"` as value, 117 | then all rows that that have a value for which `assert!(Any::is_any(value: MyType))` will still work. 118 | * prior to this bug fix these values could not be matched on, and the `any` rows would only hit 119 | if there were also rows that had that value explicitly defined. 120 | 121 | # 0.2.1 (2024-04-15) 122 | 123 | A backwards compatible patch for [v0.2.0](#020-2024-04-15), 124 | to support rows that allow any value for a specific column. 125 | 126 | Non-Breaking changes: 127 | 128 | * support `#[venndb(any)]` filters; 129 | * these are possible only for `T` filter maps, where `T: ::venndb::Any`; 130 | * `bool` filters cannot be `any` as `bool` doesn't implement the `::venndb::Any` trait; 131 | * rows that are `any` will match regardless of the query filter used for that property; 132 | 133 | Example usage: 134 | 135 | ```rust 136 | use venndb::{Any, VennDB}; 137 | #[derive(Debug, PartialEq, Eq, Clone, Hash)] 138 | pub enum Department { 139 | Any, 140 | Hr, 141 | Engineering, 142 | } 143 | 144 | impl Any for Department { 145 | fn is_any(&self) -> bool { 146 | self == Department::Any 147 | } 148 | } 149 | 150 | #[derive(Debug, VennDB)] 151 | pub struct Employee { 152 | name: String, 153 | #[venndb(filter, any)] 154 | department: Department, 155 | } 156 | 157 | let db = EmployeeDB::from_iter([ 158 | Employee { name: "Jack".to_owned(), department: Department::Any }, 159 | Employee { name: "Derby".to_owned(), department: Department::Hr }, 160 | ]); 161 | let mut query = db.query(); 162 | 163 | // will match Jack and Derby, as Jack is marked as Any, meaning it can work for w/e value 164 | let hr_employees: Vec<_> = query.department(Department::Hr).execute().unwrap().iter().collect(); 165 | assert_eq!(hr_employees.len(), 2); 166 | ``` 167 | 168 | In case you combine it with the filter map property being optional (`department: Option`), 169 | then it will still work the same, where rows with `None` are seen as nothing at all and just ignored. 170 | This has no affect on the correct functioning of `Any`. 171 | 172 | # 0.2.0 (2024-04-15) 173 | 174 | Breaking Changes: 175 | 176 | * Support Option in a special way: 177 | * for filters it means that both positive and negative bits will be set to false if the value is `None`; 178 | * for filter maps this means that the filter is not even registered; 179 | * keys cannot be optional; 180 | * While technically this is a breaking change it is not expected to actually break someone, 181 | as keys always had to be unique already and two times `None` will result in same hash... so it is unlikely 182 | that there was an `Option` already used by someone; 183 | * this is potentially breaking as some implementations from `0.1*` might have already used `Option` in a different way; 184 | 185 | While this changes behaviour of `filters` and `filter maps` it is unlikely that someone was already using 186 | `Option` for these types before, as their ergonomics have been a bit weird prior to this version. 187 | Even more so for `filter maps` it could have resulted in panics. 188 | 189 | Options, be it filters of filter maps, allow you to have rows that do not register any value for optional 190 | properties, allowing them to exist without affecting the rows which do have it. 191 | 192 | Non-Breaking Changes: 193 | 194 | * improve documentation; 195 | 196 | Updated Example from 0.1: 197 | 198 | ```rust 199 | use venndb::VennDB 200 | 201 | #[derive(Debug, VennDB)] 202 | pub struct Employee { 203 | #[venndb(key)] 204 | id: u32, 205 | name: String, 206 | is_manager: Option, 207 | is_admin: bool, 208 | #[venndb(skip)] 209 | foo: bool, 210 | #[venndb(filter)] 211 | department: Department, 212 | #[venndb(filter)] 213 | country: Option, 214 | } 215 | 216 | fn main() { 217 | let db = EmployeeDB::from_iter(/* .. */); 218 | 219 | let mut query = db.query(); 220 | let employee = query 221 | .is_admin(true) 222 | .is_manager(false) // rows which have `None` for this property will NOT match this filter 223 | .department(Department::Engineering) 224 | .execute() 225 | .expect("to have found at least one") 226 | .any(); 227 | 228 | println!("non-manager admin engineer: {:?}", employee); 229 | // as we didn't specify a `country` filter, even rows without a country specified will 230 | // match here if they match the defined (query) filters) 231 | } 232 | ``` 233 | 234 | # 0.1.1 (2024-04-10) 235 | 236 | Non-Breaking Changes: 237 | 238 | * fix clippy linter warning errors (`missing_docs`): `missing documentation for a variant`; 239 | 240 | # 0.1.0 (2024-04-09) 241 | 242 | Implement the first version of this library, `venndb`. 243 | Released as `venndb` (0.1.0) and `venndb-macros` (0.1.0). 244 | 245 | API Example: 246 | 247 | ```rust 248 | use venndb::VennDB 249 | 250 | #[derive(Debug, VennDB)] 251 | pub struct Employee { 252 | #[venndb(key)] 253 | id: u32, 254 | name: String, 255 | is_manager: bool, 256 | is_admin: bool, 257 | #[venndb(skip)] 258 | foo: bool, 259 | #[venndb(filter)] 260 | department: Department, 261 | } 262 | 263 | fn main() { 264 | let db = EmployeeDB::from_iter(/* .. */); 265 | 266 | let mut query = db.query(); 267 | let employee = query 268 | .is_admin(true) 269 | .is_manager(false) 270 | .department(Department::Engineering) 271 | .execute() 272 | .expect("to have found at least one") 273 | .any(); 274 | 275 | println!("non-manager admin engineer: {:?}", employee); 276 | } 277 | ``` 278 | 279 | This API, using nothing more then a `derive` macro allows to: 280 | 281 | - store rows of data in a generated `database`; 282 | - query the database using the defined `filter` fields; 283 | - get references to rows directly by a `key`; 284 | 285 | The crate comes with examples and a detailed README. 286 | 287 | Please share with us if you have any feedback about this first version, 288 | how you are using it, what you would like different, etc. 289 | -------------------------------------------------------------------------------- /venndb-macros/src/parse_attrs.rs: -------------------------------------------------------------------------------- 1 | use quote::ToTokens; 2 | 3 | use crate::errors::Errors; 4 | 5 | /// Attributes applied to a field of a `#![derive(VennDB)]` struct. 6 | #[derive(Default)] 7 | pub struct FieldAttrs<'a> { 8 | pub kind: Option, 9 | pub option_ty: Option<&'a syn::Type>, 10 | } 11 | 12 | pub enum FieldKind { 13 | Key, 14 | Filter, 15 | FilterMap { any: bool }, 16 | } 17 | 18 | impl<'a> FieldAttrs<'a> { 19 | pub fn parse(errors: &Errors, field: &'a syn::Field) -> Self { 20 | let mut this = Self::default(); 21 | 22 | let mut skipped = false; 23 | let mut is_key = false; 24 | let mut is_filter = false; 25 | let mut is_any = false; 26 | 27 | for attr in &field.attrs { 28 | let ml: Vec<_> = if let Some(ml) = venndb_attr_to_meta_list(errors, attr) { 29 | ml.into_iter().collect() 30 | } else { 31 | continue; 32 | }; 33 | 34 | if ml.iter().any(|meta| meta.path().is_ident("skip")) { 35 | // check first to avoid any other invalid combinations 36 | skipped = true; 37 | } else { 38 | for meta in ml { 39 | let name = meta.path(); 40 | if name.is_ident("key") { 41 | if is_filter { 42 | errors.err( 43 | &meta, 44 | concat!( 45 | "Invalid field-level `venndb` attribute\n", 46 | "Cannot have both `key` and `filter`", 47 | ), 48 | ); 49 | } else if is_any { 50 | errors.err( 51 | &meta, 52 | concat!( 53 | "Invalid field-level `venndb` attribute\n", 54 | "Cannot have both `key` and `any`", 55 | ), 56 | ); 57 | } else { 58 | is_key = true; 59 | } 60 | } else if name.is_ident("filter") { 61 | if is_key { 62 | errors.err( 63 | &meta, 64 | concat!( 65 | "Invalid field-level `venndb` attribute\n", 66 | "Cannot have both `key` and `filter`", 67 | ), 68 | ); 69 | } else { 70 | is_filter = true; 71 | } 72 | } else if name.is_ident("any") { 73 | if is_key { 74 | errors.err( 75 | &meta, 76 | concat!( 77 | "Invalid field-level `venndb` attribute\n", 78 | "Cannot have both `key` and `any`", 79 | ), 80 | ); 81 | } else { 82 | is_any = true; 83 | } 84 | } else { 85 | errors.err( 86 | &meta, 87 | concat!( 88 | "Invalid field-level `venndb` attribute\n", 89 | "Expected one of: `key`", 90 | ), 91 | ); 92 | } 93 | } 94 | } 95 | } 96 | 97 | this.option_ty = ty_inner(&["Option"], &field.ty); 98 | 99 | if skipped { 100 | this.kind = None; 101 | } else if is_key { 102 | if this.option_ty.is_some() { 103 | errors.err( 104 | &field.ty, 105 | concat!( 106 | "Invalid field-level `venndb` attribute\n", 107 | "`key` fields cannot be `Option`", 108 | ), 109 | ); 110 | } else { 111 | this.kind = Some(FieldKind::Key); 112 | } 113 | } else if is_bool(this.option_ty.unwrap_or(&field.ty)) { 114 | if is_any { 115 | errors.err( 116 | &field.ty, 117 | concat!( 118 | "Invalid field-level `venndb` attribute\n", 119 | "`any` cannot be used with `bool`", 120 | ), 121 | ); 122 | } else { 123 | this.kind = Some(FieldKind::Filter); 124 | } 125 | } else if is_filter { 126 | // bool filters are to be seen as regular filters, even when made explicitly so! 127 | this.kind = Some(FieldKind::FilterMap { any: is_any }); 128 | } else if is_any { 129 | errors.err( 130 | &field.ty, 131 | concat!( 132 | "Invalid field-level `venndb` attribute\n", 133 | "`any` can only be used with `filter`", 134 | ), 135 | ); 136 | } 137 | 138 | this 139 | } 140 | } 141 | 142 | fn is_bool(ty: &syn::Type) -> bool { 143 | if let syn::Type::Path(syn::TypePath { path, .. }) = ty { 144 | path.is_ident("bool") 145 | } else { 146 | if ty.to_token_stream().to_string().contains("bool") { 147 | panic!( 148 | "Expected bool, found {:?}", 149 | ty.to_token_stream().to_string() 150 | ); 151 | } 152 | false 153 | } 154 | } 155 | 156 | /// Represents a `#[derive(VennDB)]` type's top-level attributes. 157 | #[derive(Default)] 158 | pub struct TypeAttrs { 159 | pub name: Option, 160 | pub validator: Option, 161 | } 162 | 163 | impl TypeAttrs { 164 | /// Parse top-level `#[venndb(...)]` attributes 165 | pub fn parse(errors: &Errors, derive_input: &syn::DeriveInput) -> Self { 166 | let mut this = Self::default(); 167 | 168 | for attr in &derive_input.attrs { 169 | let ml = if let Some(ml) = venndb_attr_to_meta_list(errors, attr) { 170 | ml 171 | } else { 172 | continue; 173 | }; 174 | 175 | for meta in ml { 176 | let name = meta.path(); 177 | if name.is_ident("name") { 178 | if let Some(m) = errors.expect_meta_name_value(&meta) { 179 | this.name = errors.expect_lit_str(&m.value).cloned(); 180 | } 181 | } else if name.is_ident("validator") { 182 | if let Some(m) = errors.expect_meta_name_value(&meta) { 183 | this.validator = errors.expect_path(&m.value).cloned(); 184 | } 185 | } else { 186 | errors.err( 187 | &meta, 188 | concat!( 189 | "Invalid field-level `venndb` attribute\n", 190 | "Expected one of: `name`", 191 | ), 192 | ); 193 | } 194 | } 195 | } 196 | 197 | this 198 | } 199 | } 200 | 201 | /// Filters out non-`#[venndb(...)]` attributes and converts to a sequence of `syn::Meta`. 202 | fn venndb_attr_to_meta_list( 203 | errors: &Errors, 204 | attr: &syn::Attribute, 205 | ) -> Option> { 206 | if !is_venndb_attr(attr) { 207 | return None; 208 | } 209 | let ml = errors.expect_meta_list(&attr.meta)?; 210 | errors.ok(ml.parse_args_with( 211 | syn::punctuated::Punctuated::::parse_terminated, 212 | )) 213 | } 214 | 215 | // Whether the attribute is one like `#[ ...]` 216 | fn is_matching_attr(name: &str, attr: &syn::Attribute) -> bool { 217 | attr.path().segments.len() == 1 && attr.path().segments[0].ident == name 218 | } 219 | 220 | /// Checks for `#[venndb ...]` 221 | fn is_venndb_attr(attr: &syn::Attribute) -> bool { 222 | is_matching_attr("venndb", attr) 223 | } 224 | 225 | /// Returns `Some(T)` if a type is `wrapper_name` for any `wrapper_name` in `wrapper_names`. 226 | fn ty_inner<'a>(wrapper_names: &[&str], ty: &'a syn::Type) -> Option<&'a syn::Type> { 227 | if let syn::Type::Path(path) = ty { 228 | if path.qself.is_some() { 229 | return None; 230 | } 231 | // Since we only check the last path segment, it isn't necessarily the case that 232 | // we're referring to `std::vec::Vec` or `std::option::Option`, but there isn't 233 | // a fool proof way to check these since name resolution happens after macro expansion, 234 | // so this is likely "good enough" (so long as people don't have their own types called 235 | // `Option` or `Vec` that take one generic parameter they're looking to parse). 236 | let last_segment = path.path.segments.last()?; 237 | if !wrapper_names.iter().any(|name| last_segment.ident == *name) { 238 | return None; 239 | } 240 | if let syn::PathArguments::AngleBracketed(gen_args) = &last_segment.arguments { 241 | let generic_arg = gen_args.args.first()?; 242 | if let syn::GenericArgument::Type(ty) = &generic_arg { 243 | return Some(ty); 244 | } 245 | } 246 | } 247 | None 248 | } 249 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /benches/proxies/mod.rs: -------------------------------------------------------------------------------- 1 | use sqlite::Row; 2 | use std::{borrow::Cow, ops::Deref}; 3 | use venndb::{Any, VennDB}; 4 | 5 | pub(super) trait ProxyDB: Sized { 6 | fn create(n: usize) -> Self; 7 | 8 | fn get(&self, id: u64) -> Option>; 9 | fn any_tcp(&self, pool: &str, country: &str) -> Option>; 10 | fn any_socks5_isp(&self, pool: &str, country: &str) -> Option>; 11 | } 12 | 13 | #[derive(Debug, Clone, VennDB)] 14 | #[venndb(name = "InMemProxyDB")] 15 | pub(super) struct Proxy { 16 | #[venndb(key)] 17 | pub(super) id: u64, 18 | pub(super) address: String, 19 | pub(super) username: String, 20 | pub(super) password: String, 21 | pub(super) tcp: bool, 22 | pub(super) udp: bool, 23 | pub(super) http: bool, 24 | pub(super) socks5: bool, 25 | pub(super) datacenter: bool, 26 | pub(super) residential: bool, 27 | pub(super) mobile: bool, 28 | #[venndb(filter)] 29 | pub(super) pool: Option, 30 | #[venndb(filter, any)] 31 | pub(super) country: NormalizedString, 32 | } 33 | 34 | #[derive(Debug, Clone, PartialEq, Eq, Hash)] 35 | pub(super) struct NormalizedString(String); 36 | 37 | impl> From for NormalizedString { 38 | fn from(s: S) -> Self { 39 | Self(s.as_ref().trim().to_lowercase()) 40 | } 41 | } 42 | 43 | impl Any for NormalizedString { 44 | fn is_any(&self) -> bool { 45 | self.0 == "*" 46 | } 47 | } 48 | 49 | impl Deref for NormalizedString { 50 | type Target = str; 51 | 52 | fn deref(&self) -> &Self::Target { 53 | &self.0 54 | } 55 | } 56 | 57 | impl From for Proxy { 58 | fn from(s: String) -> Self { 59 | let mut parts = s.split(','); 60 | Self { 61 | id: parts.next().unwrap().parse().unwrap(), 62 | address: parts.next().unwrap().to_owned(), 63 | username: parts.next().unwrap().to_owned(), 64 | password: parts.next().unwrap().to_owned(), 65 | tcp: parts.next().unwrap().parse().unwrap(), 66 | udp: parts.next().unwrap().parse().unwrap(), 67 | http: parts.next().unwrap().parse().unwrap(), 68 | socks5: parts.next().unwrap().parse().unwrap(), 69 | datacenter: parts.next().unwrap().parse().unwrap(), 70 | residential: parts.next().unwrap().parse().unwrap(), 71 | mobile: parts.next().unwrap().parse().unwrap(), 72 | pool: match parts.next().unwrap() { 73 | "" => None, 74 | s => Some(s.into()), 75 | }, 76 | country: parts.next().unwrap().into(), 77 | } 78 | } 79 | } 80 | const RAW_PROXIES_CSV: &str = include_str!("fake_proxies.csv"); 81 | 82 | impl ProxyDB for InMemProxyDB { 83 | fn create(n: usize) -> Self { 84 | let mut db = Self::with_capacity(n); 85 | for line in RAW_PROXIES_CSV.lines().take(n) { 86 | db.append(Proxy::from(line.to_owned())).unwrap(); 87 | } 88 | db 89 | } 90 | 91 | fn get(&self, id: u64) -> Option> { 92 | self.get_by_id(&id).map(Cow::Borrowed) 93 | } 94 | 95 | fn any_tcp(&self, pool: &str, country: &str) -> Option> { 96 | let mut query = self.query(); 97 | query.tcp(true).pool(pool).country(country); 98 | query.execute().map(|result| { 99 | let proxy_ref = result.any(); 100 | Cow::Borrowed(proxy_ref) 101 | }) 102 | } 103 | 104 | fn any_socks5_isp(&self, pool: &str, country: &str) -> Option> { 105 | let mut query = self.query(); 106 | query 107 | .socks5(true) 108 | .datacenter(true) 109 | .residential(true) 110 | .pool(pool) 111 | .country(country); 112 | query.execute().map(|result| { 113 | let proxy_ref = result.any(); 114 | Cow::Borrowed(proxy_ref) 115 | }) 116 | } 117 | } 118 | 119 | #[derive(Debug)] 120 | pub(super) struct NaiveProxyDB { 121 | proxies: Vec, 122 | } 123 | 124 | impl ProxyDB for NaiveProxyDB { 125 | fn create(n: usize) -> Self { 126 | let proxies = RAW_PROXIES_CSV 127 | .lines() 128 | .take(n) 129 | .map(|line| Proxy::from(line.to_owned())) 130 | .collect(); 131 | Self { proxies } 132 | } 133 | 134 | fn get(&self, id: u64) -> Option> { 135 | self.proxies.iter().find(|p| p.id == id).map(Cow::Borrowed) 136 | } 137 | 138 | fn any_tcp(&self, pool: &str, country: &str) -> Option> { 139 | let found_proxies: Vec<_> = self 140 | .proxies 141 | .iter() 142 | .filter(|p| p.tcp && p.pool == Some(pool.into()) && p.country == country.into()) 143 | .collect(); 144 | if found_proxies.is_empty() { 145 | None 146 | } else { 147 | use rand::Rng; 148 | let index = rand::rng().random_range(0..found_proxies.len()); 149 | Some(Cow::Borrowed(found_proxies[index])) 150 | } 151 | } 152 | 153 | fn any_socks5_isp(&self, pool: &str, country: &str) -> Option> { 154 | let found_proxies: Vec<_> = self 155 | .proxies 156 | .iter() 157 | .filter(|p| { 158 | p.socks5 159 | && p.datacenter 160 | && p.residential 161 | && p.pool == Some(pool.into()) 162 | && p.country == country.into() 163 | }) 164 | .collect(); 165 | if found_proxies.is_empty() { 166 | None 167 | } else { 168 | use rand::Rng; 169 | let index = rand::rng().random_range(0..found_proxies.len()); 170 | Some(Cow::Borrowed(found_proxies[index])) 171 | } 172 | } 173 | } 174 | 175 | #[non_exhaustive] 176 | pub(super) struct SqlLiteProxyDB { 177 | conn: sqlite::Connection, 178 | } 179 | 180 | impl std::fmt::Debug for SqlLiteProxyDB { 181 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 182 | f.debug_struct("SqlLiteProxyDB").finish() 183 | } 184 | } 185 | 186 | impl ProxyDB for SqlLiteProxyDB { 187 | fn create(n: usize) -> Self { 188 | let conn = sqlite::open(":memory:").unwrap(); 189 | 190 | // create the DB 191 | conn.execute( 192 | "CREATE TABLE proxies ( 193 | id INTEGER PRIMARY KEY, 194 | address TEXT NOT NULL, 195 | username TEXT NOT NULL, 196 | password TEXT NOT NULL, 197 | tcp BOOLEAN NOT NULL, 198 | udp BOOLEAN NOT NULL, 199 | http BOOLEAN NOT NULL, 200 | socks5 BOOLEAN NOT NULL, 201 | datacenter BOOLEAN NOT NULL, 202 | residential BOOLEAN NOT NULL, 203 | mobile BOOLEAN NOT NULL, 204 | pool TEXT, 205 | country TEXT NOT NULL 206 | )", 207 | ) 208 | .unwrap(); 209 | 210 | // insert the rows 211 | for line in RAW_PROXIES_CSV.lines().take(n) { 212 | let proxy = Proxy::from(line.to_owned()); 213 | let statement = format!( 214 | "INSERT INTO proxies (id, address, username, password, tcp, udp, http, socks5, datacenter, residential, mobile, pool, country) 215 | VALUES ({}, '{}', '{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, '{}')", 216 | proxy.id, 217 | proxy.address, 218 | proxy.username, 219 | proxy.password, 220 | proxy.tcp as i32, 221 | proxy.udp as i32, 222 | proxy.http as i32, 223 | proxy.socks5 as i32, 224 | proxy.datacenter as i32, 225 | proxy.residential as i32, 226 | proxy.mobile as i32, 227 | proxy.pool.map_or("NULL".to_owned(), |s| format!("'{}'", s.deref())), 228 | proxy.country.0, 229 | ); 230 | conn.execute(&statement).unwrap(); 231 | } 232 | 233 | Self { conn } 234 | } 235 | 236 | fn get(&self, id: u64) -> Option> { 237 | let statement = format!("SELECT * FROM proxies WHERE id = {} LIMIT 1", id); 238 | let row = self 239 | .conn 240 | .prepare(&statement) 241 | .unwrap() 242 | .into_iter() 243 | .next()? 244 | .ok()?; 245 | let proxy = proxy_from_sql_row(&row); 246 | Some(Cow::Owned(proxy)) 247 | } 248 | 249 | fn any_tcp(&self, pool: &str, country: &str) -> Option> { 250 | let statement = format!( 251 | "SELECT * FROM proxies WHERE tcp = 1 AND pool = '{}' AND country = '{}' ORDER BY RANDOM() LIMIT 1", 252 | NormalizedString::from(pool).0, 253 | NormalizedString::from(country).0 254 | ); 255 | let row = self 256 | .conn 257 | .prepare(&statement) 258 | .unwrap() 259 | .into_iter() 260 | .next()? 261 | .ok()?; 262 | let proxy = proxy_from_sql_row(&row); 263 | Some(Cow::Owned(proxy)) 264 | } 265 | 266 | fn any_socks5_isp(&self, pool: &str, country: &str) -> Option> { 267 | let statement = format!( 268 | "SELECT * FROM proxies WHERE socks5 = 1 AND datacenter = 1 AND residential = 1 AND pool = '{}' AND country = '{}' ORDER BY RANDOM() LIMIT 1", 269 | NormalizedString::from(pool).0, 270 | NormalizedString::from(country).0 271 | ); 272 | let row = self 273 | .conn 274 | .prepare(&statement) 275 | .unwrap() 276 | .into_iter() 277 | .next()? 278 | .ok()?; 279 | let proxy = proxy_from_sql_row(&row); 280 | Some(Cow::Owned(proxy)) 281 | } 282 | } 283 | 284 | fn proxy_from_sql_row(row: &Row) -> Proxy { 285 | Proxy { 286 | id: row.read::("id") as u64, 287 | address: row.read::<&str, _>("address").to_owned(), 288 | username: row.read::<&str, _>("username").to_owned(), 289 | password: row.read::<&str, _>("password").to_owned(), 290 | tcp: row.read::("tcp") != 0, 291 | udp: row.read::("udp") != 0, 292 | http: row.read::("http") != 0, 293 | socks5: row.read::("socks5") != 0, 294 | datacenter: row.read::("datacenter") != 0, 295 | residential: row.read::("residential") != 0, 296 | mobile: row.read::("mobile") != 0, 297 | pool: row.try_read::<&str, _>("pool").ok().map(Into::into), 298 | country: row.read::<&str, _>("country").into(), 299 | } 300 | } 301 | -------------------------------------------------------------------------------- /scripts/plot_bench_charts.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | # Define input data 4 | 5 | input_data = """ 6 | proxydb fastest │ slowest │ median │ mean │ samples │ iters 7 | ├─ naive_proxy_db_100 6.499 µs │ 18.04 µs │ 7.999 µs │ 7.929 µs │ 100 │ 100 8 | │ alloc: │ │ │ │ │ 9 | │ 71 │ 71 │ 71 │ 70.29 │ │ 10 | │ 334 B │ 334 B │ 334 B │ 330.6 B │ │ 11 | │ dealloc: │ │ │ │ │ 12 | │ 71 │ 71 │ 71 │ 70.29 │ │ 13 | │ 334 B │ 334 B │ 334 B │ 330.6 B │ │ 14 | ├─ naive_proxy_db_100_000 3.79 ms │ 5.731 ms │ 3.837 ms │ 3.9 ms │ 100 │ 100 15 | │ alloc: │ │ │ │ │ 16 | │ 68046 │ 68047 │ 68046 │ 68046 │ │ 17 | │ 323.3 KB │ 323.7 KB │ 323.3 KB │ 323.3 KB │ │ 18 | │ dealloc: │ │ │ │ │ 19 | │ 68046 │ 68046 │ 68046 │ 68046 │ │ 20 | │ 328.4 KB │ 328.4 KB │ 328.4 KB │ 328.4 KB │ │ 21 | │ grow: │ │ │ │ │ 22 | │ 12 │ 12 │ 12 │ 12 │ │ 23 | │ 5.056 KB │ 5.056 KB │ 5.056 KB │ 5.056 KB │ │ 24 | ├─ naive_proxy_db_12_500 404 µs │ 478.7 µs │ 407.7 µs │ 412.7 µs │ 100 │ 100 25 | │ alloc: │ │ │ │ │ 26 | │ 8549 │ 8549 │ 8549 │ 8549 │ │ 27 | │ 40.73 KB │ 40.73 KB │ 40.73 KB │ 40.73 KB │ │ 28 | │ dealloc: │ │ │ │ │ 29 | │ 8549 │ 8549 │ 8549 │ 8549 │ │ 30 | │ 41.31 KB │ 41.31 KB │ 41.31 KB │ 41.31 KB │ │ 31 | │ grow: │ │ │ │ │ 32 | │ 6 │ 6 │ 6 │ 6 │ │ 33 | │ 576 B │ 576 B │ 576 B │ 576 B │ │ 34 | ├─ sql_lite_proxy_db_100 32.58 µs │ 302 µs │ 37.37 µs │ 42.32 µs │ 100 │ 100 35 | │ alloc: │ │ │ │ │ 36 | │ 97 │ 97 │ 97 │ 97 │ │ 37 | │ 4.043 KB │ 4.043 KB │ 4.043 KB │ 4.043 KB │ │ 38 | │ dealloc: │ │ │ │ │ 39 | │ 97 │ 97 │ 97 │ 97 │ │ 40 | │ 4.043 KB │ 4.043 KB │ 4.043 KB │ 4.043 KB │ │ 41 | ├─ sql_lite_proxy_db_100_000 8.219 ms │ 9.424 ms │ 8.298 ms │ 8.34 ms │ 100 │ 100 42 | │ alloc: │ │ │ │ │ 43 | │ 119 │ 119 │ 119 │ 119 │ │ 44 | │ 5.023 KB │ 5.015 KB │ 5.022 KB │ 5.024 KB │ │ 45 | │ dealloc: │ │ │ │ │ 46 | │ 119 │ 119 │ 119 │ 119 │ │ 47 | │ 5.023 KB │ 5.015 KB │ 5.022 KB │ 5.024 KB │ │ 48 | ├─ sql_lite_proxy_db_12_500 1.061 ms │ 1.727 ms │ 1.073 ms │ 1.094 ms │ 100 │ 100 49 | │ alloc: │ │ │ │ │ 50 | │ 119 │ 119 │ 119 │ 119 │ │ 51 | │ 5.025 KB │ 5.027 KB │ 5.023 KB │ 5.023 KB │ │ 52 | │ dealloc: │ │ │ │ │ 53 | │ 119 │ 119 │ 119 │ 119 │ │ 54 | │ 5.025 KB │ 5.027 KB │ 5.023 KB │ 5.023 KB │ │ 55 | ├─ venn_proxy_db_100 894.9 ns │ 2.738 µs │ 915.9 ns │ 946.2 ns │ 100 │ 400 56 | │ alloc: │ │ │ │ │ 57 | │ 6 │ 6 │ 6 │ 6 │ │ 58 | │ 46 B │ 46 B │ 46 B │ 46 B │ │ 59 | │ dealloc: │ │ │ │ │ 60 | │ 6 │ 6 │ 6 │ 6 │ │ 61 | │ 46 B │ 46 B │ 46 B │ 46 B │ │ 62 | ├─ venn_proxy_db_100_000 124.2 µs │ 156.3 µs │ 129.2 µs │ 130.4 µs │ 100 │ 100 63 | │ alloc: │ │ │ │ │ 64 | │ 6 │ 6 │ 6 │ 6 │ │ 65 | │ 25.02 KB │ 25.02 KB │ 25.02 KB │ 25.02 KB │ │ 66 | │ dealloc: │ │ │ │ │ 67 | │ 6 │ 6 │ 6 │ 6 │ │ 68 | │ 25.02 KB │ 25.02 KB │ 25.02 KB │ 25.02 KB │ │ 69 | ╰─ venn_proxy_db_12_500 16.04 µs │ 25.54 µs │ 16.97 µs │ 17.72 µs │ 100 │ 100 70 | alloc: │ │ │ │ │ 71 | 6 │ 6 │ 6 │ 6 │ │ 72 | 3.15 KB │ 3.15 KB │ 3.15 KB │ 3.15 KB │ │ 73 | dealloc: │ │ │ │ │ 74 | 6 │ 6 │ 6 │ 6 │ │ 75 | 3.15 KB │ 3.15 KB │ 3.15 KB │ 3.15 KB │ │ 76 | """ # noqa:E501 77 | 78 | 79 | ##################################################### 80 | #### PARSE CODE 81 | ##################################################### 82 | 83 | 84 | @dataclass 85 | # Units are in microseconds 86 | class Performance: 87 | fastest: float 88 | slowest: float 89 | median: float 90 | mean: float 91 | 92 | 93 | @dataclass 94 | # Units are in KB 95 | class Allocation: 96 | fastest: float 97 | slowest: float 98 | median: float 99 | mean: float 100 | 101 | 102 | results_performance = {} 103 | results_allocation = {} 104 | 105 | input_lines = input_data.splitlines() 106 | current_key = None 107 | while True: 108 | try: 109 | line = input_lines.pop(0) 110 | except IndexError: 111 | break 112 | 113 | line = line.strip() 114 | if not line or line.startswith("proxydb"): 115 | continue 116 | 117 | if line.startswith("├") or line.startswith("╰"): 118 | parts = [ 119 | part 120 | for part in line.split(" ") 121 | if part != "" and "│" not in part and "─" not in part 122 | ] 123 | 124 | current_key = parts[0] 125 | 126 | def get_value(parts, index): 127 | (value, unit) = parts[index : index + 2] # noqa:E203 128 | value = float(value) 129 | if unit == "ms": 130 | value *= 1000 131 | elif unit == "ns": 132 | value /= 1000 133 | elif unit != "µs": 134 | raise ValueError(f"Unexpected unit: {unit}") 135 | return value 136 | 137 | fastest = get_value(parts, 1) 138 | slowest = get_value(parts, 3) 139 | median = get_value(parts, 5) 140 | mean = get_value(parts, 7) 141 | 142 | results_performance[current_key] = Performance( 143 | fastest, 144 | slowest, 145 | median, 146 | mean, 147 | ) 148 | 149 | input_lines.pop(0) # alloc 150 | input_lines.pop(0) # count 151 | line = input_lines.pop(0) 152 | 153 | parts = [ 154 | part 155 | for part in line.split(" ") 156 | if part != "" and "│" not in part and "─" not in part 157 | ] 158 | 159 | def get_value(parts, index): 160 | (value, unit) = parts[index : index + 2] # noqa:E203 161 | value = float(value) 162 | if unit == "B": 163 | value /= 1000 164 | elif unit != "KB": 165 | raise ValueError(f"Unexpected unit: {unit}") 166 | return value 167 | 168 | fastest = get_value(parts, 0) 169 | slowest = get_value(parts, 2) 170 | median = get_value(parts, 4) 171 | mean = get_value(parts, 6) 172 | 173 | results_allocation[current_key] = Allocation( 174 | fastest, 175 | slowest, 176 | median, 177 | mean, 178 | ) 179 | 180 | 181 | 182 | ##################################################### 183 | #### PRINTER CODE 184 | ##################################################### 185 | 186 | 187 | def print_performance_chart(results_performance): 188 | # Find the maximum value for each performance metric 189 | max_fastest = max(result.fastest for result in results_performance.values()) 190 | max_slowest = max(result.slowest for result in results_performance.values()) 191 | max_median = max(result.median for result in results_performance.values()) 192 | 193 | # Print the chart header 194 | print(f"{'Performance Results'.center(80)}") 195 | print(f"{'(Units in microseconds)'.center(80)}") 196 | print(f"{'-' * 80}") 197 | 198 | # Print the chart rows 199 | for name, result in results_performance.items(): 200 | fastest_bar = "#" * int(result.fastest / max_fastest * 70) 201 | slowest_bar = "#" * int(result.slowest / max_slowest * 70) 202 | median_bar = "#" * int(result.median / max_median * 70) 203 | 204 | print( 205 | f"{name.ljust(40)} |{fastest_bar.ljust(70)}| {result.fastest:.2f} µs (Fastest)" 206 | ) 207 | print( 208 | f"{''.ljust(40)} |{median_bar.ljust(70)}| {result.median:.2f} µs (Median)" 209 | ) 210 | print( 211 | f"{''.ljust(40)} |{slowest_bar.ljust(70)}| {result.slowest:.2f} µs (Slowest)" 212 | ) 213 | print(f"{'-' * 80}") 214 | 215 | 216 | def print_performance_table(results_performance): 217 | print("| Proxy DB | Fastest (µs) | Median (µs) | Slowest (µs) |") 218 | print("| --- | --- | --- | --- |") 219 | for name, result in results_performance.items(): 220 | print( 221 | f"| {name.ljust(30)} | {result.fastest:.2f} | {result.median:.2f} | {result.slowest:.2f} |" 222 | ) 223 | 224 | 225 | def print_allocation_table(results_allocation): 226 | print("| Proxy DB | Fastest (KB) | Median (KB) | Slowest (KB) |") 227 | print("| --- | --- | --- | --- |") 228 | for name, result in results_allocation.items(): 229 | print( 230 | f"| {name.ljust(30)} | {result.fastest:.2f} | {result.median:.2f} | {result.slowest:.2f} |" 231 | ) 232 | 233 | 234 | ##################################################### 235 | #### PRINT PERFORMANCE OUTPUT 236 | ##################################################### 237 | 238 | 239 | print( 240 | """ 241 | Performance for Database with `100` records: 242 | """ 243 | ) 244 | 245 | print_performance_table( 246 | {key: value for (key, value) in results_performance.items() if key.endswith("_100")} 247 | ) 248 | 249 | 250 | print( 251 | """ 252 | Performance for Database with `12_500` records: 253 | """ 254 | ) 255 | 256 | print_performance_table( 257 | { 258 | key: value 259 | for (key, value) in results_performance.items() 260 | if key.endswith("_12_500") 261 | } 262 | ) 263 | 264 | 265 | print( 266 | """ 267 | Performance for Database with `100_000` records: 268 | """ 269 | ) 270 | 271 | print_performance_table( 272 | { 273 | key: value 274 | for (key, value) in results_performance.items() 275 | if key.endswith("_100_000") 276 | } 277 | ) 278 | 279 | print( 280 | """ 281 | """ 282 | ) 283 | 284 | 285 | ##################################################### 286 | #### PRINT Allocation OUTPUT 287 | ##################################################### 288 | 289 | 290 | print( 291 | """ 292 | Allocations for Database with `100` records: 293 | """ 294 | ) 295 | 296 | print_allocation_table( 297 | {key: value for (key, value) in results_allocation.items() if key.endswith("_100")} 298 | ) 299 | 300 | 301 | print( 302 | """ 303 | Allocations for Database with `12_500` records: 304 | """ 305 | ) 306 | 307 | print_allocation_table( 308 | { 309 | key: value 310 | for (key, value) in results_allocation.items() 311 | if key.endswith("_12_500") 312 | } 313 | ) 314 | 315 | 316 | print( 317 | """ 318 | Allocations for Database with `100_000` records: 319 | """ 320 | ) 321 | 322 | print_allocation_table( 323 | { 324 | key: value 325 | for (key, value) in results_allocation.items() 326 | if key.endswith("_100_000") 327 | } 328 | ) 329 | 330 | print( 331 | """ 332 | """ 333 | ) 334 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # VennDB 2 | 3 | An **append-only** in-memory database in Rust for rows queried using bit (flag) columns. 4 | This database is designed for a very specific use case where you have mostly static data that you typically load at startup and have to query constantly using very simple filters. Datasets 5 | like these can be large and should be both fast and compact. 6 | 7 | For the limited usecases where `venndb` can be applied to, 8 | it has less dependencies and is faster then traditional choices, 9 | such as a naive implementation or a more heavy lifted dependency such as _Sqlite_. 10 | 11 | > See [the benchmarks](#benchmarks) for more information on this topic. 12 | 13 | This project was developed originally in function of [`rama`](https://ramaproxy.org), 14 | where you can see it being used for example to provide an in-memory (upstream) proxy database. 15 | Do let us know in case you use it as well in your project, such that we can assemble a showcase list. 16 | 17 | ![venndb banner](https://raw.githubusercontent.com/plabayo/venndb/main/docs/img/banner.svg) 18 | 19 | [![Crates.io][crates-badge]][crates-url] 20 | [![Docs.rs][docs-badge]][docs-url] 21 | [![MIT License][license-mit-badge]][license-mit-url] 22 | [![Apache 2.0 License][license-apache-badge]][license-apache-url] 23 | [![rust version][rust-version-badge]][rust-version-url] 24 | [![Build Status][actions-badge]][actions-url] 25 | 26 | [![Discord][discord-badge]][discord-url] 27 | [![Buy Me A Coffee][bmac-badge]][bmac-url] 28 | [![GitHub Sponsors][ghs-badge]][ghs-url] 29 | 30 | [crates-badge]: https://img.shields.io/crates/v/venndb.svg 31 | [crates-url]: https://crates.io/crates/venndb 32 | [docs-badge]: https://img.shields.io/docsrs/venndb/latest 33 | [docs-url]: https://docs.rs/venndb/latest/venndb/index.html 34 | [license-mit-badge]: https://img.shields.io/badge/license-MIT-blue.svg 35 | [license-mit-url]: https://github.com/plabayo/venndb/blob/main/LICENSE-MIT 36 | [license-apache-badge]: https://img.shields.io/badge/license-APACHE-blue.svg 37 | [license-apache-url]: https://github.com/plabayo/venndb/blob/main/LICENSE-APACHE 38 | [rust-version-badge]: https://img.shields.io/badge/rustc-1.88+-blue?style=flat-square&logo=rust 39 | [rust-version-url]: https://www.rust-lang.org 40 | [actions-badge]: https://github.com/plabayo/venndb/workflows/CI/badge.svg 41 | [actions-url]: https://github.com/plabayo/venndb/actions 42 | 43 | [discord-badge]: https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white 44 | [discord-url]: https://discord.gg/29EetaSYCD 45 | [bmac-badge]: https://img.shields.io/badge/Buy%20Me%20a%20Coffee-ffdd00?style=for-the-badge&logo=buy-me-a-coffee&logoColor=black 46 | [bmac-url]: https://www.buymeacoffee.com/plabayo 47 | [ghs-badge]: https://img.shields.io/badge/sponsor-30363D?style=for-the-badge&logo=GitHub-Sponsors&logoColor=#EA4AAA 48 | [ghs-url]: https://github.com/sponsors/plabayo 49 | 50 | 💬 Come join us at [Discord][discord-url] on the `#venndb` public channel. To ask questions, discuss ideas and ask how venndb may be useful for you. 51 | 52 | ## Index 53 | 54 | `venndb` manual: 55 | 56 | - [Usage](#usage): quick introduction on how to use `venndb`; 57 | - [Benchmarks](#benchmarks): benchmark results to give you a rough idea how `venndb` peforms for the use case it is made for (write once, read constantly, using binary filters mostly); 58 | - [Q&A](#qa): Frequently Asked Questions (FAQ); 59 | - [Example](#example): the full example (expanded version from [Usage](#usage)), tested and documented; 60 | - [Generated Code Summary](#generated-code-summary): a documented overview of the API that `venndb` will generate for you when using `#[derive(VennDB)]` on your _named field struct_; 61 | 62 | technical information: 63 | 64 | - [Safety](#--safety) 65 | - [Compatibility](#--compatibility) 66 | - [MSRV](#minimum-supported-rust-version) (older versions might work but we make no guarantees); 67 | - [Roadmap](#--roadmap) 68 | - [License](#--license): [MIT license][mit-license] and [Apache 2.0 License][apache-license] 69 | 70 | misc: 71 | 72 | - [Contributing](#--contributing) 73 | - [Sponsors](#--sponsors) 74 | 75 | ## Usage 76 | 77 | Add `venndb` as a dependency: 78 | 79 | ```sh 80 | cargo add venndb 81 | ``` 82 | 83 | and import the `derive` macro in the module where you want to use it: 84 | 85 | ```rust,ignore 86 | use venndb::VennDB 87 | 88 | #[derive(Debug, VennDB)] 89 | pub struct Employee { 90 | #[venndb(key)] 91 | id: u32, 92 | name: String, 93 | is_manager: Option, 94 | is_admin: bool, 95 | #[venndb(skip)] 96 | foo: bool, 97 | #[venndb(filter, any)] 98 | department: Department, 99 | #[venndb(filter)] 100 | country: Option, 101 | } 102 | 103 | fn main() { 104 | let db = EmployeeDB::from_iter(/* .. */); 105 | 106 | let mut query = db.query(); 107 | let employee = query 108 | .is_admin(true) 109 | .is_manager(false) 110 | .department(Department::Engineering) 111 | .execute() 112 | .expect("to have found at least one") 113 | .any(); 114 | 115 | println!("non-manager admin engineer: {:?}", employee); 116 | } 117 | ``` 118 | 119 | See [the full example](#example) or [the "Generated Code Summary" chapter](#generated-code-summary) below 120 | to learn how to use the `VennDB` and its generated code. 121 | 122 | ## Benchmarks 123 | 124 | Benchmarks displayed here are taken on a dev machine with following specs: 125 | 126 | ```text 127 | Macbook Pro — 16 inch (2023) 128 | Chip: Apple M2 Pro 129 | Memory: 16 GB 130 | OS: Sonoma 14.2 131 | ``` 132 | 133 | The benchmarks tests 3 different implementations of a proxy database 134 | 135 | - `venndb` version (very similar to [the example below](#example)) 136 | - a `naive` version, which is just a `Vec`, over which is iterated 137 | - an `sqlite` version (using [the `sqlite` crate (version: `0.34.0`)](https://docs.rs/sqlite/0.34.0/sqlite/)) 138 | 139 | The benchmarks are created by: 140 | 141 | 1. running `just bench`; 142 | 2. copying the output into [./scripts/plot_bench_charts](./scripts/plot_bench_charts.py) and running it. 143 | 144 | Snippet that is ran for each 3 implementations: 145 | 146 | ```rust,ignore 147 | fn test_db(db: &impl ProxyDB) { 148 | let i = next_round(); 149 | 150 | let pool = POOLS[i % POOLS.len()]; 151 | let country = COUNTRIES[i % COUNTRIES.len()]; 152 | 153 | let result = db.get(i as u64); 154 | divan::black_box(result); 155 | 156 | let result = db.any_tcp(pool, country); 157 | divan::black_box(result); 158 | 159 | let result = db.any_socks5_isp(pool, country); 160 | divan::black_box(result); 161 | } 162 | ``` 163 | 164 | ### Benchmark Performance Results 165 | 166 | Performance for Database with `100` records: 167 | 168 | | Proxy DB | Fastest | Slowest | Median | 169 | | --- | --- | --- | --- | 170 | | naive_proxy_db_100 | 1.276 µs | 1.922 µs | 1.328 µs | 171 | | sql_lite_proxy_db_100 | 22.96 µs | 40.84 µs | 26.52 µs | 172 | | venn_proxy_db_100 | 156.6 ns | 243.8 ns | 183.9 ns | 173 | 174 | Performance for Database with `12_500` records: 175 | 176 | | Proxy DB | Fastest | Slowest | Median | 177 | | --- | --- | --- | --- | 178 | | naive_proxy_db_12_500 | 202.7 µs | 260 µs | 217.1 µs | 179 | | sql_lite_proxy_db_12_500 | 22 µs | 903.4 µs | 792.2 µs | 180 | | venn_proxy_db_12_500 | 1.767 µs | 3.558 µs | 2.1 µs | 181 | 182 | Performance for Database with `100_000` records: 183 | 184 | | Proxy DB | Fastest | Slowest | Median | 185 | | --- | --- | --- | --- | 186 | | naive_proxy_db_100_000 | 2.196 ms | 2.563 ms | 2.288 ms | 187 | | sql_lite_proxy_db_100_000 | 5.85 ms | 6.634 ms | 6.223 ms | 188 | | venn_proxy_db_100_000 | 13.01 µs | 30.47 µs | 17.85 µs | 189 | 190 | We are not database nor hardware experts though. Please do open an issue if you think 191 | these benchmarks are incorrect or if related improvements can be made. 192 | Contributions in the form of Pull requests are welcomed as well. 193 | 194 | See [the Contribution guidelines](#contribution) for more information. 195 | 196 | ### Benchmark Allocations Results 197 | 198 | Allocations for Database with `100` records: 199 | 200 | | Proxy DB | Fastest | Median | Slowest | 201 | | --- | --- | --- | --- | 202 | | naive_proxy_db_100 | 1.25 B | 1.25 B | 1.25 B | 203 | | sql_lite_proxy_db_100 | 1.378 KB | 1.378 KB | 1.378 KB | 204 | | venn_proxy_db_100 | 430 B | 430 B | 430 B | 205 | 206 | Allocations for Database with `12_500` records: 207 | 208 | | Proxy DB | Fastest | Median | Slowest | 209 | | --- | --- | --- | --- | 210 | | naive_proxy_db_12_500 | 40.73 KB | 40.73 KB | 40.73 KB | 211 | | sql_lite_proxy_db_12_500 | 5.149 KB | 5.141 KB | 5.144 KB | 212 | | venn_proxy_db_12_500 | 3.534 KB | 3.534 KB | 3.534 KB | 213 | 214 | Allocations for Database with `100_000` records: 215 | 216 | | Proxy DB | Fastest | Median | Slowest | 217 | | --- | --- | --- | --- | 218 | | naive_proxy_db_100_000 | 323.3 KB | 323.3 KB | 323.3 KB | 219 | | sql_lite_proxy_db_100_000 | 5.141 KB | 5.147 KB | 5.143 KB | 220 | | venn_proxy_db_100_000 | 25.4 KB | 25.4 KB | 25.4 KB | 221 | 222 | We are not database nor hardware experts though. Please do open an issue if you think 223 | these benchmarks are incorrect or if related improvements can be made. 224 | Contributions in the form of Pull requests are welcomed as well. 225 | 226 | See [the Contribution guidelines](#contribution) for more information. 227 | 228 | ## Q&A 229 | 230 | > ❓ Why use this over Database X? 231 | 232 | `venndb` is not a database, but is close enough for some specific purposes. It shines for long-lived read-only use cases where you need to filter on plenty of binary properties and get a rando matching result. 233 | 234 | Do not try to replace your usual database needs with it. 235 | 236 | > ❓ Where can I propose a new feature X or some other improvement? 237 | 238 | Please [open an issue](https://github.com/plabayo/venndb/issues) and also read [the Contribution guidelines](#contribution). We look forward to hear from you. 239 | 240 | Alternatively you can also [join our Discord][discord-url] and start a conversation / discussion over there. 241 | 242 | > ❓ Can I use _whatever_ type for a `#[venndb(filter)]` property? 243 | 244 | Yes, as long as it implements `PartialEq + Eq + Hash + Clone`. 245 | That said, we do recommend that you use `enum` values if you can, or some other highly restricted form. 246 | 247 | Using for example a `String` directly is a bad idea as that would mean that `bE` != `Be` != `BE` != `Belgium` != `Belgique` != `België`. Even though these are really referring all to the same country. In such cases a much better idea is to at the very least create a wrapper type such as `struct Country(String)`, to allow you to enforce sanitization/validation when creating the value and ensuring the hashes will be the same for those values that are conceptually the same. 248 | 249 | > ❓ How do I make a filter optional? 250 | 251 | Both filters (`bool` properties) and filter maps (`T != bool` properties with the `#[venndb(filter)]` attribute) 252 | can be made optional by wrapping the types with `Option`, resulting in `Option` and `Option`. 253 | 254 | Rows that have the `Option::None` value for such an optional column cannot filter on that property, 255 | but there is no other consequence beyond that. 256 | 257 | > ❓ Why can do keys have to be unique and non-optional? 258 | 259 | Within `venndb` keys are meant to be able to look up, 260 | a row which was previously received via filters. 261 | 262 | As such it makes no sense for such keys to be: 263 | 264 | - duplicate: it would mean: as that can result in multiple rows or the wrong row to be returned; 265 | - optional: as that would mean the row cannot be looked up when the key is not defined; 266 | 267 | > ❓ How can I allow some rows to match for _any_ value of a certain (filter) column? 268 | 269 | Filter maps can allow to have a value to match all other values. It is up to you to declare the filter as such, 270 | and to also define for that type what the _one_ value to rule them all is. 271 | 272 | Usage: 273 | 274 | ```rust,ignore 275 | use venndb::{Any, VennDB}; 276 | #[derive(Debug, PartialEq, Eq, Clone, Hash)] 277 | pub enum Department { 278 | Any, 279 | Hr, 280 | Engineering, 281 | } 282 | 283 | impl Any for Department { 284 | fn is_any(&self) -> bool { 285 | self == Department::Any 286 | } 287 | } 288 | 289 | #[derive(Debug, VennDB)] 290 | pub struct Employee { 291 | name: String, 292 | #[venndb(filter, any)] 293 | department: Department, 294 | } 295 | 296 | let db = EmployeeDB::from_iter([ 297 | Employee { name: "Jack".to_owned(), department: Department::Any }, 298 | Employee { name: "Derby".to_owned(), department: Department::Hr }, 299 | ]); 300 | let mut query = db.query(); 301 | 302 | // will match Jack and Derby, as Jack is marked as Any, meaning it can work for w/e value 303 | let hr_employees: Vec<_> = query.department(Department::Hr).execute().unwrap().iter().collect(); 304 | assert_eq!(hr_employees.len(), 2); 305 | ``` 306 | 307 | > ❓ How can I provide custom validation of rows prior to them getting appended? 308 | 309 | Is is possible to validate a row based on one or multiple of its properties? Validate in function of relationship 310 | between multiple properties? Is it possible to provide custom validation to prevent rows 311 | from getting appended that do not adhere to custom validation rules? 312 | 313 | Yes to all of the above. 314 | 315 | Example: 316 | 317 | ```rust,ignore 318 | #[derive(Debug, VennDB)] 319 | #[venndb(validator = my_validator_fn)] 320 | pub struct Value { 321 | pub foo: String, 322 | pub bar: u32, 323 | } 324 | 325 | fn my_validator_fn(value: &Value) -> bool { 326 | !value.foo.is_empty() && value.bar > 0 327 | } 328 | 329 | let mut db = ValueDB::default(); 330 | assert!(db.append(Value { 331 | foo: "".to_owned(), 332 | bar: 42, 333 | }).is_err()); // fails because foo == empty 334 | ``` 335 | 336 | > ❓ Why do `any` filter values only match rows that have an `any` value for that property? 337 | 338 | Let's say I have the following `struct`: 339 | 340 | ```rust,ignore 341 | use venndb::{Any, VennDB}; 342 | 343 | #[derive(Debug, VennDB)] 344 | pub struct Value { 345 | #[venndb(filter, any)] 346 | pub foo: MyString, 347 | pub bar: u32, 348 | } 349 | 350 | #[derive(Debug)] 351 | pub struct MyString(String); 352 | 353 | impl Any for MyString { 354 | fn is_any(&self) -> bool { 355 | self.0 == "*" 356 | } 357 | } 358 | 359 | let db = ValueDB::from_iter([ 360 | Value { 361 | foo: MyString("foo".to_owned()), 362 | bar: 8, 363 | }, 364 | Value { 365 | foo: MyString("*".to_owned()), 366 | bar: 16, 367 | } 368 | ].into_Iter()).unwrap(); 369 | 370 | let mut query = db.query(); 371 | query.foo(MyString("*".to_owned())); 372 | let value = query.execute().unwrap().any(); 373 | // this will never match the row with bar == 8, 374 | // tiven foo != an any value 375 | assert_eq!(value.bar, 16); 376 | ``` 377 | 378 | Why is this true? Because it is correct. 379 | 380 | Allowing it also to match the value `foo` would unfairly 381 | give more chances for `foo` to be selected over the _any_ value. 382 | This might not seem like a big difference, but it is. Because what if 383 | we generate a random string for `Value`s with an _any value? If we 384 | would allow all rows to be matched then that logic is now rigged, 385 | with a value of `foo` being more likely then other strings. 386 | 387 | As such the only correct answer when filtering for _any_ value, 388 | is to return rows that have _any_ value. 389 | 390 | > ❓ How can I query on multiple variants of a "filter map" property? 391 | 392 | Just call the _query_ method multiple times. It will allow you to match 393 | rows that have either of these values. 394 | 395 | Example 396 | 397 | ```rust,ignore 398 | use venndb::{Any, VennDB}; 399 | 400 | #[derive(Debug, VennDB)] 401 | pub struct Value { 402 | #[venndb(filter)] 403 | pub foo: String, 404 | pub bar: u32, 405 | } 406 | 407 | let db = ValueDB::from_iter([ 408 | Value { 409 | foo: "a".to_owned(), 410 | bar: 8, 411 | }, 412 | Value { 413 | foo: "b".to_owned(), 414 | bar: 12, 415 | }, 416 | Value { 417 | foo: "c".to_owned(), 418 | bar: 16, 419 | }, 420 | ].into_Iter()).unwrap(); 421 | 422 | let mut query = db.query(); 423 | query.foo(MyString("a".to_owned())); 424 | query.foo(MyString("c".to_owned())); 425 | let values: Vec<_> = query.execute().unwrap().iter().collect(); 426 | assert_eq!(values.len(), 2); 427 | assert_eq!(values[0].bar, 8); 428 | assert_eq!(values[0].bar, 16); 429 | ``` 430 | 431 | ## Example 432 | 433 | Here follows an example demonstrating all the features of `VennDB`. 434 | 435 | If you prefer a summary of what is generated, or do not understand something from the example below, 436 | you can also read [the "Generated Code Summary" chapter](#generated-code-summary) below. 437 | 438 | ```rust 439 | use itertools::Itertools; 440 | use venndb::VennDB; 441 | 442 | #[derive(Debug, VennDB)] 443 | // These attributes are optional, 444 | // e.g. by default the database would be called `EmployeeDB` (name + 'DB'). 445 | #[venndb(name = "EmployeeInMemDB", validator = employee_validator)] 446 | pub struct Employee { 447 | // you can use the `key` arg to be able to get an `Employee` instance 448 | // directly by this key. It will effectively establishing a mapping from key to a reference 449 | // of that Employee in the database. As such keys have to have unique values, 450 | // or else you get an error while appending / creating the DB. 451 | // 452 | // NOTE: keys do not only have to be unique, they also have to implement `Clone`!! 453 | // 454 | // A property cannot be a filter and a key at the same time, 455 | // trying to do so will result in a compile-team failure. 456 | #[venndb(key)] 457 | id: u32, 458 | name: String, 459 | is_manager: bool, 460 | is_admin: bool, 461 | // filter (booleans) can be made optional, 462 | // meaning that the row will not be able to be filtered (found) 463 | // on this column when the row has a `None` value for it 464 | is_active: Option, 465 | // booleans are automatically turned into (query) filters, 466 | // use the `skip` arg to stop this. As such it is only really needed for 467 | // bool properties :) 468 | #[venndb(skip)] 469 | foo: bool, 470 | // non-bool values can also be turned into filters, turning them into 2D filters. 471 | // For each uniquely inserted Department variant that is inserted, 472 | // a new filter is kept track of. This allows you to apply a (query) filter 473 | // based on department, a pretty useful thing to be able to do. 474 | // 475 | // NOTE: this does mean that such filter-map types have to also be: 476 | // `PartialEq + Eq + Hash + Clone`!! 477 | // 478 | // A property cannot be a filter and a key at the same time, 479 | // trying to do so will result in a compile-team failure. 480 | #[venndb(filter)] 481 | department: Department, 482 | // similar to regular bool filters, 483 | // filter maps can also be optional. 484 | // When a filter map is optional and the row's property for that filter is None, 485 | // it will not be registered and thus not be able to filtered (found) on that property 486 | #[venndb(filter)] 487 | country: Option, 488 | } 489 | 490 | fn employee_validator(employee: &Employee) -> bool { 491 | employee.id > 0 492 | } 493 | 494 | fn main() { 495 | let db = EmployeeInMemDB::from_iter([ 496 | RawCsvRow("1,John Doe,true,false,true,false,Engineering,USA"), 497 | RawCsvRow("2,Jane Doe,false,true,true,true,Sales,"), 498 | RawCsvRow("3,John Smith,false,false,,false,Marketing,"), 499 | RawCsvRow("4,Jane Smith,true,true,false,true,HR,"), 500 | RawCsvRow("5,John Johnson,true,true,true,true,Engineering,"), 501 | RawCsvRow("6,Jane Johnson,false,false,,false,Sales,BE"), 502 | RawCsvRow("7,John Brown,true,false,true,false,Marketing,BE"), 503 | RawCsvRow("8,Jane Brown,false,true,true,true,HR,BR"), 504 | ]) 505 | .expect("MemDB created without errors (e.g. no duplicate keys)"); 506 | 507 | println!(">>> Printing all employees..."); 508 | let all_employees: Vec<_> = db.iter().collect(); 509 | assert_eq!(all_employees.len(), 8); 510 | println!("All employees: {:#?}", all_employees); 511 | 512 | println!(">>> You can lookup an employee by any registered key..."); 513 | let employee = db 514 | .get_by_id(&2) 515 | .expect("to have found an employee with ID 2"); 516 | assert_eq!(employee.name, "Jane Doe"); 517 | 518 | println!(">>> Querying for all managers..."); 519 | let mut query = db.query(); 520 | query.is_manager(true); 521 | let managers: Vec<_> = query 522 | .execute() 523 | .expect("to have found at least one") 524 | .iter() 525 | .collect(); 526 | assert_eq!(managers.len(), 4); 527 | assert_eq!( 528 | managers.iter().map(|e| e.id).sorted().collect::>(), 529 | [1, 4, 5, 7] 530 | ); 531 | 532 | println!(">>> Querying for all managers with a last name of 'Johnson'..."); 533 | let managers_result = query 534 | .execute() 535 | .expect("to have found at least one") 536 | .filter(|e| e.name.ends_with("Johnson")) 537 | .expect("to have found a manager with a last name of Johnson"); 538 | let managers = managers_result.iter().collect::>(); 539 | assert_eq!(managers.len(), 1); 540 | assert_eq!(managers.iter().map(|e| e.id).collect::>(), [5]); 541 | 542 | println!(">>> You can also just get the first result if that is all you care about..."); 543 | let manager = managers_result.first(); 544 | assert_eq!(manager.id, 5); 545 | 546 | println!(">>> Querying for a random active manager in the Engineering department..."); 547 | let manager = query 548 | .reset() 549 | .is_active(true) 550 | .is_manager(true) 551 | .department(Department::Engineering) 552 | .execute() 553 | .expect("to have found at least one") 554 | .any(); 555 | assert!(manager.id == 1 || manager.id == 5); 556 | 557 | println!(">>> Optional bool filters have three possible values, where None != false. An important distinction to make..."); 558 | let mut query = db.query(); 559 | query.is_active(false); 560 | let inactive_employees: Vec<_> = query 561 | .execute() 562 | .expect("to have found at least one") 563 | .iter() 564 | .collect(); 565 | assert_eq!(inactive_employees.len(), 1); 566 | assert_eq!(inactive_employees[0].id, 4); 567 | 568 | println!(">>> If you want you can also get the Employees back as a Vec, dropping the DB data all together..."); 569 | let employees = db.into_rows(); 570 | assert_eq!(employees.len(), 8); 571 | assert!(employees[1].foo); 572 | println!("All employees: {:?}", employees); 573 | 574 | println!(">>> You can also get the DB back from the Vec, if you want start to query again..."); 575 | // of course better to just keep it as a DB to begin with, but let's pretend this is ok in this example 576 | let mut db = EmployeeInMemDB::from_rows(employees).expect("DB created without errors"); 577 | assert_eq!(db.iter().count(), 8); 578 | 579 | println!(">>> Querying for all active employees in the Sales department..."); 580 | let mut query = db.query(); 581 | query.is_active(true); 582 | query.department(Department::Sales); 583 | let sales_employees: Vec<_> = query 584 | .execute() 585 | .expect("to have found at least one") 586 | .iter() 587 | .collect(); 588 | assert_eq!(sales_employees.len(), 1); 589 | assert_eq!(sales_employees[0].name, "Jane Doe"); 590 | 591 | println!(">>> Filter maps that are optional work as well, e.g. you can query for all employees from USA..."); 592 | query.reset().country("USA".to_owned()); 593 | let usa_employees: Vec<_> = query 594 | .execute() 595 | .expect("to have found at least one") 596 | .iter() 597 | .collect(); 598 | assert_eq!(usa_employees.len(), 1); 599 | assert_eq!(usa_employees[0].id, 1); 600 | 601 | println!(">>> At any time you can also append new employees to the DB..."); 602 | assert_eq!(EmployeeInMemDBErrorKind::DuplicateKey, db 603 | .append(RawCsvRow("8,John Doe,true,false,true,false,Engineering,")) 604 | .unwrap_err().kind()); 605 | println!(">>> This will fail however if a property is not correct (e.g. ID (key) is not unique in this case), let's try this again..."); 606 | assert!(db 607 | .append(RawCsvRow("9,John Doe,false,true,true,false,Engineering,")) 608 | .is_ok()); 609 | assert_eq!(db.len(), 9); 610 | 611 | println!(">>> Rows are also validated prior to appending in case a validator is defined..."); 612 | println!(" The next insertion will fail due to the id being zero, a condition defined in the custom validator..."); 613 | assert_eq!(EmployeeInMemDBErrorKind::InvalidRow, db 614 | .append(RawCsvRow("0,John Doe,true,false,true,false,Engineering,")) 615 | .unwrap_err().kind()); 616 | 617 | println!(">>> This new employee can now also be queried for..."); 618 | let mut query = db.query(); 619 | query.department(Department::Engineering).is_manager(false); 620 | let new_employee: Vec<_> = query 621 | .execute() 622 | .expect("to have found at least one") 623 | .iter() 624 | .collect(); 625 | assert_eq!(new_employee.len(), 1); 626 | assert_eq!(new_employee[0].id, 9); 627 | 628 | println!(">>> You can also extend it using an IntoIterator..."); 629 | db.extend([ 630 | RawCsvRow("10,Glenn Doe,false,true,true,true,Engineering,"), 631 | RawCsvRow("11,Peter Miss,true,true,true,true,HR,USA"), 632 | ]) 633 | .unwrap(); 634 | let mut query = db.query(); 635 | query 636 | .department(Department::HR) 637 | .is_manager(true) 638 | .is_active(true) 639 | .is_admin(true); 640 | let employees: Vec<_> = query 641 | .execute() 642 | .expect("to have found at least one") 643 | .iter() 644 | .collect(); 645 | assert_eq!(employees.len(), 1); 646 | assert_eq!(employees[0].id, 11); 647 | 648 | println!(">>> There are now 2 employees from USA..."); 649 | query.reset().country("USA".to_owned()); 650 | let employees: Vec<_> = query 651 | .execute() 652 | .expect("to have found at least one") 653 | .iter() 654 | .collect(); 655 | assert_eq!(employees.len(), 2); 656 | assert_eq!( 657 | employees.iter().map(|e| e.id).sorted().collect::>(), 658 | [1, 11] 659 | ); 660 | 661 | println!(">>> All previously data is still there as well of course..."); 662 | query 663 | .reset() 664 | .is_active(true) 665 | .is_manager(true) 666 | .department(Department::Engineering); 667 | let managers: Vec<_> = query 668 | .execute() 669 | .expect("to have found at least one") 670 | .iter() 671 | .collect(); 672 | assert_eq!(managers.len(), 2); 673 | assert_eq!( 674 | managers.iter().map(|e| e.id).sorted().collect::>(), 675 | [1, 5] 676 | ); 677 | } 678 | 679 | #[derive(Debug)] 680 | struct RawCsvRow(S); 681 | 682 | impl From> for Employee 683 | where 684 | S: AsRef, 685 | { 686 | fn from(RawCsvRow(s): RawCsvRow) -> Employee { 687 | let mut parts = s.as_ref().split(','); 688 | let id = parts.next().unwrap().parse().unwrap(); 689 | let name = parts.next().unwrap().to_string(); 690 | let is_manager = parts.next().unwrap().parse().unwrap(); 691 | let is_admin = parts.next().unwrap().parse().unwrap(); 692 | let is_active = match parts.next().unwrap() { 693 | "" => None, 694 | s => Some(s.parse().unwrap()), 695 | }; 696 | let foo = parts.next().unwrap().parse().unwrap(); 697 | let department = parts.next().unwrap().parse().unwrap(); 698 | let country = match parts.next().unwrap() { 699 | "" => None, 700 | s => Some(s.to_string()), 701 | }; 702 | Employee { 703 | id, 704 | name, 705 | is_manager, 706 | is_admin, 707 | is_active, 708 | foo, 709 | department, 710 | country, 711 | } 712 | } 713 | } 714 | 715 | #[derive(Debug, PartialEq, Eq, Hash, Clone)] 716 | pub enum Department { 717 | Engineering, 718 | Sales, 719 | Marketing, 720 | HR, 721 | } 722 | 723 | impl std::str::FromStr for Department { 724 | type Err = (); 725 | 726 | fn from_str(s: &str) -> Result { 727 | match s { 728 | "Engineering" => Ok(Department::Engineering), 729 | "Sales" => Ok(Department::Sales), 730 | "Marketing" => Ok(Department::Marketing), 731 | "HR" => Ok(Department::HR), 732 | _ => Err(()), 733 | } 734 | } 735 | } 736 | ``` 737 | 738 | ### Generated Code Summary 739 | 740 | In this chapter we'll list the API as generated by `VennDB` for the following example code from above: 741 | 742 | ```rust,ignore 743 | #[derive(Debug, VennDB)] 744 | #[venndb(name = "EmployeeInMemDB", validator = employee_validator)] 745 | pub struct Employee { 746 | #[venndb(key)] 747 | id: u32, 748 | name: String, 749 | is_manager: bool, 750 | is_admin: bool, 751 | is_active: Option, 752 | #[venndb(skip)] 753 | foo: bool, 754 | #[venndb(filter)] 755 | department: Department, 756 | country: Option, 757 | } 758 | ``` 759 | 760 | The following public-API datastructures will be generated: 761 | 762 | - `struct EmployeeInMemDB`: the database, that can be used to query (by filters) or look up data (by keys); 763 | - `enum EmployeeInMemDBError`: the error type that is returned when mutating the DB and a property of the to be inserted row; 764 | - `enum EmployeeInMemDBErrorKind`: the kind of error that can happen as described for `EmployeeInMemDBError`; 765 | - `struct EmployeeInMemDBQuery`: the query builder that is used to build a query that can be `execute`d to query data from the db using filters; 766 | - `struct EmployeeInMemDBQueryResult`: the result when querying using `EmployeeInMemDBQuery` and at least one row was found that matched the defined filters; 767 | - `struct EmployeeInMemDBQueryResultIter`: the iterator type that is used when calling `EmployeeInMemDBQueryResult::iter`. It has no methods/api other then the fact that it is an `Iterator` and can be used as one; 768 | 769 | The visual specifiers of these datastructures will be the same as the `struct` that the `VennDB` macro is applied to. 770 | E.g. in this example `Employee` has a specifier of `pub` so the above datastructures and their public-apy methods will also be `pub`. 771 | 772 | There are also some other helper datastructures generated — all prefixed with the database name, e.g. `EmployeeInMemDB` in this example — 773 | but we do not mention here as they should not be relied upon and given the prefix it should cause no conflict. 774 | In case you do not want to expose these structures to the outside you can wrap your `struct` within its own `mod` (module). 775 | 776 | #### Generated Code Summary: Method API 777 | 778 | Database: (e.g. `EmployeeInMemDB`): 779 | 780 | | fn signature | description | 781 | | - | - | 782 | | `EmployeeInMemDB::new() -> EmployeeInMemDB` | create a new database with zero capacity | 783 | | `EmployeeInMemDB::default() -> EmployeeInMemDB` | same as `EmployeeInMemDB::new() -> EmployeeInMemDB` | 784 | | `EmployeeInMemDB::capacity(capacity: usize) -> EmployeeInMemDB` | create a new database with the given capacity, but no rows already inserted | 785 | | `EmployeeInMemDB::from_rows(rows: ::std::vec::Vec) -> EmployeeInMemDB` or `EmployeeInMemDB::from_rows(rows: ::std::vec::Vec) -> Result>>` | constructor to create the database directly from a heap-allocated list of data instances. The second version is the one used if at least one `#[venndb(key)]` property is defined, otherwise it is the first one (without the `Result`). | 786 | | `EmployeeInMemDB::from_iter(iter: impl ::std::iter::IntoIterator>) -> EmployeeInMemDB` or `EmployeeInMemDB::from_rows(iter: impl ::std::iter::IntoIterator>) -> Result>>` | Same as `from_rows` but using an iterator instead. The items do not have to be an `Employee` but can be anything that can be turned into one. E.g. in our example above we defined a struct `RawCsvRow` that was turned on the fly into an `Employee`. This happens all at once prior to inserting the database, which is why the version with a result does return a `Vec` and not an iterator. | 787 | | `EmployeeInMemDB::append(&mut self, data: impl ::std::convert::Into)` or `EmployeeInMemDB::append(&mut self, data: impl ::std::convert::Into) -> Result<(), EmployeeInMemDBError>` | append a single row to the database. Depending on whether or not a `#[venndb(key)]` property is defined it will generate the `Result` version or not. Same as `from_rows` and `from_iter` | 788 | | `EmployeeInMemDB::extend(&mut self, iter: I) where I: ::std::iter::IntoIterator, Item: ::std::convert::Into` or `EmployeeInMemDB::extend(&mut self, iter: I) -> Result<(), EmployeeInMemDBError<(Employee, I::IntoIter)>> where I: ::std::iter::IntoIterator, Item: ::std::convert::Into` | extend the database with the given iterator, once again returning a result in case such insertion can go wrong (e.g. because keys are used (duplication) or a row is invalid in case a validator is defined). Otherwise this function will return nothing. | 789 | | `EmployeeInMemDB::get_by_id(&self, data: impl ::std::convert::Into) -> Option<&Employee> where Employee ::std::borrow::Borrow, Q: ::std::hash::Hash + ::std::cmp::Eq + ?::std::marker::Sized` | look up a row by the `id` key property. This method will be generated for each property marked with `#[venndb(key)`. e.g. if you have key property named `foo: MyType` property there will be also a `get_by_foo(&self, ...)` method generated. | 790 | | `EmployeeInMemDB::query(&self) -> EmployeeInMemDBQuery` | create a `EmployeeInMemDBQuery` builder to compose a filter composition to query the database. The default builder will match all rows. See the method API for `EmployeeInMemDBQuery` for more information | 791 | 792 | Query (e.g. `EmployeeInMemDBQuery`) 793 | 794 | | fn signature | description | 795 | | - | - | 796 | | `EmployeeInMemDBQuery::reset(&mut self) -> &mut Self` | reset the query, bringing it back to the clean state it has on creation | 797 | | `EmployeeInMemDBQuery::execute(&self) -> Option>` | return the result of the query using the set filters. It will be `None` in case no rows matched the defined filters. Or put otherwise, the result will contain at least one row when `Some(_)` is returned. | 798 | | `EmployeeInMemDBQuery::is_manager(&mut self, value: bool) -> &mut Self` | a filter setter for a `bool` filter. One such method per `bool` filter (that isn't `skip`ped) will be available. E.g. if you have ` foo` filter then there will be a `EmployeeInMemDBQuery:foo` method. For _bool_ filters that are optional (`Option`) this method is also generated just the same. | 799 | | `EmployeeInMemDBQuery::department(&mut self, value: impl ::std::convert::Into) -> &mut Self` | a filter (map) setter for a non-`bool` filter. One such method per non-`bool` filter will be available. You can also `skip` these, but that's of course a bit pointless. The type will be equal to the actual field type. And the name will once again be equal to the original field name. Filter maps that have a `Option` type have exactly the same signature. Duering query you can call this method multiple times in case you wish to allow multiple variants. | 800 | 801 | Query Result (e.g. `EmployeeInMemDBQueryResult`) 802 | 803 | | fn signature | description | 804 | | - | - | 805 | | `EmployeeInMemDBQueryResult::first(&self) -> &Employee` | return a reference to the first matched employee found. An implementation detail is that this will be the matched row that was first inserted, but for compatibility reasons you best not rely on this if you do not have to. | 806 | | `EmployeeInMemDBQueryResult::any(&self) -> &Employee` | return a reference to a randomly selected matched employee. The randomness can be relied upon to be fair. | 807 | | `EmployeeInMemDBQueryResult::iter(&self) -> `EmployeeInMemDBQueryResultIter` | return an iterator for the query result, which will allow you to iterate over all found results, and as such also collect them into an owned data structure should you wish. | 808 | | `EmployeeInMemDBQueryResult::filter(&self, predicate: F) -> Option<#EmployeeInMemDBQueryResult> where F: Fn(&#name) -> bool` | return `Some(_)` `EmployeeInMemDBQueryResult` with the same reference data, but containing (and owning) only the indexes for which the linked row matches arcoding to the given `Fn` predicate | 809 | 810 | ## ⛨ | Safety 811 | 812 | This crate uses `#![forbid(unsafe_code)]` to ensure everything is implemented in 100% safe Rust. 813 | 814 | ## 🦀 | Compatibility 815 | 816 | venndb is developed mostly on MacOS M-Series machines and run in production 817 | on a variety of Linux systems. Windows support is not officially guaranteed, 818 | but is [tested using Github Actions](https://github.com/plabayo/venndb/blob/main/.github/workflows/CI.yml) with success. 819 | 820 | | platform | tested | test platform | 821 | |----------|--------|---------------| 822 | | MacOS | ✅ | M2 (developer laptop) and macos-12 Intel ([GitHub Action](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners)) | 823 | | Windows | ✅ | Windows 2022 ([GitHub Action](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners)) | 824 | | Linux | ✅ | Ubuntu 22.04 ([GitHub Action](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners)) | 825 | 826 | Please [open a ticket](https://github.com/plabayo/venndb/issues) in case you have compatibility issues for your setup/platform. 827 | Our goal is not to support all possible platformns in the world, but we do want to 828 | support as many as we reasonably can. 829 | 830 | ### Minimum supported Rust version 831 | 832 | venndb's MSRV is `1.88`. 833 | 834 | [Using GitHub Actions we also test](https://github.com/plabayo/venndb/blob/main/.github/workflows/CI.yml) if `venndb` on that version still works on 835 | the stable and beta versions of _rust_ as well. 836 | 837 | ## 🧭 | Roadmap 838 | 839 | Please refer to to know what's on the roadmap. Is there something not on the roadmap for the next version that you would really like? Please [create a feature request](https://github.com/plabayo/venndb/issues) to request it and [become a sponsor](#--sponsors) if you can. 840 | 841 | ## 💼 | License 842 | 843 | This project is dual-licensed under both the [MIT license][mit-license] and [Apache 2.0 License][apache-license]. 844 | 845 | ## 👋 | Contributing 846 | 847 | 🎈 Thanks for your help improving the project! We are so happy to have 848 | you! We have a [contributing guide][contributing] to help you get involved in the 849 | `venndb` project. 850 | 851 | Contributions often come from people who already know what they want, be it a fix for a bug they encountered, 852 | or a feature that they are missing. Please do always make a ticket if one doesn't exist already. 853 | 854 | It's possible however that you do not yet know what specifically to contribute, and yet want to help out. 855 | For that we thank you. You can take a look at the open issues, and in particular: 856 | 857 | - [`good first issue`](https://github.com/plabayo/venndb/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22): issues that are good for those new to the `venndb` codebase; 858 | - [`easy`](https://github.com/plabayo/venndb/issues?q=is%3Aissue+is%3Aopen+label%3Aeasy): issues that are seen as easy; 859 | - [`mentor available`](https://github.com/plabayo/venndb/issues?q=is%3Aissue+is%3Aopen+label%3A%22mentor+available%22): issues for which we offer mentorship; 860 | - [`low prio`](https://github.com/plabayo/venndb/issues?q=is%3Aissue+is%3Aopen+label%3A%22low+prio%22): low prio issues that have no immediate pressure to be finished quick, great in case you want to help out but can only do with limited time to spare; 861 | 862 | In general, any issue not assigned already is free to be picked up by anyone else. Please do communicate in the ticket 863 | if you are planning to pick it up, as to avoid multiple people trying to solve the same one. 864 | 865 | Should you want to contribure this project but you do not yet know how to program in Rust, you could start learning Rust with as goal to contribute as soon as possible to `venndb` by using "[the Rust 101 Learning Guide](https://rust-lang.guide/)" as your study companion. Glen can also be hired as a mentor or teacher to give you paid 1-on-1 lessons and other similar consultancy services. You can find his contact details at . 866 | 867 | ### Contribution 868 | 869 | Unless you explicitly state otherwise, any contribution intentionally submitted 870 | for inclusion in `venndb` by you, shall be licensed as both [MIT][mit-license] and [Apache 2.0][apache-license], 871 | without any additional terms or conditions. 872 | 873 | [contributing]: https://github.com/plabayo/venndb/blob/main/CONTRIBUTING.md 874 | [mit-license]: https://github.com/plabayo/venndb/blob/main/LICENSE-MIT 875 | [apache-license]: https://github.com/plabayo/venndb/blob/main/LICENSE-APACHE 876 | 877 | ### Acknowledgements 878 | 879 | Special thanks goes to all involved in developing, maintaining and supporting [the Rust programming language](https://www.rust-lang.org/). Also a big shoutout to the ["Write Powerful Rust Macros" book by _Sam Van Overmeire_](https://www.manning.com/books/write-powerful-rust-macros), which gave the courage to develop this crate. 880 | 881 | Some code was also copied/forked from [google/argh](https://github.com/google/argh), for which thank you, 882 | we are big fans of that crate. Go use it if you want to create a CLI App. 883 | 884 | ## 💖 | Sponsors 885 | 886 | venndb is **completely free, open-source software** which needs lots of effort and time to develop and maintain. 887 | 888 | Support this project by becoming a [sponsor][ghs-url]. One time payments are accepted [at GitHub][ghs-url] as well as at ["Buy me a Coffee"][bmac-url]. 889 | 890 | Sponsors help us continue to maintain and improve `venndb`, as well as other 891 | Free and Open Source (FOSS) technology. It also helps us to create 892 | educational content such as , 893 | and other open source frameworks such as . 894 | 895 | Sponsors receive perks and depending on your regular contribution it also 896 | allows you to rely on us for support and consulting. 897 | 898 | Finally, you can also support us by shopping Plabayo <3 `VennDB` merchandise 🛍️ at . 899 | 900 | [![Plabayo's Store With VennDB Merchandise](https://raw.githubusercontent.com/plabayo/venndb/main/docs/img/plabayo_mech_store_venndb.png)](https://plabayo.threadless.com/) 901 | --------------------------------------------------------------------------------