├── .gitignore ├── Cargo.toml ├── src ├── value_to_msgpack_transcoder.rs └── main.rs └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "norgberg" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | anyhow = "1.0.71" 10 | directories = "5.0.1" 11 | norgopolis-module = "2.0.1" 12 | rmpv = "1.0.1" 13 | surrealdb = { version = "1.0.0", features = ["kv-mem", "kv-rocksdb"] } 14 | tokio = { version = "1.29.1", features = ["rt-multi-thread", "macros", "process", "io-std"] } 15 | tokio-stream = "0.1.14" 16 | -------------------------------------------------------------------------------- /src/value_to_msgpack_transcoder.rs: -------------------------------------------------------------------------------- 1 | use norgopolis_module::module_communication::MessagePack; 2 | use surrealdb::sql::Value; 3 | 4 | pub fn value_to_msgpack(value: &Value) -> MessagePack { 5 | let mut bytes = Vec::new(); 6 | 7 | let transcoded = transcode_value_msgpack(value); 8 | rmpv::encode::write_value(&mut bytes, &transcoded).unwrap(); 9 | 10 | MessagePack { data: bytes } 11 | } 12 | 13 | fn transcode_value_msgpack(value: &Value) -> rmpv::Value { 14 | match value { 15 | Value::None | Value::Null => rmpv::Value::Nil, 16 | Value::Bool(bool) => rmpv::Value::Boolean(*bool), 17 | Value::Number(number) => rmpv::Value::Integer(number.to_int().into()), 18 | Value::Strand(strand) => rmpv::Value::String(strand.clone().to_raw().into()), 19 | Value::Duration(duration) => rmpv::Value::Map(Vec::from([( 20 | "duration".into(), 21 | duration.as_secs_f64().into(), 22 | )])), 23 | Value::Datetime(datetime) => rmpv::Value::Map(Vec::from([( 24 | "datetime".into(), 25 | datetime.timestamp_micros().into(), 26 | )])), 27 | Value::Uuid(uuid) => uuid.as_bytes().as_slice().into(), 28 | Value::Array(array) => array.iter().map(transcode_value_msgpack).collect(), 29 | Value::Object(object) => rmpv::Value::Map( 30 | object 31 | .iter() 32 | .map(|(k, v)| (k.clone().into(), transcode_value_msgpack(v))) 33 | .collect(), 34 | ), 35 | Value::Geometry(geometry) => transcode_value_msgpack(&geometry.as_coordinates()), 36 | Value::Bytes(bytes) => bytes.to_string().into(), 37 | Value::Thing(thing) => rmpv::Value::Map(Vec::from([ 38 | ("id".into(), thing.id.to_string().into()), 39 | ("tb".into(), thing.tb.clone().into()), 40 | ])), 41 | // As per the documentation other types will be coerced back into simpler types 42 | // before being sent back to the client and so we shouldn't worry about them here. 43 | _ => unimplemented!(), 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Norgberg 2 | 3 | Norgberg is the database services module for [Neorg](https://github.com/nvim-neorg/neorg), providing fast document data caching and module state storage. It is designed to follow the vision of the [Neorg Roadmap](https://github.com/nvim-neorg/neorg/blob/main/ROADMAP.md#external-tooling) to be a modular service that can be embedded in various different applications. 4 | 5 | Norgberg serves two purposes. 6 | 7 | 1. It is a fast structured data cache for the information in and between notes of a neorg workspace, storing properties like metadata headers, link resolutions, tasks and other structured data in the neorg standard for fast querying by other modules such as Zettelkasten and Getting Things Done (GTD). 8 | 2. It is a state storage service that can be used by Neorg modules to persist information, for purposes such as synchronisation between devices, or holding module information. 9 | 10 | Norgberg thus provides services found in other linked notes applications, as well as some utility beyond. 11 | 12 | Norgberg is ment to be a modular services for modules plugging into the Neorg ecosystem - we explicitly want other modules to store their information in one common database service that is provided for all. 13 | 14 | ## Core design 15 | 16 | The basis of Norgberg is an encapsulation of an SQLite database with custom schemas, exposed through an API. It stores primary key references for all unique files in a Neorg workspace. This allows the SQLIte database to store link resolutions between files and other such information. The database is populated by consuming the Tree-sitter Concrete Syntax Trees provided by Neorg's multi-threaded parses, and held up to date by updating database state when Neorg buffers are written to file. The API allows users to run fast queries on the DB to resolve links for jumps, aggregate and filter tasks across a workspace, and other queries of interest. 17 | 18 | In the beginning, Norgberg will only store and maintain select information caching. In the long-term, we hope to automatically cache most to all of the structured metadata in the Norg specifications by default, allowing users easy and fast access to the structured information in their workspaces with minimal custom effort. 19 | 20 | In addition, other modules may add new tables to the SQLite database to store their own information. This capability should be used with the understanding that any module state stored in the database this way will not be stored by Norgberg in any other files, and may be lost in case of database corruption or deletion. Norgberg itself will only reconstruct state that exists in the Neorg files. 21 | 22 | Long-term, we aim to introduce the caching of neorg files data in a graph database, allowing for graph queries and analysis with high performance and purposeful query languges. This should empower advanced applications using attached modifiers, tag extensions, and looking at many-notes data structures. Initial graph services will be provided through SQLite schemas for representing edge connections between nodes. 23 | 24 | ### Interface 25 | 26 | Norgberg runs as a module of the [Norgopolis common backend server](https://github.com/SevorisDoe/Norgberg). It conforms to the Norgopolis RPC standard. Methods exposed by Norgberg can be called via the common RPC router from other modules or via gRPC frontends in your application or script, as long as the method name is available and the mpack deserializes to matche the method arguments. 27 | 28 | ### Cookie system 29 | 30 | The cookie system enforces database tidieness and allows modules to automatically migrate database schemas when upgrades occur. The system stores information about what other modules have registered themselves with the database using the modules name and semantic version. On first communication after startup, modules communicate their current semantic version. Norgberg can inform the modules if version changes occured, which can be used to trigger automatic migration logic. 31 | 32 | The cookie system also tracks which tables are claimed by what module. This information can be used by the user to clean up unwanted tables cluttering up the database. 33 | 34 | ## Roadmap 35 | 36 | Currently, we are aiming to provide the core first-generation SQLite database to support the GTD and Zettelkasten modules with key cache and storage services related to inter-file links and tasks. 37 | 38 | - [ ] Design the interface API 39 | - [ ] Design the database populating code 40 | - [ ] Design the state-updating code 41 | - [ ] Design startup, shutdown and many-modules connectivity 42 | 43 | We are also aiming to develop a benchmark for graph-like information manipulation on Norgberg, helping us quantify the performance of the SQLIte graph-modelling schemas and how they compare to purposeful graph-paradigm databases we are planning to introduce long-term. For this benchmark we are looking for generalized descriptions of note-worthy link structures people have in their vaults. The more examples we have, the better! 44 | 45 | For graph databases, we are currently evaluating multiple candidates 46 | - SurrealDB 47 | - Oxigraph 48 | - CozoDB 49 | 50 | With special interest in CozoDB for its support of SQLite as a storage engine, use of the Datalog query language, and support of relational, graph, and vector paradigma. 51 | 52 | ## Contributing 53 | Especially important in this early phase of Norgberg's existence is clarity about what information needs storing and retrieving and along what schema! This is a call to everyone else looking to roll out modules for Neorg. 54 | 55 | We also need to decide on a first way in which multiple different Rust modules are linked together into one system on the user's system, in a way that is both performant at runtime and not too much effort on the user's part. 56 | 57 | Beyond that, all thoughts and experience is welcome! In the coming days, this repo will be updated with a growing set of design deliberations and decisions. 58 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use norgopolis_module::{ 3 | invoker_service::Service, module_communication::MessagePack, Code, Module, Status, 4 | }; 5 | use std::path::Path; 6 | use surrealdb::engine::local::{Db, File}; 7 | use tokio_stream::wrappers::ReceiverStream; 8 | use value_to_msgpack_transcoder::value_to_msgpack; 9 | 10 | use surrealdb::Surreal; 11 | 12 | mod value_to_msgpack_transcoder; 13 | 14 | struct Norgberg { 15 | connection: Surreal, 16 | } 17 | 18 | impl Norgberg { 19 | async fn new(file: &Path) -> Result { 20 | let connection = Surreal::new::(file.to_str().unwrap_or("memory")).await?; 21 | 22 | connection 23 | .use_ns("neorg") 24 | .use_db("neorg") 25 | .await 26 | .expect("Failed to connect to db"); 27 | 28 | connection 29 | .query("DEFINE NAMESPACE neorg; DEFINE DATABASE NEORG;") 30 | .await 31 | .expect("Unable to create neorg namespace nor db!"); 32 | 33 | connection 34 | .query( 35 | " 36 | BEGIN TRANSACTION; 37 | IF schema:current != NULL { 38 | RETURN; 39 | }; 40 | 41 | CREATE schema:current SET version = '0.0.1'; 42 | 43 | IF !string::is::semver(schema:current.version) { 44 | THROW 'Attempted to create schema with invalid semver version!'; 45 | }; 46 | 47 | DEFINE TABLE files SCHEMAFULL; 48 | DEFINE INDEX path ON TABLE files COLUMNS path UNIQUE; 49 | DEFINE FIELD path ON TABLE files TYPE string; 50 | // TODO: DEFINE FIELD metadata 51 | 52 | DEFINE TABLE vFiles SCHEMAFULL; 53 | DEFINE INDEX path ON TABLE vFiles COLUMNS path UNIQUE; 54 | DEFINE FIELD path ON TABLE vFiles TYPE string; 55 | 56 | DEFINE TABLE externalResources SCHEMAFULL; 57 | DEFINE INDEX id ON TABLE externalResources COLUMNS id UNIQUE; 58 | DEFINE FIELD id ON TABLE externalResources TYPE string; 59 | DEFINE FIELD type ON TABLE externalResources TYPE string 60 | ASSERT $value INSIDE ['uri', 'file']; 61 | COMMIT TRANSACTION; 62 | ", 63 | ) 64 | .await 65 | .expect("Unable to configure database!"); 66 | 67 | Ok(Norgberg { connection }) 68 | } 69 | } 70 | 71 | #[norgopolis_module::async_trait] 72 | impl Service for Norgberg { 73 | type Stream = ReceiverStream>; 74 | 75 | async fn call( 76 | &self, 77 | fn_name: String, 78 | args: Option, 79 | ) -> Result { 80 | let (tx, rx) = tokio::sync::mpsc::channel(8); 81 | 82 | match fn_name.as_str() { 83 | "execute-query" => match args { 84 | Some(arg) => { 85 | let query = match arg.decode::().map_err(|err| { 86 | Status::new( 87 | Code::InvalidArgument, 88 | "Invalid argument provided! Expected type `string`: ".to_string() 89 | + &err.to_string(), 90 | ) 91 | }) { 92 | Ok(val) => val, 93 | Err(err) => { 94 | tx.send(Err(err.clone())).await.unwrap(); 95 | return Err(err); 96 | } 97 | }; 98 | 99 | match self 100 | .connection 101 | .query(query) 102 | .await 103 | .map_err(|err| Status::new(Code::Cancelled, err.to_string())) 104 | { 105 | Ok(mut val) => { 106 | for i in 0..val.num_statements() { 107 | match val.take(i) { 108 | Ok(val) => { 109 | tx.send(Ok(value_to_msgpack(&val))).await.unwrap(); 110 | } 111 | Err(err) => { 112 | tx.send(Err(Status::new(Code::Cancelled, err.to_string()))) 113 | .await 114 | .unwrap(); 115 | return Ok(ReceiverStream::new(rx)); 116 | } 117 | } 118 | } 119 | } 120 | Err(err) => { 121 | tx.send(Err(err.clone())).await.unwrap(); 122 | return Err(err); 123 | } 124 | }; 125 | } 126 | None => { 127 | tx.send(Err(Status::new( 128 | Code::InvalidArgument, 129 | "Expected a string as argument, whereas nothing was provided instead.", 130 | ))) 131 | .await 132 | .unwrap(); 133 | } 134 | }, 135 | _ => todo!(), 136 | // "execute-live-query" => {}, 137 | }; 138 | 139 | Ok(ReceiverStream::new(rx)) 140 | } 141 | } 142 | 143 | #[tokio::main] 144 | async fn main() { 145 | let database_location = if cfg!(debug_assertions) { 146 | "memory".to_owned() 147 | } else { 148 | let data_dir = directories::ProjectDirs::from("org", "neorg", "norgberg").expect("Could not grab known data directories, are you running on a non-unix and non-windows system?").data_dir().to_path_buf(); 149 | 150 | let _ = std::fs::create_dir_all(&data_dir); 151 | 152 | data_dir.join("norgberg.db").to_str().unwrap().to_string() 153 | }; 154 | 155 | Module::new() 156 | .start( 157 | Norgberg::new(Path::new(&database_location)) 158 | .await 159 | .expect("Unable to connect to database!"), 160 | ) 161 | .await 162 | .unwrap() 163 | } 164 | --------------------------------------------------------------------------------