├── .env ├── .gitignore ├── Cargo.toml ├── README.md ├── build.rs ├── diesel.toml ├── input.txt ├── migrations ├── .gitkeep └── 2022-06-29-092034_grsql │ ├── down.sql │ └── up.sql ├── proto └── data.proto └── src ├── client.rs ├── entity.rs ├── schema.rs └── server.rs /.env: -------------------------------------------------------------------------------- 1 | DATABASE_URL=grsql.db 2 | BUF_LEN=1024 3 | #TLS_PEM=path/to/pem 4 | #TLS_KEY=path/to/key 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | *.swp 3 | **/*.swp 4 | *.lock 5 | *.db 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "server" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | diesel = { version = "1.4.8", features = ["sqlite"] } 10 | dotenv = "0.15.0" 11 | futures-util = "0.3.21" 12 | prost = "0.10.4" 13 | serde = { version = "1.0.137", features = ["derive"] } 14 | streamer = { version = "0.1.1", features = ["hyper"] } 15 | tokio = { version = "1.19.2", features = ["macros", "rt-multi-thread"] } 16 | tonic = "0.7.2" 17 | 18 | [[bin]] 19 | name = "server" 20 | path = "src/server.rs" 21 | 22 | [[bin]] 23 | name = "client" 24 | path = "src/client.rs" 25 | 26 | [build-dependencies] 27 | tonic-build = "0.7" 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **Grsql** is a great tool to allow you set up your remote sqlite database as service and **CRUD** (create/ read/ update/ delete) it using gRPC. 2 | 3 | ## Why Create This 4 | 5 | Since I often use micro-service and database intergration is often required. 6 | for some "heavy" database like `MySQL`, `MongoDB` or something similar, it is a luxury to use them. 7 | A sqlite database with fast server backend would be a great solution for this. 8 | 9 | This lead to the birth of `Grsql`. 10 | 11 | ## How to Use 12 | 13 | this project consist of two parts: 14 | - server 15 | - client 16 | 17 | each part can run **independently**. the `server` normally run on remote/cloud side, the `client` make request to it. 18 | 19 | use the following code run server, 20 | ```sh 21 | cargo build --bin server 22 | ``` 23 | then the server starts listening on localhost port `7749`. 24 | 25 | on another side, to run client 26 | ```sh 27 | cargo build --bin client 28 | ``` 29 | It will insert the file's content `input.txt` into remote server, then read it, update its id and finally delete it. 30 | 31 | ## Development 32 | 33 | First of all, you need to be familiar with [proto-buffer](https://developers.google.com/protocol-buffers/docs/overview), if not, go to the site and see around. 34 | 35 | since grsql is built on crates [tonic](https://github.com/hyperium/tonic), you need some pre-developing experience of [rust](https://rust-lang.org), as well as tonic. 36 | see these [examples](https://github.com/hyperium/tonic/tree/master/examples) to quick start. 37 | 38 | you can modify/add/delete the `message` and `service` in `proto/data.proto` according to your requests 39 | then implement them according to that in `src/server.rs` 40 | 41 | 42 | ## Notes 43 | 44 | - **No authentication implement so far** 45 | 46 | - gRPC is designed to deal with service that processes small and frequent communication, not for large file tranfer(file upload/download), if you do so, the performance is worse than `HTTP2`. 47 | It is better that the workload is less than 1M according to [this](https://ops.tips/blog/sending-files-via-grpc/), So use it with care! 48 | 49 | you can refer these articles for more: 50 | 51 | [Sending files via gRPC](https://ops.tips/blog/sending-files-via-grpc/) 52 | 53 | [Upload/Download performance with gRPC ](https://github.com/grpc/grpc-dotnet/issues/1186) 54 | 55 | [Use gRPC to share very large file](https://stackoverflow.com/questions/62470323/use-grpc-to-share-very-large-file) 56 | 57 | ## Feature to Add 58 | 59 | - [ ] Add Athentication 60 | 61 | Due to the issues mentioned above, it is desired to 62 | - [ ] add a http2 module to handle large file upload/download. 63 | 64 | To make transfer more efficient, it is resonable to 65 | - [ ] add a compression 66 | -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | fn main() -> Result<(), Box> { 2 | tonic_build::compile_protos("proto/data.proto")?; 3 | Ok(()) 4 | } 5 | -------------------------------------------------------------------------------- /diesel.toml: -------------------------------------------------------------------------------- 1 | # For documentation on how to configure this file, 2 | # see diesel.rs/guides/configuring-diesel-cli 3 | 4 | [print_schema] 5 | file = "src/schema.rs" 6 | -------------------------------------------------------------------------------- /input.txt: -------------------------------------------------------------------------------- 1 | 1234567890abcdefghijklmnopqrstuvw 2 | 1234567890abcdefghijklmnopqrstuvw 3 | 1234567890abcdefghijklmnopqrstuvw 4 | -------------------------------------------------------------------------------- /migrations/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hominee/grsql/e562dfcfec1a92695e75ea1bf742151bf2e5359f/migrations/.gitkeep -------------------------------------------------------------------------------- /migrations/2022-06-29-092034_grsql/down.sql: -------------------------------------------------------------------------------- 1 | -- This file should undo anything in `up.sql` -------------------------------------------------------------------------------- /migrations/2022-06-29-092034_grsql/up.sql: -------------------------------------------------------------------------------- 1 | -- Your SQL goes here 2 | CREATE TABLE IF NOT EXISTS grsql ( 3 | id Long PRIMARY KEY AUTOINCREMENT NOT NULL , 4 | name Text NOT NULL, 5 | mime Text NOT NULL, 6 | created INTEGER NOT NULL, 7 | updated INTEGER NOT NULL, 8 | content BLOB NOT NULL 9 | ); 10 | -------------------------------------------------------------------------------- /proto/data.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package tonic_sqlite; 3 | 4 | service Crud { 5 | rpc Create(stream Data) returns (ResponseCreate); 6 | rpc Read(DataQuery) returns (stream ResponseRead); 7 | rpc Update(DataUpdate) returns (ResponseChange); 8 | rpc Delete(DataQuery) returns (ResponseChange); 9 | } 10 | 11 | 12 | message Data { 13 | optional int32 id = 1; 14 | string name = 2; 15 | string mime = 3; 16 | optional int32 created = 4; 17 | optional int32 updated = 5; 18 | bytes content = 6; 19 | } 20 | 21 | message DataQuery { 22 | string query = 1; 23 | } 24 | 25 | message DataUpdate { 26 | string update = 1; 27 | optional bytes content = 2; 28 | } 29 | 30 | message ResponseCreate { 31 | bool ok = 1; 32 | int32 id = 2; 33 | string desc = 3; 34 | } 35 | 36 | message ResponseRead { 37 | bool ok = 1; 38 | optional string desc = 2; 39 | Data results = 3; 40 | } 41 | 42 | message ResponseChange { 43 | bool ok = 1; 44 | optional string desc = 2; 45 | int32 rows = 3; 46 | } 47 | -------------------------------------------------------------------------------- /src/client.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate diesel; 3 | mod entity; 4 | mod schema; 5 | 6 | use streamer::*; 7 | use tonic::Request; 8 | use tonic_sqlite::crud_client::CrudClient; 9 | use tonic_sqlite::*; 10 | 11 | use std::io::Read; 12 | 13 | pub mod tonic_sqlite { 14 | tonic::include_proto!("tonic_sqlite"); 15 | } 16 | 17 | #[tokio::main] 18 | async fn main() -> Result<(), Box> { 19 | let mut client = CrudClient::connect("http://127.0.0.1:7749").await?; 20 | 21 | let file = std::fs::File::open("./input.txt").unwrap(); 22 | let s = Streaming::from(file).chunks(16).then(|chunk| async { 23 | Data { 24 | id: None, // it is okay to set None, 25 | name: "info".into(), 26 | mime: "text/file".into(), 27 | created: None, 28 | updated: None, 29 | content: chunk, 30 | } 31 | }); 32 | let res_create = client.create(s).await?.into_inner(); 33 | let id = res_create.id; 34 | assert!(res_create.ok, "file not inserted"); 35 | 36 | // we query the inserted data 37 | let query_read = DataQuery { 38 | query: format!("select * from grsql where id = {};", id), 39 | }; 40 | let req_read = Request::new(query_read); 41 | let res_read = client.read(req_read).await?.into_inner(); 42 | let buf = res_read 43 | .map(|data| { 44 | assert!(data.is_ok(), "chunk corrupted"); 45 | data.unwrap().results.unwrap().content 46 | }) 47 | .flat_map(|en| futures_util::stream::iter(en)) 48 | .collect::>() 49 | .await; 50 | let file = std::fs::File::open("./input.txt").unwrap(); 51 | assert_eq!( 52 | file.bytes().map(|e| e.unwrap()).collect::>(), 53 | buf, 54 | "file not consistent" 55 | ); 56 | 57 | // update the inserted data 58 | let query_update = DataUpdate { 59 | update: format!("update grsql set id = 9 where id = {};", id), 60 | content: None, 61 | }; 62 | let req_update = Request::new(query_update); 63 | let res_update = client.update(req_update).await?.into_inner(); 64 | assert!(res_update.ok, "id not updated"); 65 | assert_eq!(res_update.rows, 1, "not one row changed"); 66 | 67 | // delete the data 68 | let query_delete = DataQuery { 69 | query: "delete from grsql where id = 9;".into(), 70 | }; 71 | let req_delete = Request::new(query_delete); 72 | let res_delete = client.delete(req_delete).await?.into_inner(); 73 | assert!(res_delete.ok, "data not deleted"); 74 | assert_eq!(res_delete.rows, 1, "not one row deleted"); 75 | 76 | Ok(()) 77 | } 78 | -------------------------------------------------------------------------------- /src/entity.rs: -------------------------------------------------------------------------------- 1 | use super::schema::*; 2 | use super::tonic_sqlite; 3 | use diesel::{prelude::*, sqlite::SqliteConnection, Insertable, Queryable}; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Deserialize, Serialize, PartialEq, Debug, QueryableByName, Queryable, Insertable)] 7 | #[table_name = "grsql"] 8 | pub struct Data { 9 | #[serde(default = "Default::default", skip_serializing_if = "is_zero")] 10 | pub id: i32, 11 | pub name: String, 12 | pub mime: String, 13 | #[serde(default = "now")] 14 | pub created: i32, 15 | #[serde(default = "now")] 16 | pub updated: i32, 17 | #[serde(default = "Default::default")] 18 | pub content: Vec, 19 | } 20 | 21 | impl Data { 22 | pub fn new() -> Self { 23 | let now = now(); 24 | Self { 25 | id: 0, 26 | name: String::new(), 27 | mime: "text/plain".into(), 28 | created: now, 29 | updated: now, 30 | content: Vec::new(), 31 | } 32 | } 33 | 34 | pub fn validate(&mut self) -> bool { 35 | if self.created == 0 { 36 | self.created = now(); 37 | } 38 | if self.updated == 0 { 39 | self.updated = now(); 40 | } 41 | self.id != 0 && !self.name.is_empty() && !self.mime.is_empty() && !self.content.is_empty() 42 | } 43 | } 44 | 45 | impl From for Data { 46 | fn from(d: tonic_sqlite::Data) -> Self { 47 | Self { 48 | id: d.id.unwrap_or(0) as _, 49 | name: d.name, 50 | mime: d.mime, 51 | created: d.created.unwrap_or(now()), 52 | updated: d.updated.unwrap_or(now()), 53 | content: d.content, 54 | } 55 | } 56 | } 57 | 58 | impl Into for Data { 59 | fn into(self) -> tonic_sqlite::Data { 60 | let id = match self.id { 61 | 0 => None, 62 | _ => Some(self.id), 63 | }; 64 | let created = match self.created { 65 | 0 => None, 66 | _ => Some(self.created), 67 | }; 68 | let updated = match self.updated { 69 | 0 => None, 70 | _ => Some(self.updated), 71 | }; 72 | tonic_sqlite::Data { 73 | id: id, 74 | name: self.name, 75 | mime: self.mime, 76 | created, 77 | updated, 78 | content: self.content, 79 | } 80 | } 81 | } 82 | 83 | pub fn now() -> i32 { 84 | use std::time::{SystemTime, UNIX_EPOCH}; 85 | SystemTime::now() 86 | .duration_since(UNIX_EPOCH) 87 | .unwrap() 88 | .as_secs() as _ 89 | } 90 | 91 | pub fn is_zero(en: &i32) -> bool { 92 | en == &0 93 | } 94 | 95 | #[allow(dead_code)] 96 | pub fn establish_connection() -> SqliteConnection { 97 | dotenv::dotenv().ok(); 98 | let database_url = std::env::var("DATABASE_URL").expect("database_url must not be null"); 99 | SqliteConnection::establish(&database_url) 100 | .expect(&format!("fail to connect to {}", database_url)) 101 | } 102 | -------------------------------------------------------------------------------- /src/schema.rs: -------------------------------------------------------------------------------- 1 | table! { 2 | grsql (id) { 3 | id -> Integer, 4 | name -> Text, 5 | mime -> Text, 6 | created -> Integer, 7 | updated -> Integer, 8 | content -> Binary, 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/server.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate diesel; 3 | mod entity; 4 | mod schema; 5 | 6 | use entity::establish_connection; 7 | use tonic::{transport::Server, Request, Response, Status, Streaming}; 8 | use tonic_sqlite::crud_server::{Crud, CrudServer}; 9 | use tonic_sqlite::*; 10 | 11 | use diesel::{prelude::*, sql_types::*, sqlite::SqliteConnection}; 12 | use std::{cell::Cell, env, pin::Pin, sync::RwLock}; 13 | use streamer::{Stream, StreamExt}; 14 | 15 | pub mod tonic_sqlite { 16 | tonic::include_proto!("tonic_sqlite"); 17 | } 18 | 19 | pub struct SqliteCruder { 20 | conn: RwLock, 21 | base_id: Cell, 22 | } 23 | unsafe impl Sync for SqliteCruder {} 24 | 25 | impl SqliteCruder { 26 | pub fn new() -> Self { 27 | use diesel::prelude::*; 28 | use schema::grsql::dsl::*; 29 | 30 | dotenv::dotenv().ok(); 31 | let database_url = env::var("DATABASE_URL").expect("database_url must not be null"); 32 | let conn = SqliteConnection::establish(&database_url) 33 | .expect(&format!("fail to connect to {}", database_url)); 34 | let base_id = grsql 35 | .select(id) 36 | .order(id.desc()) 37 | .get_result::(&conn) 38 | .expect("fail to load max id"); 39 | 40 | Self { 41 | conn: RwLock::new(conn), 42 | base_id: Cell::new(base_id), 43 | } 44 | } 45 | } 46 | 47 | #[tonic::async_trait] 48 | impl Crud for SqliteCruder { 49 | async fn create( 50 | &self, 51 | data: Request>, 52 | ) -> Result, Status> { 53 | use std::collections::LinkedList; 54 | dotenv::dotenv().ok(); 55 | let buf_len_limit = std::env::var("BUF_LEN") 56 | .expect("max buffer size must be set") 57 | .parse() 58 | .expect("must be valid integer"); 59 | let mut content_v = LinkedList::new(); 60 | let mut is_start = true; 61 | let mut buf_len = 0; 62 | let base_id = self.base_id.clone(); 63 | let mut en: Option = None; 64 | let mut input_stream = data.into_inner(); 65 | tokio::spawn(async move { 66 | while let Some(result) = input_stream.next().await { 67 | match result { 68 | Ok(mut v) => { 69 | if is_start { 70 | is_start = false; 71 | let v2 = Vec::new(); 72 | let c = std::mem::replace(&mut v.content, v2); 73 | buf_len += c.len(); 74 | content_v.push_back(c); 75 | en = Some(v); 76 | } else { 77 | buf_len += v.content.len(); 78 | content_v.push_back(v.content); 79 | //en.as_mut().unwrap().content.extend(v.content); 80 | } 81 | if buf_len > buf_len_limit { 82 | return Err(Status::cancelled("too long input data")); 83 | } 84 | } 85 | Err(e) => { 86 | if let Some(io_err) = match_for_io_error(&e) { 87 | if io_err.kind() == std::io::ErrorKind::BrokenPipe { 88 | eprintln!("\tclient disconnected: broken pipe"); 89 | break; 90 | } 91 | } 92 | } 93 | } 94 | } 95 | 96 | match en { 97 | None => Err(Status::aborted("Incomplete message or invalid data")), 98 | Some(mut e) => { 99 | use diesel::prelude::*; 100 | use schema::grsql::dsl::*; 101 | let mut content_vec = Vec::with_capacity(buf_len); 102 | content_v.into_iter().for_each(|c| { 103 | content_vec.extend(c); 104 | }); 105 | e.content = content_vec; 106 | let mut en: entity::Data = e.into(); 107 | //assert!(!en.content.is_empty(), "content must not be empty"); 108 | en.id = base_id.get() + 1; 109 | diesel::insert_into(grsql) 110 | .values(vec![en]) 111 | .execute(&establish_connection()) 112 | .unwrap(); 113 | Ok(Response::new(ResponseCreate { 114 | ok: true, 115 | id: base_id.get() + 1, 116 | desc: "".into(), 117 | })) 118 | } 119 | } 120 | }) 121 | .await 122 | .and_then(|en| { 123 | self.base_id.set(self.base_id.get() + 1); 124 | Ok(en) 125 | }) 126 | .map_err(|e| Status::internal(format!("runtime execution error: {:?}", e))) 127 | .unwrap() 128 | } 129 | 130 | type ReadStream = Pin> + Send>>; 131 | async fn read( 132 | &self, 133 | query: Request, 134 | //) -> Result>, Status> { 135 | ) -> Result, Status> { 136 | let query: DataQuery = query.into_inner(); 137 | match diesel::sql_query(query.query) 138 | .get_results::(&*self.conn.read().unwrap()) 139 | { 140 | Ok(data) => { 141 | let data_vec = data 142 | .into_iter() 143 | .map(|en| { 144 | Ok::<_, Status>(ResponseRead { 145 | ok: true, 146 | desc: None, 147 | results: Some(en.into()), 148 | }) 149 | }) 150 | .collect::>(); 151 | let s = futures_util::stream::iter(data_vec); 152 | 153 | Ok(Response::new(Box::pin(s))) 154 | } 155 | Err(e) => Err(Status::internal(format!( 156 | "Invalid query or Not Found: {:?}", 157 | e 158 | ))), 159 | } 160 | } 161 | 162 | async fn update(&self, query: Request) -> Result, Status> { 163 | let update_query = query.into_inner(); 164 | let query = diesel::sql_query(update_query.update); 165 | match update_query.content { 166 | Some(byte) => { 167 | match query 168 | .bind::(byte) 169 | .execute(&*self.conn.read().unwrap()) 170 | { 171 | Ok(data) => Ok(Response::new(ResponseChange { 172 | ok: true, 173 | desc: None, 174 | rows: data as _, 175 | })), 176 | Err(e) => Err(Status::not_found(format!( 177 | "Invalid query or Not Found: {:?}", 178 | e 179 | ))), 180 | //Err(e) => Err(Status::not_found("Invalid query or Not Found")), 181 | } 182 | } 183 | None => match query.execute(&*self.conn.read().unwrap()) { 184 | Ok(data) => Ok(Response::new(ResponseChange { 185 | ok: true, 186 | desc: None, 187 | rows: data as _, 188 | })), 189 | Err(e) => Err(Status::not_found(format!( 190 | "Invalid query or Not Found: {:?}", 191 | e 192 | ))), 193 | }, 194 | } 195 | } 196 | 197 | async fn delete(&self, query: Request) -> Result, Status> { 198 | use diesel::prelude::*; 199 | 200 | let delete_query = query.into_inner(); 201 | match diesel::sql_query(delete_query.query).execute(&*self.conn.read().unwrap()) { 202 | Ok(data) => Ok(Response::new(ResponseChange { 203 | ok: true, 204 | desc: None, 205 | rows: data as _, 206 | })), 207 | Err(e) => Err(Status::not_found(format!( 208 | "Invalid query or Not Found: {:?}", 209 | e 210 | ))), 211 | } 212 | } 213 | } 214 | 215 | #[test] 216 | fn test_sql_query() { 217 | use diesel::prelude::*; 218 | use diesel::sql_types::*; 219 | 220 | let conn = establish_connection(); 221 | let s = "insert into grsql (id, name, mime, created, updated, content) values (10, \"name\", \"text/plain\", 1000, 1000, ?);"; 222 | let data = diesel::sql_query(s) 223 | .bind::(vec![65, 66, 99, 100, 24, 97]) 224 | .execute(&conn) 225 | .unwrap(); 226 | dbg!(&data); 227 | let s_select = "select * from grsql where id = 10"; 228 | let data_select = diesel::sql_query(s_select) 229 | .get_result::(&conn) 230 | .unwrap(); 231 | dbg!(&data_select); 232 | let s_update = "update grsql set updated = 102358 where id = 10;"; 233 | let data_update = diesel::sql_query(s_update).execute(&conn).unwrap(); 234 | dbg!(&data_update); 235 | let s_delete = "delete from grsql where id = 10;"; 236 | let data_delete = diesel::sql_query(s_delete).execute(&conn).unwrap(); 237 | dbg!(&data_delete); 238 | assert!(false); 239 | } 240 | 241 | fn match_for_io_error(err_status: &Status) -> Option<&std::io::Error> { 242 | let mut err: &(dyn std::error::Error + 'static) = err_status; 243 | 244 | loop { 245 | if let Some(io_err) = err.downcast_ref::() { 246 | return Some(io_err); 247 | } 248 | 249 | // h2::Error do not expose std::io::Error with `source()` 250 | // https://github.com/hyperium/h2/pull/462 251 | /* 252 | *if let Some(h2_err) = err.downcast_ref::() { 253 | * if let Some(io_err) = h2_err.get_io() { 254 | * return Some(io_err); 255 | * } 256 | *} 257 | */ 258 | 259 | err = match err.source() { 260 | Some(err) => err, 261 | None => return None, 262 | }; 263 | } 264 | } 265 | 266 | #[tokio::main] 267 | async fn main() -> Result<(), Box> { 268 | let addr = "127.0.0.1:7749".parse()?; 269 | let cruder = SqliteCruder::new(); 270 | 271 | println!("Listening on: {:?}", addr); 272 | Server::builder() 273 | .add_service(CrudServer::new(cruder)) 274 | .serve(addr) 275 | .await?; 276 | 277 | Ok(()) 278 | } 279 | --------------------------------------------------------------------------------