├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .prettierignore ├── .prettierrc.json ├── .vscode ├── launch.json └── settings.json ├── Cargo.toml ├── README.md └── src ├── connection.rs ├── engine-demo.rs ├── lib.rs └── pager ├── constants.rs ├── db_header.rs ├── mod.rs └── page.rs /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | pull_request: 7 | branches: ["main"] 8 | 9 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 10 | jobs: 11 | # This workflow contains a single job called "build" 12 | build: 13 | # The type of runner that the job will run on 14 | runs-on: ubuntu-latest 15 | 16 | # Steps represent a sequence of tasks that will be executed as part of the job 17 | steps: 18 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 19 | - uses: actions/checkout@v3 20 | 21 | - name: Update local toolchain 22 | run: | 23 | rustup update 24 | rustup component add clippy 25 | rustup install stable 26 | 27 | - name: Toolchain info 28 | run: | 29 | cargo --version --verbose 30 | rustc --version 31 | cargo clippy --version 32 | 33 | - name: Verify code style (prettier) 34 | run: | 35 | npm install --global prettier-plugin-rust prettier 36 | prettier --check **/* 37 | 38 | - name: Building engine (library) 39 | run: | 40 | cargo build --lib --verbose --release 41 | 42 | - name: Building demo app (bin) 43 | run: | 44 | cargo build --bin engine-demo --verbose --release 45 | 46 | - name: Linting the app via Cargo clippy 47 | run: | 48 | cargo clippy -- -D warnings 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | /db -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | /db -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "useTabs": false, 3 | "tabWidth": 4, 4 | "printWidth": 99, 5 | "endOfLine": "lf" 6 | } 7 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "lldb", 9 | "request": "launch", 10 | "name": "Launch demo app for library (valid file)", 11 | "cargo": { 12 | "args": ["build", "--bin", "engine-demo"] 13 | }, 14 | "env": { "RUST_LOG": "DEBUG" }, 15 | "args": ["./db/database.data"] 16 | }, 17 | { 18 | "type": "lldb", 19 | "request": "launch", 20 | "name": "Launch demo app for library (invalid file)", 21 | "cargo": { 22 | "args": ["build", "--bin", "engine-demo"] 23 | }, 24 | "env": { "RUST_LOG": "DEBUG" }, 25 | "args": ["./db"] 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "lldb.displayFormat": "auto", 3 | "lldb.showDisassembly": "auto", 4 | "lldb.dereferencePointers": true, 5 | "lldb.consoleMode": "commands" 6 | } 7 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "engine" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | env_logger = "0.9.1" 8 | log = "0.4.17" 9 | bincode = "2.0.0-rc.2" 10 | 11 | [[bin]] 12 | name = "engine-demo" 13 | path = "src/engine-demo.rs" -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # About 2 | 3 | This repository is intended to store the final code for the database engine developed 4 | as part of the series of articles in the Medium written by me. 5 | 6 | [The link to the first part](https://medium.com/@valerii.maslenikov/writing-database-storage-engine-from-scratch-part-1-5303c549c26) 7 | -------------------------------------------------------------------------------- /src/connection.rs: -------------------------------------------------------------------------------- 1 | use crate::pager::Config as PagerConfig; 2 | use crate::pager::Pager; 3 | 4 | #[derive(Debug)] 5 | pub struct Config { 6 | pub cache_size_mb: u32, 7 | pub create: bool, 8 | } 9 | 10 | impl Default for Config { 11 | fn default() -> Self { 12 | Config { 13 | cache_size_mb: 100, 14 | create: true, 15 | } 16 | } 17 | } 18 | 19 | #[derive(Debug)] 20 | #[allow(dead_code)] 21 | pub struct Connection { 22 | config: Config, 23 | pager: Pager, 24 | } 25 | 26 | impl Connection { 27 | /// Connection is the main entrypoint for the library, which means that all 28 | /// initializations will be performed in this method 29 | pub fn open(main_db_path: String, config: Config) -> std::io::Result { 30 | let pager_config = PagerConfig { main_db_path }; 31 | 32 | let pager = Pager::new(pager_config)?; 33 | 34 | Ok(Connection { 35 | config, 36 | pager, 37 | }) 38 | } 39 | } -------------------------------------------------------------------------------- /src/engine-demo.rs: -------------------------------------------------------------------------------- 1 | use std::{ env }; 2 | 3 | use engine::connection::{ Connection, Config }; 4 | use log::{ info, error }; 5 | 6 | const DEFAULT_DB_PATH: &str = "./db/database.data"; 7 | fn main() { 8 | env_logger::init(); 9 | 10 | let args: Vec = env::args().collect(); 11 | 12 | // We expect the first arg to be the database path 13 | let database_path = args.get(1).get_or_insert(&DEFAULT_DB_PATH.to_string()).clone(); 14 | 15 | let connection_result = Connection::open(database_path, Config { 16 | ..Default::default() 17 | }); 18 | 19 | if connection_result.is_err() { 20 | error!("Connection to database cannot be established: {}", connection_result.unwrap_err()); 21 | } else { 22 | info!("Connection to database engine is succesfully established"); 23 | } 24 | } -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod connection; 2 | mod pager; -------------------------------------------------------------------------------- /src/pager/constants.rs: -------------------------------------------------------------------------------- 1 | /// This value defines the number of bytes the single page occupies on the disk. 2 | /// It completely depends on the Virtual Memory page size of the OS. 3 | /// 4 | /// TODO: Make this page size configurable, as far as performance depends on the 5 | /// actual page size within the OS/Hardware 6 | /// 7 | pub const PAGE_SIZE_BYTES: u32 = 4096; 8 | 9 | /// It's reserved amount of bytes for Database header. The original structure occupies 10 | /// less amount of bytes, but to avoid any restructuring for the sake of compatibliity 11 | /// we will reserve additional bytes for that purpose. 12 | pub const DATABASE_HEADER_BYTES: usize = 100; 13 | 14 | /// Defines the magic header string "Simple Data Engine", which is used to verify that the loaded 15 | /// file is the database file of this engine 16 | pub const MAGIC_HEADER_STRING: [u8; 18] = [ 17 | 83, 105, 109, 112, 108, 101, 32, 68, 97, 116, 97, 32, 69, 110, 103, 105, 110, 101, 18 | ]; -------------------------------------------------------------------------------- /src/pager/db_header.rs: -------------------------------------------------------------------------------- 1 | use bincode::{ Decode, Encode }; 2 | 3 | use super::constants::{ MAGIC_HEADER_STRING, PAGE_SIZE_BYTES }; 4 | 5 | #[derive(Encode, Decode, PartialEq, Eq, Debug)] 6 | pub struct DatabaseHeader { 7 | // The value should be always equal to MAGIC_HEADER_STRING constant 8 | magic_header_string: [u8; MAGIC_HEADER_STRING.len()], 9 | // In the current version this value will always be equal to PAGE_SIZE_BYTES, 10 | page_size_bytes: u32, 11 | } 12 | 13 | impl Default for DatabaseHeader { 14 | fn default() -> Self { 15 | DatabaseHeader { 16 | magic_header_string: MAGIC_HEADER_STRING, 17 | page_size_bytes: PAGE_SIZE_BYTES, 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /src/pager/mod.rs: -------------------------------------------------------------------------------- 1 | use constants::PAGE_SIZE_BYTES; 2 | use log::{ debug, info }; 3 | use std::fs::File; 4 | use std::io::{ Error as IOError, Write }; 5 | use std::result::Result; 6 | use std::vec; 7 | 8 | use crate::pager::constants::DATABASE_HEADER_BYTES; 9 | use crate::pager::db_header::DatabaseHeader; 10 | 11 | mod constants; 12 | mod db_header; 13 | 14 | type PagerResult = Result; 15 | 16 | #[derive(Debug)] 17 | pub struct Config { 18 | pub main_db_path: String, 19 | } 20 | 21 | #[derive(Debug)] 22 | pub struct Pager { 23 | main_db_file: File, 24 | } 25 | 26 | impl Pager { 27 | pub fn new(config: Config) -> PagerResult { 28 | debug!("Trying to open file {}, or create if it doesn't exist", &config.main_db_path); 29 | 30 | let main_db_file = File::options() 31 | .read(true) 32 | .write(true) 33 | .create(true) 34 | .open(&config.main_db_path)?; 35 | 36 | info!("Database file by path {} is opened successfully", &config.main_db_path); 37 | 38 | let mut pager = Pager { main_db_file }; 39 | 40 | pager.initialize_db_if_new()?; 41 | 42 | Ok(pager) 43 | } 44 | 45 | fn initialize_db_if_new(&mut self) -> PagerResult<()> { 46 | let file_metadata = self.main_db_file.metadata()?; 47 | 48 | let file_length = file_metadata.len(); 49 | 50 | debug!("Checking the file metadata to create the database header if it is absent"); 51 | if file_length == 0 { 52 | debug!("Database file is empty – creating the metapage with header"); 53 | let mut metapage_buffer = vec![0_u8; PAGE_SIZE_BYTES as usize]; 54 | 55 | let header = DatabaseHeader::default(); 56 | 57 | // We control the input of the encoded value, we may ignore possible errors (unwrap()) 58 | let encoded_database_header = bincode 59 | ::encode_into_slice(header, &mut metapage_buffer, bincode::config::standard()) 60 | .unwrap(); 61 | 62 | // We want to make sure that once we add new fields to the header it will not exceed 63 | // the preserved space, otherwise we will start overwriting the first page metadata 64 | assert!(encoded_database_header <= DATABASE_HEADER_BYTES); 65 | 66 | self.main_db_file.write_all(&metapage_buffer)?; 67 | 68 | info!("Database header has been written to metapage"); 69 | } else { 70 | info!("Database file is not empty, its size is {}", file_length); 71 | } 72 | Ok(()) 73 | } 74 | } -------------------------------------------------------------------------------- /src/pager/page.rs: -------------------------------------------------------------------------------- 1 | enum PageType { 2 | TableNonLeafPage, 3 | TableLeafPage, 4 | } 5 | 6 | struct PageHeader { 7 | node_type: PageType, 8 | } --------------------------------------------------------------------------------