├── .gitignore ├── .travis.yml ├── Cargo.toml ├── README.md ├── sillydb ├── Cargo.toml └── src │ └── main.rs └── src ├── disk.rs ├── lib.rs ├── page.rs └── util.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | **/*.rs.bk 3 | Cargo.lock 4 | /sillydb/target/ 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | rust: 3 | - stable 4 | - beta 5 | - nightly 6 | matrix: 7 | allow_failures: 8 | - rust: nightly 9 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "linhash" 3 | version = "0.1.0" 4 | authors = ["Samrat Man Singh "] 5 | 6 | [lib] 7 | name = "linhash" 8 | path = "src/lib.rs" 9 | 10 | [dependencies] 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rust-linhash 2 | 3 | [![Build Status](https://travis-ci.org/samrat/rust-linhash.svg?branch=master)](https://travis-ci.org/samrat/rust-linhash) 4 | 5 | A [Linear hashing][wiki] implementation in Rust. For more information, 6 | see [this blog post][blogpost]([part 2][part2] describes this project). 7 | 8 | [wiki]: https://en.wikipedia.org/wiki/Linear_hashing 9 | [blogpost]: https://samrat.me/posts/2017-11-04-kvstore-linear-hashing/ 10 | [part2]: https://samrat.me/posts/2017-11-09-kvstore-rust-hashtable/ 11 | -------------------------------------------------------------------------------- /sillydb/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sillydb" 3 | version = "0.1.0" 4 | authors = ["Samrat Man Singh "] 5 | 6 | [dependencies] 7 | linhash = { path = "../" } 8 | -------------------------------------------------------------------------------- /sillydb/src/main.rs: -------------------------------------------------------------------------------- 1 | extern crate linhash; 2 | 3 | use linhash::LinHash; 4 | use std::time::Instant; 5 | use std::fs; 6 | use linhash::util::*; 7 | 8 | #[allow(dead_code)] 9 | fn measure_perf(num_iters: i32) { 10 | // in each iteration, insert a larger number of records to see how 11 | // `insert` and `lookup` performs. `insert` should be O(n) and 12 | // `lookup` should be O(1). 13 | for i in 1..num_iters { 14 | let now = Instant::now(); 15 | let mut h2 = LinHash::open("/tmp/measure_perf", 4, 4); 16 | for k in 0..(10000*i) { 17 | h2.put(&linhash::util::i32_to_bytearray(k), 18 | &linhash::util::i32_to_bytearray(k+1)); 19 | } 20 | 21 | let time_get = Instant::now(); 22 | for k in 1000..9000 { 23 | assert_eq!(h2.get(&linhash::util::i32_to_bytearray(k)), 24 | Some(linhash::util::i32_to_bytearray(k+1).to_vec())); 25 | println!("{}", k); 26 | } 27 | let time_get_done = Instant::now(); 28 | println!("[get]{} million records {:?}", i, time_get_done.duration_since(time_get)); 29 | 30 | let new_now = Instant::now(); 31 | println!("[insert+get]{} million records {:?}", i, new_now.duration_since(now)); 32 | h2.close(); 33 | fs::remove_file("/tmp/measure_perf"); 34 | } 35 | 36 | } 37 | 38 | fn main() { 39 | let mut h = LinHash::open("/tmp/main_tests", 32, 4); 40 | h.put(b"Spin", &i32_to_bytearray(9)); 41 | h.put(b"Axis", &i32_to_bytearray(6)); 42 | h.put(b"foo", &[14]); 43 | h.put(b"bar", &[15]); 44 | h.put(b"linear", &[16]); 45 | h.put(b"hashing", &[17]); 46 | h.put(b"disk", &[18]); 47 | h.put(b"space", &[19]); 48 | h.put(b"random", &[20]); 49 | h.put(b"keys", &[21]); 50 | h.put(b"samrat", &[22]); 51 | h.put(b"linhash", &[21]); 52 | h.put(b"rust", &[21]); 53 | h.put(b"3:30", &[21]); 54 | h.put(b"xinu", &[21]); 55 | h.put(b"linhash1", &[21]); 56 | h.put(b"rust1", &[22]); 57 | h.put(b"rust2", &[51]); 58 | h.put(b"rust3", &[52]); 59 | h.put(b"rust4", &[53]); 60 | h.put(b"rust5", &[54]); 61 | 62 | h.update(b"rust1", &[99]); 63 | h.put(b"xinu3", &[24]); 64 | h.close(); 65 | 66 | measure_perf(2); 67 | 68 | println!("{:?}", h.get("rust3".as_bytes())); 69 | } 70 | -------------------------------------------------------------------------------- /src/disk.rs: -------------------------------------------------------------------------------- 1 | use std::collections::VecDeque; 2 | use std::io::prelude::*; 3 | use std::fs::File; 4 | use std::fs::OpenOptions; 5 | use std::io::SeekFrom; 6 | use std::mem; 7 | 8 | use page::{Page, PAGE_SIZE, HEADER_SIZE}; 9 | use util::*; 10 | 11 | const NUM_BUFFERS : usize = 16; 12 | 13 | pub struct SearchResult { 14 | pub page_id: Option, 15 | pub row_num: Option, 16 | pub val: Option> 17 | } 18 | 19 | fn flatten(v: Vec<(usize, Vec)>) -> Vec { 20 | let mut result = vec![]; 21 | for (_, mut i) in v { 22 | result.append(&mut i); 23 | } 24 | result 25 | } 26 | 27 | pub struct DbFile { 28 | path: String, 29 | file: File, 30 | ctrl_buffer: Page, 31 | pub buffers: VecDeque, 32 | pub records_per_page: usize, 33 | bucket_to_page: Vec, 34 | keysize: usize, 35 | valsize: usize, 36 | num_pages: usize, 37 | // overflow pages no longer in use 38 | free_list: Option, 39 | num_free: usize, 40 | } 41 | 42 | impl DbFile { 43 | pub fn new(filename: &str, keysize: usize, valsize: usize) -> DbFile { 44 | let file = OpenOptions::new() 45 | .read(true) 46 | .write(true) 47 | .create(true) 48 | .open(filename); 49 | let file = match file { 50 | Ok(f) => f, 51 | Err(e) => panic!(e), 52 | }; 53 | 54 | let total_size = keysize + valsize; 55 | let records_per_page = (PAGE_SIZE - HEADER_SIZE) / total_size; 56 | 57 | let mut buffers : VecDeque = 58 | VecDeque::with_capacity(NUM_BUFFERS); 59 | for _i in 0..NUM_BUFFERS { 60 | buffers.push_back(Page::new(keysize, valsize)); 61 | } 62 | 63 | DbFile { 64 | path: String::from(filename), 65 | file: file, 66 | ctrl_buffer: Page::new(0, 0), 67 | buffers: buffers, 68 | records_per_page: records_per_page, 69 | bucket_to_page: vec![1, 2], 70 | keysize: keysize, 71 | valsize: valsize, 72 | num_pages: 3, 73 | free_list: Some(3), 74 | num_free: 0, 75 | } 76 | } 77 | 78 | // Control page layout: 79 | // 80 | // | nbits | nitems | nbuckets | num_pages | free_list root | 81 | // num_free | bucket_to_page mappings .... | 82 | pub fn read_ctrlpage(&mut self) -> (usize, usize, usize) { 83 | self.get_ctrl_page(); 84 | let nbits : usize = bytearray_to_usize(self.ctrl_buffer.storage[0..8].to_vec()); 85 | let nitems : usize = 86 | bytearray_to_usize(self.ctrl_buffer.storage[8..16].to_vec()); 87 | let nbuckets : usize = 88 | bytearray_to_usize(self.ctrl_buffer.storage[16..24].to_vec()); 89 | 90 | self.num_pages = 91 | bytearray_to_usize(self.ctrl_buffer.storage[24..32].to_vec()); 92 | let free_list_head = bytearray_to_usize(self.ctrl_buffer.storage[32..40].to_vec()); 93 | self.free_list = 94 | if free_list_head == 0 { 95 | None 96 | } else { 97 | Some(free_list_head) 98 | }; 99 | self.num_free = 100 | bytearray_to_usize(self.ctrl_buffer.storage[40..48].to_vec()); 101 | self.bucket_to_page = 102 | bytevec_to_usize_vec(self.ctrl_buffer.storage[48..PAGE_SIZE].to_vec()); 103 | (nbits, nitems, nbuckets) 104 | } 105 | 106 | pub fn write_ctrlpage(&mut self, 107 | (nbits, nitems, nbuckets): 108 | (usize, usize, usize)) { 109 | self.get_ctrl_page(); 110 | 111 | let nbits_bytes = usize_to_bytearray(nbits); 112 | let nitems_bytes = usize_to_bytearray(nitems); 113 | let nbuckets_bytes = usize_to_bytearray(nbuckets); 114 | let num_pages_bytes = usize_to_bytearray(self.num_pages); 115 | let free_list_bytes = usize_to_bytearray(self.free_list.unwrap_or(0)); 116 | let num_free_bytes = usize_to_bytearray(self.num_free); 117 | let bucket_to_page_bytevec = usize_vec_to_bytevec(self.bucket_to_page.clone()); 118 | let mut bucket_to_page_bytearray = vec![]; 119 | bucket_to_page_bytearray.write(&bucket_to_page_bytevec) 120 | .expect("Write to ctrlpage failed"); 121 | 122 | println!("nbits: {:?} nitems: {:?} nbuckets: {:?}", nbits_bytes, 123 | nitems_bytes, nbuckets_bytes); 124 | mem_move(&mut self.ctrl_buffer.storage[0..8], 125 | &nbits_bytes); 126 | mem_move(&mut self.ctrl_buffer.storage[8..16], 127 | &nitems_bytes); 128 | mem_move(&mut self.ctrl_buffer.storage[16..24], 129 | &nbuckets_bytes); 130 | mem_move(&mut self.ctrl_buffer.storage[24..32], 131 | &num_pages_bytes); 132 | mem_move(&mut self.ctrl_buffer.storage[32..40], 133 | &free_list_bytes); 134 | mem_move(&mut self.ctrl_buffer.storage[40..48], 135 | &num_free_bytes); 136 | mem_move(&mut self.ctrl_buffer.storage[48..PAGE_SIZE], 137 | &bucket_to_page_bytearray); 138 | DbFile::write_page(&mut self.file, 139 | 0, 140 | &self.ctrl_buffer.storage); 141 | } 142 | 143 | pub fn get_ctrl_page(&mut self) { 144 | self.file.seek(SeekFrom::Start(0)) 145 | .expect("Could not seek to offset"); 146 | self.file.read(&mut self.ctrl_buffer.storage) 147 | .expect("Could not read file"); 148 | } 149 | 150 | fn bucket_to_page(&self, bucket_id: usize) -> usize { 151 | self.bucket_to_page[bucket_id] 152 | } 153 | 154 | fn search_buffer_pool(&self, page_id: usize) -> Option { 155 | for (i, b) in self.buffers.iter().enumerate() { 156 | if b.id == page_id { 157 | return Some(i); 158 | } 159 | } 160 | None 161 | } 162 | 163 | /// Reads page to self.buffer 164 | pub fn fetch_page(&mut self, page_id: usize) -> usize { 165 | let bufpool_index = self.search_buffer_pool(page_id); 166 | match bufpool_index { 167 | None => { 168 | match self.buffers.pop_front() { 169 | Some(mut old_page) => { 170 | if old_page.dirty { 171 | old_page.write_header(); 172 | DbFile::write_page(&self.file, 173 | old_page.id, 174 | &old_page.storage); 175 | } 176 | }, 177 | _ => (), 178 | } 179 | 180 | let offset = (page_id * PAGE_SIZE) as u64; 181 | let mut new_page = Page::new(self.keysize, self.valsize); 182 | new_page.id = page_id; 183 | let buffer_index = NUM_BUFFERS - 1; 184 | 185 | self.file.seek(SeekFrom::Start(offset)) 186 | .expect("Could not seek to offset"); 187 | self.file.read(&mut new_page.storage) 188 | .expect("Could not read file"); 189 | self.buffers.push_back(new_page); 190 | self.buffers[buffer_index].read_header(); 191 | 192 | buffer_index 193 | }, 194 | Some(p) => p, 195 | } 196 | } 197 | 198 | /// Writes data in `data` into page `page_id` in file. 199 | pub fn write_page(mut file: &File, page_id: usize, data: &[u8]) { 200 | let offset = (page_id * PAGE_SIZE) as u64; 201 | file.seek(SeekFrom::Start(offset)) 202 | .expect("Could not seek to offset"); 203 | file.write(data).expect("write failed"); 204 | file.flush().expect("flush failed"); 205 | } 206 | 207 | /// Write record but don't increment `num_records`. Used when 208 | /// updating already existing record. 209 | pub fn write_record(&mut self, 210 | page_id: usize, 211 | row_num: usize, 212 | key: &[u8], 213 | val: &[u8]) { 214 | let buffer_index = self.fetch_page(page_id); 215 | self.buffers[buffer_index].dirty = true; 216 | self.buffers[buffer_index].write_record(row_num, key, val); 217 | } 218 | 219 | /// Write record and increment `num_records`. Used when inserting 220 | /// new record. 221 | pub fn write_record_incr(&mut self, page_id: usize, row_num: usize, 222 | key: &[u8], val: &[u8]) { 223 | let buffer_index = self.fetch_page(page_id); 224 | self.buffers[buffer_index].incr_num_records(); 225 | self.write_record(page_id, row_num, key, val); 226 | } 227 | 228 | /// Searches for `key` in `bucket`. A bucket is a linked list of 229 | /// pages. Return value: 230 | /// 231 | /// If key is present in bucket returns as struct, SearchResult 232 | /// (page_id, row_num, val). 233 | /// 234 | /// If key is not present and: 235 | /// 1. there is enough space in last page, returns (page_id, row_num, None) 236 | /// 237 | /// 2. there is not enough space in last page, returns 238 | /// (last_page_id, None, None) 239 | pub fn search_bucket(&mut self, bucket_id: usize, key: &[u8]) -> SearchResult { 240 | let mut page_id = self.bucket_to_page(bucket_id); 241 | let mut buffer_index; 242 | let mut first_free_row = SearchResult { 243 | page_id: None, 244 | row_num: None, 245 | val: None, 246 | }; 247 | loop { 248 | buffer_index = self.fetch_page(page_id); 249 | let next_page = self.buffers[buffer_index].next; 250 | let page_records = self.all_records_in_page(page_id); 251 | 252 | let len = page_records.len(); 253 | for (row_num, (k,v)) in page_records.into_iter().enumerate() { 254 | if slices_eq(&k, key) { 255 | return SearchResult{ 256 | page_id: Some(page_id), 257 | row_num: Some(row_num), 258 | val: Some(v) 259 | } 260 | } 261 | } 262 | 263 | let row_num = if len < self.records_per_page { 264 | Some(len) 265 | } else { 266 | None 267 | }; 268 | 269 | match (first_free_row.page_id, first_free_row.row_num) { 270 | // this is the first free space for a row found, so 271 | // keep track of it. 272 | (Some(_), None) | 273 | (None, _) => { 274 | first_free_row = SearchResult { 275 | page_id: Some(page_id), 276 | row_num: row_num, 277 | val: None, 278 | } 279 | }, 280 | _ => (), 281 | } 282 | 283 | if let Some(p) = next_page { 284 | page_id = p; 285 | } else { 286 | break; 287 | } 288 | } 289 | 290 | first_free_row 291 | } 292 | 293 | /// Add a new overflow page to a `bucket`. 294 | pub fn allocate_overflow(&mut self, bucket_id: usize, 295 | last_page_id: usize) -> (usize, usize) { 296 | let physical_index = self.allocate_new_page(); 297 | 298 | let new_page_buffer_index = self.fetch_page(physical_index); 299 | self.buffers[new_page_buffer_index].next = None; 300 | self.buffers[new_page_buffer_index].dirty = true; 301 | 302 | // Write next of old page 303 | let old_page_buffer_index = self.fetch_page(last_page_id); 304 | self.buffers[old_page_buffer_index].next = Some(physical_index); 305 | self.buffers[old_page_buffer_index].dirty = true; 306 | 307 | println!("setting next of buffer_id {}(page_id: {}) to {:?}", 308 | bucket_id, 309 | self.buffers[old_page_buffer_index].id, 310 | self.buffers[old_page_buffer_index].next); 311 | 312 | (physical_index, 0) 313 | } 314 | 315 | /// Write out page in bufferpool to file. 316 | pub fn write_buffer_page(&mut self, buffer_index: usize) { 317 | // Ignore page 0(ctrlpage) 318 | if self.buffers[buffer_index].id != 0 { 319 | self.buffers[buffer_index].dirty = false; 320 | self.buffers[buffer_index].write_header(); 321 | DbFile::write_page(&mut self.file, 322 | self.buffers[buffer_index].id, 323 | &self.buffers[buffer_index].storage); 324 | } 325 | } 326 | 327 | fn all_records_in_page(&mut self, page_id: usize) 328 | -> Vec<(Vec, Vec)> { 329 | let buffer_index = self.fetch_page(page_id); 330 | let mut page_records = vec![]; 331 | for i in 0..self.buffers[buffer_index].num_records { 332 | let (k, v) = self.buffers[buffer_index].read_record(i); 333 | let (dk, dv) = (k.to_vec(), v.to_vec()); 334 | page_records.push((dk, dv)); 335 | } 336 | 337 | page_records 338 | } 339 | 340 | /// Returns a vec of (page_id, records_in_vec). ie. each inner 341 | /// vector represents the records in a page in the bucket. 342 | fn all_records_in_bucket(&mut self, bucket_id: usize) 343 | -> Vec<(usize, Vec<(Vec,Vec)>)> { 344 | let first_page_id = self.bucket_to_page(bucket_id); 345 | let buffer_index = self.fetch_page(first_page_id); 346 | let mut records = Vec::new(); 347 | records.push((self.buffers[buffer_index].id, 348 | self.all_records_in_page(first_page_id))); 349 | 350 | let mut next_page = self.buffers[buffer_index].next; 351 | while let Some(page_id) = next_page { 352 | if page_id == 0 { 353 | break; 354 | } 355 | 356 | let buffer_index = self.fetch_page(page_id); 357 | records.push((page_id, 358 | self.all_records_in_page(page_id))); 359 | 360 | next_page = self.buffers[buffer_index].next; 361 | } 362 | 363 | records 364 | } 365 | 366 | /// Allocate a new page. If available uses recycled overflow 367 | /// pages. 368 | fn allocate_new_page(&mut self) -> usize { 369 | let p = self.free_list; 370 | let page_id = p.expect("no page in free_list"); 371 | println!("[allocate_new_page] allocating page_id: {}", page_id); 372 | let buffer_index = self.fetch_page(page_id); 373 | 374 | self.free_list = match self.buffers[buffer_index].next { 375 | Some(0) | None => { 376 | self.num_pages += 1; 377 | Some(self.num_pages) 378 | }, 379 | _ => { 380 | self.num_free -= 1; 381 | self.buffers[buffer_index].next 382 | }, 383 | }; 384 | 385 | let new_page = Page::new(self.keysize, self.valsize); 386 | mem::replace(&mut self.buffers[buffer_index], new_page); 387 | self.buffers[buffer_index].id = page_id; 388 | self.buffers[buffer_index].dirty = false; 389 | self.buffers[buffer_index].next = None; 390 | 391 | page_id 392 | } 393 | 394 | /// Empties out root page for bucket. Overflow pages are added to 395 | /// `free_list` 396 | pub fn clear_bucket(&mut self, bucket_id: usize) -> Vec<(Vec,Vec)> { 397 | let all_records = self.all_records_in_bucket(bucket_id); 398 | let records = flatten(all_records.clone()); 399 | 400 | // Add overflow pages to free_list 401 | let bucket_len = all_records.len(); 402 | if bucket_len > 1 { 403 | // second page onwards are overflow pages 404 | let (second_page_id, _) = all_records[1]; 405 | println!("[clear_bucket] adding overflow chain starting page {} to free_list", second_page_id); 406 | let temp = self.free_list; 407 | self.free_list = Some(second_page_id); 408 | 409 | let second_page_buffer_index = 410 | self.fetch_page(second_page_id); 411 | // overflow pages only 412 | self.num_free += bucket_len - 1; 413 | self.buffers[second_page_buffer_index].next = temp; 414 | } 415 | 416 | let page_id = self.bucket_to_page(bucket_id); 417 | let buffer_index = self.fetch_page(page_id); 418 | let new_page = Page::new(self.keysize, self.valsize); 419 | mem::replace(&mut self.buffers[buffer_index], new_page); 420 | self.buffers[buffer_index].id = page_id; 421 | self.buffers[buffer_index].dirty = false; 422 | self.write_buffer_page(buffer_index); 423 | 424 | records 425 | } 426 | 427 | pub fn allocate_new_bucket(&mut self) { 428 | let page_id = self.allocate_new_page(); 429 | self.bucket_to_page.push(page_id); 430 | } 431 | 432 | pub fn close(&mut self) { 433 | for b in 0..NUM_BUFFERS { 434 | self.write_buffer_page(b); 435 | } 436 | } 437 | } 438 | 439 | #[cfg(test)] 440 | mod tests { 441 | use disk; 442 | use DbFile; 443 | use std::fs; 444 | 445 | #[test] 446 | fn dbfile_tests () { 447 | let mut bp = DbFile::new("/tmp/dbfile_tests", 4, 4); 448 | let bark = b"bark"; 449 | let krab = b"krab"; 450 | // write to page 1 451 | bp.write_record(1, 14, bark, krab); 452 | assert_eq!(bp.buffers[disk::NUM_BUFFERS-1].read_record(14), 453 | (&bark[..], &krab[..])); 454 | bp.close(); 455 | 456 | let mut bp2 = DbFile::new("/tmp/dbfile_tests", 4, 4); 457 | // read from page 1 458 | let buffer_index = bp2.fetch_page(1); 459 | assert_eq!(bp2.buffers[buffer_index].read_record(14), 460 | (&bark[..], &krab[..])); 461 | 462 | fs::remove_file("/tmp/dbfile_tests").ok(); 463 | } 464 | } 465 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::collections::hash_map::DefaultHasher; 2 | use std::hash::{Hash, Hasher}; 3 | use std::path::Path; 4 | 5 | // TODO: implement remove 6 | 7 | pub mod util; 8 | pub mod page; 9 | pub mod disk; 10 | 11 | use disk::{DbFile,SearchResult}; 12 | 13 | /// Linear Hashtable 14 | pub struct LinHash { 15 | buckets: DbFile, 16 | nbits: usize, // no of bits used from hash 17 | nitems: usize, // number of items in hashtable 18 | nbuckets: usize, // number of buckets 19 | } 20 | 21 | impl LinHash { 22 | /// "load factor" needed before the hashmap needs to grow. 23 | const THRESHOLD: f32 = 0.8; 24 | 25 | /// Creates a new Linear Hashtable. 26 | pub fn open(filename: &str, keysize: usize, valsize: usize) -> LinHash { 27 | let file_exists = Path::new(filename).exists(); 28 | let mut dbfile = DbFile::new(filename, keysize, valsize); 29 | let (nbits, nitems, nbuckets) = 30 | if file_exists { 31 | dbfile.read_ctrlpage() 32 | } else { 33 | (1, 0, 2) 34 | }; 35 | println!("{:?}", (nbits, nitems, nbuckets)); 36 | LinHash { 37 | buckets: dbfile, 38 | nbits: nbits, 39 | nitems: nitems, 40 | nbuckets: nbuckets, 41 | } 42 | } 43 | 44 | fn hash(&self, key: &[u8]) -> u64 { 45 | let mut s = DefaultHasher::new(); 46 | key.hash(&mut s); 47 | s.finish() 48 | } 49 | 50 | /// Which bucket to place the key-value pair in. If the target 51 | /// bucket does not yet exist, it is guaranteed that the MSB is a 52 | /// `1`. To find the bucket, the pair should be placed in, 53 | /// subtract this `1`. 54 | fn bucket(&self, key: &[u8]) -> usize { 55 | let hash = self.hash(key); 56 | let bucket = (hash & ((1 << self.nbits) - 1)) as usize; 57 | let adjusted_bucket_index = 58 | if bucket < self.nbuckets { 59 | bucket 60 | } else { 61 | bucket - (1 << (self.nbits-1)) 62 | }; 63 | 64 | adjusted_bucket_index 65 | } 66 | 67 | /// Returns true if the `load` exceeds `LinHash::THRESHOLD` 68 | fn split_needed(&self) -> bool { 69 | (self.nitems as f32 / (self.buckets.records_per_page * self.nbuckets) as f32) > 70 | LinHash::THRESHOLD 71 | } 72 | 73 | /// If necessary, allocates new bucket. If there's no more space 74 | /// in the buckets vector(ie. n > 2^i), increment number of bits 75 | /// used(i). 76 | 77 | /// Note that, the bucket split is not necessarily the one just 78 | /// inserted to. 79 | fn maybe_split(&mut self) -> bool { 80 | if self.split_needed() { 81 | self.nbuckets += 1; 82 | 83 | self.buckets.allocate_new_bucket(); 84 | if self.nbuckets > (1 << self.nbits) { 85 | self.nbits += 1; 86 | } 87 | 88 | // Take index of last item added and subtract the 1 at the 89 | // MSB position. eg: after bucket 11 is added, bucket 01 90 | // needs to be split 91 | let bucket_to_split = 92 | (self.nbuckets-1) ^ (1 << (self.nbits-1)); 93 | println!("nbits: {} nitems: {} nbuckets: {} splitting {} and {}", 94 | self.nbits, self.nitems, self.nbuckets, bucket_to_split, (self.nbuckets-1)); 95 | // Replace the bucket to split with a fresh, empty 96 | // page. And get a list of all records stored in the bucket 97 | let old_bucket_records = 98 | self.buckets.clear_bucket(bucket_to_split); 99 | 100 | // Re-hash all records in old_bucket. Ideally, about half 101 | // of the records will go into the new bucket. 102 | for (k, v) in old_bucket_records.into_iter() { 103 | self.reinsert(&k, &v); 104 | } 105 | return true 106 | } 107 | 108 | false 109 | } 110 | 111 | /// Does the hashmap contain a record with key `key`? 112 | pub fn contains(&mut self, key: &[u8]) -> bool { 113 | match self.get(key) { 114 | Some(_) => true, 115 | None => false, 116 | } 117 | } 118 | 119 | /// Update the mapping of record with key `key`. 120 | pub fn update(&mut self, key: &[u8], val: &[u8]) -> bool { 121 | let bucket_index = self.bucket(&key); 122 | match self.buckets.search_bucket(bucket_index, key.clone()) { 123 | SearchResult { page_id, row_num, val: old_val } => { 124 | match (page_id, row_num, old_val) { 125 | (Some(page_id), Some(row_num), Some(_)) => { 126 | println!("update: {:?}", (page_id, row_num, key.clone(), val.clone())); 127 | self.buckets.write_record(page_id, row_num, key, val); 128 | true 129 | } 130 | _ => false, 131 | } 132 | }, 133 | } 134 | } 135 | 136 | /// Insert (key,value) pair into the hashtable. 137 | pub fn put(&mut self, key: &[u8], val: &[u8]) { 138 | let bucket_index = self.bucket(&key); 139 | match self.buckets.search_bucket(bucket_index, key.clone()) { 140 | SearchResult { page_id, row_num, val: old_val } => { 141 | match (page_id, row_num, old_val) { 142 | // new insert 143 | (Some(page_id), Some(pos), None) => { 144 | self.buckets.write_record_incr(page_id, pos, key, val); 145 | self.nitems += 1; 146 | }, 147 | // case for update 148 | (Some(_page_id), Some(pos), Some(_old_val)) => { 149 | panic!("can't use put to reinsert old item: {:?}", (key, val)); 150 | }, 151 | // new insert, in overflow page 152 | (Some(last_page_id), None, None) => { // overflow 153 | self.buckets.allocate_overflow(bucket_index, last_page_id); 154 | self.put(key, val); 155 | }, 156 | _ => panic!("impossible case"), 157 | } 158 | }, 159 | } 160 | 161 | self.maybe_split(); 162 | self.buckets.write_ctrlpage((self.nbits, self.nitems, self.nbuckets)); 163 | } 164 | 165 | /// Re-insert (key, value) pair after a split 166 | fn reinsert(&mut self, key: &[u8], val: &[u8]) { 167 | self.put(key, val); 168 | // correct for nitems increment in `put` 169 | self.nitems -= 1; 170 | } 171 | 172 | /// Lookup `key` in hashtable 173 | pub fn get(&mut self, key: &[u8]) -> Option> { 174 | let bucket_index = self.bucket(&key); 175 | match self.buckets.search_bucket(bucket_index, key) { 176 | SearchResult { page_id, row_num, val } => { 177 | match val { 178 | Some(v) => Some(v), 179 | _ => None, 180 | } 181 | }, 182 | } 183 | } 184 | 185 | // Removes record with `key` in hashtable. 186 | // pub fn remove(&mut self, key: K) -> Option { 187 | // let bucket_index = self.bucket(&key); 188 | // let index_to_delete = self.search_bucket(bucket_index, &key); 189 | 190 | // // Delete item from bucket 191 | // match index_to_delete { 192 | // Some(x) => Some(self.buckets[bucket_index].remove(x).1), 193 | // None => None, 194 | // } 195 | // } 196 | 197 | pub fn close(&mut self) { 198 | self.buckets.write_ctrlpage((self.nbits, self.nitems, self.nbuckets)); 199 | self.buckets.close(); 200 | } 201 | } 202 | 203 | #[cfg(test)] 204 | mod tests { 205 | use LinHash; 206 | use std::fs; 207 | use util::*; 208 | 209 | #[test] 210 | fn all_ops() { 211 | let mut h = LinHash::open("/tmp/test_all_ops", 32, 4); 212 | h.put(b"hello", &[12]); 213 | h.put(b"there", &[13]); 214 | h.put(b"foo", &[42]); 215 | h.put(b"bar", &[11]); 216 | h.update(b"bar", &[22]); 217 | h.update(b"foo", &[84]); 218 | 219 | assert_eq!(h.get(b"hello"), Some(vec![12, 0, 0, 0])); 220 | assert_eq!(h.get(b"there"), Some(vec![13, 0, 0, 0])); 221 | assert_eq!(h.get(b"foo"), Some(vec![84, 0, 0, 0])); 222 | assert_eq!(h.get(b"bar"), Some(vec![22, 0, 0, 0])); 223 | 224 | // assert_eq!(h.update(String::from("doesn't exist"), 99), false); 225 | assert_eq!(h.contains(b"doesn't exist"), false); 226 | assert_eq!(h.contains(b"hello"), true); 227 | 228 | h.close(); 229 | fs::remove_file("/tmp/test_all_ops").ok(); 230 | } 231 | 232 | #[test] 233 | fn test_persistence() { 234 | let mut h = LinHash::open("/tmp/test_persistence", 32, 4); 235 | h.put(b"hello", &[12]); 236 | h.put(b"world", &[13]); 237 | h.put(b"linear", &[144]); 238 | h.put(b"hashing", &[255]); 239 | h.close(); 240 | 241 | // This reloads the file and creates a new hashtable 242 | let mut h2 = LinHash::open("/tmp/test_persistence", 32, 4); 243 | assert_eq!(h2.get(b"hello"), Some(vec![12, 0, 0, 0])); 244 | 245 | h2.close(); 246 | fs::remove_file("/tmp/test_persistence").ok(); 247 | } 248 | 249 | // TODO: figure out a better testing strategy for this. This test 250 | // currently inserts 10,000 records and checks that they are all 251 | // there. 252 | #[test] 253 | fn test_overflow_and_splitting() { 254 | let mut h = LinHash::open("/tmp/test_overflow_and_splitting", 4, 4); 255 | for k in 0..10000 { 256 | h.put(&i32_to_bytearray(k), 257 | &i32_to_bytearray(k+1)); 258 | } 259 | h.close(); 260 | 261 | let mut h2 = LinHash::open("/tmp/test_overflow_and_splitting", 4, 4); 262 | for k in 0..10000 { 263 | assert_eq!(h2.get(&i32_to_bytearray(k)), 264 | Some(i32_to_bytearray(k+1).to_vec())); 265 | } 266 | 267 | fs::remove_file("/tmp/test_overflow_and_splitting").ok(); 268 | } 269 | } 270 | -------------------------------------------------------------------------------- /src/page.rs: -------------------------------------------------------------------------------- 1 | use util::*; 2 | 3 | pub const PAGE_SIZE : usize = 4096; // bytes 4 | pub const HEADER_SIZE : usize = 16; // bytes 5 | 6 | pub struct Page { 7 | pub id: usize, 8 | pub storage: [u8; PAGE_SIZE], 9 | pub num_records: usize, 10 | // page_id of overflow bucket 11 | pub next: Option, 12 | pub dirty: bool, 13 | 14 | keysize: usize, 15 | valsize: usize, 16 | } 17 | 18 | // Row layout: 19 | // | key | val | 20 | #[derive(Debug)] 21 | struct RowOffsets { 22 | key_offset: usize, 23 | val_offset: usize, 24 | row_end: usize, 25 | } 26 | 27 | impl Page { 28 | pub fn new(keysize: usize, valsize: usize) -> Page { 29 | Page { 30 | id: 0, 31 | num_records: 0, 32 | storage: [0; PAGE_SIZE], 33 | next: None, 34 | keysize: keysize, 35 | valsize: valsize, 36 | dirty: false, 37 | } 38 | } 39 | 40 | /// Compute where in the page the row should be placed. Within the 41 | /// row, calculate the offsets of the header, key and value. 42 | fn compute_offsets(&self, row_num: usize) -> RowOffsets { 43 | let total_size = self.keysize + self.valsize; 44 | 45 | let row_offset = HEADER_SIZE + (row_num * total_size); 46 | let key_offset = row_offset; 47 | let val_offset = key_offset + self.keysize; 48 | let row_end = val_offset + self.valsize; 49 | 50 | RowOffsets { 51 | key_offset: key_offset, 52 | val_offset: val_offset, 53 | row_end: row_end, 54 | } 55 | } 56 | 57 | 58 | pub fn read_header(&mut self) { 59 | let num_records : usize = bytearray_to_usize(self.storage[0..8].to_vec()); 60 | let next : usize = bytearray_to_usize(self.storage[8..16].to_vec()); 61 | self.num_records = num_records; 62 | self.next = if next != 0 { 63 | Some(next) 64 | } else { 65 | None 66 | }; 67 | } 68 | 69 | pub fn write_header(&mut self) { 70 | mem_move(&mut self.storage[0..8], &usize_to_bytearray(self.num_records)); 71 | mem_move(&mut self.storage[8..16], &usize_to_bytearray(self.next.unwrap_or(0))); 72 | } 73 | 74 | pub fn read_record(&mut self, row_num: usize) -> (&[u8], &[u8]) { 75 | let offsets = self.compute_offsets(row_num); 76 | let key = &self.storage[offsets.key_offset..offsets.val_offset]; 77 | let val = &self.storage[offsets.val_offset..offsets.row_end]; 78 | (key, val) 79 | } 80 | 81 | /// Write record to offset specified by `row_num`. The offset is 82 | /// calculated to accomodate header as well. 83 | pub fn write_record(&mut self, row_num: usize, key: &[u8], val: &[u8]) { 84 | let offsets = self.compute_offsets(row_num); 85 | mem_move(&mut self.storage[offsets.key_offset..offsets.val_offset], 86 | key); 87 | mem_move(&mut self.storage[offsets.val_offset..offsets.row_end], 88 | val); 89 | } 90 | 91 | /// Increment number of records in page 92 | pub fn incr_num_records(&mut self) { 93 | self.num_records += 1; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/util.rs: -------------------------------------------------------------------------------- 1 | use std::mem::transmute; 2 | 3 | pub fn mem_move(dest: &mut [u8], src: &[u8]) { 4 | for (d, s) in dest.iter_mut().zip(src) { 5 | *d = *s 6 | } 7 | } 8 | 9 | pub fn usize_to_bytearray(n: usize) -> [u8; 8] { 10 | unsafe { 11 | transmute::(n) 12 | } 13 | } 14 | 15 | pub fn i32_to_bytearray(n: i32) -> [u8; 4] { 16 | unsafe { 17 | transmute::(n) 18 | } 19 | } 20 | 21 | pub fn usize_vec_to_bytevec(v: Vec) -> Vec { 22 | let mut bv : Vec = vec![]; 23 | for i in v { 24 | bv.append(&mut usize_to_bytearray(i).to_vec()); 25 | } 26 | bv 27 | } 28 | 29 | pub fn bytevec_to_usize_vec(b: Vec) -> Vec { 30 | let mut v = vec![]; 31 | for i in 0..(b.len() / 8) { 32 | v.push(bytearray_to_usize(b[i*8..(i+1)*8].to_vec())); 33 | } 34 | 35 | v 36 | } 37 | 38 | pub fn bytearray_to_usize(b: Vec) -> usize { 39 | assert_eq!(b.len(), 8); 40 | let mut a = [0; 8]; 41 | 42 | for i in 0..b.len() { 43 | a[i] = b[i]; 44 | } 45 | 46 | unsafe { 47 | transmute::<[u8;8], usize>(a) 48 | } 49 | } 50 | 51 | pub fn slices_eq(s1: &[T], s2: &[T]) -> bool { 52 | s1.iter().zip(s2).all(|(a,b)| a == b) 53 | } 54 | --------------------------------------------------------------------------------