├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md └── src ├── batch.rs ├── compare ├── bytes_compare.rs ├── compare.rs └── mod.rs ├── error.rs ├── errors ├── errors.rs └── mod.rs ├── icompare.rs ├── key.rs ├── lib.rs ├── main.rs ├── memdb ├── memdb.rs ├── memdb_node.rs └── mod.rs ├── storage ├── mod.rs └── storage.rs └── table.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .DS_Store -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "bytes" 7 | version = "1.1.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" 10 | 11 | [[package]] 12 | name = "cfg-if" 13 | version = "1.0.0" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 16 | 17 | [[package]] 18 | name = "getrandom" 19 | version = "0.2.3" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" 22 | dependencies = [ 23 | "cfg-if", 24 | "libc", 25 | "wasi", 26 | ] 27 | 28 | [[package]] 29 | name = "leveldb-rs" 30 | version = "0.1.0" 31 | dependencies = [ 32 | "bytes", 33 | "rand", 34 | "thiserror", 35 | ] 36 | 37 | [[package]] 38 | name = "libc" 39 | version = "0.2.104" 40 | source = "registry+https://github.com/rust-lang/crates.io-index" 41 | checksum = "7b2f96d100e1cf1929e7719b7edb3b90ab5298072638fccd77be9ce942ecdfce" 42 | 43 | [[package]] 44 | name = "ppv-lite86" 45 | version = "0.2.14" 46 | source = "registry+https://github.com/rust-lang/crates.io-index" 47 | checksum = "c3ca011bd0129ff4ae15cd04c4eef202cadf6c51c21e47aba319b4e0501db741" 48 | 49 | [[package]] 50 | name = "proc-macro2" 51 | version = "1.0.30" 52 | source = "registry+https://github.com/rust-lang/crates.io-index" 53 | checksum = "edc3358ebc67bc8b7fa0c007f945b0b18226f78437d61bec735a9eb96b61ee70" 54 | dependencies = [ 55 | "unicode-xid", 56 | ] 57 | 58 | [[package]] 59 | name = "quote" 60 | version = "1.0.10" 61 | source = "registry+https://github.com/rust-lang/crates.io-index" 62 | checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" 63 | dependencies = [ 64 | "proc-macro2", 65 | ] 66 | 67 | [[package]] 68 | name = "rand" 69 | version = "0.8.4" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" 72 | dependencies = [ 73 | "libc", 74 | "rand_chacha", 75 | "rand_core", 76 | "rand_hc", 77 | ] 78 | 79 | [[package]] 80 | name = "rand_chacha" 81 | version = "0.3.1" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 84 | dependencies = [ 85 | "ppv-lite86", 86 | "rand_core", 87 | ] 88 | 89 | [[package]] 90 | name = "rand_core" 91 | version = "0.6.3" 92 | source = "registry+https://github.com/rust-lang/crates.io-index" 93 | checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" 94 | dependencies = [ 95 | "getrandom", 96 | ] 97 | 98 | [[package]] 99 | name = "rand_hc" 100 | version = "0.3.1" 101 | source = "registry+https://github.com/rust-lang/crates.io-index" 102 | checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" 103 | dependencies = [ 104 | "rand_core", 105 | ] 106 | 107 | [[package]] 108 | name = "syn" 109 | version = "1.0.80" 110 | source = "registry+https://github.com/rust-lang/crates.io-index" 111 | checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194" 112 | dependencies = [ 113 | "proc-macro2", 114 | "quote", 115 | "unicode-xid", 116 | ] 117 | 118 | [[package]] 119 | name = "thiserror" 120 | version = "1.0.30" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" 123 | dependencies = [ 124 | "thiserror-impl", 125 | ] 126 | 127 | [[package]] 128 | name = "thiserror-impl" 129 | version = "1.0.30" 130 | source = "registry+https://github.com/rust-lang/crates.io-index" 131 | checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" 132 | dependencies = [ 133 | "proc-macro2", 134 | "quote", 135 | "syn", 136 | ] 137 | 138 | [[package]] 139 | name = "unicode-xid" 140 | version = "0.2.2" 141 | source = "registry+https://github.com/rust-lang/crates.io-index" 142 | checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" 143 | 144 | [[package]] 145 | name = "wasi" 146 | version = "0.10.2+wasi-snapshot-preview1" 147 | source = "registry+https://github.com/rust-lang/crates.io-index" 148 | checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" 149 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "leveldb-rs" 3 | version = "0.1.0" 4 | authors = ["kingeasternsun "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | bytes = "1" 11 | thiserror = "1.0.30" 12 | rand = "0.8.4" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 kingeasternsun 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # leveldb-rs 2 | levelDB key/value database in Rust. 3 | 4 | Inspired by [goleveldb](https://github.com/syndtr/goleveldb) 5 | 6 | THIS PROJECT MAINLY FOR LEARNING -------------------------------------------------------------------------------- /src/batch.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_camel_case_types)] 2 | #![allow(non_snake_case)] 3 | #![allow(dead_code)] 4 | use crate::coding::GetVarint64Ptr; 5 | use crate::key::internalKey; 6 | use crate::key::keyType; 7 | use crate::key::makeInternalKey; 8 | use crate::memdb; 9 | use crate::BytesComparer; 10 | use crate::GenericResult; 11 | use crate::{coding::MAX_VARINT_LEN64, key::keyTypeFromU8}; 12 | use bytes::{Buf, BufMut, BytesMut}; 13 | use std::io::Write; 14 | use std::{io::Cursor, option::Option}; 15 | 16 | // ErrBatchCorrupted records reason of batch corruption. This error will be 17 | // wrapped with errors.ErrCorrupted. 18 | // type ErrBatchCorrupted struct { 19 | // Reason string 20 | // } 21 | 22 | // func (e *ErrBatchCorrupted) Error() string { 23 | // return fmt.Sprintf("leveldb: batch corrupted: %s", e.Reason) 24 | // } 25 | 26 | // func newErrBatchCorrupted(reason string) error { 27 | // return errors.NewErrCorrupted(storage.FileDesc{}, &ErrBatchCorrupted{reason}) 28 | // } 29 | use thiserror::Error; 30 | 31 | #[derive(Error, Debug)] 32 | pub enum BatchError { 33 | #[error("decode varint error")] 34 | InvalidVarint, 35 | #[error("leveldb: batch corrupted:`{0}`")] 36 | BatchCorrupted(String), 37 | #[error("invalid key len (expected {expected:?}, found {found:?})")] 38 | InvalidKeyLen { expected: usize, found: usize }, 39 | #[error("invalid value len (expected {expected:?}, found {found:?})")] 40 | InvalidValueLen { expected: usize, found: usize }, 41 | #[error("invalid header (expected {expected:?}, found {found:?})")] 42 | InvalidHeader { expected: String, found: String }, 43 | #[error("invalid indexlen (expected {expected:?}, found {found:?})")] 44 | InvalidIndexLen { expected: usize, found: usize }, 45 | #[error("invalid seq number (expected {expected:?}, found {found:?})")] 46 | InvalidSeqNum { expected: u64, found: u64 }, 47 | #[error("invalid batch number (expected {expected:?}, found {found:?})")] 48 | InvalidBatchNum { expected: u32, found: u32 }, 49 | #[error("unknown data store error")] 50 | Unknown, 51 | } 52 | 53 | const BATCH_HEADER_LEN: u32 = 8 + 4; 54 | const BATCH_GROW_REC: u32 = 3000; 55 | const BATCH_BUFIO_SIZE: u32 = 16; 56 | 57 | pub trait BatchReplay { 58 | fn Put(&mut self, key: &[u8], value: Option<&[u8]>); 59 | fn Delete(&mut self, key: &[u8]); 60 | } 61 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] 62 | pub struct batchIndex { 63 | key_type: keyType, 64 | Key: (usize, usize), // 指向key所在的起始位置和长度 65 | Value: (usize, usize), // 指向Value所在的起始位置和长度 66 | } 67 | 68 | impl batchIndex { 69 | fn new(kt: keyType) -> Self { 70 | Self { 71 | key_type: kt, 72 | Key: (0, 0), 73 | Value: (0, 0), 74 | } 75 | } 76 | // 返回data中的key, 注意要使用lifetime 77 | fn k<'a>(&self, data: &'a [u8]) -> &'a [u8] { 78 | return &data[self.Key.0..self.Key.1]; 79 | } 80 | 81 | // 返回data中的 value 82 | fn v<'a>(&self, data: &'a [u8]) -> Option<&'a [u8]> { 83 | if self.Value.1 != 0 { 84 | Some(&data[self.Value.0..self.Value.1]) 85 | } else { 86 | None 87 | } 88 | } 89 | 90 | // 返回 data中的key 和 value 91 | fn kv<'a>(&self, data: &'a [u8]) -> (&'a [u8], Option<&'a [u8]>) { 92 | return (self.k(data), self.v(data)); 93 | } 94 | } 95 | 96 | pub struct Batch { 97 | data: BytesMut, 98 | index: Vec, 99 | internalLen: usize, 100 | } 101 | 102 | impl Batch { 103 | fn new() -> Self { 104 | Batch { 105 | data: BytesMut::new(), 106 | index: Vec::new(), 107 | internalLen: 0, 108 | } 109 | } 110 | fn encode_varint64(&mut self, v: u64) { 111 | let B: u8 = 128; 112 | let mut v = v; 113 | 114 | while v > B as u64 { 115 | self.data.put_u8(v as u8 | B); 116 | v = v >> 7; 117 | } 118 | 119 | self.data.put_u8(v as u8); 120 | } 121 | 122 | //func (b *Batch) appendRec(kt keyType, key, value []byte) { 123 | // n := 1 + binary.MaxVarintLen32 + len(key) 124 | // if kt == keyTypeVal { 125 | // n += binary.MaxVarintLen32 + len(value) 126 | // } 127 | // b.grow(n) 128 | // index := batchIndex{keyType: kt} 129 | // o := len(b.data) 130 | // data := b.data[:o+n] 131 | // data[o] = byte(kt) 132 | // o++ 133 | // o += binary.PutUvarint(data[o:], uint64(len(key))) 134 | // index.keyPos = o 135 | // index.keyLen = len(key) 136 | // o += copy(data[o:], key) 137 | // if kt == keyTypeVal { 138 | // o += binary.PutUvarint(data[o:], uint64(len(value))) 139 | // index.valuePos = o 140 | // index.valueLen = len(value) 141 | // o += copy(data[o:], value) 142 | // } 143 | // b.data = data[:o] 144 | // b.index = append(b.index, index) 145 | // b.internalLen += index.keyLen + index.valueLen + 8 146 | // } 147 | pub fn append_rec(&mut self, kt: keyType, key: &[u8], value: &[u8]) { 148 | self.data.reserve(1 + MAX_VARINT_LEN64 + key.len()); 149 | if kt == keyType::Val { 150 | self.data.reserve(MAX_VARINT_LEN64 + value.len()); 151 | } 152 | 153 | let mut index = batchIndex::new(kt); 154 | // 写入 keyType 155 | self.data.put_u8(kt as u8); 156 | // 写入 key的长度 157 | 158 | // 如果使用下面这种，数据的写入没有办法反馈到外层self.data 中 159 | // let v_len = EncodeVarint64(&mut self.data[o + 1..], key.len() as u64); 160 | // 或者只有定义一个额外的Vec，然后传入到 EncodeVarint64，再使用 put_slice将数据写入self.data 多了一层数据拷贝 161 | // 不如直接对self.data 进行写入 162 | self.encode_varint64(key.len() as u64); 163 | 164 | // 这时候，内部指针所指向的位置 ，就是key要写入的起始位置，更新 index.Key 165 | index.Key = (self.data.len(), key.len()); 166 | // 写入 key 167 | self.data.put_slice(key); 168 | 169 | if kt == keyType::Val { 170 | // 写入 value的长度 171 | self.encode_varint64(value.len() as u64); 172 | // 这时候，内部指针指向的位置，就是value要写入的起始位置，更新index.Value 173 | index.Value = (self.data.len(), value.len()); 174 | // 写入value 175 | self.data.put_slice(value); 176 | } 177 | self.internalLen += index.Key.1 + index.Value.1 + 8; 178 | self.index.push(index); 179 | } 180 | 181 | // Put appends 'put operation' of the given key/value pair to the batch. 182 | // It is safe to modify the contents of the argument after Put returns but not 183 | // before. 184 | pub fn put(&mut self, key: &[u8], value: &[u8]) { 185 | self.append_rec(keyType::Val, key, value) 186 | } 187 | 188 | // Delete appends 'delete operation' of the given key to the batch. 189 | // It is safe to modify the contents of the argument after Delete returns but 190 | // not before. 191 | pub fn delete(&mut self, key: &[u8]) { 192 | self.append_rec(keyType::Del, key, &vec![]) 193 | } 194 | 195 | // Dump dumps batch contents. The returned slice can be loaded into the 196 | // batch using Load method. 197 | // The returned slice is not its own copy, so the contents should not be 198 | // modified. 199 | pub fn dump(&self) -> &[u8] { 200 | &self.data[..] 201 | } 202 | // Load loads given slice into the batch. Previous contents of the batch 203 | // will be discarded. 204 | // The given slice will not be copied and will be used as batch buffer, so 205 | // it is not safe to modify the contents of the slice. 206 | // pub fn load(&mut self, buf: &[u8]) -> Option<()> { 207 | // self.decode(buf, 0) 208 | // } 209 | 210 | // Replay replays batch contents. 211 | pub fn replay(&self, r: &mut T) 212 | where 213 | T: BatchReplay, 214 | { 215 | for index in &self.index { 216 | match index.key_type { 217 | keyType::Val => r.Put(index.k(&self.data[..]), index.v(&self.data[..])), 218 | keyType::Del => r.Delete(index.k(&self.data[..])), 219 | keyType::Seek => {} 220 | } 221 | } 222 | } 223 | 224 | // Len returns number of records in the batch. 225 | pub fn len(&self) -> usize { 226 | self.index.len() 227 | } 228 | 229 | // Reset resets the batch. 230 | pub fn reset(&mut self) { 231 | self.data.clear(); 232 | self.index.clear(); 233 | self.internalLen = 0; 234 | } 235 | 236 | // func (b *Batch) replayInternal(fn func(i int, kt keyType, k, v []byte) error) error { 237 | // for i, index := range b.index { 238 | // if err := fn(i, index.keyType, index.k(b.data), index.v(b.data)); err != nil { 239 | // return err 240 | // } 241 | // } 242 | // return nil 243 | // } 244 | 245 | pub fn replay_internal(&self, f: F) -> GenericResult<()> 246 | where 247 | F: Fn(usize, keyType, &[u8], Option<&[u8]>) -> GenericResult, 248 | { 249 | for (i, index) in self.index.iter().enumerate() { 250 | f( 251 | i, 252 | index.key_type, 253 | index.k(&self.data[..]), 254 | index.v(&self.data[..]), 255 | )?; 256 | } 257 | Ok(()) 258 | } 259 | 260 | // func (b *Batch) append(p *Batch) { 261 | // ob := len(b.data) 262 | // oi := len(b.index) 263 | // b.data = append(b.data, p.data...) 264 | // b.index = append(b.index, p.index...) 265 | // b.internalLen += p.internalLen 266 | 267 | // // Updating index offset. 268 | // if ob != 0 { 269 | // for ; oi < len(b.index); oi++ { 270 | // index := &b.index[oi] 271 | // index.keyPos += ob 272 | // if index.valueLen != 0 { 273 | // index.valuePos += ob 274 | // } 275 | // } 276 | // } 277 | // } 278 | 279 | pub fn append(&mut self, p: &Self) { 280 | let ob = self.data.len(); 281 | let oi = self.index.len(); 282 | self.data.extend_from_slice(&p.data[..]); 283 | self.index.append(&mut p.index.clone()); 284 | self.internalLen += p.internalLen; 285 | if ob != 0 { 286 | for index in self.index.iter_mut().skip(ob) { 287 | index.Key.0 += ob; 288 | if index.Value.1 != 0 { 289 | index.Value.0 += ob; 290 | } 291 | } 292 | } 293 | } 294 | 295 | // func (b *Batch) decode(data []byte, expectedLen int) error { 296 | // b.data = data 297 | // b.index = b.index[:0] 298 | // b.internalLen = 0 299 | // err := decodeBatch(data, func(i int, index batchIndex) error { 300 | // b.index = append(b.index, index) 301 | // b.internalLen += index.keyLen + index.valueLen + 8 302 | // return nil 303 | // }) 304 | // if err != nil { 305 | // return err 306 | // } 307 | // if expectedLen >= 0 && len(b.index) != expectedLen { 308 | // return newErrBatchCorrupted(fmt.Sprintf("invalid records length: %d vs %d", expectedLen, len(b.index))) 309 | // } 310 | // return nil 311 | // } 312 | pub fn decode(&mut self, buf: &[u8], expectedLen: usize) -> GenericResult<()> { 313 | self.index.clear(); 314 | self.internalLen = 0; 315 | 316 | decode_batch(buf, |_i, index| { 317 | self.internalLen += index.Key.1 + index.Value.1 + 8; 318 | self.index.push(index); 319 | Ok(()) 320 | })?; 321 | 322 | if expectedLen > 0 && self.index.len() != expectedLen { 323 | return Err(Box::new(BatchError::InvalidIndexLen { 324 | expected: expectedLen, 325 | found: self.index.len(), 326 | })); 327 | } 328 | 329 | Ok(()) 330 | } 331 | 332 | // func (b *Batch) putMem(seq uint64, mdb *memdb.DB) error { 333 | // var ik []byte 334 | // for i, index := range b.index { 335 | // ik = makeInternalKey(ik, index.k(b.data), seq+uint64(i), index.keyType) 336 | // if err := mdb.Put(ik, index.v(b.data)); err != nil { 337 | // return err 338 | // } 339 | // } 340 | // return nil 341 | // } 342 | 343 | // 将数据放入到memdb，内存数据库中 344 | fn put_mem(&mut self, seq: u64, mdb: &mut memdb::Db) -> GenericResult<()> { 345 | let mut ik; 346 | for (i, index) in self.index.iter().enumerate() { 347 | ik = makeInternalKey(index.k(&self.data[..]), seq + i as u64, index.key_type); 348 | mdb.put(&ik, index.v(&self.data[..]))?; 349 | } 350 | Ok(()) 351 | } 352 | // func (b *Batch) revertMem(seq uint64, mdb *memdb.DB) error { 353 | // var ik []byte 354 | // for i, index := range b.index { 355 | // ik = makeInternalKey(ik, index.k(b.data), seq+uint64(i), index.keyType) 356 | // if err := mdb.Delete(ik); err != nil { 357 | // return err 358 | // } 359 | // } 360 | // return nil 361 | // } 362 | fn revert_mem(&mut self, seq: u64, mdb: &mut memdb::Db) -> GenericResult<()> { 363 | let mut ik; 364 | for (i, index) in self.index.iter().enumerate() { 365 | ik = makeInternalKey(index.k(&self.data[..]), seq + i as u64, index.key_type); 366 | mdb.delete(&ik); 367 | } 368 | Ok(()) 369 | } 370 | } 371 | // func decodeBatch(data []byte, fn func(i int, index batchIndex) error) error { 372 | // var index batchIndex 373 | // for i, o := 0, 0; o < len(data); i++ { 374 | // // Key type. 375 | // index.keyType = keyType(data[o]) 376 | // if index.keyType > keyTypeVal { 377 | // return newErrBatchCorrupted(fmt.Sprintf("bad record: invalid type %#x", uint(index.keyType))) 378 | // } 379 | // o++ 380 | 381 | // // Key. 382 | // x, n := binary.Uvarint(data[o:]) 383 | // o += n 384 | // if n <= 0 || o+int(x) > len(data) { 385 | // return newErrBatchCorrupted("bad record: invalid key length") 386 | // } 387 | // index.keyPos = o 388 | // index.keyLen = int(x) 389 | // o += index.keyLen 390 | 391 | // // Value. 392 | // if index.keyType == keyTypeVal { 393 | // x, n = binary.Uvarint(data[o:]) 394 | // o += n 395 | // if n <= 0 || o+int(x) > len(data) { 396 | // return newErrBatchCorrupted("bad record: invalid value length") 397 | // } 398 | // index.valuePos = o 399 | // index.valueLen = int(x) 400 | // o += index.valueLen 401 | // } else { 402 | // index.valuePos = 0 403 | // index.valueLen = 0 404 | // } 405 | 406 | // if err := fn(i, index); err != nil { 407 | // return err 408 | // } 409 | // } 410 | // return nil 411 | // } 412 | /// 注意这里的buf的参数定义 413 | pub fn decode_batch(buf: &[u8], mut f: F) -> GenericResult<()> 414 | where 415 | F: FnMut(usize, batchIndex) -> GenericResult<()>, 416 | { 417 | let mut buf = Cursor::new(buf); 418 | let mut index = batchIndex::new(keyType::Del); 419 | let mut cnt = 0; 420 | loop { 421 | cnt += 1; 422 | if !buf.has_remaining() { 423 | return Ok(()); 424 | } 425 | 426 | if buf.remaining() < 2 { 427 | return Err(Box::new(BatchError::BatchCorrupted(String::from( 428 | "bad record: invalid key length", 429 | )))); 430 | } 431 | 432 | // 读取key类型 433 | index.key_type = keyTypeFromU8(buf.get_u8())?; 434 | // 读取key的长度 435 | index.Key.1 = GetVarint64Ptr(&mut buf).ok_or(BatchError::InvalidVarint)? as usize; 436 | if buf.remaining() < index.Key.1 { 437 | return Err(Box::new(BatchError::InvalidKeyLen { 438 | expected: index.Key.1, 439 | found: buf.remaining(), 440 | })); 441 | } 442 | // 此刻 cursor所在的位置就是key的数据起始位置 443 | index.Key.0 = buf.position() as usize; 444 | 445 | // 跳过key的数据 446 | buf.advance(index.Key.1); 447 | 448 | if index.key_type == keyType::Val { 449 | if buf.remaining() < 2 { 450 | return Err(Box::new(BatchError::BatchCorrupted(String::from( 451 | "bad record: invalid key length", 452 | )))); 453 | } 454 | 455 | // 读取value的长度 456 | index.Value.1 = GetVarint64Ptr(&mut buf).ok_or(BatchError::InvalidVarint)? as usize; 457 | if buf.remaining() < index.Value.1 { 458 | return Err(Box::new(BatchError::InvalidKeyLen { 459 | expected: index.Value.1, 460 | found: buf.remaining(), 461 | })); 462 | } 463 | // 此刻 cursor所在的位置就是value的起始位置 464 | index.Value.0 = buf.position() as usize; 465 | 466 | // 跳过value的数据 467 | buf.advance(index.Value.1); 468 | } else { 469 | index.Value = (0, 0); 470 | } 471 | 472 | // 执行闭包函数 473 | f(cnt, index)?; 474 | } 475 | } 476 | 477 | // func decodeBatchToMem(data []byte, expectSeq uint64, mdb *memdb.DB) (seq uint64, batchLen int, err error) { 478 | // seq, batchLen, err = decodeBatchHeader(data) 479 | // if err != nil { 480 | // return 0, 0, err 481 | // } 482 | // if seq < expectSeq { 483 | // return 0, 0, newErrBatchCorrupted("invalid sequence number") 484 | // } 485 | // data = data[batchHeaderLen:] 486 | // var ik []byte 487 | // var decodedLen int 488 | // err = decodeBatch(data, func(i int, index batchIndex) error { 489 | // if i >= batchLen { 490 | // return newErrBatchCorrupted("invalid records length") 491 | // } 492 | // ik = makeInternalKey(ik, index.k(data), seq+uint64(i), index.keyType) 493 | // if err := mdb.Put(ik, index.v(data)); err != nil { 494 | // return err 495 | // } 496 | // decodedLen++ 497 | // return nil 498 | // }) 499 | // if err == nil && decodedLen != batchLen { 500 | // err = newErrBatchCorrupted(fmt.Sprintf("invalid records length: %d vs %d", batchLen, decodedLen)) 501 | // } 502 | // return 503 | // } 504 | fn decode_batch_to_mem( 505 | data: &[u8], 506 | expect_seq: u64, 507 | mdb: &mut memdb::Db, 508 | ) -> GenericResult<(u64, u32)> { 509 | let (seq, batch_len) = decode_batch_header(data)?; 510 | if seq < expect_seq { 511 | return Err(Box::new(BatchError::InvalidSeqNum { 512 | expected: expect_seq, 513 | found: seq, 514 | })); 515 | } 516 | 517 | let data = &data[BATCH_HEADER_LEN as usize..]; 518 | let mut decoded_len = 0; 519 | decode_batch(data, |i, index| { 520 | if i as u32 >= batch_len { 521 | return Err(Box::new(BatchError::InvalidBatchNum { 522 | expected: batch_len, 523 | found: i as u32, 524 | })); 525 | } 526 | 527 | let ik = makeInternalKey(index.k(data), seq + i as u64, index.key_type); 528 | mdb.put(&ik, index.v(data))?; 529 | 530 | decoded_len += 1; 531 | Ok(()) 532 | })?; 533 | if decoded_len != batch_len { 534 | return Err(Box::new(BatchError::InvalidBatchNum { 535 | expected: batch_len, 536 | found: decoded_len, 537 | })); 538 | } 539 | 540 | Ok((seq, batch_len)) 541 | } 542 | 543 | // func encodeBatchHeader(dst []byte, seq uint64, batchLen int) []byte { 544 | // dst = ensureBuffer(dst, batchHeaderLen) 545 | // binary.LittleEndian.PutUint64(dst, seq) 546 | // binary.LittleEndian.PutUint32(dst[8:], uint32(batchLen)) 547 | // return dst 548 | // } 549 | fn encocode_batch_header(mut dst: &mut [u8], seq: u64, batch_len: u32) { 550 | dst.put_u64_le(seq); 551 | dst.put_u32_le(batch_len); 552 | } 553 | // func decodeBatchHeader(data []byte) (seq uint64, batchLen int, err error) { 554 | // if len(data) < batchHeaderLen { 555 | // return 0, 0, newErrBatchCorrupted("too short") 556 | // } 557 | 558 | // seq = binary.LittleEndian.Uint64(data) 559 | // batchLen = int(binary.LittleEndian.Uint32(data[8:])) 560 | // if batchLen < 0 { 561 | // return 0, 0, newErrBatchCorrupted("invalid records length") 562 | // } 563 | // return 564 | // } 565 | fn decode_batch_header(mut data: &[u8]) -> GenericResult<(u64, u32)> { 566 | if data.len() < BATCH_HEADER_LEN as usize { 567 | return Err(Box::new(BatchError::BatchCorrupted(String::from( 568 | "too short", 569 | )))); 570 | } 571 | 572 | let mut res = (0, 0); 573 | res.0 = data.get_u64_le(); 574 | res.1 = data.get_u32_le(); 575 | Ok(res) 576 | } 577 | 578 | // func batchesLen(batches []*Batch) int { 579 | // batchLen := 0 580 | // for _, batch := range batches { 581 | // batchLen += batch.Len() 582 | // } 583 | // return batchLen 584 | // } 585 | fn batches_len(batches: &[&Batch]) -> u32 { 586 | let mut batch_len = 0; 587 | for batch in batches { 588 | batch_len += batch.len() 589 | } 590 | batch_len as u32 591 | } 592 | 593 | // func writeBatchesWithHeader(wr io.Writer, batches []*Batch, seq uint64) error { 594 | // if _, err := wr.Write(encodeBatchHeader(nil, seq, batchesLen(batches))); err != nil { 595 | // return err 596 | // } 597 | // for _, batch := range batches { 598 | // if _, err := wr.Write(batch.data); err != nil { 599 | // return err 600 | // } 601 | // } 602 | // return nil 603 | // } 604 | fn write_batches_with_header(wr: &mut T, batches: &[&Batch], seq: u64) -> GenericResult<()> 605 | where 606 | T: Write, 607 | { 608 | let mut buf = vec![0; BATCH_HEADER_LEN as usize]; 609 | encocode_batch_header(&mut buf[..], seq, batches_len(batches)); 610 | wr.write_all(&buf[..])?; 611 | 612 | for batch in batches { 613 | wr.write_all(&batch.data[..])?; 614 | } 615 | Ok(()) 616 | } 617 | 618 | #[cfg(test)] 619 | mod tests { 620 | use super::*; 621 | 622 | #[test] 623 | fn test_name() { 624 | let mut batch = Batch::new(); 625 | batch.append_rec(keyType::Val, &[b'a', b'b'], &[b'k', b'e', b's']); 626 | batch.append_rec(keyType::Val, &[b'a', b'b'], &vec![0b01; 657890]); 627 | // assert_eq!(batch.data.len(),16); 628 | assert_eq!(batch.index.len(), 2); 629 | let buf = batch.data.to_vec(); 630 | let mut batch2 = Batch::new(); 631 | batch2.decode(&buf[..], 2).unwrap(); 632 | } 633 | 634 | #[test] 635 | fn test_encode_batch_header() { 636 | let mut b = vec![0; 12]; 637 | encocode_batch_header(&mut b[..], 1, 2); 638 | encocode_batch_header(&mut b[..], 1, 2); 639 | assert_eq!(b, vec![1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0]) 640 | } 641 | } 642 | -------------------------------------------------------------------------------- /src/compare/bytes_compare.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | use crate::compare::basic_comparer; 3 | use crate::compare::Comparer; 4 | use std::cmp::Ordering; 5 | // type bytesComparer struct{} 6 | pub struct BytesComparer; 7 | 8 | // func (bytesComparer) Compare(a, b []byte) int { 9 | // return bytes.Compare(a, b) 10 | // } 11 | 12 | impl basic_comparer for BytesComparer { 13 | fn compare(&self, a: &[u8], b: &[u8]) -> Ordering { 14 | a.cmp(b) 15 | } 16 | } 17 | 18 | impl Comparer for BytesComparer { 19 | // func (bytesComparer) Name() string { 20 | // return "leveldb.BytewiseComparator" 21 | // } 22 | fn name(&self) -> String { 23 | todo!() 24 | } 25 | 26 | // func (bytesComparer) Separator(dst, a, b []byte) []byte { 27 | // i, n := 0, len(a) 28 | // if n > len(b) { 29 | // n = len(b) 30 | // } 31 | // for ; i < n && a[i] == b[i]; i++ { 32 | // } 33 | // if i >= n { 34 | // // Do not shorten if one string is a prefix of the other 35 | // } else if c := a[i]; c < 0xff && c+1 < b[i] { 36 | // dst = append(dst, a[:i+1]...) 37 | // dst[len(dst)-1]++ 38 | // return dst 39 | // } 40 | // return nil 41 | // } 42 | fn separator<'a>(&self, dst: &'a [u8], a: &'a [u8], b: &'a [u8]) -> &'a [u8] { 43 | todo!() 44 | } 45 | 46 | // func (bytesComparer) Successor(dst, b []byte) []byte { 47 | // for i, c := range b { 48 | // if c != 0xff { 49 | // dst = append(dst, b[:i+1]...) 50 | // dst[len(dst)-1]++ 51 | // return dst 52 | // } 53 | // } 54 | // return nil 55 | // } 56 | fn successor<'a>(&self, dst: &'a [u8], b: &'a [u8]) -> &'a [u8] { 57 | todo!() 58 | } 59 | } 60 | 61 | // // DefaultComparer are default implementation of the Comparer interface. 62 | // // It uses the natural ordering, consistent with bytes.Compare. 63 | // var DefaultComparer = bytesComparer{} 64 | -------------------------------------------------------------------------------- /src/compare/compare.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::Ordering; 2 | // BasicComparer is the interface that wraps the basic Compare method. 3 | // type BasicComparer interface { 4 | // // Compare returns -1, 0, or +1 depending on whether a is 'less than', 5 | // // 'equal to' or 'greater than' b. The two arguments can only be 'equal' 6 | // // if their contents are exactly equal. Furthermore, the empty slice 7 | // // must be 'less than' any non-empty slice. 8 | // Compare(a, b []byte) int 9 | // } 10 | 11 | pub trait basic_comparer { 12 | fn compare(&self, a: &[u8], b: &[u8]) -> Ordering; 13 | } 14 | /* 15 | // Comparer defines a total ordering over the space of []byte keys: a 'less 16 | // than' relationship. 17 | type Comparer interface { 18 | BasicComparer 19 | 20 | // Name returns name of the comparer. 21 | // 22 | // The Level-DB on-disk format stores the comparer name, and opening a 23 | // database with a different comparer from the one it was created with 24 | // will result in an error. 25 | // 26 | // An implementation to a new name whenever the comparer implementation 27 | // changes in a way that will cause the relative ordering of any two keys 28 | // to change. 29 | // 30 | // Names starting with "leveldb." are reserved and should not be used 31 | // by any users of this package. 32 | Name() string 33 | 34 | // Bellow are advanced functions used to reduce the space requirements 35 | // for internal data structures such as index blocks. 36 | 37 | // Separator appends a sequence of bytes x to dst such that a <= x && x < b, 38 | // where 'less than' is consistent with Compare. An implementation should 39 | // return nil if x equal to a. 40 | // 41 | // Either contents of a or b should not by any means modified. Doing so 42 | // may cause corruption on the internal state. 43 | Separator(dst, a, b []byte) []byte 44 | 45 | // Successor appends a sequence of bytes x to dst such that x >= b, where 46 | // 'less than' is consistent with Compare. An implementation should return 47 | // nil if x equal to b. 48 | // 49 | // Contents of b should not by any means modified. Doing so may cause 50 | // corruption on the internal state. 51 | Successor(dst, b []byte) []byte 52 | } 53 | */ 54 | pub trait Comparer: basic_comparer { 55 | fn name(&self) -> String; 56 | fn separator<'a>(&self, dst: &'a [u8], a: &'a [u8], b: &'a [u8]) -> &'a [u8]; 57 | fn successor<'a>(&self, dst: &'a [u8], b: &'a [u8]) -> &'a [u8]; 58 | } 59 | -------------------------------------------------------------------------------- /src/compare/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod compare; 2 | pub use compare::basic_comparer; 3 | pub use compare::Comparer; 4 | 5 | pub mod bytes_compare; 6 | pub use bytes_compare::BytesComparer; 7 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | pub type GenericError = Box; 2 | pub type GenericResult = Result; 3 | #[derive(Debug, Clone)] 4 | pub struct JsonError { 5 | pub message: String, 6 | pub line: usize, 7 | pub column: usize, 8 | } 9 | 10 | use std::fmt; 11 | // Errors should be printable. 12 | impl fmt::Display for JsonError { 13 | fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { 14 | write!(f, "{} ({}:{})", self.message, self.line, self.column) 15 | } 16 | } 17 | // Errors should implement the std::error::Error trait, 18 | // but the default definitions for the Error methods are fine. 19 | impl std::error::Error for JsonError {} 20 | 21 | // use thiserror::Error; 22 | // #[derive(Error, Debug)] 23 | // #[error("{message:} ({line:}, {column})")] 24 | // pub struct JsonError { 25 | // message: String, 26 | // line: usize, 27 | // column: usize, 28 | // } 29 | -------------------------------------------------------------------------------- /src/errors/errors.rs: -------------------------------------------------------------------------------- 1 | pub type GenericError = Box; 2 | pub type GenericResult = Result; 3 | -------------------------------------------------------------------------------- /src/errors/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod errors; 2 | pub use errors::GenericResult; 3 | -------------------------------------------------------------------------------- /src/icompare.rs: -------------------------------------------------------------------------------- 1 | use crate::key::internalKey; 2 | use crate::Comparer; 3 | use std::cmp::Ordering; 4 | 5 | // type iComparer struct { 6 | // ucmp comparer.Comparer 7 | // } 8 | 9 | pub struct IComparer { 10 | ucmp: T, 11 | } 12 | 13 | /* 14 | 15 | func (icmp *iComparer) uCompare(a, b []byte) int { 16 | return icmp.ucmp.Compare(a, b) 17 | } 18 | 19 | func (icmp *iComparer) uSeparator(dst, a, b []byte) []byte { 20 | return icmp.ucmp.Separator(dst, a, b) 21 | } 22 | 23 | func (icmp *iComparer) uSuccessor(dst, b []byte) []byte { 24 | return icmp.ucmp.Successor(dst, b) 25 | } 26 | 27 | func (icmp *iComparer) Name() string { 28 | return icmp.uName() 29 | } 30 | 31 | func (icmp *iComparer) Compare(a, b []byte) int { 32 | x := icmp.uCompare(internalKey(a).ukey(), internalKey(b).ukey()) 33 | if x == 0 { 34 | if m, n := internalKey(a).num(), internalKey(b).num(); m > n { 35 | return -1 36 | } else if m < n { 37 | return 1 38 | } 39 | } 40 | return x 41 | } 42 | 43 | func (icmp *iComparer) Separator(dst, a, b []byte) []byte { 44 | ua, ub := internalKey(a).ukey(), internalKey(b).ukey() 45 | dst = icmp.uSeparator(dst, ua, ub) 46 | if dst != nil && len(dst) < len(ua) && icmp.uCompare(ua, dst) < 0 { 47 | // Append earliest possible number. 48 | return append(dst, keyMaxNumBytes...) 49 | } 50 | return nil 51 | } 52 | 53 | func (icmp *iComparer) Successor(dst, b []byte) []byte { 54 | ub := internalKey(b).ukey() 55 | dst = icmp.uSuccessor(dst, ub) 56 | if dst != nil && len(dst) < len(ub) && icmp.uCompare(ub, dst) < 0 { 57 | // Append earliest possible number. 58 | return append(dst, keyMaxNumBytes...) 59 | } 60 | return nil 61 | } 62 | */ 63 | 64 | impl IComparer { 65 | // func (icmp *iComparer) uName() string { 66 | // return icmp.ucmp.Name() 67 | // } 68 | 69 | pub fn u_name(&self) -> String { 70 | self.ucmp.name() 71 | } 72 | 73 | pub fn u_compare(&self, a: &[u8], b: &[u8]) -> Ordering { 74 | self.ucmp.compare(a, b) 75 | } 76 | 77 | // func (icmp *iComparer) Compare(a, b []byte) int { 78 | // x := icmp.uCompare(internalKey(a).ukey(), internalKey(b).ukey()) 79 | // if x == 0 { 80 | // if m, n := internalKey(a).num(), internalKey(b).num(); m > n { 81 | // return -1 82 | // } else if m < n { 83 | // return 1 84 | // } 85 | // } 86 | // return x 87 | // } 88 | pub fn compare(&self, a: &[u8], b: &[u8]) -> Ordering { 89 | let a = internalKey::new(a); 90 | let b = internalKey::new(b); 91 | let x = self.u_compare(a.ukey(), b.ukey()); 92 | if x == Ordering::Equal { 93 | let m = a.num(); 94 | let n = b.num(); 95 | if m > n { 96 | return Ordering::Greater; 97 | } else if m < n { 98 | return Ordering::Less; 99 | } 100 | return Ordering::Equal; 101 | } 102 | x 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/key.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_camel_case_types)] 2 | #![allow(non_snake_case)] 3 | #![allow(dead_code)] 4 | 5 | use crate::GenericResult; 6 | use bytes::{Buf, BufMut}; 7 | use std::fmt; 8 | use thiserror::Error; 9 | 10 | #[derive(Error, Debug)] 11 | pub enum KeyError { 12 | #[error("leveldb: internal key {ikey:?} corrupted: {reason:?}")] 13 | InternalKeyCorrupted { ikey: Vec, reason: String }, 14 | #[error("invalid keytype :`{0}`")] 15 | InvalidKeyType(u8), 16 | #[error("unknown data store error")] 17 | Unknown, 18 | } 19 | 20 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] 21 | pub enum keyType { 22 | Del = 0, 23 | Val = 1, 24 | Seek = 2, 25 | } 26 | 27 | impl fmt::Display for keyType { 28 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 29 | match self { 30 | keyType::Del => write!(f, "d"), 31 | keyType::Val => write!(f, "v"), 32 | keyType::Seek => write!(f, "s"), 33 | } 34 | } 35 | } 36 | pub const KEY_MAX_SEQ: u64 = (1 << 56) - 1; 37 | // const ( 38 | // // Maximum value possible for sequence number; the 8-bits are 39 | // // used by value type, so its can packed together in single 40 | // // 64-bit integer. 41 | // keyMaxSeq = (uint64(1) << 56) - 1 42 | // // Maximum value possible for packed sequence number and type. 43 | // keyMaxNum = (keyMaxSeq << 8) | uint64(keyTypeSeek) 44 | // ) 45 | pub fn keyTypeFromU8(n: u8) -> GenericResult { 46 | match n { 47 | 0 => Ok(keyType::Del), 48 | 1 => Ok(keyType::Val), 49 | _ => Err(Box::new(KeyError::InvalidKeyType(n))), 50 | } 51 | } 52 | 53 | // type internalKey []byte 54 | 55 | // func makeInternalKey(dst, ukey []byte, seq uint64, kt keyType) internalKey { 56 | // if seq > keyMaxSeq { 57 | // panic("leveldb: invalid sequence number") 58 | // } else if kt > keyTypeVal { 59 | // panic("leveldb: invalid type") 60 | // } 61 | 62 | // dst = ensureBuffer(dst, len(ukey)+8) 63 | // copy(dst, ukey) 64 | // binary.LittleEndian.PutUint64(dst[len(ukey):], (seq<<8)|uint64(kt)) 65 | // return internalKey(dst) 66 | // } 67 | #[derive(Debug, Clone)] 68 | pub struct internalKey(Vec); 69 | // 将用户的key 添加 sequence number 和 keyType 变为 internalKey 70 | pub fn makeInternalKey(ukey: &[u8], seq: u64, kt: keyType) -> internalKey { 71 | let mut v = ukey.to_vec(); 72 | let mut seq = seq << 8; 73 | if kt == keyType::Val || kt == keyType::Seek { 74 | seq = seq | 1 75 | } 76 | v.put_u64_le(seq); 77 | internalKey(v) 78 | } 79 | 80 | // func parseInternalKey(ik []byte) (ukey []byte, seq uint64, kt keyType, err error) { 81 | // if len(ik) < 8 { 82 | // return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid length") 83 | // } 84 | // num := binary.LittleEndian.Uint64(ik[len(ik)-8:]) 85 | // seq, kt = uint64(num>>8), keyType(num&0xff) 86 | // if kt > keyTypeVal { 87 | // return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid type") 88 | // } 89 | // ukey = ik[:len(ik)-8] 90 | // return 91 | // } 92 | // 从 internalKey 中解析后用户的key sequence number 和 keyType 93 | pub fn parseInternalKey(ik: &[u8]) -> GenericResult<(Vec, u64, keyType)> { 94 | let mut res = (Vec::new(), 0, keyType::Del); 95 | if ik.len() < 8 { 96 | return Err(Box::new(KeyError::InternalKeyCorrupted { 97 | ikey: ik.to_vec(), 98 | reason: String::from("invalid length"), 99 | })); 100 | } 101 | 102 | let mut b = &ik[ik.len() - 8..ik.len()]; 103 | let num = b.get_u64_le(); 104 | res.1 = num >> 8; 105 | res.2 = keyTypeFromU8((num & 0xff) as u8)?; 106 | res.0 = ik[..ik.len() - 8].to_vec(); 107 | 108 | Ok(res) 109 | } 110 | 111 | impl internalKey { 112 | pub fn new(data: &[u8]) -> Self { 113 | internalKey(data.to_vec()) 114 | } 115 | 116 | pub fn ukey(&self) -> &[u8] { 117 | assert!(self.0.len() > 8); 118 | return &self.0[..self.0.len() - 8]; 119 | } 120 | pub fn data(&self) -> &[u8] { 121 | return &self.0[..]; 122 | } 123 | 124 | pub fn num(&self) -> u64 { 125 | assert!(self.0.len() > 8); 126 | let mut buf = &self.0[self.0.len() - 8..]; 127 | buf.get_u64_le() 128 | } 129 | 130 | fn parseNum(&self) -> GenericResult<(u64, keyType)> { 131 | let num = self.num(); 132 | let mut res = (0, keyType::Del); 133 | res.0 = num >> 8; 134 | res.1 = keyTypeFromU8((num & 0xff) as u8)?; 135 | Ok(res) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![feature(partition_point)] 2 | pub mod errors; 3 | pub mod key; 4 | pub mod memdb; 5 | pub use errors::GenericResult; 6 | 7 | pub mod compare; 8 | pub mod table; 9 | pub use compare::BytesComparer; 10 | pub use compare::Comparer; 11 | 12 | pub mod icompare; 13 | pub mod storage; 14 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_camel_case_types)] 2 | #![allow(non_snake_case)] 3 | #![allow(dead_code)] 4 | 5 | fn main(){ 6 | 7 | } -------------------------------------------------------------------------------- /src/memdb/memdb.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | #![warn(non_upper_case_globals)] 3 | use std::cell::RefCell; 4 | use std::cmp::Ordering; 5 | use std::{sync, vec}; 6 | 7 | use crate::key::internalKey; 8 | use crate::key::keyType; 9 | use crate::key::makeInternalKey; 10 | use crate::{Comparer, GenericResult}; 11 | use bytes::BufMut; 12 | use rand::prelude::*; 13 | const MAX_HEIGHT: usize = 12; 14 | // const ( 15 | // nKV = iota 16 | // nKey 17 | // nVal 18 | // nHeight 19 | // nNext 20 | // ) 21 | const NKEY: usize = 1; 22 | const NVAL: usize = 2; 23 | const NHEIGHT: usize = 3; 24 | const NNEXT: usize = 4; 25 | 26 | // DB is an in-memory key/value database. 27 | // type DB struct { 28 | // cmp comparer.BasicComparer 29 | // rnd *rand.Rand 30 | // mu sync.RWMutex 31 | // kvData []byte 32 | // // Node data: 33 | // // [0] : KV offset 34 | // // [1] : Key length 35 | // // [2] : Value length 36 | // // [3] : Height 37 | // // [3..height] : Next nodes 38 | // nodeData []int 39 | // prevNode [tMaxHeight]int 40 | // maxHeight int 41 | // n int 42 | // kvSize int 43 | // } 44 | #[allow(non_camel_case_types)] 45 | struct db { 46 | cmp: T, 47 | // rnd:rand 48 | // 存储实际key，value的数据 49 | kv_data: Vec, 50 | // 用于记录key,value在kv_data中的索引 ,每一个节点的格式如下 ，其中 level 表示当前节点的层数，后跟 level 个数字，分别表示当前节点的level个层中每一层的下一个节点在node_data中的位置 51 | // kvOffset1|len(key1)|len(value1)|level|next_node_1|...|next_node_i|...|next_node_h| 52 | node_data: Vec, // 前面16个字节对应于 head 节点 53 | // 在查询过程中，记录搜索过程中所经历的节点(实际是节点在node_data的位置)，主要用于插入和删除 54 | prev_node: RefCell<[usize; MAX_HEIGHT]>, 55 | // 当前skiplist的层数 56 | max_height: usize, 57 | // skiplist中的节点个数 58 | n: usize, 59 | // 数据总大小 60 | kv_size: usize, 61 | } 62 | 63 | impl db { 64 | fn new(cmp: T, capacity: usize) -> Self { 65 | db { 66 | cmp: cmp, 67 | kv_data: Vec::with_capacity(capacity), 68 | node_data: vec![0; NNEXT + MAX_HEIGHT], 69 | prev_node: RefCell::new([0; MAX_HEIGHT]), 70 | max_height: 1, 71 | n: 0, 72 | kv_size: 0, 73 | } 74 | } 75 | 76 | // func (p *DB) randHeight() (h int) { 77 | // const branching = 4 78 | // h = 1 79 | // for h < tMaxHeight && p.rnd.Int()%branching == 0 { 80 | // h++ 81 | // } 82 | // return 83 | // 用于在插入新数据的时候，获取当前的节点的层高 84 | pub fn rand_heigth(&self) -> usize { 85 | const branching: usize = 4; 86 | let mut h = 1; 87 | while h < MAX_HEIGHT && random::() % branching == 0 { 88 | h += 1; 89 | } 90 | return h; 91 | } 92 | 93 | // 计算node节点在level层的下一个节点在node_data中的位置 ,封装一下，提高代码可读性 94 | fn next_node(&self, node: usize, i: usize) -> usize { 95 | // + NNEXT 表示在node_data 中,从node位置开始，要跳过 kvOffset1|len(key1)|len(value1)|level| 这4个字节，后面再移动 i 个位置，就到达 next_node_i 了 96 | self.node_data[node + NNEXT + i] 97 | } 98 | 99 | fn set_next_node(&mut self, node: usize, i: usize, next: usize) { 100 | self.node_data[node + NNEXT + i] = next; 101 | } 102 | 103 | // 根据 node 在 node_data中的位置，求出在kv_data 中的偏移量和长度,从而得到 key 104 | fn get_key_data(&self, node: usize) -> &[u8] { 105 | let offset = self.node_data[node]; 106 | &self.kv_data[offset..offset + self.node_data[node + NKEY]] 107 | } 108 | 109 | // 根据 node 在 node_data 中的位置，求出在kv_data 中的偏移量和长度,从而得到 value 110 | fn get_value_data(&self, node: usize) -> &[u8] { 111 | let key_offset = self.node_data[node] + self.node_data[node + NKEY]; 112 | &self.kv_data[key_offset..key_offset + self.node_data[node + NVAL]] 113 | } 114 | // Must hold RW-lock if prev == true, as it use shared prevNode slice. 115 | // func (p *DB) findGE(key []byte, prev bool) (int, bool) { 116 | // node := 0 117 | // h := p.maxHeight - 1 118 | // for { 119 | // next := p.nodeData[node+nNext+h] 120 | // cmp := 1 121 | // if next != 0 { 122 | // o := p.nodeData[next] 123 | // cmp = p.cmp.Compare(p.kvData[o:o+p.nodeData[next+nKey]], key) 124 | // } 125 | // if cmp < 0 { 126 | // // Keep searching in this list 127 | // node = next 128 | // } else { 129 | // if prev { 130 | // p.prevNode[h] = node 131 | // } else if cmp == 0 { 132 | // return next, true 133 | // } 134 | // if h == 0 { 135 | // return next, cmp == 0 136 | // } 137 | // h-- 138 | // } 139 | // } 140 | // } 141 | 142 | // save_pre 标记 在搜索过程中是否要记录遍历过的节点 143 | pub fn find_great_or_equal(&self, key: &internalKey, save_pre: bool) -> (usize, bool) { 144 | let mut node = 0; 145 | // 从高层到底层开始搜索 146 | let mut i = self.max_height - 1; 147 | // println!("max_height {}", i); 148 | 149 | loop { 150 | // 下个节点在 node_data 中的位置 151 | let next = self.next_node(node, i); 152 | let mut cmp = Ordering::Greater; 153 | // 当前链表上没有走到尾 154 | if next > 0 { 155 | // 和下个节点next进行key比较 156 | cmp = self.cmp.compare(self.get_key_data(next), key.data()); 157 | } 158 | 159 | // 大于下一个节点，继续沿着当前层 向右 跳 160 | if cmp == Ordering::Less { 161 | node = next; 162 | } else { 163 | // 小于等于下一个节点 或 下一个节点是空 164 | // if save_pre { 165 | // // 对于插入或删除而进行的搜索，即使遇到相同的也要继续往下一层比较,不能立即返回 166 | // // 所以这里要先于 cmp == Ordering::Equal 的判断 167 | // self.prev_node.borrow_mut()[i] = node; 168 | // } else if cmp == Ordering::Equal { 169 | // // find_great_or_equal 跟 find_less 的一个不同就是这里返回的是 next 170 | // return (next, true); 171 | // } 172 | 173 | // 改成下面的方式可读性更高 174 | if (!save_pre) && cmp == Ordering::Equal { 175 | return (next, true); 176 | } 177 | if save_pre { 178 | self.prev_node.borrow_mut()[i] = node; 179 | } 180 | 181 | if i == 0 { 182 | return (next, cmp == Ordering::Equal); 183 | } 184 | 185 | i -= 1; 186 | } 187 | } 188 | } 189 | 190 | // func (p *DB) findLT(key []byte) int { 191 | // node := 0 192 | // h := p.maxHeight - 1 193 | // for { 194 | // next := p.nodeData[node+nNext+h] 195 | // o := p.nodeData[next] 196 | // if next == 0 || p.cmp.Compare(p.kvData[o:o+p.nodeData[next+nKey]], key) >= 0 { 197 | // if h == 0 { 198 | // break 199 | // } 200 | // h-- 201 | // } else { 202 | // node = next 203 | // } 204 | // } 205 | // return node 206 | // } 207 | 208 | // 查询小于 key 的，离key最近的节点。这个不涉及保存pre_node 逻辑简单 209 | fn find_lessthan(&self, key: &internalKey) -> usize { 210 | let mut node = 0; 211 | let mut i = self.max_height - 1; 212 | loop { 213 | let next = self.next_node(node, i); 214 | // 当前链表到尾部了 或 下个节点的key 大于等于 key， 向下一层 215 | if next == 0 || self.cmp.compare(self.get_key_data(next), key.data()) != Ordering::Less 216 | { 217 | if i == 0 { 218 | // 这里是返回 node 而不是 next 219 | return node; 220 | } 221 | i -= 1; 222 | } else { 223 | // 当前链表 跳到 下一个节点 224 | node = next; 225 | } 226 | } 227 | } 228 | 229 | // func (p *DB) findLast() int { 230 | // node := 0 231 | // h := p.maxHeight - 1 232 | // for { 233 | // next := p.nodeData[node+nNext+h] 234 | // if next == 0 { 235 | // if h == 0 { 236 | // break 237 | // } 238 | // h-- 239 | // } else { 240 | // node = next 241 | // } 242 | // } 243 | // return node 244 | // } 245 | 246 | // 查询最后一个 ,逻辑也简单，每一层沿着链表走到尾部，然后跳到下一层，再沿着链表走到底 247 | pub fn find_last(&self) -> usize { 248 | let mut node = 0; 249 | let mut i = self.max_height - 1; 250 | loop { 251 | let next = self.next_node(node, i); 252 | // 走到了尾部， 253 | if next == 0 { 254 | if i == 0 { 255 | return node; 256 | } 257 | // 跳过下一层 258 | i -= 1; 259 | } else { 260 | // 沿着链表往后面跳 261 | node = next; 262 | } 263 | } 264 | } 265 | } 266 | 267 | pub struct Db { 268 | db: sync::RwLock>, 269 | } 270 | 271 | impl Db { 272 | // Put sets the value for the given key. It overwrites any previous value 273 | // for that key; a DB is not a multi-map. 274 | // 275 | // It is safe to modify the contents of the arguments after Put returns. 276 | // func (p *DB) Put(key []byte, value []byte) error { 277 | // p.mu.Lock() 278 | // defer p.mu.Unlock() 279 | 280 | // if node, exact := p.findGE(key, true); exact { 281 | // kvOffset := len(p.kvData) 282 | // p.kvData = append(p.kvData, key...) 283 | // p.kvData = append(p.kvData, value...) 284 | // p.nodeData[node] = kvOffset 285 | // m := p.nodeData[node+nVal] 286 | // p.nodeData[node+nVal] = len(value) 287 | // p.kvSize += len(value) - m 288 | // return nil 289 | // } 290 | 291 | // h := p.randHeight() 292 | // if h > p.maxHeight { 293 | // for i := p.maxHeight; i < h; i++ { 294 | // p.prevNode[i] = 0 295 | // } 296 | // p.maxHeight = h 297 | // } 298 | 299 | // kvOffset := len(p.kvData) 300 | // p.kvData = append(p.kvData, key...) 301 | // p.kvData = append(p.kvData, value...) 302 | // // Node 303 | // node := len(p.nodeData) 304 | // p.nodeData = append(p.nodeData, kvOffset, len(key), len(value), h) 305 | // for i, n := range p.prevNode[:h] { 306 | // m := n + nNext + i 307 | // p.nodeData = append(p.nodeData, p.nodeData[m]) 308 | // p.nodeData[m] = node 309 | // } 310 | 311 | // p.kvSize += len(key) + len(value) 312 | // p.n++ 313 | // return nil 314 | // } 315 | pub fn put(&mut self, key: &internalKey, value: Option<&[u8]>) -> GenericResult<()> { 316 | let mut db = self.db.write().unwrap(); 317 | let (node, exist) = db.find_great_or_equal(key, true); 318 | if exist { 319 | // TODO 优化，如果新value的长度小于等于旧的value，直接覆盖直接的，不用重新分配 320 | // 如果key已经存在，直接覆盖之前的value 321 | // 数据是追加的方式，所以当前kv_data的长度就是node节点的在 skv_data 上的新的偏移量 322 | let offset = db.kv_data.len(); 323 | // 追加 key 和 value 的数据 324 | db.kv_data.append(&mut key.data().to_vec()); 325 | let mut v_len = 0; 326 | if let Some(value) = value { 327 | v_len = value.len(); 328 | db.kv_data.append(&mut value.to_vec()); 329 | } 330 | // 更新node的偏移量 331 | db.node_data[node] = offset; 332 | // value 的长度可能也变化了 333 | // 之前的长度 334 | let v_old = db.node_data[node + NVAL]; 335 | // 更新为新的长度 336 | db.node_data[node + NVAL] = v_len; 337 | // 更新数据总大小 338 | db.kv_size += v_len - v_old; 339 | return Ok(()); 340 | } 341 | 342 | let h = db.rand_heigth(); 343 | // 处理head节点 344 | if h > db.max_height { 345 | for i in db.max_height..h + 1 { 346 | db.prev_node.borrow_mut()[i] = 0; 347 | } 348 | db.max_height = h; 349 | println!("height {}", h); 350 | } 351 | 352 | // 新增节点在 kv_data 中的起始偏移 353 | let kv_offset = db.kv_data.len(); 354 | // 追加 key 和 value 的数据 355 | db.kv_data.append(&mut key.data().to_vec()); 356 | let mut v_len = 0; 357 | if let Some(value) = value { 358 | v_len = value.len(); 359 | db.kv_data.append(&mut value.to_vec()); 360 | } 361 | 362 | // 创建新节点,因为是追加方式，所以当前 node_data 的长度 就是新节点在 node_data 的位置 363 | let node = db.node_data.len(); 364 | // 添加新节点 365 | db.node_data.push(kv_offset); 366 | db.node_data.push(key.data().len()); 367 | db.node_data.push(v_len); 368 | db.node_data.push(h); 369 | 370 | // 这里出问题了 371 | // for (i,n )in db.prev_node.borrow()[0..h].iter().enumerate(){ 372 | // let next = db.next_node(*n, i); 373 | // db.node_data.push(next); 374 | 375 | // } 376 | // error[E0502]: cannot borrow `db` as mutable because it is also borrowed as immutable 377 | // --> src/memdb/memdb.rs:343:13 378 | // | 379 | // 341 | for (i,n )in db.prev_node.borrow()[0..h].iter().enumerate(){ 380 | // | --------------------- 381 | // | | 382 | // | immutable borrow occurs here 383 | // | a temporary with access to the immutable borrow is created here ... 384 | // 342 | let next = db.next_node(*n, i); 385 | // 343 | db.node_data.push(next); 386 | // | ^^ mutable borrow occurs here 387 | // 344 | 388 | // 345 | } 389 | // 再添加n个元素用来保存节点在h层中每一层的下个节点的位置 390 | db.node_data.resize(node + NNEXT + h, 0); 391 | // let pre_node = db.prev_node.borrow()[0..h].to_vec(); 392 | // for (i,n)in pre_node.iter().enumerate(){ 393 | // let next = db.next_node(*n, i); 394 | // db.set_next_node(node, i, next); 395 | // db.set_next_node(*n, i, node); 396 | // } 397 | for i in 0..h { 398 | let n = db.prev_node.borrow()[i]; 399 | let next = db.next_node(n, i); 400 | db.set_next_node(node, i, next); 401 | db.set_next_node(n, i, node); 402 | } 403 | 404 | db.kv_size += key.data().len() + v_len; 405 | db.n += 1; 406 | 407 | Ok(()) 408 | } 409 | 410 | pub fn delete(&mut self, key: &internalKey) -> Option<()> { 411 | let mut db = self.db.write().unwrap(); 412 | let (node, exist) = db.find_great_or_equal(key, true); 413 | if !exist { 414 | return None; 415 | } 416 | 417 | // 当前节点有几层 418 | let h = db.node_data[node + NHEIGHT]; 419 | // 开始删除, 让前一个节点指向前一个节点的下一个节点的下一个节点 pre->next = pre->next->next 420 | for i in 0..h { 421 | let pre = db.prev_node.borrow()[i]; 422 | // let pre_next = db.next_node(pre, i); 423 | // if pre_next != node { 424 | // print!("{}:{}", pre_next, node); 425 | // } 426 | 427 | // let next_next = db.next_node(pre_next, i); 428 | // db.set_next_node(pre, i, next_next); 429 | 430 | // 由于 前一个节点的下一个节点 pre_node 就是当前节点 node ，所以上面代码可以优化为 431 | let next_next = db.next_node(node, i); 432 | db.set_next_node(pre, i, next_next); 433 | } 434 | 435 | db.kv_size -= db.node_data[node + NKEY] + db.node_data[node + NVAL]; 436 | db.n -= 1; 437 | 438 | Some(()) 439 | } 440 | 441 | pub fn contains(&self, key: &internalKey) -> bool { 442 | let db = self.db.read().unwrap(); 443 | let (_, exist) = db.find_great_or_equal(key, false); 444 | return exist; 445 | } 446 | // Get gets the value for the given key. It returns error.ErrNotFound if the 447 | // DB does not contain the key. 448 | // 449 | // The caller should not modify the contents of the returned slice, but 450 | // it is safe to modify the contents of the argument after Get returns. 451 | // func (p *DB) Get(key []byte) (value []byte, err error) { 452 | // p.mu.RLock() 453 | // if node, exact := p.findGE(key, false); exact { 454 | // o := p.nodeData[node] + p.nodeData[node+nKey] 455 | // value = p.kvData[o : o+p.nodeData[node+nVal]] 456 | // } else { 457 | // err = ErrNotFound 458 | // } 459 | // p.mu.RUnlock() 460 | // return 461 | // } 462 | 463 | pub fn get(&self, key: &internalKey) -> Option> { 464 | let db = self.db.read().unwrap(); 465 | let (node, exist) = db.find_great_or_equal(key, false); 466 | if exist { 467 | Some(db.get_value_data(node).to_vec()) 468 | } else { 469 | None 470 | } 471 | } 472 | 473 | // Find finds key/value pair whose key is greater than or equal to the 474 | // given key. It returns ErrNotFound if the table doesn't contain 475 | // such pair. 476 | // 477 | // The caller should not modify the contents of the returned slice, but 478 | // it is safe to modify the contents of the argument after Find returns. 479 | // func (p *DB) Find(key []byte) (rkey, value []byte, err error) { 480 | // p.mu.RLock() 481 | // if node, _ := p.findGE(key, false); node != 0 { 482 | // n := p.nodeData[node] 483 | // m := n + p.nodeData[node+nKey] 484 | // rkey = p.kvData[n:m] 485 | // value = p.kvData[m : m+p.nodeData[node+nVal]] 486 | // } else { 487 | // err = ErrNotFound 488 | // } 489 | // p.mu.RUnlock() 490 | // return 491 | // } 492 | 493 | fn find(&self, key: &internalKey) -> Option<(Vec, Vec)> { 494 | let db = self.db.read().unwrap(); 495 | let (node, exist) = db.find_great_or_equal(key, false); 496 | if exist { 497 | Some(( 498 | db.get_key_data(node).to_vec(), 499 | db.get_value_data(node).to_vec(), 500 | )) 501 | } else { 502 | None 503 | } 504 | } 505 | 506 | fn capacity(&self) -> usize { 507 | self.db.read().unwrap().kv_data.capacity() 508 | } 509 | 510 | fn size(&self) -> usize { 511 | self.db.read().unwrap().kv_size 512 | } 513 | 514 | fn free(&self) -> usize { 515 | let db = self.db.read().unwrap(); 516 | db.kv_data.capacity() - db.kv_data.len() 517 | } 518 | 519 | fn len(&self) -> usize { 520 | self.db.read().unwrap().n 521 | } 522 | // Reset resets the DB to initial empty state. Allows reuse the buffer. 523 | // func (p *DB) Reset() { 524 | // p.mu.Lock() 525 | // p.rnd = rand.New(rand.NewSource(0xdeadbeef)) 526 | // p.maxHeight = 1 527 | // p.n = 0 528 | // p.kvSize = 0 529 | // p.kvData = p.kvData[:0] 530 | // p.nodeData = p.nodeData[:nNext+tMaxHeight] 531 | // p.nodeData[nKV] = 0 532 | // p.nodeData[nKey] = 0 533 | // p.nodeData[nVal] = 0 534 | // p.nodeData[nHeight] = tMaxHeight 535 | // for n := 0; n < tMaxHeight; n++ { 536 | // p.nodeData[nNext+n] = 0 537 | // p.prevNode[n] = 0 538 | // } 539 | // p.mu.Unlock() 540 | // } 541 | fn reset(&mut self) { 542 | let mut db = self.db.write().unwrap(); 543 | db.max_height = 1; 544 | db.n = 0; 545 | db.kv_size = 0; 546 | db.kv_data.clear(); 547 | db.node_data.resize(NNEXT + MAX_HEIGHT, 0); 548 | db.prev_node.borrow_mut().fill(0); 549 | } 550 | 551 | fn new(cmp: T, capacity: usize) -> Self { 552 | Db { 553 | db: sync::RwLock::new(db::new(cmp, capacity)), 554 | } 555 | } 556 | } 557 | 558 | #[cfg(test)] 559 | mod tests { 560 | use crate::BytesComparer; 561 | 562 | use super::*; 563 | 564 | #[test] 565 | fn test_name() { 566 | let mut db = Db::new(BytesComparer {}, 100); 567 | let max_iter_num = 1000000; 568 | for i in 1..max_iter_num { 569 | // println!("to add {}",i); 570 | let mut data = vec![]; 571 | // let i = i*100; 572 | data.put_u32_le(i); 573 | let key = makeInternalKey(&data[0..4], 10, keyType::Val); 574 | db.put(&key, Some(&data[0..4])).unwrap(); 575 | db.get(&key).unwrap(); 576 | } 577 | 578 | for i in 1..max_iter_num { 579 | // println!("to add {}",i); 580 | let mut data = vec![]; 581 | // let i = i*100; 582 | data.put_u32_le(i); 583 | let key = makeInternalKey(&data[0..4], 10, keyType::Val); 584 | db.delete(&key).unwrap(); 585 | } 586 | } 587 | } 588 | -------------------------------------------------------------------------------- /src/memdb/memdb_node.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | #![warn(non_upper_case_globals)] 3 | use std::cell::RefCell; 4 | use std::cmp::Ordering; 5 | use std::rc::Rc; 6 | use std::sync::Arc; 7 | use std::{sync, vec}; 8 | 9 | use crate::key::internalKey; 10 | use crate::key::keyType; 11 | use crate::key::makeInternalKey; 12 | use crate::{Comparer, GenericResult}; 13 | use bytes::BufMut; 14 | use rand::prelude::*; 15 | const MAX_HEIGHT: usize = 12; 16 | 17 | #[allow(non_camel_case_types)] 18 | struct db_skip { 19 | cmp: T, 20 | // 存储实际key，value的数据 ，offset从1开始， offset为0的表示head节点 21 | kv_data: Vec, 22 | head: RcNode, // 头部， 23 | // 当前skiplist的层数 24 | max_height: usize, 25 | // skiplist中的节点个数 26 | n: usize, 27 | // 数据总大小 28 | kv_size: usize, 29 | } 30 | 31 | impl db_skip { 32 | 33 | fn new(cmp:T)->Self{ 34 | let head = Rc::new(RefCell::new(node::new(0, 0, 0, 1))); 35 | Self{ 36 | cmp:cmp, 37 | kv_data:vec![], 38 | max_height:1, 39 | head:head, 40 | n:0, 41 | kv_size:0, 42 | } 43 | } 44 | // 根据 node 在 node_data中的位置，求出在kv_data 中的偏移量和长度,从而得到 key 45 | fn get_key_data(&self, node: &node) -> &[u8] { 46 | &self.kv_data[node.offset..node.offset + node.key_len] 47 | } 48 | 49 | // 根据 node 在 node_data 中的位置，求出在kv_data 中的偏移量和长度,从而得到 value 50 | fn get_value_data(&self, node: &node) -> &[u8] { 51 | &self.kv_data[node.offset + node.key_len..node.offset + node.key_len + node.value_len] 52 | } 53 | 54 | // 用于在插入新数据的时候，获取当前的节点的层高 55 | pub fn rand_heigth(&self) -> usize { 56 | const branching: usize = 4; 57 | let mut h = 1; 58 | while h < MAX_HEIGHT && random::() % branching == 0 { 59 | h += 1; 60 | } 61 | return h; 62 | } 63 | 64 | pub fn find_great_or_equal( 65 | &self, 66 | key: &internalKey, 67 | pre_node: &mut Option>, // 注意 不能用 Option<&mut Vec> 68 | ) -> (RcNode, bool) { 69 | let mut node = Rc::clone(&self.head); // 从头节点开始 70 | let mut next_node = Rc::clone(&node); 71 | let mut i = self.max_height - 1; 72 | 73 | loop { 74 | // 这里将 cmp 预先设置为 Ordering::Less 是一个非常巧妙的方式, 就可以自动包含 下个节点为空(当作是无穷大)的情况了 75 | let mut cmp = Ordering::Less; 76 | if let Some(ref next) = node.borrow().next[i] { 77 | // 下一个节点存在 78 | cmp = self 79 | .cmp 80 | .compare(key.data(), self.get_key_data(&next.borrow())); 81 | 82 | next_node = Rc::clone(&next); 83 | } 84 | 85 | // 大于下一个节点，继续沿着当前层 向右 跳 86 | if cmp == Ordering::Greater { 87 | node = Rc::clone(&next_node); 88 | continue; 89 | } 90 | 91 | // 走到这里，说明： node 小于等于下一个节点 或 下一个节点是空 92 | 93 | // 如果不保存前向节点，只是普通的搜索，找到匹配就直接返回 94 | if (pre_node.is_none()) && cmp == Ordering::Equal { 95 | return (next_node, true); 96 | } 97 | 98 | // 如果保存前向节点node 99 | if let Some(ref mut pre) = pre_node { 100 | pre.push(Rc::clone(&node)); 101 | } 102 | 103 | if i == 0 { 104 | return (next_node, cmp == Ordering::Equal); 105 | } 106 | 107 | i -= 1; 108 | } 109 | } 110 | // 查询小于 key 的，离key最近的节点。这个不涉及保存pre_node 逻辑简单 111 | fn find_lessthan(&self, key: &internalKey) -> RcNode { 112 | let mut node = Rc::clone(&self.head); // 从头节点开始 113 | let mut next_node = Rc::clone(&node); 114 | let mut i = self.max_height - 1; 115 | loop { 116 | // 这里将 cmp 预先设置为 Ordering::Less 是一个非常巧妙的方式, 就可以自动包含 下个节点为空(当作是无穷大)的情况了 117 | let mut cmp = Ordering::Less; 118 | if let Some(ref next) = node.borrow().next[i] { 119 | // 下一个节点存在 120 | cmp = self 121 | .cmp 122 | .compare(key.data(), self.get_key_data(&next.borrow())); 123 | 124 | next_node = Rc::clone(&next); 125 | } 126 | 127 | // 当前链表到尾部了 或 下个节点的key 大于等于 key， 向下一层 128 | if cmp != Ordering::Greater { 129 | if i == 0 { 130 | return node; 131 | } 132 | i -= 1; 133 | } else { 134 | // 当前链表 跳到 下一个节点 135 | node = Rc::clone(&next_node); 136 | } 137 | } 138 | } 139 | 140 | // 查询最后一个 ,逻辑也简单，每一层沿着链表走到尾部，然后跳到下一层，再沿着链表走到底 141 | pub fn find_last(&self) -> RcNode { 142 | let mut node = Rc::clone(&self.head); // 从头节点开始 143 | let mut next_node = Rc::clone(&node); 144 | let mut i = self.max_height - 1; 145 | 146 | loop { 147 | let mut end = false; 148 | if let Some(ref next) = node.borrow().next[i] { 149 | // 下一个节点存在 150 | next_node = Rc::clone(&next); 151 | end = true; 152 | } 153 | 154 | // 当前链表走到头 155 | if end { 156 | if i == 0 { 157 | return node; 158 | } 159 | i -= 1; 160 | } else { 161 | node = Rc::clone(&next_node); 162 | } 163 | } 164 | } 165 | } 166 | 167 | type RcNode = Rc>; 168 | // 每一个节点 169 | struct node { 170 | offset: usize, // 对应kv_data 中的起始位置 171 | key_len: usize, // key的长度 172 | value_len: usize, // value的长度 173 | next: Vec>, // 当前节点有 height 层, 第i元素表示第i层的下一个节点 174 | } 175 | impl node { 176 | fn new(offset: usize, key_len: usize, value_len: usize, height: usize) -> Self { 177 | node { 178 | offset: offset, 179 | key_len: key_len, 180 | value_len: value_len, 181 | next: vec![None; height], 182 | } 183 | } 184 | // 根据 node 在 node_data中的位置，求出在kv_data 中的偏移量和长度,从而得到 key 185 | fn get_key_data<'a>(&self, node: &node, kv_data: &'a [u8]) -> &'a [u8] { 186 | &kv_data[node.offset..node.offset + node.key_len] 187 | } 188 | 189 | // 根据 node 在 node_data 中的位置，求出在kv_data 中的偏移量和长度,从而得到 value 190 | fn get_value_data<'a>(&self, node: &node, kv_data: &'a [u8]) -> &'a [u8] { 191 | &kv_data[node.offset + node.key_len..node.offset + node.key_len + node.value_len] 192 | } 193 | } 194 | 195 | pub struct DBSkip { 196 | db: sync::RwLock>, 197 | } 198 | 199 | impl DBSkip { 200 | pub fn put(&mut self, key: &internalKey, value: Option<&[u8]>) -> GenericResult<()> { 201 | let mut db = self.db.write().unwrap(); 202 | let mut pre_node = Some(vec![]); 203 | let (node, exist) = db.find_great_or_equal(key, &mut pre_node); 204 | if exist { 205 | // 如果key已经存在，直接覆盖之前的value 206 | // 数据是追加的方式，所以当前kv_data的长度就是node节点的在 skv_data 上的新的偏移量 207 | let offset = db.kv_data.len(); 208 | // 追加 key 和 value 的数据 209 | db.kv_data.append(&mut key.data().to_vec()); 210 | let mut v_len = 0; 211 | if let Some(value) = value { 212 | v_len = value.len(); 213 | db.kv_data.append(&mut value.to_vec()); 214 | } 215 | // 更新node的偏移量 216 | node.borrow_mut().offset = offset; 217 | 218 | // value 的长度可能也变化了 219 | // 之前的长度 220 | let v_old = node.borrow().value_len; 221 | // 更新为新的长度 222 | node.borrow_mut().value_len = v_len; 223 | // 更新数据总大小 224 | db.kv_size += v_len - v_old; 225 | return Ok(()); 226 | } 227 | 228 | let mut pre_node = pre_node.unwrap(); 229 | let h = db.rand_heigth(); 230 | // 处理head节点 231 | if h > db.max_height { 232 | // 补充 高出的部分 233 | for i in db.max_height..h{ 234 | pre_node.push(Rc::clone(&db.head)); 235 | } 236 | db.max_height = h; 237 | println!("height {}", h); 238 | } 239 | 240 | // 新增节点在 kv_data 中的起始偏移 241 | let kv_offset = db.kv_data.len(); 242 | // 追加 key 和 value 的数据 243 | db.kv_data.append(&mut key.data().to_vec()); 244 | let mut v_len = 0; 245 | if let Some(value) = value { 246 | v_len = value.len(); 247 | db.kv_data.append(&mut value.to_vec()); 248 | } 249 | 250 | // 创建新节点 251 | let node = Rc::new(RefCell::new(node::new( 252 | kv_offset, 253 | key.data().len(), 254 | v_len, 255 | h, 256 | ))); 257 | 258 | // 执行插入 259 | for (i, pre) in pre_node.iter().enumerate() { 260 | // 新节点->next = pre->next 261 | if let Some(ref pre_next) = pre.borrow().next[i] { 262 | node.borrow_mut().next[i] = Some(Rc::clone(pre_next)); 263 | } 264 | // 因为 node.next[i]默认是Node，所以不用考虑 pre.borrow().next[i]为Node情况 265 | 266 | // pre->next = 新节点 267 | pre.borrow_mut().next[i] = Some(Rc::clone(&node)); 268 | } 269 | 270 | db.kv_size+=key.data().len()+v_len; 271 | db.n+=1; 272 | 273 | Ok(()) 274 | } 275 | 276 | pub fn delete(&mut self, key: &internalKey) -> Option<()> { 277 | let mut db = self.db.write().unwrap(); 278 | let mut pre_node = Some(vec![]); 279 | let (node, exist) = db.find_great_or_equal(key, &mut pre_node); 280 | if !exist{ 281 | return None; 282 | } 283 | let pre_node = pre_node.unwrap(); 284 | // 执行删除 285 | for (i, pre) in pre_node.iter().enumerate() { 286 | // pre->next = node->next 287 | if let Some(ref node_next) = node.borrow().next[i] { 288 | pre.borrow_mut().next[i] = Some(Rc::clone(node_next)); 289 | }else{ 290 | pre.borrow_mut().next[i]= None; 291 | } 292 | } 293 | 294 | db.kv_size-=node.borrow().key_len+node.borrow().value_len; 295 | db.n -=1; 296 | Some(()) 297 | } 298 | } 299 | -------------------------------------------------------------------------------- /src/memdb/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod memdb; 2 | pub use memdb::Db; 3 | 4 | pub mod memdb_node; 5 | -------------------------------------------------------------------------------- /src/storage/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod storage; 2 | pub use storage::FileDesc; 3 | -------------------------------------------------------------------------------- /src/storage/storage.rs: -------------------------------------------------------------------------------- 1 | // FileType represent a file type. 2 | // type FileType int 3 | 4 | // // File types. 5 | // const ( 6 | // TypeManifest FileType = 1 << iota 7 | // TypeJournal 8 | // TypeTable 9 | // TypeTemp 10 | 11 | // TypeAll = TypeManifest | TypeJournal | TypeTable | TypeTemp 12 | // ) 13 | #[derive(Debug, Clone, Copy)] 14 | pub enum FileType { 15 | TypeManifest = 1, 16 | TypeJournal = 2, 17 | TypeTable = 4, 18 | TypeTemp = 8, 19 | TypeAll = 1 | 2 | 4 | 8, 20 | } 21 | // func (t FileType) String() string { 22 | // switch t { 23 | // case TypeManifest: 24 | // return "manifest" 25 | // case TypeJournal: 26 | // return "journal" 27 | // case TypeTable: 28 | // return "table" 29 | // case TypeTemp: 30 | // return "temp" 31 | // } 32 | // return fmt.Sprintf("", t) 33 | // } 34 | 35 | // FileDesc is a 'file descriptor'. 36 | // type FileDesc struct { 37 | // Type FileType 38 | // Num int64 39 | // } 40 | #[derive(Debug, Clone, Copy)] 41 | pub struct FileDesc { 42 | pub file_type: FileType, 43 | pub num: i64, 44 | } 45 | -------------------------------------------------------------------------------- /src/table.rs: -------------------------------------------------------------------------------- 1 | #[allow(non_camel_case_types)] 2 | #[allow(dead_code)] 3 | use crate::compare::Comparer; 4 | use crate::icompare::IComparer; 5 | use crate::key::{internalKey, keyType, makeInternalKey, KEY_MAX_SEQ}; 6 | use crate::storage; 7 | use std::cmp::Ordering::{Equal, Greater, Less}; 8 | use std::sync::atomic; 9 | use std::sync::atomic::Ordering; 10 | use std::sync::Arc; 11 | // tFile holds basic information about a table. 12 | // type tFile struct { 13 | // fd storage.FileDesc 14 | // seekLeft int32 15 | // size int64 16 | // imin, imax internalKey 17 | // } 18 | #[derive(Clone)] 19 | pub struct tFile { 20 | fd: storage::FileDesc, 21 | seek_left: Arc, 22 | size: i64, 23 | imin: internalKey, 24 | imax: internalKey, 25 | } 26 | 27 | impl tFile { 28 | // Returns true if given key is after largest key of this table. 29 | // func (t *tFile) after(icmp *iComparer, ukey []byte) bool { 30 | // return ukey != nil && icmp.uCompare(ukey, t.imax.ukey()) > 0 31 | // } 32 | pub fn after(&self, icmp: &IComparer, ukey: &[u8]) -> bool 33 | where 34 | T: Comparer, 35 | { 36 | return ukey.len() > 0 && icmp.u_compare(ukey, self.imax.ukey()) == Greater; 37 | } 38 | 39 | // Returns true if given key is before smallest key of this table. 40 | // func (t *tFile) before(icmp *iComparer, ukey []byte) bool { 41 | // return ukey != nil && icmp.uCompare(ukey, t.imin.ukey()) < 0 42 | // } 43 | pub fn before(&self, icmp: &IComparer, ukey: &[u8]) -> bool 44 | where 45 | T: Comparer, 46 | { 47 | return ukey.len() > 0 && icmp.u_compare(ukey, self.imin.ukey()) > Greater; 48 | } 49 | 50 | // Returns true if given key range overlaps with this table key range. 51 | // func (t *tFile) overlaps(icmp *iComparer, umin, umax []byte) bool { 52 | // return !t.after(icmp, umin) && !t.before(icmp, umax) 53 | // } 54 | pub fn overlaps(&self, icmp: &IComparer, umin: &[u8], umax: &[u8]) -> bool 55 | where 56 | T: Comparer, 57 | { 58 | return !self.after(icmp, umin) && !self.before(icmp, umax); 59 | } 60 | 61 | // Cosumes one seek and return current seeks left. 62 | // func (t *tFile) consumeSeek() int32 { 63 | // return atomic.AddInt32(&t.seekLeft, -1) 64 | // } 65 | pub fn consume_seek(&self) -> i32 { 66 | self.seek_left.fetch_sub(1, Ordering::Relaxed) 67 | } 68 | } 69 | 70 | /* 71 | // Creates new tFile. 72 | func newTableFile(fd storage.FileDesc, size int64, imin, imax internalKey) *tFile { 73 | f := &tFile{ 74 | fd: fd, 75 | size: size, 76 | imin: imin, 77 | imax: imax, 78 | } 79 | 80 | // We arrange to automatically compact this file after 81 | // a certain number of seeks. Let's assume: 82 | // (1) One seek costs 10ms 83 | // (2) Writing or reading 1MB costs 10ms (100MB/s) 84 | // (3) A compaction of 1MB does 25MB of IO: 85 | // 1MB read from this level 86 | // 10-12MB read from next level (boundaries may be misaligned) 87 | // 10-12MB written to next level 88 | // This implies that 25 seeks cost the same as the compaction 89 | // of 1MB of data. I.e., one seek costs approximately the 90 | // same as the compaction of 40KB of data. We are a little 91 | // conservative and allow approximately one seek for every 16KB 92 | // of data before triggering a compaction. 93 | f.seekLeft = int32(size / 16384) 94 | if f.seekLeft < 100 { 95 | f.seekLeft = 100 96 | } 97 | 98 | return f 99 | } 100 | 101 | func tableFileFromRecord(r atRecord) *tFile { 102 | return newTableFile(storage.FileDesc{Type: storage.TypeTable, Num: r.num}, r.size, r.imin, r.imax) 103 | } 104 | */ 105 | 106 | // tFiles hold multiple tFile. 107 | pub struct tFiles(Vec); 108 | #[allow(dead_code)] 109 | impl tFiles { 110 | // func (tf tFiles) nums() string { 111 | // x := "[ " 112 | // for i, f := range tf { 113 | // if i != 0 { 114 | // x += ", " 115 | // } 116 | // x += fmt.Sprint(f.fd.Num) 117 | // } 118 | // x += " ]" 119 | // return x 120 | // } 121 | 122 | // 虽然 let s3 = s1 + &s2; 看起来就像它会复制两个字符串并创建一个新的字符串，而实际上这个语句会获取 s1 的所有权，附加上从 s2 中拷贝的内容，并返回结果的所有权。 123 | // 换句话说，它看起来好像生成了很多拷贝，不过实际上并没有：这个实现比拷贝要更高效。 124 | pub fn nums(&self) -> String { 125 | let mut x = String::from("[ "); 126 | /* 127 | for (i,f) in self.0.iter().enumerate(){ 128 | if i !=0{ 129 | x.push_str(", "); 130 | } 131 | x.push_str(&format!("{}",f.fd.num)); 132 | } 133 | x.push_str(" ]"); 134 | 135 | */ 136 | // 直接使用 + 137 | for (i, f) in self.0.iter().enumerate() { 138 | if i != 0 { 139 | x += ", "; 140 | } 141 | x += &format!("{}", f.fd.num); 142 | } 143 | x += " ]"; 144 | x 145 | } 146 | 147 | // Returns true if i smallest key is less than j. 148 | // This used for sort by key in ascending order. 149 | // func (tf tFiles) lessByKey(icmp *iComparer, i, j int) bool { 150 | // a, b := tf[i], tf[j] 151 | // n := icmp.Compare(a.imin, b.imin) 152 | // if n == 0 { 153 | // return a.fd.Num < b.fd.Num 154 | // } 155 | // return n < 0 156 | // } 157 | // 如果第i个tFile的最小值 小于 第j个tFile的最小值，返回true，用于递增排序, 158 | // 这个方法可以去掉, 直接使用 sort_unstable_by 进行排序 159 | pub fn less_by_key( 160 | &self, 161 | icmp: &IComparer, 162 | i: usize, 163 | j: usize, 164 | ) -> std::cmp::Ordering { 165 | let n = icmp.compare(self.0[i].imin.data(), self.0[j].imin.data()); 166 | if n == Equal { 167 | return self.0[i].fd.num.cmp(&self.0[j].fd.num); 168 | } 169 | n 170 | } 171 | // Returns true if i file number is greater than j. 172 | // This used for sort by file number in descending order. 173 | // func (tf tFiles) lessByNum(i, j int) bool { 174 | // return tf[i].fd.Num > tf[j].fd.Num 175 | // } 176 | 177 | // 如果第i个文件的file number 大于第j个文件的file number，返回true; 178 | // 用于基于 file number 递减排序, 这个方法可以去掉 179 | pub fn less_by_num(&self, i: usize, j: usize) -> bool { 180 | return self.0[i].fd.num < self.0[j].fd.num; 181 | } 182 | 183 | // Sorts tables by key in ascending order. 184 | // func (tf tFiles) sortByKey(icmp *iComparer) { 185 | // sort.Sort(&tFilesSortByKey{tFiles: tf, icmp: icmp}) 186 | // } 187 | pub fn sort_by_key(&mut self, icmp: &IComparer) { 188 | self.0.sort_unstable_by(|a, b| less_by_key(icmp, a, b)); 189 | } 190 | // Sorts tables by file number in descending order. 191 | // func (tf tFiles) sortByNum() { 192 | // sort.Sort(&tFilesSortByNum{tFiles: tf}) 193 | // } 194 | // 基于 file number 递减排序 195 | pub fn sort_by_num(&mut self) { 196 | self.0.sort_unstable_by(|a, b| a.fd.num.cmp(&b.fd.num)); 197 | } 198 | 199 | // Returns sum of all tables size. 200 | // func (tf tFiles) size() (sum int64) { 201 | // for _, t := range tf { 202 | // sum += t.size 203 | // } 204 | // return sum 205 | // } 206 | 207 | pub fn size(&self) -> i64 { 208 | let mut sum = 0; 209 | for t in &self.0 { 210 | sum += t.size; 211 | } 212 | sum 213 | } 214 | 215 | // Searches smallest index of tables whose its smallest 216 | // key is after or equal with given key. 217 | // func (tf tFiles) searchMin(icmp *iComparer, ikey internalKey) int { 218 | // return sort.Search(len(tf), func(i int) bool { 219 | // return icmp.Compare(tf[i].imin, ikey) >= 0 220 | // }) 221 | // } 222 | // 搜索 index 最小的，且最小值imin 大于或等于特定值 ikey 的 tFile 的索引 tfile.imin>=ikey 223 | pub fn search_min(&mut self, icmp: &IComparer, ikey: internalKey) -> usize { 224 | // 也就是寻找分割点 左边的是 tfile.imin < ikey，右边的是 tfile.imin >= ikey 225 | self.0 226 | .partition_point(|f| icmp.compare(f.imin.data(), ikey.data()) == Less) 227 | } 228 | 229 | // Searches smallest index of tables whose its largest 230 | // key is after or equal with given key. 231 | // func (tf tFiles) searchMax(icmp *iComparer, ikey internalKey) int { 232 | // return sort.Search(len(tf), func(i int) bool { 233 | // return icmp.Compare(tf[i].imax, ikey) >= 0 234 | // }) 235 | // } 236 | // 搜索 index 最小的，且最大值imax 大于或等于特定值 ikey 的 tFile 的索引, tfile.imax>=ikey 237 | pub fn search_max(&mut self, icmp: &IComparer, ikey: internalKey) -> usize { 238 | // 也就是寻找分割点 左边的是 tfile.imax < ikey，右边的是 tfile.imax >= ikey 239 | self.0 240 | .partition_point(|f| icmp.compare(f.imax.data(), ikey.data()) == Less) 241 | } 242 | 243 | // Searches smallest index of tables whose its file number 244 | // is smaller than the given number. 245 | // func (tf tFiles) searchNumLess(num int64) int { 246 | // return sort.Search(len(tf), func(i int) bool { 247 | // return tf[i].fd.Num < num 248 | // }) 249 | // } 250 | // 搜索 index 最小的，且 tfile.fd.num < num 251 | pub fn search_num_less(&mut self, num: i64) -> usize { 252 | // 也就是寻找分割点 左边的是 tfile.fd.num >= num 253 | self.0.partition_point(|f| f.fd.num >= num) 254 | } 255 | 256 | // Searches smallest index of tables whose its smallest 257 | // key is after the given key. 258 | // func (tf tFiles) searchMinUkey(icmp *iComparer, umin []byte) int { 259 | // return sort.Search(len(tf), func(i int) bool { 260 | // return icmp.ucmp.Compare(tf[i].imin.ukey(), umin) > 0 261 | // }) 262 | // } 263 | // 寻找 index 最小的tFile, tfile.imin.ukey()大于 ukey 264 | pub fn search_min_ukey(&self, icmp: &IComparer, ukey: &[u8]) -> usize { 265 | // 也就是寻找分割点 左边区域的是 tfile.imin.ukey() <= ukey 266 | self.0 267 | .partition_point(|f| icmp.u_compare(f.imin.ukey(), ukey) != Greater) 268 | } 269 | 270 | // Searches smallest index of tables whose its largest 271 | // key is after the given key. 272 | // func (tf tFiles) searchMaxUkey(icmp *iComparer, umax []byte) int { 273 | // return sort.Search(len(tf), func(i int) bool { 274 | // return icmp.ucmp.Compare(tf[i].imax.ukey(), umax) > 0 275 | // }) 276 | // } 277 | // 寻找 index 最小的tFile, tfile.imax.ukey()大于 ukey 278 | // 279 | pub fn search_max_ukey(&self, icmp: &IComparer, ukey: &[u8]) -> usize { 280 | // 也就是寻找分割点 左边区域的是 tfile.imax.ukey() <= ukey 281 | self.0 282 | .partition_point(|f| icmp.u_compare(f.imax.ukey(), ukey) != Greater) 283 | } 284 | 285 | // Returns true if given key range overlaps with one or more 286 | // tables key range. If unsorted is true then binary search will not be used. 287 | // func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) bool { 288 | // if unsorted { 289 | // // Check against all files. 290 | // for _, t := range tf { 291 | // if t.overlaps(icmp, umin, umax) { 292 | // return true 293 | // } 294 | // } 295 | // return false 296 | // } 297 | 298 | // i := 0 299 | // if len(umin) > 0 { 300 | // // Find the earliest possible internal key for min. 301 | // i = tf.searchMax(icmp, makeInternalKey(nil, umin, keyMaxSeq, keyTypeSeek)) 302 | // } 303 | // if i >= len(tf) { 304 | // // Beginning of range is after all files, so no overlap. 305 | // return false 306 | // } 307 | // return !tf[i].before(icmp, umax) 308 | // } 309 | // 判断区间 [umin,umax]是否有和其中某个tFile的区间有重叠 310 | fn overlaps( 311 | &mut self, 312 | icmp: &IComparer, 313 | umin: &[u8], 314 | umax: &[u8], 315 | unsorted: bool, 316 | ) -> bool { 317 | if unsorted { 318 | for t in &self.0 { 319 | if t.overlaps(icmp, umin, umax) { 320 | return true; 321 | } 322 | } 323 | return false; 324 | } 325 | 326 | let mut i = 0; 327 | if umin.len() > 0 { 328 | i = self.search_max(icmp, makeInternalKey(umin, KEY_MAX_SEQ, keyType::Seek)); 329 | } 330 | if i >= self.0.len() { 331 | return false; 332 | } 333 | return !self.0[i].before(icmp, umax); 334 | } 335 | 336 | // Returns tables whose its key range overlaps with given key range. 337 | // Range will be expanded if ukey found hop across tables. 338 | // If overlapped is true then the search will be restarted if umax 339 | // expanded. 340 | // The dst content will be overwritten. 341 | // func (tf tFiles) getOverlaps(dst tFiles, icmp *iComparer, umin, umax []byte, overlapped bool) tFiles { 342 | // // Short circuit if tf is empty 343 | // if len(tf) == 0 { 344 | // return nil 345 | // } 346 | // // For non-zero levels, there is no ukey hop across at all. 347 | // // And what's more, the files in these levels are strictly sorted, 348 | // // so use binary search instead of heavy traverse. 349 | // if !overlapped { 350 | // var begin, end int 351 | // // Determine the begin index of the overlapped file 352 | // if umin != nil { 353 | // index := tf.searchMinUkey(icmp, umin) 354 | // if index == 0 { 355 | // begin = 0 356 | // } else if bytes.Compare(tf[index-1].imax.ukey(), umin) >= 0 { 357 | // // The min ukey overlaps with the index-1 file, expand it. 358 | // begin = index - 1 359 | // } else { 360 | // begin = index 361 | // } 362 | // } 363 | // // Determine the end index of the overlapped file 364 | // if umax != nil { 365 | // index := tf.searchMaxUkey(icmp, umax) 366 | // if index == len(tf) { 367 | // end = len(tf) 368 | // } else if bytes.Compare(tf[index].imin.ukey(), umax) <= 0 { 369 | // // The max ukey overlaps with the index file, expand it. 370 | // end = index + 1 371 | // } else { 372 | // end = index 373 | // } 374 | // } else { 375 | // end = len(tf) 376 | // } 377 | // // Ensure the overlapped file indexes are valid. 378 | // if begin >= end { 379 | // return nil 380 | // } 381 | // dst = make([]*tFile, end-begin) 382 | // copy(dst, tf[begin:end]) 383 | // return dst 384 | // } 385 | 386 | // dst = dst[:0] 387 | // for i := 0; i < len(tf); { 388 | // t := tf[i] 389 | // if t.overlaps(icmp, umin, umax) { 390 | // if umin != nil && icmp.uCompare(t.imin.ukey(), umin) < 0 { 391 | // umin = t.imin.ukey() 392 | // dst = dst[:0] 393 | // i = 0 394 | // continue 395 | // } else if umax != nil && icmp.uCompare(t.imax.ukey(), umax) > 0 { 396 | // umax = t.imax.ukey() 397 | // // Restart search if it is overlapped. 398 | // dst = dst[:0] 399 | // i = 0 400 | // continue 401 | // } 402 | 403 | // dst = append(dst, t) 404 | // } 405 | // i++ 406 | // } 407 | 408 | // return dst 409 | // } 410 | 411 | // 返回和给定key区间有重叠的tFile列表 412 | // 413 | fn get_overlaps( 414 | &self, 415 | icmp: &IComparer, 416 | umin: &[u8], 417 | umax: &[u8], 418 | overlapped: bool, 419 | ) -> Self { 420 | let mut umin = umin; 421 | let mut umax = umax; 422 | let mut res = tFiles(vec![]); 423 | if self.0.len() == 0 { 424 | return res; 425 | } 426 | // 对于非0 层，ukey没有hop，这些文件都是严格排序的， 427 | if !overlapped { 428 | let mut begin = 0; 429 | let mut end = 0; 430 | if umin.len() > 0 { 431 | // 首先搜索最左边的，且文件的最小值大于umin的文件 432 | let index = self.search_min_ukey(icmp, &umin); 433 | if index == 0 { 434 | begin = index; 435 | // 如果 前一个文件的最大值大于等于umin,说明前一个文件 和区间 [umin,umax]重叠 436 | } else if icmp.u_compare(self.0[index - 1].imax.ukey(), umin) != Less { 437 | begin = index - 1; 438 | } else { 439 | begin = index; 440 | } 441 | } 442 | if umin.len() > 0 { 443 | // 首先搜索最左边的，且文件的最大值大于umax的文件 444 | let index = self.search_max_ukey(icmp, &umax); 445 | 446 | if index == self.0.len() { 447 | end = self.0.len(); 448 | // 如果当前文件的最小值小于等于umax, 说明这个文件和区间 [umin,umax]重叠 449 | } else if icmp.u_compare(self.0[index].imin.ukey(), umax) != Greater { 450 | end = index + 1; 451 | } else { 452 | end = index; 453 | } 454 | } else { 455 | end = self.0.len(); 456 | } 457 | 458 | if begin >= end { 459 | return res; 460 | } 461 | 462 | res.0.clone_from_slice(&self.0[begin..end]); 463 | } 464 | 465 | let mut i = 0; 466 | while i < self.0.len() { 467 | // 如果和区间重叠，则把两个区间合并，更新 umin umax 468 | if self.0[i].overlaps(icmp, umin, umax) { 469 | // 如果当前文件的最小值比 umin 还要小，更新umin 470 | if umin.len() > 0 && icmp.u_compare(self.0[i].imin.ukey(), umin) == Less { 471 | umin = self.0[i].imin.ukey(); 472 | res.0.clear(); 473 | i = 0; 474 | continue; 475 | // 如果当前文件的最大值比 umax 还要大，更新umax 476 | } else if umax.len() > 0 && icmp.u_compare(self.0[i].imax.ukey(), umax) == Greater { 477 | umax = self.0[i].imax.ukey(); 478 | res.0.clear(); 479 | i = 0; 480 | continue; 481 | } 482 | res.0.push(self.0[i].clone()); 483 | } 484 | i += 1; 485 | } 486 | 487 | res 488 | } 489 | 490 | // Returns tables key range. 491 | // func (tf tFiles) getRange(icmp *iComparer) (imin, imax internalKey) { 492 | // for i, t := range tf { 493 | // if i == 0 { 494 | // imin, imax = t.imin, t.imax 495 | // continue 496 | // } 497 | // if icmp.Compare(t.imin, imin) < 0 { 498 | // imin = t.imin 499 | // } 500 | // if icmp.Compare(t.imax, imax) > 0 { 501 | // imax = t.imax 502 | // } 503 | // } 504 | 505 | // return 506 | // } 507 | fn get_range(&self, icmp: &IComparer) -> (internalKey, internalKey) { 508 | // res.0 记录最小值的索引， res.1 记录最大值的索引 509 | let mut res = (0, 0); 510 | for (i, t) in self.0.iter().enumerate() { 511 | if icmp.compare(t.imin.data(), self.0[res.0].imin.data()) == Less { 512 | res.0 = i; 513 | } 514 | 515 | if icmp.compare(t.imax.data(), self.0[res.1].imax.data()) == Greater { 516 | res.1 = i 517 | } 518 | } 519 | (self.0[res.0].imin.clone(), self.0[res.1].imax.clone()) 520 | } 521 | } 522 | 523 | pub fn less_by_key(icmp: &IComparer, a: &tFile, b: &tFile) -> std::cmp::Ordering { 524 | let n = icmp.compare(a.imin.data(), b.imin.data()); 525 | if n == Equal { 526 | return a.fd.num.cmp(&b.fd.num); 527 | } 528 | n 529 | } 530 | --------------------------------------------------------------------------------