├── .gitignore ├── BUG_REPORT.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── FEATURE_REQUEST.md ├── LICENSE ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── README__V1.md ├── SECURITY.md ├── bench ├── bench.c ├── bench.go ├── bench_concurrent.c ├── bench_random.c ├── go.mod ├── go.sum └── readme.md ├── bloomfilter ├── bloomfilter.go └── bloomfilter_test.go ├── c ├── example.c ├── go.mod ├── go.sum ├── k4.go └── readme.md ├── compressor ├── compressor.go ├── compressor_test.go ├── test.png └── test2_public_domain.jpg ├── ffi ├── csharp │ ├── K4.cs │ ├── KeyValuePair.cs │ ├── KeyValuePairArray.cs │ └── readme.md ├── java │ ├── K4.java │ ├── KeyValuePair.java │ ├── KeyValuePairArray.java │ └── readme.md ├── lua │ ├── k4.lua │ └── readme.md ├── nodejs │ ├── .gitignore │ ├── k4.js │ ├── package-lock.json │ ├── package.json │ └── readme.md ├── python │ ├── k4.py │ └── readme.md ├── readme.md ├── ruby │ ├── k4.rb │ └── readme.md └── rust │ ├── k4.rs │ └── readme.md ├── fuzz ├── fuzz.go └── fuzz_test.go ├── go.mod ├── go.sum ├── graphics ├── bench-k4-2-1-6-chart-round.png ├── bench-k4-2-1-6-chart.png ├── k4-v2.png ├── k4-v2.svg ├── k4.png └── k4.svg ├── hashset ├── hashset.go └── hashset_test.go ├── k4.go ├── k4_bench_test.go ├── k4_test.go ├── murmur ├── murmur.go └── murmur_test.go ├── pager ├── pager.go └── pager_test.go ├── server_example ├── Dockerfile ├── go.mod ├── go.sum ├── main.go └── readme.md ├── skiplist ├── skiplist.go └── skiplist_test.go └── v2 ├── README.md ├── cuckoofilter ├── cuckoofilter.go └── cuckoofilter_test.go ├── go.mod ├── go.sum ├── k4.go ├── k4_bench_test.go ├── k4_test.go ├── pager ├── pager.go └── pager_test.go └── skiplist ├── skiplist.go └── skiplist_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | go.work.sum 23 | 24 | # env file 25 | .env 26 | .idea 27 | 28 | # server example data 29 | server_example/data 30 | c/data 31 | c/libk4.h.gch 32 | c/libk4.h 33 | c/libk4.so 34 | c/k4.h.gch 35 | c/example 36 | e/example.exe 37 | bench/bench 38 | bench/bench.exe 39 | bench/bench_random.c 40 | bench/bench_random.exe 41 | bench/testdb 42 | bench/bench_concurrent 43 | bench/bench_concurrent.exe -------------------------------------------------------------------------------- /BUG_REPORT.md: -------------------------------------------------------------------------------- 1 | ### Bug Report 2 | 3 | **Describe the bug** 4 | 5 | 6 | **Steps To Reproduce** 7 | 8 | **Expected behavior** 9 | 10 | 11 | **Screenshots (optional)** 12 | 13 | 14 | **K4 Version** 15 | 16 | 17 | **Additional context** 18 | 19 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Commitment 4 | 5 | We are dedicated to fostering an open and welcoming environment for everyone involved in our project and community. As contributors and maintainers, we pledge to ensure that participation is a harassment-free experience, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual orientation. 6 | 7 | ## Our Standards 8 | 9 | To cultivate a positive environment, we encourage behaviors such as: 10 | 11 | - Using inclusive and welcoming language 12 | - Respecting differing viewpoints and experiences 13 | - Gracefully accepting constructive criticism 14 | - Prioritizing the community’s well-being 15 | - Showing empathy toward fellow community members 16 | 17 | Conversely, the following behaviors are considered unacceptable: 18 | 19 | - Using sexualized language or imagery, or making unwelcome sexual advances 20 | - Engaging in trolling, derogatory comments, or personal attacks 21 | - Harassing others, either publicly or privately 22 | - Sharing others' private information without explicit permission 23 | - Any conduct deemed inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clearly communicating acceptable behavior standards and are expected to take fair and appropriate action against any unacceptable behavior. 28 | 29 | Maintainers reserve the right to remove, edit, or reject contributions that do not align with this Code of Conduct, and may temporarily or permanently ban contributors for behaviors deemed inappropriate, threatening, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies in all project spaces and extends to any situation where individuals represent the project or its community in public settings. This includes using official project email addresses, posting on official social media accounts, or acting as appointed representatives at events. Further clarifications may be provided by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | If you witness or experience abusive, harassing, or unacceptable behavior, please report it to the project team lead at me@alexpadula.com. All reports will be reviewed and investigated, resulting in appropriate responses based on the circumstances. We are committed to maintaining confidentiality for those reporting incidents. Additional enforcement details may be provided separately. 38 | 39 | Maintainers who fail to uphold or enforce the Code of Conduct in good faith may face temporary or permanent consequences, as determined by other members of the project leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the Contributor Covenant, version 1.4, which is available at [Contributor Covenant](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html). 44 | 45 | For answers to common questions regarding this Code of Conduct, please visit [FAQ](https://www.contributor-covenant.org/faq). -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to K4 2 | 3 | Thank you for your interest in contributing to the K4 project! We welcome contributions from the community and are grateful for any and all support. 4 | 5 | ## How to Contribute 6 | Firstly do review CONTRIBUTING.md and SECURITY.md. Once reviewed and understood follow fork, clone, branch, commit, push, and pull request workflow to contribute to the K4 project. 7 | -------------------------------------------------------------------------------- /FEATURE_REQUEST.md: -------------------------------------------------------------------------------- 1 | ### Feature Request 2 | 3 | **What feature are you requesting?** 4 | 5 | 6 | **Why is this feature important?** 7 | 8 | 9 | **Additional context** 10 | 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2024, Alex Gaetano Padula 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### PR Title 2 | 3 | 4 | 5 | ### Related Issue (optional) 6 | 7 | 8 | 9 | ### Type of Change 10 | 11 | - [ ] Bug fix 12 | - [ ] New feature 13 | - [ ] Breaking change 14 | - [ ] Read me update 15 | - [ ] Other 16 | 17 | ### Checklist 18 | 19 | - [ ] I have read the contributing guidelines. 20 | - [ ] I have updated the read me (if necessary). 21 | - [ ] I have tested my changes or added test coverage. 22 | 23 | ### Additional Notes 24 | 25 | -------------------------------------------------------------------------------- /README__V1.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |
4 | 5 | K4 is an open-source, high-performance, transactional, and durable storage engine based on an LSM (Log-Structured Merge) tree architecture. This design optimizes high-speed writes, making it ideal for modern data-intensive applications. 6 | 7 | ### Benchmarks 8 | ``` 9 | goos: linux 10 | goarch: amd64 11 | pkg: github.com/guycipher/k4 12 | cpu: 11th Gen Intel(R) Core(TM) i7-11700K @ 3.60GHz 13 | BenchmarkK4_Put 14 | BenchmarkK4_Put-16 158104 6862 ns/op # 145,000 ops/s 15 | 16 | RocksDB vs K4 17 | +=+=+=+=+=+=+=+ 18 | Both engines were used with default settings and similar configurations. 19 | **RocksDB v7.8.3** 1 million writes sequential key-value pairs default settings = 2.9s-3.1s 20 | **K4 v1.9.0** 1 million writes sequential key-value pairs default settings = 174.67ms-190.00ms 21 | ``` 22 | 23 | More detailed benchmarks can be found in [here](https://github.com/guycipher/k4/tree/main/bench) 24 | 25 | ### Features 26 | - High speed writes. Reads are also fast but writes are the primary focus. 27 | - Durability 28 | - Optimized for RAM and flash storage (SSD) 29 | - Variable length binary keys and values. Keys and their values can be any size that paired don't exceed available memory at time of operation. 30 | - Write-Ahead Logging (WAL). System writes PUT and DELETE operations to a log file before applying them to K4. 31 | - Atomic transactions. Multiple PUT and DELETE operations can be grouped together and applied atomically to K4. 32 | - Multi-threaded parallel paired compaction. SSTables are paired up during compaction and merged into a single SSTable(s). This reduces the number of SSTables and minimizes disk I/O for read operations. 33 | - Memtable implemented as a skip list. 34 | - Configurable memtable flush threshold 35 | - Configurable compaction interval (in seconds) 36 | - Configurable logging 37 | - Configurable skip list (max level and probability) 38 | - Optimized hashset for faster lookups. SSTable initial pages contain a hashset. The system uses the hashset to determine if a key is in the SSTable before scanning the SSTable. 39 | - Recovery from WAL 40 | - Granular page locking (sstables on scan are locked granularly) 41 | - Thread-safe (multiple readers, single writer) 42 | - TTL support (time to live). Keys can be set to expire after a certain time duration. 43 | - Murmur3 inspired hashing on compression and hash set 44 | - Optional compression support (Simple lightweight and optimized Lempel-Ziv 1977 inspired compression algorithm) 45 | - Background flushing and compaction operations for less blocking on read and write operations 46 | - Easy intuitive API(Get, Put, Delete, Range, NRange, GreaterThan, GreaterThanEq, LessThan, LessThanEq, NGet) 47 | - Iterator for iterating over key-value pairs in memtable and sstables with Next and Prev methods 48 | - Periodic pager write syncing to disk. Every 24576 writes or every minute the system will flush the file system's in-memory copy of recently written data to disk. 49 | - No dependencies 50 | 51 | ### C Library and FFIs 52 | K4 has a C library that can be used with FFIs (Foreign Function Interfaces) in other languages. The C library is built using the Go toolchain and can be found in the c directory. 53 | 54 | [Shared C Library](https://github.com/guycipher/k4/tree/main/c) 55 | 56 | #### FFIs 57 | These are FFI's using the shared K4 C library. They can be used as a building block using K4 as a storage engine in other languages. 58 | - [x] [Python](https://github.com/guycipher/k4/tree/main/ffi/python) 59 | - [x] [Ruby](https://github.com/guycipher/k4/tree/main/ffi/ruby) 60 | - [x] [Lua](https://github.com/guycipher/k4/tree/main/ffi/lua) 61 | - [x] [Java](https://github.com/guycipher/k4/tree/main/ffi/java) 62 | - [x] [Rust](https://github.com/guycipher/k4/tree/main/ffi/rust) 63 | - [x] [C#](https://github.com/guycipher/k4/tree/main/ffi/csharp) 64 | - [x] [Node.JS](https://github.com/guycipher/k4/tree/main/ffi/nodejs) 65 | 66 | 67 | ### Example usage 68 | This is **GO** code that demonstrates how to use K4. The code is simple and demonstrates PUT, GET, and DELETE operations. 69 | 70 | ```go 71 | import ( 72 | "github.com/guycipher/k4" 73 | "log" 74 | ) 75 | 76 | func main() { 77 | var err error 78 | directory := "./data" 79 | memtableFlushThreshold := (1024 * 1024) * 5 // 5MB 80 | compactionInterval := 3600 // 1 hour 81 | logging := true 82 | compression := false 83 | 84 | db, err := k4.Open(directory, memtableFlushThreshold, compactionInterval, logging, compression) 85 | if err != nil { 86 | log.Fatalf("Failed to open K4: %v", err) 87 | } 88 | 89 | defer db.Close() 90 | 91 | 92 | // Put 93 | // Putting the same key will update the value 94 | key := []byte("key") 95 | value := []byte("value") 96 | err = db.Put(key, value, nil) 97 | if err != nil { 98 | log.Fatalf("Failed to put key: %v", err) 99 | } 100 | 101 | // Get 102 | value, err = db.Get(key) 103 | if err != nil { 104 | log.Fatalf("Failed to get key: %v", err) 105 | } 106 | 107 | // Delete 108 | err = db.Delete(key) 109 | if err != nil { 110 | log.Fatalf("Failed to get key: %v", err) 111 | } 112 | } 113 | ``` 114 | 115 | ### Iteration 116 | To iterate over key-value pairs you can use an Iterator. 117 | Will iterate over key-value pairs in memtable then sstables. 118 | 119 | ```go 120 | 121 | it := NewIterator(db) 122 | 123 | for { 124 | key, value := it.Next() 125 | if key == nil { 126 | break 127 | } 128 | 129 | // .. You can also go back if you want 130 | key, value = it.Prev() 131 | if key == nil { 132 | break 133 | } 134 | } 135 | 136 | ``` 137 | 138 | ### Transactions 139 | Transactions are atomic and can be used to group multiple PUT and DELETE operations together. Transactions are committed atomically to K4. 140 | Transactions can be rolled back after their commited but before they are removed. 141 | Commits are first come first serve and are applied to K4 in the order they were committed. 142 | 143 | ```go 144 | txn := db.BeginTransaction() 145 | 146 | txn.AddOperation(k4.PUT, key, value) 147 | txn.AddOperation(k4.DELETE, key, nil) 148 | 149 | err = txn.Commit() 150 | 151 | // Once committed you can rollback before the transaction is removed 152 | // On commit error the transaction is automatically rolled back 153 | txn.Rollback() 154 | 155 | // Remove the transaction after commit or rollback 156 | txn.Remove() // txn now no longer usable nor existent 157 | 158 | ``` 159 | 160 | ### Recovery 161 | If you have a populated WAL file in the data directory but no data files aka sstables you can use `RecoverFromWAL()` which will replay the WAL file and populate K4. 162 | 163 | #### Example 164 | ```go 165 | func main() { 166 | directory := "./data" 167 | memtableFlushThreshold := 1024 * 1024 // 1MB 168 | compactionInterval := 3600 // 1 hour 169 | logging := true 170 | compression := false 171 | 172 | db, err := k4.Open(directory, memtableFlushThreshold, compactionInterval, logging, compression) 173 | if err != nil { 174 | log.Fatalf("Failed to open K4: %v", err) 175 | } 176 | 177 | defer db.Close() 178 | 179 | err := db.RecoverFromWAL() 180 | if err != nil { 181 | .. 182 | } 183 | 184 | // Continue as normal 185 | } 186 | ``` 187 | 188 | ### TTL 189 | TTL (time to live) when putting a key-value pair you can specify a time duration after which the key-value pair will be deleted. 190 | The system will mark the key with a tombstone and delete it during compaction and or flush operations. 191 | ```go 192 | key := []byte("key") 193 | value := []byte("value") 194 | ttl := 6 * time.Second 195 | 196 | err = db.Put(key, value, ttl) 197 | if err != nil { 198 | .. 199 | .. 200 | 201 | ``` 202 | 203 | ### Configuration recommendations 204 | 205 | It is advisable to configure the memtable flush threshold and compaction interval in accordance with your application's specific requirements. A minimum memtable flush size of 50-256 MB is suggested. 206 | 207 | Regarding compaction, a compaction interval of 1-6 hours is recommended, contingent upon storage usage and activity patterns. 208 | 209 | ### Compression 210 | Compression is optional and can be enabled or disabled when opening the K4 instance. 211 | Memtable keys and their values are not compressed. What is compressed is WAL entries and SSTable pages. 212 | Compression could save disk space and reduce disk I/O but it could also increase CPU usage and slow down read and write operations. 213 | 214 | 215 | ### API 216 | ```go 217 | 218 | // Open a new K4 instance 219 | // you can pass extra arguments to configure memtable such as 220 | // args[0] = max level, must be an int 221 | // args[1] = probability, must be a float64 222 | func Open(directory string, memtableFlushThreshold int, compactionInterval int, logging, compress bool, args ...interface{}) (*K4, error) 223 | 224 | // Close the K4 instance gracefully 225 | func (k4 *K4) Close() error 226 | 227 | // Put a key-value pair into the db 228 | func (k4 *K4) Put(key, value []byte, ttl *time.Duration) error 229 | 230 | // Get a value from the db 231 | func (k4 *K4) Get(key []byte) ([]byte, error) 232 | 233 | // Get all key-value pairs not equal to the key 234 | func (k4 *K4) NGet(key []byte) (*KeyValueArray, error) 235 | 236 | // Get all key-value pairs greater than the key 237 | func (k4 *K4) GreaterThan(key []byte) (*KeyValueArray, error) 238 | 239 | // Get all key-value pairs greater than or equal to the key 240 | func (k4 *K4) GreaterThanEq(key []byte) (*KeyValueArray, error) 241 | 242 | // Get all key-value pairs less than the key 243 | func (k4 *K4) LessThan(key []byte) (*KeyValueArray, error) 244 | 245 | // Get all key-value pairs less than or equal to the key 246 | func (k4 *K4) LessThanEq(key []byte) (*KeyValueArray, error) 247 | 248 | // Get all key-value pairs in the range of startKey and endKey 249 | func (k4 *K4) Range(startKey, endKey []byte) (*KeyValueArray, error) 250 | 251 | // Get all key-value pairs not in the range of startKey and endKey 252 | func (k4 *K4) NRange(startKey, endKey []byte) (*KeyValueArray, error) 253 | 254 | // Delete a key-value pair from the db 255 | func (k4 *K4) Delete(key []byte) error 256 | 257 | // Begin a transaction 258 | func (k4 *K4) BeginTransaction() *Transaction 259 | 260 | // Add a key-value pair to the transaction OPR_CODE can be PUT or DELETE 261 | func (txn *Transaction) AddOperation(op OPR_CODE, key, value []byte) 262 | 263 | // Remove a transaction after it has been committed 264 | func (txn *Transaction) Remove(k4 *K4) 265 | 266 | // Rollback a transaction (only after commit or error) 267 | func (txn *Transaction) Rollback(k4 *K4) error 268 | 269 | // Commit a transaction 270 | func (txn *Transaction) Commit(k4 *K4) error 271 | 272 | // Recover from WAL file 273 | // WAL file must be placed in the data directory 274 | func (k4 *K4) RecoverFromWAL() error 275 | ``` 276 | 277 | ### What can you build with K4? 278 | - Time series databases 279 | - Object storage 280 | - Key-value stores 281 | - Distributed databases 282 | - Blockchain 283 | - Cache 284 | - Log store 285 | - Document store 286 | - Graph databases 287 | - Search engines 288 | - Data warehouses 289 | - Data lakes 290 | - Relational databases 291 | 292 | and more! 293 | 294 | If your project is using K4, please let us know! We would love to hear about it and promote it here. 295 | 296 | ### Reporting bugs 297 | If you find a bug with K4 create an issue on this repository but please do not include any sensitive information in the issue. If you have a security concern please follow SECURITY.md. 298 | 299 | ### Contributing 300 | This repository for the K4 project welcomes contributions. Before submitting a pull request (PR), please ensure you have reviewed and adhered to our guidelines outlined in SECURITY.md and CODE_OF_CONDUCT.md. 301 | Following these documents is essential for maintaining a safe and respectful environment for all contributors. We encourage you to create well-structured PRs that address specific issues or enhancements, and to include relevant details in your description. Your contributions help improve K4, and we appreciate your commitment to fostering a collaborative and secure development process. Thank you for being part of the K4 project. 302 | 303 | ### License 304 | BSD 3-Clause License (https://opensource.org/licenses/BSD-3-Clause) -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | Reporting and Addressing Security Concerns 2 | To help us maintain a secure environment, please avoid opening GitHub issues or pull requests related to security problems, as this may expose sensitive information. Instead, we encourage you to contact the lead at me@alexpadula.com. Thank you for your cooperation! -------------------------------------------------------------------------------- /bench/bench.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #define DB_PATH "testdb" 10 | #define NUM_OPS 10000 11 | 12 | void benchmark_rocksdb(bool no_sync); 13 | void benchmark_lmdb(); 14 | void benchmark_k4(); 15 | 16 | int main() { 17 | benchmark_rocksdb(true); 18 | benchmark_lmdb(); 19 | benchmark_k4(); 20 | return 0; 21 | } 22 | 23 | void benchmark_k4() { 24 | void* db = db_open(DB_PATH, (1024*1024)*256, 3600, 0, 0); 25 | if (db == NULL) { 26 | fprintf(stderr, "Error opening K4 database\n"); 27 | return; 28 | } 29 | 30 | char key[20], value[20]; 31 | clock_t start, end; 32 | double cpu_time_used; 33 | 34 | // Benchmark Put 35 | start = clock(); 36 | for (int i = 0; i < NUM_OPS; i++) { 37 | sprintf(key, "key%d", i); 38 | sprintf(value, "value%d", i); 39 | db_put(db, key, strlen(key), value, strlen(value), 0); 40 | } 41 | end = clock(); 42 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 43 | printf("K4 Put: %f seconds\n", cpu_time_used); 44 | 45 | // Benchmark Get 46 | start = clock(); 47 | for (int i = 0; i < NUM_OPS; i++) { 48 | sprintf(key, "key%d", i); 49 | char* read_value = db_get(db, key, strlen(key)); 50 | free(read_value); 51 | } 52 | end = clock(); 53 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 54 | printf("K4 Get: %f seconds\n", cpu_time_used); 55 | 56 | // Benchmark Delete 57 | start = clock(); 58 | for (int i = 0; i < NUM_OPS; i++) { 59 | sprintf(key, "key%d", i); 60 | db_delete(db, key, strlen(key)); 61 | } 62 | end = clock(); 63 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 64 | printf("K4 Delete: %f seconds\n", cpu_time_used); 65 | 66 | db_close(db); 67 | 68 | remove(DB_PATH); 69 | } 70 | 71 | void benchmark_rocksdb(bool no_sync) { 72 | rocksdb_t *db; 73 | rocksdb_options_t *options = rocksdb_options_create(); 74 | rocksdb_options_set_create_if_missing(options, 1); 75 | char *err = NULL; 76 | 77 | db = rocksdb_open(options, DB_PATH, &err); 78 | if (err != NULL) { 79 | fprintf(stderr, "Error opening RocksDB: %s\n", err); 80 | return; 81 | } 82 | 83 | rocksdb_writeoptions_t *write_options = rocksdb_writeoptions_create(); 84 | rocksdb_writeoptions_set_sync(write_options, !no_sync); 85 | 86 | char key[20], value[20]; 87 | clock_t start, end; 88 | double cpu_time_used; 89 | 90 | // Benchmark Put 91 | start = clock(); 92 | for (int i = 0; i < NUM_OPS; i++) { 93 | sprintf(key, "key%d", i); 94 | sprintf(value, "value%d", i); 95 | rocksdb_put(db, write_options, key, strlen(key), value, strlen(value), &err); 96 | } 97 | end = clock(); 98 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 99 | printf("RocksDB Put: %f seconds\n", cpu_time_used); 100 | 101 | // Benchmark Get 102 | start = clock(); 103 | for (int i = 0; i < NUM_OPS; i++) { 104 | sprintf(key, "key%d", i); 105 | size_t read_len; 106 | char *read_value = rocksdb_get(db, rocksdb_readoptions_create(), key, strlen(key), &read_len, &err); 107 | free(read_value); 108 | } 109 | end = clock(); 110 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 111 | printf("RocksDB Get: %f seconds\n", cpu_time_used); 112 | 113 | // Benchmark Delete 114 | start = clock(); 115 | for (int i = 0; i < NUM_OPS; i++) { 116 | sprintf(key, "key%d", i); 117 | rocksdb_delete(db, write_options, key, strlen(key), &err); 118 | } 119 | end = clock(); 120 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 121 | printf("RocksDB Delete: %f seconds\n", cpu_time_used); 122 | 123 | rocksdb_writeoptions_destroy(write_options); 124 | rocksdb_close(db); 125 | rocksdb_options_destroy(options); 126 | 127 | remove(DB_PATH); 128 | } 129 | 130 | void benchmark_lmdb() { 131 | MDB_env *env; 132 | MDB_dbi dbi; 133 | MDB_val key, value; 134 | MDB_txn *txn; 135 | int rc; 136 | 137 | rc = mdb_env_create(&env); 138 | rc = mdb_env_set_maxdbs(env, 1); 139 | rc = mdb_env_open(env, DB_PATH, 0, 0664); 140 | rc = mdb_txn_begin(env, NULL, 0, &txn); 141 | rc = mdb_dbi_open(txn, NULL, 0, &dbi); 142 | rc = mdb_txn_commit(txn); 143 | 144 | char key_str[20], value_str[20]; 145 | clock_t start, end; 146 | double cpu_time_used; 147 | 148 | // Benchmark Put 149 | start = clock(); 150 | for (int i = 0; i < NUM_OPS; i++) { 151 | sprintf(key_str, "key%d", i); 152 | sprintf(value_str, "value%d", i); 153 | key.mv_size = strlen(key_str); 154 | key.mv_data = key_str; 155 | value.mv_size = strlen(value_str); 156 | value.mv_data = value_str; 157 | rc = mdb_txn_begin(env, NULL, 0, &txn); 158 | rc = mdb_put(txn, dbi, &key, &value, 0); 159 | rc = mdb_txn_commit(txn); 160 | } 161 | end = clock(); 162 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 163 | printf("LMDB Put: %f seconds\n", cpu_time_used); 164 | 165 | // Benchmark Get 166 | start = clock(); 167 | for (int i = 0; i < NUM_OPS; i++) { 168 | sprintf(key_str, "key%d", i); 169 | key.mv_size = strlen(key_str); 170 | key.mv_data = key_str; 171 | rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); 172 | rc = mdb_get(txn, dbi, &key, &value); 173 | rc = mdb_txn_commit(txn); 174 | } 175 | end = clock(); 176 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 177 | printf("LMDB Get: %f seconds\n", cpu_time_used); 178 | 179 | // Benchmark Delete 180 | start = clock(); 181 | for (int i = 0; i < NUM_OPS; i++) { 182 | sprintf(key_str, "key%d", i); 183 | key.mv_size = strlen(key_str); 184 | key.mv_data = key_str; 185 | rc = mdb_txn_begin(env, NULL, 0, &txn); 186 | rc = mdb_del(txn, dbi, &key, NULL); 187 | rc = mdb_txn_commit(txn); 188 | } 189 | end = clock(); 190 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 191 | printf("LMDB Delete: %f seconds\n", cpu_time_used); 192 | 193 | mdb_dbi_close(env, dbi); 194 | mdb_env_close(env); 195 | 196 | remove(DB_PATH); 197 | } -------------------------------------------------------------------------------- /bench/bench.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/guycipher/k4/v2" 7 | "log" 8 | "math/rand" 9 | "os" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | const ( 15 | DB_PATH = "testdb" 16 | ) 17 | 18 | func benchmarkK4(thread, numOps, flushThreshold, compactionInterval int) { 19 | 20 | db, err := k4.Open(DB_PATH, flushThreshold, compactionInterval, false, false) 21 | if err != nil { 22 | log.Fatalf("Error opening K4 database: %v", err) 23 | } 24 | defer db.Close() 25 | 26 | key := make([]byte, 20) 27 | value := make([]byte, 20) 28 | 29 | // Benchmark Put 30 | start := time.Now() 31 | for i := 0; i < numOps; i++ { 32 | key = []byte(fmt.Sprintf("key%d", i)) 33 | value = []byte(fmt.Sprintf("value%d", i)) 34 | if err := db.Put(key, value, nil); err != nil { 35 | log.Fatalf("Error putting key: %v", err) 36 | } 37 | } 38 | cpuTimeUsed := time.Since(start).Seconds() 39 | fmt.Printf("K4 Put(%d): %f seconds\n", thread, cpuTimeUsed) 40 | 41 | // Benchmark Get 42 | start = time.Now() 43 | for i := 0; i < numOps; i++ { 44 | key = []byte(fmt.Sprintf("key%d", i)) 45 | if _, err := db.Get(key); err != nil { 46 | log.Fatalf("Error getting key: %v", err) 47 | } 48 | } 49 | cpuTimeUsed = time.Since(start).Seconds() 50 | fmt.Printf("K4 Get(%d): %f seconds\n", thread, cpuTimeUsed) 51 | 52 | // Benchmark Delete 53 | start = time.Now() 54 | for i := 0; i < numOps; i++ { 55 | key = []byte(fmt.Sprintf("key%d", i)) 56 | if err := db.Delete(key); err != nil { 57 | log.Fatalf("Error deleting key: %v", err) 58 | } 59 | } 60 | cpuTimeUsed = time.Since(start).Seconds() 61 | fmt.Printf("K4 Delete (%d): %f seconds\n", thread, cpuTimeUsed) 62 | 63 | os.RemoveAll(DB_PATH) 64 | } 65 | 66 | func benchmarkK4Random(numOps, flushThreshold, compactionInterval int) { 67 | db, err := k4.Open(DB_PATH, flushThreshold, compactionInterval, false, false) 68 | if err != nil { 69 | log.Fatalf("Error opening K4 database: %v", err) 70 | } 71 | defer db.Close() 72 | 73 | key := make([]byte, 20) 74 | value := make([]byte, 20) 75 | 76 | // Seed the random number generator 77 | rand.Seed(time.Now().UnixNano()) 78 | 79 | // Benchmark Put 80 | start := time.Now() 81 | for i := 0; i < numOps; i++ { 82 | key = []byte(fmt.Sprintf("key%d", rand.Intn(numOps))) 83 | value = []byte(fmt.Sprintf("value%d", i)) 84 | if err := db.Put(key, value, nil); err != nil { 85 | log.Fatalf("Error putting key: %v", err) 86 | } 87 | } 88 | cpuTimeUsed := time.Since(start).Seconds() 89 | fmt.Printf("K4 Put: %f seconds\n", cpuTimeUsed) 90 | 91 | // Benchmark Get 92 | start = time.Now() 93 | for i := 0; i < numOps; i++ { 94 | key = []byte(fmt.Sprintf("key%d", rand.Intn(numOps))) 95 | if _, err := db.Get(key); err != nil { 96 | log.Fatalf("Error getting key: %v", err) 97 | } 98 | } 99 | cpuTimeUsed = time.Since(start).Seconds() 100 | fmt.Printf("K4 Get: %f seconds\n", cpuTimeUsed) 101 | 102 | // Benchmark Delete 103 | start = time.Now() 104 | for i := 0; i < numOps; i++ { 105 | key = []byte(fmt.Sprintf("key%d", rand.Intn(numOps))) 106 | if err := db.Delete(key); err != nil { 107 | log.Fatalf("Error deleting key: %v", err) 108 | } 109 | } 110 | cpuTimeUsed = time.Since(start).Seconds() 111 | fmt.Printf("K4 Delete: %f seconds\n", cpuTimeUsed) 112 | 113 | os.RemoveAll(DB_PATH) 114 | } 115 | 116 | func benchmarkK4Concurrent(numOps, numThreads, flushThreshold, compactionInterval int) { 117 | var wg sync.WaitGroup 118 | wg.Add(numThreads) 119 | 120 | for i := 0; i < numThreads; i++ { 121 | go func() { 122 | defer wg.Done() 123 | 124 | benchmarkK4(i, numOps, flushThreshold, compactionInterval) 125 | }() 126 | } 127 | 128 | wg.Wait() 129 | } 130 | 131 | func main() { 132 | numOps := flag.Int("num_ops", 10000, "number of operations") 133 | numThreads := flag.Int("num_threads", 4, "number of threads for concurrent operations") 134 | flushSize := flag.Int("flush_threshold", (1024*1024)*128, "flush threshold in bytes") 135 | compactionInterval := flag.Int("compaction_interval", 3600, "compaction interval") 136 | 137 | flag.Parse() // parse the flags 138 | 139 | fmt.Println("Benchmarker started with the set parameters:") 140 | fmt.Printf("Number of operations: %d\n", *numOps) 141 | fmt.Printf("Number of threads: %d\n", *numThreads) 142 | 143 | fmt.Println() 144 | 145 | fmt.Println("Benchmarking K4 non concurrent operations") 146 | benchmarkK4(-1, *numOps, *flushSize, *compactionInterval) 147 | fmt.Println() 148 | fmt.Println("Benchmarking K4 random operations") 149 | benchmarkK4Random(*numOps, *flushSize, *compactionInterval) 150 | fmt.Println() 151 | fmt.Println("Benchmarking K4 concurrent operations") 152 | benchmarkK4Concurrent(*numOps, *numThreads, *flushSize, *compactionInterval) 153 | } 154 | -------------------------------------------------------------------------------- /bench/bench_concurrent.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define DB_PATH "testdb" 11 | #define NUM_OPS 10000 12 | #define NUM_THREADS 4 13 | 14 | void* benchmark_rocksdb(void* arg); 15 | void* benchmark_lmdb(void* arg); 16 | void* benchmark_k4(void* arg); 17 | void generate_random_key(char *key, int length); 18 | 19 | int main() { 20 | srand(time(NULL)); 21 | 22 | pthread_t threads[NUM_THREADS]; 23 | int i; 24 | 25 | // Create threads for RocksDB benchmark 26 | for (i = 0; i < NUM_THREADS; i++) { 27 | pthread_create(&threads[i], NULL, benchmark_rocksdb, NULL); 28 | } 29 | for (i = 0; i < NUM_THREADS; i++) { 30 | pthread_join(threads[i], NULL); 31 | } 32 | 33 | // Create threads for LMDB benchmark 34 | for (i = 0; i < NUM_THREADS; i++) { 35 | pthread_create(&threads[i], NULL, benchmark_lmdb, NULL); 36 | } 37 | for (i = 0; i < NUM_THREADS; i++) { 38 | pthread_join(threads[i], NULL); 39 | } 40 | 41 | // Create threads for K4 benchmark 42 | for (i = 0; i < NUM_THREADS; i++) { 43 | pthread_create(&threads[i], NULL, benchmark_k4, NULL); 44 | } 45 | for (i = 0; i < NUM_THREADS; i++) { 46 | pthread_join(threads[i], NULL); 47 | } 48 | 49 | return 0; 50 | } 51 | 52 | void generate_random_key(char *key, int length) { 53 | for (int i = 0; i < length - 1; i++) { 54 | key[i] = 'a' + rand() % 26; 55 | } 56 | key[length - 1] = '\0'; 57 | } 58 | 59 | void* benchmark_k4(void* arg) { 60 | void* db = db_open(DB_PATH, (1024*1024)*256, 3600, 0, 0); 61 | if (db == NULL) { 62 | fprintf(stderr, "Error opening K4 database\n"); 63 | return NULL; 64 | } 65 | 66 | char key[20], value[20]; 67 | clock_t start, end; 68 | double cpu_time_used; 69 | 70 | // Benchmark Put 71 | start = clock(); 72 | for (int i = 0; i < NUM_OPS; i++) { 73 | generate_random_key(key, sizeof(key)); 74 | sprintf(value, "value%d", i); 75 | db_put(db, key, strlen(key), value, strlen(value), 0); 76 | } 77 | end = clock(); 78 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 79 | printf("K4 Put: %f seconds\n", cpu_time_used); 80 | 81 | // Benchmark Get 82 | start = clock(); 83 | for (int i = 0; i < NUM_OPS; i++) { 84 | generate_random_key(key, sizeof(key)); 85 | char* read_value = db_get(db, key, strlen(key)); 86 | free(read_value); 87 | } 88 | end = clock(); 89 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 90 | printf("K4 Get: %f seconds\n", cpu_time_used); 91 | 92 | // Benchmark Delete 93 | start = clock(); 94 | for (int i = 0; i < NUM_OPS; i++) { 95 | generate_random_key(key, sizeof(key)); 96 | db_delete(db, key, strlen(key)); 97 | } 98 | end = clock(); 99 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 100 | printf("K4 Delete: %f seconds\n", cpu_time_used); 101 | 102 | db_close(db); 103 | 104 | remove(DB_PATH); 105 | return NULL; 106 | } 107 | 108 | void* benchmark_rocksdb(void* arg) { 109 | rocksdb_t *db; 110 | rocksdb_options_t *options = rocksdb_options_create(); 111 | rocksdb_options_set_create_if_missing(options, 1); 112 | char *err = NULL; 113 | 114 | db = rocksdb_open(options, DB_PATH, &err); 115 | if (err != NULL) { 116 | fprintf(stderr, "Error opening RocksDB: %s\n", err); 117 | return NULL; 118 | } 119 | 120 | char key[20], value[20]; 121 | clock_t start, end; 122 | double cpu_time_used; 123 | 124 | // Benchmark Put 125 | start = clock(); 126 | for (int i = 0; i < NUM_OPS; i++) { 127 | generate_random_key(key, sizeof(key)); 128 | sprintf(value, "value%d", i); 129 | rocksdb_put(db, rocksdb_writeoptions_create(), key, strlen(key), value, strlen(value), &err); 130 | } 131 | end = clock(); 132 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 133 | printf("RocksDB Put: %f seconds\n", cpu_time_used); 134 | 135 | // Benchmark Get 136 | start = clock(); 137 | for (int i = 0; i < NUM_OPS; i++) { 138 | generate_random_key(key, sizeof(key)); 139 | size_t read_len; 140 | char *read_value = rocksdb_get(db, rocksdb_readoptions_create(), key, strlen(key), &read_len, &err); 141 | free(read_value); 142 | } 143 | end = clock(); 144 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 145 | printf("RocksDB Get: %f seconds\n", cpu_time_used); 146 | 147 | // Benchmark Delete 148 | start = clock(); 149 | for (int i = 0; i < NUM_OPS; i++) { 150 | generate_random_key(key, sizeof(key)); 151 | rocksdb_delete(db, rocksdb_writeoptions_create(), key, strlen(key), &err); 152 | } 153 | end = clock(); 154 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 155 | printf("RocksDB Delete: %f seconds\n", cpu_time_used); 156 | 157 | rocksdb_close(db); 158 | rocksdb_options_destroy(options); 159 | 160 | remove(DB_PATH); 161 | return NULL; 162 | } 163 | 164 | void* benchmark_lmdb(void* arg) { 165 | MDB_env *env; 166 | MDB_dbi dbi; 167 | MDB_val key, value; 168 | MDB_txn *txn; 169 | int rc; 170 | 171 | rc = mdb_env_create(&env); 172 | rc = mdb_env_set_maxdbs(env, 1); 173 | rc = mdb_env_open(env, DB_PATH, 0, 0664); 174 | rc = mdb_txn_begin(env, NULL, 0, &txn); 175 | rc = mdb_dbi_open(txn, NULL, 0, &dbi); 176 | rc = mdb_txn_commit(txn); 177 | 178 | char key_str[20], value_str[20]; 179 | clock_t start, end; 180 | double cpu_time_used; 181 | 182 | // Benchmark Put 183 | start = clock(); 184 | for (int i = 0; i < NUM_OPS; i++) { 185 | generate_random_key(key_str, sizeof(key_str)); 186 | sprintf(value_str, "value%d", i); 187 | key.mv_size = strlen(key_str); 188 | key.mv_data = key_str; 189 | value.mv_size = strlen(value_str); 190 | value.mv_data = value_str; 191 | rc = mdb_txn_begin(env, NULL, 0, &txn); 192 | rc = mdb_put(txn, dbi, &key, &value, 0); 193 | rc = mdb_txn_commit(txn); 194 | } 195 | end = clock(); 196 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 197 | printf("LMDB Put: %f seconds\n", cpu_time_used); 198 | 199 | // Benchmark Get 200 | start = clock(); 201 | for (int i = 0; i < NUM_OPS; i++) { 202 | generate_random_key(key_str, sizeof(key_str)); 203 | key.mv_size = strlen(key_str); 204 | key.mv_data = key_str; 205 | rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); 206 | rc = mdb_get(txn, dbi, &key, &value); 207 | rc = mdb_txn_commit(txn); 208 | } 209 | end = clock(); 210 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 211 | printf("LMDB Get: %f seconds\n", cpu_time_used); 212 | 213 | // Benchmark Delete 214 | start = clock(); 215 | for (int i = 0; i < NUM_OPS; i++) { 216 | generate_random_key(key_str, sizeof(key_str)); 217 | key.mv_size = strlen(key_str); 218 | key.mv_data = key_str; 219 | rc = mdb_txn_begin(env, NULL, 0, &txn); 220 | rc = mdb_del(txn, dbi, &key, NULL); 221 | rc = mdb_txn_commit(txn); 222 | } 223 | end = clock(); 224 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 225 | printf("LMDB Delete: %f seconds\n", cpu_time_used); 226 | 227 | mdb_dbi_close(env, dbi); 228 | mdb_env_close(env); 229 | 230 | remove(DB_PATH); 231 | return NULL; 232 | } -------------------------------------------------------------------------------- /bench/bench_random.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #define DB_PATH "testdb" 10 | #define NUM_OPS 10000 11 | 12 | void benchmark_rocksdb(); 13 | void benchmark_lmdb(); 14 | void benchmark_k4(); 15 | void generate_random_key(char *key, int length); 16 | 17 | int main() { 18 | srand(time(NULL)); 19 | benchmark_rocksdb(); 20 | benchmark_lmdb(); 21 | benchmark_k4(); 22 | return 0; 23 | } 24 | 25 | void generate_random_key(char *key, int length) { 26 | for (int i = 0; i < length - 1; i++) { 27 | key[i] = 'a' + rand() % 26; 28 | } 29 | key[length - 1] = '\0'; 30 | } 31 | 32 | void benchmark_k4() { 33 | void* db = db_open(DB_PATH, (1024*1024)*256, 3600, 0, 0); 34 | if (db == NULL) { 35 | fprintf(stderr, "Error opening K4 database\n"); 36 | return; 37 | } 38 | 39 | char key[20], value[20]; 40 | clock_t start, end; 41 | double cpu_time_used; 42 | 43 | // Benchmark Put 44 | start = clock(); 45 | for (int i = 0; i < NUM_OPS; i++) { 46 | generate_random_key(key, sizeof(key)); 47 | sprintf(value, "value%d", i); 48 | db_put(db, key, strlen(key), value, strlen(value), 0); 49 | } 50 | end = clock(); 51 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 52 | printf("K4 Put: %f seconds\n", cpu_time_used); 53 | 54 | // Benchmark Get 55 | start = clock(); 56 | for (int i = 0; i < NUM_OPS; i++) { 57 | generate_random_key(key, sizeof(key)); 58 | char* read_value = db_get(db, key, strlen(key)); 59 | free(read_value); 60 | } 61 | end = clock(); 62 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 63 | printf("K4 Get: %f seconds\n", cpu_time_used); 64 | 65 | // Benchmark Delete 66 | start = clock(); 67 | for (int i = 0; i < NUM_OPS; i++) { 68 | generate_random_key(key, sizeof(key)); 69 | db_delete(db, key, strlen(key)); 70 | } 71 | end = clock(); 72 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 73 | printf("K4 Delete: %f seconds\n", cpu_time_used); 74 | 75 | db_close(db); 76 | 77 | remove(DB_PATH); 78 | } 79 | 80 | void benchmark_rocksdb() { 81 | rocksdb_t *db; 82 | rocksdb_options_t *options = rocksdb_options_create(); 83 | rocksdb_options_set_create_if_missing(options, 1); 84 | char *err = NULL; 85 | 86 | db = rocksdb_open(options, DB_PATH, &err); 87 | if (err != NULL) { 88 | fprintf(stderr, "Error opening RocksDB: %s\n", err); 89 | return; 90 | } 91 | 92 | char key[20], value[20]; 93 | clock_t start, end; 94 | double cpu_time_used; 95 | 96 | // Benchmark Put 97 | start = clock(); 98 | for (int i = 0; i < NUM_OPS; i++) { 99 | generate_random_key(key, sizeof(key)); 100 | sprintf(value, "value%d", i); 101 | rocksdb_put(db, rocksdb_writeoptions_create(), key, strlen(key), value, strlen(value), &err); 102 | } 103 | end = clock(); 104 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 105 | printf("RocksDB Put: %f seconds\n", cpu_time_used); 106 | 107 | // Benchmark Get 108 | start = clock(); 109 | for (int i = 0; i < NUM_OPS; i++) { 110 | generate_random_key(key, sizeof(key)); 111 | size_t read_len; 112 | char *read_value = rocksdb_get(db, rocksdb_readoptions_create(), key, strlen(key), &read_len, &err); 113 | free(read_value); 114 | } 115 | end = clock(); 116 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 117 | printf("RocksDB Get: %f seconds\n", cpu_time_used); 118 | 119 | // Benchmark Delete 120 | start = clock(); 121 | for (int i = 0; i < NUM_OPS; i++) { 122 | generate_random_key(key, sizeof(key)); 123 | rocksdb_delete(db, rocksdb_writeoptions_create(), key, strlen(key), &err); 124 | } 125 | end = clock(); 126 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 127 | printf("RocksDB Delete: %f seconds\n", cpu_time_used); 128 | 129 | rocksdb_close(db); 130 | rocksdb_options_destroy(options); 131 | 132 | remove(DB_PATH); 133 | } 134 | 135 | void benchmark_lmdb() { 136 | MDB_env *env; 137 | MDB_dbi dbi; 138 | MDB_val key, value; 139 | MDB_txn *txn; 140 | int rc; 141 | 142 | rc = mdb_env_create(&env); 143 | rc = mdb_env_set_maxdbs(env, 1); 144 | rc = mdb_env_open(env, DB_PATH, 0, 0664); 145 | rc = mdb_txn_begin(env, NULL, 0, &txn); 146 | rc = mdb_dbi_open(txn, NULL, 0, &dbi); 147 | rc = mdb_txn_commit(txn); 148 | 149 | char key_str[20], value_str[20]; 150 | clock_t start, end; 151 | double cpu_time_used; 152 | 153 | // Benchmark Put 154 | start = clock(); 155 | for (int i = 0; i < NUM_OPS; i++) { 156 | generate_random_key(key_str, sizeof(key_str)); 157 | sprintf(value_str, "value%d", i); 158 | key.mv_size = strlen(key_str); 159 | key.mv_data = key_str; 160 | value.mv_size = strlen(value_str); 161 | value.mv_data = value_str; 162 | rc = mdb_txn_begin(env, NULL, 0, &txn); 163 | rc = mdb_put(txn, dbi, &key, &value, 0); 164 | rc = mdb_txn_commit(txn); 165 | } 166 | end = clock(); 167 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 168 | printf("LMDB Put: %f seconds\n", cpu_time_used); 169 | 170 | // Benchmark Get 171 | start = clock(); 172 | for (int i = 0; i < NUM_OPS; i++) { 173 | generate_random_key(key_str, sizeof(key_str)); 174 | key.mv_size = strlen(key_str); 175 | key.mv_data = key_str; 176 | rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); 177 | rc = mdb_get(txn, dbi, &key, &value); 178 | rc = mdb_txn_commit(txn); 179 | } 180 | end = clock(); 181 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 182 | printf("LMDB Get: %f seconds\n", cpu_time_used); 183 | 184 | // Benchmark Delete 185 | start = clock(); 186 | for (int i = 0; i < NUM_OPS; i++) { 187 | generate_random_key(key_str, sizeof(key_str)); 188 | key.mv_size = strlen(key_str); 189 | key.mv_data = key_str; 190 | rc = mdb_txn_begin(env, NULL, 0, &txn); 191 | rc = mdb_del(txn, dbi, &key, NULL); 192 | rc = mdb_txn_commit(txn); 193 | } 194 | end = clock(); 195 | cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC; 196 | printf("LMDB Delete: %f seconds\n", cpu_time_used); 197 | 198 | mdb_dbi_close(env, dbi); 199 | mdb_env_close(env); 200 | 201 | remove(DB_PATH); 202 | } -------------------------------------------------------------------------------- /bench/go.mod: -------------------------------------------------------------------------------- 1 | module k4__bench 2 | 3 | go 1.23.3 4 | 5 | require github.com/guycipher/k4/v2 v2.1.8 6 | 7 | require github.com/guycipher/k4 v1.9.7 // indirect 8 | -------------------------------------------------------------------------------- /bench/go.sum: -------------------------------------------------------------------------------- 1 | github.com/guycipher/k4 v1.9.7 h1:elWXP9hB8XAvyXIUPRtr/0DVzGuZJGJAFzaCoRPY98o= 2 | github.com/guycipher/k4 v1.9.7/go.mod h1:j1faqUqiDiz9I7Qiu/2l5BjeRzkk/CssoDNm8pG4g8g= 3 | github.com/guycipher/k4/v2 v2.1.1 h1:VLqvBDF9JvRJAcqD0qUvy4rZ2JxqZj17ziavesYEsAc= 4 | github.com/guycipher/k4/v2 v2.1.1/go.mod h1:97sG2QHxtszTHCJFtS/DThKITDSWIBkEz7mC0DGWLUM= 5 | github.com/guycipher/k4/v2 v2.1.2 h1:qHxHBNxP3FgawwiNWYAB3ptmTefYSDPrvSc2KfNEbNQ= 6 | github.com/guycipher/k4/v2 v2.1.2/go.mod h1:chF/U//rx6YgoD3+SsShWekwI24XCA6PiJHzNavabFM= 7 | github.com/guycipher/k4/v2 v2.1.3 h1:vOIiDIHFFaK2UEJmLcD/cEKnKTOaiJj17X8OEVWp/tk= 8 | github.com/guycipher/k4/v2 v2.1.3/go.mod h1:chF/U//rx6YgoD3+SsShWekwI24XCA6PiJHzNavabFM= 9 | github.com/guycipher/k4/v2 v2.1.4 h1:3ufXELKQLQP3i8aWWdMM8KkCO2H52yWR91cKYlJejKQ= 10 | github.com/guycipher/k4/v2 v2.1.4/go.mod h1:chF/U//rx6YgoD3+SsShWekwI24XCA6PiJHzNavabFM= 11 | github.com/guycipher/k4/v2 v2.1.5 h1:nuV2TO+QQuY4lsxpSPGRXK72kL1VECgrxGlA1eXiZO8= 12 | github.com/guycipher/k4/v2 v2.1.5/go.mod h1:chF/U//rx6YgoD3+SsShWekwI24XCA6PiJHzNavabFM= 13 | github.com/guycipher/k4/v2 v2.1.6 h1:qm2y/LtxAJDXrf9QfcH4O43CtUH9n7F5HSSiK+FKtPo= 14 | github.com/guycipher/k4/v2 v2.1.6/go.mod h1:ylj3EpecOYJCxK3U60TagKARq7OetuutkAPW+QXD4LM= 15 | github.com/guycipher/k4/v2 v2.1.7 h1:5roJ1+lIFA053Tb7S3axM5TQiraFvf9NkIb0r8ZT5v4= 16 | github.com/guycipher/k4/v2 v2.1.7/go.mod h1:ylj3EpecOYJCxK3U60TagKARq7OetuutkAPW+QXD4LM= 17 | github.com/guycipher/k4/v2 v2.1.8 h1:jcTq7lk24mczPzmzY4/RXbH7SIgIK5XVVF4NeZ/3Xa0= 18 | github.com/guycipher/k4/v2 v2.1.8/go.mod h1:ylj3EpecOYJCxK3U60TagKARq7OetuutkAPW+QXD4LM= 19 | -------------------------------------------------------------------------------- /bench/readme.md: -------------------------------------------------------------------------------- 1 | ## K4 Benchmarking 2 | These programs benchmark Put, Get, and Delete operations against RocksDB, LMDB, and K4. 3 | 4 | Their written in C and GO. 5 | 6 | You must have RocksDB, LMDB, and K4 installed on your system. 7 | 8 | 9 | ### C Bench building 10 | ``` 11 | gcc -o bench bench.c -lrocksdb -llmdb -lk4 && ./bench 12 | gcc -o bench_random bench_random.c -lrocksdb -llmdb -lk4 && ./bench_random 13 | gcc -o bench_concurrent bench_concurrent.c -lrocksdb -llmdb -lk4 && ./bench_concurrent 14 | ``` 15 | 16 | ### GO Bench building 17 | ``` 18 | go build -o bench bench.go && ./bench -num_ops 10000 -num_threads 4 -flush_threshold $((1024*1024*128)) -compaction_interval 3600 19 | ``` 20 | 21 | 10000 operations are performed for each operation type. 22 | The shared C library isn't as optimized as the Go library, so the results may vary. 23 | ``` 24 | -- These are results from bench.c on below system -- 25 | 11th Gen Intel(R) Core(TM) i7-11700K @ 3.60GHz UBuntu with WDC WDS500G2B0A-00SM50(HDD) 26 | -+ssssssssssssssssssyyssss+- OS: Ubuntu 23.04 x86_64 27 | .ossssssssssssssssssdMMMNysssso. Kernel: 6.2.0-39-generic 28 | /ssssssssssshdmmNNmmyNMMMMhssssss/ Uptime: 9 days, 16 hours, 23 mins 29 | +ssssssssshmydMMMMMMMNddddyssssssss+ Packages: 3141 (dpkg), 29 (snap) 30 | /sssssssshNMMMyhhyyyyhmNMMMNhssssssss/ Shell: bash 5.2.15 31 | .ssssssssdMMMNhsssssssssshNMMMdssssssss. Resolution: 1080x1920, 1920x1080 32 | +sssshhhyNMMNyssssssssssssyNMMMysssssss+ DE: GNOME 44.3 33 | ossyNMMMNyMMhsssssssssssssshmmmhssssssso WM: Mutter 34 | ossyNMMMNyMMhsssssssssssssshmmmhssssssso WM Theme: Adwaita 35 | +sssshhhyNMMNyssssssssssssyNMMMysssssss+ Theme: Yaru [GTK2/3] 36 | .ssssssssdMMMNhsssssssssshNMMMdssssssss. Icons: Yaru [GTK2/3] 37 | /sssssssshNMMMyhhyyyyhdNMMMNhssssssss/ Terminal: gnome-terminal 38 | +sssssssssdmydMMMMMMMMddddyssssssss+ CPU: 11th Gen Intel i7-11700K (16) @ 4.900GH 39 | /ssssssssssshdmNNNNmyNMMMMhssssss/ GPU: AMD ATI Radeon RX 5500/5500M / Pro 5500 40 | .ossssssssssssssssssdMMMNysssso. GPU: Intel RocketLake-S GT1 [UHD Graphics 75 41 | -+sssssssssssssssssyyyssss+- GPU: NVIDIA GeForce GT 730 42 | `:+ssssssssssssssssss+:` Memory: 13541MiB / 47928MiB 43 | 44 | ------------------------------------ 45 | RocksDB Put: 0.027765 seconds 46 | RocksDB Get: 0.009911 seconds 47 | RocksDB Delete: 0.034122 seconds 48 | LMDB Put: 0.308468 seconds 49 | LMDB Get: 0.001778 seconds 50 | LMDB Delete: 0.299799 seconds 51 | K4-C Put: 0.802232 seconds 52 | K4-C Get: 0.671278 seconds 53 | K4-C Delete: 0.877823 seconds 54 | K4-GO Put: 0.007235 seconds 55 | K4-GO Get: 0.003845 seconds 56 | K4-GO Delete: 0.006657 seconds 57 | 58 | Random reads and writes 59 | ------------------------------------ 60 | RocksDB Put: 0.037424 seconds 61 | RocksDB Get: 0.011199 seconds 62 | RocksDB Delete: 0.033449 seconds 63 | LMDB Put: 0.281653 seconds 64 | LMDB Get: 0.004544 seconds 65 | LMDB Delete: 0.004241 seconds 66 | K4-C Put: 0.775244 seconds 67 | K4-C Get: 0.618421 seconds 68 | K4-C Delete: 0.865813 seconds 69 | K4-GO Put: 0.008732 seconds 70 | K4-GO Get: 0.005464 seconds 71 | K4-GO Delete: 0.008889 seconds 72 | 73 | Concurrent random reads and writes 74 | ------------------------------------ 75 | RocksDB Put: 0.032743 seconds 76 | RocksDB Get: 0.011951 seconds 77 | RocksDB Delete: 0.030388 seconds 78 | LMDB Put: 0.989418 seconds 79 | LMDB Get: 0.006578 seconds 80 | LMDB Put: 0.985441 seconds 81 | LMDB Get: 0.008586 seconds 82 | LMDB Put: 1.014217 seconds 83 | LMDB Get: 0.016622 seconds 84 | LMDB Put: 1.039773 seconds 85 | LMDB Delete: 0.181927 seconds 86 | LMDB Delete: 0.215883 seconds 87 | LMDB Get: 0.168402 seconds 88 | LMDB Delete: 0.183181 seconds 89 | LMDB Delete: 0.009368 seconds 90 | K4-C Put: 0.705238 seconds 91 | K4-C Put: 0.706127 seconds 92 | K4-C Put: 0.708089 seconds 93 | K4-C Put: 0.758977 seconds 94 | K4-C Get: 0.519570 seconds 95 | K4-C Get: 0.540054 seconds 96 | K4-C Get: 0.600310 seconds 97 | K4-C Get: 0.671604 seconds 98 | K4-C Delete: 0.665265 seconds 99 | K4-C Delete: 0.657199 seconds 100 | K4-C Delete: 0.664664 seconds 101 | K4-C Delete: 0.611774 seconds 102 | K4-GO Put: 0.019521 seconds 103 | K4-GO Put: 0.020370 seconds 104 | K4-GO Put: 0.020771 seconds 105 | K4-GO Put: 0.021061 seconds 106 | K4-GO Get: 0.005092 seconds 107 | K4-GO Get: 0.004579 seconds 108 | K4-GO Get: 0.004092 seconds 109 | K4-GO Get: 0.004963 seconds 110 | K4-GO Delete: 0.010580 seconds 111 | K4-GO Delete: 0.009743 seconds 112 | K4-GO Delete: 0.010905 seconds 113 | K4-GO Delete: 0.011150 seconds 114 | 115 | 116 | 117 | ``` -------------------------------------------------------------------------------- /bloomfilter/bloomfilter.go: -------------------------------------------------------------------------------- 1 | // Package bloomfilter 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | package bloomfilter 33 | 34 | import ( 35 | "bytes" 36 | "encoding/binary" 37 | "github.com/guycipher/k4/murmur" 38 | "math" 39 | ) 40 | 41 | const GROWTH_FACTOR = 1.5 42 | const SHOULD_GROW_THRESHOLD = 0.7 43 | 44 | // BloomFilter represents a Bloom filter. 45 | type BloomFilter struct { 46 | bitset []bool // Bitset for the Bloom filter 47 | size uint // Current size of the Bloom filter 48 | hashFuncs []func([]byte, uint64) uint64 // Hash functions 49 | keys [][]byte // Original keys, maintained for resizing accuracy 50 | } 51 | 52 | // NewBloomFilter initializes a new BloomFilter. 53 | func NewBloomFilter(size uint, numHashFuncs int) *BloomFilter { 54 | bf := &BloomFilter{ 55 | bitset: make([]bool, size), // Initialize the bitset 56 | size: size, // Set the initial size 57 | hashFuncs: make([]func([]byte, uint64) uint64, numHashFuncs), // Initialize the hash functions 58 | keys: make([][]byte, 0), // Initialize the keys slice 59 | } 60 | 61 | // Initialize the hash functions 62 | for i := 0; i < numHashFuncs; i++ { 63 | bf.hashFuncs[i] = murmur.Hash64 // Use murmur hash function 64 | } 65 | 66 | return bf // Return the BloomFilter 67 | } 68 | 69 | // Add adds a key to the BloomFilter. 70 | func (bf *BloomFilter) Add(key []byte) { 71 | 72 | // check if the BloomFilter should grow 73 | if bf.shouldGrow() { 74 | bf.resize(uint(float64(bf.size) * GROWTH_FACTOR)) // Resize using the growth factor 75 | } 76 | 77 | // Add the key to the BloomFilter 78 | for _, hashFunc := range bf.hashFuncs { 79 | index := hashFunc(key, 0) % uint64(bf.size) // Use the current size 80 | bf.bitset[index] = true // Set the bit 81 | } 82 | bf.keys = append(bf.keys, key) // Add the key to the keys slice 83 | } 84 | 85 | // Check checks if a key is possibly in the BloomFilter. 86 | func (bf *BloomFilter) Check(key []byte) bool { 87 | 88 | // Check if the key is possibly in the BloomFilter 89 | for _, hashFunc := range bf.hashFuncs { 90 | // Calculate the index using the hash function and the current size 91 | index := hashFunc(key, 0) % uint64(bf.size) 92 | 93 | // If the bit is not set, the key is definitely not in the BloomFilter 94 | if !bf.bitset[index] { 95 | return false 96 | } 97 | } 98 | return true 99 | } 100 | 101 | // resize resizes the BloomFilter to a new size. 102 | func (bf *BloomFilter) resize(newSize uint) { 103 | newBitset := make([]bool, newSize) 104 | 105 | // Calculate the optimal number of hash functions 106 | numKeys := len(bf.keys) 107 | newNumHashFuncs := int(math.Ceil(float64(newSize) / float64(numKeys) * math.Ln2)) 108 | 109 | // Reinitialize the hash functions 110 | bf.hashFuncs = make([]func([]byte, uint64) uint64, newNumHashFuncs) 111 | for i := 0; i < newNumHashFuncs; i++ { 112 | bf.hashFuncs[i] = murmur.Hash64 113 | } 114 | 115 | // Re-add all keys 116 | for _, key := range bf.keys { 117 | // Add the key to the new BloomFilter 118 | for _, hashFunc := range bf.hashFuncs { 119 | index := hashFunc(key, 0) % uint64(newSize) // Use the new size 120 | newBitset[index] = true // Set the bit 121 | } 122 | } 123 | 124 | // Update the BloomFilter 125 | bf.bitset = newBitset 126 | bf.size = newSize 127 | } 128 | 129 | // shouldGrow checks if the BloomFilter should grow. 130 | func (bf *BloomFilter) shouldGrow() bool { 131 | setBits := 0 // Number of set bits 132 | for _, bit := range bf.bitset { 133 | if bit { 134 | setBits++ // Increment the number of set bits 135 | } 136 | } 137 | return setBits > int(float64(bf.size)*SHOULD_GROW_THRESHOLD) // Check if the number of set bits is greater than the threshold 138 | } 139 | 140 | // Serialize serializes the BloomFilter to a byte slice 141 | func (bf *BloomFilter) Serialize() ([]byte, error) { 142 | var buf bytes.Buffer 143 | 144 | // Write the size of the BloomFilter as uint32 145 | if err := binary.Write(&buf, binary.LittleEndian, uint32(bf.size)); err != nil { 146 | return nil, err 147 | } 148 | 149 | // Write the number of hash functions 150 | numHashFuncs := int32(len(bf.hashFuncs)) // Get the number of hash functions 151 | if err := binary.Write(&buf, binary.LittleEndian, numHashFuncs); err != nil { 152 | return nil, err 153 | } 154 | 155 | // Convert bitset to byte slice and write it 156 | bitsetBytes := make([]byte, (bf.size+7)/8) // Initialize the byte slice 157 | for i, bit := range bf.bitset { // Iterate over the bitset 158 | if bit { 159 | bitsetBytes[i/8] |= 1 << (i % 8) // Set the i-th bit 160 | } 161 | } 162 | 163 | // Write the bitset 164 | if _, err := buf.Write(bitsetBytes); err != nil { 165 | return nil, err 166 | } 167 | 168 | return buf.Bytes(), nil 169 | } 170 | 171 | // Deserialize deserializes a byte slice to a BloomFilter 172 | func Deserialize(data []byte) (*BloomFilter, error) { 173 | buf := bytes.NewReader(data) 174 | 175 | // Read the size of the BloomFilter as uint32 176 | var size uint32 177 | if err := binary.Read(buf, binary.LittleEndian, &size); err != nil { 178 | return nil, err 179 | } 180 | 181 | // Read the number of hash functions 182 | var numHashFuncs int32 183 | if err := binary.Read(buf, binary.LittleEndian, &numHashFuncs); err != nil { 184 | return nil, err 185 | } 186 | 187 | // Read the bitset 188 | bitsetBytes := make([]byte, (size+7)/8) 189 | if _, err := buf.Read(bitsetBytes); err != nil { 190 | return nil, err 191 | } 192 | bitset := make([]bool, size) // Initialize the bitset 193 | 194 | // Convert the byte slice to a bitset 195 | for i := range bitset { 196 | bitset[i] = (bitsetBytes[i/8] & (1 << (i % 8))) != 0 // Check if the i-th bit is set 197 | } 198 | 199 | // Reinitialize the hash functions 200 | hashFuncs := make([]func([]byte, uint64) uint64, numHashFuncs) 201 | for i := 0; i < int(numHashFuncs); i++ { 202 | hashFuncs[i] = murmur.Hash64 // Use murmur hash function 203 | } 204 | 205 | return &BloomFilter{ 206 | bitset: bitset, // Set the bitset 207 | size: uint(size), // Set the size 208 | hashFuncs: hashFuncs, // Set the hash functions 209 | keys: make([][]byte, 0), // Initialize keys slice 210 | }, nil 211 | } 212 | -------------------------------------------------------------------------------- /bloomfilter/bloomfilter_test.go: -------------------------------------------------------------------------------- 1 | // Package bloomfilter tests 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package bloomfilter 32 | 33 | import ( 34 | "fmt" 35 | "github.com/guycipher/k4/fuzz" 36 | "github.com/guycipher/k4/pager" 37 | "os" 38 | "testing" 39 | "time" 40 | ) 41 | 42 | func TestNewBloomFilter(t *testing.T) { 43 | size := uint(100) // Set a small size for testing 44 | numHashFuncs := 3 // Set a small number of hash functions for testing 45 | 46 | bf := NewBloomFilter(size, numHashFuncs) 47 | 48 | // Check if the BloomFilter is initialized correctly 49 | if len(bf.bitset) != int(size) { 50 | t.Errorf("Expected bitset size %d, got %d", size, len(bf.bitset)) 51 | } 52 | 53 | // Check if the hash functions are initialized correctly 54 | if len(bf.hashFuncs) != numHashFuncs { 55 | t.Errorf("Expected %d hash functions, got %d", numHashFuncs, len(bf.hashFuncs)) 56 | } 57 | 58 | // Check if the hash functions are initialized correctly 59 | for _, hashFunc := range bf.hashFuncs { 60 | if hashFunc == nil { 61 | t.Error("Expected hash function to be initialized, got nil") 62 | } 63 | } 64 | } 65 | 66 | func TestCheck(t *testing.T) { 67 | bf := NewBloomFilter(100, 3) // Small size for testing 68 | 69 | // Create two keys 70 | key := []byte("testkey") 71 | otherKey := []byte("otherkey") 72 | 73 | bf.Add(key) // Add key to BloomFilter 74 | 75 | // Check if the key is present in the BloomFilter 76 | if !bf.Check(key) { 77 | t.Error("Expected key to be present in BloomFilter, got not present") 78 | } 79 | 80 | // Check if the otherKey is not present in the BloomFilter 81 | if bf.Check(otherKey) { 82 | t.Error("Expected otherKey to be not present in BloomFilter, got present") 83 | } 84 | } 85 | 86 | func TestFalsePositives(t *testing.T) { 87 | bf := NewBloomFilter(100, 3) 88 | keys := [][]byte{ 89 | []byte("key1"), 90 | []byte("key2"), 91 | []byte("key3"), 92 | } 93 | 94 | for _, key := range keys { 95 | bf.Add(key) 96 | } 97 | 98 | falseKey := []byte("falsekey") 99 | if bf.Check(falseKey) { 100 | t.Error("Expected falseKey to be not present in BloomFilter, got present") 101 | } 102 | } 103 | 104 | func TestAddAndCheckMultipleKeys(t *testing.T) { 105 | size := uint(10) // Small size for testing, the bloom filter should resize 106 | numHashFuncs := 8 107 | bf := NewBloomFilter(size, numHashFuncs) 108 | 109 | keys := make([][]byte, 1000) 110 | for i := 0; i < 1000; i++ { 111 | keys[i] = []byte(fmt.Sprintf("key%d", i)) 112 | } 113 | 114 | // Add each key to the BloomFilter 115 | for _, key := range keys { 116 | bf.Add(key) 117 | } 118 | 119 | // Check if each key is reported as present in the BloomFilter 120 | for _, key := range keys { 121 | if !bf.Check(key) { 122 | t.Errorf("Expected key %s to be present in BloomFilter, got not present", key) 123 | } 124 | } 125 | } 126 | 127 | func TestSerialize(t *testing.T) { 128 | bf := NewBloomFilter(100, 3) 129 | keys := [][]byte{ 130 | []byte("key1"), 131 | []byte("key2"), 132 | []byte("key3"), 133 | } 134 | 135 | for _, key := range keys { 136 | bf.Add(key) 137 | } 138 | 139 | data, err := bf.Serialize() 140 | if err != nil { 141 | t.Fatalf("Failed to serialize BloomFilter: %v", err) 142 | } 143 | 144 | if data == nil { 145 | t.Error("Expected serialized data to be non-nil") 146 | } 147 | } 148 | 149 | func TestDeserialize(t *testing.T) { 150 | bf := NewBloomFilter(100, 3) 151 | keys := [][]byte{ 152 | []byte("key1"), 153 | []byte("key2"), 154 | []byte("key3"), 155 | } 156 | 157 | for _, key := range keys { 158 | bf.Add(key) 159 | } 160 | 161 | data, err := bf.Serialize() 162 | if err != nil { 163 | t.Fatalf("Failed to serialize BloomFilter: %v", err) 164 | } 165 | 166 | deserializedBF, err := Deserialize(data) 167 | if err != nil { 168 | t.Fatalf("Failed to deserialize BloomFilter: %v", err) 169 | } 170 | 171 | for _, key := range keys { 172 | if !deserializedBF.Check(key) { 173 | t.Errorf("Expected key %s to be present in deserialized BloomFilter, got not present", key) 174 | } 175 | } 176 | 177 | // Check a key that was not added 178 | falseKey := []byte("falsekey") 179 | if deserializedBF.Check(falseKey) { 180 | t.Error("Expected falseKey to be not present in deserialized BloomFilter, got present") 181 | } 182 | } 183 | 184 | func TestCheck2(t *testing.T) { 185 | tt := time.Now() 186 | bf := NewBloomFilter(1000000, 8) 187 | 188 | for i := 0; i < 10000; i++ { 189 | key := []byte("key" + fmt.Sprintf("%d", i)) 190 | bf.Add(key) 191 | } 192 | 193 | t.Logf("Time to add 10k keys to bloomfilter: %v", time.Since(tt)) 194 | 195 | serialized, err := bf.Serialize() 196 | if err != nil { 197 | t.Fatalf("Failed to serialize BloomFilter: %v", err) 198 | } 199 | 200 | bf, err = Deserialize(serialized) 201 | if err != nil { 202 | t.Fatalf("Failed to deserialize BloomFilter: %v", err) 203 | } 204 | 205 | tt = time.Now() 206 | 207 | // check all keys 208 | for i := 0; i < 10000; i++ { 209 | key := []byte("key" + fmt.Sprintf("%d", i)) 210 | if !bf.Check(key) { 211 | t.Fatalf("Expected key %s to be present in BloomFilter, got not present", key) 212 | } 213 | } 214 | 215 | t.Logf("Time to check 10k keys in bloomfilter: %v", time.Since(tt)) 216 | } 217 | 218 | func TestCheck3(t *testing.T) { 219 | tt := time.Now() 220 | bf := NewBloomFilter(1000000, 8) 221 | 222 | for i := 0; i < 10000; i++ { 223 | key := []byte("key" + fmt.Sprintf("%d", i)) 224 | bf.Add(key) 225 | } 226 | 227 | t.Logf("Time to add 10k keys to bloomfilter: %v", time.Since(tt)) 228 | 229 | serialized, err := bf.Serialize() 230 | if err != nil { 231 | t.Fatalf("Failed to serialize BloomFilter: %v", err) 232 | } 233 | 234 | // We write to file 235 | f, err := os.OpenFile("bloomfilter.test", os.O_CREATE|os.O_RDWR, 0644) 236 | if err != nil { 237 | t.Fatalf("Failed to open file: %v", err) 238 | } 239 | 240 | defer os.Remove("bloomfilter.test") 241 | 242 | f.WriteAt(serialized, 0) 243 | 244 | // We read from file 245 | serialized = make([]byte, len(serialized)) 246 | 247 | _, err = f.ReadAt(serialized, 0) 248 | if err != nil { 249 | t.Fatalf("Failed to read from file: %v", err) 250 | } 251 | 252 | bf, err = Deserialize(serialized) 253 | if err != nil { 254 | t.Fatalf("Failed to deserialize BloomFilter: %v", err) 255 | } 256 | 257 | tt = time.Now() 258 | 259 | // check all keys 260 | for i := 0; i < 10000; i++ { 261 | key := []byte("key" + fmt.Sprintf("%d", i)) 262 | if !bf.Check(key) { 263 | t.Fatalf("Expected key %s to be present in BloomFilter, got not present", key) 264 | } 265 | } 266 | 267 | t.Logf("Time to check 10k keys in bloomfilter: %v", time.Since(tt)) 268 | } 269 | 270 | func TestCheck4(t *testing.T) { 271 | tt := time.Now() 272 | bf := NewBloomFilter(100, 8) 273 | 274 | var keys [][]byte 275 | 276 | for i := 0; i < 10000; i++ { 277 | str, _ := fuzz.RandomString(10) 278 | key := []byte(str) 279 | keys = append(keys, key) 280 | bf.Add(key) 281 | } 282 | 283 | t.Logf("Time to add 10k keys to bloomfilter: %v", time.Since(tt)) 284 | 285 | serialized, err := bf.Serialize() 286 | if err != nil { 287 | t.Fatalf("Failed to serialize BloomFilter: %v", err) 288 | } 289 | 290 | // We write to file 291 | p, err := pager.OpenPager("bloomfilter.test", os.O_CREATE|os.O_RDWR, 0644) 292 | if err != nil { 293 | t.Fatalf("Failed to open file: %v", err) 294 | } 295 | 296 | defer os.Remove("bloomfilter.test") 297 | defer p.Close() 298 | p.Write(serialized) 299 | 300 | // We read from file 301 | serialized = make([]byte, len(serialized)) 302 | 303 | _, err = p.GetPage(0) 304 | if err != nil { 305 | t.Fatalf("Failed to read from file: %v", err) 306 | } 307 | 308 | bf, err = Deserialize(serialized) 309 | if err != nil { 310 | t.Fatalf("Failed to deserialize BloomFilter: %v", err) 311 | } 312 | 313 | tt = time.Now() 314 | 315 | // check all keys 316 | for _, key := range keys { 317 | if !bf.Check(key) { 318 | t.Fatalf("Expected key %s to be present in BloomFilter, got not present", key) 319 | } 320 | } 321 | 322 | t.Logf("Time to check 10k keys in bloomfilter: %v", time.Since(tt)) 323 | } 324 | -------------------------------------------------------------------------------- /c/example.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Example using K4 shared storage engine library 3 | */ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | int main() { 11 | // Open database 12 | void* db = db_open("data", 1024, 60, 1, 1); 13 | if (db == NULL) { 14 | printf("Failed to open database\n"); 15 | return 1; 16 | } 17 | 18 | // Put key-value pair 19 | char* key = "key1"; 20 | char* value = "value1"; 21 | 22 | // If you want a TTL 23 | // 5000000000 is 5 seconds from now for example 24 | // ttl := 5 * time.Second in GO resolves to 5000000000 25 | 26 | if (db_put(db, key, strlen(key), value, strlen(value), -1) != 0) { // -1 means no ttl 27 | printf("Failed to put key-value pair\n"); 28 | db_close(db); 29 | return 1; 30 | } 31 | 32 | // Get value by key 33 | char* retrieved_value = db_get(db, key, strlen(key)); 34 | if (retrieved_value == NULL) { 35 | printf("Failed to get value\n"); 36 | db_close(db); 37 | return 1; 38 | } 39 | printf("Retrieved value: %s\n", retrieved_value); 40 | free(retrieved_value); 41 | 42 | // Delete key-value pair 43 | if (db_delete(db, key, strlen(key)) != 0) { 44 | printf("Failed to delete key-value pair\n"); 45 | db_close(db); 46 | return 1; 47 | } 48 | 49 | // Begin transaction 50 | void* txn = begin_transaction(db); 51 | if (txn == NULL) { 52 | printf("Failed to begin transaction\n"); 53 | db_close(db); 54 | return 1; 55 | } 56 | 57 | // Add operation to transaction 58 | char* key2 = "key2"; 59 | char* value2 = "value2"; 60 | // 0 means put operation 61 | // 1 means delete operation 62 | if (add_operation(txn, 0, key2, strlen(key2), value2, strlen(value2)) != 0) { 63 | printf("Failed to add operation to transaction\n"); 64 | rollback_transaction(txn, db); 65 | db_close(db); 66 | return 1; 67 | } 68 | 69 | printf("commiting txn\n"); 70 | 71 | // Commit transaction 72 | if (commit_transaction(txn, db) != 0) { 73 | printf("Failed to commit transaction\n"); 74 | rollback_transaction(txn, db); 75 | db_close(db); 76 | return 1; 77 | } 78 | 79 | printf("txn committed\n"); 80 | 81 | // Define start and end keys 82 | char* startKey = "key1"; 83 | int startLen = strlen(startKey); 84 | char* endKey = "key3"; 85 | int endLen = strlen(endKey); 86 | 87 | // Call range_ function 88 | struct KeyValuePairArray result = range_(db, startKey, startLen, endKey, endLen); 89 | if (result.pairs == NULL) { 90 | printf("Failed to get range\n"); 91 | db_close(db); 92 | return 1; 93 | } 94 | 95 | // Process the result 96 | for (int i = 0; i < result.numPairs; i++) { 97 | printf("Key: %s, Value: %s\n", result.pairs[i].key, result.pairs[i].value); 98 | free(result.pairs[i].key); 99 | free(result.pairs[i].value); 100 | } 101 | 102 | // Free the allocated memory for the result array 103 | free(result.pairs); 104 | 105 | // Close database 106 | if (db_close(db) != 0) { 107 | printf("Failed to close database\n"); 108 | return 1; 109 | } 110 | 111 | return 0; 112 | } -------------------------------------------------------------------------------- /c/go.mod: -------------------------------------------------------------------------------- 1 | module k4__c 2 | 3 | go 1.23.3 4 | 5 | require github.com/guycipher/k4/v2 v2.1.8 6 | 7 | require github.com/guycipher/k4 v1.9.7 // indirect 8 | -------------------------------------------------------------------------------- /c/go.sum: -------------------------------------------------------------------------------- 1 | github.com/guycipher/k4 v1.9.1 h1:3ttslx/OKqMexw23BfvIOvtf9MvejIpPGQtFYBIdqnQ= 2 | github.com/guycipher/k4 v1.9.1/go.mod h1:j1faqUqiDiz9I7Qiu/2l5BjeRzkk/CssoDNm8pG4g8g= 3 | github.com/guycipher/k4 v1.9.2 h1:8ijeP2gE87c1uSzPtdy1xxtJWF2TruWEGrmDdVIZKJ8= 4 | github.com/guycipher/k4 v1.9.2/go.mod h1:j1faqUqiDiz9I7Qiu/2l5BjeRzkk/CssoDNm8pG4g8g= 5 | github.com/guycipher/k4 v1.9.3 h1:cxn3slGV22ExCZPFne6qtJP+UsuuWOG7pyuhqiWLW0U= 6 | github.com/guycipher/k4 v1.9.3/go.mod h1:j1faqUqiDiz9I7Qiu/2l5BjeRzkk/CssoDNm8pG4g8g= 7 | github.com/guycipher/k4 v1.9.4 h1:PEv+qxgGYszWt2mzfQKHXiU7RgwCJEfYdQYWKUR9wPk= 8 | github.com/guycipher/k4 v1.9.4/go.mod h1:nY9tmdEu0jfDmTT2pFVHQhRi0RE5+jsz4Caa9jSub+Y= 9 | github.com/guycipher/k4 v1.9.7 h1:elWXP9hB8XAvyXIUPRtr/0DVzGuZJGJAFzaCoRPY98o= 10 | github.com/guycipher/k4 v1.9.7/go.mod h1:j1faqUqiDiz9I7Qiu/2l5BjeRzkk/CssoDNm8pG4g8g= 11 | github.com/guycipher/k4/v2 v2.1.5 h1:nuV2TO+QQuY4lsxpSPGRXK72kL1VECgrxGlA1eXiZO8= 12 | github.com/guycipher/k4/v2 v2.1.5/go.mod h1:chF/U//rx6YgoD3+SsShWekwI24XCA6PiJHzNavabFM= 13 | github.com/guycipher/k4/v2 v2.1.6 h1:qm2y/LtxAJDXrf9QfcH4O43CtUH9n7F5HSSiK+FKtPo= 14 | github.com/guycipher/k4/v2 v2.1.6/go.mod h1:ylj3EpecOYJCxK3U60TagKARq7OetuutkAPW+QXD4LM= 15 | github.com/guycipher/k4/v2 v2.1.7 h1:5roJ1+lIFA053Tb7S3axM5TQiraFvf9NkIb0r8ZT5v4= 16 | github.com/guycipher/k4/v2 v2.1.7/go.mod h1:ylj3EpecOYJCxK3U60TagKARq7OetuutkAPW+QXD4LM= 17 | github.com/guycipher/k4/v2 v2.1.8 h1:jcTq7lk24mczPzmzY4/RXbH7SIgIK5XVVF4NeZ/3Xa0= 18 | github.com/guycipher/k4/v2 v2.1.8/go.mod h1:ylj3EpecOYJCxK3U60TagKARq7OetuutkAPW+QXD4LM= 19 | -------------------------------------------------------------------------------- /c/readme.md: -------------------------------------------------------------------------------- 1 | # C library for K4 2 | 3 | ## Building C library 4 | To build the K4 shared C library you require the latest version of Go installed on your system. 5 | 6 | You can get the latest version of Go lang from here https://go.dev/dl/ 7 | 8 | Once you have Go installed you can continue. 9 | ``` 10 | cd c 11 | ``` 12 | 13 | Make sure you're in k4/c directory and run the following command to build the shared library. 14 | ``` 15 | go build -o libk4.so -buildmode=c-shared k4.go 16 | ``` 17 | 18 | Once run you should get 19 | ``` 20 | libk4.so 21 | libk4.h 22 | ``` 23 | 24 | On unix copy the files to /usr/local/lib and /usr/local/include 25 | ``` 26 | sudo cp libk4.so /usr/local/lib/ 27 | sudo cp libk4.h /usr/local/include/ 28 | ``` 29 | 30 | On windows copy the files to C:\Program Files\K4 31 | ``` 32 | copy libk4.so C:\Program Files\K4 33 | copy libk4.h C:\Program Files\K4 34 | ``` 35 | 36 | On unix update the library cache 37 | ``` 38 | sudo ldconfig 39 | ``` 40 | 41 | Verify install (unix) 42 | ``` 43 | ldconfig -p | grep libk4 44 | ``` 45 | 46 | Should see 47 | ``` 48 | libk4.so (libc6,x86-64) => /usr/local/lib/libk4.so 49 | ``` 50 | 51 | Now you should include the library in your C code 52 | ```c 53 | #include 54 | ``` 55 | 56 | Example compile command 57 | ``` 58 | cc -o example example.c -L/usr/local/lib -lk4 -I/usr/local/include 59 | ``` 60 | ### Note on TTL time 61 | If you want a TTL 62 | 5000000000 is 5 seconds from now for example 63 | 64 | `ttl := 5 * time.Second` in GO resolves to 5000000000 65 | 66 | ### API 67 | ```c 68 | void* db_open(char* directory, int memtableFlushThreshold, int compactionInterval, int logging, int compress); 69 | int db_close(void* dbPtr); 70 | int db_put(void* dbPtr, char* key, int keyLen, char* value, int valueLen, int64_t ttl); 71 | char* db_get(void* dbPtr, char* key, int keyLen); 72 | int db_delete(void* dbPtr, char* key, int keyLen); 73 | void* begin_transaction(void* dbPtr); 74 | int add_operation(void* txPtr, int operation, char* key, int keyLen, char* value, int valueLen); 75 | void remove_transaction(void* dbPtr, void* txPtr); 76 | int commit_transaction(void* txPtr, void* dbPtr); 77 | int rollback_transaction(void* txPtr, void* dbPtr); 78 | int recover_from_wal(void* dbPtr); 79 | struct KeyValuePairArray range_(void* dbPtr, char* start, int startLen, char* end, int endLen); 80 | struct KeyValuePairArray nrange(void* dbPtr, char* start, int startLen, char* end, int endLen); 81 | struct KeyValuePairArray greater_than(void* dbPtr, char* key, int keyLen); 82 | struct KeyValuePairArray less_than(void* dbPtr, char* key, int keyLen); 83 | struct KeyValuePairArray nget(void* dbPtr, char* key, int keyLen); 84 | struct KeyValuePairArray greater_than_eq(void* dbPtr, char* key, int keyLen); 85 | struct KeyValuePairArray less_than_eq(void* dbPtr, char* key, int keyLen); 86 | void* new_iterator(void* dbPtr); 87 | int escalate_flush(void* dbPtr); 88 | int escalate_compaction(void* dbPtr); 89 | 90 | 91 | /* Return type for iter_next */ 92 | struct iter_next_return { 93 | char* r0; 94 | char* r1; 95 | }; 96 | 97 | struct iter_next_return iter_next(void* iterPtr); 98 | 99 | /* Return type for iter_prev */ 100 | struct iter_prev_return { 101 | char* r0; 102 | char* r1; 103 | }; 104 | 105 | struct iter_prev_return iter_prev(void* iterPtr); 106 | void iter_reset(void* iterPtr); 107 | void iter_close(void* iterPtr); 108 | 109 | ``` 110 | 111 | #### Range example 112 | ```c 113 | #include 114 | #include 115 | #include 116 | #include 117 | 118 | int main() { 119 | // Open database 120 | void* db = db_open("data", 1024, 60, 1, 1); 121 | if (db == NULL) { 122 | printf("Failed to open database\n"); 123 | return 1; 124 | } 125 | 126 | 127 | // Define start and end keys 128 | char* startKey = "key1"; 129 | int startLen = strlen(startKey); 130 | 131 | char* endKey = "key3"; 132 | int endLen = strlen(endKey); 133 | 134 | // Call range_ function 135 | struct KeyValuePairArray result = range_(db, startKey, startLen, endKey, endLen); 136 | if (result.pairs == NULL) { 137 | printf("Failed to get range\n"); 138 | db_close(db); 139 | return 1; 140 | } 141 | 142 | // Process result 143 | for (int i = 0; i < result.numPairs; i++) { 144 | printf("Key: %s, Value: %s\n", result.pairs[i].key, result.pairs[i].value); 145 | free(result.pairs[i].key); 146 | free(result.pairs[i].value); 147 | } 148 | 149 | // Free the allocated memory for the result array 150 | free(result.pairs); 151 | 152 | // Close database 153 | if (db_close(db) != 0) { 154 | printf("Failed to close database\n"); 155 | return 1; 156 | } 157 | 158 | return 0; 159 | ``` 160 | 161 | #### Iterator example 162 | ```c 163 | #include 164 | #include 165 | #include 166 | #include 167 | 168 | int main() { 169 | // Open database 170 | void* db = db_open("data", 1024, 60, 1, 1); 171 | if (db == NULL) { 172 | printf("Failed to open database\n"); 173 | return 1; 174 | } 175 | 176 | // Create a new iterator 177 | void* iter = new_iterator(db); 178 | if (iter == NULL) { 179 | printf("Failed to create iterator\n"); 180 | db_close(db); 181 | return 1; 182 | } 183 | 184 | // Iterate forward through the database 185 | struct iter_next_return next; 186 | 187 | while ((next = iter_next(iter)).r0 != NULL) { 188 | printf("Key: %s, Value: %s\n", next.r0, next.r1); 189 | free(next.r0); 190 | free(next.r1); 191 | } 192 | 193 | // Reset the iterator 194 | iter_reset(iter); 195 | 196 | // Iterate backward through the database 197 | struct iter_prev_return prev; 198 | 199 | while ((prev = iter_prev(iter)).r0 != NULL) { 200 | printf("Key: %s, Value: %s\n", prev.r0, prev.r1); 201 | free(prev.r0); 202 | free(prev.r1); 203 | } 204 | 205 | // Close the iterator 206 | iter_close(iter); 207 | 208 | // Close database 209 | if (db_close(db) != 0) { 210 | printf("Failed to close database\n"); 211 | return 1; 212 | } 213 | 214 | return 0; 215 | } 216 | ``` -------------------------------------------------------------------------------- /compressor/compressor.go: -------------------------------------------------------------------------------- 1 | // Package compressor 2 | // A simple Lempel-Ziv 1977 inspired compression algorithm 3 | // BSD 3-Clause License 4 | // 5 | // Copyright (c) 2024, Alex Gaetano Padula 6 | // All rights reserved. 7 | // 8 | // Redistribution and use in source and binary forms, with or without 9 | // modification, are permitted provided that the following conditions are met: 10 | // 11 | // 1. Redistributions of source code must retain the above copyright notice, this 12 | // list of conditions and the following disclaimer. 13 | // 14 | // 2. Redistributions in binary form must reproduce the above copyright notice, 15 | // this list of conditions and the following disclaimer in the documentation 16 | // and/or other materials provided with the distribution. 17 | // 18 | // 3. Neither the name of the copyright holder nor the names of its 19 | // contributors may be used to endorse or promote products derived from 20 | // this software without specific prior written permission. 21 | // 22 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 26 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 28 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | package compressor 33 | 34 | import ( 35 | "bytes" 36 | "encoding/binary" 37 | "fmt" 38 | "github.com/guycipher/k4/murmur" 39 | ) 40 | 41 | // Compressor is the main compression package struct 42 | type Compressor struct { 43 | windowSize int // defined window size for compression 44 | } 45 | 46 | // NewCompressor initiates a new compressor with provided window size spec 47 | func NewCompressor(windowSize int) (*Compressor, error) { 48 | if windowSize <= 0 { 49 | return nil, fmt.Errorf("window size must be greater than 0") 50 | } 51 | 52 | return &Compressor{windowSize: windowSize}, nil 53 | } 54 | 55 | // Compress compresses the provided binary array/slice 56 | func (c *Compressor) Compress(data []byte) []byte { 57 | var compressed bytes.Buffer 58 | dataLen := len(data) 59 | i := 0 60 | hashTable := make(map[uint64]int) 61 | 62 | for i < dataLen { 63 | matchLength, matchDistance := 0, 0 64 | if i+2 < dataLen { 65 | hashKey := murmur.Hash64(data[i:i+3], 0) 66 | 67 | if pos, found := hashTable[hashKey]; found && i-pos <= c.windowSize { 68 | j := 0 69 | for j < dataLen-i && data[pos+j] == data[i+j] { 70 | j++ 71 | } 72 | matchLength = j 73 | matchDistance = i - pos 74 | } 75 | hashTable[hashKey] = i 76 | } 77 | 78 | if matchLength > 0 { 79 | binary.Write(&compressed, binary.BigEndian, uint16(matchDistance)) 80 | compressed.WriteByte(byte(matchLength)) 81 | i += matchLength 82 | } else { 83 | binary.Write(&compressed, binary.BigEndian, uint16(0)) 84 | compressed.WriteByte(data[i]) 85 | i++ 86 | } 87 | } 88 | 89 | return compressed.Bytes() 90 | } 91 | 92 | // Decompress decompresses the provided binary array/slice 93 | func (c *Compressor) Decompress(data []byte) []byte { 94 | var decompressed bytes.Buffer 95 | dataLen := len(data) 96 | i := 0 97 | hashTable := make(map[uint64]int) 98 | 99 | for i < dataLen { 100 | var matchDistance uint16 101 | binary.Read(bytes.NewReader(data[i:i+2]), binary.BigEndian, &matchDistance) 102 | matchLength := int(data[i+2]) 103 | i += 3 104 | 105 | if matchDistance > 0 { 106 | start := decompressed.Len() - int(matchDistance) 107 | for j := 0; j < matchLength; j++ { 108 | decompressed.WriteByte(decompressed.Bytes()[start+j]) 109 | } 110 | } else { 111 | decompressed.WriteByte(data[i-1]) 112 | } 113 | 114 | // Update hash table with the new sequence 115 | if decompressed.Len() >= 3 { 116 | hashKey := murmur.Hash64(decompressed.Bytes()[decompressed.Len()-3:], 0) 117 | hashTable[hashKey] = decompressed.Len() - 3 118 | } 119 | } 120 | 121 | return decompressed.Bytes() 122 | } 123 | -------------------------------------------------------------------------------- /compressor/compressor_test.go: -------------------------------------------------------------------------------- 1 | // Package compressor tests 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package compressor 32 | 33 | import ( 34 | "bytes" 35 | "encoding/gob" 36 | "os" 37 | "testing" 38 | ) 39 | 40 | func TestNewCompressor(t *testing.T) { 41 | tests := []struct { 42 | windowSize int 43 | expectErr bool 44 | }{ 45 | {windowSize: 32, expectErr: false}, 46 | {windowSize: 0, expectErr: true}, 47 | {windowSize: -1, expectErr: true}, 48 | } 49 | 50 | for _, tt := range tests { 51 | _, err := NewCompressor(tt.windowSize) 52 | if (err != nil) != tt.expectErr { 53 | t.Errorf("NewCompressor(%d) error = %v, expectErr %v", tt.windowSize, err, tt.expectErr) 54 | } 55 | } 56 | } 57 | 58 | type TestStruct struct { 59 | Data []byte 60 | N int64 61 | F float64 62 | B bool 63 | } 64 | 65 | func TestCompressor_CompressDecompress(t *testing.T) { 66 | compressor, _ := NewCompressor(32) 67 | 68 | tests := [][]byte{ 69 | []byte{}, 70 | []byte("abcdef"), 71 | []byte("aaaaaa"), 72 | []byte("abcabcabcabcabcabc"), 73 | } 74 | 75 | // We will encode a test struct 76 | testStruct := TestStruct{ 77 | Data: []byte("Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source. Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of \"de Finibus Bonorum et Malorum\" (The Extremes of Good and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, \"Lorem ipsum dolor sit amet..\", comes from a line in section 1.10.32.\n\nThe standard chunk of Lorem Ipsum used since the 1500s is reproduced below for those interested. Sections 1.10.32 and 1.10.33 from \"de Finibus Bonorum et Malorum\" by Cicero are also reproduced in their exact original form, accompanied by English versions from the 1914 translation by H. Rackham."), 78 | N: 42, 79 | F: 3.14, 80 | B: true, 81 | } 82 | 83 | buff := new(bytes.Buffer) 84 | 85 | // Encode 86 | encoder := gob.NewEncoder(buff) 87 | 88 | err := encoder.Encode(testStruct) 89 | if err != nil { 90 | t.Fatalf("Failed to encode test struct: %v", err) 91 | } 92 | 93 | tests = append(tests, buff.Bytes()) 94 | 95 | for i, tt := range tests { 96 | compressed := compressor.Compress(tt) 97 | decompressed := compressor.Decompress(compressed) 98 | 99 | // check if last test, if so is struct test, decode it 100 | if i == len(tests)-1 { 101 | // Decode 102 | testStruct2 := TestStruct{} 103 | 104 | decoder := gob.NewDecoder(bytes.NewReader(decompressed)) 105 | 106 | err := decoder.Decode(&testStruct2) 107 | if err != nil { 108 | t.Fatalf("Failed to decode test struct: %v", err) 109 | } 110 | 111 | if testStruct2.N != testStruct.N { 112 | t.Errorf("CompressDecompress(%v) = %v, expected %v", tt, testStruct2.N, testStruct.N) 113 | } 114 | 115 | if testStruct2.F != testStruct.F { 116 | t.Errorf("CompressDecompress(%v) = %v, expected %v", tt, testStruct2.F, testStruct.F) 117 | } 118 | 119 | if testStruct2.B != testStruct.B { 120 | t.Errorf("CompressDecompress(%v) = %v, expected %v", tt, testStruct2.B, testStruct.B) 121 | } 122 | } else { 123 | 124 | if !bytes.Equal(decompressed, tt) { 125 | t.Errorf("CompressDecompress(%v) = %v, expected %v", tt, decompressed, tt) 126 | } 127 | } 128 | } 129 | } 130 | 131 | // We test compressing multiple files and getting % of compression 132 | func TestCompressor_Compression_Ratios(t *testing.T) { 133 | // We read test.png into memory and calculate the compression ratio 134 | 135 | // Read test.png 136 | data, err := os.ReadFile("test.png") 137 | if err != nil { 138 | t.Fatalf("Failed to read test.png: %v", err) 139 | } 140 | 141 | // Read test2_public_domain.jpg 142 | data2, err := os.ReadFile("test2_public_domain.jpg") 143 | if err != nil { 144 | t.Fatalf("Failed to read test2_public_domain.jpg: %v", err) 145 | } 146 | 147 | compressor, _ := NewCompressor(1024 * 32) 148 | 149 | compressed := compressor.Compress(data) 150 | 151 | // Calculate compression ratio 152 | compressionRatio := float64(len(compressed)) / float64(len(data)) 153 | 154 | t.Logf("test.png compression ratio: %.2f times smaller than original", compressionRatio) 155 | 156 | compressor, _ = NewCompressor(1024 * 32) 157 | 158 | compressed = compressor.Compress(data2) 159 | 160 | // Calculate compression ratio 161 | compressionRatio = float64(len(compressed)) / float64(len(data2)) 162 | 163 | t.Logf("test2_public_domain.jpg compression ratio: %.2f times smaller than original", compressionRatio) 164 | 165 | } 166 | -------------------------------------------------------------------------------- /compressor/test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guycipher/k4/4cce22a371ba223000c03d9b1f2705d940eb39fe/compressor/test.png -------------------------------------------------------------------------------- /compressor/test2_public_domain.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guycipher/k4/4cce22a371ba223000c03d9b1f2705d940eb39fe/compressor/test2_public_domain.jpg -------------------------------------------------------------------------------- /ffi/csharp/K4.cs: -------------------------------------------------------------------------------- 1 | // K4 C# FFI 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | using System; 32 | using System.Runtime.InteropServices; 33 | 34 | public static class K4 35 | { 36 | private const string DllName = "libk4.so"; 37 | 38 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 39 | public static extern IntPtr db_open(string directory, int memtableFlushThreshold, int compactionInterval, int logging, int compress); 40 | 41 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 42 | public static extern int db_close(IntPtr dbPtr); 43 | 44 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 45 | public static extern int db_put(IntPtr dbPtr, string key, int keyLen, string value, int valueLen, long ttl); 46 | 47 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 48 | public static extern IntPtr db_get(IntPtr dbPtr, string key, int keyLen); 49 | 50 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 51 | public static extern int db_delete(IntPtr dbPtr, string key, int keyLen); 52 | 53 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 54 | public static extern IntPtr begin_transaction(IntPtr dbPtr); 55 | 56 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 57 | public static extern int add_operation(IntPtr txPtr, int operation, string key, int keyLen, string value, int valueLen); 58 | 59 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 60 | public static extern void remove_transaction(IntPtr dbPtr, IntPtr txPtr); 61 | 62 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 63 | public static extern int commit_transaction(IntPtr txPtr, IntPtr dbPtr); 64 | 65 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 66 | public static extern int rollback_transaction(IntPtr txPtr, IntPtr dbPtr); 67 | 68 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 69 | public static extern int recover_from_wal(IntPtr dbPtr); 70 | 71 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 72 | public static extern KeyValuePairArray range_(IntPtr dbPtr, string start, int startLen, string end, int endLen); 73 | 74 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 75 | public static extern KeyValuePairArray nrange(IntPtr dbPtr, string start, int startLen, string end, int endLen); 76 | 77 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 78 | public static extern KeyValuePairArray greater_than(IntPtr dbPtr, string key, int keyLen); 79 | 80 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 81 | public static extern KeyValuePairArray less_than(IntPtr dbPtr, string key, int keyLen); 82 | 83 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 84 | public static extern KeyValuePairArray nget(IntPtr dbPtr, string key, int keyLen); 85 | 86 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 87 | public static extern KeyValuePairArray greater_than_eq(IntPtr dbPtr, string key, int keyLen); 88 | 89 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 90 | public static extern KeyValuePairArray less_than_eq(IntPtr dbPtr, string key, int keyLen); 91 | 92 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 93 | public static extern IntPtr new_iterator(IntPtr dbPtr); 94 | 95 | [StructLayout(LayoutKind.Sequential)] 96 | public struct IterNextReturn 97 | { 98 | public IntPtr r0; 99 | public IntPtr r1; 100 | } 101 | 102 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 103 | public static extern IterNextReturn iter_next(IntPtr iterPtr); 104 | 105 | [StructLayout(LayoutKind.Sequential)] 106 | public struct IterPrevReturn 107 | { 108 | public IntPtr r0; 109 | public IntPtr r1; 110 | } 111 | 112 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 113 | public static extern IterPrevReturn iter_prev(IntPtr iterPtr); 114 | 115 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 116 | public static extern void iter_reset(IntPtr iterPtr); 117 | 118 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 119 | public static extern void iter_close(IntPtr iterPtr); 120 | 121 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 122 | public static extern int escalate_flush(IntPtr dbPtr); 123 | 124 | [DllImport(DllName, CallingConvention = CallingConvention.Cdecl)] 125 | public static extern int escalate_compaction(IntPtr dbPtr); 126 | } -------------------------------------------------------------------------------- /ffi/csharp/KeyValuePair.cs: -------------------------------------------------------------------------------- 1 | // K4 KeyValuePair 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | using System; 32 | using System.Runtime.InteropServices; 33 | 34 | [StructLayout(LayoutKind.Sequential)] 35 | public struct KeyValuePair 36 | { 37 | public IntPtr key; 38 | public IntPtr value; 39 | } -------------------------------------------------------------------------------- /ffi/csharp/KeyValuePairArray.cs: -------------------------------------------------------------------------------- 1 | // K4 KeyValuePairArray 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | using System; 32 | using System.Runtime.InteropServices; 33 | 34 | [StructLayout(LayoutKind.Sequential)] 35 | public struct KeyValuePairArray 36 | { 37 | public IntPtr pairs; 38 | public int numPairs; 39 | } -------------------------------------------------------------------------------- /ffi/csharp/readme.md: -------------------------------------------------------------------------------- 1 | # K4 C# FFI 2 | This is an example library that demonstrates how to use a K4 FFI in C# using the shared K4 C library. 3 | 4 | ## Example 5 | ```csharp 6 | using System; 7 | using System.Runtime.InteropServices; 8 | 9 | public class ExampleUsage 10 | { 11 | public static void Main(string[] args) 12 | { 13 | IntPtr db = K4.db_open("data", 1024, 60, 1, 1); 14 | if (db == IntPtr.Zero) 15 | { 16 | Console.WriteLine("Failed to open database"); 17 | return; 18 | } 19 | 20 | string key = "key1"; 21 | string value = "value1"; 22 | 23 | if (K4.db_put(db, key, key.Length, value, value.Length, -1) != 0) 24 | { 25 | Console.WriteLine("Failed to put key-value pair"); 26 | K4.db_close(db); 27 | return; 28 | } 29 | 30 | IntPtr retrievedValuePtr = K4.db_get(db, key, key.Length); 31 | if (retrievedValuePtr == IntPtr.Zero) 32 | { 33 | Console.WriteLine("Failed to get value"); 34 | K4.db_close(db); 35 | return; 36 | } 37 | 38 | string retrievedValue = Marshal.PtrToStringAnsi(retrievedValuePtr); 39 | Console.WriteLine("Retrieved value: " + retrievedValue); 40 | 41 | if (K4.db_delete(db, key, key.Length) != 0) 42 | { 43 | Console.WriteLine("Failed to delete key-value pair"); 44 | K4.db_close(db); 45 | return; 46 | } 47 | 48 | if (K4.db_close(db) != 0) 49 | { 50 | Console.WriteLine("Failed to close database"); 51 | } 52 | } 53 | } 54 | ``` -------------------------------------------------------------------------------- /ffi/java/K4.java: -------------------------------------------------------------------------------- 1 | // K4 Java FFI 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | import com.sun.jna.Library; 32 | import com.sun.jna.Native; 33 | import com.sun.jna.Pointer; 34 | 35 | public interface K4 extends Library { 36 | K4 INSTANCE = Native.load("libk4", K4.class); // Load the shared library 37 | 38 | Pointer db_open(String directory, int memtableFlushThreshold, int compactionInterval, int logging, int compress); 39 | int db_close(Pointer dbPtr); 40 | int db_put(Pointer dbPtr, String key, int keyLen, String value, int valueLen, long ttl); 41 | Pointer db_get(Pointer dbPtr, String key, int keyLen); 42 | int db_delete(Pointer dbPtr, String key, int keyLen); 43 | Pointer begin_transaction(Pointer dbPtr); 44 | int add_operation(Pointer txPtr, int operation, String key, int keyLen, String value, int valueLen); 45 | void remove_transaction(Pointer dbPtr, Pointer txPtr); 46 | int commit_transaction(Pointer txPtr, Pointer dbPtr); 47 | int rollback_transaction(Pointer txPtr, Pointer dbPtr); 48 | int recover_from_wal(Pointer dbPtr); 49 | KeyValuePairArray.ByValue range_(Pointer dbPtr, String start, int startLen, String end, int endLen); 50 | KeyValuePairArray.ByValue nrange(Pointer dbPtr, String start, int startLen, String end, int endLen); 51 | KeyValuePairArray.ByValue greater_than(Pointer dbPtr, String key, int keyLen); 52 | KeyValuePairArray.ByValue less_than(Pointer dbPtr, String key, int keyLen); 53 | KeyValuePairArray.ByValue nget(Pointer dbPtr, String key, int keyLen); 54 | KeyValuePairArray.ByValue greater_than_eq(Pointer dbPtr, String key, int keyLen); 55 | KeyValuePairArray.ByValue less_than_eq(Pointer dbPtr, String key, int keyLen); 56 | Pointer new_iterator(Pointer dbPtr); 57 | K4.iter_next_return iter_next(Pointer iterPtr); 58 | K4.iter_prev_return iter_prev(Pointer iterPtr); 59 | void iter_reset(Pointer iterPtr); 60 | void iter_close(Pointer iterPtr); 61 | int escalate_flush(Pointer dbPtr); 62 | int escalate_compaction(Pointer dbPtr); 63 | 64 | class iter_next_return extends Structure { 65 | public Pointer r0; 66 | public Pointer r1; 67 | 68 | @Override 69 | protected List getFieldOrder() { 70 | return Arrays.asList("r0", "r1"); 71 | } 72 | 73 | public static class ByReference extends iter_next_return implements Structure.ByReference {} 74 | public static class ByValue extends iter_next_return implements Structure.ByValue {} 75 | } 76 | 77 | class iter_prev_return extends Structure { 78 | public Pointer r0; 79 | public Pointer r1; 80 | 81 | @Override 82 | protected List getFieldOrder() { 83 | return Arrays.asList("r0", "r1"); 84 | } 85 | 86 | public static class ByReference extends iter_prev_return implements Structure.ByReference {} 87 | public static class ByValue extends iter_prev_return implements Structure.ByValue {} 88 | } 89 | } -------------------------------------------------------------------------------- /ffi/java/KeyValuePair.java: -------------------------------------------------------------------------------- 1 | // K4 KeyValuePair 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | import com.sun.jna.Pointer; 32 | import com.sun.jna.Structure; 33 | import java.util.Arrays; 34 | import java.util.List; 35 | 36 | public class KeyValuePair extends Structure { 37 | public Pointer key; 38 | public Pointer value; 39 | 40 | @Override 41 | protected List getFieldOrder() { 42 | return Arrays.asList("key", "value"); 43 | } 44 | 45 | public static class ByReference extends KeyValuePair implements Structure.ByReference {} 46 | public static class ByValue extends KeyValuePair implements Structure.ByValue {} 47 | } -------------------------------------------------------------------------------- /ffi/java/KeyValuePairArray.java: -------------------------------------------------------------------------------- 1 | // K4 KeyValuePairArray 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | import com.sun.jna.Pointer; 32 | import com.sun.jna.Structure; 33 | import java.util.Arrays; 34 | import java.util.List; 35 | 36 | public class KeyValuePairArray extends Structure { 37 | public Pointer pairs; 38 | public int numPairs; 39 | 40 | @Override 41 | protected List getFieldOrder() { 42 | return Arrays.asList("pairs", "numPairs"); 43 | } 44 | 45 | public static class ByReference extends KeyValuePairArray implements Structure.ByReference {} 46 | public static class ByValue extends KeyValuePairArray implements Structure.ByValue {} 47 | } -------------------------------------------------------------------------------- /ffi/java/readme.md: -------------------------------------------------------------------------------- 1 | # K4 Java FFI 2 | This is an example library that demonstrates how to use a K4 FFI in Java using the shared K4 C library. 3 | 4 | ## Example 5 | ```java 6 | public class ExampleUsage { 7 | public static void main(String[] args) { 8 | Pointer db = K4.INSTANCE.db_open("data", 1024, 60, 1, 1); 9 | if (db == null) { 10 | System.out.println("Failed to open database"); 11 | return; 12 | } 13 | 14 | String key = "key1"; 15 | String value = "value1"; 16 | if (K4.INSTANCE.db_put(db, key, key.length(), value, value.length(), -1) != 0) { 17 | System.out.println("Failed to put key-value pair"); 18 | K4.INSTANCE.db_close(db); 19 | return; 20 | } 21 | 22 | Pointer retrievedValue = K4.INSTANCE.db_get(db, key, key.length()); 23 | if (retrievedValue == null) { 24 | System.out.println("Failed to get value"); 25 | K4.INSTANCE.db_close(db); 26 | return; 27 | } 28 | 29 | System.out.println("Retrieved value: " + retrievedValue.getString(0)); 30 | 31 | if (K4.INSTANCE.db_delete(db, key, key.length()) != 0) { 32 | System.out.println("Failed to delete key-value pair"); 33 | K4.INSTANCE.db_close(db); 34 | return; 35 | } 36 | 37 | if (K4.INSTANCE.db_close(db) != 0) { 38 | System.out.println("Failed to close database"); 39 | } 40 | } 41 | } 42 | 43 | ``` -------------------------------------------------------------------------------- /ffi/lua/k4.lua: -------------------------------------------------------------------------------- 1 | -- K4 Lua FFI 2 | -- BSD 3-Clause License 3 | -- 4 | -- Copyright (c) 2024, Alex Gaetano Padula 5 | -- All rights reserved. 6 | -- 7 | -- Redistribution and use in source and binary forms, with or without 8 | -- modification, are permitted provided that the following conditions are met: 9 | -- 10 | -- 1. Redistributions of source code must retain the above copyright notice, this 11 | -- list of conditions and the following disclaimer. 12 | -- 13 | -- 2. Redistributions in binary form must reproduce the above copyright notice, 14 | -- this list of conditions and the following disclaimer in the documentation 15 | -- and/or other materials provided with the distribution. 16 | -- 17 | -- 3. Neither the name of the copyright holder nor the names of its 18 | -- contributors may be used to endorse or promote products derived from 19 | -- this software without specific prior written permission. 20 | -- 21 | -- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | -- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | -- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | -- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | -- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | -- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | -- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | -- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | -- OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | -- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | local ffi = require("ffi") 33 | 34 | -- Load the shared library 35 | local k4 = ffi.load("libk4.so") -- you gotta specify the path to the shared library 36 | 37 | -- KeyValuePair is a structure that holds a key-value pair 38 | ffi.cdef[[ 39 | typedef struct { 40 | char* key; 41 | char* value; 42 | } KeyValuePair; 43 | ]] 44 | 45 | -- KeyValuePairArray is a structure that holds an array of KeyValuePair structures 46 | ffi.cdef[[ 47 | typedef struct { 48 | KeyValuePair* pairs; 49 | int numPairs; 50 | } KeyValuePairArray; 51 | ]] 52 | 53 | -- Define the iter_next_return structure 54 | ffi.cdef[[ 55 | typedef struct { 56 | char* r0; 57 | char* r1; 58 | } IterNextReturn; 59 | ]] 60 | 61 | -- Define the iter_prev_return structure 62 | ffi.cdef[[ 63 | typedef struct { 64 | char* r0; 65 | char* r1; 66 | } IterPrevReturn; 67 | ]] 68 | 69 | -- Define the function prototypes 70 | ffi.cdef[[ 71 | void* db_open(const char* directory, int memtableFlushThreshold, int compactionInterval, int logging, int compress); 72 | int db_close(void* dbPtr); 73 | int db_put(void* dbPtr, const char* key, int keyLen, const char* value, int valueLen, int64_t ttl); 74 | char* db_get(void* dbPtr, const char* key, int keyLen); 75 | int db_delete(void* dbPtr, const char* key, int keyLen); 76 | void* begin_transaction(void* dbPtr); 77 | int add_operation(void* txPtr, int operation, const char* key, int keyLen, const char* value, int valueLen); 78 | void remove_transaction(void* dbPtr, void* txPtr); 79 | int commit_transaction(void* txPtr, void* dbPtr); 80 | int rollback_transaction(void* txPtr, void* dbPtr); 81 | int recover_from_wal(void* dbPtr); 82 | KeyValuePairArray range_(void* dbPtr, const char* start, int startLen, const char* end, int endLen); 83 | KeyValuePairArray nrange(void* dbPtr, const char* start, int startLen, const char* end, int endLen); 84 | KeyValuePairArray greater_than(void* dbPtr, const char* key, int keyLen); 85 | KeyValuePairArray less_than(void* dbPtr, const char* key, int keyLen); 86 | KeyValuePairArray nget(void* dbPtr, const char* key, int keyLen); 87 | KeyValuePairArray greater_than_eq(void* dbPtr, const char* key, int keyLen); 88 | KeyValuePairArray less_than_eq(void* dbPtr, const char* key, int keyLen); 89 | void* new_iterator(void* dbPtr); 90 | IterNextReturn iter_next(void* iterPtr); 91 | IterPrevReturn iter_prev(void* iterPtr); 92 | void iter_reset(void* iterPtr); 93 | void iter_close(void* iterPtr); 94 | int escalate_flush(void* dbPtr); 95 | int escalate_compaction(void* dbPtr); 96 | ]] -------------------------------------------------------------------------------- /ffi/lua/readme.md: -------------------------------------------------------------------------------- 1 | # K4 Lua FFI 2 | This is an example library that demonstrates how to use a K4 FFI in Lua using the shared K4 C library. 3 | 4 | ## Example 5 | ``` 6 | -- Open the database 7 | local db = k4.db_open("data", 1024, 60, 1, 1) 8 | if db == nil then 9 | print("Failed to open database") 10 | return 11 | end 12 | 13 | -- Put a key-value pair 14 | local key = "key1" 15 | local value = "value1" 16 | if k4.db_put(db, key, #key, value, #value, -1) ~= 0 then -- -1 means no expiration 17 | print("Failed to put key-value pair") 18 | k4.db_close(db) 19 | return 20 | end 21 | 22 | -- Get the value for the key 23 | local retrieved_value = k4.db_get(db, key, #key) 24 | if retrieved_value == nil then 25 | print("Failed to get value") 26 | k4.db_close(db) 27 | return 28 | end 29 | 30 | print("Retrieved value:", ffi.string(retrieved_value)) 31 | 32 | -- Delete the key-value pair 33 | if k4.db_delete(db, key, #key) ~= 0 then 34 | print("Failed to delete key-value pair") 35 | k4.db_close(db) 36 | return 37 | end 38 | 39 | -- Close the database 40 | if k4.db_close(db) ~= 0 then 41 | print("Failed to close database") 42 | end 43 | ``` -------------------------------------------------------------------------------- /ffi/nodejs/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json -------------------------------------------------------------------------------- /ffi/nodejs/k4.js: -------------------------------------------------------------------------------- 1 | // K4 Node.JS FFI 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | const ffi = require('ffi-napi'); 32 | const ref = require('ref-napi'); 33 | 34 | // KeyValuePair is a struct with two fields: key and value 35 | const KeyValuePair = new ffi.StructType({ 36 | key: 'string', 37 | value: 'string' 38 | }); 39 | 40 | // KeyValuePairArray is an array of KeyValuePair structs 41 | const KeyValuePairArray = new ffi.StructType({ 42 | pairs: KeyValuePair, 43 | numPairs: 'int' 44 | }); 45 | 46 | // IterNextReturn is the return type of the iter_next function 47 | const IterNextReturn = new ffi.StructType({ 48 | r0: 'string', 49 | r1: 'string' 50 | }); 51 | 52 | // IterPrevReturn is the return type of the iter_prev function 53 | const IterPrevReturn = new ffi.StructType({ 54 | r0: 'string', 55 | r1: 'string' 56 | }); 57 | 58 | // Load the K4 library 59 | // note you must have the library installed in your system 60 | const k4 = ffi.Library('libk4', { 61 | 'db_open': ['pointer', ['string', 'int', 'int', 'int', 'int']], 62 | 'db_close': ['int', ['pointer']], 63 | 'db_put': ['int', ['pointer', 'string', 'int', 'string', 'int', 'int64']], 64 | 'db_get': ['string', ['pointer', 'string', 'int']], 65 | 'db_delete': ['int', ['pointer', 'string', 'int']], 66 | 'begin_transaction': ['pointer', ['pointer']], 67 | 'add_operation': ['int', ['pointer', 'int', 'string', 'int', 'string', 'int']], 68 | 'remove_transaction': ['void', ['pointer', 'pointer']], 69 | 'commit_transaction': ['int', ['pointer', 'pointer']], 70 | 'rollback_transaction': ['int', ['pointer', 'pointer']], 71 | 'recover_from_wal': ['int', ['pointer']], 72 | 'range_': [KeyValuePairArray, ['pointer', 'string', 'int', 'string', 'int']], 73 | 'nrange': [KeyValuePairArray, ['pointer', 'string', 'int', 'string', 'int']], 74 | 'greater_than': [KeyValuePairArray, ['pointer', 'string', 'int']], 75 | 'less_than': [KeyValuePairArray, ['pointer', 'string', 'int']], 76 | 'nget': [KeyValuePairArray, ['pointer', 'string', 'int']], 77 | 'greater_than_eq': [KeyValuePairArray, ['pointer', 'string', 'int']], 78 | 'less_than_eq': [KeyValuePairArray, ['pointer', 'string', 'int']], 79 | 'new_iterator': ['pointer', ['pointer']], 80 | 'iter_next': [IterNextReturn, ['pointer']], 81 | 'iter_prev': [IterPrevReturn, ['pointer']], 82 | 'iter_reset': ['void', ['pointer']], 83 | 'iter_close': ['void', ['pointer']], 84 | 'escalate_flush': ['int', ['pointer']], 85 | 'escalate_compaction': ['int', ['pointer']] 86 | }); 87 | 88 | module.exports = k4; // Export the k4 object -------------------------------------------------------------------------------- /ffi/nodejs/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "k4", 3 | "version": "0.9.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "k4", 9 | "version": "0.9.0", 10 | "license": "BSD-3-Clause", 11 | "dependencies": { 12 | "ffi-napi": "^4.0.3", 13 | "ref-napi": "^3.0.3" 14 | } 15 | }, 16 | "node_modules/debug": { 17 | "version": "4.3.7", 18 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", 19 | "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", 20 | "dependencies": { 21 | "ms": "^2.1.3" 22 | }, 23 | "engines": { 24 | "node": ">=6.0" 25 | }, 26 | "peerDependenciesMeta": { 27 | "supports-color": { 28 | "optional": true 29 | } 30 | } 31 | }, 32 | "node_modules/ffi-napi": { 33 | "version": "4.0.3", 34 | "resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-4.0.3.tgz", 35 | "integrity": "sha512-PMdLCIvDY9mS32RxZ0XGb95sonPRal8aqRhLbeEtWKZTe2A87qRFG9HjOhvG8EX2UmQw5XNRMIOT+1MYlWmdeg==", 36 | "hasInstallScript": true, 37 | "dependencies": { 38 | "debug": "^4.1.1", 39 | "get-uv-event-loop-napi-h": "^1.0.5", 40 | "node-addon-api": "^3.0.0", 41 | "node-gyp-build": "^4.2.1", 42 | "ref-napi": "^2.0.1 || ^3.0.2", 43 | "ref-struct-di": "^1.1.0" 44 | }, 45 | "engines": { 46 | "node": ">=10" 47 | } 48 | }, 49 | "node_modules/get-symbol-from-current-process-h": { 50 | "version": "1.0.2", 51 | "resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz", 52 | "integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw==" 53 | }, 54 | "node_modules/get-uv-event-loop-napi-h": { 55 | "version": "1.0.6", 56 | "resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz", 57 | "integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==", 58 | "dependencies": { 59 | "get-symbol-from-current-process-h": "^1.0.1" 60 | } 61 | }, 62 | "node_modules/ms": { 63 | "version": "2.1.3", 64 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 65 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" 66 | }, 67 | "node_modules/node-addon-api": { 68 | "version": "3.2.1", 69 | "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz", 70 | "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==" 71 | }, 72 | "node_modules/node-gyp-build": { 73 | "version": "4.8.2", 74 | "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.2.tgz", 75 | "integrity": "sha512-IRUxE4BVsHWXkV/SFOut4qTlagw2aM8T5/vnTsmrHJvVoKueJHRc/JaFND7QDDc61kLYUJ6qlZM3sqTSyx2dTw==", 76 | "bin": { 77 | "node-gyp-build": "bin.js", 78 | "node-gyp-build-optional": "optional.js", 79 | "node-gyp-build-test": "build-test.js" 80 | } 81 | }, 82 | "node_modules/ref-napi": { 83 | "version": "3.0.3", 84 | "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.3.tgz", 85 | "integrity": "sha512-LiMq/XDGcgodTYOMppikEtJelWsKQERbLQsYm0IOOnzhwE9xYZC7x8txNnFC9wJNOkPferQI4vD4ZkC0mDyrOA==", 86 | "hasInstallScript": true, 87 | "dependencies": { 88 | "debug": "^4.1.1", 89 | "get-symbol-from-current-process-h": "^1.0.2", 90 | "node-addon-api": "^3.0.0", 91 | "node-gyp-build": "^4.2.1" 92 | }, 93 | "engines": { 94 | "node": ">= 10.0" 95 | } 96 | }, 97 | "node_modules/ref-struct-di": { 98 | "version": "1.1.1", 99 | "resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz", 100 | "integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==", 101 | "dependencies": { 102 | "debug": "^3.1.0" 103 | } 104 | }, 105 | "node_modules/ref-struct-di/node_modules/debug": { 106 | "version": "3.2.7", 107 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", 108 | "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", 109 | "dependencies": { 110 | "ms": "^2.1.1" 111 | } 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /ffi/nodejs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "k4", 3 | "version": "1.9.5", 4 | "description": "K4 storage engine Node.JS FFI", 5 | "main": "k4.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "Alex Gaetano Padula", 10 | "license": "BSD-3-Clause", 11 | "dependencies": { 12 | "ffi-napi": "^4.0.3", 13 | "ref-napi": "^3.0.3" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /ffi/nodejs/readme.md: -------------------------------------------------------------------------------- 1 | # K4 Node.JS FFI 2 | This is an example library that demonstrates how to use a K4 FFI in Node.JS using the shared K4 C library. 3 | 4 | ## Examples 5 | ```javascript 6 | const k4 = require('./k4'); 7 | 8 | const db = k4.db_open('data', 1024, 60, 1, 1); 9 | if (db.isNull()) { 10 | console.error('Failed to open database'); 11 | process.exit(1); 12 | } 13 | 14 | const key = 'key1'; 15 | const value = 'value1'; 16 | if (k4.db_put(db, key, key.length, value, value.length, -1) !== 0) { // ttl = -1 means no expiration 17 | console.error('Failed to put key-value pair'); 18 | libk4.db_close(db); 19 | process.exit(1); 20 | } 21 | 22 | const retrievedValue = k4.db_get(db, key, key.length); 23 | if (retrievedValue.isNull()) { 24 | console.error('Failed to get value'); 25 | libk4.db_close(db); 26 | process.exit(1); 27 | } 28 | 29 | console.log('Retrieved value:', retrievedValue); 30 | 31 | if (k4.db_delete(db, key, key.length) !== 0) { 32 | console.error('Failed to delete key-value pair'); 33 | libk4.db_close(db); 34 | process.exit(1); 35 | } 36 | 37 | if (k4.db_close(db) !== 0) { 38 | console.error('Failed to close database'); 39 | process.exit(1); 40 | } 41 | ``` -------------------------------------------------------------------------------- /ffi/python/k4.py: -------------------------------------------------------------------------------- 1 | # K4 Python FFI 2 | # BSD 3-Clause License 3 | # 4 | # Copyright (c) 2024, Alex Gaetano Padula 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are met: 9 | # 10 | # 1. Redistributions of source code must retain the above copyright notice, this 11 | # list of conditions and the following disclaimer. 12 | # 13 | # 2. Redistributions in binary form must reproduce the above copyright notice, 14 | # this list of conditions and the following disclaimer in the documentation 15 | # and/or other materials provided with the distribution. 16 | # 17 | # 3. Neither the name of the copyright holder nor the names of its 18 | # contributors may be used to endorse or promote products derived from 19 | # this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | import ctypes 32 | from ctypes import c_char_p, c_int, c_void_p, c_int64, Structure, POINTER 33 | 34 | # Load the shared library 35 | k4 = ctypes.CDLL('libk4.so') # you gotta specify the path to the shared library 36 | 37 | # KeyValuePair is a structure that holds a key-value pair 38 | class KeyValuePair(Structure): 39 | _fields_ = [("key", c_char_p), 40 | ("value", c_char_p)] 41 | 42 | # KeyValuePairArray is a structure that holds an array of KeyValuePair's 43 | class KeyValuePairArray(Structure): 44 | _fields_ = [("pairs", POINTER(KeyValuePair)), 45 | ("numPairs", c_int)] 46 | 47 | # Define the iter_next_return structure 48 | class IterNextReturn(Structure): 49 | _fields_ = [("r0", c_char_p), 50 | ("r1", c_char_p)] 51 | 52 | # Define the iter_prev_return structure 53 | class IterPrevReturn(Structure): 54 | _fields_ = [("r0", c_char_p), 55 | ("r1", c_char_p)] 56 | 57 | # Define the K4 function prototypes 58 | k4.db_open.argtypes = [c_char_p, c_int, c_int, c_int, c_int] 59 | k4.db_open.restype = c_void_p 60 | 61 | k4.db_close.argtypes = [c_void_p] 62 | k4.db_close.restype = c_int 63 | 64 | k4.db_put.argtypes = [c_void_p, c_char_p, c_int, c_char_p, c_int, c_int64] 65 | k4.db_put.restype = c_int 66 | 67 | k4.db_get.argtypes = [c_void_p, c_char_p, c_int] 68 | k4.db_get.restype = c_char_p 69 | 70 | k4.db_delete.argtypes = [c_void_p, c_char_p, c_int] 71 | k4.db_delete.restype = c_int 72 | 73 | k4.begin_transaction.argtypes = [c_void_p] 74 | k4.begin_transaction.restype = c_void_p 75 | 76 | k4.add_operation.argtypes = [c_void_p, c_int, c_char_p, c_int, c_char_p, c_int] 77 | k4.add_operation.restype = c_int 78 | 79 | k4.remove_transaction.argtypes = [c_void_p, c_void_p] 80 | 81 | k4.commit_transaction.argtypes = [c_void_p, c_void_p] 82 | k4.commit_transaction.restype = c_int 83 | 84 | k4.rollback_transaction.argtypes = [c_void_p, c_void_p] 85 | k4.rollback_transaction.restype = c_int 86 | 87 | k4.recover_from_wal.argtypes = [c_void_p] 88 | k4.recover_from_wal.restype = c_int 89 | 90 | k4.range_.argtypes = [c_void_p, c_char_p, c_int, c_char_p, c_int] 91 | k4.range_.restype = KeyValuePairArray 92 | 93 | k4.nrange.argtypes = [c_void_p, c_char_p, c_int, c_char_p, c_int] 94 | k4.nrange.restype = KeyValuePairArray 95 | 96 | k4.greater_than.argtypes = [c_void_p, c_char_p, c_int] 97 | k4.greater_than.restype = KeyValuePairArray 98 | 99 | k4.less_than.argtypes = [c_void_p, c_char_p, c_int] 100 | k4.less_than.restype = KeyValuePairArray 101 | 102 | k4.nget.argtypes = [c_void_p, c_char_p, c_int] 103 | k4.nget.restype = KeyValuePairArray 104 | 105 | k4.greater_than_eq.argtypes = [c_void_p, c_char_p, c_int] 106 | k4.greater_than_eq.restype = KeyValuePairArray 107 | 108 | k4.less_than_eq.argtypes = [c_void_p, c_char_p, c_int] 109 | k4.less_than_eq.restype = KeyValuePairArray 110 | 111 | k4.new_iterator.argtypes = [c_void_p] 112 | k4.new_iterator.restype = c_void_p 113 | 114 | k4.iter_next.argtypes = [c_void_p] 115 | k4.iter_next.restype = IterNextReturn 116 | 117 | k4.iter_prev.argtypes = [c_void_p] 118 | k4.iter_prev.restype = IterPrevReturn 119 | 120 | k4.iter_reset.argtypes = [c_void_p] 121 | 122 | k4.iter_close.argtypes = [c_void_p] 123 | 124 | k4.escalate_flush.argtypes = [c_void_p] 125 | k4.escalate_flush.restype = c_int 126 | 127 | k4.escalate_compaction.argtypes = [c_void_p] 128 | k4.escalate_compaction.restype = c_int -------------------------------------------------------------------------------- /ffi/python/readme.md: -------------------------------------------------------------------------------- 1 | # K4 Python FFI 2 | This is an example library that demonstrates how to use a K4 FFI in Python using the shared K4 C library. 3 | 4 | 5 | ## Example 6 | ``` 7 | def main(): 8 | # Open the database 9 | db = k4.db_open(b"data", 1024, 60, 1, 1) 10 | if not db: 11 | print("Failed to open database") 12 | return 13 | 14 | # Put a key-value pair 15 | key = b"key1" 16 | value = b"value1" 17 | 18 | if k4.db_put(db, key, len(key), value, len(value), -1) != 0: # -1 means no expiration 19 | print("Failed to put key-value pair") 20 | k4.db_close(db) 21 | return 22 | 23 | # Get the value for the key 24 | retrieved_value = k4.db_get(db, key, len(key)) 25 | if not retrieved_value: 26 | print("Failed to get value") 27 | k4.db_close(db) 28 | return 29 | 30 | print("Retrieved value:", retrieved_value.decode('utf-8')) 31 | 32 | # Delete the key-value pair 33 | if k4.db_delete(db, key, len(key)) != 0: 34 | print("Failed to delete key-value pair") 35 | k4.db_close(db) 36 | return 37 | 38 | # Close the database 39 | if k4.db_close(db) != 0: 40 | print("Failed to close database") 41 | 42 | if __name__ == "__main__": 43 | main() 44 | ``` -------------------------------------------------------------------------------- /ffi/readme.md: -------------------------------------------------------------------------------- 1 | ## FFIs (foreign function interfaces) 2 | K4 offers foreign function interfaces for a variety of languages. 3 | 4 | You must make sure you have the shared C library installed on your system. 5 | 6 | You can follow here [C Library](https://github.com/guycipher/k4/tree/main/c) 7 | 8 | -------------------------------------------------------------------------------- /ffi/ruby/k4.rb: -------------------------------------------------------------------------------- 1 | =begin 2 | K4 Ruby FFI 3 | BSD 3-Clause License 4 | 5 | Copyright (c) 2024, Alex Gaetano Padula 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are met: 10 | 11 | 1. Redistributions of source code must retain the above copyright notice, this 12 | list of conditions and the following disclaimer. 13 | 14 | 2. Redistributions in binary form must reproduce the above copyright notice, 15 | this list of conditions and the following disclaimer in the documentation 16 | and/or other materials provided with the distribution. 17 | 18 | 3. Neither the name of the copyright holder nor the names of its 19 | contributors may be used to endorse or promote products derived from 20 | this software without specific prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 26 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 28 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | =end 33 | require 'ffi' 34 | 35 | module K4 36 | extend FFI::Library 37 | ffi_lib 'libk4.so' # specify the path to the shared library 38 | 39 | class KeyValuePair < FFI::Struct 40 | layout :key, :pointer, 41 | :value, :pointer 42 | end 43 | 44 | class KeyValuePairArray < FFI::Struct 45 | layout :pairs, :pointer, 46 | :numPairs, :int 47 | end 48 | 49 | class IterNextReturn < FFI::Struct 50 | layout :r0, :pointer, 51 | :r1, :pointer 52 | end 53 | 54 | class IterPrevReturn < FFI::Struct 55 | layout :r0, :pointer, 56 | :r1, :pointer 57 | end 58 | 59 | attach_function :db_open, [:string, :int, :int, :int, :int], :pointer 60 | attach_function :db_close, [:pointer], :int 61 | attach_function :db_put, [:pointer, :string, :int, :string, :int, :int64], :int 62 | attach_function :db_get, [:pointer, :string, :int], :string 63 | attach_function :db_delete, [:pointer, :string, :int], :int 64 | attach_function :begin_transaction, [:pointer], :pointer 65 | attach_function :add_operation, [:pointer, :int, :string, :int, :string, :int], :int 66 | attach_function :remove_transaction, [:pointer, :pointer], :void 67 | attach_function :commit_transaction, [:pointer, :pointer], :int 68 | attach_function :rollback_transaction, [:pointer, :pointer], :int 69 | attach_function :recover_from_wal, [:pointer], :int 70 | attach_function :range_, [:pointer, :string, :int, :string, :int], KeyValuePairArray.by_value 71 | attach_function :nrange, [:pointer, :string, :int, :string, :int], KeyValuePairArray.by_value 72 | attach_function :greater_than, [:pointer, :string, :int], KeyValuePairArray.by_value 73 | attach_function :less_than, [:pointer, :string, :int], KeyValuePairArray.by_value 74 | attach_function :nget, [:pointer, :string, :int], KeyValuePairArray.by_value 75 | attach_function :greater_than_eq, [:pointer, :string, :int], KeyValuePairArray.by_value 76 | attach_function :less_than_eq, [:pointer, :string, :int], KeyValuePairArray.by_value 77 | attach_function :new_iterator, [:pointer], :pointer 78 | attach_function :iter_next, [:pointer], IterNextReturn.by_value 79 | attach_function :iter_prev, [:pointer], IterPrevReturn.by_value 80 | attach_function :iter_reset, [:pointer], :void 81 | attach_function :iter_close, [:pointer], :void 82 | attach_function :escalate_flush, [:pointer], :int 83 | attach_function :escalate_compaction, [:pointer], :int 84 | end -------------------------------------------------------------------------------- /ffi/ruby/readme.md: -------------------------------------------------------------------------------- 1 | # K4 Ruby FFI 2 | This is an example library that demonstrates how to use a K4 FFI in Ruby using the shared K4 C library. 3 | 4 | ## Example 5 | ``` 6 | db = K4.db_open("data", 1024, 60, 1, 1) 7 | if db.null? 8 | puts "Failed to open database" 9 | exit 10 | end 11 | 12 | key = "key1" 13 | value = "value1" 14 | if K4.db_put(db, key, key.length, value, value.length, -1) != 0 15 | puts "Failed to put key-value pair" 16 | K4.db_close(db) 17 | exit 18 | end 19 | 20 | retrieved_value = K4.db_get(db, key, key.length) 21 | if retrieved_value.null? 22 | puts "Failed to get value" 23 | K4.db_close(db) 24 | exit 25 | end 26 | 27 | puts "Retrieved value: #{retrieved_value.read_string}" 28 | 29 | if K4.db_delete(db, key, key.length) != 0 30 | puts "Failed to delete key-value pair" 31 | K4.db_close(db) 32 | exit 33 | end 34 | 35 | if K4.db_close(db) != 0 36 | puts "Failed to close database" 37 | end 38 | ``` -------------------------------------------------------------------------------- /ffi/rust/k4.rs: -------------------------------------------------------------------------------- 1 | // K4 Rust FFI 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | extern crate libc; 32 | 33 | use libc::{c_char, c_int, c_void, int64_t}; 34 | use std::ffi::CStr; 35 | use std::ptr; 36 | 37 | #[repr(C)] 38 | pub struct KeyValuePair { 39 | key: *mut c_char, 40 | value: *mut c_char, 41 | } 42 | 43 | #[repr(C)] 44 | pub struct KeyValuePairArray { 45 | pairs: *mut KeyValuePair, 46 | numPairs: c_int, 47 | } 48 | 49 | #[repr(C)] 50 | pub struct IterNextReturn { 51 | r0: *mut c_char, 52 | r1: *mut c_char, 53 | } 54 | 55 | #[repr(C)] 56 | pub struct IterPrevReturn { 57 | r0: *mut c_char, 58 | r1: *mut c_char, 59 | } 60 | 61 | #[link(name = "libk4")] // Link to K4 C library 62 | extern "C" { 63 | pub fn db_open(directory: *const c_char, memtableFlushThreshold: c_int, compactionInterval: c_int, logging: c_int, compress: c_int) -> *mut c_void; 64 | pub fn db_close(dbPtr: *mut c_void) -> c_int; 65 | pub fn db_put(dbPtr: *mut c_void, key: *const c_char, keyLen: c_int, value: *const c_char, valueLen: c_int, ttl: int64_t) -> c_int; 66 | pub fn db_get(dbPtr: *mut c_void, key: *const c_char, keyLen: c_int) -> *mut c_char; 67 | pub fn db_delete(dbPtr: *mut c_void, key: *const c_char, keyLen: c_int) -> c_int; 68 | pub fn begin_transaction(dbPtr: *mut c_void) -> *mut c_void; 69 | pub fn add_operation(txPtr: *mut c_void, operation: c_int, key: *const c_char, keyLen: c_int, value: *const c_char, valueLen: c_int) -> c_int; 70 | pub fn remove_transaction(dbPtr: *mut c_void, txPtr: *mut c_void); 71 | pub fn commit_transaction(txPtr: *mut c_void, dbPtr: *mut c_void) -> c_int; 72 | pub fn rollback_transaction(txPtr: *mut c_void, dbPtr: *mut c_void) -> c_int; 73 | pub fn recover_from_wal(dbPtr: *mut c_void) -> c_int; 74 | pub fn range_(dbPtr: *mut c_void, start: *const c_char, startLen: c_int, end: *const c_char, endLen: c_int) -> KeyValuePairArray; 75 | pub fn nrange(dbPtr: *mut c_void, start: *const c_char, startLen: c_int, end: *const c_char, endLen: c_int) -> KeyValuePairArray; 76 | pub fn greater_than(dbPtr: *mut c_void, key: *const c_char, keyLen: c_int) -> KeyValuePairArray; 77 | pub fn less_than(dbPtr: *mut c_void, key: *const c_char, keyLen: c_int) -> KeyValuePairArray; 78 | pub fn nget(dbPtr: *mut c_void, key: *const c_char, keyLen: c_int) -> KeyValuePairArray; 79 | pub fn greater_than_eq(dbPtr: *mut c_void, key: *const c_char, keyLen: c_int) -> KeyValuePairArray; 80 | pub fn less_than_eq(dbPtr: *mut c_void, key: *const c_char, keyLen: c_int) -> KeyValuePairArray; 81 | pub fn new_iterator(dbPtr: *mut c_void) -> *mut c_void; 82 | pub fn iter_next(iterPtr: *mut c_void) -> IterNextReturn; 83 | pub fn iter_prev(iterPtr: *mut c_void) -> IterPrevReturn; 84 | pub fn iter_reset(iterPtr: *mut c_void); 85 | pub fn iter_close(iterPtr: *mut c_void); 86 | pub fn escalate_flush(dbPtr: *mut c_void) -> c_int; 87 | pub fn escalate_compaction(dbPtr: *mut c_void) -> c_int; 88 | } -------------------------------------------------------------------------------- /ffi/rust/readme.md: -------------------------------------------------------------------------------- 1 | # K4 Rust FFI 2 | This is an example library that demonstrates how to use a K4 FFI in Rust using the shared K4 C library. 3 | 4 | ## Example 5 | An example of using the FFI to interact with a K4 database is shown below. 6 | ```rust 7 | use std::ffi::CString; 8 | use std::ptr; 9 | 10 | fn main() { 11 | unsafe { 12 | // Open a database 13 | let directory = CString::new("data").expect("CString::new failed"); 14 | let db = db_open(directory.as_ptr(), 1024, 60, 1, 1); 15 | if db.is_null() { 16 | eprintln!("Failed to open database"); 17 | return; 18 | } 19 | 20 | // Put a key-value pair 21 | let key = CString::new("key1").expect("CString::new failed"); 22 | let value = CString::new("value1").expect("CString::new failed"); 23 | if db_put(db, key.as_ptr(), key.as_bytes().len() as c_int, value.as_ptr(), value.as_bytes().len() as c_int, -1) != 0 { 24 | eprintln!("Failed to put key-value pair"); 25 | db_close(db); 26 | return; 27 | } 28 | 29 | // Get the value for the key 30 | let retrieved_value_ptr = db_get(db, key.as_ptr(), key.as_bytes().len() as c_int); 31 | if retrieved_value_ptr.is_null() { 32 | eprintln!("Failed to get value"); 33 | db_close(db); 34 | return; 35 | } 36 | 37 | let retrieved_value = CStr::from_ptr(retrieved_value_ptr).to_string_lossy().into_owned(); 38 | println!("Retrieved value: {}", retrieved_value); 39 | 40 | // Delete the key-value pair 41 | if db_delete(db, key.as_ptr(), key.as_bytes().len() as c_int) != 0 { 42 | eprintln!("Failed to delete key-value pair"); 43 | db_close(db); 44 | return; 45 | } 46 | 47 | // Close the database 48 | if db_close(db) != 0 { 49 | eprintln!("Failed to close database"); 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /fuzz/fuzz.go: -------------------------------------------------------------------------------- 1 | // Package fuzz 2 | // Fuzz generates random byte arrays, strings and key-value pairs for fuzz testing 3 | // BSD 3-Clause License 4 | // 5 | // Copyright (c) 2024, Alex Gaetano Padula 6 | // All rights reserved. 7 | // 8 | // Redistribution and use in source and binary forms, with or without 9 | // modification, are permitted provided that the following conditions are met: 10 | // 11 | // 1. Redistributions of source code must retain the above copyright notice, this 12 | // list of conditions and the following disclaimer. 13 | // 14 | // 2. Redistributions in binary form must reproduce the above copyright notice, 15 | // this list of conditions and the following disclaimer in the documentation 16 | // and/or other materials provided with the distribution. 17 | // 18 | // 3. Neither the name of the copyright holder nor the names of its 19 | // contributors may be used to endorse or promote products derived from 20 | // this software without specific prior written permission. 21 | // 22 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 26 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 28 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | package fuzz 33 | 34 | import ( 35 | "crypto/rand" 36 | "fmt" 37 | "math/big" 38 | "time" 39 | ) 40 | 41 | // RandomString generates random strings of a given length 42 | func RandomString(length int) (string, error) { 43 | const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 44 | b := make([]byte, length) 45 | for i := range b { 46 | num, err := rand.Int(rand.Reader, big.NewInt(int64(len(charset)))) 47 | if err != nil { 48 | return "", err 49 | } 50 | b[i] = charset[num.Int64()] 51 | } 52 | uniqueSuffix := fmt.Sprintf("%d", time.Now().UnixNano())[2:] 53 | return string(b) + uniqueSuffix, nil 54 | } 55 | 56 | // RandomByteArr generates random byte arrays of a given length 57 | func RandomByteArr(length int) ([]byte, error) { 58 | const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 59 | b := make([]byte, length) 60 | for i := range b { 61 | num, err := rand.Int(rand.Reader, big.NewInt(int64(len(charset)))) 62 | if err != nil { 63 | return nil, err 64 | } 65 | b[i] = charset[num.Int64()] 66 | } 67 | return b, nil 68 | } 69 | 70 | // GenerateKeyValuePairs generates a map of random key-value pairs 71 | func GenerateKeyValuePairs(numPairs int) map[string]interface{} { 72 | pairs := make(map[string]interface{}) 73 | 74 | for i := 0; i < numPairs; i++ { 75 | key, _ := RandomString(5 + i%10) 76 | value, _ := RandomString(5 + i%10) 77 | pairs[key] = value 78 | } 79 | 80 | return pairs 81 | } 82 | -------------------------------------------------------------------------------- /fuzz/fuzz_test.go: -------------------------------------------------------------------------------- 1 | // Package fuzz tests 2 | // Fuzz generates random byte arrays, strings and key-value pairs for fuzz testing 3 | // BSD 3-Clause License 4 | // 5 | // Copyright (c) 2024, Alex Gaetano Padula 6 | // All rights reserved. 7 | // 8 | // Redistribution and use in source and binary forms, with or without 9 | // modification, are permitted provided that the following conditions are met: 10 | // 11 | // 1. Redistributions of source code must retain the above copyright notice, this 12 | // list of conditions and the following disclaimer. 13 | // 14 | // 2. Redistributions in binary form must reproduce the above copyright notice, 15 | // this list of conditions and the following disclaimer in the documentation 16 | // and/or other materials provided with the distribution. 17 | // 18 | // 3. Neither the name of the copyright holder nor the names of its 19 | // contributors may be used to endorse or promote products derived from 20 | // this software without specific prior written permission. 21 | // 22 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 26 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 28 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | package fuzz 33 | 34 | import "testing" 35 | 36 | func TestRandomStringUniqueness(t *testing.T) { 37 | const numGenerations = 1000000 38 | const strLength = 10 39 | 40 | seen := make(map[string]struct{}, numGenerations) 41 | 42 | for i := 0; i < numGenerations; i++ { 43 | s, _ := RandomString(strLength) 44 | if _, exists := seen[s]; exists { 45 | t.Fatalf("Duplicate string found: %s", s) 46 | } 47 | seen[s] = struct{}{} 48 | } 49 | } 50 | 51 | func TestRandomByteArrUniqueness(t *testing.T) { 52 | const numGenerations = 1000000 53 | const byteArrLength = 10 54 | 55 | seen := make(map[string]struct{}, numGenerations) 56 | 57 | for i := 0; i < numGenerations; i++ { 58 | b, _ := RandomByteArr(byteArrLength) 59 | s := string(b) 60 | if _, exists := seen[s]; exists { 61 | t.Fatalf("Duplicate byte array found: %s", s) 62 | } 63 | seen[s] = struct{}{} 64 | } 65 | } 66 | 67 | func TestGenerateKeyValuePairs(t *testing.T) { 68 | const numPairs = 1000000 69 | 70 | pairs := GenerateKeyValuePairs(numPairs) 71 | if len(pairs) != numPairs { 72 | t.Fatalf("Expected %d pairs, but got %d", numPairs, len(pairs)) 73 | } 74 | 75 | seenKeys := make(map[string]struct{}, numPairs) 76 | for key := range pairs { 77 | if _, exists := seenKeys[key]; exists { 78 | t.Fatalf("Duplicate key found: %s", key) 79 | } 80 | seenKeys[key] = struct{}{} 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/guycipher/k4 2 | 3 | go 1.23.3 4 | 5 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/guycipher/k4 v1.9.3 h1:cxn3slGV22ExCZPFne6qtJP+UsuuWOG7pyuhqiWLW0U= 2 | github.com/guycipher/k4 v1.9.3/go.mod h1:j1faqUqiDiz9I7Qiu/2l5BjeRzkk/CssoDNm8pG4g8g= 3 | -------------------------------------------------------------------------------- /graphics/bench-k4-2-1-6-chart-round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guycipher/k4/4cce22a371ba223000c03d9b1f2705d940eb39fe/graphics/bench-k4-2-1-6-chart-round.png -------------------------------------------------------------------------------- /graphics/bench-k4-2-1-6-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guycipher/k4/4cce22a371ba223000c03d9b1f2705d940eb39fe/graphics/bench-k4-2-1-6-chart.png -------------------------------------------------------------------------------- /graphics/k4-v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guycipher/k4/4cce22a371ba223000c03d9b1f2705d940eb39fe/graphics/k4-v2.png -------------------------------------------------------------------------------- /graphics/k4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guycipher/k4/4cce22a371ba223000c03d9b1f2705d940eb39fe/graphics/k4.png -------------------------------------------------------------------------------- /hashset/hashset.go: -------------------------------------------------------------------------------- 1 | // Package hashset 2 | // This hashset is mainly in replacement for the bloom filter implementation in K4 3 | // Bloom filters are more compact yes, but they are not as efficient as a hashset 4 | // BSD 3-Clause License 5 | // 6 | // Copyright (c) 2024, Alex Gaetano Padula 7 | // All rights reserved. 8 | // 9 | // Redistribution and use in source and binary forms, with or without 10 | // modification, are permitted provided that the following conditions are met: 11 | // 12 | // 1. Redistributions of source code must retain the above copyright notice, this 13 | // list of conditions and the following disclaimer. 14 | // 15 | // 2. Redistributions in binary form must reproduce the above copyright notice, 16 | // this list of conditions and the following disclaimer in the documentation 17 | // and/or other materials provided with the distribution. 18 | // 19 | // 3. Neither the name of the copyright holder nor the names of its 20 | // contributors may be used to endorse or promote products derived from 21 | // this software without specific prior written permission. 22 | // 23 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 26 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 27 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 29 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 30 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 31 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | package hashset 34 | 35 | import ( 36 | "bytes" 37 | "encoding/gob" 38 | "github.com/guycipher/k4/murmur" 39 | ) 40 | 41 | const initialCapacity = 32 // initial hashset capacity 42 | const loadFactorThreshold = 0.7 // load factor threshold 43 | 44 | // HashSet represents a hash set. 45 | type HashSet struct { 46 | Buckets [][]interface{} // Buckets to store elements 47 | Size int // Number of elements in the set 48 | Capacity int // Capacity of the set 49 | } 50 | 51 | // NewHashSet creates a new instance of HashSet. 52 | func NewHashSet() *HashSet { 53 | return &HashSet{ 54 | Buckets: make([][]interface{}, initialCapacity), // Initialize buckets 55 | Capacity: initialCapacity, // Set initial capacity 56 | } 57 | } 58 | 59 | // Hash function to compute the index for a given value. 60 | func (h *HashSet) hash(value []byte, capacity int) int { 61 | return int(murmur.Hash64(value, 4) % uint64(capacity)) // Use murmur hash 62 | } 63 | 64 | // Add inserts a new element into the set. 65 | func (h *HashSet) Add(value []byte) { 66 | 67 | index := h.hash(value, h.Capacity) // Compute the index 68 | 69 | // Check if the element already exists 70 | for _, item := range h.Buckets[index] { 71 | if bytes.Equal(item.([]byte), value) { 72 | return // Element already exists 73 | } 74 | } 75 | 76 | // Add the element to the set 77 | h.Buckets[index] = append(h.Buckets[index], value) 78 | h.Size++ // Increment the size 79 | 80 | // Resize if the load factor is too high 81 | if float64(h.Size)/float64(h.Capacity) > loadFactorThreshold { // Load factor 82 | h.resize() // Resize the hash set 83 | } 84 | } 85 | 86 | // Resize increases the capacity of the hash set. 87 | func (h *HashSet) resize() { 88 | newCapacity := h.Capacity * 2 // new capacity 89 | newBuckets := make([][]interface{}, newCapacity) // new buckets 90 | 91 | for _, bucket := range h.Buckets { 92 | for _, value := range bucket { 93 | newIndex := h.hash(value.([]byte), newCapacity) // Compute the new index 94 | newBuckets[newIndex] = append(newBuckets[newIndex], value) // Add the value 95 | } 96 | } 97 | 98 | h.Buckets = newBuckets // Update the buckets 99 | h.Capacity = newCapacity // Update the capacity 100 | } 101 | 102 | // Remove deletes an element from the set. 103 | func (h *HashSet) Remove(value []byte) { 104 | index := h.hash(value, h.Capacity) // Compute the index 105 | 106 | // Find the element and remove it 107 | for i, item := range h.Buckets[index] { 108 | if bytes.Equal(item.([]byte), value) { // Element found 109 | h.Buckets[index] = append(h.Buckets[index][:i], h.Buckets[index][i+1:]...) // Remove the element 110 | h.Size-- // Decrement the size 111 | return 112 | } 113 | } 114 | } 115 | 116 | // Contains checks if an element is in the set. 117 | func (h *HashSet) Contains(value []byte) bool { 118 | index := h.hash(value, h.Capacity) // Compute the index 119 | for _, item := range h.Buckets[index] { // Check if the element exists 120 | if bytes.Equal(item.([]byte), value) { // Element found 121 | return true // Element exists 122 | } 123 | } 124 | return false 125 | } 126 | 127 | // Clear removes all elements from the set. 128 | func (h *HashSet) Clear() { 129 | h.Buckets = make([][]interface{}, initialCapacity) // Reset the buckets 130 | h.Size = 0 // Reset the size 131 | h.Capacity = initialCapacity // Reset the capacity 132 | } 133 | 134 | // Serialize encodes the HashSet into a byte slice. 135 | func (h *HashSet) Serialize() ([]byte, error) { 136 | // We just use gob to encode the HashSet 137 | var buf bytes.Buffer 138 | enc := gob.NewEncoder(&buf) 139 | err := enc.Encode(h) 140 | if err != nil { 141 | return nil, err 142 | } 143 | return buf.Bytes(), nil 144 | } 145 | 146 | // Deserialize decodes the byte slice into a HashSet. 147 | func Deserialize(data []byte) (*HashSet, error) { 148 | // We just use gob to decode the byte slice 149 | var h HashSet 150 | buf := bytes.NewBuffer(data) 151 | dec := gob.NewDecoder(buf) 152 | err := dec.Decode(&h) 153 | if err != nil { 154 | return nil, err 155 | } 156 | return &h, nil 157 | } 158 | -------------------------------------------------------------------------------- /hashset/hashset_test.go: -------------------------------------------------------------------------------- 1 | // Package hashset tests 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package hashset 32 | 33 | import ( 34 | "fmt" 35 | "github.com/guycipher/k4/pager" 36 | "os" 37 | "testing" 38 | ) 39 | 40 | func TestHashSet_Add(t *testing.T) { 41 | set := NewHashSet() 42 | value := []byte("test") 43 | 44 | set.Add(value) 45 | if !set.Contains(value) { 46 | t.Errorf("Expected set to contain %v", value) 47 | } 48 | } 49 | 50 | func TestHashSet_Remove(t *testing.T) { 51 | set := NewHashSet() 52 | value := []byte("test") 53 | 54 | set.Add(value) 55 | set.Remove(value) 56 | if set.Contains(value) { 57 | t.Errorf("Expected set to not contain %v", value) 58 | } 59 | } 60 | 61 | func TestHashSet_Contains(t *testing.T) { 62 | set := NewHashSet() 63 | value := []byte("test") 64 | 65 | if set.Contains(value) { 66 | t.Errorf("Expected set to not contain %v", value) 67 | } 68 | 69 | set.Add(value) 70 | if !set.Contains(value) { 71 | t.Errorf("Expected set to contain %v", value) 72 | } 73 | } 74 | 75 | func TestHashSet_Size(t *testing.T) { 76 | set := NewHashSet() 77 | value1 := []byte("test1") 78 | value2 := []byte("test2") 79 | 80 | if set.Size != 0 { 81 | t.Errorf("Expected size to be 0, got %d", set.Size) 82 | } 83 | 84 | set.Add(value1) 85 | if set.Size != 1 { 86 | t.Errorf("Expected size to be 1, got %d", set.Size) 87 | } 88 | 89 | set.Add(value2) 90 | if set.Size != 2 { 91 | t.Errorf("Expected size to be 2, got %d", set.Size) 92 | } 93 | 94 | set.Remove(value1) 95 | if set.Size != 1 { 96 | t.Errorf("Expected size to be 1, got %d", set.Size) 97 | } 98 | } 99 | 100 | func TestHashSet_Clear(t *testing.T) { 101 | set := NewHashSet() 102 | value := []byte("test") 103 | 104 | set.Add(value) 105 | set.Clear() 106 | if set.Size != 0 { 107 | t.Errorf("Expected size to be 0 after clear, got %d", set.Size) 108 | } 109 | if set.Contains(value) { 110 | t.Errorf("Expected set to not contain %v after clear", value) 111 | } 112 | } 113 | 114 | func TestHashSetAddCheckManyValues(t *testing.T) { 115 | set := NewHashSet() 116 | 117 | for i := 0; i < 10_000; i++ { 118 | value := []byte("test" + fmt.Sprintf("%d", i)) 119 | set.Add(value) 120 | if !set.Contains(value) { 121 | t.Errorf("Expected set to contain %v", value) 122 | } 123 | } 124 | } 125 | 126 | func TestHashSet_SerializeDeserialize(t *testing.T) { 127 | set := NewHashSet() 128 | values := [][]byte{ 129 | []byte("test1"), 130 | []byte("test2"), 131 | []byte("test3"), 132 | } 133 | 134 | for _, value := range values { 135 | set.Add(value) 136 | } 137 | 138 | serialized, err := set.Serialize() 139 | if err != nil { 140 | t.Error(err) 141 | } 142 | 143 | deserializedSet, err := Deserialize(serialized) 144 | if err != nil { 145 | t.Error(err) 146 | } 147 | 148 | for _, value := range values { 149 | if !deserializedSet.Contains(value) { 150 | t.Errorf("Expected deserialized set to contain %v", value) 151 | } 152 | } 153 | 154 | if deserializedSet.Size != set.Size { 155 | t.Errorf("Expected deserialized set size to be %d, got %d", set.Size, deserializedSet.Size) 156 | } 157 | } 158 | 159 | func TestHashSet_SerializeDeserialize_Pager(t *testing.T) { 160 | defer os.Remove("hashset.test") 161 | 162 | p, err := pager.OpenPager("hashset.test", os.O_CREATE|os.O_RDWR, 0644) 163 | if err != nil { 164 | t.Fatalf("Failed to open file: %v", err) 165 | } 166 | 167 | set := NewHashSet() 168 | values := [][]byte{ 169 | []byte("test1"), 170 | []byte("test2"), 171 | []byte("test3"), 172 | } 173 | 174 | for _, value := range values { 175 | set.Add(value) 176 | } 177 | 178 | serialized, err := set.Serialize() 179 | if err != nil { 180 | t.Error(err) 181 | } 182 | 183 | pgNum, err := p.Write(serialized) 184 | if err != nil { 185 | return 186 | } 187 | 188 | data, err := p.GetPage(pgNum) 189 | if err != nil { 190 | return 191 | } 192 | 193 | deserializedSet, err := Deserialize(data) 194 | if err != nil { 195 | t.Error(err) 196 | } 197 | 198 | for _, value := range values { 199 | if !deserializedSet.Contains(value) { 200 | t.Errorf("Expected deserialized set to contain %v", value) 201 | } 202 | } 203 | 204 | if deserializedSet.Size != set.Size { 205 | t.Errorf("Expected deserialized set size to be %d, got %d", set.Size, deserializedSet.Size) 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /k4_bench_test.go: -------------------------------------------------------------------------------- 1 | // Package k4 benchmarking 2 | // BSD 3-Clause License 3 | // @depreciated please refer to github.com/guycipher/k4/v2 for the latest version 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package k4 32 | 33 | import ( 34 | "fmt" 35 | "math/rand" 36 | "os" 37 | "testing" 38 | ) 39 | 40 | func BenchmarkK4_Put(b *testing.B) { 41 | // Setup 42 | dir := "testdata" 43 | os.MkdirAll(dir, 0755) 44 | defer os.RemoveAll(dir) 45 | 46 | k4, err := Open(dir, (1024*1024)*50, 60, false, false) 47 | if err != nil { 48 | b.Fatalf("Failed to open K4: %v", err) 49 | } 50 | defer k4.Close() 51 | 52 | // Benchmark 53 | b.ResetTimer() 54 | for i := 0; i < b.N; i++ { 55 | key := []byte(fmt.Sprintf("key-%d", i)) 56 | value := []byte(fmt.Sprintf("value-%d", i)) 57 | err := k4.Put(key, value, nil) 58 | if err != nil { 59 | b.Fatalf("Failed to put key-value pair: %v", err) 60 | } 61 | } 62 | } 63 | 64 | func BenchmarkK4_Get(b *testing.B) { 65 | // Setup 66 | dir := "testdata" 67 | os.MkdirAll(dir, 0755) 68 | defer os.RemoveAll(dir) 69 | 70 | k4, err := Open(dir, 1024*1024, 60, false, false) 71 | if err != nil { 72 | b.Fatalf("Failed to open K4: %v", err) 73 | } 74 | defer k4.Close() 75 | 76 | // Insert some data 77 | for i := 0; i < 1000; i++ { 78 | key := []byte(fmt.Sprintf("key-%d", i)) 79 | value := []byte(fmt.Sprintf("value-%d", i)) 80 | err := k4.Put(key, value, nil) 81 | if err != nil { 82 | b.Fatalf("Failed to put key-value pair: %v", err) 83 | } 84 | } 85 | 86 | // Benchmark 87 | b.ResetTimer() 88 | for i := 0; i < b.N; i++ { 89 | key := []byte(fmt.Sprintf("key-%d", rand.Intn(1000))) 90 | _, err := k4.Get(key) 91 | if err != nil { 92 | b.Fatalf("Failed to get key: %v", err) 93 | } 94 | } 95 | } 96 | 97 | func BenchmarkK4_Delete(b *testing.B) { 98 | // Setup 99 | dir := "testdata" 100 | os.MkdirAll(dir, 0755) 101 | defer os.RemoveAll(dir) 102 | 103 | k4, err := Open(dir, 1024*1024, 60, false, false) 104 | if err != nil { 105 | b.Fatalf("Failed to open K4: %v", err) 106 | } 107 | defer k4.Close() 108 | 109 | // Insert some data 110 | for i := 0; i < 1000; i++ { 111 | key := []byte(fmt.Sprintf("key-%d", i)) 112 | value := []byte(fmt.Sprintf("value-%d", i)) 113 | err := k4.Put(key, value, nil) 114 | if err != nil { 115 | b.Fatalf("Failed to put key-value pair: %v", err) 116 | } 117 | } 118 | 119 | // Benchmark 120 | b.ResetTimer() 121 | for i := 0; i < b.N; i++ { 122 | key := []byte(fmt.Sprintf("key-%d", rand.Intn(1000))) 123 | err := k4.Delete(key) 124 | if err != nil { 125 | b.Fatalf("Failed to delete key: %v", err) 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /murmur/murmur.go: -------------------------------------------------------------------------------- 1 | // Package murmur 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package murmur 32 | 33 | import ( 34 | "encoding/binary" 35 | ) 36 | 37 | const ( 38 | // Constants for 64-bit hash 39 | m64 = 0xff51afd7ed558ccd 40 | seed64 = 0xc4ceb9fe1a85ec53 41 | 42 | // Constants for 32-bit hash 43 | m32 = 0xcc9e2d51 44 | seed32 = 0x1b873593 45 | ) 46 | 47 | // scramble64 performs the scrambling operation for 64-bit hash 48 | func scramble64(k uint64) uint64 { 49 | k *= m64 50 | k = (k << 31) | (k >> 33) // Rotate left by 31 bits 51 | k *= seed64 52 | return k 53 | } 54 | 55 | // Hash64 computes a 64-bit MurmurHash3 hash for the given key and seed 56 | func Hash64(key []byte, seed uint64) uint64 { 57 | h := seed // initialize hash with seed 58 | var k uint64 59 | 60 | // Process the input in 8-byte chunks 61 | for i := 0; i < len(key)/8; i++ { 62 | k = binary.LittleEndian.Uint64(key[i*8:]) 63 | h ^= scramble64(k) 64 | h = (h << 27) | (h >> 37) // Rotate left by 27 bits 65 | h = h*5 + 0x52dce729 66 | } 67 | 68 | // Process the remaining bytes 69 | k = 0 70 | for i := 0; i < len(key)&7; i++ { 71 | k <<= 8 72 | k |= uint64(key[len(key)-1-i]) 73 | } 74 | h ^= scramble64(k) 75 | 76 | // Finalize the hash 77 | h ^= uint64(len(key)) 78 | h ^= h >> 33 79 | h *= m64 80 | h ^= h >> 33 81 | h *= seed64 82 | h ^= h >> 33 83 | 84 | return h // Return the final hash 85 | } 86 | 87 | // scramble32 performs the scrambling operation for 32-bit hash 88 | func scramble32(k uint32) uint32 { 89 | k *= m32 90 | k = (k << 15) | (k >> 17) // Rotate left by 15 bits 91 | k *= seed32 92 | return k 93 | } 94 | 95 | // Hash32 computes a 32-bit MurmurHash3 hash for the given key and seed 96 | func Hash32(key []byte, seed uint32) uint32 { 97 | h := seed // Initialize hash with seed 98 | var k uint32 99 | 100 | // Process the input in 4-byte chunks 101 | for i := 0; i < len(key)/4; i++ { 102 | k = binary.LittleEndian.Uint32(key[i*4:]) 103 | h ^= scramble32(k) 104 | h = (h << 13) | (h >> 19) // Rotate left by 13 bits 105 | h = h*5 + 0xe6546b64 106 | } 107 | 108 | // Process the remaining bytes 109 | k = 0 110 | for i := 0; i < len(key)&3; i++ { 111 | k <<= 8 112 | k |= uint32(key[len(key)-1-i]) 113 | } 114 | h ^= scramble32(k) 115 | 116 | // Finalize the hash 117 | h ^= uint32(len(key)) 118 | h ^= h >> 16 119 | h *= 0x85ebca6b 120 | h ^= h >> 13 121 | h *= 0xc2b2ae35 122 | h ^= h >> 16 123 | 124 | return h 125 | } 126 | -------------------------------------------------------------------------------- /murmur/murmur_test.go: -------------------------------------------------------------------------------- 1 | // Package murmur tests 2 | // murmur3 inspired 3 | // BSD 3-Clause License 4 | // 5 | // Copyright (c) 2024, Alex Gaetano Padula 6 | // All rights reserved. 7 | // 8 | // Redistribution and use in source and binary forms, with or without 9 | // modification, are permitted provided that the following conditions are met: 10 | // 11 | // 1. Redistributions of source code must retain the above copyright notice, this 12 | // list of conditions and the following disclaimer. 13 | // 14 | // 2. Redistributions in binary form must reproduce the above copyright notice, 15 | // this list of conditions and the following disclaimer in the documentation 16 | // and/or other materials provided with the distribution. 17 | // 18 | // 3. Neither the name of the copyright holder nor the names of its 19 | // contributors may be used to endorse or promote products derived from 20 | // this software without specific prior written permission. 21 | // 22 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 26 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 28 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | package murmur 33 | 34 | import ( 35 | "github.com/guycipher/k4/fuzz" 36 | "testing" 37 | "time" 38 | ) 39 | 40 | func TestHash64(t *testing.T) { 41 | 42 | tests := []struct { 43 | key []byte 44 | seed uint64 45 | want uint64 46 | }{ 47 | {[]byte("hello"), 0, 0xf369cd39c641eb89}, 48 | {[]byte("world"), 0, 0x96a5312ceeb4b275}, 49 | {[]byte("murmur"), 0, 0xc40377c960d8b391}, 50 | {[]byte("hash"), 0, 0xe7fcedc45a9406da}, 51 | } // Can fail on different machines 52 | 53 | for _, tt := range tests { 54 | t.Run(string(tt.key), func(t *testing.T) { 55 | if got := Hash64(tt.key, tt.seed); got != tt.want { 56 | t.Errorf("Hash64() = %v, want %v", got, tt.want) 57 | } 58 | }) 59 | } 60 | } 61 | 62 | func TestHash64Many(t *testing.T) { 63 | // Generate a large number of keys 64 | keys := make([][]byte, 100000) 65 | for i := range keys { 66 | str, _ := fuzz.RandomString(10) 67 | keys[i] = []byte(str) 68 | } 69 | 70 | tt := time.Now() 71 | // Compute the hash of each key 72 | for _, key := range keys { 73 | _ = Hash64(key, 0) 74 | } 75 | 76 | t.Logf("Time taken to hash 100k keys %v", time.Since(tt)) 77 | 78 | } 79 | 80 | func TestHash32(t *testing.T) { 81 | 82 | tests := []struct { 83 | key []byte 84 | seed uint32 85 | want uint32 86 | }{ 87 | {[]byte("hello"), 0, 0x248bfa47}, 88 | {[]byte("world"), 0, 0xfb963cfb}, 89 | {[]byte("murmur"), 0, 0x73f313cd}, 90 | {[]byte("hash"), 0, 0x56c454fb}, 91 | } 92 | 93 | for _, tt := range tests { 94 | t.Run(string(tt.key), func(t *testing.T) { 95 | if got := Hash32(tt.key, tt.seed); got != tt.want { 96 | t.Errorf("Hash32() = %v, want %v", got, tt.want) 97 | } 98 | }) 99 | } 100 | } 101 | 102 | func TestHash32Many(t *testing.T) { 103 | // Generate a large number of keys 104 | keys := make([][]byte, 100000) 105 | for i := range keys { 106 | str, _ := fuzz.RandomString(10) 107 | keys[i] = []byte(str) 108 | } 109 | 110 | tt := time.Now() 111 | // Compute the hash of each key 112 | for _, key := range keys { 113 | _ = Hash32(key, 0) 114 | } 115 | 116 | t.Logf("Time taken to hash 100k keys %v", time.Since(tt)) 117 | 118 | } 119 | 120 | func BenchmarkHash64(b *testing.B) { 121 | key := []byte("benchmarking 64-bit murmur3 hash function") 122 | seed := uint64(0) 123 | 124 | for i := 0; i < b.N; i++ { 125 | _ = Hash64(key, seed) 126 | } 127 | } 128 | 129 | func BenchmarkHash32(b *testing.B) { 130 | key := []byte("benchmarking 32-bit murmur3 hash function") 131 | seed := uint32(0) 132 | 133 | for i := 0; i < b.N; i++ { 134 | _ = Hash32(key, seed) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /pager/pager_test.go: -------------------------------------------------------------------------------- 1 | // Package pager tests 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package pager 32 | 33 | import ( 34 | "bytes" 35 | "os" 36 | "sync" 37 | "testing" 38 | ) 39 | 40 | func TestOpenPager(t *testing.T) { 41 | file, err := os.CreateTemp("", "pager_test") 42 | if err != nil { 43 | t.Fatalf("Failed to create temp file: %v", err) 44 | } 45 | defer os.Remove(file.Name()) 46 | 47 | p, err := OpenPager(file.Name(), os.O_RDWR|os.O_CREATE, 0666) 48 | if err != nil { 49 | t.Fatalf("Failed to open pager: %v", err) 50 | } 51 | if p == nil { 52 | t.Fatalf("Pager is nil") 53 | } 54 | if err := p.Close(); err != nil { 55 | t.Fatalf("Failed to close pager: %v", err) 56 | } 57 | } 58 | 59 | func TestWriteAndGetPage(t *testing.T) { 60 | file, err := os.CreateTemp("", "pager_test") 61 | if err != nil { 62 | t.Fatalf("Failed to create temp file: %v", err) 63 | } 64 | defer os.Remove(file.Name()) 65 | 66 | p, err := OpenPager(file.Name(), os.O_RDWR|os.O_CREATE, 0666) 67 | if err != nil { 68 | t.Fatalf("Failed to open pager: %v", err) 69 | } 70 | defer p.Close() 71 | 72 | data := []byte("Hello, World!") 73 | pageID, err := p.Write(data) 74 | if err != nil { 75 | t.Fatalf("Failed to write data: %v", err) 76 | } 77 | if pageID != 0 { 78 | t.Fatalf("Expected pageID 0, got %d", pageID) 79 | } 80 | 81 | readData, err := p.GetPage(pageID) 82 | if err != nil { 83 | t.Fatalf("Failed to get page: %v", err) 84 | } 85 | if !bytes.Equal(data, bytes.Trim(readData, "\x00")) { 86 | t.Fatalf("Expected %s, got %s", data, readData) 87 | } 88 | } 89 | 90 | func TestWriteToMultiplePages(t *testing.T) { 91 | file, err := os.CreateTemp("", "pager_test") 92 | if err != nil { 93 | t.Fatalf("Failed to create temp file: %v", err) 94 | } 95 | defer os.Remove(file.Name()) 96 | 97 | p, err := OpenPager(file.Name(), os.O_RDWR|os.O_CREATE, 0666) 98 | if err != nil { 99 | t.Fatalf("Failed to open pager: %v", err) 100 | } 101 | defer p.Close() 102 | 103 | data := make([]byte, PAGE_SIZE*2) 104 | str := "Hello, World!" 105 | 106 | for i := len(data) - len(str); i < len(data); i += len(str) { 107 | copy(data[i:], str) 108 | } 109 | 110 | pageID, err := p.Write(data) 111 | if err != nil { 112 | t.Fatalf("Failed to write data: %v", err) 113 | } 114 | 115 | // Read back the first page 116 | readData, err := p.GetPage(pageID) 117 | if err != nil { 118 | t.Fatalf("Failed to get first page: %v", err) 119 | } 120 | 121 | // end of readData should be str 122 | if !bytes.Equal([]byte(str), readData[len(readData)-len(str):]) { 123 | t.Fatalf("Expected %s, got %s", str, readData[len(readData)-len(str):]) 124 | 125 | } 126 | 127 | } 128 | 129 | func TestPagerSizeAndCount(t *testing.T) { 130 | file, err := os.CreateTemp("", "pager_test") 131 | if err != nil { 132 | t.Fatalf("Failed to create temp file: %v", err) 133 | } 134 | defer os.Remove(file.Name()) 135 | 136 | p, err := OpenPager(file.Name(), os.O_RDWR|os.O_CREATE, 0666) 137 | if err != nil { 138 | t.Fatalf("Failed to open pager: %v", err) 139 | } 140 | defer p.Close() 141 | 142 | data := []byte("Hello, World!") 143 | _, err = p.Write(data) 144 | if err != nil { 145 | t.Fatalf("Failed to write data: %v", err) 146 | } 147 | 148 | expectedSize := int64(PAGE_SIZE + HEADER_SIZE) 149 | if p.Size() != expectedSize { 150 | t.Fatalf("Expected size %d, got %d", expectedSize, p.Size()) 151 | } 152 | if p.Count() != 1 { 153 | t.Fatalf("Expected count 1, got %d", p.Count()) 154 | } 155 | } 156 | 157 | func TestPagerConcurrency(t *testing.T) { 158 | file, err := os.CreateTemp("", "pager_test") 159 | if err != nil { 160 | t.Fatalf("Failed to create temp file: %v", err) 161 | } 162 | defer os.Remove(file.Name()) 163 | 164 | p, err := OpenPager(file.Name(), os.O_RDWR|os.O_CREATE, 0666) 165 | if err != nil { 166 | t.Fatalf("Failed to open pager: %v", err) 167 | } 168 | defer p.Close() 169 | 170 | data := []byte("Hello, World!") 171 | var wg sync.WaitGroup 172 | 173 | for i := 0; i < 10; i++ { 174 | wg.Add(1) 175 | go func() { 176 | defer wg.Done() 177 | _, err := p.Write(data) 178 | if err != nil { 179 | t.Errorf("Failed to write data: %v", err) 180 | } 181 | }() 182 | } 183 | 184 | wg.Wait() 185 | if p.Count() != 10 { 186 | t.Fatalf("Expected count 10, got %d", p.Count()) 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /server_example/Dockerfile: -------------------------------------------------------------------------------- 1 | # use version that k4 is using 2 | FROM golang:1.23.0-alpine 3 | 4 | # set the working directory 5 | WORKDIR /app 6 | 7 | # copy mod files and download dependencies 8 | COPY go.mod go.sum ./ 9 | RUN go mod download 10 | 11 | # copy server files 12 | COPY . . 13 | 14 | # build the application 15 | RUN go build -o server_example main.go 16 | 17 | # expose server port 18 | EXPOSE 8000 19 | 20 | # command to run the server 21 | CMD ["./server_example"] -------------------------------------------------------------------------------- /server_example/go.mod: -------------------------------------------------------------------------------- 1 | module k4_server_example 2 | 3 | go 1.23.3 4 | 5 | require github.com/guycipher/k4/v2 v2.1.8 6 | 7 | require github.com/guycipher/k4 v1.9.7 // indirect 8 | -------------------------------------------------------------------------------- /server_example/go.sum: -------------------------------------------------------------------------------- 1 | github.com/guycipher/k4 v1.9.0 h1:sZQfs1fDDI7ljRKZWlBsp8agpIcHg2l0xFTnQYturIo= 2 | github.com/guycipher/k4 v1.9.0/go.mod h1:j1faqUqiDiz9I7Qiu/2l5BjeRzkk/CssoDNm8pG4g8g= 3 | github.com/guycipher/k4 v1.9.4 h1:PEv+qxgGYszWt2mzfQKHXiU7RgwCJEfYdQYWKUR9wPk= 4 | github.com/guycipher/k4 v1.9.4/go.mod h1:nY9tmdEu0jfDmTT2pFVHQhRi0RE5+jsz4Caa9jSub+Y= 5 | github.com/guycipher/k4 v1.9.7 h1:elWXP9hB8XAvyXIUPRtr/0DVzGuZJGJAFzaCoRPY98o= 6 | github.com/guycipher/k4 v1.9.7/go.mod h1:j1faqUqiDiz9I7Qiu/2l5BjeRzkk/CssoDNm8pG4g8g= 7 | github.com/guycipher/k4/v2 v2.1.2 h1:qHxHBNxP3FgawwiNWYAB3ptmTefYSDPrvSc2KfNEbNQ= 8 | github.com/guycipher/k4/v2 v2.1.2/go.mod h1:chF/U//rx6YgoD3+SsShWekwI24XCA6PiJHzNavabFM= 9 | github.com/guycipher/k4/v2 v2.1.3 h1:vOIiDIHFFaK2UEJmLcD/cEKnKTOaiJj17X8OEVWp/tk= 10 | github.com/guycipher/k4/v2 v2.1.3/go.mod h1:chF/U//rx6YgoD3+SsShWekwI24XCA6PiJHzNavabFM= 11 | github.com/guycipher/k4/v2 v2.1.4 h1:3ufXELKQLQP3i8aWWdMM8KkCO2H52yWR91cKYlJejKQ= 12 | github.com/guycipher/k4/v2 v2.1.4/go.mod h1:chF/U//rx6YgoD3+SsShWekwI24XCA6PiJHzNavabFM= 13 | github.com/guycipher/k4/v2 v2.1.5 h1:nuV2TO+QQuY4lsxpSPGRXK72kL1VECgrxGlA1eXiZO8= 14 | github.com/guycipher/k4/v2 v2.1.5/go.mod h1:chF/U//rx6YgoD3+SsShWekwI24XCA6PiJHzNavabFM= 15 | github.com/guycipher/k4/v2 v2.1.6 h1:qm2y/LtxAJDXrf9QfcH4O43CtUH9n7F5HSSiK+FKtPo= 16 | github.com/guycipher/k4/v2 v2.1.6/go.mod h1:ylj3EpecOYJCxK3U60TagKARq7OetuutkAPW+QXD4LM= 17 | github.com/guycipher/k4/v2 v2.1.7 h1:5roJ1+lIFA053Tb7S3axM5TQiraFvf9NkIb0r8ZT5v4= 18 | github.com/guycipher/k4/v2 v2.1.7/go.mod h1:ylj3EpecOYJCxK3U60TagKARq7OetuutkAPW+QXD4LM= 19 | github.com/guycipher/k4/v2 v2.1.8 h1:jcTq7lk24mczPzmzY4/RXbH7SIgIK5XVVF4NeZ/3Xa0= 20 | github.com/guycipher/k4/v2 v2.1.8/go.mod h1:ylj3EpecOYJCxK3U60TagKARq7OetuutkAPW+QXD4LM= 21 | -------------------------------------------------------------------------------- /server_example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "fmt" 8 | "github.com/guycipher/k4/v2" 9 | "log" 10 | "net" 11 | "os" 12 | "os/signal" 13 | "syscall" 14 | ) 15 | 16 | // handleConnection reads commands from the client and executes them on the K4 database. 17 | func handleConnection(conn net.Conn, db *k4.K4) { 18 | defer conn.Close() // defer closing the connection 19 | 20 | reader := bufio.NewReader(conn) 21 | for { 22 | message, err := reader.ReadBytes('\n') // Read up to the next newline 23 | if err != nil { 24 | log.Println("Error reading from client:", err) 25 | return 26 | } 27 | 28 | message = bytes.TrimSpace(message) 29 | parts := bytes.SplitN(message, []byte(" "), 3) 30 | if len(parts) < 2 { 31 | conn.Write([]byte("Invalid command\n")) 32 | continue 33 | } 34 | 35 | command := bytes.ToUpper(parts[0]) 36 | key := []byte(parts[1]) 37 | var response string 38 | 39 | switch { 40 | case bytes.HasPrefix(command, []byte("PUT")): 41 | // i.e. PUT key value 42 | if len(parts) < 3 { 43 | response = "PUT command requires a value\n" 44 | } else { 45 | value := []byte(parts[2]) 46 | err := db.Put(key, value, nil) 47 | if err != nil { 48 | response = fmt.Sprintf("Error putting key: %v\n", err) 49 | } else { 50 | response = "OK\n" 51 | } 52 | } 53 | case bytes.HasPrefix(command, []byte("GET")): 54 | // i.e. GET key 55 | value, err := db.Get(key) 56 | if err != nil { 57 | response = fmt.Sprintf("Error getting key: %v\n", err) 58 | } else { 59 | response = fmt.Sprintf("Value: %s\n", value) 60 | } 61 | case bytes.HasPrefix(command, []byte("DELETE")): 62 | // i.e. DELETE key 63 | err := db.Delete(key) 64 | if err != nil { 65 | response = fmt.Sprintf("Error deleting key: %v\n", err) 66 | } else { 67 | response = "OK\n" 68 | } 69 | default: 70 | response = "Unknown command\n" 71 | } 72 | 73 | conn.Write([]byte(response)) 74 | } 75 | } 76 | 77 | // startServer starts a TCP server that listens for incoming connections and handles them 78 | // using the handleConnection function 79 | func startServer(ctx context.Context, address string, db *k4.K4) { 80 | listener, err := net.Listen("tcp", address) 81 | if err != nil { 82 | log.Fatalf("Error starting server: %v", err) 83 | } 84 | defer listener.Close() 85 | 86 | log.Printf("Server started on %s\n", address) 87 | for { 88 | conn, err := listener.Accept() 89 | if err != nil { 90 | select { 91 | case <-ctx.Done(): 92 | log.Println("Server shutting down...") 93 | return 94 | default: 95 | log.Println("Error accepting connection:", err) 96 | continue 97 | } 98 | } 99 | go handleConnection(conn, db) 100 | } 101 | } 102 | 103 | func main() { 104 | directory := "./data" 105 | memtableFlushThreshold := 1024 * 1024 // 1MB 106 | compactionInterval := 3600 // 1 hour 107 | logging := true 108 | compression := false 109 | 110 | db, err := k4.Open(directory, memtableFlushThreshold, compactionInterval, logging, compression) 111 | if err != nil { 112 | log.Fatalf("Failed to open K4: %v", err) 113 | } 114 | 115 | // Create a context that is canceled on SIGINT or SIGTERM 116 | ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) 117 | defer stop() 118 | 119 | go startServer(ctx, ":8000", db) 120 | 121 | // Wait for the context to be canceled 122 | <-ctx.Done() 123 | 124 | db.Close() // Close the database 125 | 126 | log.Println("Server stopped") 127 | } 128 | -------------------------------------------------------------------------------- /server_example/readme.md: -------------------------------------------------------------------------------- 1 | ## K4 Server example 2 | This is a basic server example using K4 as a key-value store. The server will listen on port 8000. 3 | 4 | The protocol is simple, it's a simple text protocol with the following commands 5 | ``` 6 | PUT key value 7 | GET key 8 | DELETE key 9 | ``` 10 | 11 | ### Example 12 | ``` 13 | $ nc localhost 8000 14 | put hello world 15 | OK 16 | get hello 17 | Value: world 18 | ``` -------------------------------------------------------------------------------- /skiplist/skiplist_test.go: -------------------------------------------------------------------------------- 1 | // Package skiplist tests 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package skiplist 32 | 33 | import ( 34 | "testing" 35 | "time" 36 | ) 37 | 38 | func TestNewSkipList(t *testing.T) { 39 | sl := NewSkipList(12, 0.25) 40 | if sl == nil { 41 | t.Fatal("Expected new skip list to be created") 42 | } 43 | if sl.level != 0 { 44 | t.Fatalf("Expected level to be 0, got %d", sl.level) 45 | } 46 | if sl.size != 0 { 47 | t.Fatalf("Expected size to be 0, got %d", sl.size) 48 | } 49 | } 50 | 51 | func TestInsertAndSearch(t *testing.T) { 52 | sl := NewSkipList(12, 0.25) 53 | key := []byte("key1") 54 | value := []byte("value1") 55 | sl.Insert(key, value, nil) 56 | 57 | val, found := sl.Search(key) 58 | if !found { 59 | t.Fatal("Expected to find the key") 60 | } 61 | if string(val) != string(value) { 62 | t.Fatalf("Expected value %s, got %s", value, val) 63 | } 64 | } 65 | 66 | func TestInsertWithTTL(t *testing.T) { 67 | sl := NewSkipList(12, 0.25) 68 | key := []byte("key1") 69 | value := []byte("value1") 70 | ttl := 1 * time.Second 71 | sl.Insert(key, value, &ttl) 72 | 73 | time.Sleep(2 * time.Second) 74 | _, found := sl.Search(key) 75 | if found { 76 | t.Fatal("Expected key to be expired and not found") 77 | } 78 | } 79 | 80 | func TestDelete(t *testing.T) { 81 | sl := NewSkipList(12, 0.25) 82 | key := []byte("key1") 83 | value := []byte("value1") 84 | sl.Insert(key, value, nil) 85 | 86 | sl.Delete(key) 87 | _, found := sl.Search(key) 88 | if found { 89 | t.Fatal("Expected key to be deleted") 90 | } 91 | } 92 | 93 | func TestSize(t *testing.T) { 94 | sl := NewSkipList(12, 0.25) 95 | key := []byte("key1") 96 | value := []byte("value1") 97 | sl.Insert(key, value, nil) 98 | 99 | // Correct the expected size calculation 100 | expectedSize := len(key) + len(value) + (sl.level+1)*8 101 | if sl.Size() != expectedSize { 102 | t.Fatalf("Expected size %d, got %d", expectedSize, sl.Size()) 103 | } 104 | } 105 | 106 | func TestIterator(t *testing.T) { 107 | sl := NewSkipList(12, 0.25) 108 | key1 := []byte("key1") 109 | value1 := []byte("value1") 110 | key2 := []byte("key2") 111 | value2 := []byte("value2") 112 | sl.Insert(key1, value1, nil) 113 | sl.Insert(key2, value2, nil) 114 | 115 | it := NewIterator(sl) 116 | if !it.Next() { 117 | t.Fatal("Expected iterator to move to the first element") 118 | } 119 | k, v, _ := it.Current() 120 | if string(k) != string(key1) || string(v) != string(value1) { 121 | t.Fatalf("Expected key %s and value %s, got key %s and value %s", key1, value1, k, v) 122 | } 123 | if !it.Next() { 124 | t.Fatal("Expected iterator to move to the second element") 125 | } 126 | k, v, _ = it.Current() 127 | if string(k) != string(key2) || string(v) != string(value2) { 128 | t.Fatalf("Expected key %s and value %s, got key %s and value %s", key2, value2, k, v) 129 | } 130 | if it.Next() { 131 | t.Fatal("Expected iterator to be at the end") 132 | } 133 | } 134 | 135 | func TestSearchNil(t *testing.T) { 136 | sl := NewSkipList(12, 0.25) 137 | key := []byte("key1") 138 | 139 | _, found := sl.Search(key) 140 | if found { 141 | t.Fatal("Expected key to not be found") 142 | } 143 | 144 | // and again, just in case 145 | _, found = sl.Search(key) 146 | if found { 147 | t.Fatal("Expected key to not be found") 148 | } 149 | } 150 | 151 | func TestInsertTombstone(t *testing.T) { 152 | sl := NewSkipList(12, 0.25) 153 | key := []byte("key1") 154 | value := []byte("$tombstone") 155 | 156 | sl.Insert(key, value, nil) 157 | _, found := sl.Search(key) 158 | if found { 159 | t.Fatal("Expected key to be deleted") 160 | } 161 | 162 | key = []byte("key2") 163 | value = []byte("$tombstone") 164 | 165 | sl.Insert(key, value, nil) 166 | _, found = sl.Search(key) 167 | if found { 168 | t.Fatal("Expected key to be deleted") 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /v2/cuckoofilter/cuckoofilter.go: -------------------------------------------------------------------------------- 1 | // Package cuckoofilter implements a custom cuckoo filter data structure that allows for inserting and looking up keys with associated prefixes. 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package cuckoofilter 32 | 33 | import ( 34 | "bytes" 35 | "encoding/gob" 36 | "github.com/guycipher/k4/murmur" 37 | ) 38 | 39 | // Cuckoo Filter Configuration 40 | const ( 41 | initialFilterSize = 1000 // initial number of buckets in the filter 42 | maxBucketSize = 8 // max number of elements per bucket 43 | ) 44 | 45 | /* 46 | cf := NewCuckooFilter() 47 | numEntries := 10000000 // 10 million entries 48 | 49 | for i := 0; i < numEntries; i++ { 50 | key := []byte(fmt.Sprintf("key%d", i)) 51 | prefix := int64(i) 52 | cf.Insert(prefix, key) 53 | } 54 | ... 55 | 56 | initialFilterSize = 10000 57 | maxBucketSize = 4 58 | Number of buckets: 163840000 59 | Number of entries: 10000000 60 | Encoded size 375340330 61 | 62 | initialFilterSize = 1000 63 | maxBucketSize = 4 64 | Number of buckets: 131072000 65 | Number of entries: 10000000 66 | Encoded size 342572346 67 | 68 | initialFilterSize = 100 69 | maxBucketSize = 4 70 | Number of buckets: 209715200 71 | Number of entries: 10000000 72 | Encoded size 421215578 73 | 74 | initialFilterSize = 1000 75 | maxBucketSize = 8 76 | Number of buckets: 65536000 77 | Number of entries: 10000000 78 | 277036394 = 264.2 megabytes 79 | 80 | */ 81 | 82 | // CuckooFilter structure 83 | type CuckooFilter struct { 84 | Buckets []uint64 85 | KeyPrefixMap map[uint64]int64 86 | } 87 | 88 | // NewCuckooFilter creates a new cuckoo filter 89 | func NewCuckooFilter() *CuckooFilter { 90 | return &CuckooFilter{ 91 | Buckets: make([]uint64, initialFilterSize*maxBucketSize), 92 | KeyPrefixMap: make(map[uint64]int64), 93 | } 94 | } 95 | 96 | // Hash the key into a single value using murmur 97 | func (cf *CuckooFilter) hashKey(key []byte) uint64 { 98 | return murmur.Hash64(key, 0) // Using 0 as the seed 99 | } 100 | 101 | // Get two possible indices in the cuckoo filter for a hashed key 102 | func (cf *CuckooFilter) getHashIndices(hashedKey uint64) (int, int) { 103 | filterSize := len(cf.Buckets) / maxBucketSize 104 | index1 := int(hashedKey % uint64(filterSize)) 105 | index2 := int((hashedKey >> 32) % uint64(filterSize)) 106 | return index1, index2 107 | } 108 | 109 | // Resize the cuckoo filter by doubling its size 110 | func (cf *CuckooFilter) resize() { 111 | newFilterSize := len(cf.Buckets) * 2 112 | newBuckets := make([]uint64, newFilterSize) 113 | 114 | // Rehash all existing keys into the new buckets 115 | for i := 0; i < len(cf.Buckets); i++ { 116 | if cf.Buckets[i] != 0 { 117 | hashedKey := cf.Buckets[i] 118 | index1, index2 := cf.getHashIndices(hashedKey) 119 | inserted := false 120 | for k := 0; k < maxBucketSize; k++ { 121 | if newBuckets[index1*maxBucketSize+k] == 0 { 122 | newBuckets[index1*maxBucketSize+k] = hashedKey 123 | inserted = true 124 | break 125 | } 126 | } 127 | if !inserted { 128 | for k := 0; k < maxBucketSize; k++ { 129 | if newBuckets[index2*maxBucketSize+k] == 0 { 130 | newBuckets[index2*maxBucketSize+k] = hashedKey 131 | break 132 | } 133 | } 134 | } 135 | } 136 | } 137 | 138 | cf.Buckets = newBuckets 139 | } 140 | 141 | // Insert a key into the cuckoo filter with its prefix 142 | func (cf *CuckooFilter) Insert(prefix int64, key []byte) bool { 143 | hashedKey := cf.hashKey(key) 144 | index1, index2 := cf.getHashIndices(hashedKey) 145 | 146 | // Try to insert into the first index 147 | for i := 0; i < maxBucketSize; i++ { 148 | if cf.Buckets[index1*maxBucketSize+i] == 0 { 149 | cf.Buckets[index1*maxBucketSize+i] = hashedKey 150 | cf.KeyPrefixMap[hashedKey] = prefix 151 | return true 152 | } 153 | } 154 | 155 | // If index1 is full, try to insert into index2 156 | for i := 0; i < maxBucketSize; i++ { 157 | if cf.Buckets[index2*maxBucketSize+i] == 0 { 158 | cf.Buckets[index2*maxBucketSize+i] = hashedKey 159 | cf.KeyPrefixMap[hashedKey] = prefix 160 | return true 161 | } 162 | } 163 | 164 | // If both buckets are full, resize and retry 165 | cf.resize() 166 | return cf.Insert(prefix, key) 167 | } 168 | 169 | // Lookup a key in the cuckoo filter and return its prefix if found 170 | func (cf *CuckooFilter) Lookup(key []byte) (int64, bool) { 171 | hashedKey := cf.hashKey(key) 172 | index1, index2 := cf.getHashIndices(hashedKey) 173 | 174 | // Check the first index 175 | for i := 0; i < maxBucketSize; i++ { 176 | if cf.Buckets[index1*maxBucketSize+i] == hashedKey { 177 | return cf.KeyPrefixMap[hashedKey], true 178 | } 179 | } 180 | 181 | // Check the second index 182 | for i := 0; i < maxBucketSize; i++ { 183 | if cf.Buckets[index2*maxBucketSize+i] == hashedKey { 184 | return cf.KeyPrefixMap[hashedKey], true 185 | } 186 | } 187 | 188 | return 0, false 189 | } 190 | 191 | // Serialize the cuckoo filter to a byte slice 192 | func (cf *CuckooFilter) Serialize() ([]byte, error) { 193 | var buf bytes.Buffer 194 | enc := gob.NewEncoder(&buf) 195 | err := enc.Encode(cf) 196 | if err != nil { 197 | return nil, err 198 | } 199 | return buf.Bytes(), nil 200 | } 201 | 202 | // Deserialize the cuckoo filter from a byte slice 203 | func Deserialize(data []byte) (*CuckooFilter, error) { 204 | var cf CuckooFilter 205 | buf := bytes.NewBuffer(data) 206 | dec := gob.NewDecoder(buf) 207 | err := dec.Decode(&cf) 208 | if err != nil { 209 | return nil, err 210 | } 211 | return &cf, nil 212 | } 213 | -------------------------------------------------------------------------------- /v2/cuckoofilter/cuckoofilter_test.go: -------------------------------------------------------------------------------- 1 | // Package cuckoofilter tests 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package cuckoofilter 32 | 33 | import ( 34 | "fmt" 35 | "testing" 36 | ) 37 | 38 | func TestNewCuckooFilter(t *testing.T) { 39 | cf := NewCuckooFilter() 40 | 41 | if len(cf.Buckets) != 400 { 42 | t.Errorf("expected 400 buckets, got %d", len(cf.Buckets)) 43 | } 44 | for _, bucket := range cf.Buckets { 45 | if bucket != 0 { 46 | t.Errorf("expected all buckets to be empty, got %d", bucket) 47 | } 48 | } 49 | } 50 | 51 | func TestInsertAndLookup(t *testing.T) { 52 | cf := NewCuckooFilter() 53 | key := []byte("testkey") 54 | prefix := int64(42) 55 | 56 | if !cf.Insert(prefix, key) { 57 | t.Errorf("expected Insert to return true") 58 | } 59 | 60 | if p, found := cf.Lookup(key); !found || p != prefix { 61 | t.Errorf("expected Lookup to find the key with prefix %d, got %d", prefix, p) 62 | } 63 | } 64 | 65 | func TestResize(t *testing.T) { 66 | cf := NewCuckooFilter() 67 | for i := 0; i < 1000; i++ { 68 | key := []byte{byte(i)} 69 | cf.Insert(int64(i), key) 70 | } 71 | 72 | fmt.Println(len(cf.Buckets)) 73 | 74 | if len(cf.Buckets) < 1000 { 75 | t.Errorf("expected buckets to be greater than 1000, got %d", len(cf.Buckets)) 76 | } 77 | } 78 | 79 | func TestSerializeDeserialize(t *testing.T) { 80 | cf := NewCuckooFilter() 81 | key := []byte("testkey") 82 | prefix := int64(42) 83 | cf.Insert(prefix, key) 84 | 85 | data, err := cf.Serialize() 86 | if err != nil { 87 | t.Fatalf("expected no error during serialization, got %v", err) 88 | } 89 | 90 | deserializedCF, err := Deserialize(data) 91 | if err != nil { 92 | t.Fatalf("expected no error during deserialization, got %v", err) 93 | } 94 | 95 | if p, found := deserializedCF.Lookup(key); !found || p != prefix { 96 | t.Errorf("expected Lookup to find the key with prefix %d, got %d", prefix, p) 97 | } 98 | } 99 | 100 | func TestCuckooFilterSizeFor10MillionEntries(t *testing.T) { 101 | cf := NewCuckooFilter() 102 | numEntries := 10000000 103 | 104 | for i := 0; i < numEntries; i++ { 105 | key := []byte(fmt.Sprintf("key%d", i)) 106 | prefix := int64(i) 107 | cf.Insert(prefix, key) 108 | } 109 | 110 | fmt.Printf("Number of buckets: %d\n", len(cf.Buckets)) 111 | fmt.Printf("Number of entries: %d\n", numEntries) 112 | 113 | // serialize and deserialize to check for errors 114 | data, err := cf.Serialize() 115 | if err != nil { 116 | t.Fatalf("expected no error during serialization, got %v", err) 117 | } 118 | 119 | fmt.Println(len(data)) 120 | } 121 | -------------------------------------------------------------------------------- /v2/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/guycipher/k4/v2 2 | 3 | go 1.23.3 4 | 5 | require github.com/guycipher/k4 v1.9.7 6 | -------------------------------------------------------------------------------- /v2/go.sum: -------------------------------------------------------------------------------- 1 | github.com/guycipher/k4 v1.9.7 h1:elWXP9hB8XAvyXIUPRtr/0DVzGuZJGJAFzaCoRPY98o= 2 | github.com/guycipher/k4 v1.9.7/go.mod h1:j1faqUqiDiz9I7Qiu/2l5BjeRzkk/CssoDNm8pG4g8g= 3 | -------------------------------------------------------------------------------- /v2/k4_bench_test.go: -------------------------------------------------------------------------------- 1 | // Package k4 benchmarking 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package k4 32 | 33 | import ( 34 | "fmt" 35 | "math/rand" 36 | "os" 37 | "testing" 38 | ) 39 | 40 | func BenchmarkK4_Put(b *testing.B) { 41 | // Setup 42 | dir := "testdata" 43 | os.MkdirAll(dir, 0755) 44 | defer os.RemoveAll(dir) 45 | 46 | k4, err := Open(dir, (1024*1024)*50, 60, false, false) 47 | if err != nil { 48 | b.Fatalf("Failed to open K4: %v", err) 49 | } 50 | defer k4.Close() 51 | 52 | // Benchmark 53 | b.ResetTimer() 54 | for i := 0; i < b.N; i++ { 55 | key := []byte(fmt.Sprintf("key-%d", i)) 56 | value := []byte(fmt.Sprintf("value-%d", i)) 57 | err := k4.Put(key, value, nil) 58 | if err != nil { 59 | b.Fatalf("Failed to put key-value pair: %v", err) 60 | } 61 | } 62 | } 63 | 64 | func BenchmarkK4_Get(b *testing.B) { 65 | // Setup 66 | dir := "testdata" 67 | os.MkdirAll(dir, 0755) 68 | defer os.RemoveAll(dir) 69 | 70 | k4, err := Open(dir, 1024*1024, 60, false, false) 71 | if err != nil { 72 | b.Fatalf("Failed to open K4: %v", err) 73 | } 74 | defer k4.Close() 75 | 76 | // Insert some data 77 | for i := 0; i < 1000; i++ { 78 | key := []byte(fmt.Sprintf("key-%d", i)) 79 | value := []byte(fmt.Sprintf("value-%d", i)) 80 | err := k4.Put(key, value, nil) 81 | if err != nil { 82 | b.Fatalf("Failed to put key-value pair: %v", err) 83 | } 84 | } 85 | 86 | // Benchmark 87 | b.ResetTimer() 88 | for i := 0; i < b.N; i++ { 89 | key := []byte(fmt.Sprintf("key-%d", rand.Intn(1000))) 90 | _, err := k4.Get(key) 91 | if err != nil { 92 | b.Fatalf("Failed to get key: %v", err) 93 | } 94 | } 95 | } 96 | 97 | func BenchmarkK4_Delete(b *testing.B) { 98 | // Setup 99 | dir := "testdata" 100 | os.MkdirAll(dir, 0755) 101 | defer os.RemoveAll(dir) 102 | 103 | k4, err := Open(dir, 1024*1024, 60, false, false) 104 | if err != nil { 105 | b.Fatalf("Failed to open K4: %v", err) 106 | } 107 | defer k4.Close() 108 | 109 | // Insert some data 110 | for i := 0; i < 1000; i++ { 111 | key := []byte(fmt.Sprintf("key-%d", i)) 112 | value := []byte(fmt.Sprintf("value-%d", i)) 113 | err := k4.Put(key, value, nil) 114 | if err != nil { 115 | b.Fatalf("Failed to put key-value pair: %v", err) 116 | } 117 | } 118 | 119 | // Benchmark 120 | b.ResetTimer() 121 | for i := 0; i < b.N; i++ { 122 | key := []byte(fmt.Sprintf("key-%d", rand.Intn(1000))) 123 | err := k4.Delete(key) 124 | if err != nil { 125 | b.Fatalf("Failed to delete key: %v", err) 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /v2/pager/pager_test.go: -------------------------------------------------------------------------------- 1 | // Package pager tests 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package pager 32 | 33 | import ( 34 | "bytes" 35 | "os" 36 | "sync" 37 | "testing" 38 | ) 39 | 40 | func TestOpenPager(t *testing.T) { 41 | file, err := os.CreateTemp("", "pager_test") 42 | if err != nil { 43 | t.Fatalf("Failed to create temp file: %v", err) 44 | } 45 | defer os.Remove(file.Name()) 46 | 47 | p, err := OpenPager(file.Name(), os.O_RDWR|os.O_CREATE, 0666) 48 | if err != nil { 49 | t.Fatalf("Failed to open pager: %v", err) 50 | } 51 | if p == nil { 52 | t.Fatalf("Pager is nil") 53 | } 54 | if err := p.Close(); err != nil { 55 | t.Fatalf("Failed to close pager: %v", err) 56 | } 57 | } 58 | 59 | func TestWriteAndGetPage(t *testing.T) { 60 | file, err := os.CreateTemp("", "pager_test") 61 | if err != nil { 62 | t.Fatalf("Failed to create temp file: %v", err) 63 | } 64 | defer os.Remove(file.Name()) 65 | 66 | p, err := OpenPager(file.Name(), os.O_RDWR|os.O_CREATE, 0666) 67 | if err != nil { 68 | t.Fatalf("Failed to open pager: %v", err) 69 | } 70 | defer p.Close() 71 | 72 | data := []byte("Hello, World!") 73 | pageID, err := p.Write(data) 74 | if err != nil { 75 | t.Fatalf("Failed to write data: %v", err) 76 | } 77 | if pageID != 0 { 78 | t.Fatalf("Expected pageID 0, got %d", pageID) 79 | } 80 | 81 | readData, err := p.GetPage(pageID) 82 | if err != nil { 83 | t.Fatalf("Failed to get page: %v", err) 84 | } 85 | if !bytes.Equal(data, bytes.Trim(readData, "\x00")) { 86 | t.Fatalf("Expected %s, got %s", data, readData) 87 | } 88 | } 89 | 90 | func TestWriteToMultiplePages(t *testing.T) { 91 | file, err := os.CreateTemp("", "pager_test") 92 | if err != nil { 93 | t.Fatalf("Failed to create temp file: %v", err) 94 | } 95 | defer os.Remove(file.Name()) 96 | 97 | p, err := OpenPager(file.Name(), os.O_RDWR|os.O_CREATE, 0666) 98 | if err != nil { 99 | t.Fatalf("Failed to open pager: %v", err) 100 | } 101 | defer p.Close() 102 | 103 | data := make([]byte, PAGE_SIZE*2) 104 | str := "Hello, World!" 105 | 106 | for i := len(data) - len(str); i < len(data); i += len(str) { 107 | copy(data[i:], str) 108 | } 109 | 110 | pageID, err := p.Write(data) 111 | if err != nil { 112 | t.Fatalf("Failed to write data: %v", err) 113 | } 114 | 115 | // Read back the first page 116 | readData, err := p.GetPage(pageID) 117 | if err != nil { 118 | t.Fatalf("Failed to get first page: %v", err) 119 | } 120 | 121 | // end of readData should be str 122 | if !bytes.Equal([]byte(str), readData[len(readData)-len(str):]) { 123 | t.Fatalf("Expected %s, got %s", str, readData[len(readData)-len(str):]) 124 | 125 | } 126 | 127 | } 128 | 129 | func TestPagerSizeAndCount(t *testing.T) { 130 | file, err := os.CreateTemp("", "pager_test") 131 | if err != nil { 132 | t.Fatalf("Failed to create temp file: %v", err) 133 | } 134 | defer os.Remove(file.Name()) 135 | 136 | p, err := OpenPager(file.Name(), os.O_RDWR|os.O_CREATE, 0666) 137 | if err != nil { 138 | t.Fatalf("Failed to open pager: %v", err) 139 | } 140 | defer p.Close() 141 | 142 | data := []byte("Hello, World!") 143 | _, err = p.Write(data) 144 | if err != nil { 145 | t.Fatalf("Failed to write data: %v", err) 146 | } 147 | 148 | expectedSize := int64(PAGE_SIZE + HEADER_SIZE) 149 | if p.Size() != expectedSize { 150 | t.Fatalf("Expected size %d, got %d", expectedSize, p.Size()) 151 | } 152 | if p.Count() != 1 { 153 | t.Fatalf("Expected count 1, got %d", p.Count()) 154 | } 155 | } 156 | 157 | func TestPagerConcurrency(t *testing.T) { 158 | file, err := os.CreateTemp("", "pager_test") 159 | if err != nil { 160 | t.Fatalf("Failed to create temp file: %v", err) 161 | } 162 | defer os.Remove(file.Name()) 163 | 164 | p, err := OpenPager(file.Name(), os.O_RDWR|os.O_CREATE, 0666) 165 | if err != nil { 166 | t.Fatalf("Failed to open pager: %v", err) 167 | } 168 | defer p.Close() 169 | 170 | data := []byte("Hello, World!") 171 | var wg sync.WaitGroup 172 | 173 | for i := 0; i < 10; i++ { 174 | wg.Add(1) 175 | go func() { 176 | defer wg.Done() 177 | _, err := p.Write(data) 178 | if err != nil { 179 | t.Errorf("Failed to write data: %v", err) 180 | } 181 | }() 182 | } 183 | 184 | wg.Wait() 185 | if p.Count() != 10 { 186 | t.Fatalf("Expected count 10, got %d", p.Count()) 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /v2/skiplist/skiplist_test.go: -------------------------------------------------------------------------------- 1 | // Package skiplist tests 2 | // BSD 3-Clause License 3 | // 4 | // Copyright (c) 2024, Alex Gaetano Padula 5 | // All rights reserved. 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions are met: 9 | // 10 | // 1. Redistributions of source code must retain the above copyright notice, this 11 | // list of conditions and the following disclaimer. 12 | // 13 | // 2. Redistributions in binary form must reproduce the above copyright notice, 14 | // this list of conditions and the following disclaimer in the documentation 15 | // and/or other materials provided with the distribution. 16 | // 17 | // 3. Neither the name of the copyright holder nor the names of its 18 | // contributors may be used to endorse or promote products derived from 19 | // this software without specific prior written permission. 20 | // 21 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | package skiplist 32 | 33 | import ( 34 | "testing" 35 | "time" 36 | ) 37 | 38 | func TestNewSkipList(t *testing.T) { 39 | sl := NewSkipList(12, 0.25) 40 | if sl == nil { 41 | t.Fatal("Expected new skip list to be created") 42 | } 43 | if sl.level != 0 { 44 | t.Fatalf("Expected level to be 0, got %d", sl.level) 45 | } 46 | if sl.size != 0 { 47 | t.Fatalf("Expected size to be 0, got %d", sl.size) 48 | } 49 | } 50 | 51 | func TestInsertAndSearch(t *testing.T) { 52 | sl := NewSkipList(12, 0.25) 53 | key := []byte("key1") 54 | value := []byte("value1") 55 | sl.Insert(key, value, nil) 56 | 57 | val, found := sl.Search(key) 58 | if !found { 59 | t.Fatal("Expected to find the key") 60 | } 61 | if string(val) != string(value) { 62 | t.Fatalf("Expected value %s, got %s", value, val) 63 | } 64 | } 65 | 66 | func TestInsertWithTTL(t *testing.T) { 67 | sl := NewSkipList(12, 0.25) 68 | key := []byte("key1") 69 | value := []byte("value1") 70 | ttl := 1 * time.Second 71 | sl.Insert(key, value, &ttl) 72 | 73 | time.Sleep(2 * time.Second) 74 | _, found := sl.Search(key) 75 | if found { 76 | t.Fatal("Expected key to be expired and not found") 77 | } 78 | } 79 | 80 | func TestDelete(t *testing.T) { 81 | sl := NewSkipList(12, 0.25) 82 | key := []byte("key1") 83 | value := []byte("value1") 84 | sl.Insert(key, value, nil) 85 | 86 | sl.Delete(key) 87 | _, found := sl.Search(key) 88 | if found { 89 | t.Fatal("Expected key to be deleted") 90 | } 91 | } 92 | 93 | func TestSize(t *testing.T) { 94 | sl := NewSkipList(12, 0.25) 95 | key := []byte("key1") 96 | value := []byte("value1") 97 | sl.Insert(key, value, nil) 98 | 99 | // Correct the expected size calculation 100 | expectedSize := len(key) + len(value) + (sl.level+1)*8 101 | if sl.Size() != expectedSize { 102 | t.Fatalf("Expected size %d, got %d", expectedSize, sl.Size()) 103 | } 104 | } 105 | 106 | func TestIterator(t *testing.T) { 107 | sl := NewSkipList(12, 0.25) 108 | key1 := []byte("key1") 109 | value1 := []byte("value1") 110 | key2 := []byte("key2") 111 | value2 := []byte("value2") 112 | sl.Insert(key1, value1, nil) 113 | sl.Insert(key2, value2, nil) 114 | 115 | it := NewIterator(sl) 116 | if !it.Next() { 117 | t.Fatal("Expected iterator to move to the first element") 118 | } 119 | k, v, _ := it.Current() 120 | if string(k) != string(key1) || string(v) != string(value1) { 121 | t.Fatalf("Expected key %s and value %s, got key %s and value %s", key1, value1, k, v) 122 | } 123 | if !it.Next() { 124 | t.Fatal("Expected iterator to move to the second element") 125 | } 126 | k, v, _ = it.Current() 127 | if string(k) != string(key2) || string(v) != string(value2) { 128 | t.Fatalf("Expected key %s and value %s, got key %s and value %s", key2, value2, k, v) 129 | } 130 | if it.Next() { 131 | t.Fatal("Expected iterator to be at the end") 132 | } 133 | } 134 | 135 | func TestSearchNil(t *testing.T) { 136 | sl := NewSkipList(12, 0.25) 137 | key := []byte("key1") 138 | 139 | _, found := sl.Search(key) 140 | if found { 141 | t.Fatal("Expected key to not be found") 142 | } 143 | 144 | // and again, just in case 145 | _, found = sl.Search(key) 146 | if found { 147 | t.Fatal("Expected key to not be found") 148 | } 149 | } 150 | 151 | func TestInsertTombstone(t *testing.T) { 152 | sl := NewSkipList(12, 0.25) 153 | key := []byte("key1") 154 | value := []byte("$tombstone") 155 | 156 | sl.Insert(key, value, nil) 157 | _, found := sl.Search(key) 158 | if found { 159 | t.Fatal("Expected key to be deleted") 160 | } 161 | 162 | key = []byte("key2") 163 | value = []byte("$tombstone") 164 | 165 | sl.Insert(key, value, nil) 166 | _, found = sl.Search(key) 167 | if found { 168 | t.Fatal("Expected key to be deleted") 169 | } 170 | } 171 | --------------------------------------------------------------------------------