├── .github └── workflows │ └── starskey_ci.yml ├── .gitignore ├── LICENSE ├── README.md ├── artwork ├── starskey-avatar-large.png ├── starskey_logo.png └── starskey_logo.svg ├── bloomfilter ├── bloomfilter.go └── bloomfilter_test.go ├── go.mod ├── go.sum ├── pager ├── pager.go └── pager_test.go ├── starskey.go ├── starskey_test.go ├── surf ├── surf.go └── surf_test.go └── ttree ├── ttree.go └── ttree_test.go /.github/workflows/starskey_ci.yml: -------------------------------------------------------------------------------- 1 | name: Starskey CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v2 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v3 21 | with: 22 | go-version: '1.23' 23 | 24 | - name: Install dependencies 25 | run: go mod tidy 26 | 27 | - name: Run bloom filter tests 28 | run: go test ./bloomfilter -v 29 | 30 | - name: Run pager tests 31 | run: go test ./pager -v 32 | 33 | - name: Run ttree tests 34 | run: go test ./ttree -v 35 | 36 | - name: Run surf tests 37 | run: go test ./surf -v 38 | 39 | - name: Run core tests 40 | run: go test -v -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.exe~ 3 | *.dll 4 | *.so 5 | *.dylib 6 | *.test 7 | *.out 8 | go.work 9 | go.work.sum 10 | .env 11 | .idea 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |
4 | 5 | [![Go Reference](https://pkg.go.dev/badge/github.com/starskey-io/starskey.svg)](https://pkg.go.dev/github.com/starskey-io/starskey) 6 | 7 | Starskey is a fast embedded key-value store package for GO! Starskey implements a multi-level, durable log structured merge tree. 8 | 9 | ## Features 10 | - **Leveled partial merge compaction** Compactions occur on writes. If any disk level reaches its maximum size, half of the sstables are merged into a new sstable and placed into the next level. This algorithm is recursive until the last level. At the last level, if full, we merge all sstables into a new sstable. During merge operations, tombstones (deleted keys) are removed when a key reaches the last level. 11 | - **Simple API** with Put, Get, Delete, Range, FilterKeys, Update (for txns), PrefixSearch, LongestPrefixSearch, DeleteByRange, DeleteByFilter, DeleteByPrefix. 12 | - **Acid transactions** You can group multiple operations into a single atomic transaction. If any operation fails the entire transaction is rolled back. Only committed operations within a transaction roll back. These transactions would be considered fully serializable. Transactions themselves are also thread safe so you can add operations to them safety. 13 | - **Configurable options** You can configure many options such as max levels, memtable threshold, bloom filter,succinct range filters, logging, compression and more. 14 | - **WAL with recovery** Starskey uses a write ahead log to ensure durability. Memtable is replayed if a flush did not occur prior to shut down. On sorted runs to disk the WAL is truncated. 15 | - **Key value separation** Keys and values are stored separately for sstables within a klog and vlog respectively. 16 | - **Bloom filters** Each sstable has an in memory bloom filter to reduce disk reads. Bloom filters are used to check if a key exists in an SST instead of scanning it entirely. 17 | - **Succinct Range Filters** If enabled, each sstable will use a SuRF instead of a bloom filter; This will speed up range, prefix queries. Will use more memory than bloom filters. Only a bloom filter OR a SuRF filter can be enabled. 18 | - **Fast** up to 400k+ ops per second. 19 | - **Compression** S2, and Snappy compression is available. 20 | - **Logging** Logging to file is available. Will write to standard out if not enabled. 21 | - **Thread safe** Starskey is thread safe. Multiple goroutines can read and write to Starskey concurrently. Starskey uses one global lock to keep things consistent. 22 | - **T-Tree memtable** the memory table is a balanced in-memory tree data structure, designed as an alternative to AVL trees and B-Trees for main-memory. 23 | - **Channel logging** You can log to a channel instead of a file or standard output. 24 | 25 | ## Discord 26 | Chat everything Starskey @ our [Discord Server](https://discord.gg/HVxkhyys3R) 27 | 28 | ## Bench 29 | Use the benchmark program at [bench](https://github.com/starskey-io/bench) to compare Starskey with other popular key value stores/engines. 30 | 31 | ## Basic Example 32 | Below is a basic example of how to use Starskey. 33 | 34 | **Mind you examples use `skey` as the variable name for opened starskey instance(s).** 35 | 36 | ```go 37 | import ( 38 | "fmt" 39 | "github.com/starskey-io/starskey" 40 | ) 41 | 42 | func main() { 43 | skey, err := starskey.Open(&starskey.Config{ 44 | Permission: 0755, // Dir, file permission 45 | Directory: "db_dir", // Directory to store data 46 | FlushThreshold: (1024 * 1024) * 24, // 24mb Flush threshold in bytes, for production use 64mb or higher 47 | MaxLevel: 3, // Max levels number of disk levels 48 | SizeFactor: 10, // Size factor for each level. Say 10 that's 10 * the FlushThreshold at each level. So level 1 is 10MB, level 2 is 100MB, level 3 is 1GB. 49 | BloomFilter: false, // If you want to use bloom filters 50 | SuRF: false, // If enabled will speed up range queries as we check if an sstable has the keys we are looking for. 51 | Logging: true, // Enable logging to file 52 | Compression: false, // Enable compression 53 | CompressionOption: starskey.NoCompression, // Or SnappyCompression, S2Compression 54 | 55 | // Internal options 56 | // Optional: &OptionalConfig{ 57 | // BackgroundFSync: .. If you don't want to fsync writes to disk (default is true) 58 | // BackgroundFSyncInterval: .. Interval for background fsync, if configured true (default is 256ms) 59 | // TTreeMin: .. Minimum degree of the T-Tree 60 | // TTreeMax: .. Maximum degree of the T-Tree 61 | // PageSize: .. Page size for internal pagers 62 | // BloomFilterProbability: .. Bloom filter probability 63 | // LogChannel chan string // Log channel, instead of log file or standard output we log to a channel 64 | // }, 65 | }) // Config cannot be nil** 66 | if err != nil { 67 | // ..handle error 68 | } 69 | 70 | key := []byte("some_key") 71 | value := []byte("some_value") 72 | 73 | // Write key-value pair 74 | if err := skey.Put(key, value); err != nil { 75 | // ..handle error 76 | } 77 | 78 | // Read key-value pair 79 | v, err := skey.Get(key) 80 | if err != nil { 81 | // ..handle error 82 | } 83 | 84 | // Value is nil if key does not exist 85 | if v == nil { 86 | // ..handle error 87 | } 88 | 89 | fmt.Println(string(key), string(v)) 90 | 91 | // Close starskey 92 | if err := skey.Close(); err != nil { 93 | // ..handle error 94 | } 95 | 96 | } 97 | 98 | ``` 99 | 100 | ## Range Keys 101 | You can provide a start and end key to retrieve a range of keys values. 102 | ```go 103 | results, err := skey.Range([]byte("key900"), []byte("key980")) 104 | if err != nil { 105 | // ..handle error 106 | } 107 | 108 | // results is [][]byte where each element is a value 109 | for _, value := range results { 110 | fmt.Println(string(value)) // Each result is just the value 111 | } 112 | ``` 113 | 114 | ## Filter Keys 115 | You can filter keys to retrieve values based on a filter/compare method. 116 | ```go 117 | compareFunc := func(key []byte) bool { 118 | // if has prefix "c" return true 119 | return bytes.HasPrefix(key, []byte("c")) 120 | } 121 | 122 | results, err := skey.FilterKeys(compareFunc) 123 | if err != nil { 124 | // ..handle error 125 | } 126 | ``` 127 | 128 | ## Prefix Searches 129 | Starskey supports optimized prefix searches. 130 | 131 | ### Longest Prefix Search 132 | You can search for longest key with a common prefix. 133 | ```go 134 | result, n, err := skey.LongestPrefixSearch([]byte("common_prefix")) 135 | if err != nil { 136 | // ..handle error 137 | } 138 | ``` 139 | 140 | ### Prefix Search 141 | You can search for keys value's with a common prefix. 142 | ```go 143 | results, err := skey.PrefixSearch([]byte("ke")) 144 | if err != nil { 145 | // ..handle error 146 | } 147 | ``` 148 | 149 | ## Acid Transactions 150 | Using atomic transactions to group multiple operations into a single atomic transaction. If any operation fails the entire transaction is rolled back. Only committed operations roll back. 151 | ```go 152 | txn := skey.BeginTxn() 153 | if txn == nil { 154 | // ..handle error 155 | } 156 | 157 | txn.Put([]byte("key"), []byte("value")) 158 | // or txn.Delete([]byte("key")) 159 | 160 | if err := txn.Commit(); err != nil { 161 | // ..handle error 162 | } 163 | ``` 164 | 165 | **OR** 166 | 167 | You should use `Update` if you want to say Get a value, modify it and then Put it back as an atomic operation. 168 | Say you want to increment a value, below is an example of how to do that. 169 | ```go 170 | err = skey.Update(func(txn *starskey.Txn) error { 171 | value, err := txn.Get([]byte("key")) 172 | if err != nil { 173 | return err 174 | } 175 | 176 | // Increment value 177 | i, err := strconv.Atoi(string(value)) 178 | if err != nil { 179 | return err 180 | } 181 | 182 | i++ // Increment value 183 | 184 | txn.Put([]byte("key"), []byte(strconv.Itoa(i))) // Put back incremented value, this update is atomic 185 | 186 | return nil // Commit 187 | }) 188 | if err != nil { 189 | // ..handle error 190 | } 191 | ``` 192 | 193 | ```go 194 | err = skey.Update(func(txn *starskey.Txn) error { 195 | txn.Put([]byte("key"), []byte("value")) // or txn.Delete, txn.Get 196 | // .. 197 | return nil // Commit 198 | }) 199 | if err != nil { 200 | // ..handle error 201 | } 202 | ``` 203 | 204 | ## Delete 205 | Delete a key value pair from starskey. 206 | ```go 207 | if err := skey.Delete([]byte("key")); err != nil { 208 | // ..handle error 209 | } 210 | ``` 211 | 212 | **Delete operations removing multiple key values return an `n` value which is the amount of keys deleted.** 213 | 214 | ### Delete by range 215 | Delete key values within a range. 216 | ```go 217 | if n, err := skey.DeleteByRange([]byte("startKey"), []byte("endKey")); err != nil { 218 | // ..handle error 219 | } 220 | // n is amount of keys deleted 221 | ``` 222 | 223 | ### Delete by filter 224 | Delete key values based on a filter/compare method. You're comparing a key with a function and if it returns true the key is deleted. 225 | ```go 226 | compareFunc := func(key []byte) bool { 227 | // if has prefix "c" return true 228 | return bytes.HasPrefix(key, []byte("c")) 229 | } 230 | 231 | if n, err := skey.DeleteByFilter(compareFunc); err != nil { 232 | // ..handle error 233 | } 234 | // n is amount of keys deleted 235 | ``` 236 | 237 | ### Delete by key prefix 238 | Delete key values with a common prefix. 239 | ```go 240 | if n, err := skey.DeleteByPrefix([]byte("key")); err != nil { 241 | // ..handle error 242 | } 243 | // n is amount of keys deleted 244 | ``` 245 | 246 | ## Ingesting log messages 247 | ```go 248 | package main 249 | 250 | import ( 251 | "fmt" 252 | "github.com/starskey-io/starskey" 253 | "sync" 254 | "time" 255 | ) 256 | 257 | func main() { 258 | // Create a buffered channel for logs 259 | logChannel := make(chan string, 1000) 260 | 261 | // Create Starskey instance with log channel configured 262 | skey, err := starskey.Open(&starskey.Config{ 263 | Permission: 0755, 264 | Directory: "db_dir", 265 | FlushThreshold: (1024 * 1024) * 24, // 24MB 266 | MaxLevel: 3, 267 | SizeFactor: 10, 268 | BloomFilter: false, 269 | SuRF: false, 270 | Logging: true, 271 | Compression: false, 272 | CompressionOption: starskey.NoCompression, 273 | 274 | // Configure the LogChannel in OptionalConfig 275 | Optional: &starskey.OptionalConfig{ 276 | LogChannel: logChannel, 277 | }, 278 | }) 279 | if err != nil { 280 | fmt.Printf("Failed to open Starskey: %v\n", err) 281 | return 282 | } 283 | defer skey.Close() 284 | 285 | // Start a goroutine to consume and process logs in real-time 286 | var wg sync.WaitGroup 287 | 288 | wg.Add(1) // Add one goroutine to wait for 289 | go func() { 290 | defer wg.Done() 291 | for logMsg := range logChannel { 292 | // Process log messages in real-time 293 | timestamp := time.Now().Format("2006-01-02 15:04:05.000") 294 | 295 | fmt.Printf("[%s] %s\n", timestamp, logMsg) 296 | } 297 | }() 298 | 299 | // Use Starskey for your normal operations 300 | for i := 0; i < 100; i++ { 301 | key := []byte(fmt.Sprintf("key%d", i)) 302 | value := []byte(fmt.Sprintf("value%d", i)) 303 | 304 | if err := skey.Put(key, value); err != nil { 305 | fmt.Printf("Failed to put key-value: %v\n", err) 306 | } 307 | } 308 | 309 | // The log channel will keep receiving logs until Starskey is closed 310 | time.Sleep(2 * time.Second) // Give some time for operations to complete 311 | 312 | // Close starskey as we are done 313 | skey.Close() 314 | 315 | // close the channel as Starskey doesn't close it 316 | close(logChannel) 317 | 318 | // Wait for log processing to complete 319 | wg.Wait() 320 | } 321 | ``` 322 | 323 | ## Key Lifecycle 324 | A key once inserted will live in the memtable until it is flushed to disk. 325 | Once flushed to disk it will live in a sstable at l1 until it is compacted. Once compacted it will be merged into a new sstable at the next level. This process is recursive until the last level. At the last level if full we merge all sstables into a new sstable. 326 | 327 | If a key is deleted it will live on the same way until it reaches last level at which point it will be removed entirely. 328 | 329 | ## Memory and disk sorting 330 | The sorting internally would be lexicographical (alphabetical), meaning it will sort based on the byte-by-byte comparisons of slices. 331 | We use bytes.Compare to sort keys in memory and on disk. 332 | 333 | 334 | -------------------------------------------------------------------------------- /artwork/starskey-avatar-large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/starskey-io/starskey/d0e32d404509f85c9f25a7e37d8cdb35f7bba061/artwork/starskey-avatar-large.png -------------------------------------------------------------------------------- /artwork/starskey_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/starskey-io/starskey/d0e32d404509f85c9f25a7e37d8cdb35f7bba061/artwork/starskey_logo.png -------------------------------------------------------------------------------- /bloomfilter/bloomfilter.go: -------------------------------------------------------------------------------- 1 | // Package bloomfilter 2 | // 3 | // (C) Copyright Starskey 4 | // 5 | // Original Author: Alex Gaetano Padula 6 | // 7 | // Licensed under the Mozilla Public License, v. 2.0 (the "License"); 8 | // you may not use this file except in compliance with the License. 9 | // You may obtain a copy of the License at 10 | // 11 | // https://www.mozilla.org/en-US/MPL/2.0/ 12 | // 13 | // Unless required by applicable law or agreed to in writing, software 14 | // distributed under the License is distributed on an "AS IS" BASIS, 15 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | // See the License for the specific language governing permissions and 17 | // limitations under the License. 18 | package bloomfilter 19 | 20 | import ( 21 | "bytes" 22 | "encoding/gob" 23 | "errors" 24 | "github.com/zeebo/xxh3" 25 | "hash" 26 | "hash/fnv" 27 | "math" 28 | // We could compare against github.com/cespare/xxhash/v2 and github.com/zeebo/xxh3 29 | // but may not be backwards compatible. We use both currently(cespare is used for SuRF) which may be redundant to import 2 packages doing same thing 30 | ) 31 | 32 | // BloomFilter struct represents a Bloom filter 33 | type BloomFilter struct { 34 | Bitset []int8 // Bitset, each int8 can store 8 bits 35 | Size uint // Size of the bit array 36 | hashFuncs []hash.Hash64 // Hash functions (can't be exported on purpose for serialization purposes..) 37 | } 38 | 39 | // New creates a new Bloom filter with an expected number of items and false positive rate 40 | func New(expectedItems uint, falsePositiveRate float64) (*BloomFilter, error) { 41 | if expectedItems == 0 { 42 | return nil, errors.New("expectedItems must be greater than 0") 43 | } 44 | 45 | if falsePositiveRate <= 0 || falsePositiveRate >= 1 { 46 | return nil, errors.New("falsePositiveRate must be between 0 and 1") 47 | } 48 | 49 | size := optimalSize(expectedItems, falsePositiveRate) 50 | hashCount := optimalHashCount(size, expectedItems) 51 | 52 | bf := &BloomFilter{ 53 | Bitset: make([]int8, (size+7)/8), // Allocate enough int8s to store the bits 54 | Size: size, 55 | hashFuncs: make([]hash.Hash64, hashCount), 56 | } 57 | 58 | // Initialize hash functions with different seeds 59 | for i := uint(0); i < hashCount; i++ { 60 | bf.hashFuncs[i] = fnv.New64a() 61 | } 62 | 63 | return bf, nil 64 | } 65 | 66 | // Add adds an item to the Bloom filter 67 | func (bf *BloomFilter) Add(data []byte) { 68 | for _, hf := range bf.hashFuncs { 69 | hf.Reset() 70 | hf.Write(data) 71 | hash := hf.Sum64() 72 | position := hash % uint64(bf.Size) 73 | bf.Bitset[position/8] |= 1 << (position % 8) 74 | } 75 | } 76 | 77 | // Contains checks if an item might exist in the Bloom filter 78 | func (bf *BloomFilter) Contains(data []byte) bool { 79 | for _, hf := range bf.hashFuncs { 80 | hf.Reset() 81 | 82 | var err error 83 | _, err = hf.Write(data) 84 | if err != nil { 85 | return false 86 | } 87 | h := hf.Sum64() 88 | position := h % uint64(bf.Size) 89 | if bf.Bitset[position/8]&(1<<(position%8)) == 0 { 90 | return false // Definitely not in set 91 | } 92 | } 93 | return true // Might be in set 94 | } 95 | 96 | // optimalSize calculates the optimal size of the bit array 97 | func optimalSize(n uint, p float64) uint { 98 | return uint(math.Ceil(-float64(n) * math.Log(p) / math.Pow(math.Log(2), 2))) 99 | } 100 | 101 | // optimalHashCount calculates the optimal number of hash functions 102 | func optimalHashCount(size uint, n uint) uint { 103 | return uint(math.Ceil(float64(size) / float64(n) * math.Log(2))) 104 | } 105 | 106 | // Serialize converts the BloomFilter to a byte slice 107 | func (bf *BloomFilter) Serialize() ([]byte, error) { 108 | var buf bytes.Buffer 109 | encoder := gob.NewEncoder(&buf) 110 | err := encoder.Encode(bf) 111 | if err != nil { 112 | return nil, err 113 | } 114 | return buf.Bytes(), nil 115 | } 116 | 117 | // Deserialize reconstructs a BloomFilter from a byte slice 118 | func Deserialize(data []byte) (*BloomFilter, error) { 119 | var bf BloomFilter 120 | buf := bytes.NewBuffer(data) 121 | decoder := gob.NewDecoder(buf) 122 | err := decoder.Decode(&bf) 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | // Reinitialize hash functions 128 | bf.hashFuncs = make([]hash.Hash64, len(bf.hashFuncs)) 129 | for i := range bf.hashFuncs { 130 | bf.hashFuncs[i] = xxh3.New() 131 | } 132 | 133 | return &bf, nil 134 | } 135 | -------------------------------------------------------------------------------- /bloomfilter/bloomfilter_test.go: -------------------------------------------------------------------------------- 1 | // Package bloomfilter 2 | // 3 | // (C) Copyright Starskey 4 | // 5 | // Original Author: Alex Gaetano Padula 6 | // 7 | // Licensed under the Mozilla Public License, v. 2.0 (the "License"); 8 | // you may not use this file except in compliance with the License. 9 | // You may obtain a copy of the License at 10 | // 11 | // https://www.mozilla.org/en-US/MPL/2.0/ 12 | // 13 | // Unless required by applicable law or agreed to in writing, software 14 | // distributed under the License is distributed on an "AS IS" BASIS, 15 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | // See the License for the specific language governing permissions and 17 | // limitations under the License. 18 | package bloomfilter 19 | 20 | import ( 21 | "fmt" 22 | "testing" 23 | ) 24 | 25 | func TestNewBloomFilter(t *testing.T) { 26 | bf, err := New(1000, 0.01) 27 | if err != nil { 28 | t.Errorf("Error creating BloomFilter: %v", err) 29 | } 30 | 31 | if bf.Size == 0 { 32 | t.Errorf("Expected non-zero size, got %d", bf.Size) 33 | } 34 | if len(bf.hashFuncs) == 0 { 35 | t.Errorf("Expected non-zero hash count, got %d", len(bf.hashFuncs)) 36 | } 37 | } 38 | 39 | func TestNewBloomFilterSerialization(t *testing.T) { 40 | bf, err := New(1000, 0.01) 41 | if err != nil { 42 | t.Errorf("Error creating BloomFilter: %v", err) 43 | } 44 | 45 | for i := 0; i < 100; i++ { 46 | bf.Add([]byte(fmt.Sprintf("testdata%d", i))) 47 | } 48 | 49 | serialized, err := bf.Serialize() 50 | if err != nil { 51 | t.Errorf("Error serializing BloomFilter: %v", err) 52 | 53 | } 54 | deserialized, err := Deserialize(serialized) 55 | if err != nil { 56 | t.Errorf("Error deserializing BloomFilter: %v", err) 57 | 58 | } 59 | 60 | if deserialized.Size != bf.Size { 61 | t.Errorf("Expected size %d, got %d", bf.Size, deserialized.Size) 62 | } 63 | 64 | for i := 0; i < 100; i++ { 65 | data := []byte(fmt.Sprintf("testdata%d", i)) 66 | if !deserialized.Contains(data) { 67 | t.Errorf("Expected deserialized BloomFilter to contain data") 68 | } 69 | 70 | } 71 | } 72 | 73 | func TestAddAndContains(t *testing.T) { 74 | bf, err := New(1000, 0.01) 75 | if err != nil { 76 | t.Errorf("Error creating BloomFilter: %v", err) 77 | } 78 | 79 | data := []byte("testdata") 80 | 81 | bf.Add(data) 82 | if !bf.Contains(data) { 83 | t.Errorf("Expected BloomFilter to contain data") 84 | } 85 | 86 | nonExistentData := []byte("nonexistent") 87 | if bf.Contains(nonExistentData) { 88 | t.Errorf("Expected BloomFilter to not contain non-existent data") 89 | } 90 | } 91 | 92 | func TestNewBloomFilterSerializationSize(t *testing.T) { 93 | bf, err := New(1_000_000, 0.01) 94 | if err != nil { 95 | t.Errorf("Error creating BloomFilter: %v", err) 96 | 97 | } 98 | 99 | for i := 0; i < 1_000_000; i++ { 100 | bf.Add([]byte(fmt.Sprintf("testdata%d", i))) 101 | } 102 | 103 | serialized, err := bf.Serialize() 104 | if err != nil { 105 | t.Errorf("Error serializing BloomFilter: %v", err) 106 | 107 | } 108 | 109 | t.Logf("Size of serialized bloom filter at 1m items %.2f MB\n", float64(len(serialized))/1024/1024) 110 | } 111 | 112 | func BenchmarkAdd(b *testing.B) { 113 | bf, err := New(1000, 0.01) 114 | if err != nil { 115 | b.Errorf("Error creating BloomFilter: %v", err) 116 | } 117 | 118 | data := []byte("testdata") 119 | 120 | for i := 0; i < b.N; i++ { 121 | bf.Add(data) 122 | } 123 | } 124 | 125 | func BenchmarkContains(b *testing.B) { 126 | bf, err := New(1000, 0.01) 127 | if err != nil { 128 | b.Errorf("Error creating BloomFilter: %v", err) 129 | } 130 | 131 | data := []byte("testdata") 132 | bf.Add(data) 133 | 134 | for i := 0; i < b.N; i++ { 135 | bf.Contains(data) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/starskey-io/starskey 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/cespare/xxhash/v2 v2.3.0 7 | github.com/klauspost/compress v1.16.7 8 | github.com/zeebo/xxh3 v1.0.2 9 | go.mongodb.org/mongo-driver v1.17.2 10 | ) 11 | 12 | require github.com/klauspost/cpuid/v2 v2.0.9 // indirect 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 2 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 6 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 7 | github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= 8 | github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 9 | github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= 10 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 11 | github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= 12 | github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= 13 | github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= 14 | github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= 15 | go.mongodb.org/mongo-driver v1.17.2 h1:gvZyk8352qSfzyZ2UMWcpDpMSGEr1eqE4T793SqyhzM= 16 | go.mongodb.org/mongo-driver v1.17.2/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= 17 | -------------------------------------------------------------------------------- /pager/pager.go: -------------------------------------------------------------------------------- 1 | // Package pager 2 | // 3 | // (C) Copyright Starskey 4 | // 5 | // Original Author: Alex Gaetano Padula 6 | // 7 | // Licensed under the Mozilla Public License, v. 2.0 (the "License"); 8 | // you may not use this file except in compliance with the License. 9 | // You may obtain a copy of the License at 10 | // 11 | // https://www.mozilla.org/en-US/MPL/2.0/ 12 | // 13 | // Unless required by applicable law or agreed to in writing, software 14 | // distributed under the License is distributed on an "AS IS" BASIS, 15 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | // See the License for the specific language governing permissions and 17 | // limitations under the License. 18 | package pager 19 | 20 | // Append only file pager 21 | // We essentially only care about appending data to the file but keeping each page equal in size. 22 | // The pager handles overflowing data by creating new pages and linking them together, if need be. 23 | // The iterator is able to traverse through the pages reliable skipping and gathering pages as needed. 24 | 25 | import ( 26 | "encoding/binary" 27 | "errors" 28 | "fmt" 29 | "math" 30 | "os" 31 | "sync" 32 | "sync/atomic" 33 | "time" 34 | ) 35 | 36 | // Pager is the main pager struct 37 | type Pager struct { 38 | file *os.File // File to use for paging 39 | pageSize int // Size of each page.. if data overflows new pages are created and linked 40 | syncQuit chan struct{} // Channel to quit background fsync 41 | wg *sync.WaitGroup // WaitGroup for background fsync 42 | syncInterval time.Duration // File sync interval 43 | sync bool // To sync or not to sync 44 | closed atomic.Bool // We use to track if we have closed the pager already to prevent double closing 45 | } 46 | 47 | // Iterator is the iterator struct used for 48 | // iterator through pages within paged file 49 | type Iterator struct { 50 | pager *Pager // Pager for iterator 51 | pageStack []int // Stack of page numbers 52 | currentPage int // Current page number 53 | CurrentData []byte // Current data 54 | maxPages int // Max pages based on file size calculation 55 | } 56 | 57 | // Open opens a file for paging 58 | func Open(filename string, flag int, perm os.FileMode, pageSize int, syncOn bool, syncInterval time.Duration) (*Pager, error) { 59 | var err error 60 | pager := &Pager{pageSize: pageSize, syncQuit: make(chan struct{}), wg: &sync.WaitGroup{}, syncInterval: syncInterval, sync: syncOn} 61 | 62 | // Open the file for reading and writing 63 | pager.file, err = os.OpenFile(filename, flag, perm) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | if !pager.sync { 69 | return pager, nil 70 | } 71 | 72 | // Initialize closed to false 73 | pager.closed.Store(false) 74 | 75 | // Start background sync 76 | pager.wg.Add(1) 77 | go pager.backgroundSync() 78 | 79 | return pager, nil 80 | } 81 | 82 | // Close closes the pager gracefully 83 | func (p *Pager) Close() error { 84 | if p == nil { 85 | return nil 86 | } 87 | 88 | if p.file == nil { 89 | return nil 90 | } 91 | 92 | // Only close channel if sync is enabled and we haven't closed before 93 | if p.sync && !p.closed.Swap(true) { 94 | close(p.syncQuit) 95 | p.wg.Wait() 96 | } 97 | 98 | return p.file.Close() 99 | } 100 | 101 | // Truncate truncates the file 102 | func (p *Pager) Truncate() error { 103 | if err := p.file.Truncate(0); err != nil { 104 | return err 105 | } 106 | return nil 107 | } 108 | 109 | // Size returns the size of the file 110 | func (p *Pager) Size() int64 { 111 | fileInfo, err := p.file.Stat() 112 | if err != nil { 113 | return 0 114 | } 115 | return fileInfo.Size() 116 | } 117 | 118 | // backgroundSync is a goroutine that syncs the file in the background every syncInterval 119 | func (p *Pager) backgroundSync() { 120 | defer p.wg.Done() 121 | ticker := time.NewTicker(p.syncInterval) 122 | defer ticker.Stop() 123 | for { 124 | select { 125 | case <-ticker.C: 126 | _ = p.file.Sync() 127 | case <-p.syncQuit: 128 | _ = p.file.Sync() // Escalate then return.. 129 | return 130 | } 131 | } 132 | } 133 | 134 | // chunk splits a byte slice into n chunks 135 | func chunk(data []byte, n int) ([][]byte, error) { 136 | if n <= 0 { 137 | return nil, fmt.Errorf("n must be greater than 0") 138 | } 139 | 140 | // Calculate the chunk size 141 | chunkSize := int(math.Ceil(float64(len(data)) / float64(n))) 142 | 143 | var chunks [][]byte 144 | 145 | // Loop to slice the data into chunks 146 | for i := 0; i < len(data); i += chunkSize { 147 | end := i + chunkSize 148 | if end > len(data) { 149 | end = len(data) 150 | } 151 | chunks = append(chunks, data[i:end]) 152 | } 153 | 154 | return chunks, nil 155 | } 156 | 157 | // Write writes data to the pager 158 | func (p *Pager) Write(data []byte) (int, error) { 159 | if len(data) == 0 { 160 | return -1, errors.New("data is empty") 161 | } 162 | 163 | initialPgN := -1 164 | 165 | if len(data) > p.pageSize { 166 | chunks, err := chunk(data, p.pageSize) 167 | if err != nil { 168 | return -1, err 169 | } 170 | 171 | // Each chunk is a page 172 | for i, c := range chunks { 173 | 174 | if i == len(chunks)-1 { 175 | if _, err := p.writePage(c, false); err != nil { 176 | return -1, err 177 | } 178 | break 179 | } 180 | 181 | if pg, err := p.writePage(c, true); err != nil { 182 | return -1, err 183 | } else { 184 | 185 | if initialPgN == -1 { 186 | initialPgN = pg 187 | } 188 | } 189 | } 190 | return initialPgN, nil 191 | } 192 | 193 | return p.writePage(data, false) 194 | } 195 | 196 | // GetPageSize returns the page size 197 | func (p *Pager) GetPageSize() int { 198 | return p.pageSize 199 | } 200 | 201 | // writePage writes a page to the file 202 | func (p *Pager) writePage(data []byte, overflow bool) (int, error) { 203 | pageNumber := p.newPageNumber() 204 | 205 | // Create a buffer to hold the header and the data, ensuring it is the size of a page 206 | buffer := make([]byte, p.pageSize+16) 207 | 208 | // Write the size of the data (int64) to the buffer 209 | binary.LittleEndian.PutUint64(buffer[0:], uint64(len(data))) 210 | 211 | // Write the overflow flag (int64) to the buffer 212 | if overflow { 213 | binary.LittleEndian.PutUint64(buffer[8:], 1) 214 | } else { 215 | binary.LittleEndian.PutUint64(buffer[8:], 0) 216 | } 217 | 218 | // Write the actual data to the buffer, ensuring it does not exceed the page size 219 | copy(buffer[16:], data) 220 | 221 | // write to end of file 222 | // we seek to the end of the file 223 | _, err := p.file.Seek(0, 2) 224 | if err != nil { 225 | return 0, err 226 | } 227 | 228 | _, err = p.file.Write(buffer) 229 | if err != nil { 230 | return -1, err 231 | } 232 | return int(pageNumber), nil 233 | } 234 | 235 | // newPageNumber returns the next page number 236 | func (p *Pager) newPageNumber() int64 { 237 | // Get the file size 238 | fileInfo, err := p.file.Stat() 239 | if err != nil { 240 | return 0 241 | } 242 | return fileInfo.Size() / int64(p.pageSize+16) // We add 16 bytes for the page header 243 | } 244 | 245 | // Read reads a page from the file 246 | func (p *Pager) Read(pg int) ([]byte, int, error) { 247 | var data []byte 248 | 249 | for { 250 | // Seek to the start of the page 251 | offset := int64(pg) * int64(p.pageSize+16) 252 | _, err := p.file.Seek(offset, 0) 253 | if err != nil { 254 | return nil, -1, err 255 | } 256 | 257 | // Read the header 258 | header := make([]byte, 16) 259 | _, err = p.file.Read(header) 260 | if err != nil { 261 | return nil, -1, err 262 | } 263 | 264 | // Get the size of the data 265 | dataSize := binary.LittleEndian.Uint64(header[0:8]) 266 | 267 | // Read the data 268 | pageData := make([]byte, dataSize) 269 | _, err = p.file.Read(pageData) 270 | if err != nil { 271 | return nil, -1, err 272 | } 273 | 274 | // Append the data to the result 275 | data = append(data, pageData...) 276 | 277 | // Check the overflow flag 278 | overflow := binary.LittleEndian.Uint64(header[8:16]) 279 | if overflow == 0 { 280 | break 281 | } 282 | 283 | // Move to the next page 284 | pg++ 285 | } 286 | // We return the last page number read 287 | return data, pg, nil 288 | } 289 | 290 | // PageCount returns the number of pages in the file 291 | func (p *Pager) PageCount() int { 292 | // We could use an iterator and gather a better count but this works as well.. 293 | fileInfo, err := p.file.Stat() 294 | if err != nil { 295 | return 0 296 | } 297 | return int(fileInfo.Size()) / (p.pageSize + 16) 298 | } 299 | 300 | // Name returns the name of the file 301 | func (p *Pager) Name() string { 302 | return p.file.Name() 303 | 304 | } 305 | 306 | // NewIterator returns a new iterator 307 | func NewIterator(pager *Pager) *Iterator { 308 | return &Iterator{maxPages: pager.PageCount(), pager: pager, currentPage: 0} 309 | } 310 | 311 | // Next moves the iterator to the next page 312 | func (it *Iterator) Next() bool { 313 | if it.currentPage < it.maxPages { 314 | it.stackAdd(it.currentPage) 315 | read, lastPg, err := it.pager.Read(it.currentPage) 316 | if err != nil { 317 | return false 318 | } 319 | 320 | it.currentPage = lastPg + 1 321 | it.CurrentData = read 322 | return true 323 | } 324 | return false 325 | } 326 | 327 | // Prev moves the iterator to the previous page 328 | func (it *Iterator) Prev() bool { 329 | if len(it.pageStack) == 0 { 330 | return false 331 | } 332 | 333 | // Pop the last page number from the stack 334 | it.currentPage = it.pageStack[len(it.pageStack)-1] 335 | it.pageStack = it.pageStack[:len(it.pageStack)-1] 336 | 337 | // Read the data for the current page 338 | read, _, err := it.pager.Read(it.currentPage) 339 | if err != nil { 340 | return false 341 | } 342 | 343 | it.CurrentData = read 344 | return true 345 | } 346 | 347 | // Read reads the current page 348 | func (it *Iterator) Read() ([]byte, error) { 349 | return it.CurrentData, nil 350 | } 351 | 352 | // stackAdd adds a page number to the stack 353 | func (it *Iterator) stackAdd(pg int) { 354 | // We avoid adding the same page number to the stack 355 | for _, p := range it.pageStack { 356 | if p == pg { 357 | return 358 | } 359 | } 360 | 361 | it.pageStack = append(it.pageStack, pg) 362 | } 363 | 364 | // FileName returns the pager underlying file name 365 | func (p *Pager) FileName() string { 366 | return p.file.Name() 367 | } 368 | 369 | // EscalateFSync escalates a disk fsync 370 | func (p *Pager) EscalateFSync() { 371 | _ = p.file.Sync() // Is thread safe 372 | } 373 | -------------------------------------------------------------------------------- /pager/pager_test.go: -------------------------------------------------------------------------------- 1 | // Package pager 2 | // 3 | // (C) Copyright Starskey 4 | // 5 | // Original Author: Alex Gaetano Padula 6 | // 7 | // Licensed under the Mozilla Public License, v. 2.0 (the "License"); 8 | // you may not use this file except in compliance with the License. 9 | // You may obtain a copy of the License at 10 | // 11 | // https://www.mozilla.org/en-US/MPL/2.0/ 12 | // 13 | // Unless required by applicable law or agreed to in writing, software 14 | // distributed under the License is distributed on an "AS IS" BASIS, 15 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | // See the License for the specific language governing permissions and 17 | // limitations under the License. 18 | package pager 19 | 20 | import ( 21 | "log" 22 | "os" 23 | "testing" 24 | "time" 25 | ) 26 | 27 | func TestOpen(t *testing.T) { 28 | defer os.Remove("test.bin") 29 | p, err := Open("test.bin", os.O_CREATE|os.O_RDWR, 0777, 1024, true, time.Millisecond*128) 30 | if err != nil { 31 | t.Errorf("Error opening file: %v", err) 32 | } 33 | 34 | // Close 35 | if err := p.Close(); err != nil { 36 | t.Errorf("Error closing file: %v", err) 37 | } 38 | } 39 | 40 | func TestChunk(t *testing.T) { 41 | data := []byte("hello world") 42 | chunks, err := chunk(data, 5) 43 | if err != nil { 44 | t.Errorf("Error chunking data: %v", err) 45 | 46 | } 47 | for i, c := range chunks { 48 | t.Logf("Chunk %d: %s", i, string(c)) 49 | 50 | } 51 | if len(chunks) != 4 { 52 | t.Errorf("Expected 2 chunks, got %d", len(chunks)) 53 | } 54 | } 55 | 56 | func TestPager_Write(t *testing.T) { 57 | defer os.Remove("test.bin") 58 | p, err := Open("test.bin", os.O_CREATE|os.O_RDWR, 0777, 4, true, time.Millisecond*128) 59 | if err != nil { 60 | t.Errorf("Error opening file: %v", err) 61 | } 62 | 63 | defer p.Close() 64 | 65 | pg, err := p.Write([]byte("hello world")) 66 | if err != nil { 67 | t.Errorf("Error writing to file: %v", err) 68 | } 69 | 70 | // Expect page 0 initially 71 | if pg != 0 { 72 | t.Errorf("Expected page 0, got %d", pg) 73 | } 74 | 75 | pg, err = p.Write([]byte("hello world")) 76 | if err != nil { 77 | t.Errorf("Error writing to file: %v", err) 78 | } 79 | 80 | // Expect page 4 81 | if pg != 4 { 82 | t.Errorf("Expected page 0, got %d", pg) 83 | } 84 | } 85 | 86 | func TestPager_Read(t *testing.T) { 87 | defer os.Remove("test.bin") 88 | p, err := Open("test.bin", os.O_CREATE|os.O_RDWR, 0777, 4, true, time.Millisecond*128) 89 | if err != nil { 90 | t.Errorf("Error opening file: %v", err) 91 | } 92 | 93 | defer p.Close() 94 | 95 | pg, err := p.Write([]byte("hello world")) 96 | if err != nil { 97 | t.Errorf("Error writing to file: %v", err) 98 | } 99 | 100 | log.Println(pg) 101 | 102 | pg, err = p.Write([]byte("hello world2")) 103 | if err != nil { 104 | t.Errorf("Error writing to file: %v", err) 105 | } 106 | 107 | log.Println(pg) 108 | pg, err = p.Write([]byte("hello world")) 109 | if err != nil { 110 | t.Errorf("Error writing to file: %v", err) 111 | } 112 | 113 | log.Println(pg) 114 | 115 | data, _, err := p.Read(4) 116 | if err != nil { 117 | t.Errorf("Error reading from file: %v", err) 118 | } 119 | 120 | if string(data) != "hello world2" { 121 | t.Errorf("Expected 'hello world2', got %s", string(data)) 122 | } 123 | } 124 | 125 | func TestPagerIterator(t *testing.T) { 126 | defer os.Remove("test.bin") 127 | p, err := Open("test.bin", os.O_CREATE|os.O_RDWR, 0777, 4, true, time.Millisecond*128) 128 | if err != nil { 129 | t.Errorf("Error opening file: %v", err) 130 | } 131 | 132 | defer p.Close() 133 | 134 | p.Write([]byte("hello world")) 135 | p.Write([]byte("hello world2")) 136 | p.Write([]byte("hello world3")) 137 | 138 | it := NewIterator(p) 139 | for it.Next() { 140 | data, err := it.Read() 141 | if err != nil { 142 | break 143 | } 144 | log.Println(string(data)) 145 | 146 | } 147 | 148 | for { 149 | if !it.Prev() { 150 | break 151 | } 152 | 153 | data, err := it.Read() 154 | if err != nil { 155 | break 156 | } 157 | log.Println(string(data)) 158 | } 159 | } 160 | 161 | func TestPager_Truncate(t *testing.T) { 162 | defer os.Remove("test.bin") 163 | p, err := Open("test.bin", os.O_CREATE|os.O_RDWR, 0777, 1024, true, time.Millisecond*128) 164 | if err != nil { 165 | t.Errorf("Error opening file: %v", err) 166 | } 167 | defer p.Close() 168 | 169 | _, err = p.Write([]byte("hello world")) 170 | if err != nil { 171 | t.Errorf("Error writing to file: %v", err) 172 | } 173 | 174 | err = p.Truncate() 175 | if err != nil { 176 | t.Errorf("Error truncating file: %v", err) 177 | } 178 | 179 | size := p.Size() 180 | if size != 0 { 181 | t.Errorf("Expected file size 0, got %d", size) 182 | } 183 | } 184 | 185 | func TestPager_Size(t *testing.T) { 186 | defer os.Remove("test.bin") 187 | p, err := Open("test.bin", os.O_CREATE|os.O_RDWR, 0777, 1024, true, time.Millisecond*128) 188 | if err != nil { 189 | t.Errorf("Error opening file: %v", err) 190 | } 191 | defer p.Close() 192 | 193 | _, err = p.Write([]byte("hello world")) 194 | if err != nil { 195 | t.Errorf("Error writing to file: %v", err) 196 | } 197 | 198 | size := p.Size() 199 | if size <= 0 { 200 | t.Errorf("Expected file size greater than 0, got %d", size) 201 | } 202 | } 203 | 204 | func TestPager_EscalateFSync(t *testing.T) { 205 | defer os.Remove("test.bin") 206 | p, err := Open("test.bin", os.O_CREATE|os.O_RDWR, 0777, 1024, true, time.Millisecond*128) 207 | if err != nil { 208 | t.Errorf("Error opening file: %v", err) 209 | } 210 | defer p.Close() 211 | 212 | _, err = p.Write([]byte("hello world")) 213 | if err != nil { 214 | t.Errorf("Error writing to file: %v", err) 215 | } 216 | 217 | p.EscalateFSync() 218 | // No direct way to verify fsync 219 | } 220 | 221 | func TestPager_PageCount(t *testing.T) { 222 | defer os.Remove("test.bin") 223 | p, err := Open("test.bin", os.O_CREATE|os.O_RDWR, 0777, 4, true, time.Millisecond*128) 224 | if err != nil { 225 | t.Errorf("Error opening file: %v", err) 226 | } 227 | defer p.Close() 228 | 229 | _, err = p.Write([]byte("hello world")) 230 | if err != nil { 231 | t.Errorf("Error writing to file: %v", err) 232 | } 233 | 234 | pageCount := p.PageCount() 235 | if pageCount != 4 { 236 | t.Errorf("Expected 4 pages, got %d", pageCount) 237 | } 238 | } 239 | 240 | func BenchmarkPager_Write(b *testing.B) { 241 | defer os.Remove("test.bin") 242 | p, err := Open("test.bin", os.O_CREATE|os.O_RDWR, 0777, 128, true, time.Millisecond*128) 243 | if err != nil { 244 | b.Fatalf("Error opening file: %v", err) 245 | } 246 | defer p.Close() 247 | 248 | data := []byte("hello world") 249 | b.ResetTimer() 250 | for i := 0; i < b.N; i++ { 251 | if _, err := p.Write(data); err != nil { 252 | b.Fatalf("Error writing to file: %v", err) 253 | } 254 | } 255 | } 256 | -------------------------------------------------------------------------------- /starskey_test.go: -------------------------------------------------------------------------------- 1 | // Package starskey 2 | // 3 | // (C) Copyright Starskey 4 | // 5 | // Original Author: Alex Gaetano Padula 6 | // 7 | // Licensed under the Mozilla Public License, v. 2.0 (the "License"); 8 | // you may not use this file except in compliance with the License. 9 | // You may obtain a copy of the License at 10 | // 11 | // https://www.mozilla.org/en-US/MPL/2.0/ 12 | // 13 | // Unless required by applicable law or agreed to in writing, software 14 | // distributed under the License is distributed on an "AS IS" BASIS, 15 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | // See the License for the specific language governing permissions and 17 | // limitations under the License. 18 | package starskey 19 | 20 | import ( 21 | "bytes" 22 | "fmt" 23 | "log" 24 | "os" 25 | "reflect" 26 | "strings" 27 | "sync" 28 | "testing" 29 | "time" 30 | ) 31 | 32 | func TestSerializeDeserializeWalRecord(t *testing.T) { 33 | originalRecord := &WALRecord{ 34 | Key: []byte("testKey"), 35 | Value: []byte("testValue"), 36 | Op: Put, 37 | } 38 | 39 | // Serialize the original record 40 | serializedData, err := serializeWalRecord(originalRecord, false, NoCompression) 41 | if err != nil { 42 | t.Fatalf("Failed to serialize WAL record: %v", err) 43 | } 44 | 45 | // Deserialize the data back into a WALRecord 46 | deserializedRecord, err := deserializeWalRecord(serializedData, false, NoCompression) 47 | if err != nil { 48 | t.Fatalf("Failed to deserialize WAL record: %v", err) 49 | } 50 | 51 | // Check if the deserialized record matches the original record 52 | if !reflect.DeepEqual(originalRecord, deserializedRecord) { 53 | t.Errorf("Deserialized record does not match the original record.\nOriginal: %+v\nDeserialized: %+v", originalRecord, deserializedRecord) 54 | } 55 | } 56 | 57 | func TestSerializeDeserializeWalRecordInvalid(t *testing.T) { 58 | _, err := serializeWalRecord(nil, false, NoCompression) 59 | if err == nil { 60 | t.Fatalf("Failed to serialize WAL record: %v", err) 61 | 62 | } 63 | 64 | } 65 | 66 | func TestSerializeDeserializeKLogRecord(t *testing.T) { 67 | originalRecord := &KLogRecord{ 68 | Key: []byte("testKey"), 69 | ValPageNum: 12345, 70 | } 71 | 72 | // Serialize the original record 73 | serializedData, err := serializeKLogRecord(originalRecord, false, NoCompression) 74 | if err != nil { 75 | t.Fatalf("Failed to serialize KLog record: %v", err) 76 | } 77 | 78 | // Deserialize the data back into a KLogRecord 79 | deserializedRecord, err := deserializeKLogRecord(serializedData, false, NoCompression) 80 | if err != nil { 81 | t.Fatalf("Failed to deserialize KLog record: %v", err) 82 | 83 | } 84 | 85 | // Check if the deserialized record matches the original record 86 | if !reflect.DeepEqual(originalRecord, deserializedRecord) { 87 | t.Errorf("Deserialized record does not match the original record.\nOriginal: %+v\nDeserialized: %+v", originalRecord, deserializedRecord) 88 | } 89 | } 90 | 91 | func TestSerializeDeserializeKLogRecordInvalid(t *testing.T) { 92 | _, err := serializeKLogRecord(nil, false, NoCompression) 93 | if err == nil { 94 | t.Fatalf("Failed to serialize KLog record: %v", err) 95 | 96 | } 97 | 98 | } 99 | 100 | func TestSerializeDeserializeVLogRecord(t *testing.T) { 101 | originalRecord := &VLogRecord{ 102 | Value: []byte("testValue"), 103 | } 104 | 105 | // Serialize the original record 106 | serializedData, err := serializeVLogRecord(originalRecord, false, NoCompression) 107 | if err != nil { 108 | t.Fatalf("Failed to serialize VLog record: %v", err) 109 | } 110 | 111 | // Deserialize the data back into a VLogRecord 112 | deserializedRecord, err := deserializeVLogRecord(serializedData, false, NoCompression) 113 | if err != nil { 114 | t.Fatalf("Failed to deserialize VLog record: %v", err) 115 | } 116 | 117 | // Check if the deserialized record matches the original record 118 | if !reflect.DeepEqual(originalRecord, deserializedRecord) { 119 | t.Errorf("Deserialized record does not match the original record.\nOriginal: %+v\nDeserialized: %+v", originalRecord, deserializedRecord) 120 | } 121 | } 122 | 123 | func TestSerializeDeserializeVLogRecordInvalid(t *testing.T) { 124 | _, err := serializeVLogRecord(nil, false, NoCompression) 125 | if err == nil { 126 | t.Fatalf("Failed to serialize VLog record: %v", err) 127 | 128 | } 129 | 130 | } 131 | 132 | func TestSerializeDeserializeWalRecordCompress(t *testing.T) { 133 | originalRecord := &WALRecord{ 134 | Key: []byte("testKey"), 135 | Value: []byte("testValue"), 136 | Op: Put, 137 | } 138 | 139 | // Serialize the original record 140 | serializedData, err := serializeWalRecord(originalRecord, true, SnappyCompression) 141 | if err != nil { 142 | t.Fatalf("Failed to serialize WAL record: %v", err) 143 | } 144 | 145 | // Deserialize the data back into a WALRecord 146 | deserializedRecord, err := deserializeWalRecord(serializedData, true, SnappyCompression) 147 | if err != nil { 148 | t.Fatalf("Failed to deserialize WAL record: %v", err) 149 | } 150 | 151 | // Check if the deserialized record matches the original record 152 | if !reflect.DeepEqual(originalRecord, deserializedRecord) { 153 | t.Errorf("Deserialized record does not match the original record.\nOriginal: %+v\nDeserialized: %+v", originalRecord, deserializedRecord) 154 | } 155 | } 156 | 157 | func TestSerializeDeserializeKLogRecordCompress(t *testing.T) { 158 | originalRecord := &KLogRecord{ 159 | Key: []byte("testKey"), 160 | ValPageNum: 12345, 161 | } 162 | 163 | // Serialize the original record 164 | serializedData, err := serializeKLogRecord(originalRecord, true, SnappyCompression) 165 | if err != nil { 166 | t.Fatalf("Failed to serialize KLog record: %v", err) 167 | } 168 | 169 | // Deserialize the data back into a KLogRecord 170 | deserializedRecord, err := deserializeKLogRecord(serializedData, true, SnappyCompression) 171 | if err != nil { 172 | t.Fatalf("Failed to deserialize KLog record: %v", err) 173 | 174 | } 175 | 176 | // Check if the deserialized record matches the original record 177 | if !reflect.DeepEqual(originalRecord, deserializedRecord) { 178 | t.Errorf("Deserialized record does not match the original record.\nOriginal: %+v\nDeserialized: %+v", originalRecord, deserializedRecord) 179 | } 180 | } 181 | 182 | func TestSerializeDeserializeVLogRecordCompress(t *testing.T) { 183 | originalRecord := &VLogRecord{ 184 | Value: []byte("testValue"), 185 | } 186 | 187 | // Serialize the original record 188 | serializedData, err := serializeVLogRecord(originalRecord, true, SnappyCompression) 189 | if err != nil { 190 | t.Fatalf("Failed to serialize VLog record: %v", err) 191 | } 192 | 193 | // Deserialize the data back into a VLogRecord 194 | deserializedRecord, err := deserializeVLogRecord(serializedData, true, SnappyCompression) 195 | if err != nil { 196 | t.Fatalf("Failed to deserialize VLog record: %v", err) 197 | } 198 | 199 | // Check if the deserialized record matches the original record 200 | if !reflect.DeepEqual(originalRecord, deserializedRecord) { 201 | t.Errorf("Deserialized record does not match the original record.\nOriginal: %+v\nDeserialized: %+v", originalRecord, deserializedRecord) 202 | } 203 | } 204 | 205 | func TestSerializeDeserializeWalRecordCompress_S2(t *testing.T) { 206 | originalRecord := &WALRecord{ 207 | Key: []byte("testKey"), 208 | Value: []byte("testValue"), 209 | Op: Put, 210 | } 211 | 212 | // Serialize the original record 213 | serializedData, err := serializeWalRecord(originalRecord, true, S2Compression) 214 | if err != nil { 215 | t.Fatalf("Failed to serialize WAL record: %v", err) 216 | } 217 | 218 | // Deserialize the data back into a WALRecord 219 | deserializedRecord, err := deserializeWalRecord(serializedData, true, S2Compression) 220 | if err != nil { 221 | t.Fatalf("Failed to deserialize WAL record: %v", err) 222 | } 223 | 224 | // Check if the deserialized record matches the original record 225 | if !reflect.DeepEqual(originalRecord, deserializedRecord) { 226 | t.Errorf("Deserialized record does not match the original record.\nOriginal: %+v\nDeserialized: %+v", originalRecord, deserializedRecord) 227 | } 228 | } 229 | 230 | func TestSerializeDeserializeKLogRecordCompress_S2(t *testing.T) { 231 | originalRecord := &KLogRecord{ 232 | Key: []byte("testKey"), 233 | ValPageNum: 12345, 234 | } 235 | 236 | // Serialize the original record 237 | serializedData, err := serializeKLogRecord(originalRecord, true, S2Compression) 238 | if err != nil { 239 | t.Fatalf("Failed to serialize KLog record: %v", err) 240 | } 241 | 242 | // Deserialize the data back into a KLogRecord 243 | deserializedRecord, err := deserializeKLogRecord(serializedData, true, S2Compression) 244 | if err != nil { 245 | t.Fatalf("Failed to deserialize KLog record: %v", err) 246 | 247 | } 248 | 249 | // Check if the deserialized record matches the original record 250 | if !reflect.DeepEqual(originalRecord, deserializedRecord) { 251 | t.Errorf("Deserialized record does not match the original record.\nOriginal: %+v\nDeserialized: %+v", originalRecord, deserializedRecord) 252 | } 253 | } 254 | 255 | func TestSerializeDeserializeVLogRecordCompress_S2(t *testing.T) { 256 | originalRecord := &VLogRecord{ 257 | Value: []byte("testValue"), 258 | } 259 | 260 | // Serialize the original record 261 | serializedData, err := serializeVLogRecord(originalRecord, true, S2Compression) 262 | if err != nil { 263 | t.Fatalf("Failed to serialize VLog record: %v", err) 264 | } 265 | 266 | // Deserialize the data back into a VLogRecord 267 | deserializedRecord, err := deserializeVLogRecord(serializedData, true, S2Compression) 268 | if err != nil { 269 | t.Fatalf("Failed to deserialize VLog record: %v", err) 270 | } 271 | 272 | // Check if the deserialized record matches the original record 273 | if !reflect.DeepEqual(originalRecord, deserializedRecord) { 274 | t.Errorf("Deserialized record does not match the original record.\nOriginal: %+v\nDeserialized: %+v", originalRecord, deserializedRecord) 275 | } 276 | } 277 | 278 | func TestOpen(t *testing.T) { 279 | os.RemoveAll("test") 280 | defer os.RemoveAll("test") 281 | 282 | // Define a valid configuration 283 | config := &Config{ 284 | Permission: 0755, 285 | Directory: "test", 286 | FlushThreshold: 1024 * 1024, 287 | MaxLevel: 3, 288 | SizeFactor: 10, 289 | BloomFilter: false, 290 | } 291 | 292 | // Test opening starskey with a valid configuration 293 | starskey, err := Open(config) 294 | if err != nil { 295 | t.Fatalf("Failed to open starskey: %v", err) 296 | } 297 | 298 | // We verify the db directory is created with configured levels within 299 | for i := uint64(0); i < config.MaxLevel; i++ { 300 | if _, err := os.Stat(fmt.Sprintf("%s%sl%d", config.Directory, string(os.PathSeparator), i+1)); os.IsNotExist(err) { 301 | t.Fatalf("Failed to create directory for level %d", i) 302 | } 303 | 304 | } 305 | 306 | // Check if WAL exists 307 | if _, err := os.Stat(fmt.Sprintf("%s%s%s", config.Directory, string(os.PathSeparator), WALExtension)); os.IsNotExist(err) { 308 | t.Fatalf("Failed to create WAL file") 309 | } 310 | 311 | // Close starskey 312 | if err := starskey.Close(); err != nil { 313 | t.Fatalf("Failed to close starskey: %v", err) 314 | } 315 | } 316 | 317 | func TestOpenOptionalInternalConfig(t *testing.T) { 318 | os.RemoveAll("test") 319 | defer os.RemoveAll("test") 320 | 321 | // Define a valid configuration 322 | config := &Config{ 323 | Permission: 0755, 324 | Directory: "test", 325 | FlushThreshold: 1024 * 1024, 326 | MaxLevel: 3, 327 | SizeFactor: 10, 328 | BloomFilter: false, 329 | Optional: &OptionalConfig{ 330 | BackgroundFSync: false, 331 | BackgroundFSyncInterval: 0, 332 | TTreeMin: 32, 333 | TTreeMax: 64, 334 | PageSize: 1024, 335 | BloomFilterProbability: 0.25, 336 | }, 337 | } 338 | 339 | // Test opening starskey with a valid configuration 340 | starskey, err := Open(config) 341 | if err != nil { 342 | t.Fatalf("Failed to open starskey: %v", err) 343 | } 344 | 345 | // We verify the db directory is created with configured levels within 346 | for i := uint64(0); i < config.MaxLevel; i++ { 347 | if _, err := os.Stat(fmt.Sprintf("%s%sl%d", config.Directory, string(os.PathSeparator), i+1)); os.IsNotExist(err) { 348 | t.Fatalf("Failed to create directory for level %d", i) 349 | } 350 | 351 | } 352 | 353 | // Check if WAL exists 354 | if _, err := os.Stat(fmt.Sprintf("%s%s%s", config.Directory, string(os.PathSeparator), WALExtension)); os.IsNotExist(err) { 355 | t.Fatalf("Failed to create WAL file") 356 | } 357 | 358 | // Close starskey 359 | if err := starskey.Close(); err != nil { 360 | t.Fatalf("Failed to close starskey: %v", err) 361 | } 362 | } 363 | 364 | func TestOpenInvalid(t *testing.T) { 365 | os.RemoveAll("test") 366 | defer os.RemoveAll("test") 367 | 368 | _, err := Open(nil) 369 | if err == nil { 370 | t.Fatalf("Failed to open starskey: %v", err) 371 | 372 | } 373 | } 374 | 375 | func TestOpenInvalidCompression(t *testing.T) { 376 | os.RemoveAll("test") 377 | defer os.RemoveAll("test") 378 | 379 | config := &Config{ 380 | Permission: 0755, 381 | Directory: "test", 382 | FlushThreshold: 1024 * 1024, 383 | MaxLevel: 3, 384 | SizeFactor: 10, 385 | BloomFilter: false, 386 | Compression: true, 387 | CompressionOption: NoCompression, 388 | } 389 | 390 | _, err := Open(config) 391 | if err == nil { 392 | t.Fatalf("Failed to open starskey: %v", err) 393 | 394 | } 395 | } 396 | 397 | func TestStarskey_Put(t *testing.T) { 398 | _ = os.RemoveAll("test") 399 | defer func() { 400 | _ = os.RemoveAll("test") 401 | }() 402 | 403 | // Define a valid configuration 404 | config := &Config{ 405 | Permission: 0755, 406 | Directory: "test", 407 | FlushThreshold: 13780 / 2, 408 | MaxLevel: 3, 409 | SizeFactor: 10, 410 | BloomFilter: false, 411 | Logging: false, 412 | } 413 | 414 | starskey, err := Open(config) 415 | if err != nil { 416 | t.Fatalf("Failed to open starskey: %v", err) 417 | } 418 | 419 | size := 0 420 | 421 | for i := 0; i < 2000; i++ { 422 | key := []byte(fmt.Sprintf("key%03d", i)) 423 | value := []byte(fmt.Sprintf("value%03d", i)) 424 | 425 | if err := starskey.Put(key, value); err != nil { 426 | t.Fatalf("Failed to put key-value pair: %v", err) 427 | } 428 | 429 | size += len(key) + len(value) 430 | 431 | } 432 | 433 | for i := 0; i < 2000; i++ { 434 | key := []byte(fmt.Sprintf("key%03d", i)) 435 | value := []byte(fmt.Sprintf("value%03d", i)) 436 | 437 | val, err := starskey.Get(key) 438 | if err != nil { 439 | t.Fatalf("Failed to get key-value pair: %v", err) 440 | } 441 | 442 | if !reflect.DeepEqual(val, value) { 443 | t.Fatalf("Value does not match expected value") 444 | } 445 | 446 | } 447 | 448 | log.Println(size) 449 | 450 | if err := starskey.Close(); err != nil { 451 | t.Fatalf("Failed to close starskey: %v", err) 452 | } 453 | 454 | } 455 | 456 | func TestStarskey_Put_Invalid(t *testing.T) { 457 | _ = os.RemoveAll("test") 458 | defer func() { 459 | _ = os.RemoveAll("test") 460 | }() 461 | 462 | // Define a valid configuration 463 | config := &Config{ 464 | Permission: 0755, 465 | Directory: "test", 466 | FlushThreshold: 13780 / 2, 467 | MaxLevel: 3, 468 | SizeFactor: 10, 469 | BloomFilter: false, 470 | Logging: false, 471 | } 472 | 473 | starskey, err := Open(config) 474 | if err != nil { 475 | t.Fatalf("Failed to open starskey: %v", err) 476 | } 477 | 478 | err = starskey.Put(nil, nil) 479 | if err == nil { 480 | t.Fatalf("Failed to put key-value pair: %v", err) 481 | } 482 | 483 | if err := starskey.Close(); err != nil { 484 | t.Fatalf("Failed to close starskey: %v", err) 485 | 486 | } 487 | 488 | } 489 | 490 | func TestStarskey_Put_Get_Concurrent(t *testing.T) { 491 | _ = os.RemoveAll("test") 492 | defer func() { 493 | _ = os.RemoveAll("test") 494 | }() 495 | 496 | // Define a valid configuration 497 | config := &Config{ 498 | Permission: 0755, 499 | Directory: "test", 500 | FlushThreshold: 13780 / 2, 501 | MaxLevel: 3, 502 | SizeFactor: 10, 503 | BloomFilter: false, 504 | Logging: false, 505 | } 506 | 507 | starskey, err := Open(config) 508 | if err != nil { 509 | t.Fatalf("Failed to open starskey: %v", err) 510 | } 511 | 512 | var size int 513 | var mu sync.Mutex 514 | 515 | routines := 10 516 | batch := 2000 / routines 517 | wg := &sync.WaitGroup{} 518 | 519 | // Concurrently put key-value pairs 520 | for i := 0; i < routines; i++ { 521 | wg.Add(1) 522 | go func(i int) { 523 | defer wg.Done() 524 | for j := 0; j < batch; j++ { 525 | key := []byte(fmt.Sprintf("key%03d", i*batch+j)) 526 | value := []byte(fmt.Sprintf("value%03d", i*batch+j)) 527 | 528 | if err := starskey.Put(key, value); err != nil { 529 | t.Fatalf("Failed to put key-value pair: %v", err) 530 | } 531 | 532 | mu.Lock() 533 | size += len(key) + len(value) 534 | mu.Unlock() 535 | } 536 | }(i) 537 | } 538 | 539 | wg.Wait() 540 | 541 | // Concurrently get and check key-value pairs 542 | for i := 0; i < routines; i++ { 543 | wg.Add(1) 544 | go func(i int) { 545 | defer wg.Done() 546 | for j := 0; j < batch; j++ { 547 | key := []byte(fmt.Sprintf("key%03d", i*batch+j)) 548 | expectedValue := []byte(fmt.Sprintf("value%03d", i*batch+j)) 549 | 550 | val, err := starskey.Get(key) 551 | if err != nil { 552 | t.Fatalf("Failed to get key-value pair: %v", err) 553 | } 554 | 555 | if !reflect.DeepEqual(val, expectedValue) { 556 | t.Fatalf("Value does not match expected value for key %s: got %s, want %s", key, val, expectedValue) 557 | } 558 | } 559 | }(i) 560 | } 561 | 562 | wg.Wait() 563 | 564 | if err := starskey.Close(); err != nil { 565 | t.Fatalf("Failed to close starskey: %v", err) 566 | } 567 | } 568 | 569 | func TestStarskey_BeginTxn(t *testing.T) { 570 | _ = os.RemoveAll("test") 571 | defer func() { 572 | _ = os.RemoveAll("test") 573 | }() 574 | 575 | // Define a valid configuration 576 | config := &Config{ 577 | Permission: 0755, 578 | Directory: "test", 579 | FlushThreshold: 13780 / 2, 580 | MaxLevel: 3, 581 | SizeFactor: 10, 582 | BloomFilter: false, 583 | Logging: false, 584 | } 585 | 586 | starskey, err := Open(config) 587 | if err != nil { 588 | t.Fatalf("Failed to open starskey: %v", err) 589 | } 590 | 591 | txn := starskey.BeginTxn() 592 | if txn == nil { 593 | t.Fatalf("Failed to begin transaction") 594 | } 595 | 596 | txn.Put([]byte("key"), []byte("value")) 597 | 598 | if err := txn.Commit(); err != nil { 599 | t.Fatalf("Failed to commit transaction: %v", err) 600 | } 601 | 602 | // Get 603 | val, err := starskey.Get([]byte("key")) 604 | if err != nil { 605 | t.Fatalf("Failed to get key-value pair: %v", err) 606 | } 607 | 608 | if !reflect.DeepEqual(val, []byte("value")) { 609 | t.Fatalf("Value does not match expected value") 610 | 611 | } 612 | 613 | if err := starskey.Close(); err != nil { 614 | t.Fatalf("Failed to close starskey: %v", err) 615 | } 616 | 617 | } 618 | 619 | func TestStarskey_Update(t *testing.T) { 620 | _ = os.RemoveAll("test") 621 | defer func() { 622 | _ = os.RemoveAll("test") 623 | }() 624 | 625 | // Define a valid configuration 626 | config := &Config{ 627 | Permission: 0755, 628 | Directory: "test", 629 | FlushThreshold: 13780 / 2, 630 | MaxLevel: 3, 631 | SizeFactor: 10, 632 | BloomFilter: false, 633 | Logging: false, 634 | } 635 | 636 | starskey, err := Open(config) 637 | if err != nil { 638 | t.Fatalf("Failed to open starskey: %v", err) 639 | } 640 | 641 | err = starskey.Update(func(txn *Txn) error { 642 | txn.Put([]byte("key"), []byte("value")) 643 | return nil 644 | }) 645 | if err != nil { 646 | t.Fatalf("Failed to update: %v", err) 647 | } 648 | 649 | // Get 650 | val, err := starskey.Get([]byte("key")) 651 | if err != nil { 652 | t.Fatalf("Failed to get key-value pair: %v", err) 653 | } 654 | 655 | if !reflect.DeepEqual(val, []byte("value")) { 656 | t.Fatalf("Value does not match expected value") 657 | } 658 | 659 | if err := starskey.Close(); err != nil { 660 | t.Fatalf("Failed to close starskey: %v", err) 661 | } 662 | } 663 | 664 | func TestStarskey_Reopen(t *testing.T) { 665 | _ = os.RemoveAll("test") 666 | defer func() { 667 | _ = os.RemoveAll("test") 668 | }() 669 | 670 | // Define a valid configuration 671 | config := &Config{ 672 | Permission: 0755, 673 | Directory: "test", 674 | FlushThreshold: 13780 / 2, 675 | MaxLevel: 3, 676 | SizeFactor: 10, 677 | BloomFilter: false, 678 | Logging: false, 679 | } 680 | 681 | starskey, err := Open(config) 682 | if err != nil { 683 | t.Fatalf("Failed to open starskey: %v", err) 684 | } 685 | 686 | size := 0 687 | 688 | for i := 0; i < 2000; i++ { 689 | key := []byte(fmt.Sprintf("key%03d", i)) 690 | value := []byte(fmt.Sprintf("value%03d", i)) 691 | 692 | if err := starskey.Put(key, value); err != nil { 693 | t.Fatalf("Failed to put key-value pair: %v", err) 694 | } 695 | 696 | size += len(key) + len(value) 697 | 698 | } 699 | 700 | if err := starskey.Close(); err != nil { 701 | t.Fatalf("Failed to close starskey: %v", err) 702 | } 703 | 704 | starskey, err = Open(config) 705 | if err != nil { 706 | t.Fatalf("Failed to open starskey: %v", err) 707 | } 708 | 709 | for i := 0; i < 2000; i++ { 710 | key := []byte(fmt.Sprintf("key%03d", i)) 711 | value := []byte(fmt.Sprintf("value%03d", i)) 712 | 713 | val, err := starskey.Get(key) 714 | if err != nil { 715 | t.Fatalf("Failed to get key-value pair: %v", err) 716 | } 717 | 718 | if !reflect.DeepEqual(val, value) { 719 | t.Fatalf("Value does not match expected value") 720 | } 721 | 722 | } 723 | 724 | if err := starskey.Close(); err != nil { 725 | t.Fatalf("Failed to close starskey: %v", err) 726 | } 727 | 728 | } 729 | 730 | func TestStarskey_Put_Bloom(t *testing.T) { 731 | _ = os.RemoveAll("test") 732 | defer func() { 733 | _ = os.RemoveAll("test") 734 | }() 735 | 736 | config := &Config{ 737 | Permission: 0755, 738 | Directory: "test", 739 | FlushThreshold: 13780 / 2, 740 | MaxLevel: 3, 741 | SizeFactor: 10, 742 | BloomFilter: true, 743 | } 744 | 745 | starskey, err := Open(config) 746 | if err != nil { 747 | t.Fatalf("Failed to open starskey: %v", err) 748 | } 749 | 750 | size := 0 751 | 752 | for i := 0; i < 2000; i++ { 753 | key := []byte(fmt.Sprintf("key%03d", i)) 754 | value := []byte(fmt.Sprintf("value%03d", i)) 755 | 756 | if err := starskey.Put(key, value); err != nil { 757 | t.Fatalf("Failed to put key-value pair: %v", err) 758 | } 759 | 760 | size += len(key) + len(value) 761 | 762 | } 763 | 764 | for i := 0; i < 2000; i++ { 765 | key := []byte(fmt.Sprintf("key%03d", i)) 766 | value := []byte(fmt.Sprintf("value%03d", i)) 767 | 768 | val, err := starskey.Get(key) 769 | if err != nil { 770 | t.Fatalf("Failed to get key-value pair: %v", err) 771 | } 772 | 773 | if !reflect.DeepEqual(val, value) { 774 | t.Fatalf("Value does not match expected value") 775 | } 776 | 777 | } 778 | 779 | log.Println(size) 780 | 781 | if err := starskey.Close(); err != nil { 782 | t.Fatalf("Failed to close starskey: %v", err) 783 | } 784 | 785 | } 786 | 787 | func TestStarskey_Bloom_Reopen(t *testing.T) { 788 | _ = os.RemoveAll("test") 789 | defer func() { 790 | _ = os.RemoveAll("test") 791 | }() 792 | 793 | // Define a valid configuration 794 | config := &Config{ 795 | Permission: 0755, 796 | Directory: "test", 797 | FlushThreshold: 13780 / 2, 798 | MaxLevel: 3, 799 | SizeFactor: 10, 800 | BloomFilter: true, 801 | Logging: false, 802 | } 803 | 804 | starskey, err := Open(config) 805 | if err != nil { 806 | t.Fatalf("Failed to open starskey: %v", err) 807 | } 808 | 809 | size := 0 810 | 811 | for i := 0; i < 2000; i++ { 812 | key := []byte(fmt.Sprintf("key%03d", i)) 813 | value := []byte(fmt.Sprintf("value%03d", i)) 814 | 815 | if err := starskey.Put(key, value); err != nil { 816 | t.Fatalf("Failed to put key-value pair: %v", err) 817 | } 818 | 819 | size += len(key) + len(value) 820 | 821 | } 822 | 823 | if err := starskey.Close(); err != nil { 824 | t.Fatalf("Failed to close starskey: %v", err) 825 | } 826 | 827 | starskey, err = Open(config) 828 | if err != nil { 829 | t.Fatalf("Failed to open starskey: %v", err) 830 | } 831 | 832 | for i := 0; i < 2000; i++ { 833 | key := []byte(fmt.Sprintf("key%03d", i)) 834 | value := []byte(fmt.Sprintf("value%03d", i)) 835 | 836 | val, err := starskey.Get(key) 837 | if err != nil { 838 | t.Fatalf("Failed to get key-value pair: %v", err) 839 | } 840 | 841 | if !reflect.DeepEqual(val, value) { 842 | t.Fatalf("Value does not match expected value") 843 | } 844 | 845 | } 846 | 847 | if err := starskey.Close(); err != nil { 848 | t.Fatalf("Failed to close starskey: %v", err) 849 | } 850 | 851 | } 852 | 853 | func TestStarskey_FilterKeys(t *testing.T) { 854 | _ = os.RemoveAll("test") 855 | defer func() { 856 | _ = os.RemoveAll("test") 857 | }() 858 | 859 | config := &Config{ 860 | Permission: 0755, 861 | Directory: "test", 862 | FlushThreshold: 13780 / 2, 863 | MaxLevel: 3, 864 | SizeFactor: 10, 865 | BloomFilter: false, 866 | } 867 | 868 | starskey, err := Open(config) 869 | if err != nil { 870 | t.Fatalf("Failed to open starskey: %v", err) 871 | } 872 | 873 | size := 0 874 | 875 | for i := 0; i < 250; i++ { 876 | key := []byte(fmt.Sprintf("aey%03d", i)) 877 | value := []byte(fmt.Sprintf("value%03d", i)) 878 | 879 | if err := starskey.Put(key, value); err != nil { 880 | t.Fatalf("Failed to put key-value pair: %v", err) 881 | } 882 | 883 | size += len(key) + len(value) 884 | 885 | } 886 | 887 | for i := 0; i < 250; i++ { 888 | key := []byte(fmt.Sprintf("bey%03d", i)) 889 | value := []byte(fmt.Sprintf("value%03d", i)) 890 | 891 | if err := starskey.Put(key, value); err != nil { 892 | t.Fatalf("Failed to put key-value pair: %v", err) 893 | } 894 | 895 | size += len(key) + len(value) 896 | 897 | } 898 | 899 | expect := make(map[string]bool) 900 | 901 | for i := 0; i < 500; i++ { 902 | key := []byte(fmt.Sprintf("cey%03d", i)) 903 | value := []byte(fmt.Sprintf("value%03d", i)) 904 | expect[string(value)] = false 905 | 906 | if err := starskey.Put(key, value); err != nil { 907 | t.Fatalf("Failed to put key-value pair: %v", err) 908 | } 909 | 910 | size += len(key) + len(value) 911 | 912 | } 913 | 914 | compareFunc := func(key []byte) bool { 915 | // if has prefix "c" return true 916 | return bytes.HasPrefix(key, []byte("c")) 917 | } 918 | 919 | results, err := starskey.FilterKeys(compareFunc) 920 | if err != nil { 921 | t.Fatalf("Failed to filter: %v", err) 922 | } 923 | 924 | for _, key := range results { 925 | if _, ok := expect[string(key)]; ok { 926 | expect[string(key)] = true 927 | } 928 | 929 | } 930 | 931 | for _, v := range expect { 932 | if !v { 933 | t.Fatalf("Value does not match expected value") 934 | } 935 | } 936 | 937 | if err := starskey.Close(); err != nil { 938 | t.Fatalf("Failed to close starskey: %v", err) 939 | } 940 | } 941 | 942 | func TestStarskey_FilterKeys_Invalid(t *testing.T) { 943 | _ = os.RemoveAll("test") 944 | defer func() { 945 | _ = os.RemoveAll("test") 946 | }() 947 | 948 | config := &Config{ 949 | Permission: 0755, 950 | Directory: "test", 951 | FlushThreshold: 13780 / 2, 952 | MaxLevel: 3, 953 | SizeFactor: 10, 954 | BloomFilter: false, 955 | } 956 | 957 | starskey, err := Open(config) 958 | if err != nil { 959 | t.Fatalf("Failed to open starskey: %v", err) 960 | } 961 | 962 | // Test invalid compare function 963 | _, err = starskey.FilterKeys(nil) 964 | if err == nil { 965 | t.Fatalf("Failed to filter: %v", err) 966 | } 967 | 968 | if err := starskey.Close(); err != nil { 969 | t.Fatalf("Failed to close starskey: %v", err) 970 | } 971 | 972 | } 973 | 974 | func TestStarskey_DeleteByRange_Sequential(t *testing.T) { 975 | _ = os.RemoveAll("test") 976 | defer func() { 977 | _ = os.RemoveAll("test") 978 | }() 979 | 980 | config := &Config{ 981 | Permission: 0755, 982 | Directory: "test", 983 | FlushThreshold: 13780 / 2, 984 | MaxLevel: 3, 985 | SizeFactor: 10, 986 | BloomFilter: false, 987 | } 988 | 989 | starskey, err := Open(config) 990 | if err != nil { 991 | t.Fatalf("Failed to open starskey: %v", err) 992 | } 993 | 994 | // Insert sequential data like in Range test 995 | for i := 0; i < 1000; i++ { 996 | key := []byte(fmt.Sprintf("key%03d", i)) 997 | value := []byte(fmt.Sprintf("value%03d", i)) 998 | 999 | if err := starskey.Put(key, value); err != nil { 1000 | t.Fatalf("Failed to put key-value pair: %v", err) 1001 | } 1002 | } 1003 | 1004 | // Delete a range 1005 | startKey := []byte("key200") 1006 | endKey := []byte("key299") 1007 | 1008 | if _, err := starskey.DeleteByRange(startKey, endKey); err != nil { 1009 | t.Fatalf("Failed to delete range: %v", err) 1010 | } 1011 | 1012 | // Verify deletion 1013 | for i := 0; i < 1000; i++ { 1014 | key := []byte(fmt.Sprintf("key%03d", i)) 1015 | value, err := starskey.Get(key) 1016 | if err != nil { 1017 | t.Fatalf("Failed to get key: %v", err) 1018 | } 1019 | 1020 | // Keys in deleted range should return nil 1021 | if i >= 200 && i <= 299 { 1022 | if value != nil { 1023 | t.Errorf("Key %s should be deleted but has value %s", key, value) 1024 | } 1025 | } else { 1026 | // Other keys should still have their values 1027 | expectedValue := []byte(fmt.Sprintf("value%03d", i)) 1028 | if !bytes.Equal(value, expectedValue) { 1029 | t.Errorf("Key %s has wrong value. Got %s, want %s", key, value, expectedValue) 1030 | } 1031 | } 1032 | } 1033 | 1034 | if err := starskey.Close(); err != nil { 1035 | t.Fatalf("Failed to close starskey: %v", err) 1036 | } 1037 | } 1038 | 1039 | func TestStarskey_Range(t *testing.T) { 1040 | _ = os.RemoveAll("test") 1041 | defer func() { 1042 | _ = os.RemoveAll("test") 1043 | }() 1044 | 1045 | config := &Config{ 1046 | Permission: 0755, 1047 | Directory: "test", 1048 | FlushThreshold: 13780 / 2, 1049 | MaxLevel: 3, 1050 | SizeFactor: 10, 1051 | BloomFilter: false, 1052 | } 1053 | 1054 | starskey, err := Open(config) 1055 | if err != nil { 1056 | t.Fatalf("Failed to open starskey: %v", err) 1057 | } 1058 | 1059 | size := 0 1060 | 1061 | for i := 0; i < 1000; i++ { 1062 | key := []byte(fmt.Sprintf("key%03d", i)) 1063 | value := []byte(fmt.Sprintf("value%03d", i)) 1064 | 1065 | if err := starskey.Put(key, value); err != nil { 1066 | t.Fatalf("Failed to put key-value pair: %v", err) 1067 | } 1068 | 1069 | size += len(key) + len(value) 1070 | 1071 | } 1072 | 1073 | results, err := starskey.Range([]byte("key900"), []byte("key980")) 1074 | if err != nil { 1075 | t.Fatalf("Failed to range: %v", err) 1076 | } 1077 | 1078 | for i := 0; i < 80; i++ { 1079 | i += 900 1080 | value := []byte(fmt.Sprintf("value%03d", i)) 1081 | 1082 | i -= 900 1083 | if !reflect.DeepEqual(results[i], value) { 1084 | t.Fatalf("Value does not match expected value") 1085 | } 1086 | 1087 | } 1088 | 1089 | if err := starskey.Close(); err != nil { 1090 | t.Fatalf("Failed to close starskey: %v", err) 1091 | } 1092 | } 1093 | 1094 | func TestStarskey_Range_Invalid(t *testing.T) { 1095 | _ = os.RemoveAll("test") 1096 | defer func() { 1097 | _ = os.RemoveAll("test") 1098 | }() 1099 | 1100 | config := &Config{ 1101 | Permission: 0755, 1102 | Directory: "test", 1103 | FlushThreshold: 13780 / 2, 1104 | MaxLevel: 3, 1105 | SizeFactor: 10, 1106 | BloomFilter: false, 1107 | } 1108 | 1109 | starskey, err := Open(config) 1110 | if err != nil { 1111 | t.Fatalf("Failed to open starskey: %v", err) 1112 | } 1113 | 1114 | // Test invalid range 1115 | _, err = starskey.Range([]byte("key900"), []byte("key800")) 1116 | if err == nil { 1117 | t.Fatalf("Failed to range: %v", err) 1118 | } 1119 | 1120 | if err := starskey.Close(); err != nil { 1121 | t.Fatalf("Failed to close starskey: %v", err) 1122 | } 1123 | } 1124 | 1125 | func TestStarskey_Range_Invalid2(t *testing.T) { 1126 | _ = os.RemoveAll("test") 1127 | defer func() { 1128 | _ = os.RemoveAll("test") 1129 | }() 1130 | 1131 | config := &Config{ 1132 | Permission: 0755, 1133 | Directory: "test", 1134 | FlushThreshold: 13780 / 2, 1135 | MaxLevel: 3, 1136 | SizeFactor: 10, 1137 | BloomFilter: false, 1138 | } 1139 | 1140 | starskey, err := Open(config) 1141 | if err != nil { 1142 | t.Fatalf("Failed to open starskey: %v", err) 1143 | } 1144 | 1145 | // Test invalid range 1146 | _, err = starskey.Range(nil, nil) 1147 | if err == nil { 1148 | t.Fatalf("Failed to range: %v", err) 1149 | } 1150 | 1151 | if err := starskey.Close(); err != nil { 1152 | t.Fatalf("Failed to close starskey: %v", err) 1153 | } 1154 | } 1155 | 1156 | func TestStarskey_Delete_Invalid(t *testing.T) { 1157 | _ = os.RemoveAll("test") 1158 | defer func() { 1159 | _ = os.RemoveAll("test") 1160 | }() 1161 | 1162 | config := &Config{ 1163 | Permission: 0755, 1164 | Directory: "test", 1165 | FlushThreshold: 13780 / 2, 1166 | MaxLevel: 3, 1167 | SizeFactor: 10, 1168 | BloomFilter: false, 1169 | } 1170 | 1171 | starskey, err := Open(config) 1172 | if err != nil { 1173 | t.Fatalf("Failed to open starskey: %v", err) 1174 | } 1175 | 1176 | // Test invalid delete 1177 | err = starskey.Delete(nil) 1178 | if err == nil { 1179 | t.Fatalf("Failed to delete key: %v", err) 1180 | } 1181 | 1182 | if err := starskey.Close(); err != nil { 1183 | t.Fatalf("Failed to close starskey: %v", err) 1184 | } 1185 | } 1186 | 1187 | func TestStarskey_Delete(t *testing.T) { 1188 | _ = os.RemoveAll("test") 1189 | defer func() { 1190 | _ = os.RemoveAll("test") 1191 | }() 1192 | 1193 | config := &Config{ 1194 | Permission: 0755, 1195 | Directory: "test", 1196 | FlushThreshold: 13780 / 2, 1197 | MaxLevel: 3, 1198 | SizeFactor: 10, 1199 | BloomFilter: false, 1200 | } 1201 | 1202 | starskey, err := Open(config) 1203 | if err != nil { 1204 | t.Fatalf("Failed to open starskey: %v", err) 1205 | } 1206 | 1207 | size := 0 1208 | 1209 | for i := 0; i < 1000; i++ { 1210 | key := []byte(fmt.Sprintf("key%03d", i)) 1211 | value := []byte(fmt.Sprintf("value%03d", i)) 1212 | 1213 | if err := starskey.Put(key, value); err != nil { 1214 | t.Fatalf("Failed to put key-value pair: %v", err) 1215 | } 1216 | 1217 | size += len(key) + len(value) 1218 | 1219 | } 1220 | 1221 | for i := 0; i < 1000; i++ { 1222 | key := []byte(fmt.Sprintf("key%03d", i)) 1223 | 1224 | if err := starskey.Delete(key); err != nil { 1225 | t.Fatalf("Failed to delete key: %v", err) 1226 | } 1227 | 1228 | } 1229 | 1230 | for i := 0; i < 1000; i++ { 1231 | key := []byte(fmt.Sprintf("key%03d", i)) 1232 | val, _ := starskey.Get(key) 1233 | if val != nil { 1234 | t.Fatalf("Failed to delete key: %v", val) 1235 | } 1236 | 1237 | } 1238 | 1239 | if err := starskey.Close(); err != nil { 1240 | t.Fatalf("Failed to close starskey: %v", err) 1241 | } 1242 | } 1243 | 1244 | func TestStarskey_DeleteByRange_SuRF(t *testing.T) { 1245 | _ = os.RemoveAll("test") 1246 | defer func() { 1247 | _ = os.RemoveAll("test") 1248 | }() 1249 | 1250 | config := &Config{ 1251 | Permission: 0755, 1252 | Directory: "test", 1253 | FlushThreshold: 1024, 1254 | MaxLevel: 3, 1255 | SizeFactor: 10, 1256 | BloomFilter: false, 1257 | SuRF: true, 1258 | } 1259 | 1260 | starskey, err := Open(config) 1261 | if err != nil { 1262 | t.Fatalf("Failed to open starskey: %v", err) 1263 | } 1264 | 1265 | // Insert test data in different ranges 1266 | ranges := []struct { 1267 | prefix string 1268 | start int 1269 | end int 1270 | }{ 1271 | {"aaa", 0, 100}, 1272 | {"bbb", 100, 200}, 1273 | {"ccc", 200, 300}, 1274 | } 1275 | 1276 | // Track inserted keys for verification 1277 | insertedKeys := make(map[string]bool) 1278 | 1279 | // Insert data and force flush after each range to ensure separate SSTables 1280 | for _, r := range ranges { 1281 | for i := r.start; i < r.end; i++ { 1282 | key := []byte(fmt.Sprintf("%s%03d", r.prefix, i)) 1283 | value := []byte(fmt.Sprintf("value%03d", i)) 1284 | 1285 | if err := starskey.Put(key, value); err != nil { 1286 | t.Fatalf("Failed to put key-value pair: %v", err) 1287 | } 1288 | insertedKeys[string(key)] = true 1289 | } 1290 | // Force flush after each range 1291 | if err := starskey.run(); err != nil { 1292 | t.Fatalf("Failed to force flush: %v", err) 1293 | } 1294 | } 1295 | 1296 | // Test range deletion spanning multiple SSTables 1297 | startKey := []byte("bbb150") 1298 | endKey := []byte("ccc250") 1299 | 1300 | // Delete range 1301 | if _, err := starskey.DeleteByRange(startKey, endKey); err != nil { 1302 | t.Fatalf("Failed to delete range: %v", err) 1303 | } 1304 | 1305 | // Test partial range within single SSTable 1306 | startKey = []byte("aaa030") 1307 | endKey = []byte("aaa060") 1308 | 1309 | if _, err := starskey.DeleteByRange(startKey, endKey); err != nil { 1310 | t.Fatalf("Failed to delete range: %v", err) 1311 | } 1312 | 1313 | // Verify final state with actual gets 1314 | t.Log("Verifying deletions...") 1315 | for key := range insertedKeys { 1316 | inFirstRange := bytes.Compare([]byte(key), []byte("aaa030")) >= 0 && 1317 | bytes.Compare([]byte(key), []byte("aaa060")) <= 0 1318 | inSecondRange := bytes.Compare([]byte(key), []byte("bbb150")) >= 0 && 1319 | bytes.Compare([]byte(key), []byte("ccc250")) <= 0 1320 | shouldExist := !inFirstRange && !inSecondRange 1321 | 1322 | val, err := starskey.Get([]byte(key)) 1323 | if err != nil { 1324 | t.Fatalf("Failed to get key-value pair: %v", err) 1325 | } 1326 | 1327 | if shouldExist && val == nil { 1328 | t.Errorf("Key %s should exist but doesn't", key) 1329 | } 1330 | if !shouldExist && val != nil { 1331 | t.Errorf("Key %s should be deleted but exists with value %s", key, val) 1332 | } 1333 | } 1334 | 1335 | if err := starskey.Close(); err != nil { 1336 | t.Fatalf("Failed to close starskey: %v", err) 1337 | } 1338 | } 1339 | 1340 | func TestStarskey_DeleteByFilter(t *testing.T) { 1341 | _ = os.RemoveAll("test") 1342 | defer func() { 1343 | _ = os.RemoveAll("test") 1344 | }() 1345 | 1346 | config := &Config{ 1347 | Permission: 0755, 1348 | Directory: "test", 1349 | FlushThreshold: 13780 / 2, 1350 | MaxLevel: 3, 1351 | SizeFactor: 10, 1352 | BloomFilter: false, 1353 | SuRF: false, 1354 | } 1355 | 1356 | starskey, err := Open(config) 1357 | if err != nil { 1358 | t.Fatalf("Failed to open starskey: %v", err) 1359 | } 1360 | 1361 | // Insert test data with different key prefixes 1362 | for i := 0; i < 1000; i++ { 1363 | // Create keys with different prefixes to test filter deletion 1364 | var key []byte 1365 | if i < 300 { 1366 | key = []byte(fmt.Sprintf("test1_%03d", i)) // test1_000 - test1_299 1367 | } else if i < 600 { 1368 | key = []byte(fmt.Sprintf("test2_%03d", i)) // test2_300 - test2_599 1369 | } else { 1370 | key = []byte(fmt.Sprintf("other_%03d", i)) // other_600 - other_999 1371 | } 1372 | value := []byte(fmt.Sprintf("value%03d", i)) 1373 | 1374 | if err := starskey.Put(key, value); err != nil { 1375 | t.Fatalf("Failed to put key-value pair: %v", err) 1376 | } 1377 | } 1378 | 1379 | // Create filter function to delete all keys with "test" prefix 1380 | filterFunc := func(key []byte) bool { 1381 | return bytes.HasPrefix(key, []byte("test")) 1382 | } 1383 | 1384 | // Delete all keys matching the filter 1385 | if _, err := starskey.DeleteByFilter(filterFunc); err != nil { 1386 | t.Fatalf("Failed to delete by filter: %v", err) 1387 | } 1388 | 1389 | // Verify all "test" prefixed keys are deleted 1390 | for i := 0; i < 600; i++ { 1391 | var key []byte 1392 | if i < 300 { 1393 | key = []byte(fmt.Sprintf("test1_%03d", i)) 1394 | } else { 1395 | key = []byte(fmt.Sprintf("test2_%03d", i)) 1396 | } 1397 | 1398 | val, err := starskey.Get(key) 1399 | if err != nil { 1400 | t.Fatalf("Failed to get key-value pair: %v", err) 1401 | } 1402 | if val != nil { 1403 | t.Fatalf("Key %s should have been deleted", key) 1404 | } 1405 | } 1406 | 1407 | // Verify other keys are still present 1408 | for i := 600; i < 1000; i++ { 1409 | key := []byte(fmt.Sprintf("other_%03d", i)) 1410 | value := []byte(fmt.Sprintf("value%03d", i)) 1411 | 1412 | val, err := starskey.Get(key) 1413 | if err != nil { 1414 | t.Fatalf("Failed to get key-value pair: %v", err) 1415 | } 1416 | if !reflect.DeepEqual(val, value) { 1417 | t.Fatalf("Value does not match expected value for key %s", key) 1418 | } 1419 | } 1420 | 1421 | // Test invalid filter function 1422 | _, err = starskey.DeleteByFilter(nil) 1423 | if err == nil { 1424 | t.Fatal("DeleteByFilter should return error when filter function is nil") 1425 | } 1426 | 1427 | if err := starskey.Close(); err != nil { 1428 | t.Fatalf("Failed to close starskey: %v", err) 1429 | } 1430 | } 1431 | 1432 | func TestStarskey_LongestPrefixSearch(t *testing.T) { 1433 | _ = os.RemoveAll("test") 1434 | defer func() { 1435 | _ = os.RemoveAll("test") 1436 | }() 1437 | 1438 | config := &Config{ 1439 | Permission: 0755, 1440 | Directory: "test", 1441 | FlushThreshold: 13780 / 2, // Using same threshold as other tests 1442 | MaxLevel: 3, 1443 | SizeFactor: 10, 1444 | BloomFilter: false, 1445 | } 1446 | 1447 | starskey, err := Open(config) 1448 | if err != nil { 1449 | t.Fatalf("Failed to open starskey: %v", err) 1450 | } 1451 | 1452 | // Insert test data with different key prefixes 1453 | testData := []struct { 1454 | key string 1455 | value string 1456 | }{ 1457 | {"com", "root domain"}, 1458 | {"com.example", "example domain"}, 1459 | {"com.example.www", "www subdomain"}, 1460 | {"com.example.mail", "mail subdomain"}, 1461 | {"com.test", "test domain"}, 1462 | {"org", "org domain"}, 1463 | {"org.example", "example org"}, 1464 | } 1465 | 1466 | // Insert the test data 1467 | for _, data := range testData { 1468 | if err := starskey.Put([]byte(data.key), []byte(data.value)); err != nil { 1469 | t.Fatalf("Failed to put key-value pair: %v", err) 1470 | } 1471 | } 1472 | 1473 | // Force a flush to ensure data is in SSTables 1474 | if err := starskey.run(); err != nil { 1475 | t.Fatalf("Failed to force flush: %v", err) 1476 | } 1477 | 1478 | // Test cases 1479 | tests := []struct { 1480 | name string 1481 | searchKey string 1482 | expectedValue string 1483 | expectedLen int 1484 | expectError bool 1485 | }{ 1486 | { 1487 | name: "Exact match", 1488 | searchKey: "com.example", 1489 | expectedValue: "example domain", 1490 | expectedLen: 11, 1491 | expectError: false, 1492 | }, 1493 | { 1494 | name: "Longer key with existing prefix", 1495 | searchKey: "com.example.www.subdomain", 1496 | expectedValue: "www subdomain", 1497 | expectedLen: 15, 1498 | expectError: false, 1499 | }, 1500 | { 1501 | name: "Partial match", 1502 | searchKey: "com.another", 1503 | expectedValue: "root domain", 1504 | expectedLen: 3, 1505 | expectError: false, 1506 | }, 1507 | { 1508 | name: "No match", 1509 | searchKey: "net.example", 1510 | expectedValue: "", 1511 | expectedLen: 0, 1512 | expectError: false, 1513 | }, 1514 | { 1515 | name: "Empty key", 1516 | searchKey: "", 1517 | expectedValue: "", 1518 | expectedLen: 0, 1519 | expectError: true, 1520 | }, 1521 | } 1522 | 1523 | // Run test cases 1524 | for _, tc := range tests { 1525 | t.Run(tc.name, func(t *testing.T) { 1526 | value, length, err := starskey.LongestPrefixSearch([]byte(tc.searchKey)) 1527 | 1528 | // Check error expectation 1529 | if tc.expectError { 1530 | if err == nil { 1531 | t.Error("Expected error but got none") 1532 | } 1533 | return 1534 | } 1535 | 1536 | if err != nil { 1537 | t.Fatalf("Unexpected error: %v", err) 1538 | } 1539 | 1540 | // For no match case 1541 | if tc.expectedLen == 0 { 1542 | if value != nil { 1543 | t.Errorf("Expected no match, but got value: %s", value) 1544 | } 1545 | return 1546 | } 1547 | 1548 | // Check results 1549 | if length != tc.expectedLen { 1550 | t.Errorf("Expected length %d, got %d", tc.expectedLen, length) 1551 | } 1552 | 1553 | if !bytes.Equal(value, []byte(tc.expectedValue)) { 1554 | t.Errorf("Expected value %s, got %s", tc.expectedValue, value) 1555 | } 1556 | }) 1557 | } 1558 | 1559 | if err := starskey.Close(); err != nil { 1560 | t.Fatalf("Failed to close starskey: %v", err) 1561 | } 1562 | 1563 | // Test with SuRF enabled 1564 | surfConfig := &Config{ 1565 | Permission: 0755, 1566 | Directory: "test_surf", 1567 | FlushThreshold: 13780 / 2, // Using same threshold as other tests 1568 | MaxLevel: 3, 1569 | SizeFactor: 10, 1570 | BloomFilter: false, 1571 | SuRF: true, 1572 | } 1573 | 1574 | _ = os.RemoveAll("test_surf") 1575 | defer os.RemoveAll("test_surf") 1576 | 1577 | starskeyWithSurf, err := Open(surfConfig) 1578 | if err != nil { 1579 | t.Fatalf("Failed to open starskey with SuRF: %v", err) 1580 | } 1581 | 1582 | // Insert the same test data 1583 | for _, data := range testData { 1584 | if err := starskeyWithSurf.Put([]byte(data.key), []byte(data.value)); err != nil { 1585 | t.Fatalf("Failed to put key-value pair with SuRF: %v", err) 1586 | } 1587 | } 1588 | 1589 | // Force a flush to ensure SuRF is used 1590 | if err := starskeyWithSurf.run(); err != nil { 1591 | t.Fatalf("Failed to force flush: %v", err) 1592 | } 1593 | 1594 | // Run same tests with SuRF 1595 | for _, tc := range tests { 1596 | t.Run(tc.name+"_with_surf", func(t *testing.T) { 1597 | value, length, err := starskeyWithSurf.LongestPrefixSearch([]byte(tc.searchKey)) 1598 | 1599 | // Check error expectation 1600 | if tc.expectError { 1601 | if err == nil { 1602 | t.Error("Expected error but got none") 1603 | } 1604 | return 1605 | } 1606 | 1607 | if err != nil { 1608 | t.Fatalf("Unexpected error: %v", err) 1609 | } 1610 | 1611 | // For no match case 1612 | if tc.expectedLen == 0 { 1613 | if value != nil { 1614 | t.Errorf("Expected no match, but got value: %s", value) 1615 | } 1616 | return 1617 | } 1618 | 1619 | // Check results 1620 | if length != tc.expectedLen { 1621 | t.Errorf("Expected length %d, got %d", tc.expectedLen, length) 1622 | } 1623 | 1624 | if !bytes.Equal(value, []byte(tc.expectedValue)) { 1625 | t.Errorf("Expected value %s, got %s", tc.expectedValue, value) 1626 | } 1627 | }) 1628 | } 1629 | 1630 | if err := starskeyWithSurf.Close(); err != nil { 1631 | t.Fatalf("Failed to close starskey with SuRF: %v", err) 1632 | } 1633 | } 1634 | 1635 | func TestStarskey_DeleteByPrefix(t *testing.T) { 1636 | _ = os.RemoveAll("test") 1637 | defer func() { 1638 | _ = os.RemoveAll("test") 1639 | }() 1640 | 1641 | config := &Config{ 1642 | Permission: 0755, 1643 | Directory: "test", 1644 | FlushThreshold: 13780 / 2, 1645 | MaxLevel: 3, 1646 | SizeFactor: 10, 1647 | BloomFilter: false, 1648 | } 1649 | 1650 | starskey, err := Open(config) 1651 | if err != nil { 1652 | t.Fatalf("Failed to open starskey: %v", err) 1653 | } 1654 | 1655 | // Insert test data with different prefixes 1656 | prefixes := []string{"test", "demo", "sample"} 1657 | keysPerPrefix := 100 1658 | 1659 | // Insert data 1660 | for _, prefix := range prefixes { 1661 | for i := 0; i < keysPerPrefix; i++ { 1662 | key := []byte(fmt.Sprintf("%s_%03d", prefix, i)) 1663 | value := []byte(fmt.Sprintf("value_%s_%03d", prefix, i)) 1664 | if err := starskey.Put(key, value); err != nil { 1665 | t.Fatalf("Failed to put key-value pair: %v", err) 1666 | } 1667 | } 1668 | } 1669 | 1670 | // Force flush to ensure data is in SSTables 1671 | if err := starskey.run(); err != nil { 1672 | t.Fatalf("Failed to force flush: %v", err) 1673 | } 1674 | 1675 | // Delete all keys with prefix "test" 1676 | deletedCount, err := starskey.DeleteByPrefix([]byte("test")) 1677 | if err != nil { 1678 | t.Fatalf("Failed to delete by prefix: %v", err) 1679 | } 1680 | 1681 | // Verify deletion count 1682 | if deletedCount != keysPerPrefix { 1683 | t.Errorf("Expected %d deletions, got %d", keysPerPrefix, deletedCount) 1684 | } 1685 | 1686 | // Verify deletions and remaining keys 1687 | for _, prefix := range prefixes { 1688 | for i := 0; i < keysPerPrefix; i++ { 1689 | key := []byte(fmt.Sprintf("%s_%03d", prefix, i)) 1690 | value := []byte(fmt.Sprintf("value_%s_%03d", prefix, i)) 1691 | val, err := starskey.Get(key) 1692 | if err != nil { 1693 | t.Fatalf("Failed to get key %s: %v", key, err) 1694 | } 1695 | 1696 | if prefix == "test" { 1697 | if val != nil { 1698 | t.Errorf("Key %s should be deleted but has value %s", key, val) 1699 | } 1700 | } else { 1701 | if !bytes.Equal(val, value) { 1702 | t.Errorf("Expected value %s for key %s, got %s", value, key, val) 1703 | } 1704 | } 1705 | } 1706 | } 1707 | 1708 | // Test invalid prefix 1709 | count, err := starskey.DeleteByPrefix(nil) 1710 | if err == nil { 1711 | t.Error("Expected error for nil prefix but got none") 1712 | } 1713 | if count != 0 { 1714 | t.Errorf("Expected 0 deletions for nil prefix, got %d", count) 1715 | } 1716 | 1717 | if err := starskey.Close(); err != nil { 1718 | t.Fatalf("Failed to close starskey: %v", err) 1719 | } 1720 | } 1721 | 1722 | func TestStarskey_DeleteByPrefix_SuRF(t *testing.T) { 1723 | _ = os.RemoveAll("test") 1724 | defer func() { 1725 | _ = os.RemoveAll("test") 1726 | }() 1727 | 1728 | config := &Config{ 1729 | Permission: 0755, 1730 | Directory: "test", 1731 | FlushThreshold: 13780 / 2, 1732 | MaxLevel: 3, 1733 | SizeFactor: 10, 1734 | BloomFilter: false, 1735 | SuRF: true, 1736 | } 1737 | 1738 | starskey, err := Open(config) 1739 | if err != nil { 1740 | t.Fatalf("Failed to open starskey: %v", err) 1741 | } 1742 | 1743 | prefixes := []string{"test", "demo", "sample"} 1744 | keysPerPrefix := 100 1745 | 1746 | // Insert data and force flush after each prefix 1747 | for _, prefix := range prefixes { 1748 | for i := 0; i < keysPerPrefix; i++ { 1749 | key := []byte(fmt.Sprintf("%s_%03d", prefix, i)) 1750 | value := []byte(fmt.Sprintf("value_%s_%03d", prefix, i)) 1751 | if err := starskey.Put(key, value); err != nil { 1752 | t.Fatalf("Failed to put key-value pair: %v", err) 1753 | } 1754 | } 1755 | 1756 | } 1757 | 1758 | // Force flush after each prefix 1759 | if err := starskey.run(); err != nil { 1760 | t.Fatalf("Failed to force flush: %v", err) 1761 | } 1762 | 1763 | // Delete all keys with prefix "test" 1764 | deletedCount, err := starskey.DeleteByPrefix([]byte("test")) 1765 | if err != nil { 1766 | t.Fatalf("Failed to delete by prefix: %v", err) 1767 | } 1768 | 1769 | if deletedCount != keysPerPrefix { 1770 | t.Errorf("Expected %d deletions, got %d", keysPerPrefix, deletedCount) 1771 | } 1772 | 1773 | // Verify deletions and remaining keys 1774 | for _, prefix := range prefixes { 1775 | for i := 0; i < keysPerPrefix; i++ { 1776 | key := []byte(fmt.Sprintf("%s_%03d", prefix, i)) 1777 | value := []byte(fmt.Sprintf("value_%s_%03d", prefix, i)) 1778 | val, err := starskey.Get(key) 1779 | if err != nil { 1780 | t.Fatalf("Failed to get key %s: %v", key, err) 1781 | } 1782 | 1783 | if prefix == "test" { 1784 | if val != nil { 1785 | t.Errorf("Key %s should be deleted but has value %s", key, val) 1786 | } 1787 | } else { 1788 | if !bytes.Equal(val, value) { 1789 | t.Errorf("Expected value %s for key %s, got %s", value, key, val) 1790 | } 1791 | } 1792 | } 1793 | } 1794 | 1795 | if err := starskey.Close(); err != nil { 1796 | t.Fatalf("Failed to close starskey: %v", err) 1797 | } 1798 | } 1799 | 1800 | func TestStarskey_PrefixSearch(t *testing.T) { 1801 | _ = os.RemoveAll("test") 1802 | defer func() { 1803 | _ = os.RemoveAll("test") 1804 | }() 1805 | 1806 | config := &Config{ 1807 | Permission: 0755, 1808 | Directory: "test", 1809 | FlushThreshold: 13780 / 2, 1810 | MaxLevel: 3, 1811 | SizeFactor: 10, 1812 | BloomFilter: false, 1813 | } 1814 | 1815 | starskey, err := Open(config) 1816 | if err != nil { 1817 | t.Fatalf("Failed to open starskey: %v", err) 1818 | } 1819 | 1820 | // Insert test data with different prefixes 1821 | testData := map[string][]struct { 1822 | key string 1823 | value string 1824 | }{ 1825 | "com": { 1826 | {"com.example.www", "website"}, 1827 | {"com.example.mail", "email"}, 1828 | {"com.test", "test site"}, 1829 | }, 1830 | "org": { 1831 | {"org.example", "org site"}, 1832 | {"org.test", "test org"}, 1833 | }, 1834 | "net": { 1835 | {"net.demo", "demo net"}, 1836 | }, 1837 | } 1838 | 1839 | // Insert all test data 1840 | for _, entries := range testData { 1841 | for _, entry := range entries { 1842 | if err := starskey.Put([]byte(entry.key), []byte(entry.value)); err != nil { 1843 | t.Fatalf("Failed to put key-value pair: %v", err) 1844 | } 1845 | } 1846 | } 1847 | 1848 | // Force flush to ensure data is in SSTables 1849 | if err := starskey.run(); err != nil { 1850 | t.Fatalf("Failed to force flush: %v", err) 1851 | } 1852 | 1853 | // Test cases 1854 | tests := []struct { 1855 | prefix string 1856 | expectedKeys int 1857 | }{ 1858 | {"com", 3}, 1859 | {"com.example", 2}, 1860 | {"org", 2}, 1861 | {"net", 1}, 1862 | {"invalid", 0}, 1863 | } 1864 | 1865 | for _, tc := range tests { 1866 | t.Run(fmt.Sprintf("Prefix_%s", tc.prefix), func(t *testing.T) { 1867 | results, err := starskey.PrefixSearch([]byte(tc.prefix)) 1868 | if err != nil { 1869 | t.Fatalf("Failed to search prefix %s: %v", tc.prefix, err) 1870 | } 1871 | 1872 | if len(results) != tc.expectedKeys { 1873 | t.Errorf("Expected %d results for prefix %s, got %d", tc.expectedKeys, tc.prefix, len(results)) 1874 | } 1875 | 1876 | // Verify each result corresponds to a valid key with the prefix 1877 | resultMap := make(map[string]bool) 1878 | for _, result := range results { 1879 | resultMap[string(result)] = true 1880 | } 1881 | 1882 | matchCount := 0 1883 | for _, entries := range testData { 1884 | for _, entry := range entries { 1885 | if strings.HasPrefix(entry.key, tc.prefix) { 1886 | matchCount++ 1887 | if !resultMap[entry.value] { 1888 | t.Errorf("Expected to find value %s for key %s with prefix %s", entry.value, entry.key, tc.prefix) 1889 | } 1890 | } 1891 | } 1892 | } 1893 | 1894 | if matchCount != len(results) { 1895 | t.Errorf("Number of matches (%d) doesn't match number of results (%d) for prefix %s", matchCount, len(results), tc.prefix) 1896 | } 1897 | }) 1898 | } 1899 | 1900 | // Test invalid prefix 1901 | results, err := starskey.PrefixSearch(nil) 1902 | if err == nil { 1903 | t.Error("Expected error for nil prefix but got none") 1904 | } 1905 | if results != nil { 1906 | t.Errorf("Expected nil results for nil prefix, got %v", results) 1907 | } 1908 | 1909 | if err := starskey.Close(); err != nil { 1910 | t.Fatalf("Failed to close starskey: %v", err) 1911 | } 1912 | } 1913 | 1914 | func TestStarskey_PrefixSearch_SuRF(t *testing.T) { 1915 | _ = os.RemoveAll("test") 1916 | defer func() { 1917 | _ = os.RemoveAll("test") 1918 | }() 1919 | 1920 | config := &Config{ 1921 | Permission: 0755, 1922 | Directory: "test", 1923 | FlushThreshold: 13780 / 2, 1924 | MaxLevel: 3, 1925 | SizeFactor: 10, 1926 | BloomFilter: false, 1927 | SuRF: true, 1928 | } 1929 | 1930 | starskey, err := Open(config) 1931 | if err != nil { 1932 | t.Fatalf("Failed to open starskey: %v", err) 1933 | } 1934 | 1935 | // Insert test data with different prefixes 1936 | testData := map[string][]struct { 1937 | key string 1938 | value string 1939 | }{ 1940 | "com": { 1941 | {"com.example.www", "website"}, 1942 | {"com.example.mail", "email"}, 1943 | {"com.test", "test site"}, 1944 | }, 1945 | "org": { 1946 | {"org.example", "org site"}, 1947 | {"org.test", "test org"}, 1948 | }, 1949 | "net": { 1950 | {"net.demo", "demo net"}, 1951 | }, 1952 | } 1953 | 1954 | // Insert data and force flush after each prefix group 1955 | for prefix, entries := range testData { 1956 | for _, entry := range entries { 1957 | if err := starskey.Put([]byte(entry.key), []byte(entry.value)); err != nil { 1958 | t.Fatalf("Failed to put key-value pair: %v", err) 1959 | } 1960 | } 1961 | // Force flush after each prefix group 1962 | if err := starskey.run(); err != nil { 1963 | t.Fatalf("Failed to force flush after prefix %s: %v", prefix, err) 1964 | } 1965 | } 1966 | 1967 | // Test cases 1968 | tests := []struct { 1969 | prefix string 1970 | expectedKeys int 1971 | }{ 1972 | {"com", 3}, 1973 | {"com.example", 2}, 1974 | {"org", 2}, 1975 | {"net", 1}, 1976 | {"invalid", 0}, 1977 | } 1978 | 1979 | for _, tc := range tests { 1980 | t.Run(fmt.Sprintf("Prefix_%s_SuRF", tc.prefix), func(t *testing.T) { 1981 | results, err := starskey.PrefixSearch([]byte(tc.prefix)) 1982 | if err != nil { 1983 | t.Fatalf("Failed to search prefix %s: %v", tc.prefix, err) 1984 | } 1985 | 1986 | if len(results) != tc.expectedKeys { 1987 | t.Errorf("Expected %d results for prefix %s, got %d", tc.expectedKeys, tc.prefix, len(results)) 1988 | } 1989 | 1990 | // Verify each result corresponds to a valid key with the prefix 1991 | resultMap := make(map[string]bool) 1992 | for _, result := range results { 1993 | resultMap[string(result)] = true 1994 | } 1995 | 1996 | matchCount := 0 1997 | for _, entries := range testData { 1998 | for _, entry := range entries { 1999 | if strings.HasPrefix(entry.key, tc.prefix) { 2000 | matchCount++ 2001 | if !resultMap[entry.value] { 2002 | t.Errorf("Expected to find value %s for key %s with prefix %s", entry.value, entry.key, tc.prefix) 2003 | } 2004 | } 2005 | } 2006 | } 2007 | 2008 | if matchCount != len(results) { 2009 | t.Errorf("Number of matches (%d) doesn't match number of results (%d) for prefix %s", matchCount, len(results), tc.prefix) 2010 | } 2011 | }) 2012 | } 2013 | 2014 | if err := starskey.Close(); err != nil { 2015 | t.Fatalf("Failed to close starskey: %v", err) 2016 | } 2017 | } 2018 | 2019 | func TestStarskey_ChanneledLogging(t *testing.T) { 2020 | defer os.RemoveAll("db_dir") 2021 | // Create a buffered channel for logs 2022 | logChannel := make(chan string, 1000) 2023 | 2024 | // Create Starskey instance with log channel configured 2025 | skey, err := Open(&Config{ 2026 | Permission: 0755, 2027 | Directory: "db_dir", 2028 | FlushThreshold: (1024 * 1024) * 24, // 24MB 2029 | MaxLevel: 3, 2030 | SizeFactor: 10, 2031 | BloomFilter: false, 2032 | SuRF: false, 2033 | Logging: true, 2034 | Compression: false, 2035 | CompressionOption: NoCompression, 2036 | 2037 | // Configure the LogChannel in OptionalConfig 2038 | Optional: &OptionalConfig{ 2039 | LogChannel: logChannel, 2040 | }, 2041 | }) 2042 | if err != nil { 2043 | fmt.Printf("Failed to open Starskey: %v\n", err) 2044 | return 2045 | } 2046 | defer skey.Close() 2047 | 2048 | // Start a goroutine to consume and process logs in real-time 2049 | var wg sync.WaitGroup 2050 | wg.Add(1) 2051 | 2052 | go func() { 2053 | defer wg.Done() 2054 | for logMsg := range logChannel { 2055 | // Process log messages in real-time 2056 | timestamp := time.Now().Format("2006-01-02 15:04:05.000") 2057 | 2058 | fmt.Printf("[%s] %s\n", timestamp, logMsg) 2059 | } 2060 | }() 2061 | 2062 | // Use Starskey for your normal operations 2063 | for i := 0; i < 100; i++ { 2064 | key := []byte(fmt.Sprintf("key%d", i)) 2065 | value := []byte(fmt.Sprintf("value%d", i)) 2066 | 2067 | if err := skey.Put(key, value); err != nil { 2068 | fmt.Printf("Failed to put key-value: %v\n", err) 2069 | } 2070 | } 2071 | 2072 | // The log channel will keep receiving logs until Starskey is closed 2073 | time.Sleep(2 * time.Second) // Give some time for operations to complete 2074 | 2075 | // Close starskey as we are done 2076 | skey.Close() 2077 | 2078 | // close the channel as Starskey doesn't close it 2079 | close(logChannel) 2080 | 2081 | // Wait for log processing to complete 2082 | wg.Wait() 2083 | 2084 | } 2085 | 2086 | func BenchmarkStarskey_Put(b *testing.B) { 2087 | _ = os.RemoveAll("test") 2088 | defer func() { 2089 | _ = os.RemoveAll("test") 2090 | }() 2091 | 2092 | config := &Config{ 2093 | Permission: 0755, 2094 | Directory: "test", 2095 | FlushThreshold: (1024 * 1024) * 64, 2096 | MaxLevel: 3, 2097 | SizeFactor: 10, 2098 | BloomFilter: false, 2099 | Logging: true, 2100 | } 2101 | 2102 | starskey, err := Open(config) 2103 | if err != nil { 2104 | b.Fatalf("Failed to open starskey: %v", err) 2105 | } 2106 | defer starskey.Close() 2107 | 2108 | b.ResetTimer() 2109 | for i := 0; i < b.N; i++ { 2110 | key := []byte(fmt.Sprintf("key%06d", i)) 2111 | value := []byte(fmt.Sprintf("value%06d", i)) 2112 | if err := starskey.Put(key, value); err != nil { 2113 | b.Fatalf("Failed to put key-value pair: %v", err) 2114 | } 2115 | } 2116 | } 2117 | 2118 | func BenchmarkStarskey_Get(b *testing.B) { 2119 | _ = os.RemoveAll("test") 2120 | defer func() { 2121 | _ = os.RemoveAll("test") 2122 | }() 2123 | 2124 | config := &Config{ 2125 | Permission: 0755, 2126 | Directory: "test", 2127 | FlushThreshold: (1024 * 1024) * 64, 2128 | MaxLevel: 3, 2129 | SizeFactor: 10, 2130 | BloomFilter: false, 2131 | Logging: true, 2132 | } 2133 | 2134 | starskey, err := Open(config) 2135 | if err != nil { 2136 | b.Fatalf("Failed to open starskey: %v", err) 2137 | } 2138 | defer starskey.Close() 2139 | 2140 | for i := 0; i < 1000; i++ { 2141 | key := []byte(fmt.Sprintf("key%06d", i)) 2142 | value := []byte(fmt.Sprintf("value%06d", i)) 2143 | if err := starskey.Put(key, value); err != nil { 2144 | b.Fatalf("Failed to put key-value pair: %v", err) 2145 | } 2146 | } 2147 | 2148 | b.ResetTimer() 2149 | for i := 0; i < b.N; i++ { 2150 | key := []byte(fmt.Sprintf("key%06d", i%1000)) 2151 | if _, err := starskey.Get(key); err != nil { 2152 | b.Fatalf("Failed to get key-value pair: %v", err) 2153 | } 2154 | } 2155 | } 2156 | 2157 | func BenchmarkStarskey_Delete(b *testing.B) { 2158 | _ = os.RemoveAll("test") 2159 | defer func() { 2160 | _ = os.RemoveAll("test") 2161 | }() 2162 | 2163 | config := &Config{ 2164 | Permission: 0755, 2165 | Directory: "test", 2166 | FlushThreshold: (1024 * 1024) * 64, 2167 | MaxLevel: 3, 2168 | SizeFactor: 10, 2169 | BloomFilter: false, 2170 | Logging: true, 2171 | } 2172 | 2173 | starskey, err := Open(config) 2174 | if err != nil { 2175 | b.Fatalf("Failed to open starskey: %v", err) 2176 | } 2177 | defer starskey.Close() 2178 | 2179 | for i := 0; i < 1000; i++ { 2180 | key := []byte(fmt.Sprintf("key%06d", i)) 2181 | value := []byte(fmt.Sprintf("value%06d", i)) 2182 | if err := starskey.Put(key, value); err != nil { 2183 | b.Fatalf("Failed to put key-value pair: %v", err) 2184 | } 2185 | } 2186 | 2187 | b.ResetTimer() 2188 | for i := 0; i < b.N; i++ { 2189 | key := []byte(fmt.Sprintf("key%06d", i%1000)) 2190 | if err := starskey.Delete(key); err != nil { 2191 | b.Fatalf("Failed to delete key: %v", err) 2192 | } 2193 | } 2194 | } 2195 | -------------------------------------------------------------------------------- /surf/surf.go: -------------------------------------------------------------------------------- 1 | // Package ttree 2 | // 3 | // (C) Copyright Starskey 4 | // 5 | // Original Author: Alex Gaetano Padula 6 | // 7 | // Licensed under the Mozilla Public License, v. 2.0 (the "License"); 8 | // you may not use this file except in compliance with the License. 9 | // You may obtain a copy of the License at 10 | // 11 | // https://www.mozilla.org/en-US/MPL/2.0/ 12 | // 13 | // Unless required by applicable law or agreed to in writing, software 14 | // distributed under the License is distributed on an "AS IS" BASIS, 15 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | // See the License for the specific language governing permissions and 17 | // limitations under the License. 18 | package surf 19 | 20 | import ( 21 | "bytes" 22 | "encoding/gob" 23 | "math" 24 | 25 | "github.com/cespare/xxhash/v2" 26 | ) 27 | 28 | // SuRF represents a succinct filter inspired by https://www.cs.cmu.edu/~huanche1/publications/surf_paper.pdf 29 | type SuRF struct { 30 | Trie *TrieNode // Root node of the trie 31 | SuffixBits int // Number of suffix bits for better filtering 32 | HashBits int // Number of hash bits for better filtering 33 | TotalKeys uint64 // Track total keys for adaptive suffix sizing 34 | } 35 | 36 | // TrieNode represents a node in the SuRF trie 37 | type TrieNode struct { 38 | Children map[byte]*TrieNode // Map of children nodes 39 | IsLeaf bool // Indicates if the node is a leaf 40 | Suffix uint64 // Suffix bits for better filtering 41 | Hash uint64 // Additional hash for better filtering 42 | Height int // Track node height for balanced deletion 43 | } 44 | 45 | // New initializes a new SuRF 46 | func New(expectedKeys int) *SuRF { 47 | return &SuRF{ 48 | Trie: &TrieNode{Children: make(map[byte]*TrieNode), Height: 1}, 49 | SuffixBits: OptimalSuffixBits(expectedKeys), 50 | HashBits: OptimalHashBits(expectedKeys), 51 | TotalKeys: 0, 52 | } 53 | } 54 | 55 | // OptimalSuffixBits calculates the optimal number of suffix bits for better filtering 56 | func OptimalSuffixBits(totalKeys int) int { 57 | entropy := math.Log2(float64(totalKeys)) 58 | baseBits := int(math.Ceil(entropy / 4)) 59 | if baseBits < 4 { 60 | return 4 61 | } 62 | if baseBits > 16 { 63 | return 16 64 | } 65 | return baseBits 66 | } 67 | 68 | // OptimalHashBits calculates the optimal number of hash bits for better filtering 69 | func OptimalHashBits(totalKeys int) int { 70 | return OptimalSuffixBits(totalKeys) * 2 71 | } 72 | 73 | // Add inserts a key into the SuRF 74 | func (s *SuRF) Add(key []byte) { 75 | node := s.Trie 76 | path := make([]byte, 0, len(key)) 77 | 78 | for _, b := range key { 79 | path = append(path, b) 80 | if _, exists := node.Children[b]; !exists { 81 | node.Children[b] = &TrieNode{ 82 | Children: make(map[byte]*TrieNode), 83 | Height: len(path), 84 | } 85 | } 86 | node = node.Children[b] 87 | } 88 | 89 | node.IsLeaf = true 90 | node.Suffix = MixedSuffixBits(key, s.SuffixBits) 91 | node.Hash = ComputeNodeHash(key, path, s.HashBits) 92 | s.TotalKeys++ 93 | 94 | // Rebalance if needed 95 | s.rebalanceAfterAdd(node) 96 | } 97 | 98 | // CheckRange checks if a range of keys exists in the SuRF 99 | func (s *SuRF) CheckRange(lower, upper []byte) bool { 100 | return s.checkRangeHelper(s.Trie, lower, upper, 0, make([]byte, 0)) 101 | } 102 | 103 | // checkRangeHelper recursively checks if a range of keys exists in the SuRF 104 | func (s *SuRF) checkRangeHelper(node *TrieNode, lower, upper []byte, depth int, path []byte) bool { 105 | if node == nil { 106 | return false 107 | } 108 | 109 | if node.IsLeaf { 110 | suffix := MixedSuffixBits(lower, s.SuffixBits) 111 | upperSuffix := MixedSuffixBits(upper, s.SuffixBits) 112 | 113 | if node.Suffix >= suffix && node.Suffix <= upperSuffix { 114 | nodeHash := ComputeNodeHash(path, path, s.HashBits) 115 | if node.Hash == nodeHash { 116 | return true 117 | } 118 | } 119 | return false 120 | } 121 | 122 | for b, child := range node.Children { 123 | newPath := append(path, b) 124 | if depth < len(lower) && depth < len(upper) { 125 | if b >= lower[depth] && b <= upper[depth] { 126 | if s.checkRangeHelper(child, lower, upper, depth+1, newPath) { 127 | return true 128 | } 129 | } 130 | } else if depth >= len(lower) && depth < len(upper) { 131 | if b <= upper[depth] { 132 | if s.checkRangeHelper(child, lower, upper, depth+1, newPath) { 133 | return true 134 | } 135 | } 136 | } else if depth < len(lower) && depth >= len(upper) { 137 | if b >= lower[depth] { 138 | if s.checkRangeHelper(child, lower, upper, depth+1, newPath) { 139 | return true 140 | } 141 | } 142 | } else { 143 | if s.checkRangeHelper(child, lower, upper, depth+1, newPath) { 144 | return true 145 | } 146 | } 147 | } 148 | return false 149 | } 150 | 151 | // MixedSuffixBits computes a combined suffix using both hash and real suffix 152 | func MixedSuffixBits(key []byte, bits int) uint64 { 153 | if len(key) == 0 { 154 | return 0 155 | } 156 | 157 | hashValue := xxhash.Sum64(key) 158 | var realSuffix uint64 159 | 160 | suffixStart := max(0, len(key)-bits/4) 161 | for i := suffixStart; i < len(key); i++ { 162 | realSuffix = (realSuffix << 8) | uint64(key[i]) 163 | } 164 | 165 | mask := uint64((1 << bits) - 1) 166 | return ((hashValue & (mask >> 1)) << (bits / 2)) | (realSuffix & ((1 << (bits / 2)) - 1)) 167 | } 168 | 169 | // ComputeNodeHash computes a hash for a node using both key and path 170 | func ComputeNodeHash(key []byte, path []byte, bits int) uint64 { 171 | combined := append(path, key...) 172 | hash := xxhash.Sum64(combined) 173 | return hash & ((1 << bits) - 1) 174 | } 175 | 176 | // Delete removes a key from the SuRF 177 | func (s *SuRF) Delete(key []byte) bool { 178 | success, _ := s.deleteHelper(s.Trie, key, 0, nil) 179 | if success { 180 | s.TotalKeys-- 181 | } 182 | return success 183 | } 184 | 185 | // deleteHelper returns two booleans, the first indicates if the key was successfully deleted 186 | // and the second indicates if the node should be deleted as well 187 | func (s *SuRF) deleteHelper(node *TrieNode, key []byte, depth int, parent *TrieNode) (bool, bool) { 188 | if node == nil { 189 | return false, false 190 | } 191 | 192 | if depth == len(key) { 193 | if !node.IsLeaf { 194 | return false, false 195 | } 196 | 197 | node.IsLeaf = false 198 | shouldDelete := len(node.Children) == 0 199 | 200 | // Update heights 201 | if parent != nil && shouldDelete { 202 | s.updateHeights(parent) 203 | } 204 | 205 | return true, shouldDelete 206 | } 207 | 208 | b := key[depth] 209 | child, exists := node.Children[b] 210 | if !exists { 211 | return false, false 212 | } 213 | 214 | deleted, shouldDelete := s.deleteHelper(child, key, depth+1, node) 215 | if shouldDelete { 216 | delete(node.Children, b) 217 | 218 | // Clean up empty non-leaf node 219 | if !node.IsLeaf && len(node.Children) == 0 { 220 | return deleted, true 221 | } 222 | 223 | // Rebalance if needed 224 | s.rebalanceAfterDelete(node) 225 | } 226 | 227 | return deleted, false 228 | } 229 | 230 | // rebalanceAfterAdd rebalances the tree after an addition 231 | func (s *SuRF) rebalanceAfterAdd(node *TrieNode) { 232 | if node == nil { 233 | return 234 | } 235 | 236 | maxHeight := 0 237 | minHeight := math.MaxInt32 238 | 239 | for _, child := range node.Children { 240 | if child.Height > maxHeight { 241 | maxHeight = child.Height 242 | } 243 | if child.Height < minHeight { 244 | minHeight = child.Height 245 | } 246 | } 247 | 248 | // Update node height 249 | node.Height = maxHeight + 1 250 | 251 | // Check if rebalancing is needed 252 | if maxHeight-minHeight > 1 { 253 | s.rotateNode(node) 254 | } 255 | } 256 | 257 | // rebalanceAfterDelete rebalances the tree after a deletion 258 | func (s *SuRF) rebalanceAfterDelete(node *TrieNode) { 259 | if node == nil { 260 | return 261 | } 262 | 263 | s.updateHeights(node) 264 | 265 | // Check balance factor 266 | balance := s.getBalance(node) 267 | if math.Abs(float64(balance)) > 1 { 268 | s.rotateNode(node) 269 | } 270 | } 271 | 272 | // updateHeights updates the height of a node based on its children 273 | func (s *SuRF) updateHeights(node *TrieNode) { 274 | if node == nil { 275 | return 276 | } 277 | 278 | maxHeight := 0 279 | for _, child := range node.Children { 280 | if child.Height > maxHeight { 281 | maxHeight = child.Height 282 | } 283 | } 284 | 285 | node.Height = maxHeight + 1 286 | } 287 | 288 | // getBalance returns the balance factor of a node 289 | func (s *SuRF) getBalance(node *TrieNode) int { 290 | if node == nil { 291 | return 0 292 | } 293 | 294 | leftHeight := 0 295 | rightHeight := 0 296 | 297 | // Find leftmost and rightmost children 298 | for _, child := range node.Children { 299 | if child.Height > leftHeight { 300 | leftHeight = child.Height 301 | } 302 | if child.Height > rightHeight { 303 | rightHeight = child.Height 304 | } 305 | } 306 | 307 | return leftHeight - rightHeight 308 | } 309 | 310 | // rotateNode performs the necessary rotations to balance the node 311 | func (s *SuRF) rotateNode(node *TrieNode) { 312 | balance := s.getBalance(node) 313 | 314 | // Get the highest and lowest byte keys 315 | var highestByte, lowestByte byte 316 | first := true 317 | for b := range node.Children { 318 | if first { 319 | highestByte = b 320 | lowestByte = b 321 | first = false 322 | continue 323 | } 324 | if b > highestByte { 325 | highestByte = b 326 | } 327 | if b < lowestByte { 328 | lowestByte = b 329 | } 330 | } 331 | 332 | if balance > 1 { // Left-heavy 333 | leftChild := node.Children[lowestByte] 334 | 335 | // Determine rotation type 336 | leftBalance := s.getBalance(leftChild) 337 | if leftBalance < 0 { 338 | // Left-Right rotation needed 339 | s.rotateLeft(leftChild) 340 | } 341 | s.rotateRight(node) 342 | 343 | } else if balance < -1 { // Right-heavy 344 | rightChild := node.Children[highestByte] 345 | 346 | // Determine rotation type 347 | rightBalance := s.getBalance(rightChild) 348 | if rightBalance > 0 { 349 | // Right-Left rotation needed 350 | s.rotateRight(rightChild) 351 | } 352 | s.rotateLeft(node) 353 | } 354 | } 355 | 356 | // rotateLeft rotates the node to the left 357 | func (s *SuRF) rotateLeft(node *TrieNode) { 358 | // Find the rightmost child 359 | var rightmostByte byte 360 | var rightChild *TrieNode 361 | for b, child := range node.Children { 362 | if rightChild == nil || b > rightmostByte { 363 | rightmostByte = b 364 | rightChild = child 365 | } 366 | } 367 | 368 | if rightChild == nil { 369 | return // Cannot rotate if no right child 370 | } 371 | 372 | // Store original node's children except rightmost 373 | oldChildren := make(map[byte]*TrieNode) 374 | for b, child := range node.Children { 375 | if b != rightmostByte { 376 | oldChildren[b] = child 377 | } 378 | } 379 | 380 | // Move appropriate children from right child to node 381 | var lowestByteInRight byte 382 | first := true 383 | for b, _ := range rightChild.Children { 384 | if first { 385 | lowestByteInRight = b 386 | first = false 387 | } 388 | if b < lowestByteInRight { 389 | lowestByteInRight = b 390 | } 391 | } 392 | 393 | // Redistribute children 394 | newNodeChildren := make(map[byte]*TrieNode) 395 | for b, child := range rightChild.Children { 396 | if b < lowestByteInRight { 397 | newNodeChildren[b] = child 398 | delete(rightChild.Children, b) 399 | } 400 | } 401 | 402 | // Update the children maps 403 | for b, child := range oldChildren { 404 | newNodeChildren[b] = child 405 | } 406 | node.Children = newNodeChildren 407 | 408 | // Update heights 409 | s.updateHeights(node) 410 | s.updateHeights(rightChild) 411 | } 412 | 413 | // rotateRight rotates the node to the right 414 | func (s *SuRF) rotateRight(node *TrieNode) { 415 | // Find the leftmost child 416 | var leftmostByte byte 417 | var leftChild *TrieNode 418 | first := true 419 | for b, child := range node.Children { 420 | if first || b < leftmostByte { 421 | leftmostByte = b 422 | leftChild = child 423 | first = false 424 | } 425 | } 426 | 427 | if leftChild == nil { 428 | return // Cannot rotate if no left child 429 | } 430 | 431 | // Store original node's children except leftmost 432 | oldChildren := make(map[byte]*TrieNode) 433 | for b, child := range node.Children { 434 | if b != leftmostByte { 435 | oldChildren[b] = child 436 | } 437 | } 438 | 439 | // Move appropriate children from left child to node 440 | var highestByteInLeft byte 441 | first = true 442 | for b, _ := range leftChild.Children { 443 | if first { 444 | highestByteInLeft = b 445 | first = false 446 | continue 447 | } 448 | if b > highestByteInLeft { 449 | highestByteInLeft = b 450 | } 451 | } 452 | 453 | // Redistribute children 454 | newNodeChildren := make(map[byte]*TrieNode) 455 | for b, child := range leftChild.Children { 456 | if b > highestByteInLeft { 457 | newNodeChildren[b] = child 458 | delete(leftChild.Children, b) 459 | } 460 | } 461 | 462 | // Update the children maps 463 | for b, child := range oldChildren { 464 | newNodeChildren[b] = child 465 | } 466 | node.Children = newNodeChildren 467 | 468 | // Handle leaf status and suffix/hash transfer if needed 469 | if node.IsLeaf { 470 | leftChild.IsLeaf = true 471 | leftChild.Suffix = node.Suffix 472 | leftChild.Hash = node.Hash 473 | node.IsLeaf = false 474 | } 475 | 476 | // Update heights 477 | s.updateHeights(node) 478 | s.updateHeights(leftChild) 479 | } 480 | 481 | // Helper function to get the leftmost and rightmost bytes of a node's children 482 | func (s *SuRF) getExtremeBytes(node *TrieNode) (byte, byte) { 483 | var lowest, highest byte 484 | first := true 485 | 486 | for b := range node.Children { 487 | if first { 488 | lowest = b 489 | highest = b 490 | first = false 491 | continue 492 | } 493 | if b < lowest { 494 | lowest = b 495 | } 496 | if b > highest { 497 | highest = b 498 | } 499 | } 500 | 501 | return lowest, highest 502 | } 503 | 504 | // Helper function to get the height of a child node 505 | func (s *SuRF) getChildHeight(node *TrieNode, b byte) int { 506 | if child, exists := node.Children[b]; exists { 507 | return child.Height 508 | } 509 | return 0 510 | } 511 | 512 | // Serialize converts the SuRF to a byte slice 513 | func (s *SuRF) Serialize() ([]byte, error) { 514 | var buf bytes.Buffer 515 | enc := gob.NewEncoder(&buf) 516 | if err := enc.Encode(s); err != nil { 517 | return nil, err 518 | } 519 | return buf.Bytes(), nil 520 | } 521 | 522 | // Deserialize reconstructs a SuRF from a byte slice 523 | func Deserialize(data []byte) (*SuRF, error) { 524 | var s SuRF 525 | buf := bytes.NewReader(data) 526 | dec := gob.NewDecoder(buf) 527 | if err := dec.Decode(&s); err != nil { 528 | return nil, err 529 | } 530 | return &s, nil 531 | } 532 | 533 | // Contains checks if a key exists in the SuRF 534 | func (s *SuRF) Contains(key []byte) bool { 535 | node := s.Trie 536 | path := make([]byte, 0, len(key)) 537 | 538 | // Traverse the trie following the key path 539 | for _, b := range key { 540 | path = append(path, b) 541 | child, exists := node.Children[b] 542 | if !exists { 543 | return false 544 | } 545 | node = child 546 | } 547 | 548 | // We've reached the end of the key path 549 | if !node.IsLeaf { 550 | return false 551 | } 552 | 553 | // Verify both suffix and hash match for better filtering 554 | expectedSuffix := MixedSuffixBits(key, s.SuffixBits) 555 | if node.Suffix != expectedSuffix { 556 | return false 557 | } 558 | 559 | expectedHash := ComputeNodeHash(key, path, s.HashBits) 560 | return node.Hash == expectedHash 561 | } 562 | 563 | // LongestPrefixSearch returns the longest matching prefix of the given key 564 | // along with its length. If no prefix is found, returns nil and 0. 565 | func (s *SuRF) LongestPrefixSearch(key []byte) ([]byte, int) { 566 | if len(key) == 0 { 567 | return nil, 0 568 | } 569 | 570 | node := s.Trie 571 | path := make([]byte, 0, len(key)) 572 | lastMatchPos := -1 573 | 574 | // Track last valid match for prefix 575 | var lastMatch []byte 576 | 577 | // Traverse the trie following the key path 578 | for i, b := range key { 579 | // If we can't go further, break 580 | child, exists := node.Children[b] 581 | if !exists { 582 | break 583 | } 584 | 585 | path = append(path, b) 586 | node = child 587 | 588 | // If this is a leaf node, verify suffix and hash 589 | if node.IsLeaf { 590 | expectedSuffix := MixedSuffixBits(path, s.SuffixBits) 591 | expectedHash := ComputeNodeHash(path, path, s.HashBits) 592 | 593 | // Only update match if both suffix and hash match 594 | if node.Suffix == expectedSuffix && node.Hash == expectedHash { 595 | lastMatchPos = i 596 | lastMatch = make([]byte, len(path)) 597 | copy(lastMatch, path) 598 | } 599 | } 600 | } 601 | 602 | if lastMatchPos == -1 { 603 | return nil, 0 604 | } 605 | 606 | return lastMatch, len(lastMatch) 607 | } 608 | 609 | // PrefixExists checks if a given prefix exists in the trie 610 | func (s *SuRF) PrefixExists(prefix []byte) bool { 611 | if len(prefix) == 0 { 612 | return false 613 | } 614 | 615 | node := s.Trie 616 | for _, b := range prefix { 617 | child, exists := node.Children[b] 618 | if !exists { 619 | return false 620 | } 621 | node = child 622 | } 623 | 624 | // If the node exists, the prefix exists. 625 | return true 626 | } 627 | -------------------------------------------------------------------------------- /surf/surf_test.go: -------------------------------------------------------------------------------- 1 | // Package ttree 2 | // 3 | // (C) Copyright Starskey 4 | // 5 | // Original Author: Alex Gaetano Padula 6 | // 7 | // Licensed under the Mozilla Public License, v. 2.0 (the "License"); 8 | // you may not use this file except in compliance with the License. 9 | // You may obtain a copy of the License at 10 | // 11 | // https://www.mozilla.org/en-US/MPL/2.0/ 12 | // 13 | // Unless required by applicable law or agreed to in writing, software 14 | // distributed under the License is distributed on an "AS IS" BASIS, 15 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | // See the License for the specific language governing permissions and 17 | // limitations under the License. 18 | package surf 19 | 20 | import ( 21 | "bytes" 22 | "fmt" 23 | "math/rand" 24 | "testing" 25 | "time" 26 | ) 27 | 28 | // TestSuRF tests the SuRF implementation with 1 million random keys and range checks 29 | func TestSuRF(t *testing.T) { 30 | rand.Seed(time.Now().UnixNano()) 31 | surf := New(8) 32 | keys := make([][]byte, 1000000) 33 | 34 | // Insert 1 million random keys 35 | for i := 0; i < 1000000; i++ { 36 | key := []byte(fmt.Sprintf("%08d", rand.Intn(99999999))) 37 | keys[i] = key 38 | surf.Add(key) 39 | } 40 | 41 | // Perform range checks 42 | checks := []struct { 43 | lower, upper []byte 44 | }{ 45 | {[]byte("00001000"), []byte("00002000")}, 46 | {[]byte("50000000"), []byte("60000000")}, 47 | {[]byte("90000000"), []byte("99999999")}, 48 | {[]byte("12345678"), []byte("23456789")}, 49 | {[]byte("70000000"), []byte("80000000")}, 50 | } 51 | 52 | for _, check := range checks { 53 | exists := surf.CheckRange(check.lower, check.upper) 54 | t.Logf("CheckRange(%s, %s) = %v", check.lower, check.upper, exists) 55 | } 56 | } 57 | 58 | // Generates a sequentially increasing set of keys 59 | func generateSequentialKeys(n int) [][]byte { 60 | keys := make([][]byte, n) 61 | for i := 0; i < n; i++ { 62 | keys[i] = []byte(fmt.Sprintf("%08d", i)) 63 | } 64 | return keys 65 | } 66 | 67 | // Generates keys with Zipfian distribution 68 | func generateZipfianKeys(n int) [][]byte { 69 | rand.Seed(time.Now().UnixNano()) 70 | zipf := rand.NewZipf(rand.New(rand.NewSource(time.Now().UnixNano())), 1.1, 2, uint64(n-1)) 71 | keys := make([][]byte, n) 72 | for i := 0; i < n; i++ { 73 | keys[i] = []byte(fmt.Sprintf("%08d", zipf.Uint64())) 74 | } 75 | return keys 76 | } 77 | 78 | func TestSuRF_2(t *testing.T) { 79 | rand.Seed(time.Now().UnixNano()) 80 | 81 | // Initialize SuRF with an estimated number of keys 82 | surf := New(1000000) 83 | 84 | // Insert sequential keys 85 | seqKeys := generateSequentialKeys(100000) 86 | for _, key := range seqKeys { 87 | surf.Add(key) 88 | } 89 | 90 | // Insert Zipfian distributed keys 91 | zipfKeys := generateZipfianKeys(100000) 92 | for _, key := range zipfKeys { 93 | surf.Add(key) 94 | } 95 | 96 | // Perform range queries (Realistic sparse and dense ranges) 97 | rangeChecks := []struct { 98 | lower, upper []byte 99 | }{ 100 | {[]byte("00005000"), []byte("00010000")}, // Small range (sparse) 101 | {[]byte("01000000"), []byte("09000000")}, // Large range (dense) 102 | {[]byte("09990000"), []byte("10000000")}, // Edge case boundary 103 | {[]byte("50000000"), []byte("60000000")}, // Mid-range values 104 | {[]byte("90000000"), []byte("99999999")}, // Upper range 105 | } 106 | 107 | for _, check := range rangeChecks { 108 | exists := surf.CheckRange(check.lower, check.upper) 109 | t.Logf("CheckRange(%s, %s) = %v", check.lower, check.upper, exists) 110 | } 111 | 112 | // Delete some random keys and test their absence 113 | deleteKeys := [][]byte{ 114 | []byte("00005000"), 115 | []byte("01000000"), 116 | []byte("09990000"), 117 | } 118 | 119 | for _, key := range deleteKeys { 120 | surf.Delete(key) 121 | if surf.CheckRange(key, key) { 122 | t.Errorf("Failed to delete key %s", key) 123 | } else { 124 | t.Logf("Successfully deleted key %s", key) 125 | } 126 | } 127 | 128 | // Introduce false positive tests (ensure we have some) 129 | falsePositiveKeys := [][]byte{ 130 | []byte("88888888"), // Unlikely inserted 131 | []byte("77777777"), 132 | []byte("66666666"), 133 | } 134 | 135 | for _, key := range falsePositiveKeys { 136 | exists := surf.CheckRange(key, key) 137 | t.Logf("False positive check for key %s = %v", key, exists) 138 | } 139 | } 140 | 141 | // Generates keys with specific prefixes 142 | func generatePrefixedKeys(prefix string, n int) [][]byte { 143 | keys := make([][]byte, n) 144 | for i := 0; i < n; i++ { 145 | keys[i] = []byte(fmt.Sprintf("%s%05d", prefix, i)) 146 | } 147 | return keys 148 | } 149 | 150 | func TestSuRF_3(t *testing.T) { 151 | rand.Seed(time.Now().UnixNano()) 152 | 153 | // Initialize SuRF with an estimated number of keys 154 | surf := New(1000000) 155 | 156 | // Define key categories 157 | keyPrefixes := []string{"usr", "prod", "txn", "log", "sys"} 158 | 159 | // Insert keys with specific prefixes 160 | for _, prefix := range keyPrefixes { 161 | prefixedKeys := generatePrefixedKeys(prefix, 100000) 162 | for _, key := range prefixedKeys { 163 | surf.Add(key) 164 | } 165 | } 166 | 167 | // Perform range queries for each prefix 168 | for _, prefix := range keyPrefixes { 169 | lower := []byte(fmt.Sprintf("%s%05d", prefix, 0)) 170 | upper := []byte(fmt.Sprintf("%s%05d", prefix, 99999)) 171 | exists := surf.CheckRange(lower, upper) 172 | t.Logf("CheckRange(%s, %s) = %v", lower, upper, exists) 173 | } 174 | 175 | // Perform additional range queries with mixed prefixes 176 | mixedRangeChecks := []struct { 177 | lower, upper []byte 178 | }{ 179 | {[]byte("usr00000"), []byte("prod00000")}, 180 | {[]byte("txn00000"), []byte("log00000")}, 181 | {[]byte("log00000"), []byte("sys00000")}, 182 | } 183 | for _, check := range mixedRangeChecks { 184 | exists := surf.CheckRange(check.lower, check.upper) 185 | t.Logf("CheckRange(%s, %s) = %v", check.lower, check.upper, exists) 186 | } 187 | 188 | // Delete some keys with specific prefixes and test their absence 189 | deleteKeys := [][]byte{ 190 | []byte("usr00000"), 191 | []byte("prod00000"), 192 | []byte("txn00000"), 193 | []byte("log00000"), 194 | []byte("sys00000"), 195 | } 196 | 197 | for _, key := range deleteKeys { 198 | surf.Delete(key) 199 | if surf.CheckRange(key, key) { 200 | t.Errorf("Failed to delete key %s", key) 201 | } else { 202 | t.Logf("Successfully deleted key %s", key) 203 | } 204 | } 205 | 206 | // Test false positives with keys that were not inserted 207 | falsePositiveKeys := [][]byte{ 208 | []byte("usr99999"), 209 | []byte("prod99999"), 210 | []byte("txn99999"), 211 | []byte("log99999"), 212 | []byte("sys99999"), 213 | } 214 | 215 | for _, key := range falsePositiveKeys { 216 | exists := surf.CheckRange(key, key) 217 | t.Logf("False positive check for key %s = %v", key, exists) 218 | } 219 | 220 | // Test edge cases with empty and very large ranges 221 | edgeCaseChecks := []struct { 222 | lower, upper []byte 223 | }{ 224 | {[]byte(""), []byte("usr00000")}, 225 | {[]byte("usr00000"), []byte("")}, 226 | {[]byte(""), []byte("")}, 227 | {[]byte("usr00000"), []byte("usr99999")}, 228 | {[]byte("usr00000"), []byte("usr00001")}, 229 | } 230 | 231 | for _, check := range edgeCaseChecks { 232 | exists := surf.CheckRange(check.lower, check.upper) 233 | t.Logf("Edge case CheckRange(%s, %s) = %v", check.lower, check.upper, exists) 234 | } 235 | } 236 | 237 | func TestSuRF_Contains(t *testing.T) { 238 | rand.Seed(time.Now().UnixNano()) 239 | 240 | // Initialize SuRF with an estimated number of keys 241 | surf := New(1000000) 242 | 243 | // Test 1 Basic sequential keys 244 | t.Run("Sequential Keys", func(t *testing.T) { 245 | seqKeys := generateSequentialKeys(1000) 246 | for _, key := range seqKeys { 247 | surf.Add(key) 248 | } 249 | 250 | // Verify all keys were added 251 | for _, key := range seqKeys { 252 | if !surf.Contains(key) { 253 | t.Errorf("Key %s should exist but Contains returned false", key) 254 | } 255 | } 256 | 257 | // Test non-existent keys 258 | nonExistentKey := []byte("99999999") 259 | if surf.Contains(nonExistentKey) { 260 | t.Errorf("Key %s should not exist but Contains returned true", nonExistentKey) 261 | } 262 | }) 263 | 264 | // Test 2 Zipfian distributed keys 265 | t.Run("Zipfian Keys", func(t *testing.T) { 266 | zipfKeys := generateZipfianKeys(1000) 267 | for _, key := range zipfKeys { 268 | surf.Add(key) 269 | } 270 | 271 | // Verify all Zipfian keys were added 272 | for _, key := range zipfKeys { 273 | if !surf.Contains(key) { 274 | t.Errorf("Zipfian key %s should exist but Contains returned false", key) 275 | } 276 | } 277 | }) 278 | 279 | // Test 3 Prefixed keys 280 | t.Run("Prefixed Keys", func(t *testing.T) { 281 | prefixes := []string{"usr", "prod", "txn", "log", "sys"} 282 | addedKeys := make([][]byte, 0) 283 | 284 | for _, prefix := range prefixes { 285 | prefixedKeys := generatePrefixedKeys(prefix, 100) 286 | for _, key := range prefixedKeys { 287 | surf.Add(key) 288 | addedKeys = append(addedKeys, key) 289 | } 290 | } 291 | 292 | // Verify all prefixed keys were added 293 | for _, key := range addedKeys { 294 | if !surf.Contains(key) { 295 | t.Errorf("Prefixed key %s should exist but Contains returned false", key) 296 | } 297 | } 298 | 299 | // Test non-existent prefixed keys 300 | nonExistentPrefixedKeys := [][]byte{ 301 | []byte("usr99999"), 302 | []byte("prod99999"), 303 | []byte("txn99999"), 304 | []byte("log99999"), 305 | []byte("sys99999"), 306 | } 307 | 308 | for _, key := range nonExistentPrefixedKeys { 309 | if surf.Contains(key) { 310 | t.Errorf("Non-existent key %s should not exist but Contains returned true", key) 311 | } 312 | } 313 | }) 314 | 315 | // Test 4 Edge cases 316 | t.Run("Edge Cases", func(t *testing.T) { 317 | edgeCases := []struct { 318 | key []byte 319 | exists bool 320 | }{ 321 | {[]byte(""), false}, // Empty key 322 | {[]byte("a"), false}, // Single character 323 | {[]byte("usr"), false}, // Prefix only 324 | {[]byte("usr00000"), true}, // Already added key 325 | {[]byte("nonexistent"), false}, // Non-existent key 326 | {[]byte("usr000001234567890"), false}, // Very long key 327 | } 328 | 329 | for _, tc := range edgeCases { 330 | if got := surf.Contains(tc.key); got != tc.exists { 331 | t.Errorf("Contains(%s) = %v, want %v", tc.key, got, tc.exists) 332 | } 333 | } 334 | }) 335 | 336 | // Test 5 Deletion verification 337 | t.Run("Deletion Verification", func(t *testing.T) { 338 | key := []byte("deletetest") 339 | surf.Add(key) 340 | 341 | if !surf.Contains(key) { 342 | t.Errorf("Key %s should exist before deletion", key) 343 | } 344 | 345 | surf.Delete(key) 346 | 347 | if surf.Contains(key) { 348 | t.Errorf("Key %s should not exist after deletion", key) 349 | } 350 | }) 351 | 352 | // Test 6 Large number of random keys 353 | t.Run("Random Keys", func(t *testing.T) { 354 | // Add 10000 random keys 355 | randomKeys := make([][]byte, 10000) 356 | for i := 0; i < 10000; i++ { 357 | key := []byte(fmt.Sprintf("%08d", rand.Intn(99999999))) 358 | randomKeys[i] = key 359 | surf.Add(key) 360 | } 361 | 362 | // Verify random sampling of added keys 363 | for i := 0; i < 1000; i++ { 364 | idx := rand.Intn(len(randomKeys)) 365 | if !surf.Contains(randomKeys[idx]) { 366 | t.Errorf("Random key %s should exist but Contains returned false", randomKeys[idx]) 367 | } 368 | } 369 | }) 370 | } 371 | 372 | func TestSuRF_PrefixSearch(t *testing.T) { 373 | rand.Seed(time.Now().UnixNano()) 374 | 375 | // Initialize SuRF with an estimated number of keys 376 | surf := New(1000000) 377 | 378 | // We test basic prefix matching 379 | t.Run("Basic Prefix Matching", func(t *testing.T) { 380 | // Add some test keys 381 | testKeys := [][]byte{ 382 | []byte("user123"), 383 | []byte("user456"), 384 | []byte("user789"), 385 | []byte("product123"), 386 | []byte("product456"), 387 | } 388 | 389 | for _, key := range testKeys { 390 | surf.Add(key) 391 | } 392 | 393 | // Test cases 394 | tests := []struct { 395 | search []byte 396 | wantPrefix []byte 397 | wantLength int 398 | }{ 399 | {[]byte("user123456"), []byte("user123"), 7}, 400 | {[]byte("user"), nil, 0}, // No exact prefix match 401 | {[]byte("product123789"), []byte("product123"), 10}, 402 | {[]byte("nonexistent"), nil, 0}, 403 | {[]byte("use"), nil, 0}, 404 | } 405 | 406 | for _, tc := range tests { 407 | gotPrefix, gotLength := surf.LongestPrefixSearch(tc.search) 408 | if !bytes.Equal(gotPrefix, tc.wantPrefix) || gotLength != tc.wantLength { 409 | t.Errorf("LongestPrefixSearch(%s) = (%s, %d), want (%s, %d)", 410 | tc.search, gotPrefix, gotLength, tc.wantPrefix, tc.wantLength) 411 | } 412 | } 413 | }) 414 | 415 | // We test hierarchical prefixes 416 | t.Run("Hierarchical Prefixes", func(t *testing.T) { 417 | // Clear previous keys 418 | surf = New(1000000) 419 | 420 | // Add hierarchical keys 421 | hierarchicalKeys := [][]byte{ 422 | []byte("/usr/local/bin"), 423 | []byte("/usr/local/lib"), 424 | []byte("/usr/share/doc"), 425 | []byte("/etc/config"), 426 | } 427 | 428 | for _, key := range hierarchicalKeys { 429 | surf.Add(key) 430 | } 431 | 432 | tests := []struct { 433 | search []byte 434 | wantPrefix []byte 435 | wantLength int 436 | }{ 437 | {[]byte("/usr/local/bin/program"), []byte("/usr/local/bin"), 14}, 438 | {[]byte("/usr/local/unknown"), nil, 0}, // No exact match exists 439 | {[]byte("/usr/share/doc/readme"), []byte("/usr/share/doc"), 14}, 440 | {[]byte("/etc/config/settings"), []byte("/etc/config"), 11}, 441 | {[]byte("/var/log"), nil, 0}, 442 | } 443 | 444 | for _, tc := range tests { 445 | gotPrefix, gotLength := surf.LongestPrefixSearch(tc.search) 446 | if !bytes.Equal(gotPrefix, tc.wantPrefix) || gotLength != tc.wantLength { 447 | t.Errorf("LongestPrefixSearch(%s) = (%s, %d), want (%s, %d)", 448 | tc.search, gotPrefix, gotLength, tc.wantPrefix, tc.wantLength) 449 | } 450 | } 451 | }) 452 | 453 | // We test some edge cases 454 | t.Run("Edge Cases", func(t *testing.T) { 455 | // Clear previous keys 456 | surf = New(1000000) 457 | 458 | edgeCaseKeys := [][]byte{ 459 | []byte(""), 460 | []byte("a"), 461 | []byte("ab"), 462 | []byte("abc"), 463 | []byte("abcd"), 464 | } 465 | 466 | for _, key := range edgeCaseKeys { 467 | surf.Add(key) 468 | } 469 | 470 | tests := []struct { 471 | search []byte 472 | wantPrefix []byte 473 | wantLength int 474 | }{ 475 | {[]byte(""), nil, 0}, 476 | {[]byte("abcdef"), []byte("abcd"), 4}, 477 | {[]byte("a"), []byte("a"), 1}, 478 | {[]byte("ab"), []byte("ab"), 2}, 479 | {[]byte("abc"), []byte("abc"), 3}, 480 | } 481 | 482 | for _, tc := range tests { 483 | gotPrefix, gotLength := surf.LongestPrefixSearch(tc.search) 484 | if !bytes.Equal(gotPrefix, tc.wantPrefix) || gotLength != tc.wantLength { 485 | t.Errorf("LongestPrefixSearch(%s) = (%s, %d), want (%s, %d)", 486 | tc.search, gotPrefix, gotLength, tc.wantPrefix, tc.wantLength) 487 | } 488 | } 489 | }) 490 | 491 | // We test random prefixes with common beginnings 492 | t.Run("Random Prefixes", func(t *testing.T) { 493 | // Clear previous keys 494 | surf = New(1000000) 495 | 496 | // Generate keys with common prefixes 497 | prefixes := []string{"com", "org", "net", "edu"} 498 | var randomKeys [][]byte 499 | 500 | for _, prefix := range prefixes { 501 | for i := 0; i < 100; i++ { 502 | key := []byte(fmt.Sprintf("%s.domain%d.example", prefix, i)) 503 | randomKeys = append(randomKeys, key) 504 | surf.Add(key) 505 | } 506 | } 507 | 508 | // Test random searches 509 | for i := 0; i < 100; i++ { 510 | // Pick a random key and extend it 511 | originalKey := randomKeys[rand.Intn(len(randomKeys))] 512 | searchKey := append(originalKey, []byte(fmt.Sprintf(".extra%d", i))...) 513 | 514 | prefix, _ := surf.LongestPrefixSearch(searchKey) 515 | if !bytes.Equal(prefix, originalKey) { 516 | t.Errorf("LongestPrefixSearch(%s) failed: got prefix %s, want %s", 517 | searchKey, prefix, originalKey) 518 | } 519 | } 520 | }) 521 | 522 | // We test prefixExists function 523 | t.Run("PrefixExists", func(t *testing.T) { 524 | // Clear previous keys 525 | surf = New(1000000) 526 | 527 | // Add test keys 528 | testKeys := [][]byte{ 529 | []byte("http://example.com"), 530 | []byte("https://example.com"), 531 | []byte("ftp://example.com"), 532 | } 533 | 534 | for _, key := range testKeys { 535 | surf.Add(key) 536 | } 537 | 538 | tests := []struct { 539 | prefix []byte 540 | want bool 541 | }{ 542 | {[]byte("http"), true}, 543 | {[]byte("http://"), true}, 544 | {[]byte("http://example.co"), true}, 545 | {[]byte("https://example.co"), true}, 546 | {[]byte("nonexistent"), false}, 547 | } 548 | 549 | for _, tc := range tests { 550 | if got := surf.PrefixExists(tc.prefix); got != tc.want { 551 | t.Errorf("PrefixExists(%s) = %v, want %v", tc.prefix, got, tc.want) 552 | } 553 | } 554 | }) 555 | } 556 | 557 | func BenchmarkAdd(b *testing.B) { 558 | sf := New(b.N) // Initialize with benchmark size 559 | data := []byte("testdata") 560 | 561 | b.ResetTimer() 562 | for i := 0; i < b.N; i++ { 563 | sf.Add(data) 564 | } 565 | } 566 | 567 | func BenchmarkContains(b *testing.B) { 568 | sf := New(1000) 569 | data := []byte("testdata") 570 | sf.Add(data) 571 | 572 | b.ResetTimer() 573 | for i := 0; i < b.N; i++ { 574 | sf.Contains(data) 575 | } 576 | } 577 | -------------------------------------------------------------------------------- /ttree/ttree.go: -------------------------------------------------------------------------------- 1 | // Package ttree 2 | // 3 | // (C) Copyright Starskey 4 | // 5 | // Original Author: Alex Gaetano Padula 6 | // 7 | // Licensed under the Mozilla Public License, v. 2.0 (the "License"); 8 | // you may not use this file except in compliance with the License. 9 | // You may obtain a copy of the License at 10 | // 11 | // https://www.mozilla.org/en-US/MPL/2.0/ 12 | // 13 | // Unless required by applicable law or agreed to in writing, software 14 | // distributed under the License is distributed on an "AS IS" BASIS, 15 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | // See the License for the specific language governing permissions and 17 | // limitations under the License. 18 | package ttree 19 | 20 | import ( 21 | "bytes" 22 | "fmt" 23 | ) 24 | 25 | // Entry represents a key-value pair in the T-Tree. 26 | type Entry struct { 27 | Key []byte // Key 28 | Value []byte // Value 29 | } 30 | 31 | // Node represents a node in the T-Tree. 32 | type Node struct { 33 | parent *Node // Parent node 34 | left *Node // Left child node 35 | right *Node // Right child node 36 | data []*Entry // Data stored in the node 37 | height int // Height of the node 38 | minItems int // Minimum number of items in the node 39 | maxItems int // Maximum number of items in the node 40 | } 41 | 42 | // TTree represents a T-Tree. 43 | type TTree struct { 44 | root *Node // Root node 45 | minItems int // Minimum number of items in a node 46 | maxItems int // Maximum number of items in a node 47 | SizeOfTree uint64 // Size of the tree 48 | } 49 | 50 | // Iterator represents an iterator for the T-Tree. 51 | type Iterator struct { 52 | tree *TTree // T-Tree 53 | node *Node // Current node 54 | position int // Current position in the node 55 | reverse bool // Reverse iteration 56 | } 57 | 58 | // New creates a new T-Tree with the specified minimum and maximum number of items in a node. 59 | func New(minItems, maxItems int) *TTree { 60 | return &TTree{ 61 | minItems: minItems, 62 | maxItems: maxItems, 63 | } 64 | } 65 | 66 | // NewIterator creates a new iterator for the T-Tree. 67 | func (t *TTree) NewIterator(reverse bool) *Iterator { 68 | if t.root == nil { 69 | return &Iterator{tree: t} 70 | } 71 | 72 | var node *Node 73 | if reverse { 74 | node = t.findRightmostNode(t.root) 75 | } else { 76 | node = t.findLeftmostNode(t.root) 77 | } 78 | 79 | position := 0 80 | if reverse && node != nil { 81 | position = len(node.data) - 1 82 | } 83 | 84 | return &Iterator{ 85 | tree: t, 86 | node: node, 87 | position: position, 88 | reverse: reverse, 89 | } 90 | } 91 | 92 | // Put inserts a key-value pair into the T-Tree. 93 | func (t *TTree) Put(key, value []byte) error { 94 | entry := &Entry{ 95 | Key: key, 96 | Value: value, 97 | } 98 | 99 | if t.root == nil { 100 | t.root = &Node{ 101 | data: make([]*Entry, 0, t.maxItems), 102 | minItems: t.minItems, 103 | maxItems: t.maxItems, 104 | height: 1, 105 | } 106 | t.root.data = append(t.root.data, entry) 107 | t.SizeOfTree += uint64(len(entry.Key) + len(entry.Value)) 108 | return nil 109 | } 110 | 111 | node := t.findBoundingNode(entry.Key) 112 | if node != nil { 113 | if err := node.putEntry(entry, t); err != nil { 114 | return err 115 | } 116 | return nil 117 | } 118 | 119 | parent := t.findPutParent(entry.Key) 120 | newNode := &Node{ 121 | data: make([]*Entry, 0, t.maxItems), 122 | parent: parent, 123 | minItems: t.minItems, 124 | maxItems: t.maxItems, 125 | height: 1, 126 | } 127 | newNode.data = append(newNode.data, entry) 128 | t.SizeOfTree += uint64(len(entry.Key) + len(entry.Value)) 129 | 130 | if bytes.Compare(entry.Key, parent.data[0].Key) < 0 { 131 | parent.left = newNode 132 | } else { 133 | parent.right = newNode 134 | } 135 | 136 | t.updateHeightsAndBalance(parent) 137 | 138 | return nil 139 | } 140 | 141 | // findBoundingNode finds the node that bounds the specified key. 142 | func (t *TTree) findBoundingNode(key []byte) *Node { 143 | current := t.root 144 | for current != nil { 145 | if len(current.data) > 0 { 146 | if bytes.Compare(key, current.data[0].Key) >= 0 && bytes.Compare(key, current.data[len(current.data)-1].Key) <= 0 { 147 | return current 148 | } 149 | if bytes.Compare(key, current.data[0].Key) < 0 { 150 | current = current.left 151 | } else { 152 | current = current.right 153 | } 154 | } else { 155 | return current 156 | } 157 | } 158 | return nil 159 | } 160 | 161 | // putEntry inserts an entry into the node. 162 | func (n *Node) putEntry(entry *Entry, tree *TTree) error { 163 | if len(n.data) >= n.maxItems { 164 | return fmt.Errorf("node is full") 165 | } 166 | 167 | // Binary search for put position 168 | i, j := 0, len(n.data) 169 | for i < j { 170 | h := int(uint(i+j) >> 1) 171 | cmp := bytes.Compare(n.data[h].Key, entry.Key) 172 | if cmp == 0 { 173 | // Key exists, replace the value 174 | tree.SizeOfTree -= uint64(len(n.data[h].Value)) 175 | tree.SizeOfTree += uint64(len(entry.Value)) 176 | n.data[h].Value = entry.Value 177 | return nil 178 | } 179 | if cmp < 0 { 180 | i = h + 1 181 | } else { 182 | j = h 183 | } 184 | } 185 | 186 | // Key doesn't exist, put new entry 187 | n.data = append(n.data, nil) 188 | copy(n.data[i+1:], n.data[i:]) 189 | n.data[i] = entry 190 | tree.SizeOfTree += uint64(len(entry.Key) + len(entry.Value)) 191 | return nil 192 | } 193 | 194 | // findPutParent finds the parent node for inserting a key. 195 | func (t *TTree) findPutParent(key []byte) *Node { 196 | current := t.root 197 | var parent *Node 198 | for current != nil { 199 | parent = current 200 | if bytes.Compare(key, current.data[0].Key) < 0 { 201 | current = current.left 202 | } else { 203 | current = current.right 204 | } 205 | } 206 | return parent 207 | } 208 | 209 | // updateHeightsAndBalance updates the heights and balances of the nodes in the T-Tree. 210 | func (t *TTree) updateHeightsAndBalance(node *Node) { 211 | for node != nil { 212 | leftHeight := t.getHeight(node.left) 213 | rightHeight := t.getHeight(node.right) 214 | 215 | newHeight := max(leftHeight, rightHeight) + 1 216 | if newHeight == node.height { 217 | break 218 | } 219 | node.height = newHeight 220 | 221 | balance := rightHeight - leftHeight 222 | if balance > 1 { 223 | if t.getHeight(node.right.right) >= t.getHeight(node.right.left) { 224 | t.rotateLeft(node) 225 | } else { 226 | t.rotateRight(node.right) 227 | t.rotateLeft(node) 228 | } 229 | } else if balance < -1 { 230 | if t.getHeight(node.left.left) >= t.getHeight(node.left.right) { 231 | t.rotateRight(node) 232 | } else { 233 | t.rotateLeft(node.left) 234 | t.rotateRight(node) 235 | } 236 | } 237 | 238 | node = node.parent 239 | } 240 | } 241 | 242 | // getHeight returns the height of the node. 243 | func (t *TTree) getHeight(node *Node) int { 244 | if node == nil { 245 | return 0 246 | } 247 | return node.height 248 | } 249 | 250 | // rotateLeft rotates the node to the left. 251 | func (t *TTree) rotateLeft(node *Node) { 252 | rightChild := node.right 253 | node.right = rightChild.left 254 | if rightChild.left != nil { 255 | rightChild.left.parent = node 256 | } 257 | rightChild.parent = node.parent 258 | if node.parent == nil { 259 | t.root = rightChild 260 | } else if node == node.parent.left { 261 | node.parent.left = rightChild 262 | } else { 263 | node.parent.right = rightChild 264 | } 265 | rightChild.left = node 266 | node.parent = rightChild 267 | 268 | node.height = max(t.getHeight(node.left), t.getHeight(node.right)) + 1 269 | rightChild.height = max(t.getHeight(rightChild.left), t.getHeight(rightChild.right)) + 1 270 | } 271 | 272 | // rotateRight rotates the node to the right. 273 | func (t *TTree) rotateRight(node *Node) { 274 | leftChild := node.left 275 | node.left = leftChild.right 276 | if leftChild.right != nil { 277 | leftChild.right.parent = node 278 | } 279 | leftChild.parent = node.parent 280 | if node.parent == nil { 281 | t.root = leftChild 282 | } else if node == node.parent.right { 283 | node.parent.right = leftChild 284 | } else { 285 | node.parent.left = leftChild 286 | } 287 | leftChild.right = node 288 | node.parent = leftChild 289 | 290 | node.height = max(t.getHeight(node.left), t.getHeight(node.right)) + 1 291 | leftChild.height = max(t.getHeight(leftChild.left), t.getHeight(leftChild.right)) + 1 292 | } 293 | 294 | // Current returns the current entry in the iterator. 295 | func (it *Iterator) Current() (*Entry, bool) { 296 | if !it.Valid() { 297 | return nil, false 298 | } 299 | return it.node.data[it.position], true 300 | } 301 | 302 | // Valid returns true if the iterator is valid. 303 | func (it *Iterator) Valid() bool { 304 | return it.node != nil && it.position >= 0 && it.position < len(it.node.data) 305 | } 306 | 307 | // HasNext returns true if there is a next entry in the iterator. 308 | func (it *Iterator) HasNext() bool { 309 | if !it.Valid() { 310 | return false 311 | } 312 | 313 | if it.position < len(it.node.data)-1 { 314 | return true 315 | } 316 | 317 | // Try to find successor 318 | tmpNode := it.node 319 | tmpPos := it.position 320 | 321 | hasNext := it.moveNext() 322 | 323 | // Restore original position 324 | it.node = tmpNode 325 | it.position = tmpPos 326 | 327 | return hasNext 328 | } 329 | 330 | // HasPrev returns true if there is a previous entry in the iterator. 331 | func (it *Iterator) HasPrev() bool { 332 | if !it.Valid() { 333 | return false 334 | } 335 | 336 | if it.position > 0 { 337 | return true 338 | } 339 | 340 | // Try to find predecessor 341 | tmpNode := it.node 342 | tmpPos := it.position 343 | 344 | hasPrev := it.movePrev() 345 | 346 | // Restore original position 347 | it.node = tmpNode 348 | it.position = tmpPos 349 | 350 | return hasPrev 351 | } 352 | 353 | // Next moves the iterator to the next entry. 354 | func (it *Iterator) Next() bool { 355 | if it.node == nil { 356 | return false 357 | } 358 | 359 | if it.reverse { 360 | return it.movePrev() 361 | } 362 | return it.moveNext() 363 | } 364 | 365 | // Prev moves the iterator to the previous entry. 366 | func (it *Iterator) Prev() bool { 367 | if it.node == nil { 368 | return false 369 | } 370 | 371 | if it.reverse { 372 | return it.moveNext() 373 | } 374 | return it.movePrev() 375 | } 376 | 377 | // moveNext moves the iterator to the next entry. 378 | func (it *Iterator) moveNext() bool { 379 | if it.position < len(it.node.data)-1 { 380 | it.position++ 381 | return true 382 | } 383 | 384 | nextNode := it.findSuccessor(it.node) 385 | if nextNode == nil { 386 | return false 387 | } 388 | 389 | it.node = nextNode 390 | it.position = 0 391 | return true 392 | } 393 | 394 | // movePrev moves the iterator to the previous entry. 395 | func (it *Iterator) movePrev() bool { 396 | if it.position > 0 { 397 | it.position-- 398 | return true 399 | } 400 | 401 | prevNode := it.findPredecessor(it.node) 402 | if prevNode == nil { 403 | return false 404 | } 405 | 406 | it.node = prevNode 407 | it.position = len(prevNode.data) - 1 408 | return true 409 | } 410 | 411 | // findSuccessor finds the successor node of the specified node. 412 | func (it *Iterator) findSuccessor(node *Node) *Node { 413 | if node.right != nil { 414 | return it.tree.findLeftmostNode(node.right) 415 | } 416 | 417 | current := node 418 | parent := node.parent 419 | for parent != nil && current == parent.right { 420 | current = parent 421 | parent = parent.parent 422 | } 423 | return parent 424 | } 425 | 426 | // findPredecessor finds the predecessor node of the specified node. 427 | func (it *Iterator) findPredecessor(node *Node) *Node { 428 | if node.left != nil { 429 | return it.tree.findRightmostNode(node.left) 430 | } 431 | 432 | current := node 433 | parent := node.parent 434 | for parent != nil && current == parent.left { 435 | current = parent 436 | parent = parent.parent 437 | } 438 | return parent 439 | } 440 | 441 | // findLeftmostNode finds the leftmost node starting from the specified node. 442 | func (t *TTree) findLeftmostNode(start *Node) *Node { 443 | current := start 444 | for current != nil && current.left != nil { 445 | current = current.left 446 | } 447 | return current 448 | } 449 | 450 | // findRightmostNode finds the rightmost node starting from the specified node. 451 | func (t *TTree) findRightmostNode(start *Node) *Node { 452 | current := start 453 | for current != nil && current.right != nil { 454 | current = current.right 455 | } 456 | return current 457 | } 458 | 459 | // Get retrieves the value for the specified key. 460 | func (t *TTree) Get(key []byte) (*Entry, bool) { 461 | node := t.findBoundingNode(key) 462 | if node == nil { 463 | return nil, false 464 | } 465 | 466 | // Binary search within node's data array 467 | i, j := 0, len(node.data) 468 | for i < j { 469 | h := int(uint(i+j) >> 1) 470 | cmp := bytes.Compare(node.data[h].Key, key) 471 | if cmp == 0 { 472 | return node.data[h], true 473 | } 474 | if cmp < 0 { 475 | i = h + 1 476 | } else { 477 | j = h 478 | } 479 | } 480 | return nil, false 481 | } 482 | 483 | // Range retrieves all entries within the specified range. 484 | func (t *TTree) Range(start, end []byte) []*Entry { 485 | var result []*Entry 486 | t.rangeHelper(t.root, start, end, &result) 487 | return result 488 | } 489 | 490 | // rangeHelper retrieves all entries within the specified range. 491 | func (t *TTree) rangeHelper(node *Node, start, end []byte, result *[]*Entry) { 492 | if node == nil { 493 | return 494 | } 495 | 496 | // Traverse the left subtree if the start key is less than the current node's first key 497 | if bytes.Compare(start, node.data[0].Key) < 0 { 498 | t.rangeHelper(node.left, start, end, result) 499 | } 500 | 501 | // Collect entries within the range 502 | for _, entry := range node.data { 503 | if bytes.Compare(entry.Key, start) >= 0 && bytes.Compare(entry.Key, end) <= 0 { 504 | *result = append(*result, entry) 505 | } 506 | } 507 | 508 | // Traverse the right subtree if the end key is greater than the current node's last key 509 | if bytes.Compare(end, node.data[len(node.data)-1].Key) > 0 { 510 | t.rangeHelper(node.right, start, end, result) 511 | } 512 | } 513 | 514 | // CountEntries returns the number of entries in the T-Tree. 515 | func (t *TTree) CountEntries() int { 516 | count := 0 517 | iter := t.NewIterator(false) 518 | for iter.Valid() { 519 | if _, ok := iter.Current(); ok { 520 | count++ 521 | } 522 | if !iter.HasNext() { 523 | break 524 | } 525 | iter.Next() 526 | } 527 | 528 | return count 529 | 530 | } 531 | -------------------------------------------------------------------------------- /ttree/ttree_test.go: -------------------------------------------------------------------------------- 1 | // Package ttree 2 | // 3 | // (C) Copyright Starskey 4 | // 5 | // Original Author: Alex Gaetano Padula 6 | // 7 | // Licensed under the Mozilla Public License, v. 2.0 (the "License"); 8 | // you may not use this file except in compliance with the License. 9 | // You may obtain a copy of the License at 10 | // 11 | // https://www.mozilla.org/en-US/MPL/2.0/ 12 | // 13 | // Unless required by applicable law or agreed to in writing, software 14 | // distributed under the License is distributed on an "AS IS" BASIS, 15 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | // See the License for the specific language governing permissions and 17 | // limitations under the License. 18 | package ttree 19 | 20 | import ( 21 | "fmt" 22 | "testing" 23 | ) 24 | 25 | func TestTTree_Put_Get(t *testing.T) { 26 | tree := New(12, 32) 27 | err := tree.Put([]byte("hello"), []byte("world")) 28 | if err != nil { 29 | t.Errorf("Error putting key 'hello': %v", err) 30 | } 31 | err = tree.Put([]byte("foo"), []byte("bar")) 32 | if err != nil { 33 | t.Errorf("Error putting key 'foo': %v", err) 34 | } 35 | 36 | val, ok := tree.Get([]byte("hello")) 37 | if !ok { 38 | t.Errorf("Expected to find key 'hello'") 39 | } 40 | 41 | if string(val.Value) != "world" { 42 | t.Errorf("Expected value 'world', got %s", val) 43 | } 44 | 45 | val, ok = tree.Get([]byte("foo")) 46 | if !ok { 47 | t.Errorf("Expected to find key 'foo\"'") 48 | } 49 | 50 | if string(val.Value) != "bar" { 51 | t.Errorf("Expected value 'bar', got %s", val) 52 | } 53 | 54 | err = tree.Put([]byte("foo"), []byte("chocolate")) 55 | if err != nil { 56 | t.Errorf("Error putting key 'foo': %v", err) 57 | } 58 | 59 | val, ok = tree.Get([]byte("foo")) 60 | if !ok { 61 | t.Errorf("Expected to find key 'foo\"'") 62 | } 63 | 64 | if string(val.Value) != "chocolate" { 65 | t.Errorf("Expected value 'chocolate', got %s", val) 66 | } 67 | 68 | } 69 | 70 | func TestTTree_Range(t *testing.T) { 71 | tree := New(12, 32) 72 | err := tree.Put([]byte("a"), []byte("1")) 73 | if err != nil { 74 | t.Errorf("Error putting key 'a': %v", err) 75 | } 76 | err = tree.Put([]byte("b"), []byte("2")) 77 | if err != nil { 78 | t.Errorf("Error putting key 'b': %v", err) 79 | } 80 | err = tree.Put([]byte("c"), []byte("3")) 81 | if err != nil { 82 | t.Errorf("Error putting key 'c': %v", err) 83 | } 84 | err = tree.Put([]byte("d"), []byte("4")) 85 | if err != nil { 86 | t.Errorf("Error putting key 'd': %v", err) 87 | } 88 | 89 | entries := tree.Range([]byte("b"), []byte("c")) 90 | if len(entries) != 2 { 91 | t.Errorf("Expected 2 entries, got %d", len(entries)) 92 | } 93 | 94 | if string(entries[0].Key) != "b" || string(entries[1].Key) != "c" { 95 | t.Errorf("Expected keys 'b' and 'c', got %s and %s", entries[0].Key, entries[1].Key) 96 | } 97 | } 98 | 99 | func TestTTree_Iterator(t *testing.T) { 100 | tree := New(12, 32) 101 | err := tree.Put([]byte("a"), []byte("1")) 102 | if err != nil { 103 | t.Errorf("Error putting key 'a': %v", err) 104 | } 105 | err = tree.Put([]byte("b"), []byte("2")) 106 | if err != nil { 107 | t.Errorf("Error putting key 'b': %v", err) 108 | } 109 | err = tree.Put([]byte("c"), []byte("3")) 110 | if err != nil { 111 | t.Errorf("Error putting key 'c': %v", err) 112 | } 113 | 114 | expect := make(map[string]bool) 115 | for _, key := range []string{"a", "b", "c"} { 116 | expect[key] = false 117 | 118 | } 119 | 120 | iter := tree.NewIterator(false) 121 | for iter.Valid() { 122 | if entry, ok := iter.Current(); ok { 123 | expect[string(entry.Key)] = true 124 | } 125 | if !iter.HasNext() { 126 | break 127 | } 128 | iter.Next() 129 | } 130 | 131 | for key, found := range expect { 132 | if !found { 133 | t.Errorf("Expected key %s not found", key) 134 | } 135 | } 136 | 137 | // Reset 138 | for key := range expect { 139 | expect[key] = false 140 | 141 | } 142 | 143 | for iter.Valid() { 144 | if entry, ok := iter.Current(); ok { 145 | expect[string(entry.Key)] = true 146 | } 147 | if !iter.HasPrev() { 148 | break 149 | } 150 | iter.Prev() 151 | } 152 | 153 | for key, found := range expect { 154 | if !found { 155 | t.Errorf("Expected key %s not found", key) 156 | } 157 | 158 | } 159 | } 160 | 161 | func TestCountEntries(t *testing.T) { 162 | tree := New(12, 32) 163 | tree.Put([]byte("a"), []byte("1")) 164 | tree.Put([]byte("b"), []byte("2")) 165 | tree.Put([]byte("c"), []byte("3")) 166 | 167 | count := tree.CountEntries() 168 | if count != 3 { 169 | t.Errorf("Expected 3 entries, got %d", count) 170 | } 171 | } 172 | 173 | func TestTTree_Update(t *testing.T) { 174 | tree := New(12, 32) 175 | tree.Put([]byte("a"), []byte("1")) 176 | tree.Put([]byte("b"), []byte("2")) 177 | 178 | err := tree.Put([]byte("b"), []byte("updated")) 179 | if err != nil { 180 | t.Errorf("Error updating key 'b': %v", err) 181 | } 182 | 183 | val, ok := tree.Get([]byte("b")) 184 | if !ok { 185 | t.Errorf("Expected to find key 'b'") 186 | } 187 | 188 | if string(val.Value) != "updated" { 189 | t.Errorf("Expected value 'updated', got %s", val.Value) 190 | } 191 | } 192 | 193 | func TestTTree_EmptyTree(t *testing.T) { 194 | tree := New(12, 32) 195 | 196 | _, ok := tree.Get([]byte("a")) 197 | if ok { 198 | t.Errorf("Expected not to find key 'a' in an empty tree") 199 | } 200 | 201 | count := tree.CountEntries() 202 | if count != 0 { 203 | t.Errorf("Expected 0 entries, got %d", count) 204 | } 205 | } 206 | 207 | func BenchmarkTTree_Put(b *testing.B) { 208 | tree := New(12, 32) 209 | for i := 0; i < b.N; i++ { 210 | key := []byte(fmt.Sprintf("key%d", i)) 211 | value := []byte(fmt.Sprintf("value%d", i)) 212 | tree.Put(key, value) 213 | } 214 | } 215 | 216 | func BenchmarkTTree_Get(b *testing.B) { 217 | tree := New(12, 32) 218 | for i := 0; i < 1000; i++ { 219 | key := []byte(fmt.Sprintf("key%d", i)) 220 | value := []byte(fmt.Sprintf("value%d", i)) 221 | tree.Put(key, value) 222 | } 223 | 224 | b.ResetTimer() 225 | for i := 0; i < b.N; i++ { 226 | key := []byte(fmt.Sprintf("key%d", i%1000)) 227 | tree.Get(key) 228 | } 229 | } 230 | --------------------------------------------------------------------------------