├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── rangestore.go ├── rangestore_test.go └── weighted_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | os: 3 | - linux 4 | - osx 5 | go: 6 | - 1.8 7 | - 1.9 8 | - master 9 | before_install: 10 | - go get github.com/axw/gocov/gocov 11 | - go get github.com/mattn/goveralls 12 | script: 13 | - go test -v -tags ci ./... 14 | - $GOPATH/bin/goveralls -service=travis-ci 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Go Range Store 2 | ============== 3 | 4 | [![Build Status](https://travis-ci.org/tenta-browser/go-range-store.svg?branch=master)](https://travis-ci.org/tenta-browser/go-range-store/builds) 5 | [![Coverage Status](https://coveralls.io/repos/github/tenta-browser/go-range-store/badge.svg?branch=master)](https://coveralls.io/github/tenta-browser/go-range-store?branch=master) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/tenta-browser/go-range-store)](https://goreportcard.com/report/github.com/tenta-browser/go-range-store) 7 | [![GoDoc](https://godoc.org/github.com/tenta-browser/go-range-store?status.svg)](https://godoc.org/github.com/tenta-browser/go-range-store) 8 | 9 | Range store provides a simple datastructure providing efficient storage of a single value to many (consecutive) keys. Inspired by wanting to write: 10 | 11 | ```go 12 | store := make(map[uint64]interface{}) 13 | store[10:20] = "hello" 14 | greet := store[15] 15 | // Greet is "hello" 16 | ``` 17 | 18 | Contact: developer@tenta.io 19 | 20 | Installation 21 | ============ 22 | 23 | 1. `go get github.com/tenta-browser/go-range-store` 24 | 25 | Usage 26 | ===== 27 | 28 | The range store provides a compact and efficient tree based method of storing a single value associated with a range of keys. You may 29 | either use the provided `DefaultRangedValue` to store data. In addition, a `Ranged` interface is provided allowing for storage of 30 | arbitrary types. For example, using US zip codes (postal codes), to map zip codes to a regional office: 31 | 32 | ```go 33 | items := make([]Ranged,0) 34 | items = append(items, DefaultRangedValue{0, 25505, "New York"}) 35 | items = append(items, DefaultRangedValue(25506, 67890, "Chicago"}) 36 | items = append(items, DefaultRangedValue(67891, 89000, "Phoenix"}) 37 | items = append(items, DefaultRangedValue(89001, 99999, "Los Angeles"}) 38 | 39 | n, err := NewRangeStoreFromSorted(items) 40 | // Check error 41 | 42 | city := n.RangeSearch(85716) 43 | 44 | // City is "Phoenix" 45 | ``` 46 | 47 | The range store must be constructed with a continuously inscreasing set of non-negative integers which don't overlap and contain 48 | no discontinuities. To simplify operation when using values where only weights matter, and not explicit ranges, a `Weighted` interface 49 | and `DefaultWeightedValue` are provided. For example, so select fairly among servers with different weights: 50 | 51 | ```go 52 | items := make([]Weighted, 0) 53 | items = append(items, DefaultWeightedValue{10, "a.example.com"}) 54 | items = append(items, DefaultWeightedValue(10, "b.example.com")} 55 | items = append(items, DefaultWeightedValue(20, "c.example.com")} 56 | 57 | n, err := NewRangeStoreFromWeighted(items) 58 | // Check error 59 | 60 | p := rand.Intn(40) 61 | server := n.RangeSearch(uint64(p)) 62 | 63 | // Server has a 25% chance of being a or b and a 50% chance of being c 64 | ``` 65 | 66 | Performance 67 | =========== 68 | 69 | We wrote it to be fast and we use it in production. The tests include a totally arbitrary benchmark against a very naive implementation using 70 | a raw map with an entry for every value in the range. Unsurprisingly, for large ranges, the range store massively outperforms the map. However, 71 | even for small ranges, the range store still outperforms the map, as indexing requires only fast integer operations. In addition, the range store 72 | always outperforms the map on construction: 73 | 74 | ``` 75 | Benchmark_NewNodeSorted_Small-4 10000000 219 ns/op 76 | Benchmark_NewMapSorted_Small-4 300000 5389 ns/op 77 | Benchmark_NewNodeSorted_Large-4 10000000 224 ns/op 78 | Benchmark_NewMapSorted_Large-4 5 203757320 ns/op 79 | Benchmark_RangeSearch_Node-4 20000000 63 ns/op 80 | Benchmark_RangeSearch_Map-4 10000000 186 ns/op 81 | ``` 82 | 83 | License 84 | ======= 85 | 86 | Licensed under the Apache License, Version 2.0 (the "License"); 87 | you may not use this file except in compliance with the License. 88 | You may obtain a copy of the License at 89 | 90 | http://www.apache.org/licenses/LICENSE-2.0 91 | 92 | Unless required by applicable law or agreed to in writing, software 93 | distributed under the License is distributed on an "AS IS" BASIS, 94 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 95 | See the License for the specific language governing permissions and 96 | limitations under the License. 97 | 98 | For any questions, please contact developer@tenta.io 99 | 100 | Contributing 101 | ============ 102 | 103 | We welcome contributions, feedback and plain old complaining. Feel free to open 104 | an issue or shoot us a message to developer@tenta.io. If you'd like to contribute, 105 | please open a pull request and send us an email to sign a contributor agreement. 106 | 107 | About Tenta 108 | =========== 109 | 110 | This range store library is brought to you by Team Tenta. Tenta is your [private, encrypted browser](https://tenta.com) that protects your data instead of selling. We're building a next-generation browser that combines all the privacy tools you need, including built-in OpenVPN. Everything is encrypted by default. That means your bookmarks, saved tabs, web history, web traffic, downloaded files, IP address and DNS. A truly incognito browser that's fast and easy. 111 | -------------------------------------------------------------------------------- /rangestore.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Go Range Store 3 | * 4 | * Copyright 2017 Tenta, LLC 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | * For any questions, please contact developer@tenta.io 19 | * 20 | * rangestore.go: Range Store Implementation 21 | */ 22 | 23 | package rangestore 24 | 25 | import ( 26 | "fmt" 27 | ) 28 | 29 | type Node struct { 30 | max uint64 31 | value interface{} 32 | left, right *Node 33 | } 34 | 35 | type Weighted interface { 36 | GetWeight() uint64 37 | GetValue() interface{} 38 | } 39 | type DefaultWeightedValue struct { 40 | Weight uint64 41 | Value interface{} 42 | } 43 | 44 | func (w DefaultWeightedValue) GetWeight() uint64 { 45 | return w.Weight 46 | } 47 | func (w DefaultWeightedValue) GetValue() interface{} { 48 | return w.Value 49 | } 50 | 51 | type Ranged interface { 52 | GetMin() uint64 53 | GetMax() uint64 54 | GetValue() interface{} 55 | } 56 | type DefaultRangedValue struct { 57 | min, max uint64 58 | value interface{} 59 | } 60 | 61 | func (r DefaultRangedValue) GetMin() uint64 { 62 | return r.min 63 | } 64 | func (r DefaultRangedValue) GetMax() uint64 { 65 | return r.max 66 | } 67 | func (r DefaultRangedValue) GetValue() interface{} { 68 | return r.value 69 | } 70 | 71 | type ErrUnsignedIntegerOverflow struct { 72 | a, b uint64 73 | } 74 | 75 | func (ex ErrUnsignedIntegerOverflow) Error() string { 76 | return fmt.Sprintf("Overflow adding %d + %d", ex.a, ex.b) 77 | } 78 | 79 | type ErrDiscontinuity struct { 80 | x, y uint64 81 | } 82 | 83 | func (ex ErrDiscontinuity) Error() string { 84 | return fmt.Sprintf("Discontinuity detected from %d -> %d", ex.x, ex.y) 85 | } 86 | 87 | type ErrOutOfRange struct { 88 | s uint64 89 | } 90 | 91 | func (ex ErrOutOfRange) Error() string { 92 | return fmt.Sprintf("Value %d is out of range", ex.s) 93 | } 94 | 95 | type ErrOverlap struct { 96 | a, b uint64 97 | } 98 | 99 | func (ex ErrOverlap) Error() string { 100 | return fmt.Sprintf("Overlap detected between %d -> %d", ex.a, ex.b) 101 | } 102 | 103 | type ErrEmptyInput struct{} 104 | 105 | func (ex ErrEmptyInput) Error() string { 106 | return "Input list is empty" 107 | } 108 | 109 | func NewRangeStoreFromWeighted(items []Weighted) (*Node, error) { 110 | if len(items) < 1 { 111 | return nil, ErrEmptyInput{} 112 | } 113 | totalWeight := uint64(0) 114 | ranges := make([]Ranged, 0) 115 | for _, item := range items { 116 | w := item.GetWeight() 117 | ranges = append(ranges, DefaultRangedValue{totalWeight + 1, totalWeight + w, item.GetValue()}) 118 | newSum := totalWeight + w 119 | if newSum < totalWeight || newSum < w { 120 | return nil, ErrUnsignedIntegerOverflow{totalWeight, w} 121 | } 122 | totalWeight = newSum 123 | } 124 | 125 | return NewRangeStoreFromSorted(ranges) 126 | } 127 | 128 | // Builds a optimal(ish) tree containing the range values as 129 | // node values. For the computation of optimality, we assume 130 | // that every value in the aggregate ranges is equally likely 131 | // to be looked up. We then choose pivots so that roughly equal 132 | // amounts of range are in each subtree. 133 | // 134 | // That is, with a set of values like: 135 | // * A [0,1] 136 | // * B [1,2] 137 | // * C [2,3] 138 | // The tree produced will look like 139 | // 140 | // B 141 | // [2] 142 | // ___|___ 143 | // | | 144 | // A C 145 | // [1] [3] 146 | // 147 | // However with non-equal weights, such as 148 | // * A [0,1] 149 | // * B [1,2] 150 | // * C [2,100] 151 | // The tree produced will look more like 152 | // 153 | // C 154 | // [100] 155 | // ___| 156 | // | 157 | // A 158 | // [1] 159 | // |___ 160 | // | 161 | // B 162 | // [2] 163 | // 164 | // Although this tree is degenerate based on a counting of *nodes* 165 | // it is optimal based on lookup frequency, since ~98% of lookups 166 | // will terminate at the root node and only very infrequently will 167 | // recursion down the tree occur. 168 | // 169 | // _Note_: The items passed to RangeStoreFRomSorted must contain 170 | // a monotonically increasing and continuous sequence of min and 171 | // max values 172 | // 173 | // _Note_: Construction of the tree is done using Mehlhorn's 174 | // approximation for balancing the tree and uses an effective floor 175 | // (unsigned integer division) when computing pivots. As a result, 176 | // the produced data structure approaches, but may not always be 177 | // exactly, optimal. 178 | func NewRangeStoreFromSorted(items []Ranged) (*Node, error) { 179 | return rangeStoreFromSortedChecked(items, true) 180 | } 181 | 182 | // Helper function which takes a bool whether the ranges have already been checked 183 | // Breaking this into two functions is a small optimization on big sets, but not 184 | // having to do all of the overlap and discontinuity checking on every 185 | // recursive call. We know that if we're calling recursively that we have 186 | // only part of a range that's previously been through this function, so 187 | // we can skip the checks for monotonicity. 188 | func rangeStoreFromSortedChecked(items []Ranged, check bool) (*Node, error) { 189 | if len(items) < 1 { 190 | return nil, ErrEmptyInput{} 191 | } 192 | n := &Node{} 193 | // Easy base case: We've got one item. Just set it and forget it 194 | if len(items) == 1 { 195 | n.max = items[0].GetMax() 196 | n.value = items[0].GetValue() 197 | } else { 198 | // Compute the total weight in this slice 199 | // Also, check for discontinuities 200 | start := uint64(0) 201 | total := uint64(0) 202 | for idx, item := range items { 203 | if idx == 0 { 204 | start = item.GetMin() 205 | } else if check { 206 | // Check for discontinuity 207 | prev := items[idx-1].GetMax() 208 | curr := item.GetMin() 209 | if curr > prev+1 { 210 | return nil, ErrDiscontinuity{prev, curr} 211 | } 212 | // Check for overlap 213 | if curr < prev+1 { 214 | return nil, ErrOverlap{prev, curr} 215 | } 216 | } 217 | a := (item.GetMax() - item.GetMin()) + 1 218 | newSum := total + a 219 | if newSum < total || newSum < a { 220 | return nil, ErrUnsignedIntegerOverflow{total, a} 221 | } 222 | total = newSum 223 | } 224 | 225 | // Compute the pivot 226 | pivot := total / 2 227 | 228 | // Walk the list backwards and find the index of the item which has 229 | // a min less than the pivot 230 | var ridx int 231 | for ridx = len(items) - 1; ridx >= 0; ridx -= 1 { 232 | if items[ridx].GetMin() < pivot+start { 233 | break 234 | } 235 | } 236 | 237 | // Fill the node based on the current item 238 | n.max = items[ridx].GetMax() 239 | n.value = items[ridx].GetValue() 240 | 241 | // If we didn't pick the first item for the pivot, build the left subtree 242 | if ridx != 0 { 243 | // Explicitly ignore the error, since we've indicated we've already checked 244 | lft, _ := rangeStoreFromSortedChecked(items[:ridx], false) 245 | n.left = lft 246 | } 247 | // If we didn't pick the last item for the pivot, build the right subtree 248 | if ridx != len(items)-1 { 249 | // Explicitly ignore the error, since we've indicated we've already checked 250 | rht, _ := rangeStoreFromSortedChecked(items[ridx+1:], false) 251 | n.right = rht 252 | } 253 | } 254 | return n, nil 255 | } 256 | 257 | // Searches for the range which contains the specified key 258 | // and returns the associated value, or an error if the 259 | // value is out of range 260 | func (n *Node) RangeSearch(val uint64) (interface{}, error) { 261 | if n.max < val { 262 | if n.right == nil { 263 | return nil, ErrOutOfRange{val} 264 | } 265 | return n.right.RangeSearch(val) 266 | } else { 267 | if n.left != nil { 268 | val, err := n.left.RangeSearch(val) 269 | if err == nil { 270 | return val, nil 271 | } 272 | } 273 | return n.value, nil 274 | } 275 | } 276 | 277 | // Creates a nicely formatter string representation of the Range Store. Useful for understanding how the data is 278 | // internally stored and represented. 279 | func (n *Node) String() string { 280 | return n.formattedString("") 281 | } 282 | func (n *Node) formattedString(prefix string) string { 283 | ret := fmt.Sprintf("%s-%s [max: %d]\n", prefix, n.value, n.max) 284 | if n.left != nil { 285 | ret += n.left.formattedString(prefix + " |") 286 | } 287 | if n.right != nil { 288 | ret += n.right.formattedString(prefix + " !") 289 | } 290 | return ret 291 | } 292 | -------------------------------------------------------------------------------- /rangestore_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Go Range Store 3 | * 4 | * Copyright 2017 Tenta, LLC 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | * For any questions, please contact developer@tenta.io 19 | * 20 | * rangestore_test.go: Tests on the core range store 21 | */ 22 | 23 | package rangestore 24 | 25 | import ( 26 | "math/rand" 27 | "reflect" 28 | "testing" 29 | ) 30 | 31 | func TestRangeStoreFromSorted_Basic(t *testing.T) { 32 | items := make([]Ranged, 0) 33 | 34 | items = append(items, DefaultRangedValue{0, 9, "A"}) 35 | items = append(items, DefaultRangedValue{10, 19, "B"}) 36 | items = append(items, DefaultRangedValue{20, 29, "C"}) 37 | 38 | n, err := NewRangeStoreFromSorted(items) 39 | 40 | if err != nil { 41 | t.Fatalf("Error while constructing range store: %s", err.Error()) 42 | } 43 | 44 | // -B [max: 19] 45 | // |-A [max: 9] 46 | // !-C [max: 29] 47 | if n.value != "B" { 48 | t.Fatalf("Expected B at the root") 49 | } 50 | if n.max != 19 { 51 | t.Fatalf("Expected 19 max at the root") 52 | } 53 | if n.left.value != "A" { 54 | t.Fatalf("Expected A as the right child") 55 | } 56 | if n.left.max != 9 { 57 | t.Fatalf("Expected 9 max as the right child") 58 | } 59 | if n.right.value != "C" { 60 | t.Fatalf("Expected C as the right child") 61 | } 62 | if n.right.max != 29 { 63 | t.Fatalf("Expected 29 max as the right child") 64 | } 65 | 66 | if n.left.left != nil { 67 | t.Fatalf("Expected children to have null leaves") 68 | } 69 | if n.left.right != nil { 70 | t.Fatalf("Expected children to have null leaves") 71 | } 72 | if n.right.left != nil { 73 | t.Fatalf("Expected children to have null leaves") 74 | } 75 | if n.right.right != nil { 76 | t.Fatalf("Expected children to have null leaves") 77 | } 78 | } 79 | 80 | func TestRangeStoreFromSorted_Overflow(t *testing.T) { 81 | items := make([]Ranged, 0) 82 | 83 | items = append(items, DefaultRangedValue{0, (1 << 63), "A"}) 84 | items = append(items, DefaultRangedValue{(1 << 63) + 1, 19, "B"}) 85 | items = append(items, DefaultRangedValue{20, 29, "C"}) 86 | 87 | _, err := NewRangeStoreFromSorted(items) 88 | 89 | if err == nil { 90 | t.Fatalf("Expecting integer overflow error and got none") 91 | } 92 | if reflect.TypeOf(err).Name() != reflect.TypeOf(ErrUnsignedIntegerOverflow{}).Name() { 93 | t.Fatalf("Expecting an ErrUnsignedIntegerOverflow, but got something else") 94 | } 95 | msg := err.Error() 96 | if msg != "Overflow adding 9223372036854775809 + 9223372036854775827" { 97 | t.Fatalf("Wrong error message: %s", msg) 98 | } 99 | } 100 | 101 | func TestRangeStoreFromSorted_Overlap(t *testing.T) { 102 | items := make([]Ranged, 0) 103 | 104 | items = append(items, DefaultRangedValue{0, 10, "A"}) 105 | items = append(items, DefaultRangedValue{9, 19, "B"}) 106 | items = append(items, DefaultRangedValue{20, 29, "C"}) 107 | 108 | _, err := NewRangeStoreFromSorted(items) 109 | 110 | if err == nil { 111 | t.Fatalf("Expecting overlap error and got none") 112 | } 113 | if reflect.TypeOf(err).Name() != reflect.TypeOf(ErrOverlap{}).Name() { 114 | t.Fatalf("Expecting an ErrOverlap, but got something else") 115 | } 116 | msg := err.Error() 117 | if msg != "Overlap detected between 10 -> 9" { 118 | t.Fatalf("Wrong error message: %s", msg) 119 | } 120 | } 121 | 122 | func TestNode_RangeSearch(t *testing.T) { 123 | items := make([]Ranged, 0) 124 | 125 | items = append(items, DefaultRangedValue{0, 9, "A"}) 126 | items = append(items, DefaultRangedValue{10, 19, "B"}) 127 | items = append(items, DefaultRangedValue{20, 29, "C"}) 128 | 129 | n, err := NewRangeStoreFromSorted(items) 130 | 131 | if err != nil { 132 | t.Fatalf("Error while constructing range store: %s", err.Error()) 133 | } 134 | 135 | a0, err := n.RangeSearch(0) 136 | if err != nil { 137 | t.Fatalf("Got an error while searching: %s", err.Error()) 138 | } 139 | if a0 != "A" { 140 | t.Fatalf("Got invalid value back %s [%s]", a0, "A") 141 | } 142 | a3, err := n.RangeSearch(3) 143 | if err != nil { 144 | t.Fatalf("Got an error while searching: %s", err.Error()) 145 | } 146 | if a3 != "A" { 147 | t.Fatalf("Got invalid value back %s [%s]", a3, "A") 148 | } 149 | a9, err := n.RangeSearch(9) 150 | if err != nil { 151 | t.Fatalf("Got an error while searching: %s", err.Error()) 152 | } 153 | if a0 != "A" { 154 | t.Fatalf("Got invalid value back %s [%s]", a9, "A") 155 | } 156 | 157 | b0, err := n.RangeSearch(10) 158 | if err != nil { 159 | t.Fatalf("Got an error while searching: %s", err.Error()) 160 | } 161 | if b0 != "B" { 162 | t.Fatalf("Got invalid value back %s [%s]", b0, "B") 163 | } 164 | b5, err := n.RangeSearch(15) 165 | if err != nil { 166 | t.Fatalf("Got an error while searching: %s", err.Error()) 167 | } 168 | if b5 != "B" { 169 | t.Fatalf("Got invalid value back %s [%s]", b5, "B") 170 | } 171 | b9, err := n.RangeSearch(19) 172 | if err != nil { 173 | t.Fatalf("Got an error while searching: %s", err.Error()) 174 | } 175 | if b9 != "B" { 176 | t.Fatalf("Got invalid value back %s [%s]", b9, "B") 177 | } 178 | 179 | c0, err := n.RangeSearch(20) 180 | if err != nil { 181 | t.Fatalf("Got an error while searching: %s", err.Error()) 182 | } 183 | if c0 != "C" { 184 | t.Fatalf("Got invalid value back %s [%s]", c0, "C") 185 | } 186 | c7, err := n.RangeSearch(27) 187 | if err != nil { 188 | t.Fatalf("Got an error while searching: %s", err.Error()) 189 | } 190 | if c7 != "C" { 191 | t.Fatalf("Got invalid value back %s [%s]", c7, "C") 192 | } 193 | c9, err := n.RangeSearch(29) 194 | if err != nil { 195 | t.Fatalf("Got an error while searching: %s", err.Error()) 196 | } 197 | if c9 != "C" { 198 | t.Fatalf("Got invalid value back %s [%s]", c9, "C") 199 | } 200 | 201 | _, err = n.RangeSearch(30) 202 | if err == nil { 203 | t.Fatalf("Expected an error while performing an out of range search, got nothing") 204 | } 205 | if reflect.TypeOf(err).Name() != reflect.TypeOf(ErrOutOfRange{}).Name() { 206 | t.Fatalf("Expecting an ErrOutOfRange, but got something else") 207 | } 208 | msg := err.Error() 209 | if msg != "Value 30 is out of range" { 210 | t.Fatalf("Wrong error message: %s", msg) 211 | } 212 | } 213 | 214 | func TestRangeStoreFromSorted_Lots(t *testing.T) { 215 | items := make([]Ranged, 0) 216 | 217 | items = append(items, DefaultRangedValue{0, 9, "A"}) 218 | items = append(items, DefaultRangedValue{10, 19, "B"}) 219 | items = append(items, DefaultRangedValue{20, 29, "C"}) 220 | items = append(items, DefaultRangedValue{30, 39, "D"}) 221 | items = append(items, DefaultRangedValue{40, 49, "E"}) 222 | items = append(items, DefaultRangedValue{50, 59, "F"}) 223 | items = append(items, DefaultRangedValue{60, 69, "G"}) 224 | items = append(items, DefaultRangedValue{70, 79, "H"}) 225 | items = append(items, DefaultRangedValue{80, 89, "I"}) 226 | items = append(items, DefaultRangedValue{90, 99, "J"}) 227 | 228 | _, err := NewRangeStoreFromSorted(items) 229 | 230 | if err != nil { 231 | t.Fatalf("Error while constructing range store: %s", err.Error()) 232 | } 233 | } 234 | 235 | func TestRangeStoreFromSorted_LongTail(t *testing.T) { 236 | items := make([]Ranged, 0) 237 | 238 | items = append(items, DefaultRangedValue{0, 2, "A"}) 239 | items = append(items, DefaultRangedValue{3, 5, "B"}) 240 | items = append(items, DefaultRangedValue{6, 29, "C"}) 241 | 242 | n, err := NewRangeStoreFromSorted(items) 243 | 244 | if err != nil { 245 | t.Fatalf("Error while constructing range store: %s", err.Error()) 246 | } 247 | 248 | //-C [max: 29] 249 | // |-A [max: 2] 250 | // | !-B [max: 5] 251 | if n.value != "C" { 252 | t.Fatalf("Expected B at the root") 253 | } 254 | if n.max != 29 { 255 | t.Fatalf("Expected 19 max at the root") 256 | } 257 | if n.left.value != "A" { 258 | t.Fatalf("Expected A as the left child") 259 | } 260 | if n.left.max != 2 { 261 | t.Fatalf("Expected 2 max as the left child") 262 | } 263 | if n.right != nil { 264 | t.Fatalf("Expected nil right child") 265 | } 266 | if n.left.right.value != "B" { 267 | t.Fatalf("Expected B as the left-right grandchild") 268 | } 269 | if n.left.right.max != 5 { 270 | t.Fatalf("Expected 5 as the max left-right grandchild") 271 | } 272 | 273 | if n.left.left != nil { 274 | t.Fatalf("Expected children to have null leaves") 275 | } 276 | if n.left.right.left != nil { 277 | t.Fatalf("Expected children to have null leaves") 278 | } 279 | if n.left.right.right != nil { 280 | t.Fatalf("Expected children to have null leaves") 281 | } 282 | } 283 | 284 | func TestRangeStoreFromSorted_Discontinuity(t *testing.T) { 285 | items := make([]Ranged, 0) 286 | 287 | items = append(items, DefaultRangedValue{0, 9, "A"}) 288 | items = append(items, DefaultRangedValue{11, 19, "B"}) 289 | items = append(items, DefaultRangedValue{20, 29, "C"}) 290 | 291 | _, err := NewRangeStoreFromSorted(items) 292 | 293 | if err == nil { 294 | t.Fatalf("Error while constructing range store: Expected an error, but none generated") 295 | } 296 | if reflect.TypeOf(err).Name() != reflect.TypeOf(ErrDiscontinuity{}).Name() { 297 | t.Fatalf("Expecting an ErrDiscontinuity, but got something else") 298 | } 299 | msg := err.Error() 300 | if msg != "Discontinuity detected from 9 -> 11" { 301 | t.Fatalf("Wrong error message: %s", msg) 302 | } 303 | } 304 | 305 | func TestRangeStoreFromSorted_Empty(t *testing.T) { 306 | items := make([]Ranged, 0) 307 | 308 | _, err := NewRangeStoreFromSorted(items) 309 | 310 | if err == nil { 311 | t.Fatalf("Error while constructing range store: Expected an error, but none generated") 312 | } 313 | if reflect.TypeOf(err).Name() != reflect.TypeOf(ErrEmptyInput{}).Name() { 314 | t.Fatalf("Expecting an ErrEmptyInput, but got something else") 315 | } 316 | msg := err.Error() 317 | if msg != "Input list is empty" { 318 | t.Fatalf("Wrong error message: %s", msg) 319 | } 320 | } 321 | 322 | func Benchmark_NewNodeSorted_Small(b *testing.B) { 323 | items := make([]Ranged, 0) 324 | items = append(items, DefaultRangedValue{0, 9, "A"}) 325 | items = append(items, DefaultRangedValue{10, 19, "B"}) 326 | items = append(items, DefaultRangedValue{20, 29, "C"}) 327 | b.ResetTimer() 328 | for n := 0; n < b.N; n += 1 { 329 | _, err := NewRangeStoreFromSorted(items) 330 | if err != nil { 331 | b.Fatalf("Got an error while benchmarking: %s", err.Error()) 332 | } 333 | } 334 | } 335 | 336 | func Benchmark_NewMapSorted_Small(b *testing.B) { 337 | items := make([]Ranged, 0) 338 | items = append(items, DefaultRangedValue{0, 9, "A"}) 339 | items = append(items, DefaultRangedValue{10, 19, "B"}) 340 | items = append(items, DefaultRangedValue{20, 29, "C"}) 341 | b.ResetTimer() 342 | for n := 0; n < b.N; n += 1 { 343 | ret := make(map[uint64]interface{}) 344 | for _, item := range items { 345 | for i := item.GetMin(); i <= item.GetMax(); i += 1 { 346 | ret[i] = item.GetValue() 347 | } 348 | } 349 | if _, ok := ret[0]; !ok { 350 | b.Fatalf("Failed to generate a valid map") 351 | } 352 | } 353 | } 354 | 355 | func Benchmark_NewNodeSorted_Large(b *testing.B) { 356 | items := make([]Ranged, 0) 357 | items = append(items, DefaultRangedValue{0, 199999, "A"}) 358 | items = append(items, DefaultRangedValue{200000, 399999, "B"}) 359 | items = append(items, DefaultRangedValue{400000, 599999, "C"}) 360 | b.ResetTimer() 361 | for n := 0; n < b.N; n += 1 { 362 | _, err := NewRangeStoreFromSorted(items) 363 | if err != nil { 364 | b.Fatalf("Got an error while benchmarking: %s", err.Error()) 365 | } 366 | } 367 | } 368 | 369 | func Benchmark_NewMapSorted_Large(b *testing.B) { 370 | items := make([]Ranged, 0) 371 | items = append(items, DefaultRangedValue{0, 199999, "A"}) 372 | items = append(items, DefaultRangedValue{200000, 399999, "B"}) 373 | items = append(items, DefaultRangedValue{400000, 599999, "C"}) 374 | b.ResetTimer() 375 | for n := 0; n < b.N; n += 1 { 376 | ret := make(map[uint64]interface{}) 377 | for _, item := range items { 378 | for i := item.GetMin(); i <= item.GetMax(); i += 1 { 379 | ret[i] = item.GetValue() 380 | } 381 | } 382 | if _, ok := ret[0]; !ok { 383 | b.Fatalf("Failed to generate a valid map") 384 | } 385 | } 386 | } 387 | 388 | func Benchmark_RangeSearch_Node(b *testing.B) { 389 | items := make([]Ranged, 0) 390 | items = append(items, DefaultRangedValue{0, 199999, "A"}) 391 | items = append(items, DefaultRangedValue{200000, 399999, "B"}) 392 | items = append(items, DefaultRangedValue{400000, 600000, "C"}) 393 | n, _ := NewRangeStoreFromSorted(items) 394 | b.ResetTimer() 395 | for i := 0; i < b.N; i += 1 { 396 | v := uint64(rand.Int() % 600000) 397 | e, err := n.RangeSearch(v) 398 | if v >= 0 && v < 200000 && e != "A" { 399 | b.Fatalf("Wrong value %d -> %s", v, e) 400 | } 401 | if v >= 200000 && v < 400000 && e != "B" { 402 | b.Fatalf("Wrong value %d -> %s", v, e) 403 | } 404 | if v >= 400000 && v < 600000 && e != "C" { 405 | b.Fatalf("Wrong value %d -> %s", v, e) 406 | } 407 | if err != nil { 408 | b.Fatalf("Got an error while searching: %s", err.Error()) 409 | } 410 | } 411 | } 412 | 413 | func Benchmark_RangeSearch_Map(b *testing.B) { 414 | items := make([]Ranged, 0) 415 | items = append(items, DefaultRangedValue{0, 199999, "A"}) 416 | items = append(items, DefaultRangedValue{200000, 399999, "B"}) 417 | items = append(items, DefaultRangedValue{400000, 600000, "C"}) 418 | ret := make(map[uint64]interface{}) 419 | for _, item := range items { 420 | for i := item.GetMin(); i <= item.GetMax(); i += 1 { 421 | ret[i] = item.GetValue() 422 | } 423 | } 424 | b.ResetTimer() 425 | for i := 0; i < b.N; i += 1 { 426 | v := uint64(rand.Int() % 600000) 427 | e, ok := ret[v] 428 | if !ok { 429 | b.Fatalf("Error while testing, unable to get in range value") 430 | } 431 | if v >= 0 && v < 200000 && e != "A" { 432 | b.Fatalf("Wrong value %d -> %s", v, e) 433 | } 434 | if v >= 200000 && v < 400000 && e != "B" { 435 | b.Fatalf("Wrong value %d -> %s", v, e) 436 | } 437 | if v >= 400000 && v < 600000 && e != "C" { 438 | b.Fatalf("Wrong value %d -> %s", v, e) 439 | } 440 | } 441 | } 442 | 443 | func TestNode_String(t *testing.T) { 444 | // Meh, print the string representation to get coverage on that code. 445 | items := make([]Ranged, 0) 446 | 447 | items = append(items, DefaultRangedValue{0, 9, "A"}) 448 | items = append(items, DefaultRangedValue{10, 19, "B"}) 449 | items = append(items, DefaultRangedValue{20, 29, "C"}) 450 | 451 | n, err := NewRangeStoreFromSorted(items) 452 | 453 | if err != nil { 454 | t.Fatalf("Error while constructing range store: %s", err.Error()) 455 | } 456 | 457 | R := `-B [max: 19] 458 | |-A [max: 9] 459 | !-C [max: 29] 460 | ` 461 | 462 | str := n.String() 463 | 464 | if str != R { 465 | t.Fatalf("Wrong string output form:\n%s\n%s", str, R) 466 | } 467 | } 468 | -------------------------------------------------------------------------------- /weighted_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Go Range Store 3 | * 4 | * Copyright 2017 Tenta, LLC 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | * For any questions, please contact developer@tenta.io 19 | * 20 | * weighted_test.go: Tests on the weighted wrapper 21 | */ 22 | 23 | package rangestore 24 | 25 | import ( 26 | "reflect" 27 | "testing" 28 | ) 29 | 30 | func TestRangeStoreFromWeighted_Basic(t *testing.T) { 31 | vals := make([]Weighted, 0) 32 | vals = append(vals, &DefaultWeightedValue{Weight: 9, Value: "A"}) 33 | vals = append(vals, &DefaultWeightedValue{Weight: 10, Value: "B"}) 34 | vals = append(vals, &DefaultWeightedValue{Weight: 10, Value: "C"}) 35 | 36 | n, err := NewRangeStoreFromWeighted(vals) 37 | 38 | if err != nil { 39 | t.Fatalf("Error while constructing range store: %s", err.Error()) 40 | } 41 | 42 | // -B [max: 19] 43 | // |-A [max: 9] 44 | // !-C [max: 29] 45 | if n.value != "B" { 46 | t.Fatalf("Expected B at the root") 47 | } 48 | if n.max != 19 { 49 | t.Fatalf("Expected 19 max at the root") 50 | } 51 | if n.left.value != "A" { 52 | t.Fatalf("Expected A as the right child") 53 | } 54 | if n.left.max != 9 { 55 | t.Fatalf("Expected 9 max as the right child") 56 | } 57 | if n.right.value != "C" { 58 | t.Fatalf("Expected C as the right child") 59 | } 60 | if n.right.max != 29 { 61 | t.Fatalf("Expected 29 max as the right child") 62 | } 63 | 64 | if n.left.left != nil { 65 | t.Fatalf("Expected children to have null leaves") 66 | } 67 | if n.left.right != nil { 68 | t.Fatalf("Expected children to have null leaves") 69 | } 70 | if n.right.left != nil { 71 | t.Fatalf("Expected children to have null leaves") 72 | } 73 | if n.right.right != nil { 74 | t.Fatalf("Expected children to have null leaves") 75 | } 76 | } 77 | 78 | func TestRangeStoreFromWeighted_Empty(t *testing.T) { 79 | items := make([]Weighted, 0) 80 | 81 | n, err := NewRangeStoreFromWeighted(items) 82 | if n != nil { 83 | t.Fatal("Expected no return but got one") 84 | } 85 | if err == nil { 86 | t.Fatalf("Error while constructing range store: Expected an error, but none generated") 87 | } 88 | if reflect.TypeOf(err).Name() != reflect.TypeOf(ErrEmptyInput{}).Name() { 89 | t.Fatalf("Expecting an ErrEmptyInput, but got something else") 90 | } 91 | msg := err.Error() 92 | if msg != "Input list is empty" { 93 | t.Fatalf("Wrong error message: %s", msg) 94 | } 95 | } 96 | 97 | func TestRangeStoreFromWeighted_Overflow(t *testing.T) { 98 | items := make([]Weighted, 0) 99 | items = append(items, DefaultWeightedValue{1 << 63, "A"}) 100 | items = append(items, DefaultWeightedValue{1 << 63, "B"}) 101 | 102 | _, err := NewRangeStoreFromWeighted(items) 103 | 104 | if err == nil { 105 | t.Fatalf("Expecting integer overflow error and got none") 106 | } 107 | if reflect.TypeOf(err).Name() != reflect.TypeOf(ErrUnsignedIntegerOverflow{}).Name() { 108 | t.Fatalf("Expecting an ErrUnsignedIntegerOverflow, but got something else") 109 | } 110 | msg := err.Error() 111 | if msg != "Overflow adding 9223372036854775808 + 9223372036854775808" { 112 | t.Fatalf("Wrong error message: %s", msg) 113 | } 114 | } 115 | --------------------------------------------------------------------------------