├── README.md ├── judy1.go ├── judy1_test.go ├── judyl.go └── judyl_test.go /README.md: -------------------------------------------------------------------------------- 1 | # go-judy 2 | Go docs can be found here: [http://godoc.org/github.com/gnoso/go-judy](http://godoc.org/github.com/gnoso/go-judy) 3 | 4 | **go-judy** is a Go language wrapper of the Judy array implementation at [http://judy.sourceforge.net](http://judy.sourceforge.net). 5 | 6 | Judy arrays are a fast and memory efficient dynamic array structure. Judy arrays were invented by Doug Baskins and implemented by Hewlett-Packard. 7 | 8 | Judy is designed to avoid cache-line fills wherever possible. There are several different variants of Judy arrays. This package implements the Judy1 bitvector and the JudyL integer map currently. Adding other variants should be relatively simple, however. 9 | 10 | Counting and range counting operations are particularly fast, and do not require a scan of the array. 11 | 12 | **NOTE:** The Judy array is implemented in C and allocates memory directly from the operating system. It is NOT garbage collected by the Go runtime. **It is very important that you call Free() on a Judy array after using it to prevent memory leaks.** The "defer" pattern is a great way to accomplish this. 13 | 14 | Here are some examples of set/unset/test bit operations: 15 | 16 | ```go 17 | j := Judy1{} // declare empty Judy1 bit vector array 18 | defer j.Free() // make sure the array is freed when finished 19 | 20 | j.Set(11235) // returns true 21 | j.Set(11235) // returns false (already set) 22 | j.Test(11235) // returns true 23 | j.Unset(11235) // returns true 24 | j.Unset(11235) // returns false (already unset) 25 | ``` 26 | 27 | #### Integer Map 28 | ```go 29 | j := JudyL{} // declare empty JudyL integer map array 30 | defer j.Free() // make sure the array is freed when finished 31 | j.Insert(11235, 1123) // returns true 32 | val, ok := j.Get(11235) // val == 1123, ok == true 33 | _, ok = j.Get(1) // ok == false (not found) 34 | j.Delete(11235) // returns true 35 | j.Delete(11235) // returns false (doesn't exist) 36 | ``` 37 | 38 | #### Count of an empty array 39 | ```go 40 | j := Judy1{} // declare empty Judy1 bit vector array 41 | defer j.Free() // make sure the array is freed when finished 42 | j.CountAll() // return 0 43 | ``` 44 | 45 | #### Check memory used by the array 46 | ```go 47 | j := Judy1{} // declare empty Judy1 bit vector array 48 | defer j.Free() // make sure the array is freed when finished 49 | for i := 0; i < 10; i++ { 50 | j.Set(uint64(i)) 51 | } 52 | j.CountAll() // return 10 53 | j.MemoryUsed() // returns memory usage info 54 | ``` 55 | -------------------------------------------------------------------------------- /judy1.go: -------------------------------------------------------------------------------- 1 | // Go language wrapper for Judy arrays (as found at http://judy.sourceforge.net) 2 | // 3 | // Judy arrays are a fast and memory efficient dynamic array structure. Judy arrays were invented by Doug Baskins 4 | // and implemented by Hewlett-Packard. 5 | // 6 | // Judy is designed to avoid cache-line fills wherever possible. There are several different variants of Judy 7 | // arrays. This package implements the Judy1 bitvector and the JudyL integer map currently. Adding other 8 | // variants should be relatively simple, however. 9 | // 10 | // Counting and range counting operations are particularly fast, and do not require a scan of the array. 11 | package judy 12 | 13 | /* 14 | #cgo LDFLAGS: -lJudy 15 | #include 16 | */ 17 | import "C" 18 | 19 | import ( 20 | "math" 21 | "unsafe" 22 | ) 23 | 24 | // A Judy1 array is the equivalent of a bit array or bit map. A bit is addressed by an index (key). The array may be sparse, and the index is a uint64 value. If an index is present, it represents a set bit (a bit set represents an index present). If an index is absent, it represents an unset bit (a bit unset represents an absent index). 25 | // The default value of this struct is a valid empty Judy1 array. 26 | // 27 | // j := Judy1{} 28 | // defer j.Free() 29 | // 30 | // j.Set(5142) 31 | // fmt.Printf("Number of items: %v", j.CountAll()) 32 | // 33 | // 34 | // Memory to support the array is allocated as bits are set, and released as bits are unset. If the Judy1 array is freed ( by calling .Free() ), all bits are unset (and the Judy1 array requires no memory). 35 | // As with an ordinary array, a Judy1 array contains no duplicate indexes. 36 | // 37 | // NOTE: The Judy array is implemented in C and allocates memory directly from the operating system. It is NOT 38 | // garbage collected by the Go runtime. It is very important that you call Free() on a Judy array after using 39 | // it to prevent memory leaks. The "defer" pattern is a great way to accomplish this. 40 | type Judy1 struct { 41 | array unsafe.Pointer 42 | } 43 | 44 | // Set index's bit in the Judy1 array. 45 | // Return true if index's bit was previously unset (successful), otherwise false if the bit was already set (unsuccessful). 46 | func (j *Judy1) Set(index uint64) bool { 47 | return C.Judy1Set(C.PPvoid_t(&j.array), C.Word_t(index), nil) != 0 48 | } 49 | 50 | // Unset index's bit in the Judy1 array. 51 | // Return true if index's bit was previously set (successful), otherwise false if the bit was already unset (unsuccessful). 52 | func (j *Judy1) Unset(index uint64) bool { 53 | return C.Judy1Unset(C.PPvoid_t(&j.array), C.Word_t(index), nil) != 0 54 | } 55 | 56 | // Test if index's bit is set in the Judy1 array. 57 | // Return true if index's bit is set (index is present), false if it is unset (index is absent). 58 | func (j *Judy1) Test(index uint64) bool { 59 | return C.Judy1Test(C.Pcvoid_t(j.array), C.Word_t(index), nil) != 0 60 | } 61 | 62 | // Free the entire Judy1 array. 63 | // Return the number of bytes freed. 64 | // 65 | // NOTE: The Judy array allocates memory directly from the operating system and is NOT garbage collected by the 66 | // Go runtime. It is very important that you call Free() on a Judy array after using it to prevent memory leaks. 67 | func (j *Judy1) Free() uint64 { 68 | return uint64(C.Judy1FreeArray(C.PPvoid_t(&j.array), nil)) 69 | } 70 | 71 | // Count the number of indexes present in the Judy1 array. 72 | // A return value of 0 can be valid as a count, or it can indicate a special case for fully populated array (32-bit machines only). See libjudy docs for ways to resolve this. 73 | func (j *Judy1) CountAll() uint64 { 74 | return uint64(C.Judy1Count(C.Pcvoid_t(j.array), 0, math.MaxUint64, nil)) 75 | } 76 | 77 | // Count the number of indexes present in the Judy1 array between indexA and indexB (inclusive). 78 | // A return value of 0 can be valid as a count, or it can indicate a special case for fully populated array (32-bit machines only). See libjudy docs for ways to resolve this. 79 | func (j *Judy1) CountFrom(indexA, indexB uint64) uint64 { 80 | return uint64(C.Judy1Count(C.Pcvoid_t(j.array), C.Word_t(indexA), C.Word_t(indexB), nil)) 81 | } 82 | 83 | // Return the number of bytes of memory currently in use by Judy1 array. This is a very fast routine, and may be used with little performance impact. 84 | func (j *Judy1) MemoryUsed() uint64 { 85 | return uint64(C.Judy1MemUsed(C.Pcvoid_t(j.array))) 86 | } 87 | 88 | // Search (inclusive) for the first index present that is equal to or greater than the passed index. 89 | // (Start with index = 0 to find the first index in the array.) This is typically used to begin a sorted-order scan of the indexes present in a Judy1 array. 90 | // 91 | // index - search index 92 | // returns uint64 - value of the first index that is equal to or greater than the passed index (only if bool return value is true) 93 | // bool - true if the search was successful, false if an index was not found 94 | func (j *Judy1) First(index uint64) (uint64, bool) { 95 | var idx C.Word_t = C.Word_t(index) 96 | 97 | if C.Judy1First(C.Pcvoid_t(j.array), &idx, nil) != 0 { 98 | return uint64(idx), true 99 | } else { 100 | return 0, false 101 | } 102 | } 103 | 104 | // Search (exclusive) for the first index present that is greater than the passed index. 105 | // This is typically used to continue a sorted-order scan of the indexes present in a Judy1 array. 106 | // 107 | // index - search index 108 | // returns uint64 - value of the first index that is greater than the passed index (only if bool return value is true) 109 | // bool - true if the search was successful, false if an index was not found 110 | func (j *Judy1) Next(index uint64) (uint64, bool) { 111 | var idx C.Word_t = C.Word_t(index) 112 | 113 | if C.Judy1Next(C.Pcvoid_t(j.array), &idx, nil) != 0 { 114 | return uint64(idx), true 115 | } else { 116 | return 0, false 117 | } 118 | } 119 | 120 | // Search (inclusive) for the last index present that is equal to or less than than the passed index. 121 | // (Start with index = math.MaxUint64 to find the last index in the array.) This is typically used to begin a reverse-sorted-order scan of the indexes present in a Judy1 array. 122 | // 123 | // index - search index 124 | // returns uint64 - value of the last index that is equal to or less than the passed index (only if bool return value is true) 125 | // bool - true if the search was successful, false if an index was not found 126 | func (j *Judy1) Last(index uint64) (uint64, bool) { 127 | var idx C.Word_t = C.Word_t(index) 128 | 129 | if C.Judy1Last(C.Pcvoid_t(j.array), &idx, nil) != 0 { 130 | return uint64(idx), true 131 | } else { 132 | return 0, false 133 | } 134 | } 135 | 136 | // Search (exclusive) for the last index present that is less than the passed index. 137 | // This is typically used to continue a reverse sorted-order scan of the indexes present in a Judy1 array. 138 | // 139 | // index - search index 140 | // returns uint64 - value of the last index that is less than the passed index (only if bool return value is true) 141 | // bool - true if the search was successful, false if an index was not found 142 | func (j *Judy1) Prev(index uint64) (uint64, bool) { 143 | var idx C.Word_t = C.Word_t(index) 144 | 145 | if C.Judy1Prev(C.Pcvoid_t(j.array), &idx, nil) != 0 { 146 | return uint64(idx), true 147 | } else { 148 | return 0, false 149 | } 150 | } 151 | 152 | // Locate the Nth index that is present in the Judy1 array (Nth = 1 returns the first index present). 153 | // 154 | // nth - nth index to find 155 | // returns uint64 - nth index (unless return false)) 156 | // bool - true if the search was successful, false if an index was not found 157 | func (j *Judy1) ByCount(nth uint64) (uint64, bool) { 158 | var idx C.Word_t 159 | 160 | if C.Judy1ByCount(C.Pcvoid_t(j.array), C.Word_t(nth), &idx, nil) != 0 { 161 | return uint64(idx), true 162 | } else { 163 | return 0, false 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /judy1_test.go: -------------------------------------------------------------------------------- 1 | package judy 2 | 3 | import ( 4 | "math" 5 | "math/rand" 6 | "testing" 7 | ) 8 | 9 | func TestEmptyJudy1Array(t *testing.T) { 10 | 11 | j := Judy1{} 12 | r := j.Free() 13 | 14 | if r != 0 { 15 | t.Errorf("Free should return 0, returned %v", r) 16 | } 17 | } 18 | 19 | func TestJudy1Count(t *testing.T) { 20 | 21 | j := Judy1{} 22 | defer j.Free() 23 | 24 | if ct := j.CountAll(); ct != 0 { 25 | t.Errorf("Count should be zero, was %v", ct) 26 | } 27 | 28 | var i uint64 29 | for i = 0; i < 100; i++ { 30 | j.Set(i) 31 | } 32 | 33 | var ct uint64 34 | if ct = j.CountAll(); ct != 100 { 35 | t.Errorf("Count should be 100, was %v", ct) 36 | } 37 | 38 | if ct = j.CountFrom(0, 1000); ct != 100 { 39 | t.Errorf("Count should be 100, was %v", ct) 40 | } 41 | if ct = j.CountFrom(200, 1000); ct != 0 { 42 | t.Errorf("Count should be 0, was %v", ct) 43 | } 44 | if ct = j.CountFrom(5, 5); ct != 1 { 45 | t.Errorf("Count should be 1, was %v", ct) 46 | } 47 | if ct = j.CountFrom(20, 29); ct != 10 { 48 | t.Errorf("Count should be 10, was %v", ct) 49 | } 50 | 51 | } 52 | 53 | func TestJudy1Set(t *testing.T) { 54 | 55 | j := Judy1{} 56 | defer j.Free() 57 | 58 | var i uint64 59 | for i = 0; i < 100; i++ { 60 | j.Set(i * 100000) 61 | } 62 | 63 | for i = 0; i < 100; i++ { 64 | if !j.Test(i * 100000) { 65 | t.Errorf("Index %v not set", i*100000) 66 | } 67 | } 68 | 69 | for i = 1; i < 100; i++ { 70 | if j.Test(i * 99999) { 71 | t.Errorf("Index %v incorrectly set", i*99999) 72 | } 73 | } 74 | 75 | } 76 | 77 | func TestJudy1SetReturn(t *testing.T) { 78 | j := Judy1{} 79 | defer j.Free() 80 | 81 | if !j.Set(12345) { 82 | t.Error("First set should return true") 83 | } 84 | if j.Set(12345) { 85 | t.Error("Second set should return false") 86 | } 87 | if !j.Test(12345) { 88 | t.Error("Data should be set") 89 | } 90 | } 91 | 92 | func TestJudy1UnsetReturn(t *testing.T) { 93 | j := Judy1{} 94 | defer j.Free() 95 | 96 | if j.Unset(12345) { 97 | t.Error("First unset should return false") 98 | } 99 | j.Set(12345) 100 | if !j.Unset(12345) { 101 | t.Error("Second unset should return true") 102 | } 103 | if j.Test(12345) { 104 | t.Error("Data should be unset") 105 | } 106 | } 107 | 108 | func TestJudy1Unset(t *testing.T) { 109 | 110 | j := Judy1{} 111 | defer j.Free() 112 | 113 | var i uint64 114 | for i = 0; i < 100; i++ { 115 | j.Set(i * 100000) 116 | } 117 | 118 | for i = 0; i < 100; i++ { 119 | j.Unset(i * 100000) 120 | } 121 | 122 | for i = 0; i < 100; i++ { 123 | if j.Test(i * 100000) { 124 | t.Errorf("Index %v incorrectly set", i*100000) 125 | } 126 | } 127 | 128 | if ct := j.CountAll(); ct != 0 { 129 | t.Errorf("Count should be zero, was %v", ct) 130 | } 131 | } 132 | 133 | func TestJudy1ByCount(t *testing.T) { 134 | 135 | j := Judy1{} 136 | defer j.Free() 137 | 138 | j.Set(12345) 139 | j.Set(11235) 140 | j.Set(54321) 141 | 142 | if idx, ok := j.ByCount(1); ok && idx != 11235 { 143 | t.Errorf("ByCount should return 11235,true but was %v, %v", idx, ok) 144 | } 145 | if idx, ok := j.ByCount(2); ok && idx != 12345 { 146 | t.Errorf("ByCount should return 12345,true but was %v, %v", idx, ok) 147 | } 148 | if idx, ok := j.ByCount(3); ok && idx != 54321 { 149 | t.Errorf("ByCount should return 54321,true but was %v, %v", idx, ok) 150 | } 151 | if _, ok := j.ByCount(0); ok { 152 | t.Error("There should be no return value for 0") 153 | } 154 | if _, ok := j.ByCount(4); ok { 155 | t.Error("There should be no return value for 4") 156 | } 157 | } 158 | 159 | func TestJudy1First(t *testing.T) { 160 | 161 | j := Judy1{} 162 | defer j.Free() 163 | 164 | var i uint64 165 | for i = 0; i < 100; i++ { 166 | j.Set(i * 2) 167 | } 168 | 169 | if next, ok := j.First(20); ok && next != 20 { 170 | t.Errorf("First(20) should be 20, was %v", next) 171 | } 172 | if next, ok := j.First(21); ok && next != 22 { 173 | t.Errorf("First(21) should be 22, was %v", next) 174 | } 175 | if _, ok := j.First(201); ok { 176 | t.Errorf("First(201) should not be found") 177 | } 178 | 179 | } 180 | 181 | func TestJudy1Last(t *testing.T) { 182 | 183 | j := Judy1{} 184 | defer j.Free() 185 | 186 | var i uint64 187 | for i = 1; i < 100; i++ { 188 | j.Set(i * 2) 189 | } 190 | 191 | if next, ok := j.Last(20); ok && next != 20 { 192 | t.Errorf("Last(20) should be 20, was %v", next) 193 | } 194 | if next, ok := j.Last(21); ok && next != 20 { 195 | t.Errorf("Last(21) should be 20, was %v", next) 196 | } 197 | if _, ok := j.Last(1); ok { 198 | t.Errorf("Last(1) should not be found") 199 | } 200 | } 201 | 202 | func TestJudy1Next(t *testing.T) { 203 | 204 | j := Judy1{} 205 | defer j.Free() 206 | 207 | var i uint64 208 | for i = 0; i < 100; i++ { 209 | j.Set(i * 2) 210 | } 211 | 212 | if next, ok := j.Next(20); ok && next != 22 { 213 | t.Errorf("Next(20) should be 22, was %v", next) 214 | } 215 | if next, ok := j.Next(21); ok && next != 22 { 216 | t.Errorf("Next(21) should be 22, was %v", next) 217 | } 218 | if _, ok := j.Next(200); ok { 219 | t.Errorf("Next(200) should not be found") 220 | } 221 | 222 | } 223 | 224 | func TestJudy1Prev(t *testing.T) { 225 | 226 | j := Judy1{} 227 | defer j.Free() 228 | 229 | var i uint64 230 | for i = 1; i < 100; i++ { 231 | j.Set(i * 2) 232 | } 233 | 234 | if next, ok := j.Prev(20); ok && next != 18 { 235 | t.Errorf("Prev(20) should be 18, was %v", next) 236 | } 237 | if next, ok := j.Prev(21); ok && next != 20 { 238 | t.Errorf("Prev(21) should be 20, was %v", next) 239 | } 240 | if _, ok := j.Prev(2); ok { 241 | t.Errorf("Prev(2) should not be found") 242 | } 243 | 244 | } 245 | 246 | func runOrderedJudy1MemUsageTest(t *testing.T, n int) { 247 | j := Judy1{} 248 | defer j.Free() 249 | 250 | for i := 0; i < n; i++ { 251 | j.Set(uint64(i * 10000)) 252 | } 253 | 254 | if ct := j.CountAll(); int(ct) != n { 255 | t.Errorf("Count should be %v, was %v", n, ct) 256 | } 257 | t.Logf("Memory Usage with %7v ordered bits %8v", n, j.MemoryUsed()) 258 | } 259 | 260 | func runRandomJudy1MemUsageTest(t *testing.T, n int) { 261 | j := Judy1{} 262 | defer j.Free() 263 | 264 | for i := 0; i < n; i++ { 265 | j.Set(uint64(rand.Int63())) 266 | } 267 | 268 | if ct := j.CountAll(); int(ct) != n { 269 | t.Errorf("Count should be %v, was %v", n, ct) 270 | } 271 | t.Logf("Memory Usage with %7v random bits %8v", n, j.MemoryUsed()) 272 | } 273 | 274 | func TestJudy1MemUsage(t *testing.T) { 275 | 276 | runOrderedJudy1MemUsageTest(t, 1000) 277 | runRandomJudy1MemUsageTest(t, 1000) 278 | runOrderedJudy1MemUsageTest(t, 10000) 279 | runRandomJudy1MemUsageTest(t, 10000) 280 | runOrderedJudy1MemUsageTest(t, 100000) 281 | runRandomJudy1MemUsageTest(t, 100000) 282 | runOrderedJudy1MemUsageTest(t, 1000000) 283 | runRandomJudy1MemUsageTest(t, 1000000) 284 | 285 | //t.Fail() // Uncomment to see the log output 286 | } 287 | 288 | func BenchmarkJudy1CountAllRand1000(b *testing.B) { 289 | j := Judy1{} 290 | defer j.Free() 291 | 292 | n := 1000 293 | for i := 0; i < n; i++ { 294 | j.Set(uint64(rand.Int63())) 295 | } 296 | 297 | for loops := 0; loops < b.N; loops++ { 298 | if ct := j.CountAll(); int(ct) != n { 299 | b.Errorf("Count should be %v, was %v", n, ct) 300 | } 301 | } 302 | } 303 | 304 | func BenchmarkJudy1CountAllRand1000000(b *testing.B) { 305 | j := Judy1{} 306 | defer j.Free() 307 | 308 | n := 1000000 309 | for i := 0; i < n; i++ { 310 | j.Set(uint64(rand.Int63())) 311 | } 312 | 313 | for loops := 0; loops < b.N; loops++ { 314 | if ct := j.CountAll(); int(ct) != n { 315 | b.Errorf("Count should be %v, was %v", n, ct) 316 | } 317 | } 318 | } 319 | 320 | func BenchmarkJudy1CountAllOrd1000(b *testing.B) { 321 | j := Judy1{} 322 | defer j.Free() 323 | 324 | n := 1000 325 | for i := 0; i < n; i++ { 326 | j.Set(uint64(i)) 327 | } 328 | 329 | for loops := 0; loops < b.N; loops++ { 330 | if ct := j.CountAll(); int(ct) != n { 331 | b.Errorf("Count should be %v, was %v", n, ct) 332 | } 333 | } 334 | } 335 | 336 | func BenchmarkJudy1CountAllOrd1000000(b *testing.B) { 337 | j := Judy1{} 338 | defer j.Free() 339 | 340 | n := 1000000 341 | for i := 0; i < n; i++ { 342 | j.Set(uint64(i)) 343 | } 344 | 345 | for loops := 0; loops < b.N; loops++ { 346 | if ct := j.CountAll(); int(ct) != n { 347 | b.Errorf("Count should be %v, was %v", n, ct) 348 | } 349 | } 350 | } 351 | 352 | func BenchmarkJudy1CountRangeRand1000(b *testing.B) { 353 | j := Judy1{} 354 | defer j.Free() 355 | 356 | n := 1000 357 | for i := 0; i < n; i++ { 358 | j.Set(uint64(rand.Int63())) 359 | } 360 | 361 | for loops := 0; loops < b.N; loops++ { 362 | if ct := j.CountFrom(math.MaxUint64/8, (math.MaxUint64/8)*7); int(ct) < n/2 { 363 | b.Errorf("Count should > %v, was %v", n/2, ct) 364 | } 365 | } 366 | } 367 | 368 | func BenchmarkJudy1CountRangeRand1000000(b *testing.B) { 369 | j := Judy1{} 370 | defer j.Free() 371 | 372 | n := 1000000 373 | for i := 0; i < n; i++ { 374 | j.Set(uint64(rand.Int63())) 375 | } 376 | 377 | for loops := 0; loops < b.N; loops++ { 378 | if ct := j.CountFrom(math.MaxUint64/8, (math.MaxUint64/8)*7); int(ct) < n/2 { 379 | b.Errorf("Count should > %v, was %v", n/2, ct) 380 | } 381 | } 382 | } 383 | -------------------------------------------------------------------------------- /judyl.go: -------------------------------------------------------------------------------- 1 | package judy 2 | 3 | /* 4 | #cgo LDFLAGS: -lJudy 5 | #include 6 | */ 7 | import "C" 8 | 9 | import ( 10 | "math" 11 | "unsafe" 12 | ) 13 | 14 | // A JudyL array is the equivalent of a dynamic array of uint64 values. A value is addressed by an index (key). 15 | // The array may be sparse, and the index may be any uint64 number. 16 | // 17 | // The default value of this struct is a valid empty JudyL array. 18 | // 19 | // j := JudyL{} 20 | // defer j.Free() 21 | // 22 | // j.Insert(5142, 142) 23 | // fmt.Printf("Number of items: %v", j.CountAll()) 24 | // 25 | // 26 | // Memory to support the array is allocated as index/value pairs are inserted, and released as index/value 27 | // pairs are deleted. A JudyL array can also be thought of as a mapper, that is "map" a word to another 28 | // word. If the JudyL array is freed ( by calling .Free() ), all words are removed (and the JudyL array requires no memory). 29 | // As with an ordinary array, a JudyL array contains no duplicate indexes. 30 | // 31 | // NOTE: The Judy array is implemented in C and allocates memory directly from the operating system. It is NOT 32 | // garbage collected by the Go runtime. It is very important that you call Free() on a Judy array after using 33 | // it to prevent memory leaks. The "defer" pattern is a great way to accomplish this. 34 | type JudyL struct { 35 | array unsafe.Pointer 36 | } 37 | 38 | // Insert an Index and Value into the JudyL array. If the Index is successfully inserted, the Value is 39 | // initialized as well. If the Index was already present, the current Value is replaced with the provided Value. 40 | func (j *JudyL) Insert(index uint64, value uint64) { 41 | pval := unsafe.Pointer(C.JudyLIns(C.PPvoid_t(&j.array), C.Word_t(index), nil)) 42 | *((*C.Word_t)(pval)) = C.Word_t(value) 43 | } 44 | 45 | // Delete the Index/Value pair from the JudyL array. 46 | // Returns true if successful. Returns false if Index was not present. 47 | func (j *JudyL) Delete(index uint64) bool { 48 | return C.JudyLDel(C.PPvoid_t(&j.array), C.Word_t(index), nil) != 0 49 | } 50 | 51 | // Get the Value associated with Index in the Judy array 52 | // returns (value, true) if the index was found 53 | // returns (_, false) if the index was not found 54 | func (j *JudyL) Get(index uint64) (uint64, bool) { 55 | pval := unsafe.Pointer(C.JudyLGet(C.Pcvoid_t(j.array), C.Word_t(index), nil)) 56 | if pval == nil { 57 | return 0, false 58 | } else { 59 | return uint64(*((*C.Word_t)(pval))), true 60 | } 61 | } 62 | 63 | // Free the entire JudyL array. 64 | // Return the number of bytes freed. 65 | // 66 | // NOTE: The Judy array allocates memory directly from the operating system and is NOT garbage collected by the 67 | // Go runtime. It is very important that you call Free() on a Judy array after using it to prevent memory leaks. 68 | func (j *JudyL) Free() uint64 { 69 | return uint64(C.JudyLFreeArray(C.PPvoid_t(&j.array), nil)) 70 | } 71 | 72 | // Count the number of indexes present in the JudyL array. 73 | // Returns the count. A return value of 0 can be valid as a count, or it can indicate a special case for fully populated array (32-bit machines only). See libjudy docs for ways to resolve this. 74 | func (j *JudyL) CountAll() uint64 { 75 | return uint64(C.JudyLCount(C.Pcvoid_t(j.array), 0, math.MaxUint64, nil)) 76 | } 77 | 78 | // Count the number of indexes present in the JudyL array between indexA and indexB (inclusive). 79 | // Returns the count. A return value of 0 can be valid as a count, or it can indicate a special case for fully populated array (32-bit machines only). See libjudy docs for ways to resolve this. 80 | func (j *JudyL) CountFrom(indexA, indexB uint64) uint64 { 81 | return uint64(C.JudyLCount(C.Pcvoid_t(j.array), C.Word_t(indexA), C.Word_t(indexB), nil)) 82 | } 83 | 84 | // Return the number of bytes of memory currently in use by JudyL array. This is a very fast routine, 85 | // and may be used with little performance impact. 86 | func (j *JudyL) MemoryUsed() uint64 { 87 | return uint64(C.JudyLMemUsed(C.Pcvoid_t(j.array))) 88 | } 89 | 90 | // Search (inclusive) for the first index present that is equal to or greater than the passed index. 91 | // (Start with index = 0 to find the first index in the array.) This is typically used to begin a sorted-order scan of the indexes present in a JudyL array. 92 | // 93 | // index - search index 94 | // returns uint64 - value of the first index that is equal to or greater than the passed index 95 | // (only if bool return value is true) 96 | // uint64 - value pointed to by the index 97 | // bool - true if the search was successful, false if an index was not found 98 | func (j *JudyL) First(index uint64) (uint64, uint64, bool) { 99 | idx := C.Word_t(index) 100 | pval := unsafe.Pointer(C.JudyLFirst(C.Pcvoid_t(j.array), &idx, nil)) 101 | 102 | if pval == nil { 103 | return 0, 0, false 104 | } else { 105 | return uint64(idx), uint64(*((*C.Word_t)(pval))), true 106 | } 107 | } 108 | 109 | // Search (exclusive) for the first index present that is greater than the passed index. 110 | // This is typically used to continue a sorted-order scan of the indexes present in a JudyL array. 111 | // 112 | // index - search index 113 | // returns uint64 - value of the first index that is greater than the passed index 114 | // (only if bool return value is true) 115 | // uint64 - value pointed to by the index 116 | // bool - true if the search was successful, false if an index was not found 117 | func (j *JudyL) Next(index uint64) (uint64, uint64, bool) { 118 | idx := C.Word_t(index) 119 | pval := unsafe.Pointer(C.JudyLNext(C.Pcvoid_t(j.array), &idx, nil)) 120 | 121 | if pval == nil { 122 | return 0, 0, false 123 | } else { 124 | return uint64(idx), uint64(*((*C.Word_t)(pval))), true 125 | } 126 | } 127 | 128 | // Search (inclusive) for the last index present that is equal to or less than than the passed index. 129 | // (Start with index = math.MaxUint64 to find the last index in the array.) This is typically used to begin a reverse-sorted-order scan of the indexes present in a JudyL array. 130 | // 131 | // index - search index 132 | // returns uint64 - value of the last index that is equal to or less than the passed index 133 | // (only if bool return value is true) 134 | // uint64 - value pointed to by the index 135 | // bool - true if the search was successful, false if an index was not found 136 | func (j *JudyL) Last(index uint64) (uint64, uint64, bool) { 137 | idx := C.Word_t(index) 138 | pval := unsafe.Pointer(C.JudyLLast(C.Pcvoid_t(j.array), &idx, nil)) 139 | 140 | if pval == nil { 141 | return 0, 0, false 142 | } else { 143 | return uint64(idx), uint64(*((*C.Word_t)(pval))), true 144 | } 145 | } 146 | 147 | // Search (exclusive) for the last index present that is less than the passed index. 148 | // This is typically used to continue a reverse sorted-order scan of the indexes present in a JudyL array. 149 | // 150 | // index - search index 151 | // returns uint64 - value of the last index that is less than the passed index 152 | // (only if bool return value is true) 153 | // uint64 - value pointed to by the index 154 | // bool - true if the search was successful, false if an index was not found 155 | func (j *JudyL) Prev(index uint64) (uint64, uint64, bool) { 156 | idx := C.Word_t(index) 157 | pval := unsafe.Pointer(C.JudyLPrev(C.Pcvoid_t(j.array), &idx, nil)) 158 | 159 | if pval == nil { 160 | return 0, 0, false 161 | } else { 162 | return uint64(idx), uint64(*((*C.Word_t)(pval))), true 163 | } 164 | } 165 | 166 | // Locate the Nth index that is present in the JudyL array (Nth = 1 returns the first index present). 167 | // 168 | // nth - nth index to find 169 | // returns uint64 - nth index (unless return false) 170 | // uint64 - nth value (unless return false) 171 | // bool - true if the search was successful, false if an index was not found 172 | func (j *JudyL) ByCount(nth uint64) (uint64, uint64, bool) { 173 | var idx C.Word_t 174 | pval := unsafe.Pointer(C.JudyLByCount(C.Pcvoid_t(j.array), C.Word_t(nth), &idx, nil)) 175 | 176 | if pval == nil { 177 | return 0, 0, false 178 | } else { 179 | return uint64(idx), uint64(*((*C.Word_t)(pval))), true 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /judyl_test.go: -------------------------------------------------------------------------------- 1 | package judy 2 | 3 | import ( 4 | "math" 5 | "math/rand" 6 | "testing" 7 | ) 8 | 9 | func TestEmptyJudyLArray(t *testing.T) { 10 | 11 | j := JudyL{} 12 | r := j.Free() 13 | 14 | if r != 0 { 15 | t.Errorf("Free should return 0, returned %v", r) 16 | } 17 | } 18 | 19 | func TestJudyLCount(t *testing.T) { 20 | 21 | j := JudyL{} 22 | defer j.Free() 23 | 24 | if ct := j.CountAll(); ct != 0 { 25 | t.Errorf("Count should be zero, was %v", ct) 26 | } 27 | 28 | var i uint64 29 | for i = 0; i < 100; i++ { 30 | j.Insert(i, i) 31 | } 32 | 33 | var ct uint64 34 | if ct = j.CountAll(); ct != 100 { 35 | t.Errorf("Count should be 100, was %v", ct) 36 | } 37 | 38 | if ct = j.CountFrom(0, 1000); ct != 100 { 39 | t.Errorf("Count should be 100, was %v", ct) 40 | } 41 | if ct = j.CountFrom(200, 1000); ct != 0 { 42 | t.Errorf("Count should be 0, was %v", ct) 43 | } 44 | if ct = j.CountFrom(5, 5); ct != 1 { 45 | t.Errorf("Count should be 1, was %v", ct) 46 | } 47 | if ct = j.CountFrom(20, 29); ct != 10 { 48 | t.Errorf("Count should be 10, was %v", ct) 49 | } 50 | 51 | } 52 | 53 | func TestJudyLInsertGet(t *testing.T) { 54 | 55 | j := JudyL{} 56 | defer j.Free() 57 | 58 | var i uint64 59 | for i = 0; i < 100; i++ { 60 | j.Insert(i*100000, i) 61 | } 62 | 63 | for i = 0; i < 100; i++ { 64 | if v, ok := j.Get(i * 100000); ok && v != i { 65 | t.Errorf("Index %v should be %v, but was %v", i*100000, i, v) 66 | } 67 | } 68 | 69 | for i = 1; i < 100; i++ { 70 | if _, ok := j.Get(i * 99999); ok { 71 | t.Errorf("Index %v incorrectly set", i*99999) 72 | } 73 | } 74 | 75 | } 76 | 77 | func TestJudyLByCount(t *testing.T) { 78 | 79 | j := JudyL{} 80 | defer j.Free() 81 | 82 | j.Insert(12345, 234) 83 | j.Insert(11235, 11235) 84 | j.Insert(54321, 4321) 85 | 86 | if idx, val, ok := j.ByCount(1); ok && (idx != 11235 || val != 11235) { 87 | t.Errorf("ByCount should return 11235,11235,true but was %v, %v, %v", idx, val, ok) 88 | } 89 | if idx, val, ok := j.ByCount(2); ok && (idx != 12345 || val != 234) { 90 | t.Errorf("ByCount should return 12345,234,true but was %v, %v, %v", idx, val, ok) 91 | } 92 | if idx, val, ok := j.ByCount(3); ok && (idx != 54321 || val != 4321) { 93 | t.Errorf("ByCount should return 54321,4321,true but was %v, %v, %v", idx, val, ok) 94 | } 95 | if _, _, ok := j.ByCount(0); ok { 96 | t.Error("There should be no return value for 0") 97 | } 98 | if _, _, ok := j.ByCount(4); ok { 99 | t.Error("There should be no return value for 4") 100 | } 101 | } 102 | 103 | func TestJudyLDelete(t *testing.T) { 104 | 105 | j := JudyL{} 106 | defer j.Free() 107 | 108 | j.Insert(12345, 234) 109 | j.Insert(11235, 11235) 110 | j.Insert(54321, 4321) 111 | 112 | if ct := j.CountAll(); ct != 3 { 113 | t.Errorf("Count should be 3") 114 | } 115 | if ok := j.Delete(11235); !ok { 116 | t.Errorf("Delete should return ok") 117 | } 118 | if ct := j.CountAll(); ct != 2 { 119 | t.Errorf("Count should be 2") 120 | } 121 | if _, ok := j.Get(11235); ok { 122 | t.Errorf("Value should be removed") 123 | } 124 | if ok := j.Delete(11235); ok { 125 | t.Errorf("Delete not should return ok") 126 | } 127 | 128 | } 129 | 130 | func TestJudyLFirst(t *testing.T) { 131 | 132 | j := JudyL{} 133 | defer j.Free() 134 | 135 | var i uint64 136 | for i = 0; i < 100; i++ { 137 | j.Insert(i*2, i) 138 | } 139 | 140 | if next, val, ok := j.First(20); ok && (next != 20 || val != 10) { 141 | t.Errorf("First(20) should be 20, 10 was %v,%v", next, val) 142 | } 143 | if next, val, ok := j.First(21); ok && (next != 22 || val != 11) { 144 | t.Errorf("First(21) should be 22, 11 was %v,%v", next, val) 145 | } 146 | if _, _, ok := j.First(201); ok { 147 | t.Errorf("First(201) should not be found") 148 | } 149 | 150 | } 151 | 152 | func TestJudyLLast(t *testing.T) { 153 | 154 | j := JudyL{} 155 | defer j.Free() 156 | 157 | var i uint64 158 | for i = 1; i < 100; i++ { 159 | j.Insert(i*2, i) 160 | } 161 | 162 | if next, val, ok := j.Last(20); ok && (next != 20 || val != 10) { 163 | t.Errorf("Last(20) should be 20,10 was %v,%v", next, val) 164 | } 165 | if next, val, ok := j.Last(21); ok && (next != 20 || val != 10) { 166 | t.Errorf("Last(21) should be 20, 10 was %v,%v", next, val) 167 | } 168 | if _, _, ok := j.Last(1); ok { 169 | t.Errorf("Last(1) should not be found") 170 | } 171 | } 172 | 173 | func TestJudyLNext(t *testing.T) { 174 | 175 | j := JudyL{} 176 | defer j.Free() 177 | 178 | var i uint64 179 | for i = 0; i < 100; i++ { 180 | j.Insert(i*2, i) 181 | } 182 | 183 | if next, val, ok := j.Next(20); ok && (next != 22 || val != 11) { 184 | t.Errorf("Next(20) should be 22,11 was %v,%v", next, val) 185 | } 186 | if next, val, ok := j.Next(21); ok && (next != 22 || val != 11) { 187 | t.Errorf("Next(21) should be 22,11 was %v,%v", next, val) 188 | } 189 | if _, _, ok := j.Next(200); ok { 190 | t.Errorf("Next(200) should not be found") 191 | } 192 | 193 | } 194 | 195 | func TestJudyLPrev(t *testing.T) { 196 | 197 | j := JudyL{} 198 | defer j.Free() 199 | 200 | var i uint64 201 | for i = 1; i < 100; i++ { 202 | j.Insert(i*2, i) 203 | } 204 | 205 | if next, val, ok := j.Prev(20); ok && (next != 18 || val != 9) { 206 | t.Errorf("Prev(20) should be 18,9 was %v,%v", next, val) 207 | } 208 | if next, val, ok := j.Prev(21); ok && (next != 20 || val != 10) { 209 | t.Errorf("Prev(21) should be 20,10 was %v", next, val) 210 | } 211 | if _, _, ok := j.Prev(2); ok { 212 | t.Errorf("Prev(2) should not be found") 213 | } 214 | 215 | } 216 | 217 | func runOrderedJudyLMemUsageTest(t *testing.T, n int) { 218 | j := JudyL{} 219 | defer j.Free() 220 | 221 | for i := 0; i < n; i++ { 222 | j.Insert(uint64(i*10000), uint64(i)) 223 | } 224 | 225 | if ct := j.CountAll(); int(ct) != n { 226 | t.Errorf("Count should be %v, was %v", n, ct) 227 | } 228 | t.Logf("Memory Usage with %7v ordered bits %8v", n, j.MemoryUsed()) 229 | } 230 | 231 | func runRandomJudyLMemUsageTest(t *testing.T, n int) { 232 | j := JudyL{} 233 | defer j.Free() 234 | 235 | for i := 0; i < n; i++ { 236 | j.Insert(uint64(rand.Int63()), uint64(rand.Int63())) 237 | } 238 | 239 | if ct := j.CountAll(); int(ct) != n { 240 | t.Errorf("Count should be %v, was %v", n, ct) 241 | } 242 | t.Logf("Memory Usage with %7v random bits %8v", n, j.MemoryUsed()) 243 | } 244 | 245 | func TestMemUsage(t *testing.T) { 246 | 247 | runOrderedJudyLMemUsageTest(t, 1000) 248 | runRandomJudyLMemUsageTest(t, 1000) 249 | runOrderedJudyLMemUsageTest(t, 10000) 250 | runRandomJudyLMemUsageTest(t, 10000) 251 | runOrderedJudyLMemUsageTest(t, 100000) 252 | runRandomJudyLMemUsageTest(t, 100000) 253 | runOrderedJudyLMemUsageTest(t, 1000000) 254 | runRandomJudyLMemUsageTest(t, 1000000) 255 | 256 | //t.Fail() // Uncomment to see the log output 257 | } 258 | 259 | func BenchmarkJudyLCountAllRand1000(b *testing.B) { 260 | j := JudyL{} 261 | defer j.Free() 262 | 263 | n := 1000 264 | for i := 0; i < n; i++ { 265 | j.Insert(uint64(rand.Int63()), uint64(rand.Int63())) 266 | } 267 | 268 | for loops := 0; loops < b.N; loops++ { 269 | if ct := j.CountAll(); int(ct) != n { 270 | b.Errorf("Count should be %v, was %v", n, ct) 271 | } 272 | } 273 | } 274 | 275 | func BenchmarkJudyLCountAllRand1000000(b *testing.B) { 276 | j := JudyL{} 277 | defer j.Free() 278 | 279 | n := 1000000 280 | for i := 0; i < n; i++ { 281 | j.Insert(uint64(rand.Int63()), uint64(rand.Int63())) 282 | } 283 | 284 | for loops := 0; loops < b.N; loops++ { 285 | if ct := j.CountAll(); int(ct) != n { 286 | b.Errorf("Count should be %v, was %v", n, ct) 287 | } 288 | } 289 | } 290 | 291 | func BenchmarkJudyLCountAllOrd1000(b *testing.B) { 292 | j := JudyL{} 293 | defer j.Free() 294 | 295 | n := 1000 296 | for i := 0; i < n; i++ { 297 | j.Insert(uint64(i), uint64(i)) 298 | } 299 | 300 | for loops := 0; loops < b.N; loops++ { 301 | if ct := j.CountAll(); int(ct) != n { 302 | b.Errorf("Count should be %v, was %v", n, ct) 303 | } 304 | } 305 | } 306 | 307 | func BenchmarkJudyLCountAllOrd1000000(b *testing.B) { 308 | j := JudyL{} 309 | defer j.Free() 310 | 311 | n := 1000000 312 | for i := 0; i < n; i++ { 313 | j.Insert(uint64(i), uint64(i)) 314 | } 315 | 316 | for loops := 0; loops < b.N; loops++ { 317 | if ct := j.CountAll(); int(ct) != n { 318 | b.Errorf("Count should be %v, was %v", n, ct) 319 | } 320 | } 321 | } 322 | 323 | func BenchmarkJudyLCountRangeRand1000(b *testing.B) { 324 | j := JudyL{} 325 | defer j.Free() 326 | 327 | n := 1000 328 | for i := 0; i < n; i++ { 329 | j.Insert(uint64(rand.Int63()), uint64(rand.Int63())) 330 | } 331 | 332 | for loops := 0; loops < b.N; loops++ { 333 | if ct := j.CountFrom(math.MaxUint64/8, (math.MaxUint64/8)*7); int(ct) < n/2 { 334 | b.Errorf("Count should > %v, was %v", n/2, ct) 335 | } 336 | } 337 | } 338 | 339 | func BenchmarkJudyLCountRangeRand1000000(b *testing.B) { 340 | j := JudyL{} 341 | defer j.Free() 342 | 343 | n := 1000000 344 | for i := 0; i < n; i++ { 345 | j.Insert(uint64(rand.Int63()), uint64(rand.Int63())) 346 | } 347 | 348 | for loops := 0; loops < b.N; loops++ { 349 | if ct := j.CountFrom(math.MaxUint64/8, (math.MaxUint64/8)*7); int(ct) < n/2 { 350 | b.Errorf("Count should > %v, was %v", n/2, ct) 351 | } 352 | } 353 | } 354 | --------------------------------------------------------------------------------