├── .travis.yml ├── LICENSE ├── README.md ├── doc.go ├── errors.go ├── example_object_test.go ├── example_prefix_queue_test.go ├── example_priority_queue_test.go ├── example_queue_test.go ├── example_stack_test.go ├── file.go ├── go.mod ├── go.sum ├── item.go ├── prefix_queue.go ├── prefix_queue_test.go ├── priority_queue.go ├── priority_queue_test.go ├── queue.go ├── queue_test.go ├── stack.go └── stack_test.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.13 5 | - tip 6 | 7 | script: 8 | - go test -v ./... -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Conner Hewitt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Goque [![GoDoc](http://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/beeker1121/goque) [![License](http://img.shields.io/badge/license-mit-blue.svg)](https://raw.githubusercontent.com/beeker1121/goque/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/beeker1121/goque)](https://goreportcard.com/report/github.com/beeker1121/goque) [![Build Status](https://travis-ci.org/beeker1121/goque.svg?branch=master)](https://travis-ci.org/beeker1121/goque) 2 | 3 | Goque provides embedded, disk-based implementations of stack and queue data structures. 4 | 5 | Motivation for creating this project was the need for a persistent priority queue that remained performant while growing well beyond the available memory of a given machine. While there are many packages for Go offering queues, they all seem to be memory based and/or standalone solutions that are not embeddable within an application. 6 | 7 | Instead of using an in-memory heap structure to store data, everything is stored using the [Go port of LevelDB](https://github.com/syndtr/goleveldb). This results in very little memory being used no matter the size of the database, while read and write performance remains near constant. 8 | 9 | > [!CAUTION] 10 | > This project uses the Go port of LevelDB, which may not be production safe. The author of this project has used this package in production and has experienced extreme database slowdown. If used in production, please make sure to make regular backups and test recovery should the need arise. 11 | 12 | ## Features 13 | 14 | - Provides stack (LIFO), queue (FIFO), priority queue, and prefix queue structures. 15 | - Stacks and queues (but not priority queues or prefix queues) are interchangeable. 16 | - Persistent, disk-based. 17 | - Optimized for fast inserts and reads. 18 | - Goroutine safe. 19 | - Designed to work with large datasets outside of RAM/memory. 20 | 21 | ## Installation 22 | 23 | Fetch the package from GitHub: 24 | 25 | ```sh 26 | go get github.com/beeker1121/goque 27 | ``` 28 | 29 | Import to your project: 30 | 31 | ```go 32 | import "github.com/beeker1121/goque" 33 | ``` 34 | 35 | ## Usage 36 | 37 | ### Stack 38 | 39 | Stack is a LIFO (last in, first out) data structure. 40 | 41 | Create or open a stack: 42 | 43 | ```go 44 | s, err := goque.OpenStack("data_dir") 45 | ... 46 | defer s.Close() 47 | ``` 48 | 49 | Push an item: 50 | 51 | ```go 52 | item, err := s.Push([]byte("item value")) 53 | // or 54 | item, err := s.PushString("item value") 55 | // or 56 | item, err := s.PushObject(Object{X:1}) 57 | // or 58 | item, err := s.PushObjectAsJSON(Object{X:1}) 59 | ``` 60 | 61 | Pop an item: 62 | 63 | ```go 64 | item, err := s.Pop() 65 | ... 66 | fmt.Println(item.ID) // 1 67 | fmt.Println(item.Key) // [0 0 0 0 0 0 0 1] 68 | fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 69 | fmt.Println(item.ToString()) // item value 70 | 71 | // Decode to object. 72 | var obj Object 73 | err := item.ToObject(&obj) 74 | ... 75 | fmt.Printf("%+v\n", obj) // {X:1} 76 | 77 | // Decode to object from JSON. 78 | var obj Object 79 | err := item.ToObjectFromJSON(&obj) 80 | ... 81 | fmt.Printf("%+v\n", obj) // {X:1} 82 | ``` 83 | 84 | Peek the next stack item: 85 | 86 | ```go 87 | item, err := s.Peek() 88 | // or 89 | item, err := s.PeekByOffset(1) 90 | // or 91 | item, err := s.PeekByID(1) 92 | ``` 93 | 94 | Update an item in the stack: 95 | 96 | ```go 97 | item, err := s.Update(1, []byte("new value")) 98 | // or 99 | item, err := s.UpdateString(1, "new value") 100 | // or 101 | item, err := s.UpdateObject(1, Object{X:2}) 102 | // or 103 | item, err := s.UpdateObjectAsJSON(1, Object{X:2}) 104 | ``` 105 | 106 | Delete the stack and underlying database: 107 | 108 | ```go 109 | s.Drop() 110 | ``` 111 | 112 | ### Queue 113 | 114 | Queue is a FIFO (first in, first out) data structure. 115 | 116 | #### Methods 117 | 118 | Create or open a queue: 119 | 120 | ```go 121 | q, err := goque.OpenQueue("data_dir") 122 | ... 123 | defer q.Close() 124 | ``` 125 | 126 | Enqueue an item: 127 | 128 | ```go 129 | item, err := q.Enqueue([]byte("item value")) 130 | // or 131 | item, err := q.EnqueueString("item value") 132 | // or 133 | item, err := q.EnqueueObject(Object{X:1}) 134 | // or 135 | item, err := q.EnqueueObjectAsJSON(Object{X:1}) 136 | ``` 137 | 138 | Dequeue an item: 139 | 140 | ```go 141 | item, err := q.Dequeue() 142 | ... 143 | fmt.Println(item.ID) // 1 144 | fmt.Println(item.Key) // [0 0 0 0 0 0 0 1] 145 | fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 146 | fmt.Println(item.ToString()) // item value 147 | 148 | // Decode to object. 149 | var obj Object 150 | err := item.ToObject(&obj) 151 | ... 152 | fmt.Printf("%+v\n", obj) // {X:1} 153 | 154 | // Decode to object from JSON. 155 | var obj Object 156 | err := item.ToObjectFromJSON(&obj) 157 | ... 158 | fmt.Printf("%+v\n", obj) // {X:1} 159 | ``` 160 | 161 | Peek the next queue item: 162 | 163 | ```go 164 | item, err := q.Peek() 165 | // or 166 | item, err := q.PeekByOffset(1) 167 | // or 168 | item, err := q.PeekByID(1) 169 | ``` 170 | 171 | Update an item in the queue: 172 | 173 | ```go 174 | item, err := q.Update(1, []byte("new value")) 175 | // or 176 | item, err := q.UpdateString(1, "new value") 177 | // or 178 | item, err := q.UpdateObject(1, Object{X:2}) 179 | // or 180 | item, err := q.UpdateObjectAsJSON(1, Object{X:2}) 181 | ``` 182 | 183 | Delete the queue and underlying database: 184 | 185 | ```go 186 | q.Drop() 187 | ``` 188 | 189 | ### Priority Queue 190 | 191 | PriorityQueue is a FIFO (first in, first out) queue with priority levels. 192 | 193 | #### Methods 194 | 195 | Create or open a priority queue: 196 | 197 | ```go 198 | pq, err := goque.OpenPriorityQueue("data_dir", goque.ASC) 199 | ... 200 | defer pq.Close() 201 | ``` 202 | 203 | Enqueue an item: 204 | 205 | ```go 206 | item, err := pq.Enqueue(0, []byte("item value")) 207 | // or 208 | item, err := pq.EnqueueString(0, "item value") 209 | // or 210 | item, err := pq.EnqueueObject(0, Object{X:1}) 211 | // or 212 | item, err := pq.EnqueueObjectAsJSON(0, Object{X:1}) 213 | ``` 214 | 215 | Dequeue an item: 216 | 217 | ```go 218 | item, err := pq.Dequeue() 219 | // or 220 | item, err := pq.DequeueByPriority(0) 221 | ... 222 | fmt.Println(item.ID) // 1 223 | fmt.Println(item.Priority) // 0 224 | fmt.Println(item.Key) // [0 58 0 0 0 0 0 0 0 1] 225 | fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 226 | fmt.Println(item.ToString()) // item value 227 | 228 | // Decode to object. 229 | var obj Object 230 | err := item.ToObject(&obj) 231 | ... 232 | fmt.Printf("%+v\n", obj) // {X:1} 233 | 234 | // Decode to object from JSON. 235 | var obj Object 236 | err := item.ToObjectFromJSON(&obj) 237 | ... 238 | fmt.Printf("%+v\n", obj) // {X:1} 239 | ``` 240 | 241 | Peek the next priority queue item: 242 | 243 | ```go 244 | item, err := pq.Peek() 245 | // or 246 | item, err := pq.PeekByOffset(1) 247 | // or 248 | item, err := pq.PeekByPriorityID(0, 1) 249 | ``` 250 | 251 | Update an item in the priority queue: 252 | 253 | ```go 254 | item, err := pq.Update(0, 1, []byte("new value")) 255 | // or 256 | item, err := pq.UpdateString(0, 1, "new value") 257 | // or 258 | item, err := pq.UpdateObject(0, 1, Object{X:2}) 259 | // or 260 | item, err := pq.UpdateObjectAsJSON(0, 1, Object{X:2}) 261 | ``` 262 | 263 | Delete the priority queue and underlying database: 264 | 265 | ```go 266 | pq.Drop() 267 | ``` 268 | 269 | ### Prefix Queue 270 | 271 | PrefixQueue is a FIFO (first in, first out) data structure that separates each given prefix into its own queue. 272 | 273 | #### Methods 274 | 275 | Create or open a prefix queue: 276 | 277 | ```go 278 | pq, err := goque.OpenPrefixQueue("data_dir") 279 | ... 280 | defer pq.Close() 281 | ``` 282 | 283 | Enqueue an item: 284 | 285 | ```go 286 | item, err := pq.Enqueue([]byte("prefix"), []byte("item value")) 287 | // or 288 | item, err := pq.EnqueueString("prefix", "item value") 289 | // or 290 | item, err := pq.EnqueueObject([]byte("prefix"), Object{X:1}) 291 | // or 292 | item, err := pq.EnqueueObjectAsJSON([]byte("prefix"), Object{X:1}) 293 | ``` 294 | 295 | Dequeue an item: 296 | 297 | ```go 298 | item, err := pq.Dequeue([]byte("prefix")) 299 | // or 300 | item, err := pq.DequeueString("prefix") 301 | ... 302 | fmt.Println(item.ID) // 1 303 | fmt.Println(item.Key) // [112 114 101 102 105 120 0 0 0 0 0 0 0 0 1] 304 | fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 305 | fmt.Println(item.ToString()) // item value 306 | 307 | // Decode to object. 308 | var obj Object 309 | err := item.ToObject(&obj) 310 | ... 311 | fmt.Printf("%+v\n", obj) // {X:1} 312 | 313 | // Decode to object from JSON. 314 | var obj Object 315 | err := item.ToObjectFromJSON(&obj) 316 | ... 317 | fmt.Printf("%+v\n", obj) // {X:1} 318 | ``` 319 | 320 | Peek the next prefix queue item: 321 | 322 | ```go 323 | item, err := pq.Peek([]byte("prefix")) 324 | // or 325 | item, err := pq.PeekString("prefix") 326 | // or 327 | item, err := pq.PeekByID([]byte("prefix"), 1) 328 | // or 329 | item, err := pq.PeekByIDString("prefix", 1) 330 | ``` 331 | 332 | Update an item in the prefix queue: 333 | 334 | ```go 335 | item, err := pq.Update([]byte("prefix"), 1, []byte("new value")) 336 | // or 337 | item, err := pq.UpdateString("prefix", 1, "new value") 338 | // or 339 | item, err := pq.UpdateObject([]byte("prefix"), 1, Object{X:2}) 340 | // or 341 | item, err := pq.UpdateObjectAsJSON([]byte("prefix"), 1, Object{X:2}) 342 | ``` 343 | 344 | Delete the prefix queue and underlying database: 345 | 346 | ```go 347 | pq.Drop() 348 | ``` 349 | 350 | ## Benchmarks 351 | 352 | Benchmarks were ran on a Google Compute Engine n1-standard-1 machine (1 vCPU 3.75 GB of RAM): 353 | 354 | Go 1.6: 355 | 356 | ``` 357 | $ go test -bench=. 358 | PASS 359 | BenchmarkPriorityQueueEnqueue 200000 8104 ns/op 522 B/op 7 allocs/op 360 | BenchmarkPriorityQueueDequeue 200000 18622 ns/op 1166 B/op 17 allocs/op 361 | BenchmarkQueueEnqueue 200000 8049 ns/op 487 B/op 7 allocs/op 362 | BenchmarkQueueDequeue 200000 18970 ns/op 1089 B/op 17 allocs/op 363 | BenchmarkStackPush 200000 8145 ns/op 487 B/op 7 allocs/op 364 | BenchmarkStackPop 200000 18947 ns/op 1097 B/op 17 allocs/op 365 | ok github.com/beeker1121/goque 22.549s 366 | ``` 367 | 368 | Go 1.8: 369 | 370 | ``` 371 | $ go test -bench=. 372 | BenchmarkPrefixQueueEnqueue 20000 60553 ns/op 10532 B/op 242 allocs/op 373 | BenchmarkPrefixQueueDequeue 10000 100727 ns/op 18519 B/op 444 allocs/op 374 | BenchmarkPriorityQueueEnqueue 300000 4781 ns/op 557 B/op 9 allocs/op 375 | BenchmarkPriorityQueueDequeue 200000 11656 ns/op 1206 B/op 19 allocs/op 376 | BenchmarkQueueEnqueue 300000 4625 ns/op 513 B/op 9 allocs/op 377 | BenchmarkQueueDequeue 200000 11537 ns/op 1125 B/op 19 allocs/op 378 | BenchmarkStackPush 300000 4631 ns/op 513 B/op 9 allocs/op 379 | BenchmarkStackPop 200000 9629 ns/op 1116 B/op 19 allocs/op 380 | PASS 381 | ok github.com/beeker1121/goque 18.135s 382 | ``` 383 | 384 | ## Thanks 385 | 386 | **syndtr** ([https://github.com/syndtr](https://github.com/syndtr)) - LevelDB port to Go 387 | **bogdanovich** ([https://github.com/bogdanovich/siberite](https://github.com/bogdanovich/siberite)) - Server based queue for Go using LevelDB 388 | **connor4312** ([https://github.com/connor4312](https://github.com/connor4312)) - Recommending BoltDB/LevelDB, helping with structure 389 | **bwmarrin** ([https://github.com/bwmarrin](https://github.com/bwmarrin)) - Recommending BoltDB/LevelDB 390 | **zeroZshadow** ([https://github.com/zeroZshadow](https://github.com/zeroZshadow)) - Code review and optimization 391 | **nstafie** ([https://github.com/nstafie](https://github.com/nstafie)) - Help with structure -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package goque provides embedded, disk-based implementations of stack, queue, and priority queue data structures. 2 | // 3 | // Motivation for creating this project was the need for a persistent priority queue that remained performant while growing well beyond the available memory of a given machine. While there are many packages for Go offering queues, they all seem to be memory based and/or standalone solutions that are not embeddable within an application. 4 | // 5 | // Instead of using an in-memory heap structure to store data, everything is stored using the Go port of LevelDB (https://github.com/syndtr/goleveldb). This results in very little memory being used no matter the size of the database, while read and write performance remains near constant. 6 | // 7 | // See README.md or visit https://github.com/beeker1121/goque for more info. 8 | package goque 9 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | // ErrIncompatibleType is returned when the opener type is 9 | // incompatible with the stored Goque type. 10 | ErrIncompatibleType = errors.New("goque: Opener type is incompatible with stored Goque type") 11 | 12 | // ErrEmpty is returned when the stack or queue is empty. 13 | ErrEmpty = errors.New("goque: Stack or queue is empty") 14 | 15 | // ErrOutOfBounds is returned when the ID used to lookup an item 16 | // is outside of the range of the stack or queue. 17 | ErrOutOfBounds = errors.New("goque: ID used is outside range of stack or queue") 18 | 19 | // ErrDBClosed is returned when the Close function has already 20 | // been called, causing the stack or queue to close, as well as 21 | // its underlying database. 22 | ErrDBClosed = errors.New("goque: Database is closed") 23 | ) 24 | -------------------------------------------------------------------------------- /example_object_test.go: -------------------------------------------------------------------------------- 1 | package goque_test 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/beeker1121/goque" 7 | ) 8 | 9 | // ExampleObject demonstrates enqueuing a struct object. 10 | func Example_object() { 11 | // Open/create a queue. 12 | q, err := goque.OpenQueue("data_dir") 13 | if err != nil { 14 | fmt.Println(err) 15 | return 16 | } 17 | defer q.Close() 18 | 19 | // Define our struct. 20 | type object struct { 21 | X int 22 | Y int 23 | } 24 | 25 | // Enqueue an object. 26 | item, err := q.EnqueueObject(object{X: 1, Y: 2}) 27 | if err != nil { 28 | fmt.Println(err) 29 | return 30 | } 31 | 32 | fmt.Println(item.ID) // 1 33 | fmt.Println(item.Key) // [0 0 0 0 0 0 0 1] 34 | 35 | // Dequeue an item. 36 | deqItem, err := q.Dequeue() 37 | if err != nil { 38 | fmt.Println(err) 39 | return 40 | } 41 | 42 | // Create variable to hold our object in. 43 | var obj object 44 | 45 | // Decode item into our struct type. 46 | if err := deqItem.ToObject(&obj); err != nil { 47 | fmt.Println(err) 48 | return 49 | } 50 | 51 | fmt.Printf("%+v\n", obj) // {X:1 Y:2} 52 | 53 | // Delete the queue and its database. 54 | q.Drop() 55 | } 56 | -------------------------------------------------------------------------------- /example_prefix_queue_test.go: -------------------------------------------------------------------------------- 1 | package goque_test 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/beeker1121/goque" 7 | ) 8 | 9 | // ExamplePrefixQueue demonstrates the implementation of a Goque queue. 10 | func Example_prefixQueue() { 11 | // Open/create a prefix queue. 12 | pq, err := goque.OpenPrefixQueue("data_dir") 13 | if err != nil { 14 | fmt.Println(err) 15 | return 16 | } 17 | defer pq.Close() 18 | 19 | // Enqueue an item. 20 | item, err := pq.Enqueue([]byte("prefix"), []byte("item value")) 21 | if err != nil { 22 | fmt.Println(err) 23 | return 24 | } 25 | 26 | fmt.Println(item.ID) // 1 27 | fmt.Println(item.Key) // [112 114 101 102 105 120 0 0 0 0 0 0 0 0 1] 28 | fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 29 | fmt.Println(item.ToString()) // item value 30 | 31 | // Change the item value in the queue. 32 | item, err = pq.Update([]byte("prefix"), item.ID, []byte("new item value")) 33 | if err != nil { 34 | fmt.Println(err) 35 | return 36 | } 37 | 38 | fmt.Println(item.ToString()) // new item value 39 | 40 | // Dequeue the next item. 41 | deqItem, err := pq.Dequeue([]byte("prefix")) 42 | if err != nil { 43 | fmt.Println(err) 44 | return 45 | } 46 | 47 | fmt.Println(deqItem.ToString()) // new item value 48 | 49 | // Delete the queue and its database. 50 | pq.Drop() 51 | } 52 | -------------------------------------------------------------------------------- /example_priority_queue_test.go: -------------------------------------------------------------------------------- 1 | package goque_test 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/beeker1121/goque" 7 | ) 8 | 9 | // ExamplePriorityQueue demonstrates the implementation of a Goque queue. 10 | func Example_priorityQueue() { 11 | // Open/create a priority queue. 12 | pq, err := goque.OpenPriorityQueue("data_dir", goque.ASC) 13 | if err != nil { 14 | fmt.Println(err) 15 | return 16 | } 17 | defer pq.Close() 18 | 19 | // Enqueue the item. 20 | item, err := pq.Enqueue(0, []byte("item value")) 21 | if err != nil { 22 | fmt.Println(err) 23 | return 24 | } 25 | 26 | fmt.Println(item.ID) // 1 27 | fmt.Println(item.Priority) // 0 28 | fmt.Println(item.Key) // [0 58 0 0 0 0 0 0 0 1] 29 | fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 30 | fmt.Println(item.ToString()) // item value 31 | 32 | // Change the item value in the queue. 33 | item, err = pq.Update(item.Priority, item.ID, []byte("new item value")) 34 | if err != nil { 35 | fmt.Println(err) 36 | return 37 | } 38 | 39 | fmt.Println(item.ToString()) // new item value 40 | 41 | // Dequeue the next item. 42 | deqItem, err := pq.Dequeue() 43 | if err != nil { 44 | fmt.Println(err) 45 | return 46 | } 47 | 48 | fmt.Println(deqItem.ToString()) // new item value 49 | 50 | // Delete the queue and its database. 51 | pq.Drop() 52 | } 53 | -------------------------------------------------------------------------------- /example_queue_test.go: -------------------------------------------------------------------------------- 1 | package goque_test 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/beeker1121/goque" 7 | ) 8 | 9 | // ExampleQueue demonstrates the implementation of a Goque queue. 10 | func Example_queue() { 11 | // Open/create a queue. 12 | q, err := goque.OpenQueue("data_dir") 13 | if err != nil { 14 | fmt.Println(err) 15 | return 16 | } 17 | defer q.Close() 18 | 19 | // Enqueue an item. 20 | item, err := q.Enqueue([]byte("item value")) 21 | if err != nil { 22 | fmt.Println(err) 23 | return 24 | } 25 | 26 | fmt.Println(item.ID) // 1 27 | fmt.Println(item.Key) // [0 0 0 0 0 0 0 1] 28 | fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 29 | fmt.Println(item.ToString()) // item value 30 | 31 | // Change the item value in the queue. 32 | item, err = q.Update(item.ID, []byte("new item value")) 33 | if err != nil { 34 | fmt.Println(err) 35 | return 36 | } 37 | 38 | fmt.Println(item.ToString()) // new item value 39 | 40 | // Dequeue the next item. 41 | deqItem, err := q.Dequeue() 42 | if err != nil { 43 | fmt.Println(err) 44 | return 45 | } 46 | 47 | fmt.Println(deqItem.ToString()) // new item value 48 | 49 | // Delete the queue and its database. 50 | q.Drop() 51 | } 52 | -------------------------------------------------------------------------------- /example_stack_test.go: -------------------------------------------------------------------------------- 1 | package goque_test 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/beeker1121/goque" 7 | ) 8 | 9 | // ExampleStack demonstrates the implementation of a Goque stack. 10 | func Example_stack() { 11 | // Open/create a stack. 12 | s, err := goque.OpenStack("data_dir") 13 | if err != nil { 14 | fmt.Println(err) 15 | return 16 | } 17 | defer s.Close() 18 | 19 | // Push an item onto the stack. 20 | item, err := s.Push([]byte("item value")) 21 | if err != nil { 22 | fmt.Println(err) 23 | return 24 | } 25 | 26 | fmt.Println(item.ID) // 1 27 | fmt.Println(item.Key) // [0 0 0 0 0 0 0 1] 28 | fmt.Println(item.Value) // [105 116 101 109 32 118 97 108 117 101] 29 | fmt.Println(item.ToString()) // item value 30 | 31 | // Change the item value in the stack. 32 | item, err = s.Update(item.ID, []byte("new item value")) 33 | if err != nil { 34 | fmt.Println(err) 35 | return 36 | } 37 | 38 | fmt.Println(item.ToString()) // new item value 39 | 40 | // Pop an item off the stack. 41 | popItem, err := s.Pop() 42 | if err != nil { 43 | fmt.Println(err) 44 | return 45 | } 46 | 47 | fmt.Println(popItem.ToString()) // new item value 48 | 49 | // Delete the stack and its database. 50 | s.Drop() 51 | } 52 | -------------------------------------------------------------------------------- /file.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | // goqueType defines the type of Goque data structure used. 9 | type goqueType uint8 10 | 11 | // The possible Goque types, used to determine compatibility when 12 | // one stored type is trying to be opened by a different type. 13 | const ( 14 | goqueStack goqueType = iota 15 | goqueQueue 16 | goquePriorityQueue 17 | goquePrefixQueue 18 | ) 19 | 20 | // checkGoqueType checks if the type of Goque data structure 21 | // trying to be opened is compatible with the opener type. 22 | // 23 | // A file named 'GOQUE' within the data directory used by 24 | // the structure stores the structure type, using the constants 25 | // declared above. 26 | // 27 | // Stacks and Queues are 100% compatible with each other, while 28 | // a PriorityQueue is incompatible with both. 29 | // 30 | // Returns true if types are compatible and false if incompatible. 31 | func checkGoqueType(dataDir string, gt goqueType) (bool, error) { 32 | // Set the path to 'GOQUE' file. 33 | path := filepath.Join(dataDir, "GOQUE") 34 | 35 | // Read 'GOQUE' file for this directory. 36 | f, err := os.OpenFile(path, os.O_RDONLY, 0) 37 | if os.IsNotExist(err) { 38 | f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) 39 | if err != nil { 40 | return false, err 41 | } 42 | defer f.Close() 43 | 44 | // Create byte slice of goqueType. 45 | gtb := make([]byte, 1) 46 | gtb[0] = byte(gt) 47 | 48 | _, err = f.Write(gtb) 49 | if err != nil { 50 | return false, err 51 | } 52 | 53 | return true, nil 54 | } 55 | if err != nil { 56 | return false, err 57 | } 58 | defer f.Close() 59 | 60 | // Get the saved type from the file. 61 | fb := make([]byte, 1) 62 | _, err = f.Read(fb) 63 | if err != nil { 64 | return false, err 65 | } 66 | 67 | // Convert the file byte to its goqueType. 68 | filegt := goqueType(fb[0]) 69 | 70 | // Compare the types. 71 | if filegt == gt { 72 | return true, nil 73 | } else if filegt == goqueStack && gt == goqueQueue { 74 | return true, nil 75 | } else if filegt == goqueQueue && gt == goqueStack { 76 | return true, nil 77 | } 78 | 79 | return false, nil 80 | } 81 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/beeker1121/goque 2 | 3 | go 1.13 4 | 5 | require github.com/syndtr/goleveldb v1.0.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 2 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 3 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= 4 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 5 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 6 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 7 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 8 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 9 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 10 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 11 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 12 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 13 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 14 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 15 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 16 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 17 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 18 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 19 | -------------------------------------------------------------------------------- /item.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "encoding/gob" 7 | "encoding/json" 8 | ) 9 | 10 | // Item represents an entry in either a stack or queue. 11 | type Item struct { 12 | ID uint64 13 | Key []byte 14 | Value []byte 15 | } 16 | 17 | // ToString returns the item value as a string. 18 | func (i *Item) ToString() string { 19 | return string(i.Value) 20 | } 21 | 22 | // ToObject decodes the item value into the given value type using 23 | // encoding/gob. 24 | // 25 | // The value passed to this method should be a pointer to a variable 26 | // of the type you wish to decode into. The variable pointed to will 27 | // hold the decoded object. 28 | // 29 | // Objects containing pointers with zero values will decode to nil 30 | // when using this function. This is due to how the encoding/gob 31 | // package works. Because of this, you should only use this function 32 | // to decode simple types. 33 | func (i *Item) ToObject(value interface{}) error { 34 | buffer := bytes.NewBuffer(i.Value) 35 | dec := gob.NewDecoder(buffer) 36 | return dec.Decode(value) 37 | } 38 | 39 | // ToObjectFromJSON decodes the item value into the given value type 40 | // using encoding/json. 41 | // 42 | // The value passed to this method should be a pointer to a variable 43 | // of the type you wish to decode into. The variable pointed to will 44 | // hold the decoded object. 45 | func (i *Item) ToObjectFromJSON(value interface{}) error { 46 | return json.Unmarshal(i.Value, value) 47 | } 48 | 49 | // PriorityItem represents an entry in a priority queue. 50 | type PriorityItem struct { 51 | ID uint64 52 | Priority uint8 53 | Key []byte 54 | Value []byte 55 | } 56 | 57 | // ToString returns the priority item value as a string. 58 | func (pi *PriorityItem) ToString() string { 59 | return string(pi.Value) 60 | } 61 | 62 | // ToObject decodes the item value into the given value type using 63 | // encoding/gob. 64 | // 65 | // The value passed to this method should be a pointer to a variable 66 | // of the type you wish to decode into. The variable pointed to will 67 | // hold the decoded object. 68 | // 69 | // Objects containing pointers with zero values will decode to nil 70 | // when using this function. This is due to how the encoding/gob 71 | // package works. Because of this, you should only use this function 72 | // to decode simple types. 73 | func (pi *PriorityItem) ToObject(value interface{}) error { 74 | buffer := bytes.NewBuffer(pi.Value) 75 | dec := gob.NewDecoder(buffer) 76 | return dec.Decode(value) 77 | } 78 | 79 | // ToObjectFromJSON decodes the item value into the given value type 80 | // using encoding/json. 81 | // 82 | // The value passed to this method should be a pointer to a variable 83 | // of the type you wish to decode into. The variable pointed to will 84 | // hold the decoded object. 85 | func (pi *PriorityItem) ToObjectFromJSON(value interface{}) error { 86 | return json.Unmarshal(pi.Value, value) 87 | } 88 | 89 | // idToKey converts and returns the given ID to a key. 90 | func idToKey(id uint64) []byte { 91 | key := make([]byte, 8) 92 | binary.BigEndian.PutUint64(key, id) 93 | return key 94 | } 95 | 96 | // keyToID converts and returns the given key to an ID. 97 | func keyToID(key []byte) uint64 { 98 | return binary.BigEndian.Uint64(key) 99 | } 100 | -------------------------------------------------------------------------------- /prefix_queue.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "encoding/gob" 7 | "encoding/json" 8 | "os" 9 | "sync" 10 | 11 | "github.com/syndtr/goleveldb/leveldb" 12 | "github.com/syndtr/goleveldb/leveldb/errors" 13 | ) 14 | 15 | // prefixDelimiter defines the delimiter used to separate a prefix from an 16 | // item ID within the LevelDB database. We use the lowest possible value for 17 | // a single byte, 0x00 (null), as the delimiter. 18 | const prefixDelimiter byte = '\x00' 19 | 20 | // queue defines the unique queue for a prefix. 21 | type queue struct { 22 | Head uint64 23 | Tail uint64 24 | } 25 | 26 | // Length returns the total number of items in the queue. 27 | func (q *queue) Length() uint64 { 28 | return q.Tail - q.Head 29 | } 30 | 31 | // PrefixQueue is a standard FIFO (first in, first out) queue that separates 32 | // each given prefix into its own queue. 33 | type PrefixQueue struct { 34 | sync.RWMutex 35 | DataDir string 36 | db *leveldb.DB 37 | size uint64 38 | isOpen bool 39 | } 40 | 41 | // OpenPrefixQueue opens a prefix queue if one exists at the given directory. 42 | // If one does not already exist, a new prefix queue is created. 43 | func OpenPrefixQueue(dataDir string) (*PrefixQueue, error) { 44 | var err error 45 | 46 | // Create a new Queue. 47 | pq := &PrefixQueue{ 48 | DataDir: dataDir, 49 | db: &leveldb.DB{}, 50 | isOpen: false, 51 | } 52 | 53 | // Open database for the prefix queue. 54 | pq.db, err = leveldb.OpenFile(dataDir, nil) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | // Check if this Goque type can open the requested data directory. 60 | ok, err := checkGoqueType(dataDir, goquePrefixQueue) 61 | if err != nil { 62 | return nil, err 63 | } 64 | if !ok { 65 | return nil, ErrIncompatibleType 66 | } 67 | 68 | // Set isOpen and return. 69 | pq.isOpen = true 70 | return pq, pq.init() 71 | } 72 | 73 | // Enqueue adds an item to the queue. 74 | func (pq *PrefixQueue) Enqueue(prefix, value []byte) (*Item, error) { 75 | pq.Lock() 76 | defer pq.Unlock() 77 | 78 | // Check if queue is closed. 79 | if !pq.isOpen { 80 | return nil, ErrDBClosed 81 | } 82 | 83 | // Get the queue for this prefix. 84 | q, err := pq.getOrCreateQueue(prefix) 85 | if err != nil { 86 | return nil, err 87 | } 88 | 89 | // Create new Item. 90 | item := &Item{ 91 | ID: q.Tail + 1, 92 | Key: generateKeyPrefixID(prefix, q.Tail+1), 93 | Value: value, 94 | } 95 | 96 | // Add it to the queue. 97 | if err := pq.db.Put(item.Key, item.Value, nil); err != nil { 98 | return nil, err 99 | } 100 | 101 | // Increment tail position and prefix queue size. 102 | q.Tail++ 103 | pq.size++ 104 | 105 | // Save the queue. 106 | if err := pq.saveQueue(prefix, q); err != nil { 107 | return nil, err 108 | } 109 | 110 | // Save main prefix queue data. 111 | if err := pq.save(); err != nil { 112 | return nil, err 113 | } 114 | 115 | return item, nil 116 | } 117 | 118 | // EnqueueString is a helper function for Enqueue that accepts the prefix and 119 | // value as a string rather than a byte slice. 120 | func (pq *PrefixQueue) EnqueueString(prefix, value string) (*Item, error) { 121 | return pq.Enqueue([]byte(prefix), []byte(value)) 122 | } 123 | 124 | // EnqueueObject is a helper function for Enqueue that accepts any 125 | // value type, which is then encoded into a byte slice using 126 | // encoding/gob. 127 | // 128 | // Objects containing pointers with zero values will decode to nil 129 | // when using this function. This is due to how the encoding/gob 130 | // package works. Because of this, you should only use this function 131 | // to encode simple types. 132 | func (pq *PrefixQueue) EnqueueObject(prefix []byte, value interface{}) (*Item, error) { 133 | var buffer bytes.Buffer 134 | enc := gob.NewEncoder(&buffer) 135 | if err := enc.Encode(value); err != nil { 136 | return nil, err 137 | } 138 | 139 | return pq.Enqueue(prefix, buffer.Bytes()) 140 | } 141 | 142 | // EnqueueObjectAsJSON is a helper function for Enqueue that accepts 143 | // any value type, which is then encoded into a JSON byte slice using 144 | // encoding/json. 145 | // 146 | // Use this function to handle encoding of complex types. 147 | func (pq *PrefixQueue) EnqueueObjectAsJSON(prefix []byte, value interface{}) (*Item, error) { 148 | jsonBytes, err := json.Marshal(value) 149 | if err != nil { 150 | return nil, err 151 | } 152 | 153 | return pq.Enqueue(prefix, jsonBytes) 154 | } 155 | 156 | // Dequeue removes the next item in the prefix queue and returns it. 157 | func (pq *PrefixQueue) Dequeue(prefix []byte) (*Item, error) { 158 | pq.Lock() 159 | defer pq.Unlock() 160 | 161 | // Check if queue is closed. 162 | if !pq.isOpen { 163 | return nil, ErrDBClosed 164 | } 165 | 166 | // Get the queue for this prefix. 167 | q, err := pq.getQueue(prefix) 168 | if err != nil { 169 | return nil, err 170 | } 171 | 172 | // Try to get the next item in the queue. 173 | item, err := pq.getItemByPrefixID(prefix, q.Head+1) 174 | if err != nil { 175 | return nil, err 176 | } 177 | 178 | // Remove this item from the queue. 179 | if err := pq.db.Delete(item.Key, nil); err != nil { 180 | return nil, err 181 | } 182 | 183 | // Increment head position and decrement prefix queue size. 184 | q.Head++ 185 | pq.size-- 186 | 187 | // Save the queue. 188 | if err := pq.saveQueue(prefix, q); err != nil { 189 | return nil, err 190 | } 191 | 192 | // Save main prefix queue data. 193 | if err := pq.save(); err != nil { 194 | return nil, err 195 | } 196 | 197 | return item, nil 198 | } 199 | 200 | // DequeueString is a helper function for Dequeue that accepts the prefix as a 201 | // string rather than a byte slice. 202 | func (pq *PrefixQueue) DequeueString(prefix string) (*Item, error) { 203 | return pq.Dequeue([]byte(prefix)) 204 | } 205 | 206 | // Peek returns the next item in the given queue without removing it. 207 | func (pq *PrefixQueue) Peek(prefix []byte) (*Item, error) { 208 | pq.RLock() 209 | defer pq.RUnlock() 210 | 211 | // Check if queue is closed. 212 | if !pq.isOpen { 213 | return nil, ErrDBClosed 214 | } 215 | 216 | // Get the queue for this prefix. 217 | q, err := pq.getQueue(prefix) 218 | if err != nil { 219 | return nil, err 220 | } 221 | 222 | return pq.getItemByPrefixID(prefix, q.Head+1) 223 | } 224 | 225 | // PeekString is a helper function for Peek that accepts the prefix as a 226 | // string rather than a byte slice. 227 | func (pq *PrefixQueue) PeekString(prefix string) (*Item, error) { 228 | return pq.Peek([]byte(prefix)) 229 | } 230 | 231 | // PeekByID returns the item with the given ID without removing it. 232 | func (pq *PrefixQueue) PeekByID(prefix []byte, id uint64) (*Item, error) { 233 | pq.RLock() 234 | defer pq.RUnlock() 235 | 236 | // Check if queue is closed. 237 | if !pq.isOpen { 238 | return nil, ErrDBClosed 239 | } 240 | 241 | return pq.getItemByPrefixID(prefix, id) 242 | } 243 | 244 | // PeekByIDString is a helper function for Peek that accepts the prefix as a 245 | // string rather than a byte slice. 246 | func (pq *PrefixQueue) PeekByIDString(prefix string, id uint64) (*Item, error) { 247 | return pq.PeekByID([]byte(prefix), id) 248 | } 249 | 250 | // Update updates an item in the given queue without changing its position. 251 | func (pq *PrefixQueue) Update(prefix []byte, id uint64, newValue []byte) (*Item, error) { 252 | pq.Lock() 253 | defer pq.Unlock() 254 | 255 | // Check if queue is closed. 256 | if !pq.isOpen { 257 | return nil, ErrDBClosed 258 | } 259 | 260 | // Get the queue for this prefix. 261 | q, err := pq.getQueue(prefix) 262 | if err != nil { 263 | return nil, err 264 | } 265 | 266 | // Check if item exists in queue. 267 | if id <= q.Head || id > q.Tail { 268 | return nil, ErrOutOfBounds 269 | } 270 | 271 | // Create new Item. 272 | item := &Item{ 273 | ID: id, 274 | Key: generateKeyPrefixID(prefix, id), 275 | Value: newValue, 276 | } 277 | 278 | // Update this item in the queue. 279 | if err := pq.db.Put(item.Key, item.Value, nil); err != nil { 280 | return nil, err 281 | } 282 | 283 | return item, nil 284 | } 285 | 286 | // UpdateString is a helper function for Update that accepts the prefix and 287 | // value as a string rather than a byte slice. 288 | func (pq *PrefixQueue) UpdateString(prefix string, id uint64, value string) (*Item, error) { 289 | return pq.Update([]byte(prefix), id, []byte(value)) 290 | } 291 | 292 | // UpdateObject is a helper function for Update that accepts any 293 | // value type, which is then encoded into a byte slice using 294 | // encoding/gob. 295 | // 296 | // Objects containing pointers with zero values will decode to nil 297 | // when using this function. This is due to how the encoding/gob 298 | // package works. Because of this, you should only use this function 299 | // to encode simple types. 300 | func (pq *PrefixQueue) UpdateObject(prefix []byte, id uint64, newValue interface{}) (*Item, error) { 301 | var buffer bytes.Buffer 302 | enc := gob.NewEncoder(&buffer) 303 | if err := enc.Encode(newValue); err != nil { 304 | return nil, err 305 | } 306 | return pq.Update(prefix, id, buffer.Bytes()) 307 | } 308 | 309 | // UpdateObjectAsJSON is a helper function for Update that accepts 310 | // any value type, which is then encoded into a JSON byte slice using 311 | // encoding/json. 312 | // 313 | // Use this function to handle encoding of complex types. 314 | func (pq *PrefixQueue) UpdateObjectAsJSON(prefix []byte, id uint64, newValue interface{}) (*Item, error) { 315 | jsonBytes, err := json.Marshal(newValue) 316 | if err != nil { 317 | return nil, err 318 | } 319 | 320 | return pq.Update(prefix, id, jsonBytes) 321 | } 322 | 323 | // Length returns the total number of items in the prefix queue. 324 | func (pq *PrefixQueue) Length() uint64 { 325 | return pq.size 326 | } 327 | 328 | // Close closes the LevelDB database of the prefix queue. 329 | func (pq *PrefixQueue) Close() error { 330 | pq.Lock() 331 | defer pq.Unlock() 332 | 333 | // Check if queue is already closed. 334 | if !pq.isOpen { 335 | return nil 336 | } 337 | 338 | // Close the LevelDB database. 339 | if err := pq.db.Close(); err != nil { 340 | return err 341 | } 342 | 343 | // Reset size and set isOpen to false. 344 | pq.size = 0 345 | pq.isOpen = false 346 | 347 | return nil 348 | } 349 | 350 | // Drop closes and deletes the LevelDB database of the prefix queue. 351 | func (pq *PrefixQueue) Drop() error { 352 | if err := pq.Close(); err != nil { 353 | return err 354 | } 355 | 356 | return os.RemoveAll(pq.DataDir) 357 | } 358 | 359 | // getQueue gets the unique queue for the given prefix. 360 | func (pq *PrefixQueue) getQueue(prefix []byte) (*queue, error) { 361 | // Try to get the queue gob value. 362 | qval, err := pq.db.Get(generateKeyPrefixData(prefix), nil) 363 | if err == errors.ErrNotFound { 364 | return nil, ErrEmpty 365 | } else if err != nil { 366 | return nil, err 367 | } 368 | 369 | // Decode gob to our queue type. 370 | q := &queue{} 371 | buffer := bytes.NewBuffer(qval) 372 | dec := gob.NewDecoder(buffer) 373 | return q, dec.Decode(q) 374 | } 375 | 376 | // getOrCreateQueue gets the unique queue for the given prefix. If one does not 377 | // already exist, a new queue is created. 378 | func (pq *PrefixQueue) getOrCreateQueue(prefix []byte) (*queue, error) { 379 | // Try to get the queue gob value. 380 | qval, err := pq.db.Get(generateKeyPrefixData(prefix), nil) 381 | if err == errors.ErrNotFound { 382 | return &queue{}, nil 383 | } else if err != nil { 384 | return nil, err 385 | } 386 | 387 | // Decode gob to our queue type. 388 | q := &queue{} 389 | buffer := bytes.NewBuffer(qval) 390 | dec := gob.NewDecoder(buffer) 391 | return q, dec.Decode(q) 392 | } 393 | 394 | // savePrefixQueue saves the given queue for the given prefix. 395 | func (pq *PrefixQueue) saveQueue(prefix []byte, q *queue) error { 396 | // Encode the queue using gob. 397 | var buffer bytes.Buffer 398 | enc := gob.NewEncoder(&buffer) 399 | if err := enc.Encode(q); err != nil { 400 | return err 401 | } 402 | 403 | // Save it to the database. 404 | return pq.db.Put(generateKeyPrefixData(prefix), buffer.Bytes(), nil) 405 | } 406 | 407 | // save saves the main prefix queue data. 408 | func (pq *PrefixQueue) save() error { 409 | val := make([]byte, 8) 410 | binary.BigEndian.PutUint64(val, pq.size) 411 | return pq.db.Put(pq.getDataKey(), val, nil) 412 | } 413 | 414 | // getDataKey generates the main prefix queue data key. 415 | func (pq *PrefixQueue) getDataKey() []byte { 416 | var key []byte 417 | key = append(key, prefixDelimiter) 418 | return append(key, []byte(":main_data")...) 419 | } 420 | 421 | // getItemByPrefixID returns an item, if found, for the given prefix and ID. 422 | func (pq *PrefixQueue) getItemByPrefixID(prefix []byte, id uint64) (*Item, error) { 423 | // Check if empty. 424 | if pq.size == 0 { 425 | return nil, ErrEmpty 426 | } 427 | 428 | // Get the queue for this prefix. 429 | q, err := pq.getQueue(prefix) 430 | if err != nil { 431 | return nil, err 432 | } 433 | 434 | // Check if out of bounds. 435 | if id <= q.Head || id > q.Tail { 436 | return nil, ErrOutOfBounds 437 | } 438 | 439 | // Get item from database. 440 | item := &Item{ 441 | ID: id, 442 | Key: generateKeyPrefixID(prefix, id), 443 | } 444 | 445 | if item.Value, err = pq.db.Get(item.Key, nil); err != nil { 446 | return nil, err 447 | } 448 | 449 | return item, nil 450 | } 451 | 452 | // init initializes the prefix queue data. 453 | func (pq *PrefixQueue) init() error { 454 | // Get the main prefix queue data. 455 | val, err := pq.db.Get(pq.getDataKey(), nil) 456 | if err == errors.ErrNotFound { 457 | return nil 458 | } else if err != nil { 459 | return err 460 | } 461 | 462 | pq.size = binary.BigEndian.Uint64(val) 463 | return nil 464 | } 465 | 466 | // generateKeyPrefixData generates a data key using the given prefix. This key 467 | // should be used to get the stored queue struct for the given prefix. 468 | func generateKeyPrefixData(prefix []byte) []byte { 469 | return append(prefix, []byte(":data")...) 470 | } 471 | 472 | // generateKeyPrefixID generates a key using the given prefix and ID. 473 | func generateKeyPrefixID(prefix []byte, id uint64) []byte { 474 | // Handle the prefix. 475 | key := append(prefix, prefixDelimiter) 476 | 477 | // Handle the item ID. 478 | key = append(key, idToKey(id)...) 479 | 480 | return key 481 | } 482 | -------------------------------------------------------------------------------- /prefix_queue_test.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestPrefixQueueClose(t *testing.T) { 11 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 12 | pq, err := OpenPrefixQueue(file) 13 | if err != nil { 14 | t.Error(err) 15 | } 16 | defer pq.Drop() 17 | 18 | if _, err = pq.EnqueueString("prefix", "value"); err != nil { 19 | t.Error(err) 20 | } 21 | 22 | if pq.Length() != 1 { 23 | t.Errorf("Expected queue length of 1, got %d", pq.Length()) 24 | } 25 | 26 | pq.Close() 27 | 28 | if _, err = pq.DequeueString("prefix"); err != ErrDBClosed { 29 | t.Errorf("Expected to get database closed error, got %s", err.Error()) 30 | } 31 | 32 | if pq.Length() != 0 { 33 | t.Errorf("Expected queue length of 0, got %d", pq.Length()) 34 | } 35 | } 36 | 37 | func TestPrefixQueueDrop(t *testing.T) { 38 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 39 | pq, err := OpenPrefixQueue(file) 40 | if err != nil { 41 | t.Error(err) 42 | } 43 | 44 | if _, err = os.Stat(file); os.IsNotExist(err) { 45 | t.Error(err) 46 | } 47 | 48 | pq.Drop() 49 | 50 | if _, err = os.Stat(file); err == nil { 51 | t.Error("Expected directory for test database to have been deleted") 52 | } 53 | } 54 | 55 | func TestPrefixQueueIncompatibleType(t *testing.T) { 56 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 57 | prq, err := OpenPriorityQueue(file, ASC) 58 | if err != nil { 59 | t.Error(err) 60 | } 61 | defer prq.Drop() 62 | prq.Close() 63 | 64 | if _, err = OpenPrefixQueue(file); err != ErrIncompatibleType { 65 | t.Error("Expected priority queue to return ErrIncompatibleTypes when opening goquePriorityQueue") 66 | } 67 | } 68 | 69 | func TestPrefixQueueEnqueue(t *testing.T) { 70 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 71 | pq, err := OpenPrefixQueue(file) 72 | if err != nil { 73 | t.Error(err) 74 | } 75 | defer pq.Drop() 76 | 77 | for i := 1; i <= 10; i++ { 78 | if _, err = pq.EnqueueString("prefix", fmt.Sprintf("value for item %d", i)); err != nil { 79 | t.Error(err) 80 | } 81 | } 82 | 83 | if pq.Length() != 10 { 84 | t.Errorf("Expected queue size of 10, got %d", pq.Length()) 85 | } 86 | } 87 | 88 | func TestPrefixQueueDequeue(t *testing.T) { 89 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 90 | pq, err := OpenPrefixQueue(file) 91 | if err != nil { 92 | t.Error(err) 93 | } 94 | defer pq.Drop() 95 | 96 | for i := 1; i <= 10; i++ { 97 | if _, err = pq.EnqueueString("prefix", fmt.Sprintf("value for item %d", i)); err != nil { 98 | t.Error(err) 99 | } 100 | } 101 | 102 | if pq.Length() != 10 { 103 | t.Errorf("Expected queue length of 10, got %d", pq.Length()) 104 | } 105 | 106 | deqItem, err := pq.DequeueString("prefix") 107 | if err != nil { 108 | t.Error(err) 109 | } 110 | 111 | if pq.Length() != 9 { 112 | t.Errorf("Expected queue length of 9, got %d", pq.Length()) 113 | } 114 | 115 | compStr := "value for item 1" 116 | 117 | if deqItem.ToString() != compStr { 118 | t.Errorf("Expected string to be '%s', got '%s'", compStr, deqItem.ToString()) 119 | } 120 | } 121 | 122 | func TestPrefixQueueEncodeDecodePointerJSON(t *testing.T) { 123 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 124 | pq, err := OpenPrefixQueue(file) 125 | if err != nil { 126 | t.Error(err) 127 | } 128 | defer pq.Drop() 129 | 130 | type subObject struct { 131 | Value *int 132 | } 133 | 134 | type object struct { 135 | Value int 136 | SubObject subObject 137 | } 138 | 139 | val := 0 140 | obj := object{ 141 | Value: 0, 142 | SubObject: subObject{ 143 | Value: &val, 144 | }, 145 | } 146 | 147 | if _, err = pq.EnqueueObjectAsJSON([]byte("prefix"), obj); err != nil { 148 | t.Error(err) 149 | } 150 | 151 | item, err := pq.Dequeue([]byte("prefix")) 152 | if err != nil { 153 | t.Error(err) 154 | } 155 | 156 | var itemObj object 157 | if err := item.ToObjectFromJSON(&itemObj); err != nil { 158 | t.Error(err) 159 | } 160 | 161 | if *itemObj.SubObject.Value != 0 { 162 | t.Errorf("Expected object subobject value to be '0', got '%v'", *itemObj.SubObject.Value) 163 | } 164 | } 165 | 166 | func TestPrefixQueuePeek(t *testing.T) { 167 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 168 | pq, err := OpenPrefixQueue(file) 169 | if err != nil { 170 | t.Error(err) 171 | } 172 | defer pq.Drop() 173 | 174 | compStr := "value for item" 175 | 176 | if _, err = pq.EnqueueString("prefix", compStr); err != nil { 177 | t.Error(err) 178 | } 179 | 180 | peekItem, err := pq.PeekString("prefix") 181 | if err != nil { 182 | t.Error(err) 183 | } 184 | 185 | if peekItem.ToString() != compStr { 186 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 187 | } 188 | 189 | if pq.Length() != 1 { 190 | t.Errorf("Expected queue length of 1, got %d", pq.Length()) 191 | } 192 | } 193 | 194 | func TestPrefixQueuePeekByID(t *testing.T) { 195 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 196 | pq, err := OpenPrefixQueue(file) 197 | if err != nil { 198 | t.Error(err) 199 | } 200 | defer pq.Drop() 201 | 202 | for i := 1; i <= 10; i++ { 203 | if _, err = pq.EnqueueString("prefix", fmt.Sprintf("value for item %d", i)); err != nil { 204 | t.Error(err) 205 | } 206 | } 207 | 208 | compStr := "value for item 3" 209 | 210 | peekItem, err := pq.PeekByIDString("prefix", 3) 211 | if err != nil { 212 | t.Error(err) 213 | } 214 | 215 | if peekItem.ToString() != compStr { 216 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 217 | } 218 | 219 | if pq.Length() != 10 { 220 | t.Errorf("Expected queue length of 10, got %d", pq.Length()) 221 | } 222 | } 223 | 224 | func TestPrefixQueueUpdate(t *testing.T) { 225 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 226 | pq, err := OpenPrefixQueue(file) 227 | if err != nil { 228 | t.Error(err) 229 | } 230 | defer pq.Drop() 231 | 232 | for i := 1; i <= 10; i++ { 233 | if _, err = pq.EnqueueString("prefix", fmt.Sprintf("value for item %d", i)); err != nil { 234 | t.Error(err) 235 | } 236 | } 237 | 238 | item, err := pq.PeekByIDString("prefix", 3) 239 | if err != nil { 240 | t.Error(err) 241 | } 242 | 243 | oldCompStr := "value for item 3" 244 | newCompStr := "new value for item 3" 245 | 246 | if item.ToString() != oldCompStr { 247 | t.Errorf("Expected string to be '%s', got '%s'", oldCompStr, item.ToString()) 248 | } 249 | 250 | updatedItem, err := pq.UpdateString("prefix", item.ID, newCompStr) 251 | if err != nil { 252 | t.Error(err) 253 | } 254 | 255 | if updatedItem.ToString() != newCompStr { 256 | t.Errorf("Expected current item value to be '%s', got '%s'", newCompStr, item.ToString()) 257 | } 258 | 259 | newItem, err := pq.PeekByIDString("prefix", 3) 260 | if err != nil { 261 | t.Error(err) 262 | } 263 | 264 | if newItem.ToString() != newCompStr { 265 | t.Errorf("Expected new item value to be '%s', got '%s'", newCompStr, item.ToString()) 266 | } 267 | } 268 | 269 | func TestPrefixQueueUpdateString(t *testing.T) { 270 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 271 | pq, err := OpenPrefixQueue(file) 272 | if err != nil { 273 | t.Error(err) 274 | } 275 | defer pq.Drop() 276 | 277 | for i := 1; i <= 10; i++ { 278 | if _, err = pq.EnqueueString("prefix", fmt.Sprintf("value for item %d", i)); err != nil { 279 | t.Error(err) 280 | } 281 | } 282 | 283 | item, err := pq.PeekByIDString("prefix", 3) 284 | if err != nil { 285 | t.Error(err) 286 | } 287 | 288 | oldCompStr := "value for item 3" 289 | newCompStr := "new value for item 3" 290 | 291 | if item.ToString() != oldCompStr { 292 | t.Errorf("Expected string to be '%s', got '%s'", oldCompStr, item.ToString()) 293 | } 294 | 295 | updatedItem, err := pq.UpdateString("prefix", item.ID, newCompStr) 296 | if err != nil { 297 | t.Error(err) 298 | } 299 | 300 | if updatedItem.ToString() != newCompStr { 301 | t.Errorf("Expected current item value to be '%s', got '%s'", newCompStr, item.ToString()) 302 | } 303 | 304 | newItem, err := pq.PeekByIDString("prefix", 3) 305 | if err != nil { 306 | t.Error(err) 307 | } 308 | 309 | if newItem.ToString() != newCompStr { 310 | t.Errorf("Expected new item value to be '%s', got '%s'", newCompStr, item.ToString()) 311 | } 312 | } 313 | 314 | func TestPrefixQueueUpdateObject(t *testing.T) { 315 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 316 | pq, err := OpenPrefixQueue(file) 317 | if err != nil { 318 | t.Error(err) 319 | } 320 | defer pq.Drop() 321 | 322 | type object struct { 323 | Value int 324 | } 325 | 326 | for i := 1; i <= 10; i++ { 327 | if _, err = pq.EnqueueObject([]byte("prefix"), object{i}); err != nil { 328 | t.Error(err) 329 | } 330 | } 331 | 332 | item, err := pq.PeekByIDString("prefix", 3) 333 | if err != nil { 334 | t.Error(err) 335 | } 336 | 337 | oldCompObj := object{3} 338 | newCompObj := object{33} 339 | 340 | var obj object 341 | if err := item.ToObject(&obj); err != nil { 342 | t.Error(err) 343 | } 344 | 345 | if obj != oldCompObj { 346 | t.Errorf("Expected object to be '%+v', got '%+v'", oldCompObj, obj) 347 | } 348 | 349 | updatedItem, err := pq.UpdateObject([]byte("prefix"), item.ID, newCompObj) 350 | if err != nil { 351 | t.Error(err) 352 | } 353 | 354 | if err := updatedItem.ToObject(&obj); err != nil { 355 | t.Error(err) 356 | } 357 | 358 | if obj != newCompObj { 359 | t.Errorf("Expected current object to be '%+v', got '%+v'", newCompObj, obj) 360 | } 361 | 362 | newItem, err := pq.PeekByIDString("prefix", 3) 363 | if err != nil { 364 | t.Error(err) 365 | } 366 | 367 | if err := newItem.ToObject(&obj); err != nil { 368 | t.Error(err) 369 | } 370 | 371 | if obj != newCompObj { 372 | t.Errorf("Expected new object to be '%+v', got '%+v'", newCompObj, obj) 373 | } 374 | } 375 | 376 | func TestPrefixQueueUpdateObjectAsJSON(t *testing.T) { 377 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 378 | pq, err := OpenPrefixQueue(file) 379 | if err != nil { 380 | t.Error(err) 381 | } 382 | defer pq.Drop() 383 | 384 | type subObject struct { 385 | Value *int 386 | } 387 | 388 | type object struct { 389 | Value int 390 | SubObject subObject 391 | } 392 | 393 | for i := 1; i <= 10; i++ { 394 | obj := object{ 395 | Value: i, 396 | SubObject: subObject{ 397 | Value: &i, 398 | }, 399 | } 400 | 401 | if _, err = pq.EnqueueObjectAsJSON([]byte("prefix"), obj); err != nil { 402 | t.Error(err) 403 | } 404 | } 405 | 406 | item, err := pq.PeekByIDString("prefix", 3) 407 | if err != nil { 408 | t.Error(err) 409 | } 410 | 411 | oldCompObjVal := 3 412 | oldCompObj := object{ 413 | Value: 3, 414 | SubObject: subObject{ 415 | Value: &oldCompObjVal, 416 | }, 417 | } 418 | newCompObjVal := 33 419 | newCompObj := object{ 420 | Value: 33, 421 | SubObject: subObject{ 422 | Value: &newCompObjVal, 423 | }, 424 | } 425 | 426 | var obj object 427 | if err := item.ToObjectFromJSON(&obj); err != nil { 428 | t.Error(err) 429 | } 430 | 431 | if *obj.SubObject.Value != *oldCompObj.SubObject.Value { 432 | t.Errorf("Expected object subobject value to be '%+v', got '%+v'", *oldCompObj.SubObject.Value, *obj.SubObject.Value) 433 | } 434 | 435 | updatedItem, err := pq.UpdateObjectAsJSON([]byte("prefix"), item.ID, newCompObj) 436 | if err != nil { 437 | t.Error(err) 438 | } 439 | 440 | if err := updatedItem.ToObjectFromJSON(&obj); err != nil { 441 | t.Error(err) 442 | } 443 | 444 | if *obj.SubObject.Value != *newCompObj.SubObject.Value { 445 | t.Errorf("Expected current object subobject value to be '%+v', got '%+v'", *newCompObj.SubObject.Value, *obj.SubObject.Value) 446 | } 447 | 448 | newItem, err := pq.PeekByIDString("prefix", 3) 449 | if err != nil { 450 | t.Error(err) 451 | } 452 | 453 | if err := newItem.ToObjectFromJSON(&obj); err != nil { 454 | t.Error(err) 455 | } 456 | 457 | if *obj.SubObject.Value != *newCompObj.SubObject.Value { 458 | t.Errorf("Expected current object subobject value to be '%+v', got '%+v'", *newCompObj.SubObject.Value, *obj.SubObject.Value) 459 | } 460 | } 461 | 462 | func TestPrefixQueueUpdateOutOfBounds(t *testing.T) { 463 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 464 | pq, err := OpenPrefixQueue(file) 465 | if err != nil { 466 | t.Error(err) 467 | } 468 | defer pq.Drop() 469 | 470 | for i := 1; i <= 10; i++ { 471 | if _, err = pq.EnqueueString("prefix", fmt.Sprintf("value for item %d", i)); err != nil { 472 | t.Error(err) 473 | } 474 | } 475 | 476 | if pq.Length() != 10 { 477 | t.Errorf("Expected queue length of 10, got %d", pq.Length()) 478 | } 479 | 480 | deqItem, err := pq.DequeueString("prefix") 481 | if err != nil { 482 | t.Error(err) 483 | } 484 | 485 | if pq.Length() != 9 { 486 | t.Errorf("Expected queue length of 9, got %d", pq.Length()) 487 | } 488 | 489 | if _, err = pq.Update([]byte("prefix"), deqItem.ID, []byte(`new value`)); err != ErrOutOfBounds { 490 | t.Errorf("Expected to get queue out of bounds error, got %s", err.Error()) 491 | } 492 | 493 | if _, err = pq.Update([]byte("prefix"), deqItem.ID+1, []byte(`new value`)); err != nil { 494 | t.Error(err) 495 | } 496 | } 497 | 498 | func TestPrefixQueueEmpty(t *testing.T) { 499 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 500 | pq, err := OpenPrefixQueue(file) 501 | if err != nil { 502 | t.Error(err) 503 | } 504 | defer pq.Drop() 505 | 506 | _, err = pq.EnqueueString("prefix", "value for item") 507 | if err != nil { 508 | t.Error(err) 509 | } 510 | 511 | _, err = pq.DequeueString("prefix") 512 | if err != nil { 513 | t.Error(err) 514 | } 515 | 516 | _, err = pq.DequeueString("prefix") 517 | if err != ErrEmpty { 518 | t.Errorf("Expected to get empty error, got %s", err.Error()) 519 | } 520 | } 521 | 522 | func TestPrefixQueueOutOfBounds(t *testing.T) { 523 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 524 | pq, err := OpenPrefixQueue(file) 525 | if err != nil { 526 | t.Error(err) 527 | } 528 | defer pq.Drop() 529 | 530 | _, err = pq.EnqueueString("prefix", "value for item") 531 | if err != nil { 532 | t.Error(err) 533 | } 534 | 535 | _, err = pq.PeekByIDString("prefix", 2) 536 | if err != ErrOutOfBounds { 537 | t.Errorf("Expected to get queue out of bounds error, got %s", err.Error()) 538 | } 539 | } 540 | 541 | func BenchmarkPrefixQueueEnqueue(b *testing.B) { 542 | // Open test database 543 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 544 | pq, err := OpenPrefixQueue(file) 545 | if err != nil { 546 | b.Error(err) 547 | } 548 | defer pq.Drop() 549 | 550 | b.ResetTimer() 551 | b.ReportAllocs() 552 | 553 | for n := 0; n < b.N; n++ { 554 | _, _ = pq.Enqueue([]byte("prefix"), []byte("value")) 555 | } 556 | } 557 | 558 | func BenchmarkPrefixQueueDequeue(b *testing.B) { 559 | // Open test database 560 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 561 | pq, err := OpenPrefixQueue(file) 562 | if err != nil { 563 | b.Error(err) 564 | } 565 | defer pq.Drop() 566 | 567 | // Fill with dummy data 568 | for n := 0; n < b.N; n++ { 569 | if _, err = pq.Enqueue([]byte("prefix"), []byte("value")); err != nil { 570 | b.Error(err) 571 | } 572 | } 573 | 574 | // Start benchmark 575 | b.ResetTimer() 576 | b.ReportAllocs() 577 | 578 | for n := 0; n < b.N; n++ { 579 | _, _ = pq.Dequeue([]byte("prefix")) 580 | } 581 | } 582 | -------------------------------------------------------------------------------- /priority_queue.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "encoding/json" 7 | "os" 8 | "sync" 9 | 10 | "github.com/syndtr/goleveldb/leveldb" 11 | "github.com/syndtr/goleveldb/leveldb/util" 12 | ) 13 | 14 | // prefixSep is the prefix separator for each item key. 15 | var prefixSep []byte = []byte(":") 16 | 17 | // order defines the priority ordering of the queue. 18 | type order int 19 | 20 | // Defines which priority order to dequeue in. 21 | const ( 22 | ASC order = iota // Set priority level 0 as most important. 23 | DESC // Set priority level 255 as most important. 24 | ) 25 | 26 | // priorityLevel holds the head and tail position of a priority 27 | // level within the queue. 28 | type priorityLevel struct { 29 | head uint64 30 | tail uint64 31 | } 32 | 33 | // length returns the total number of items in this priority level. 34 | func (pl *priorityLevel) length() uint64 { 35 | return pl.tail - pl.head 36 | } 37 | 38 | // PriorityQueue is a standard FIFO (first in, first out) queue with 39 | // priority levels. 40 | type PriorityQueue struct { 41 | sync.RWMutex 42 | DataDir string 43 | db *leveldb.DB 44 | order order 45 | levels [256]*priorityLevel 46 | curLevel uint8 47 | isOpen bool 48 | } 49 | 50 | // OpenPriorityQueue opens a priority queue if one exists at the given 51 | // directory. If one does not already exist, a new priority queue is 52 | // created. 53 | func OpenPriorityQueue(dataDir string, order order) (*PriorityQueue, error) { 54 | var err error 55 | 56 | // Create a new PriorityQueue. 57 | pq := &PriorityQueue{ 58 | DataDir: dataDir, 59 | db: &leveldb.DB{}, 60 | order: order, 61 | isOpen: false, 62 | } 63 | 64 | // Open database for the priority queue. 65 | pq.db, err = leveldb.OpenFile(dataDir, nil) 66 | if err != nil { 67 | return pq, err 68 | } 69 | 70 | // Check if this Goque type can open the requested data directory. 71 | ok, err := checkGoqueType(dataDir, goquePriorityQueue) 72 | if err != nil { 73 | return pq, err 74 | } 75 | if !ok { 76 | return pq, ErrIncompatibleType 77 | } 78 | 79 | // Set isOpen and return. 80 | pq.isOpen = true 81 | return pq, pq.init() 82 | } 83 | 84 | // Enqueue adds an item to the priority queue. 85 | func (pq *PriorityQueue) Enqueue(priority uint8, value []byte) (*PriorityItem, error) { 86 | pq.Lock() 87 | defer pq.Unlock() 88 | 89 | // Check if queue is closed. 90 | if !pq.isOpen { 91 | return nil, ErrDBClosed 92 | } 93 | 94 | // Get the priorityLevel. 95 | level := pq.levels[priority] 96 | 97 | // Create new PriorityItem. 98 | item := &PriorityItem{ 99 | ID: level.tail + 1, 100 | Priority: priority, 101 | Key: pq.generateKey(priority, level.tail+1), 102 | Value: value, 103 | } 104 | 105 | // Add it to the priority queue. 106 | if err := pq.db.Put(item.Key, item.Value, nil); err != nil { 107 | return nil, err 108 | } 109 | 110 | // Increment tail position. 111 | level.tail++ 112 | 113 | // If this priority level is more important than the curLevel. 114 | if pq.cmpAsc(priority) || pq.cmpDesc(priority) { 115 | pq.curLevel = priority 116 | } 117 | 118 | return item, nil 119 | } 120 | 121 | // EnqueueString is a helper function for Enqueue that accepts a 122 | // value as a string rather than a byte slice. 123 | func (pq *PriorityQueue) EnqueueString(priority uint8, value string) (*PriorityItem, error) { 124 | return pq.Enqueue(priority, []byte(value)) 125 | } 126 | 127 | // EnqueueObject is a helper function for Enqueue that accepts any 128 | // value type, which is then encoded into a byte slice using 129 | // encoding/gob. 130 | // 131 | // Objects containing pointers with zero values will decode to nil 132 | // when using this function. This is due to how the encoding/gob 133 | // package works. Because of this, you should only use this function 134 | // to encode simple types. 135 | func (pq *PriorityQueue) EnqueueObject(priority uint8, value interface{}) (*PriorityItem, error) { 136 | var buffer bytes.Buffer 137 | enc := gob.NewEncoder(&buffer) 138 | if err := enc.Encode(value); err != nil { 139 | return nil, err 140 | } 141 | 142 | return pq.Enqueue(priority, buffer.Bytes()) 143 | } 144 | 145 | // EnqueueObjectAsJSON is a helper function for Enqueue that accepts 146 | // any value type, which is then encoded into a JSON byte slice using 147 | // encoding/json. 148 | // 149 | // Use this function to handle encoding of complex types. 150 | func (pq *PriorityQueue) EnqueueObjectAsJSON(priority uint8, value interface{}) (*PriorityItem, error) { 151 | jsonBytes, err := json.Marshal(value) 152 | if err != nil { 153 | return nil, err 154 | } 155 | 156 | return pq.Enqueue(priority, jsonBytes) 157 | } 158 | 159 | // Dequeue removes the next item in the priority queue and returns it. 160 | func (pq *PriorityQueue) Dequeue() (*PriorityItem, error) { 161 | pq.Lock() 162 | defer pq.Unlock() 163 | 164 | // Check if queue is closed. 165 | if !pq.isOpen { 166 | return nil, ErrDBClosed 167 | } 168 | 169 | // Try to get the next item. 170 | item, err := pq.getNextItem() 171 | if err != nil { 172 | return nil, err 173 | } 174 | 175 | // Remove this item from the priority queue. 176 | if err = pq.db.Delete(item.Key, nil); err != nil { 177 | return nil, err 178 | } 179 | 180 | // Increment head position. 181 | pq.levels[pq.curLevel].head++ 182 | 183 | return item, nil 184 | } 185 | 186 | // DequeueByPriority removes the next item in the given priority level 187 | // and returns it. 188 | func (pq *PriorityQueue) DequeueByPriority(priority uint8) (*PriorityItem, error) { 189 | pq.Lock() 190 | defer pq.Unlock() 191 | 192 | // Check if queue is closed. 193 | if !pq.isOpen { 194 | return nil, ErrDBClosed 195 | } 196 | 197 | // Try to get the next item in the given priority level. 198 | item, err := pq.getItemByPriorityID(priority, pq.levels[priority].head+1) 199 | if err != nil { 200 | return nil, err 201 | } 202 | 203 | // Remove this item from the priority queue. 204 | if err = pq.db.Delete(item.Key, nil); err != nil { 205 | return nil, err 206 | } 207 | 208 | // Increment head position. 209 | pq.levels[priority].head++ 210 | 211 | return item, nil 212 | } 213 | 214 | // Peek returns the next item in the priority queue without removing it. 215 | func (pq *PriorityQueue) Peek() (*PriorityItem, error) { 216 | pq.RLock() 217 | defer pq.RUnlock() 218 | 219 | // Check if queue is closed. 220 | if !pq.isOpen { 221 | return nil, ErrDBClosed 222 | } 223 | 224 | return pq.getNextItem() 225 | } 226 | 227 | // PeekByOffset returns the item located at the given offset, 228 | // starting from the head of the queue, without removing it. 229 | func (pq *PriorityQueue) PeekByOffset(offset uint64) (*PriorityItem, error) { 230 | pq.RLock() 231 | defer pq.RUnlock() 232 | 233 | // Check if queue is closed. 234 | if !pq.isOpen { 235 | return nil, ErrDBClosed 236 | } 237 | 238 | // Check if queue is empty. 239 | if pq.Length() == 0 { 240 | return nil, ErrEmpty 241 | } 242 | 243 | // If the offset is within the current priority level. 244 | if pq.levels[pq.curLevel].length() >= offset+1 { 245 | return pq.getItemByPriorityID(pq.curLevel, pq.levels[pq.curLevel].head+offset+1) 246 | } 247 | 248 | return pq.findOffset(offset) 249 | } 250 | 251 | // PeekByPriorityID returns the item with the given ID and priority without 252 | // removing it. 253 | func (pq *PriorityQueue) PeekByPriorityID(priority uint8, id uint64) (*PriorityItem, error) { 254 | pq.RLock() 255 | defer pq.RUnlock() 256 | 257 | // Check if queue is closed. 258 | if !pq.isOpen { 259 | return nil, ErrDBClosed 260 | } 261 | 262 | return pq.getItemByPriorityID(priority, id) 263 | } 264 | 265 | // Update updates an item in the priority queue without changing its 266 | // position. 267 | func (pq *PriorityQueue) Update(priority uint8, id uint64, newValue []byte) (*PriorityItem, error) { 268 | pq.Lock() 269 | defer pq.Unlock() 270 | 271 | // Check if queue is closed. 272 | if !pq.isOpen { 273 | return nil, ErrDBClosed 274 | } 275 | 276 | // Check if item exists in queue. 277 | if id <= pq.levels[priority].head || id > pq.levels[priority].tail { 278 | return nil, ErrOutOfBounds 279 | } 280 | 281 | // Create new PriorityItem. 282 | item := &PriorityItem{ 283 | ID: id, 284 | Priority: priority, 285 | Key: pq.generateKey(priority, id), 286 | Value: newValue, 287 | } 288 | 289 | // Update this item in the queue. 290 | if err := pq.db.Put(item.Key, item.Value, nil); err != nil { 291 | return nil, err 292 | } 293 | 294 | return item, nil 295 | } 296 | 297 | // UpdateString is a helper function for Update that accepts a value 298 | // as a string rather than a byte slice. 299 | func (pq *PriorityQueue) UpdateString(priority uint8, id uint64, newValue string) (*PriorityItem, error) { 300 | return pq.Update(priority, id, []byte(newValue)) 301 | } 302 | 303 | // UpdateObject is a helper function for Update that accepts any 304 | // value type, which is then encoded into a byte slice using 305 | // encoding/gob. 306 | // 307 | // Objects containing pointers with zero values will decode to nil 308 | // when using this function. This is due to how the encoding/gob 309 | // package works. Because of this, you should only use this function 310 | // to encode simple types. 311 | func (pq *PriorityQueue) UpdateObject(priority uint8, id uint64, newValue interface{}) (*PriorityItem, error) { 312 | var buffer bytes.Buffer 313 | enc := gob.NewEncoder(&buffer) 314 | if err := enc.Encode(newValue); err != nil { 315 | return nil, err 316 | } 317 | return pq.Update(priority, id, buffer.Bytes()) 318 | } 319 | 320 | // UpdateObjectAsJSON is a helper function for Update that accepts 321 | // any value type, which is then encoded into a JSON byte slice using 322 | // encoding/json. 323 | // 324 | // Use this function to handle encoding of complex types. 325 | func (pq *PriorityQueue) UpdateObjectAsJSON(priority uint8, id uint64, newValue interface{}) (*PriorityItem, error) { 326 | jsonBytes, err := json.Marshal(newValue) 327 | if err != nil { 328 | return nil, err 329 | } 330 | 331 | return pq.Update(priority, id, jsonBytes) 332 | } 333 | 334 | // Length returns the total number of items in the priority queue. 335 | func (pq *PriorityQueue) Length() uint64 { 336 | pq.RLock() 337 | defer pq.RUnlock() 338 | 339 | var length uint64 340 | for _, v := range pq.levels { 341 | length += v.length() 342 | } 343 | 344 | return length 345 | } 346 | 347 | // Close closes the LevelDB database of the priority queue. 348 | func (pq *PriorityQueue) Close() error { 349 | pq.Lock() 350 | defer pq.Unlock() 351 | 352 | // Check if queue is already closed. 353 | if !pq.isOpen { 354 | return nil 355 | } 356 | 357 | // Close the LevelDB database. 358 | if err := pq.db.Close(); err != nil { 359 | return err 360 | } 361 | 362 | // Reset head and tail of each priority level 363 | // and set isOpen to false. 364 | for i := 0; i <= 255; i++ { 365 | pq.levels[uint8(i)].head = 0 366 | pq.levels[uint8(i)].tail = 0 367 | } 368 | pq.isOpen = false 369 | 370 | return nil 371 | } 372 | 373 | // Drop closes and deletes the LevelDB database of the priority queue. 374 | func (pq *PriorityQueue) Drop() error { 375 | if err := pq.Close(); err != nil { 376 | return err 377 | } 378 | 379 | return os.RemoveAll(pq.DataDir) 380 | } 381 | 382 | // cmpAsc returns wehther the given priority level is higher than the 383 | // current priority level based on ascending order. 384 | func (pq *PriorityQueue) cmpAsc(priority uint8) bool { 385 | return pq.order == ASC && priority < pq.curLevel 386 | } 387 | 388 | // cmpAsc returns wehther the given priority level is higher than the 389 | // current priority level based on descending order. 390 | func (pq *PriorityQueue) cmpDesc(priority uint8) bool { 391 | return pq.order == DESC && priority > pq.curLevel 392 | } 393 | 394 | // resetCurrentLevel resets the current priority level of the queue 395 | // so the highest level can be found. 396 | func (pq *PriorityQueue) resetCurrentLevel() { 397 | if pq.order == ASC { 398 | pq.curLevel = 255 399 | } else if pq.order == DESC { 400 | pq.curLevel = 0 401 | } 402 | } 403 | 404 | // findOffset finds the given offset from the current queue position 405 | // based on priority order. 406 | func (pq *PriorityQueue) findOffset(offset uint64) (*PriorityItem, error) { 407 | var length uint64 408 | var curLevel uint8 = pq.curLevel 409 | var newLevel int 410 | 411 | // Handle newLevel initialization for descending order. 412 | if pq.order == DESC { 413 | newLevel = 255 414 | } 415 | 416 | // For condition expression. 417 | condExpr := func(level int) bool { 418 | if pq.order == ASC { 419 | return level <= 255 420 | } 421 | return level >= 0 422 | } 423 | 424 | // For loop expression. 425 | loopExpr := func(level *int) { 426 | if pq.order == ASC { 427 | *level++ 428 | } else if pq.order == DESC { 429 | *level-- 430 | } 431 | } 432 | 433 | // Level comparison. 434 | cmpLevels := func(newLevel, curLevel uint8) bool { 435 | if pq.order == ASC { 436 | return newLevel >= curLevel 437 | } 438 | return newLevel <= curLevel 439 | } 440 | 441 | // Loop through the priority levels. 442 | for ; condExpr(newLevel); loopExpr(&newLevel) { 443 | // If this level is lower than the current level based on ordering and contains items. 444 | if cmpLevels(uint8(newLevel), curLevel) && pq.levels[uint8(newLevel)].length() > 0 { 445 | curLevel = uint8(newLevel) 446 | newLength := pq.levels[curLevel].length() 447 | 448 | // If the offset is within the current priority level. 449 | if length+newLength >= offset+1 { 450 | return pq.getItemByPriorityID(curLevel, offset-length+1) 451 | } 452 | 453 | length += newLength 454 | } 455 | } 456 | 457 | return nil, ErrOutOfBounds 458 | } 459 | 460 | // getNextItem returns the next item in the priority queue, updating 461 | // the current priority level of the queue if necessary. 462 | func (pq *PriorityQueue) getNextItem() (*PriorityItem, error) { 463 | // If the current priority level is empty. 464 | if pq.levels[pq.curLevel].length() == 0 { 465 | // Set starting value for curLevel. 466 | pq.resetCurrentLevel() 467 | 468 | // Try to get the next priority level. 469 | for i := 0; i <= 255; i++ { 470 | if (pq.cmpAsc(uint8(i)) || pq.cmpDesc(uint8(i))) && pq.levels[uint8(i)].length() > 0 { 471 | pq.curLevel = uint8(i) 472 | } 473 | } 474 | 475 | // If still empty, return queue empty error. 476 | if pq.levels[pq.curLevel].length() == 0 { 477 | return nil, ErrEmpty 478 | } 479 | } 480 | 481 | // Try to get the next item in the current priority level. 482 | return pq.getItemByPriorityID(pq.curLevel, pq.levels[pq.curLevel].head+1) 483 | } 484 | 485 | // getItemByID returns an item, if found, for the given ID. 486 | func (pq *PriorityQueue) getItemByPriorityID(priority uint8, id uint64) (*PriorityItem, error) { 487 | // Check if empty or out of bounds. 488 | if pq.levels[priority].length() == 0 { 489 | return nil, ErrEmpty 490 | } else if id <= pq.levels[priority].head || id > pq.levels[priority].tail { 491 | return nil, ErrOutOfBounds 492 | } 493 | 494 | // Get item from database. 495 | var err error 496 | item := &PriorityItem{ID: id, Priority: priority, Key: pq.generateKey(priority, id)} 497 | if item.Value, err = pq.db.Get(item.Key, nil); err != nil { 498 | return nil, err 499 | } 500 | 501 | return item, nil 502 | } 503 | 504 | // generatePrefix creates the key prefix for the given priority level. 505 | func (pq *PriorityQueue) generatePrefix(level uint8) []byte { 506 | // priority + prefixSep = 1 + 1 = 2 507 | prefix := make([]byte, 2) 508 | prefix[0] = byte(level) 509 | prefix[1] = prefixSep[0] 510 | return prefix 511 | } 512 | 513 | // generateKey create a key to be used with LevelDB. 514 | func (pq *PriorityQueue) generateKey(priority uint8, id uint64) []byte { 515 | // prefix + key = 2 + 8 = 10 516 | key := make([]byte, 10) 517 | copy(key[0:2], pq.generatePrefix(priority)) 518 | copy(key[2:], idToKey(id)) 519 | return key 520 | } 521 | 522 | // init initializes the priority queue data. 523 | func (pq *PriorityQueue) init() error { 524 | // Set starting value for curLevel. 525 | pq.resetCurrentLevel() 526 | 527 | // Loop through each priority level. 528 | for i := 0; i <= 255; i++ { 529 | // Create a new LevelDB Iterator for this priority level. 530 | prefix := pq.generatePrefix(uint8(i)) 531 | iter := pq.db.NewIterator(util.BytesPrefix(prefix), nil) 532 | 533 | // Create a new priorityLevel. 534 | pl := &priorityLevel{ 535 | head: 0, 536 | tail: 0, 537 | } 538 | 539 | // Set priority level head to the first item. 540 | if iter.First() { 541 | pl.head = keyToID(iter.Key()[2:]) - 1 542 | 543 | // Since this priority level has item(s), handle updating curLevel. 544 | if pq.cmpAsc(uint8(i)) || pq.cmpDesc(uint8(i)) { 545 | pq.curLevel = uint8(i) 546 | } 547 | } 548 | 549 | // Set priority level tail to the last item. 550 | if iter.Last() { 551 | pl.tail = keyToID(iter.Key()[2:]) 552 | } 553 | 554 | if iter.Error() != nil { 555 | return iter.Error() 556 | } 557 | 558 | pq.levels[i] = pl 559 | iter.Release() 560 | } 561 | 562 | return nil 563 | } 564 | -------------------------------------------------------------------------------- /priority_queue_test.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestPriorityQueueClose(t *testing.T) { 12 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 13 | pq, err := OpenPriorityQueue(file, ASC) 14 | if err != nil { 15 | t.Error(err) 16 | } 17 | defer pq.Drop() 18 | 19 | for p := 0; p <= 4; p++ { 20 | for i := 1; i <= 10; i++ { 21 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 22 | t.Error(err) 23 | } 24 | } 25 | } 26 | 27 | if pq.Length() != 50 { 28 | t.Errorf("Expected queue length of 1, got %d", pq.Length()) 29 | } 30 | 31 | pq.Close() 32 | 33 | if _, err = pq.Dequeue(); err != ErrDBClosed { 34 | t.Errorf("Expected to get database closed error, got %s", err.Error()) 35 | } 36 | 37 | if pq.Length() != 0 { 38 | t.Errorf("Expected queue length of 0, got %d", pq.Length()) 39 | } 40 | } 41 | 42 | func TestPriorityQueueDrop(t *testing.T) { 43 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 44 | pq, err := OpenPriorityQueue(file, ASC) 45 | if err != nil { 46 | t.Error(err) 47 | } 48 | 49 | if _, err = os.Stat(file); os.IsNotExist(err) { 50 | t.Error(err) 51 | } 52 | 53 | pq.Drop() 54 | 55 | if _, err = os.Stat(file); err == nil { 56 | t.Error("Expected directory for test database to have been deleted") 57 | } 58 | } 59 | 60 | func TestPriorityQueueIncompatibleType(t *testing.T) { 61 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 62 | q, err := OpenQueue(file) 63 | if err != nil { 64 | t.Error(err) 65 | } 66 | defer q.Drop() 67 | q.Close() 68 | 69 | if _, err = OpenPriorityQueue(file, ASC); err != ErrIncompatibleType { 70 | t.Error("Expected priority queue to return ErrIncompatibleTypes when opening Queue") 71 | } 72 | } 73 | 74 | func TestPriorityQueueEnqueue(t *testing.T) { 75 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 76 | pq, err := OpenPriorityQueue(file, ASC) 77 | if err != nil { 78 | t.Error(err) 79 | } 80 | defer pq.Drop() 81 | 82 | for p := 0; p <= 4; p++ { 83 | for i := 1; i <= 10; i++ { 84 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 85 | t.Error(err) 86 | } 87 | } 88 | } 89 | 90 | if pq.Length() != 50 { 91 | t.Errorf("Expected queue size of 50, got %d", pq.Length()) 92 | } 93 | } 94 | 95 | func TestPriorityQueueDequeueAsc(t *testing.T) { 96 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 97 | pq, err := OpenPriorityQueue(file, ASC) 98 | if err != nil { 99 | t.Error(err) 100 | } 101 | defer pq.Drop() 102 | 103 | for p := 0; p <= 4; p++ { 104 | for i := 1; i <= 10; i++ { 105 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 106 | t.Error(err) 107 | } 108 | } 109 | } 110 | 111 | if pq.Length() != 50 { 112 | t.Errorf("Expected queue length of 1, got %d", pq.Length()) 113 | } 114 | 115 | deqItem, err := pq.Dequeue() 116 | if err != nil { 117 | t.Error(err) 118 | } 119 | 120 | if pq.Length() != 49 { 121 | t.Errorf("Expected queue length of 49, got %d", pq.Length()) 122 | } 123 | 124 | compStr := "value for item 1" 125 | 126 | if deqItem.Priority != 0 { 127 | t.Errorf("Expected priority level to be 0, got %d", deqItem.Priority) 128 | } 129 | 130 | if deqItem.ToString() != compStr { 131 | t.Errorf("Expected string to be '%s', got '%s'", compStr, deqItem.ToString()) 132 | } 133 | } 134 | 135 | func TestPriorityQueueDequeueDesc(t *testing.T) { 136 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 137 | pq, err := OpenPriorityQueue(file, DESC) 138 | if err != nil { 139 | t.Error(err) 140 | } 141 | defer pq.Drop() 142 | 143 | for p := 0; p <= 4; p++ { 144 | for i := 1; i <= 10; i++ { 145 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 146 | t.Error(err) 147 | } 148 | } 149 | } 150 | 151 | if pq.Length() != 50 { 152 | t.Errorf("Expected queue length of 1, got %d", pq.Length()) 153 | } 154 | 155 | deqItem, err := pq.Dequeue() 156 | if err != nil { 157 | t.Error(err) 158 | } 159 | 160 | if pq.Length() != 49 { 161 | t.Errorf("Expected queue length of 49, got %d", pq.Length()) 162 | } 163 | 164 | compStr := "value for item 1" 165 | 166 | if deqItem.Priority != 4 { 167 | t.Errorf("Expected priority level to be 4, got %d", deqItem.Priority) 168 | } 169 | 170 | if deqItem.ToString() != compStr { 171 | t.Errorf("Expected string to be '%s', got '%s'", compStr, deqItem.ToString()) 172 | } 173 | } 174 | 175 | func TestPriorityQueueDequeueByPriority(t *testing.T) { 176 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 177 | pq, err := OpenPriorityQueue(file, ASC) 178 | if err != nil { 179 | t.Error(err) 180 | } 181 | defer pq.Drop() 182 | 183 | for p := 0; p <= 4; p++ { 184 | for i := 1; i <= 10; i++ { 185 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 186 | t.Error(err) 187 | } 188 | } 189 | } 190 | 191 | if pq.Length() != 50 { 192 | t.Errorf("Expected queue length of 50, got %d", pq.Length()) 193 | } 194 | 195 | deqItem, err := pq.DequeueByPriority(3) 196 | if err != nil { 197 | t.Error(err) 198 | } 199 | 200 | if pq.Length() != 49 { 201 | t.Errorf("Expected queue length of 49, got %d", pq.Length()) 202 | } 203 | 204 | compStr := "value for item 1" 205 | 206 | if deqItem.Priority != 3 { 207 | t.Errorf("Expected priority level to be 1, got %d", deqItem.Priority) 208 | } 209 | 210 | if deqItem.ToString() != compStr { 211 | t.Errorf("Expected string to be '%s', got '%s'", compStr, deqItem.ToString()) 212 | } 213 | } 214 | 215 | func TestPriorityQueueEncodeDecodePointerJSON(t *testing.T) { 216 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 217 | pq, err := OpenPriorityQueue(file, DESC) 218 | if err != nil { 219 | t.Error(err) 220 | } 221 | defer pq.Drop() 222 | 223 | type subObject struct { 224 | Value *int 225 | } 226 | 227 | type object struct { 228 | Value int 229 | SubObject subObject 230 | } 231 | 232 | val := 0 233 | obj := object{ 234 | Value: 0, 235 | SubObject: subObject{ 236 | Value: &val, 237 | }, 238 | } 239 | 240 | if _, err = pq.EnqueueObjectAsJSON(0, obj); err != nil { 241 | t.Error(err) 242 | } 243 | 244 | item, err := pq.Dequeue() 245 | if err != nil { 246 | t.Error(err) 247 | } 248 | 249 | var itemObj object 250 | if err := item.ToObjectFromJSON(&itemObj); err != nil { 251 | t.Error(err) 252 | } 253 | 254 | if *itemObj.SubObject.Value != 0 { 255 | t.Errorf("Expected object subobject value to be '0', got '%v'", *itemObj.SubObject.Value) 256 | } 257 | } 258 | 259 | func TestPriorityQueuePeek(t *testing.T) { 260 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 261 | pq, err := OpenPriorityQueue(file, ASC) 262 | if err != nil { 263 | t.Error(err) 264 | } 265 | defer pq.Drop() 266 | 267 | for p := 0; p <= 4; p++ { 268 | for i := 1; i <= 10; i++ { 269 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 270 | t.Error(err) 271 | } 272 | } 273 | } 274 | 275 | compStr := "value for item 1" 276 | 277 | peekItem, err := pq.Peek() 278 | if err != nil { 279 | t.Error(err) 280 | } 281 | 282 | if peekItem.Priority != 0 { 283 | t.Errorf("Expected priority level to be 0, got %d", peekItem.Priority) 284 | } 285 | 286 | if peekItem.ToString() != compStr { 287 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 288 | } 289 | 290 | if pq.Length() != 50 { 291 | t.Errorf("Expected queue length of 50, got %d", pq.Length()) 292 | } 293 | } 294 | 295 | func TestPriorityQueuePeekByOffsetEmptyAsc(t *testing.T) { 296 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 297 | pq, err := OpenPriorityQueue(file, ASC) 298 | if err != nil { 299 | t.Error(err) 300 | } 301 | defer pq.Drop() 302 | 303 | _, err = pq.PeekByOffset(0) 304 | if err != ErrEmpty { 305 | t.Errorf("Expected to get empty error, got %s", err.Error()) 306 | } 307 | 308 | if _, err = pq.EnqueueString(0, "value"); err != nil { 309 | t.Error(err) 310 | } 311 | 312 | _, err = pq.PeekByOffset(0) 313 | if err != nil { 314 | t.Errorf("Expected to get nil error, got %s", err.Error()) 315 | } 316 | 317 | if _, err = pq.Dequeue(); err != nil { 318 | t.Error(err) 319 | } 320 | 321 | _, err = pq.PeekByOffset(0) 322 | if err != ErrEmpty { 323 | t.Errorf("Expected to get empty error, got %s", err.Error()) 324 | } 325 | } 326 | 327 | func TestPriorityQueuePeekByOffsetEmptyDesc(t *testing.T) { 328 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 329 | pq, err := OpenPriorityQueue(file, DESC) 330 | if err != nil { 331 | t.Error(err) 332 | } 333 | defer pq.Drop() 334 | 335 | _, err = pq.PeekByOffset(0) 336 | if err != ErrEmpty { 337 | t.Errorf("Expected to get empty error, got %s", err.Error()) 338 | } 339 | 340 | if _, err = pq.EnqueueString(0, "value"); err != nil { 341 | t.Error(err) 342 | } 343 | 344 | _, err = pq.PeekByOffset(0) 345 | if err != nil { 346 | t.Errorf("Expected to get nil error, got %s", err.Error()) 347 | } 348 | 349 | if _, err = pq.Dequeue(); err != nil { 350 | t.Error(err) 351 | } 352 | 353 | _, err = pq.PeekByOffset(0) 354 | if err != ErrEmpty { 355 | t.Errorf("Expected to get empty error, got %s", err.Error()) 356 | } 357 | } 358 | 359 | func TestPriorityQueuePeekByOffsetBoundsAsc(t *testing.T) { 360 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 361 | pq, err := OpenPriorityQueue(file, ASC) 362 | if err != nil { 363 | t.Error(err) 364 | } 365 | defer pq.Drop() 366 | 367 | _, err = pq.PeekByOffset(0) 368 | if err != ErrEmpty { 369 | t.Errorf("Expected to get empty error, got %s", err.Error()) 370 | } 371 | 372 | if _, err = pq.EnqueueString(0, "value"); err != nil { 373 | t.Error(err) 374 | } 375 | 376 | _, err = pq.PeekByOffset(0) 377 | if err != nil { 378 | t.Errorf("Expected to get nil error, got %s", err.Error()) 379 | } 380 | 381 | _, err = pq.PeekByOffset(1) 382 | if err != ErrOutOfBounds { 383 | t.Errorf("Expected to get queue out of bounds error, got %s", err.Error()) 384 | } 385 | 386 | for p := 0; p <= 4; p++ { 387 | for i := 1; i <= 10; i++ { 388 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 389 | t.Error(err) 390 | } 391 | } 392 | } 393 | 394 | _, err = pq.PeekByOffset(50) 395 | if err != nil { 396 | t.Errorf("Expected to get nil error, got %s", err.Error()) 397 | } 398 | 399 | _, err = pq.PeekByOffset(51) 400 | if err != ErrOutOfBounds { 401 | t.Errorf("Expected to get queue out of bounds error, got %s", err.Error()) 402 | } 403 | } 404 | 405 | func TestPriorityQueuePeekByOffsetBoundsDesc(t *testing.T) { 406 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 407 | pq, err := OpenPriorityQueue(file, DESC) 408 | if err != nil { 409 | t.Error(err) 410 | } 411 | defer pq.Drop() 412 | 413 | _, err = pq.PeekByOffset(0) 414 | if err != ErrEmpty { 415 | t.Errorf("Expected to get empty error, got %s", err.Error()) 416 | } 417 | 418 | if _, err = pq.EnqueueString(0, "value"); err != nil { 419 | t.Error(err) 420 | } 421 | 422 | _, err = pq.PeekByOffset(0) 423 | if err != nil { 424 | t.Errorf("Expected to get nil error, got %s", err.Error()) 425 | } 426 | 427 | _, err = pq.PeekByOffset(1) 428 | if err != ErrOutOfBounds { 429 | t.Errorf("Expected to get queue out of bounds error, got %s", err.Error()) 430 | } 431 | 432 | for p := 0; p <= 4; p++ { 433 | for i := 1; i <= 10; i++ { 434 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 435 | t.Error(err) 436 | } 437 | } 438 | } 439 | 440 | _, err = pq.PeekByOffset(50) 441 | if err != nil { 442 | t.Errorf("Expected to get nil error, got %s", err.Error()) 443 | } 444 | 445 | _, err = pq.PeekByOffset(51) 446 | if err != ErrOutOfBounds { 447 | t.Errorf("Expected to get queue out of bounds error, got %s", err.Error()) 448 | } 449 | } 450 | 451 | func TestPriorityQueuePeekByOffsetAsc(t *testing.T) { 452 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 453 | pq, err := OpenPriorityQueue(file, ASC) 454 | if err != nil { 455 | t.Error(err) 456 | } 457 | defer pq.Drop() 458 | 459 | for p := 0; p <= 4; p++ { 460 | for i := 1; i <= 10; i++ { 461 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 462 | t.Error(err) 463 | } 464 | } 465 | } 466 | 467 | compStrFirst := "value for item 1" 468 | compStrLast := "value for item 10" 469 | compStr := "value for item 3" 470 | 471 | peekFirstItem, err := pq.PeekByOffset(0) 472 | if err != nil { 473 | t.Error(err) 474 | } 475 | 476 | if peekFirstItem.Priority != 0 { 477 | t.Errorf("Expected priority level to be 0, got %d", peekFirstItem.Priority) 478 | } 479 | 480 | if peekFirstItem.ToString() != compStrFirst { 481 | t.Errorf("Expected string to be '%s', got '%s'", compStrFirst, peekFirstItem.ToString()) 482 | } 483 | 484 | peekLastItem, err := pq.PeekByOffset(49) 485 | if err != nil { 486 | t.Error(err) 487 | } 488 | 489 | if peekLastItem.Priority != 4 { 490 | t.Errorf("Expected priority level to be 4, got %d", peekLastItem.Priority) 491 | } 492 | 493 | if peekLastItem.ToString() != compStrLast { 494 | t.Errorf("Expected string to be '%s', got '%s'", compStrLast, peekLastItem.ToString()) 495 | } 496 | 497 | peekItem, err := pq.PeekByOffset(22) 498 | if err != nil { 499 | t.Error(err) 500 | } 501 | 502 | if peekItem.Priority != 2 { 503 | t.Errorf("Expected priority level to be 2, got %d", peekItem.Priority) 504 | } 505 | 506 | if peekItem.ToString() != compStr { 507 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 508 | } 509 | 510 | if pq.Length() != 50 { 511 | t.Errorf("Expected queue length of 50, got %d", pq.Length()) 512 | } 513 | } 514 | 515 | func TestPriorityQueuePeekByOffsetDesc(t *testing.T) { 516 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 517 | pq, err := OpenPriorityQueue(file, DESC) 518 | if err != nil { 519 | t.Error(err) 520 | } 521 | defer pq.Drop() 522 | 523 | for p := 0; p <= 4; p++ { 524 | for i := 1; i <= 10; i++ { 525 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 526 | t.Error(err) 527 | } 528 | } 529 | } 530 | 531 | compStrFirst := "value for item 1" 532 | compStrLast := "value for item 10" 533 | compStr := "value for item 3" 534 | 535 | peekFirstItem, err := pq.PeekByOffset(0) 536 | if err != nil { 537 | t.Error(err) 538 | } 539 | 540 | if peekFirstItem.Priority != 4 { 541 | t.Errorf("Expected priority level to be 4, got %d", peekFirstItem.Priority) 542 | } 543 | 544 | if peekFirstItem.ToString() != compStrFirst { 545 | t.Errorf("Expected string to be '%s', got '%s'", compStrFirst, peekFirstItem.ToString()) 546 | } 547 | 548 | peekLastItem, err := pq.PeekByOffset(49) 549 | if err != nil { 550 | t.Error(err) 551 | } 552 | 553 | if peekLastItem.Priority != 0 { 554 | t.Errorf("Expected priority level to be 0, got %d", peekLastItem.Priority) 555 | } 556 | 557 | if peekLastItem.ToString() != compStrLast { 558 | t.Errorf("Expected string to be '%s', got '%s'", compStrLast, peekLastItem.ToString()) 559 | } 560 | 561 | peekItem, err := pq.PeekByOffset(32) 562 | if err != nil { 563 | t.Error(err) 564 | } 565 | 566 | if peekItem.Priority != 1 { 567 | t.Errorf("Expected priority level to be 0, got %d", peekItem.Priority) 568 | } 569 | 570 | if peekItem.ToString() != compStr { 571 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 572 | } 573 | 574 | if pq.Length() != 50 { 575 | t.Errorf("Expected queue length of 50, got %d", pq.Length()) 576 | } 577 | } 578 | 579 | func TestPriorityQueuePeekByPriorityID(t *testing.T) { 580 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 581 | pq, err := OpenPriorityQueue(file, ASC) 582 | if err != nil { 583 | t.Error(err) 584 | } 585 | defer pq.Drop() 586 | 587 | for p := 0; p <= 4; p++ { 588 | for i := 1; i <= 10; i++ { 589 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 590 | t.Error(err) 591 | } 592 | } 593 | } 594 | 595 | compStr := "value for item 3" 596 | 597 | peekItem, err := pq.PeekByPriorityID(1, 3) 598 | if err != nil { 599 | t.Error(err) 600 | } 601 | 602 | if peekItem.Priority != 1 { 603 | t.Errorf("Expected priority level to be 1, got %d", peekItem.Priority) 604 | } 605 | 606 | if peekItem.ToString() != compStr { 607 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 608 | } 609 | 610 | if pq.Length() != 50 { 611 | t.Errorf("Expected queue length of 50, got %d", pq.Length()) 612 | } 613 | } 614 | 615 | func TestPriorityQueueUpdate(t *testing.T) { 616 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 617 | pq, err := OpenPriorityQueue(file, ASC) 618 | if err != nil { 619 | t.Error(err) 620 | } 621 | defer pq.Drop() 622 | 623 | for p := 0; p <= 4; p++ { 624 | for i := 1; i <= 10; i++ { 625 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 626 | t.Error(err) 627 | } 628 | } 629 | } 630 | 631 | item, err := pq.PeekByPriorityID(0, 3) 632 | if err != nil { 633 | t.Error(err) 634 | } 635 | 636 | oldCompStr := "value for item 3" 637 | newCompStr := "new value for item 3" 638 | 639 | if item.ToString() != oldCompStr { 640 | t.Errorf("Expected string to be '%s', got '%s'", oldCompStr, item.ToString()) 641 | } 642 | 643 | updatedItem, err := pq.Update(item.Priority, item.ID, []byte(newCompStr)) 644 | if err != nil { 645 | t.Error(err) 646 | } 647 | 648 | if updatedItem.Priority != 0 { 649 | t.Errorf("Expected priority level to be 0, got %d", item.Priority) 650 | } 651 | 652 | if updatedItem.ToString() != newCompStr { 653 | t.Errorf("Expected current item value to be '%s', got '%s'", newCompStr, item.ToString()) 654 | } 655 | 656 | newItem, err := pq.PeekByPriorityID(0, 3) 657 | if err != nil { 658 | t.Error(err) 659 | } 660 | 661 | if newItem.Priority != 0 { 662 | t.Errorf("Expected priority level to be 0, got %d", newItem.Priority) 663 | } 664 | 665 | if newItem.ToString() != newCompStr { 666 | t.Errorf("Expected new item value to be '%s', got '%s'", newCompStr, item.ToString()) 667 | } 668 | } 669 | 670 | func TestPriorityQueueUpdateString(t *testing.T) { 671 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 672 | pq, err := OpenPriorityQueue(file, ASC) 673 | if err != nil { 674 | t.Error(err) 675 | } 676 | defer pq.Drop() 677 | 678 | for p := 0; p <= 4; p++ { 679 | for i := 1; i <= 10; i++ { 680 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 681 | t.Error(err) 682 | } 683 | } 684 | } 685 | 686 | item, err := pq.PeekByPriorityID(0, 3) 687 | if err != nil { 688 | t.Error(err) 689 | } 690 | 691 | oldCompStr := "value for item 3" 692 | newCompStr := "new value for item 3" 693 | 694 | if item.ToString() != oldCompStr { 695 | t.Errorf("Expected string to be '%s', got '%s'", oldCompStr, item.ToString()) 696 | } 697 | 698 | updatedItem, err := pq.UpdateString(item.Priority, item.ID, newCompStr) 699 | if err != nil { 700 | t.Error(err) 701 | } 702 | 703 | if updatedItem.Priority != 0 { 704 | t.Errorf("Expected priority level to be 0, got %d", item.Priority) 705 | } 706 | 707 | if updatedItem.ToString() != newCompStr { 708 | t.Errorf("Expected current item value to be '%s', got '%s'", newCompStr, item.ToString()) 709 | } 710 | 711 | newItem, err := pq.PeekByPriorityID(0, 3) 712 | if err != nil { 713 | t.Error(err) 714 | } 715 | 716 | if newItem.Priority != 0 { 717 | t.Errorf("Expected priority level to be 0, got %d", newItem.Priority) 718 | } 719 | 720 | if newItem.ToString() != newCompStr { 721 | t.Errorf("Expected new item value to be '%s', got '%s'", newCompStr, item.ToString()) 722 | } 723 | } 724 | 725 | func TestPriorityQueueUpdateObject(t *testing.T) { 726 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 727 | pq, err := OpenPriorityQueue(file, ASC) 728 | if err != nil { 729 | t.Error(err) 730 | } 731 | defer pq.Drop() 732 | 733 | type object struct { 734 | Priority uint8 735 | Value int 736 | } 737 | 738 | for p := 0; p <= 4; p++ { 739 | for i := 1; i <= 10; i++ { 740 | if _, err = pq.EnqueueObject(uint8(p), object{uint8(p), i}); err != nil { 741 | t.Error(err) 742 | } 743 | } 744 | } 745 | 746 | item, err := pq.PeekByPriorityID(0, 3) 747 | if err != nil { 748 | t.Error(err) 749 | } 750 | 751 | oldCompObj := object{0, 3} 752 | newCompObj := object{0, 33} 753 | 754 | var obj object 755 | if err := item.ToObject(&obj); err != nil { 756 | t.Error(err) 757 | } 758 | 759 | if obj != oldCompObj { 760 | t.Errorf("Expected object to be '%+v', got '%+v'", oldCompObj, obj) 761 | } 762 | 763 | updatedItem, err := pq.UpdateObject(item.Priority, item.ID, newCompObj) 764 | if err != nil { 765 | t.Error(err) 766 | } 767 | 768 | if updatedItem.Priority != 0 { 769 | t.Errorf("Expected priority level to be 0, got %d", item.Priority) 770 | } 771 | 772 | if err := updatedItem.ToObject(&obj); err != nil { 773 | t.Error(err) 774 | } 775 | 776 | if obj != newCompObj { 777 | t.Errorf("Expected current object to be '%+v', got '%+v'", newCompObj, obj) 778 | } 779 | 780 | newItem, err := pq.PeekByPriorityID(0, 3) 781 | if err != nil { 782 | t.Error(err) 783 | } 784 | 785 | if newItem.Priority != 0 { 786 | t.Errorf("Expected priority level to be 0, got %d", newItem.Priority) 787 | } 788 | 789 | if err := newItem.ToObject(&obj); err != nil { 790 | t.Error(err) 791 | } 792 | 793 | if obj != newCompObj { 794 | t.Errorf("Expected new object to be '%+v', got '%+v'", newCompObj, obj) 795 | } 796 | } 797 | 798 | func TestPriorityQueueUpdateObjectAsJSON(t *testing.T) { 799 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 800 | pq, err := OpenPriorityQueue(file, ASC) 801 | if err != nil { 802 | t.Error(err) 803 | } 804 | defer pq.Drop() 805 | 806 | type subObject struct { 807 | Value *int 808 | } 809 | 810 | type object struct { 811 | Priority uint8 812 | Value int 813 | SubObject subObject 814 | } 815 | 816 | for p := 0; p <= 4; p++ { 817 | for i := 1; i <= 10; i++ { 818 | obj := object{ 819 | Priority: uint8(p), 820 | Value: i, 821 | SubObject: subObject{ 822 | Value: &i, 823 | }, 824 | } 825 | 826 | if _, err = pq.EnqueueObjectAsJSON(uint8(p), obj); err != nil { 827 | t.Error(err) 828 | } 829 | } 830 | } 831 | 832 | item, err := pq.PeekByPriorityID(0, 3) 833 | if err != nil { 834 | t.Error(err) 835 | } 836 | 837 | oldCompObjVal := 3 838 | oldCompObj := object{ 839 | Priority: 0, 840 | Value: 3, 841 | SubObject: subObject{ 842 | Value: &oldCompObjVal, 843 | }, 844 | } 845 | newCompObjVal := 33 846 | newCompObj := object{ 847 | Priority: 0, 848 | Value: 33, 849 | SubObject: subObject{ 850 | Value: &newCompObjVal, 851 | }, 852 | } 853 | 854 | var obj object 855 | if err := item.ToObjectFromJSON(&obj); err != nil { 856 | t.Error(err) 857 | } 858 | 859 | if *obj.SubObject.Value != *oldCompObj.SubObject.Value { 860 | t.Errorf("Expected object subobject value to be '%+v', got '%+v'", *oldCompObj.SubObject.Value, *obj.SubObject.Value) 861 | } 862 | 863 | updatedItem, err := pq.UpdateObjectAsJSON(item.Priority, item.ID, newCompObj) 864 | if err != nil { 865 | t.Error(err) 866 | } 867 | 868 | if updatedItem.Priority != 0 { 869 | t.Errorf("Expected priority level to be 0, got %d", item.Priority) 870 | } 871 | 872 | if err := updatedItem.ToObjectFromJSON(&obj); err != nil { 873 | t.Error(err) 874 | } 875 | 876 | if *obj.SubObject.Value != *newCompObj.SubObject.Value { 877 | t.Errorf("Expected current object subobject value to be '%+v', got '%+v'", *newCompObj.SubObject.Value, *obj.SubObject.Value) 878 | } 879 | 880 | newItem, err := pq.PeekByPriorityID(0, 3) 881 | if err != nil { 882 | t.Error(err) 883 | } 884 | 885 | if newItem.Priority != 0 { 886 | t.Errorf("Expected priority level to be 0, got %d", newItem.Priority) 887 | } 888 | 889 | if err := newItem.ToObjectFromJSON(&obj); err != nil { 890 | t.Error(err) 891 | } 892 | 893 | if *obj.SubObject.Value != *newCompObj.SubObject.Value { 894 | t.Errorf("Expected current object subobject value to be '%+v', got '%+v'", *newCompObj.SubObject.Value, *obj.SubObject.Value) 895 | } 896 | } 897 | 898 | func TestPriorityQueueUpdateOutOfBounds(t *testing.T) { 899 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 900 | pq, err := OpenPriorityQueue(file, ASC) 901 | if err != nil { 902 | t.Error(err) 903 | } 904 | defer pq.Drop() 905 | 906 | for p := 0; p <= 4; p++ { 907 | for i := 1; i <= 10; i++ { 908 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 909 | t.Error(err) 910 | } 911 | } 912 | } 913 | 914 | if pq.Length() != 50 { 915 | t.Errorf("Expected queue length of 50, got %d", pq.Length()) 916 | } 917 | 918 | deqItem, err := pq.DequeueByPriority(3) 919 | if err != nil { 920 | t.Error(err) 921 | } 922 | 923 | if pq.Length() != 49 { 924 | t.Errorf("Expected queue length of 49, got %d", pq.Length()) 925 | } 926 | 927 | if _, err = pq.Update(deqItem.Priority, deqItem.ID, []byte(`new value`)); err != ErrOutOfBounds { 928 | t.Errorf("Expected to get queue out of bounds error, got %s", err.Error()) 929 | } 930 | 931 | if _, err = pq.Update(deqItem.Priority, deqItem.ID+1, []byte(`new value`)); err != nil { 932 | t.Error(err) 933 | } 934 | } 935 | 936 | func TestPriorityQueueHigherPriorityAsc(t *testing.T) { 937 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 938 | pq, err := OpenPriorityQueue(file, ASC) 939 | if err != nil { 940 | t.Error(err) 941 | } 942 | defer pq.Drop() 943 | 944 | for p := 5; p <= 9; p++ { 945 | for i := 1; i <= 10; i++ { 946 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 947 | t.Error(err) 948 | } 949 | } 950 | } 951 | 952 | item, err := pq.Dequeue() 953 | if err != nil { 954 | t.Error(err) 955 | } 956 | 957 | if item.Priority != 5 { 958 | t.Errorf("Expected priority level to be 5, got %d", item.Priority) 959 | } 960 | 961 | _, err = pq.EnqueueString(2, "value") 962 | if err != nil { 963 | t.Error(err) 964 | } 965 | 966 | higherItem, err := pq.Dequeue() 967 | if err != nil { 968 | t.Error(err) 969 | } 970 | 971 | if higherItem.Priority != 2 { 972 | t.Errorf("Expected priority level to be 2, got %d", higherItem.Priority) 973 | } 974 | } 975 | 976 | func TestPriorityQueueHigherPriorityDesc(t *testing.T) { 977 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 978 | pq, err := OpenPriorityQueue(file, DESC) 979 | if err != nil { 980 | t.Error(err) 981 | } 982 | defer pq.Drop() 983 | 984 | for p := 5; p <= 9; p++ { 985 | for i := 1; i <= 10; i++ { 986 | if _, err = pq.EnqueueString(uint8(p), fmt.Sprintf("value for item %d", i)); err != nil { 987 | t.Error(err) 988 | } 989 | } 990 | } 991 | 992 | item, err := pq.Dequeue() 993 | if err != nil { 994 | t.Error(err) 995 | } 996 | 997 | if item.Priority != 9 { 998 | t.Errorf("Expected priority level to be 9, got %d", item.Priority) 999 | } 1000 | 1001 | _, err = pq.EnqueueString(12, "value") 1002 | if err != nil { 1003 | t.Error(err) 1004 | } 1005 | 1006 | higherItem, err := pq.Dequeue() 1007 | if err != nil { 1008 | t.Error(err) 1009 | } 1010 | 1011 | if higherItem.Priority != 12 { 1012 | t.Errorf("Expected priority level to be 12, got %d", higherItem.Priority) 1013 | } 1014 | } 1015 | 1016 | func TestPriorityQueueEmpty(t *testing.T) { 1017 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 1018 | pq, err := OpenPriorityQueue(file, ASC) 1019 | if err != nil { 1020 | t.Error(err) 1021 | } 1022 | defer pq.Drop() 1023 | 1024 | _, err = pq.EnqueueString(0, "value for item") 1025 | if err != nil { 1026 | t.Error(err) 1027 | } 1028 | 1029 | _, err = pq.Dequeue() 1030 | if err != nil { 1031 | t.Error(err) 1032 | } 1033 | 1034 | _, err = pq.Dequeue() 1035 | if err != ErrEmpty { 1036 | t.Errorf("Expected to get empty error, got %s", err.Error()) 1037 | } 1038 | } 1039 | 1040 | func TestPriorityQueueOutOfBounds(t *testing.T) { 1041 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 1042 | pq, err := OpenPriorityQueue(file, ASC) 1043 | if err != nil { 1044 | t.Error(err) 1045 | } 1046 | defer pq.Drop() 1047 | 1048 | _, err = pq.EnqueueString(0, "value for item") 1049 | if err != nil { 1050 | t.Error(err) 1051 | } 1052 | 1053 | _, err = pq.PeekByOffset(2) 1054 | if err != ErrOutOfBounds { 1055 | t.Errorf("Expected to get queue out of bounds error, got %s", err.Error()) 1056 | } 1057 | } 1058 | 1059 | func BenchmarkPriorityQueueEnqueue(b *testing.B) { 1060 | // Open test database 1061 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 1062 | pq, err := OpenPriorityQueue(file, ASC) 1063 | if err != nil { 1064 | b.Error(err) 1065 | } 1066 | defer pq.Drop() 1067 | 1068 | b.ResetTimer() 1069 | b.ReportAllocs() 1070 | 1071 | for n := 0; n < b.N; n++ { 1072 | _, _ = pq.EnqueueString(0, "value") 1073 | } 1074 | } 1075 | 1076 | func BenchmarkPriorityQueueDequeue(b *testing.B) { 1077 | // Open test database 1078 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 1079 | pq, err := OpenPriorityQueue(file, ASC) 1080 | if err != nil { 1081 | b.Error(err) 1082 | } 1083 | defer pq.Drop() 1084 | 1085 | // Fill with dummy data 1086 | for n := 0; n < b.N; n++ { 1087 | if _, err = pq.EnqueueString(uint8(math.Mod(float64(n), 255)), "value"); err != nil { 1088 | b.Error(err) 1089 | } 1090 | } 1091 | 1092 | // Start benchmark 1093 | b.ResetTimer() 1094 | b.ReportAllocs() 1095 | 1096 | for n := 0; n < b.N; n++ { 1097 | _, _ = pq.Dequeue() 1098 | } 1099 | } 1100 | -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "encoding/json" 7 | "os" 8 | "sync" 9 | 10 | "github.com/syndtr/goleveldb/leveldb" 11 | ) 12 | 13 | // Queue is a standard FIFO (first in, first out) queue. 14 | type Queue struct { 15 | sync.RWMutex 16 | DataDir string 17 | db *leveldb.DB 18 | head uint64 19 | tail uint64 20 | isOpen bool 21 | } 22 | 23 | // OpenQueue opens a queue if one exists at the given directory. If one 24 | // does not already exist, a new queue is created. 25 | func OpenQueue(dataDir string) (*Queue, error) { 26 | var err error 27 | 28 | // Create a new Queue. 29 | q := &Queue{ 30 | DataDir: dataDir, 31 | db: &leveldb.DB{}, 32 | head: 0, 33 | tail: 0, 34 | isOpen: false, 35 | } 36 | 37 | // Open database for the queue. 38 | q.db, err = leveldb.OpenFile(dataDir, nil) 39 | if err != nil { 40 | return q, err 41 | } 42 | 43 | // Check if this Goque type can open the requested data directory. 44 | ok, err := checkGoqueType(dataDir, goqueQueue) 45 | if err != nil { 46 | return q, err 47 | } 48 | if !ok { 49 | return q, ErrIncompatibleType 50 | } 51 | 52 | // Set isOpen and return. 53 | q.isOpen = true 54 | return q, q.init() 55 | } 56 | 57 | // Enqueue adds an item to the queue. 58 | func (q *Queue) Enqueue(value []byte) (*Item, error) { 59 | q.Lock() 60 | defer q.Unlock() 61 | 62 | // Check if queue is closed. 63 | if !q.isOpen { 64 | return nil, ErrDBClosed 65 | } 66 | 67 | // Create new Item. 68 | item := &Item{ 69 | ID: q.tail + 1, 70 | Key: idToKey(q.tail + 1), 71 | Value: value, 72 | } 73 | 74 | // Add it to the queue. 75 | if err := q.db.Put(item.Key, item.Value, nil); err != nil { 76 | return nil, err 77 | } 78 | 79 | // Increment tail position. 80 | q.tail++ 81 | 82 | return item, nil 83 | } 84 | 85 | // EnqueueString is a helper function for Enqueue that accepts a 86 | // value as a string rather than a byte slice. 87 | func (q *Queue) EnqueueString(value string) (*Item, error) { 88 | return q.Enqueue([]byte(value)) 89 | } 90 | 91 | // EnqueueObject is a helper function for Enqueue that accepts any 92 | // value type, which is then encoded into a byte slice using 93 | // encoding/gob. 94 | // 95 | // Objects containing pointers with zero values will decode to nil 96 | // when using this function. This is due to how the encoding/gob 97 | // package works. Because of this, you should only use this function 98 | // to encode simple types. 99 | func (q *Queue) EnqueueObject(value interface{}) (*Item, error) { 100 | var buffer bytes.Buffer 101 | enc := gob.NewEncoder(&buffer) 102 | if err := enc.Encode(value); err != nil { 103 | return nil, err 104 | } 105 | 106 | return q.Enqueue(buffer.Bytes()) 107 | } 108 | 109 | // EnqueueObjectAsJSON is a helper function for Enqueue that accepts 110 | // any value type, which is then encoded into a JSON byte slice using 111 | // encoding/json. 112 | // 113 | // Use this function to handle encoding of complex types. 114 | func (q *Queue) EnqueueObjectAsJSON(value interface{}) (*Item, error) { 115 | jsonBytes, err := json.Marshal(value) 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | return q.Enqueue(jsonBytes) 121 | } 122 | 123 | // Dequeue removes the next item in the queue and returns it. 124 | func (q *Queue) Dequeue() (*Item, error) { 125 | q.Lock() 126 | defer q.Unlock() 127 | 128 | // Check if queue is closed. 129 | if !q.isOpen { 130 | return nil, ErrDBClosed 131 | } 132 | 133 | // Try to get the next item in the queue. 134 | item, err := q.getItemByID(q.head + 1) 135 | if err != nil { 136 | return nil, err 137 | } 138 | 139 | // Remove this item from the queue. 140 | if err := q.db.Delete(item.Key, nil); err != nil { 141 | return nil, err 142 | } 143 | 144 | // Increment head position. 145 | q.head++ 146 | 147 | return item, nil 148 | } 149 | 150 | // Peek returns the next item in the queue without removing it. 151 | func (q *Queue) Peek() (*Item, error) { 152 | q.RLock() 153 | defer q.RUnlock() 154 | 155 | // Check if queue is closed. 156 | if !q.isOpen { 157 | return nil, ErrDBClosed 158 | } 159 | 160 | return q.getItemByID(q.head + 1) 161 | } 162 | 163 | // PeekByOffset returns the item located at the given offset, 164 | // starting from the head of the queue, without removing it. 165 | func (q *Queue) PeekByOffset(offset uint64) (*Item, error) { 166 | q.RLock() 167 | defer q.RUnlock() 168 | 169 | // Check if queue is closed. 170 | if !q.isOpen { 171 | return nil, ErrDBClosed 172 | } 173 | 174 | return q.getItemByID(q.head + offset + 1) 175 | } 176 | 177 | // PeekByID returns the item with the given ID without removing it. 178 | func (q *Queue) PeekByID(id uint64) (*Item, error) { 179 | q.RLock() 180 | defer q.RUnlock() 181 | 182 | // Check if queue is closed. 183 | if !q.isOpen { 184 | return nil, ErrDBClosed 185 | } 186 | 187 | return q.getItemByID(id) 188 | } 189 | 190 | // Update updates an item in the queue without changing its position. 191 | func (q *Queue) Update(id uint64, newValue []byte) (*Item, error) { 192 | q.Lock() 193 | defer q.Unlock() 194 | 195 | // Check if queue is closed. 196 | if !q.isOpen { 197 | return nil, ErrDBClosed 198 | } 199 | 200 | // Check if item exists in queue. 201 | if id <= q.head || id > q.tail { 202 | return nil, ErrOutOfBounds 203 | } 204 | 205 | // Create new Item. 206 | item := &Item{ 207 | ID: id, 208 | Key: idToKey(id), 209 | Value: newValue, 210 | } 211 | 212 | // Update this item in the queue. 213 | if err := q.db.Put(item.Key, item.Value, nil); err != nil { 214 | return nil, err 215 | } 216 | 217 | return item, nil 218 | } 219 | 220 | // UpdateString is a helper function for Update that accepts a value 221 | // as a string rather than a byte slice. 222 | func (q *Queue) UpdateString(id uint64, newValue string) (*Item, error) { 223 | return q.Update(id, []byte(newValue)) 224 | } 225 | 226 | // UpdateObject is a helper function for Update that accepts any 227 | // value type, which is then encoded into a byte slice using 228 | // encoding/gob. 229 | // 230 | // Objects containing pointers with zero values will decode to nil 231 | // when using this function. This is due to how the encoding/gob 232 | // package works. Because of this, you should only use this function 233 | // to encode simple types. 234 | func (q *Queue) UpdateObject(id uint64, newValue interface{}) (*Item, error) { 235 | var buffer bytes.Buffer 236 | enc := gob.NewEncoder(&buffer) 237 | if err := enc.Encode(newValue); err != nil { 238 | return nil, err 239 | } 240 | return q.Update(id, buffer.Bytes()) 241 | } 242 | 243 | // UpdateObjectAsJSON is a helper function for Update that accepts 244 | // any value type, which is then encoded into a JSON byte slice using 245 | // encoding/json. 246 | // 247 | // Use this function to handle encoding of complex types. 248 | func (q *Queue) UpdateObjectAsJSON(id uint64, newValue interface{}) (*Item, error) { 249 | jsonBytes, err := json.Marshal(newValue) 250 | if err != nil { 251 | return nil, err 252 | } 253 | 254 | return q.Update(id, jsonBytes) 255 | } 256 | 257 | // Length returns the total number of items in the queue. 258 | func (q *Queue) Length() uint64 { 259 | return q.tail - q.head 260 | } 261 | 262 | // Close closes the LevelDB database of the queue. 263 | func (q *Queue) Close() error { 264 | q.Lock() 265 | defer q.Unlock() 266 | 267 | // Check if queue is already closed. 268 | if !q.isOpen { 269 | return nil 270 | } 271 | 272 | // Close the LevelDB database. 273 | if err := q.db.Close(); err != nil { 274 | return err 275 | } 276 | 277 | // Reset queue head and tail and set 278 | // isOpen to false. 279 | q.head = 0 280 | q.tail = 0 281 | q.isOpen = false 282 | 283 | return nil 284 | } 285 | 286 | // Drop closes and deletes the LevelDB database of the queue. 287 | func (q *Queue) Drop() error { 288 | if err := q.Close(); err != nil { 289 | return err 290 | } 291 | 292 | return os.RemoveAll(q.DataDir) 293 | } 294 | 295 | // getItemByID returns an item, if found, for the given ID. 296 | func (q *Queue) getItemByID(id uint64) (*Item, error) { 297 | // Check if empty or out of bounds. 298 | if q.Length() == 0 { 299 | return nil, ErrEmpty 300 | } else if id <= q.head || id > q.tail { 301 | return nil, ErrOutOfBounds 302 | } 303 | 304 | // Get item from database. 305 | var err error 306 | item := &Item{ID: id, Key: idToKey(id)} 307 | if item.Value, err = q.db.Get(item.Key, nil); err != nil { 308 | return nil, err 309 | } 310 | 311 | return item, nil 312 | } 313 | 314 | // init initializes the queue data. 315 | func (q *Queue) init() error { 316 | // Create a new LevelDB Iterator. 317 | iter := q.db.NewIterator(nil, nil) 318 | defer iter.Release() 319 | 320 | // Set queue head to the first item. 321 | if iter.First() { 322 | q.head = keyToID(iter.Key()) - 1 323 | } 324 | 325 | // Set queue tail to the last item. 326 | if iter.Last() { 327 | q.tail = keyToID(iter.Key()) 328 | } 329 | 330 | return iter.Error() 331 | } 332 | -------------------------------------------------------------------------------- /queue_test.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestQueueClose(t *testing.T) { 11 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 12 | q, err := OpenQueue(file) 13 | if err != nil { 14 | t.Error(err) 15 | } 16 | defer q.Drop() 17 | 18 | if _, err = q.EnqueueString("value"); err != nil { 19 | t.Error(err) 20 | } 21 | 22 | if q.Length() != 1 { 23 | t.Errorf("Expected queue length of 1, got %d", q.Length()) 24 | } 25 | 26 | q.Close() 27 | 28 | if _, err = q.Dequeue(); err != ErrDBClosed { 29 | t.Errorf("Expected to get database closed error, got %s", err.Error()) 30 | } 31 | 32 | if q.Length() != 0 { 33 | t.Errorf("Expected queue length of 0, got %d", q.Length()) 34 | } 35 | } 36 | 37 | func TestQueueDrop(t *testing.T) { 38 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 39 | q, err := OpenQueue(file) 40 | if err != nil { 41 | t.Error(err) 42 | } 43 | 44 | if _, err = os.Stat(file); os.IsNotExist(err) { 45 | t.Error(err) 46 | } 47 | 48 | q.Drop() 49 | 50 | if _, err = os.Stat(file); err == nil { 51 | t.Error("Expected directory for test database to have been deleted") 52 | } 53 | } 54 | 55 | func TestQueueIncompatibleType(t *testing.T) { 56 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 57 | pq, err := OpenPriorityQueue(file, ASC) 58 | if err != nil { 59 | t.Error(err) 60 | } 61 | defer pq.Drop() 62 | pq.Close() 63 | 64 | if _, err = OpenQueue(file); err != ErrIncompatibleType { 65 | t.Error("Expected priority queue to return ErrIncompatibleTypes when opening goquePriorityQueue") 66 | } 67 | } 68 | 69 | func TestQueueEnqueue(t *testing.T) { 70 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 71 | q, err := OpenQueue(file) 72 | if err != nil { 73 | t.Error(err) 74 | } 75 | defer q.Drop() 76 | 77 | for i := 1; i <= 10; i++ { 78 | if _, err = q.EnqueueString(fmt.Sprintf("value for item %d", i)); err != nil { 79 | t.Error(err) 80 | } 81 | } 82 | 83 | if q.Length() != 10 { 84 | t.Errorf("Expected queue size of 10, got %d", q.Length()) 85 | } 86 | } 87 | 88 | func TestQueueDequeue(t *testing.T) { 89 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 90 | q, err := OpenQueue(file) 91 | if err != nil { 92 | t.Error(err) 93 | } 94 | defer q.Drop() 95 | 96 | for i := 1; i <= 10; i++ { 97 | if _, err = q.EnqueueString(fmt.Sprintf("value for item %d", i)); err != nil { 98 | t.Error(err) 99 | } 100 | } 101 | 102 | if q.Length() != 10 { 103 | t.Errorf("Expected queue length of 10, got %d", q.Length()) 104 | } 105 | 106 | deqItem, err := q.Dequeue() 107 | if err != nil { 108 | t.Error(err) 109 | } 110 | 111 | if q.Length() != 9 { 112 | t.Errorf("Expected queue length of 9, got %d", q.Length()) 113 | } 114 | 115 | compStr := "value for item 1" 116 | 117 | if deqItem.ToString() != compStr { 118 | t.Errorf("Expected string to be '%s', got '%s'", compStr, deqItem.ToString()) 119 | } 120 | } 121 | 122 | func TestQueueEncodeDecodePointerJSON(t *testing.T) { 123 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 124 | q, err := OpenQueue(file) 125 | if err != nil { 126 | t.Error(err) 127 | } 128 | defer q.Drop() 129 | 130 | type subObject struct { 131 | Value *int 132 | } 133 | 134 | type object struct { 135 | Value int 136 | SubObject subObject 137 | } 138 | 139 | val := 0 140 | obj := object{ 141 | Value: 0, 142 | SubObject: subObject{ 143 | Value: &val, 144 | }, 145 | } 146 | 147 | if _, err = q.EnqueueObjectAsJSON(obj); err != nil { 148 | t.Error(err) 149 | } 150 | 151 | item, err := q.Dequeue() 152 | if err != nil { 153 | t.Error(err) 154 | } 155 | 156 | var itemObj object 157 | if err := item.ToObjectFromJSON(&itemObj); err != nil { 158 | t.Error(err) 159 | } 160 | 161 | if *itemObj.SubObject.Value != 0 { 162 | t.Errorf("Expected object subobject value to be '0', got '%v'", *itemObj.SubObject.Value) 163 | } 164 | } 165 | 166 | func TestQueuePeek(t *testing.T) { 167 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 168 | q, err := OpenQueue(file) 169 | if err != nil { 170 | t.Error(err) 171 | } 172 | defer q.Drop() 173 | 174 | compStr := "value for item" 175 | 176 | if _, err = q.EnqueueString(compStr); err != nil { 177 | t.Error(err) 178 | } 179 | 180 | peekItem, err := q.Peek() 181 | if err != nil { 182 | t.Error(err) 183 | } 184 | 185 | if peekItem.ToString() != compStr { 186 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 187 | } 188 | 189 | if q.Length() != 1 { 190 | t.Errorf("Expected queue length of 1, got %d", q.Length()) 191 | } 192 | } 193 | 194 | func TestQueuePeekByOffset(t *testing.T) { 195 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 196 | q, err := OpenQueue(file) 197 | if err != nil { 198 | t.Error(err) 199 | } 200 | defer q.Drop() 201 | 202 | for i := 1; i <= 10; i++ { 203 | if _, err = q.EnqueueString(fmt.Sprintf("value for item %d", i)); err != nil { 204 | t.Error(err) 205 | } 206 | } 207 | 208 | compStrFirst := "value for item 1" 209 | compStrLast := "value for item 10" 210 | compStr := "value for item 4" 211 | 212 | peekFirstItem, err := q.PeekByOffset(0) 213 | if err != nil { 214 | t.Error(err) 215 | } 216 | 217 | if peekFirstItem.ToString() != compStrFirst { 218 | t.Errorf("Expected string to be '%s', got '%s'", compStrFirst, peekFirstItem.ToString()) 219 | } 220 | 221 | peekLastItem, err := q.PeekByOffset(9) 222 | if err != nil { 223 | t.Error(err) 224 | } 225 | 226 | if peekLastItem.ToString() != compStrLast { 227 | t.Errorf("Expected string to be '%s', got '%s'", compStrLast, peekLastItem.ToString()) 228 | } 229 | 230 | peekItem, err := q.PeekByOffset(3) 231 | if err != nil { 232 | t.Error(err) 233 | } 234 | 235 | if peekItem.ToString() != compStr { 236 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 237 | } 238 | 239 | if q.Length() != 10 { 240 | t.Errorf("Expected queue length of 10, got %d", q.Length()) 241 | } 242 | } 243 | 244 | func TestQueuePeekByID(t *testing.T) { 245 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 246 | q, err := OpenQueue(file) 247 | if err != nil { 248 | t.Error(err) 249 | } 250 | defer q.Drop() 251 | 252 | for i := 1; i <= 10; i++ { 253 | if _, err = q.EnqueueString(fmt.Sprintf("value for item %d", i)); err != nil { 254 | t.Error(err) 255 | } 256 | } 257 | 258 | compStr := "value for item 3" 259 | 260 | peekItem, err := q.PeekByID(3) 261 | if err != nil { 262 | t.Error(err) 263 | } 264 | 265 | if peekItem.ToString() != compStr { 266 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 267 | } 268 | 269 | if q.Length() != 10 { 270 | t.Errorf("Expected queue length of 10, got %d", q.Length()) 271 | } 272 | } 273 | 274 | func TestQueueUpdate(t *testing.T) { 275 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 276 | q, err := OpenQueue(file) 277 | if err != nil { 278 | t.Error(err) 279 | } 280 | defer q.Drop() 281 | 282 | for i := 1; i <= 10; i++ { 283 | if _, err = q.EnqueueString(fmt.Sprintf("value for item %d", i)); err != nil { 284 | t.Error(err) 285 | } 286 | } 287 | 288 | item, err := q.PeekByID(3) 289 | if err != nil { 290 | t.Error(err) 291 | } 292 | 293 | oldCompStr := "value for item 3" 294 | newCompStr := "new value for item 3" 295 | 296 | if item.ToString() != oldCompStr { 297 | t.Errorf("Expected string to be '%s', got '%s'", oldCompStr, item.ToString()) 298 | } 299 | 300 | updatedItem, err := q.Update(item.ID, []byte(newCompStr)) 301 | if err != nil { 302 | t.Error(err) 303 | } 304 | 305 | if updatedItem.ToString() != newCompStr { 306 | t.Errorf("Expected current item value to be '%s', got '%s'", newCompStr, item.ToString()) 307 | } 308 | 309 | newItem, err := q.PeekByID(3) 310 | if err != nil { 311 | t.Error(err) 312 | } 313 | 314 | if newItem.ToString() != newCompStr { 315 | t.Errorf("Expected new item value to be '%s', got '%s'", newCompStr, item.ToString()) 316 | } 317 | } 318 | 319 | func TestQueueUpdateString(t *testing.T) { 320 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 321 | q, err := OpenQueue(file) 322 | if err != nil { 323 | t.Error(err) 324 | } 325 | defer q.Drop() 326 | 327 | for i := 1; i <= 10; i++ { 328 | if _, err = q.EnqueueString(fmt.Sprintf("value for item %d", i)); err != nil { 329 | t.Error(err) 330 | } 331 | } 332 | 333 | item, err := q.PeekByID(3) 334 | if err != nil { 335 | t.Error(err) 336 | } 337 | 338 | oldCompStr := "value for item 3" 339 | newCompStr := "new value for item 3" 340 | 341 | if item.ToString() != oldCompStr { 342 | t.Errorf("Expected string to be '%s', got '%s'", oldCompStr, item.ToString()) 343 | } 344 | 345 | updatedItem, err := q.UpdateString(item.ID, newCompStr) 346 | if err != nil { 347 | t.Error(err) 348 | } 349 | 350 | if updatedItem.ToString() != newCompStr { 351 | t.Errorf("Expected current item value to be '%s', got '%s'", newCompStr, item.ToString()) 352 | } 353 | 354 | newItem, err := q.PeekByID(3) 355 | if err != nil { 356 | t.Error(err) 357 | } 358 | 359 | if newItem.ToString() != newCompStr { 360 | t.Errorf("Expected new item value to be '%s', got '%s'", newCompStr, item.ToString()) 361 | } 362 | } 363 | 364 | func TestQueueUpdateObject(t *testing.T) { 365 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 366 | q, err := OpenQueue(file) 367 | if err != nil { 368 | t.Error(err) 369 | } 370 | defer q.Drop() 371 | 372 | type object struct { 373 | Value int 374 | } 375 | 376 | for i := 1; i <= 10; i++ { 377 | if _, err = q.EnqueueObject(object{i}); err != nil { 378 | t.Error(err) 379 | } 380 | } 381 | 382 | item, err := q.PeekByID(3) 383 | if err != nil { 384 | t.Error(err) 385 | } 386 | 387 | oldCompObj := object{3} 388 | newCompObj := object{33} 389 | 390 | var obj object 391 | if err := item.ToObject(&obj); err != nil { 392 | t.Error(err) 393 | } 394 | 395 | if obj != oldCompObj { 396 | t.Errorf("Expected object to be '%+v', got '%+v'", oldCompObj, obj) 397 | } 398 | 399 | updatedItem, err := q.UpdateObject(item.ID, newCompObj) 400 | if err != nil { 401 | t.Error(err) 402 | } 403 | 404 | if err := updatedItem.ToObject(&obj); err != nil { 405 | t.Error(err) 406 | } 407 | 408 | if obj != newCompObj { 409 | t.Errorf("Expected current object to be '%+v', got '%+v'", newCompObj, obj) 410 | } 411 | 412 | newItem, err := q.PeekByID(3) 413 | if err != nil { 414 | t.Error(err) 415 | } 416 | 417 | if err := newItem.ToObject(&obj); err != nil { 418 | t.Error(err) 419 | } 420 | 421 | if obj != newCompObj { 422 | t.Errorf("Expected new object to be '%+v', got '%+v'", newCompObj, obj) 423 | } 424 | } 425 | 426 | func TestQueueUpdateObjectAsJSON(t *testing.T) { 427 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 428 | q, err := OpenQueue(file) 429 | if err != nil { 430 | t.Error(err) 431 | } 432 | defer q.Drop() 433 | 434 | type subObject struct { 435 | Value *int 436 | } 437 | 438 | type object struct { 439 | Value int 440 | SubObject subObject 441 | } 442 | 443 | for i := 1; i <= 10; i++ { 444 | obj := object{ 445 | Value: i, 446 | SubObject: subObject{ 447 | Value: &i, 448 | }, 449 | } 450 | 451 | if _, err = q.EnqueueObjectAsJSON(obj); err != nil { 452 | t.Error(err) 453 | } 454 | } 455 | 456 | item, err := q.PeekByID(3) 457 | if err != nil { 458 | t.Error(err) 459 | } 460 | 461 | oldCompObjVal := 3 462 | oldCompObj := object{ 463 | Value: 3, 464 | SubObject: subObject{ 465 | Value: &oldCompObjVal, 466 | }, 467 | } 468 | newCompObjVal := 33 469 | newCompObj := object{ 470 | Value: 33, 471 | SubObject: subObject{ 472 | Value: &newCompObjVal, 473 | }, 474 | } 475 | 476 | var obj object 477 | if err := item.ToObjectFromJSON(&obj); err != nil { 478 | t.Error(err) 479 | } 480 | 481 | if *obj.SubObject.Value != *oldCompObj.SubObject.Value { 482 | t.Errorf("Expected object subobject value to be '%+v', got '%+v'", *oldCompObj.SubObject.Value, *obj.SubObject.Value) 483 | } 484 | 485 | updatedItem, err := q.UpdateObjectAsJSON(item.ID, newCompObj) 486 | if err != nil { 487 | t.Error(err) 488 | } 489 | 490 | if err := updatedItem.ToObjectFromJSON(&obj); err != nil { 491 | t.Error(err) 492 | } 493 | 494 | if *obj.SubObject.Value != *newCompObj.SubObject.Value { 495 | t.Errorf("Expected current object subobject value to be '%+v', got '%+v'", *newCompObj.SubObject.Value, *obj.SubObject.Value) 496 | } 497 | 498 | newItem, err := q.PeekByID(3) 499 | if err != nil { 500 | t.Error(err) 501 | } 502 | 503 | if err := newItem.ToObjectFromJSON(&obj); err != nil { 504 | t.Error(err) 505 | } 506 | 507 | if *obj.SubObject.Value != *newCompObj.SubObject.Value { 508 | t.Errorf("Expected current object subobject value to be '%+v', got '%+v'", *newCompObj.SubObject.Value, *obj.SubObject.Value) 509 | } 510 | } 511 | 512 | func TestQueueUpdateOutOfBounds(t *testing.T) { 513 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 514 | q, err := OpenQueue(file) 515 | if err != nil { 516 | t.Error(err) 517 | } 518 | defer q.Drop() 519 | 520 | for i := 1; i <= 10; i++ { 521 | if _, err = q.EnqueueString(fmt.Sprintf("value for item %d", i)); err != nil { 522 | t.Error(err) 523 | } 524 | } 525 | 526 | if q.Length() != 10 { 527 | t.Errorf("Expected queue length of 10, got %d", q.Length()) 528 | } 529 | 530 | deqItem, err := q.Dequeue() 531 | if err != nil { 532 | t.Error(err) 533 | } 534 | 535 | if q.Length() != 9 { 536 | t.Errorf("Expected queue length of 9, got %d", q.Length()) 537 | } 538 | 539 | if _, err = q.Update(deqItem.ID, []byte(`new value`)); err != ErrOutOfBounds { 540 | t.Errorf("Expected to get queue out of bounds error, got %s", err.Error()) 541 | } 542 | 543 | if _, err = q.Update(deqItem.ID+1, []byte(`new value`)); err != nil { 544 | t.Error(err) 545 | } 546 | } 547 | 548 | func TestQueueEmpty(t *testing.T) { 549 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 550 | q, err := OpenQueue(file) 551 | if err != nil { 552 | t.Error(err) 553 | } 554 | defer q.Drop() 555 | 556 | _, err = q.EnqueueString("value for item") 557 | if err != nil { 558 | t.Error(err) 559 | } 560 | 561 | _, err = q.Dequeue() 562 | if err != nil { 563 | t.Error(err) 564 | } 565 | 566 | _, err = q.Dequeue() 567 | if err != ErrEmpty { 568 | t.Errorf("Expected to get empty error, got %s", err.Error()) 569 | } 570 | } 571 | 572 | func TestQueueOutOfBounds(t *testing.T) { 573 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 574 | q, err := OpenQueue(file) 575 | if err != nil { 576 | t.Error(err) 577 | } 578 | defer q.Drop() 579 | 580 | _, err = q.EnqueueString("value for item") 581 | if err != nil { 582 | t.Error(err) 583 | } 584 | 585 | _, err = q.PeekByOffset(2) 586 | if err != ErrOutOfBounds { 587 | t.Errorf("Expected to get queue out of bounds error, got %s", err.Error()) 588 | } 589 | } 590 | 591 | func BenchmarkQueueEnqueue(b *testing.B) { 592 | // Open test database 593 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 594 | q, err := OpenQueue(file) 595 | if err != nil { 596 | b.Error(err) 597 | } 598 | defer q.Drop() 599 | 600 | b.ResetTimer() 601 | b.ReportAllocs() 602 | 603 | for n := 0; n < b.N; n++ { 604 | _, _ = q.Enqueue([]byte("value")) 605 | } 606 | } 607 | 608 | func BenchmarkQueueDequeue(b *testing.B) { 609 | // Open test database 610 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 611 | q, err := OpenQueue(file) 612 | if err != nil { 613 | b.Error(err) 614 | } 615 | defer q.Drop() 616 | 617 | // Fill with dummy data 618 | for n := 0; n < b.N; n++ { 619 | if _, err = q.Enqueue([]byte("value")); err != nil { 620 | b.Error(err) 621 | } 622 | } 623 | 624 | // Start benchmark 625 | b.ResetTimer() 626 | b.ReportAllocs() 627 | 628 | for n := 0; n < b.N; n++ { 629 | _, _ = q.Dequeue() 630 | } 631 | } 632 | -------------------------------------------------------------------------------- /stack.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "encoding/json" 7 | "os" 8 | "sync" 9 | 10 | "github.com/syndtr/goleveldb/leveldb" 11 | ) 12 | 13 | // Stack is a standard LIFO (last in, first out) stack. 14 | type Stack struct { 15 | sync.RWMutex 16 | DataDir string 17 | db *leveldb.DB 18 | head uint64 19 | tail uint64 20 | isOpen bool 21 | } 22 | 23 | // OpenStack opens a stack if one exists at the given directory. If one 24 | // does not already exist, a new stack is created. 25 | func OpenStack(dataDir string) (*Stack, error) { 26 | var err error 27 | 28 | // Create a new Stack. 29 | s := &Stack{ 30 | DataDir: dataDir, 31 | db: &leveldb.DB{}, 32 | head: 0, 33 | tail: 0, 34 | isOpen: false, 35 | } 36 | 37 | // Open database for the stack. 38 | s.db, err = leveldb.OpenFile(dataDir, nil) 39 | if err != nil { 40 | return s, err 41 | } 42 | 43 | // Check if this Goque type can open the requested data directory. 44 | ok, err := checkGoqueType(dataDir, goqueStack) 45 | if err != nil { 46 | return s, err 47 | } 48 | if !ok { 49 | return s, ErrIncompatibleType 50 | } 51 | 52 | // Set isOpen and return. 53 | s.isOpen = true 54 | return s, s.init() 55 | } 56 | 57 | // Push adds an item to the stack. 58 | func (s *Stack) Push(value []byte) (*Item, error) { 59 | s.Lock() 60 | defer s.Unlock() 61 | 62 | // Check if stack is closed. 63 | if !s.isOpen { 64 | return nil, ErrDBClosed 65 | } 66 | 67 | // Create new Item. 68 | item := &Item{ 69 | ID: s.head + 1, 70 | Key: idToKey(s.head + 1), 71 | Value: value, 72 | } 73 | 74 | // Add it to the stack. 75 | if err := s.db.Put(item.Key, item.Value, nil); err != nil { 76 | return nil, err 77 | } 78 | 79 | // Increment head position. 80 | s.head++ 81 | 82 | return item, nil 83 | } 84 | 85 | // PushString is a helper function for Push that accepts a 86 | // value as a string rather than a byte slice. 87 | func (s *Stack) PushString(value string) (*Item, error) { 88 | return s.Push([]byte(value)) 89 | } 90 | 91 | // PushObject is a helper function for Push that accepts any 92 | // value type, which is then encoded into a byte slice using 93 | // encoding/gob. 94 | // 95 | // Objects containing pointers with zero values will decode to nil 96 | // when using this function. This is due to how the encoding/gob 97 | // package works. Because of this, you should only use this function 98 | // to encode simple types. 99 | func (s *Stack) PushObject(value interface{}) (*Item, error) { 100 | var buffer bytes.Buffer 101 | enc := gob.NewEncoder(&buffer) 102 | if err := enc.Encode(value); err != nil { 103 | return nil, err 104 | } 105 | 106 | return s.Push(buffer.Bytes()) 107 | } 108 | 109 | // PushObjectAsJSON is a helper function for Push that accepts any 110 | // value type, which is then encoded into a JSON byte slice using 111 | // encoding/json. 112 | // 113 | // Use this function to handle encoding of complex types. 114 | func (s *Stack) PushObjectAsJSON(value interface{}) (*Item, error) { 115 | jsonBytes, err := json.Marshal(value) 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | return s.Push(jsonBytes) 121 | } 122 | 123 | // Pop removes the next item in the stack and returns it. 124 | func (s *Stack) Pop() (*Item, error) { 125 | s.Lock() 126 | defer s.Unlock() 127 | 128 | // Check if stack is closed. 129 | if !s.isOpen { 130 | return nil, ErrDBClosed 131 | } 132 | 133 | // Try to get the next item in the stack. 134 | item, err := s.getItemByID(s.head) 135 | if err != nil { 136 | return nil, err 137 | } 138 | 139 | // Remove this item from the stack. 140 | if err := s.db.Delete(item.Key, nil); err != nil { 141 | return nil, err 142 | } 143 | 144 | // Decrement head position. 145 | s.head-- 146 | 147 | return item, nil 148 | } 149 | 150 | // Peek returns the next item in the stack without removing it. 151 | func (s *Stack) Peek() (*Item, error) { 152 | s.RLock() 153 | defer s.RUnlock() 154 | 155 | // Check if stack is closed. 156 | if !s.isOpen { 157 | return nil, ErrDBClosed 158 | } 159 | 160 | return s.getItemByID(s.head) 161 | } 162 | 163 | // PeekByOffset returns the item located at the given offset, 164 | // starting from the head of the stack, without removing it. 165 | func (s *Stack) PeekByOffset(offset uint64) (*Item, error) { 166 | s.RLock() 167 | defer s.RUnlock() 168 | 169 | // Check if stack is closed. 170 | if !s.isOpen { 171 | return nil, ErrDBClosed 172 | } 173 | 174 | return s.getItemByID(s.head - offset) 175 | } 176 | 177 | // PeekByID returns the item with the given ID without removing it. 178 | func (s *Stack) PeekByID(id uint64) (*Item, error) { 179 | s.RLock() 180 | defer s.RUnlock() 181 | 182 | // Check if stack is closed. 183 | if !s.isOpen { 184 | return nil, ErrDBClosed 185 | } 186 | 187 | return s.getItemByID(id) 188 | } 189 | 190 | // Update updates an item in the stack without changing its position. 191 | func (s *Stack) Update(id uint64, newValue []byte) (*Item, error) { 192 | s.Lock() 193 | defer s.Unlock() 194 | 195 | // Check if stack is closed. 196 | if !s.isOpen { 197 | return nil, ErrDBClosed 198 | } 199 | 200 | // Check if item exists in stack. 201 | if id > s.head || id <= s.tail { 202 | return nil, ErrOutOfBounds 203 | } 204 | 205 | // Create new Item. 206 | item := &Item{ 207 | ID: id, 208 | Key: idToKey(id), 209 | Value: newValue, 210 | } 211 | 212 | // Update this item in the stack. 213 | if err := s.db.Put(item.Key, item.Value, nil); err != nil { 214 | return nil, err 215 | } 216 | 217 | return item, nil 218 | } 219 | 220 | // UpdateString is a helper function for Update that accepts a value 221 | // as a string rather than a byte slice. 222 | func (s *Stack) UpdateString(id uint64, newValue string) (*Item, error) { 223 | return s.Update(id, []byte(newValue)) 224 | } 225 | 226 | // UpdateObject is a helper function for Update that accepts any 227 | // value type, which is then encoded into a byte slice using 228 | // encoding/gob. 229 | // 230 | // Objects containing pointers with zero values will decode to nil 231 | // when using this function. This is due to how the encoding/gob 232 | // package works. Because of this, you should only use this function 233 | // to encode simple types. 234 | func (s *Stack) UpdateObject(id uint64, newValue interface{}) (*Item, error) { 235 | var buffer bytes.Buffer 236 | enc := gob.NewEncoder(&buffer) 237 | if err := enc.Encode(newValue); err != nil { 238 | return nil, err 239 | } 240 | return s.Update(id, buffer.Bytes()) 241 | } 242 | 243 | // UpdateObjectAsJSON is a helper function for Update that accepts 244 | // any value type, which is then encoded into a JSON byte slice using 245 | // encoding/json. 246 | // 247 | // Use this function to handle encoding of complex types. 248 | func (s *Stack) UpdateObjectAsJSON(id uint64, newValue interface{}) (*Item, error) { 249 | jsonBytes, err := json.Marshal(newValue) 250 | if err != nil { 251 | return nil, err 252 | } 253 | 254 | return s.Update(id, jsonBytes) 255 | } 256 | 257 | // Length returns the total number of items in the stack. 258 | func (s *Stack) Length() uint64 { 259 | return s.head - s.tail 260 | } 261 | 262 | // Close closes the LevelDB database of the stack. 263 | func (s *Stack) Close() error { 264 | s.Lock() 265 | defer s.Unlock() 266 | 267 | // Check if stack is already closed. 268 | if !s.isOpen { 269 | return nil 270 | } 271 | 272 | // Close the LevelDB database. 273 | if err := s.db.Close(); err != nil { 274 | return err 275 | } 276 | 277 | // Reset stack head and tail and set 278 | // isOpen to false. 279 | s.head = 0 280 | s.tail = 0 281 | s.isOpen = false 282 | 283 | return nil 284 | } 285 | 286 | // Drop closes and deletes the LevelDB database of the stack. 287 | func (s *Stack) Drop() error { 288 | if err := s.Close(); err != nil { 289 | return err 290 | } 291 | 292 | return os.RemoveAll(s.DataDir) 293 | } 294 | 295 | // getItemByID returns an item, if found, for the given ID. 296 | func (s *Stack) getItemByID(id uint64) (*Item, error) { 297 | // Check if empty or out of bounds. 298 | if s.Length() == 0 { 299 | return nil, ErrEmpty 300 | } else if id <= s.tail || id > s.head { 301 | return nil, ErrOutOfBounds 302 | } 303 | 304 | // Get item from database. 305 | var err error 306 | item := &Item{ID: id, Key: idToKey(id)} 307 | if item.Value, err = s.db.Get(item.Key, nil); err != nil { 308 | return nil, err 309 | } 310 | 311 | return item, nil 312 | } 313 | 314 | // init initializes the stack data. 315 | func (s *Stack) init() error { 316 | // Create a new LevelDB Iterator. 317 | iter := s.db.NewIterator(nil, nil) 318 | defer iter.Release() 319 | 320 | // Set stack head to the last item. 321 | if iter.Last() { 322 | s.head = keyToID(iter.Key()) 323 | } 324 | 325 | // Set stack tail to the first item. 326 | if iter.First() { 327 | s.tail = keyToID(iter.Key()) - 1 328 | } 329 | 330 | return iter.Error() 331 | } 332 | -------------------------------------------------------------------------------- /stack_test.go: -------------------------------------------------------------------------------- 1 | package goque 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestStackClose(t *testing.T) { 11 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 12 | s, err := OpenStack(file) 13 | if err != nil { 14 | t.Error(err) 15 | } 16 | defer s.Drop() 17 | 18 | if _, err = s.PushString("value"); err != nil { 19 | t.Error(err) 20 | } 21 | 22 | if s.Length() != 1 { 23 | t.Errorf("Expected stack length of 1, got %d", s.Length()) 24 | } 25 | 26 | s.Close() 27 | 28 | if _, err = s.Pop(); err != ErrDBClosed { 29 | t.Errorf("Expected to get database closed error, got %s", err.Error()) 30 | } 31 | 32 | if s.Length() != 0 { 33 | t.Errorf("Expected stack length of 0, got %d", s.Length()) 34 | } 35 | } 36 | 37 | func TestStackDrop(t *testing.T) { 38 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 39 | s, err := OpenStack(file) 40 | if err != nil { 41 | t.Error(err) 42 | } 43 | 44 | if _, err = os.Stat(file); os.IsNotExist(err) { 45 | t.Error(err) 46 | } 47 | 48 | s.Drop() 49 | 50 | if _, err = os.Stat(file); err == nil { 51 | t.Error("Expected directory for test database to have been deleted") 52 | } 53 | } 54 | 55 | func TestStackIncompatibleType(t *testing.T) { 56 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 57 | pq, err := OpenPriorityQueue(file, ASC) 58 | if err != nil { 59 | t.Error(err) 60 | } 61 | defer pq.Drop() 62 | pq.Close() 63 | 64 | if _, err = OpenStack(file); err != ErrIncompatibleType { 65 | t.Error("Expected stack to return ErrIncompatibleTypes when opening goquePriorityQueue") 66 | } 67 | } 68 | 69 | func TestStackPush(t *testing.T) { 70 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 71 | s, err := OpenStack(file) 72 | if err != nil { 73 | t.Error(err) 74 | } 75 | defer s.Drop() 76 | 77 | for i := 1; i <= 10; i++ { 78 | if _, err = s.PushString(fmt.Sprintf("value for item %d", i)); err != nil { 79 | t.Error(err) 80 | } 81 | } 82 | 83 | if s.Length() != 10 { 84 | t.Errorf("Expected stack size of 10, got %d", s.Length()) 85 | } 86 | } 87 | 88 | func TestStackPop(t *testing.T) { 89 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 90 | s, err := OpenStack(file) 91 | if err != nil { 92 | t.Error(err) 93 | } 94 | defer s.Drop() 95 | 96 | for i := 1; i <= 10; i++ { 97 | if _, err = s.PushString(fmt.Sprintf("value for item %d", i)); err != nil { 98 | t.Error(err) 99 | } 100 | } 101 | 102 | if s.Length() != 10 { 103 | t.Errorf("Expected stack length of 10, got %d", s.Length()) 104 | } 105 | 106 | popItem, err := s.Pop() 107 | if err != nil { 108 | t.Error(err) 109 | } 110 | 111 | if s.Length() != 9 { 112 | t.Errorf("Expected stack length of 9, got %d", s.Length()) 113 | } 114 | 115 | compStr := "value for item 10" 116 | 117 | if popItem.ToString() != compStr { 118 | t.Errorf("Expected string to be '%s', got '%s'", compStr, popItem.ToString()) 119 | } 120 | } 121 | 122 | func TestStackPushPopPointerJSON(t *testing.T) { 123 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 124 | q, err := OpenStack(file) 125 | if err != nil { 126 | t.Error(err) 127 | } 128 | defer q.Drop() 129 | 130 | type subObject struct { 131 | Value *int 132 | } 133 | 134 | type object struct { 135 | Value int 136 | SubObject subObject 137 | } 138 | 139 | val := 0 140 | obj := object{ 141 | Value: 0, 142 | SubObject: subObject{ 143 | Value: &val, 144 | }, 145 | } 146 | 147 | if _, err = q.PushObjectAsJSON(obj); err != nil { 148 | t.Error(err) 149 | } 150 | 151 | item, err := q.Pop() 152 | if err != nil { 153 | t.Error(err) 154 | } 155 | 156 | var itemObj object 157 | if err := item.ToObjectFromJSON(&itemObj); err != nil { 158 | t.Error(err) 159 | } 160 | 161 | if *itemObj.SubObject.Value != 0 { 162 | t.Errorf("Expected object subobject value to be '0', got '%v'", *itemObj.SubObject.Value) 163 | } 164 | } 165 | 166 | func TestStackPeek(t *testing.T) { 167 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 168 | s, err := OpenStack(file) 169 | if err != nil { 170 | t.Error(err) 171 | } 172 | defer s.Drop() 173 | 174 | compStr := "value for item" 175 | 176 | if _, err = s.PushString(compStr); err != nil { 177 | t.Error(err) 178 | } 179 | 180 | peekItem, err := s.Peek() 181 | if err != nil { 182 | t.Error(err) 183 | } 184 | 185 | if peekItem.ToString() != compStr { 186 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 187 | } 188 | 189 | if s.Length() != 1 { 190 | t.Errorf("Expected stack length of 1, got %d", s.Length()) 191 | } 192 | } 193 | 194 | func TestStackPeekByOffset(t *testing.T) { 195 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 196 | s, err := OpenStack(file) 197 | if err != nil { 198 | t.Error(err) 199 | } 200 | defer s.Drop() 201 | 202 | for i := 1; i <= 10; i++ { 203 | if _, err = s.PushString(fmt.Sprintf("value for item %d", i)); err != nil { 204 | t.Error(err) 205 | } 206 | } 207 | 208 | compStrFirst := "value for item 10" 209 | compStrLast := "value for item 1" 210 | compStr := "value for item 7" 211 | 212 | peekFirstItem, err := s.PeekByOffset(0) 213 | if err != nil { 214 | t.Error(err) 215 | } 216 | 217 | if peekFirstItem.ToString() != compStrFirst { 218 | t.Errorf("Expected string to be '%s', got '%s'", compStrFirst, peekFirstItem.ToString()) 219 | } 220 | 221 | peekLastItem, err := s.PeekByOffset(9) 222 | if err != nil { 223 | t.Error(err) 224 | } 225 | 226 | if peekLastItem.ToString() != compStrLast { 227 | t.Errorf("Expected string to be '%s', got '%s'", compStrLast, peekLastItem.ToString()) 228 | } 229 | 230 | peekItem, err := s.PeekByOffset(3) 231 | if err != nil { 232 | t.Error(err) 233 | } 234 | 235 | if peekItem.ToString() != compStr { 236 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 237 | } 238 | 239 | if s.Length() != 10 { 240 | t.Errorf("Expected stack length of 10, got %d", s.Length()) 241 | } 242 | } 243 | 244 | func TestStackPeekByID(t *testing.T) { 245 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 246 | s, err := OpenStack(file) 247 | if err != nil { 248 | t.Error(err) 249 | } 250 | defer s.Drop() 251 | 252 | for i := 1; i <= 10; i++ { 253 | if _, err = s.PushString(fmt.Sprintf("value for item %d", i)); err != nil { 254 | t.Error(err) 255 | } 256 | } 257 | 258 | compStr := "value for item 3" 259 | 260 | peekItem, err := s.PeekByID(3) 261 | if err != nil { 262 | t.Error(err) 263 | } 264 | 265 | if peekItem.ToString() != compStr { 266 | t.Errorf("Expected string to be '%s', got '%s'", compStr, peekItem.ToString()) 267 | } 268 | 269 | if s.Length() != 10 { 270 | t.Errorf("Expected stack length of 10, got %d", s.Length()) 271 | } 272 | } 273 | 274 | func TestStackUpdate(t *testing.T) { 275 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 276 | s, err := OpenStack(file) 277 | if err != nil { 278 | t.Error(err) 279 | } 280 | defer s.Drop() 281 | 282 | for i := 1; i <= 10; i++ { 283 | if _, err = s.PushString(fmt.Sprintf("value for item %d", i)); err != nil { 284 | t.Error(err) 285 | } 286 | } 287 | 288 | item, err := s.PeekByID(3) 289 | if err != nil { 290 | t.Error(err) 291 | } 292 | 293 | oldCompStr := "value for item 3" 294 | newCompStr := "new value for item 3" 295 | 296 | if item.ToString() != oldCompStr { 297 | t.Errorf("Expected string to be '%s', got '%s'", oldCompStr, item.ToString()) 298 | } 299 | 300 | updatedItem, err := s.Update(item.ID, []byte(newCompStr)) 301 | if err != nil { 302 | t.Error(err) 303 | } 304 | 305 | if updatedItem.ToString() != newCompStr { 306 | t.Errorf("Expected current item value to be '%s', got '%s'", newCompStr, item.ToString()) 307 | } 308 | 309 | newItem, err := s.PeekByID(3) 310 | if err != nil { 311 | t.Error(err) 312 | } 313 | 314 | if newItem.ToString() != newCompStr { 315 | t.Errorf("Expected new item value to be '%s', got '%s'", newCompStr, item.ToString()) 316 | } 317 | } 318 | 319 | func TestStackUpdateString(t *testing.T) { 320 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 321 | s, err := OpenStack(file) 322 | if err != nil { 323 | t.Error(err) 324 | } 325 | defer s.Drop() 326 | 327 | for i := 1; i <= 10; i++ { 328 | if _, err = s.PushString(fmt.Sprintf("value for item %d", i)); err != nil { 329 | t.Error(err) 330 | } 331 | } 332 | 333 | item, err := s.PeekByID(3) 334 | if err != nil { 335 | t.Error(err) 336 | } 337 | 338 | oldCompStr := "value for item 3" 339 | newCompStr := "new value for item 3" 340 | 341 | if item.ToString() != oldCompStr { 342 | t.Errorf("Expected string to be '%s', got '%s'", oldCompStr, item.ToString()) 343 | } 344 | 345 | updatedItem, err := s.UpdateString(item.ID, newCompStr) 346 | if err != nil { 347 | t.Error(err) 348 | } 349 | 350 | if updatedItem.ToString() != newCompStr { 351 | t.Errorf("Expected current item value to be '%s', got '%s'", newCompStr, item.ToString()) 352 | } 353 | 354 | newItem, err := s.PeekByID(3) 355 | if err != nil { 356 | t.Error(err) 357 | } 358 | 359 | if newItem.ToString() != newCompStr { 360 | t.Errorf("Expected new item value to be '%s', got '%s'", newCompStr, item.ToString()) 361 | } 362 | } 363 | 364 | func TestStackUpdateObject(t *testing.T) { 365 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 366 | s, err := OpenStack(file) 367 | if err != nil { 368 | t.Error(err) 369 | } 370 | defer s.Drop() 371 | 372 | type object struct { 373 | Value int 374 | } 375 | 376 | for i := 1; i <= 10; i++ { 377 | if _, err = s.PushObject(object{i}); err != nil { 378 | t.Error(err) 379 | } 380 | } 381 | 382 | item, err := s.PeekByID(3) 383 | if err != nil { 384 | t.Error(err) 385 | } 386 | 387 | oldCompObj := object{3} 388 | newCompObj := object{33} 389 | 390 | var obj object 391 | if err := item.ToObject(&obj); err != nil { 392 | t.Error(err) 393 | } 394 | 395 | if obj != oldCompObj { 396 | t.Errorf("Expected object to be '%+v', got '%+v'", oldCompObj, obj) 397 | } 398 | 399 | updatedItem, err := s.UpdateObject(item.ID, newCompObj) 400 | if err != nil { 401 | t.Error(err) 402 | } 403 | 404 | if err := updatedItem.ToObject(&obj); err != nil { 405 | t.Error(err) 406 | } 407 | 408 | if obj != newCompObj { 409 | t.Errorf("Expected current object to be '%+v', got '%+v'", newCompObj, obj) 410 | } 411 | 412 | newItem, err := s.PeekByID(3) 413 | if err != nil { 414 | t.Error(err) 415 | } 416 | 417 | if err := newItem.ToObject(&obj); err != nil { 418 | t.Error(err) 419 | } 420 | 421 | if obj != newCompObj { 422 | t.Errorf("Expected new object to be '%+v', got '%+v'", newCompObj, obj) 423 | } 424 | } 425 | 426 | func TestStackUpdateObjectAsJSON(t *testing.T) { 427 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 428 | s, err := OpenStack(file) 429 | if err != nil { 430 | t.Error(err) 431 | } 432 | defer s.Drop() 433 | 434 | type subObject struct { 435 | Value *int 436 | } 437 | 438 | type object struct { 439 | Value int 440 | SubObject subObject 441 | } 442 | 443 | for i := 1; i <= 10; i++ { 444 | obj := object{ 445 | Value: i, 446 | SubObject: subObject{ 447 | Value: &i, 448 | }, 449 | } 450 | 451 | if _, err = s.PushObjectAsJSON(obj); err != nil { 452 | t.Error(err) 453 | } 454 | } 455 | 456 | item, err := s.PeekByID(3) 457 | if err != nil { 458 | t.Error(err) 459 | } 460 | 461 | oldCompObjVal := 3 462 | oldCompObj := object{ 463 | Value: 3, 464 | SubObject: subObject{ 465 | Value: &oldCompObjVal, 466 | }, 467 | } 468 | newCompObjVal := 33 469 | newCompObj := object{ 470 | Value: 33, 471 | SubObject: subObject{ 472 | Value: &newCompObjVal, 473 | }, 474 | } 475 | 476 | var obj object 477 | if err := item.ToObjectFromJSON(&obj); err != nil { 478 | t.Error(err) 479 | } 480 | 481 | if *obj.SubObject.Value != *oldCompObj.SubObject.Value { 482 | t.Errorf("Expected object subobject value to be '%+v', got '%+v'", *oldCompObj.SubObject.Value, *obj.SubObject.Value) 483 | } 484 | 485 | updatedItem, err := s.UpdateObjectAsJSON(item.ID, newCompObj) 486 | if err != nil { 487 | t.Error(err) 488 | } 489 | 490 | if err := updatedItem.ToObjectFromJSON(&obj); err != nil { 491 | t.Error(err) 492 | } 493 | 494 | if *obj.SubObject.Value != *newCompObj.SubObject.Value { 495 | t.Errorf("Expected current object subobject value to be '%+v', got '%+v'", *newCompObj.SubObject.Value, *obj.SubObject.Value) 496 | } 497 | 498 | newItem, err := s.PeekByID(3) 499 | if err != nil { 500 | t.Error(err) 501 | } 502 | 503 | if err := newItem.ToObjectFromJSON(&obj); err != nil { 504 | t.Error(err) 505 | } 506 | 507 | if *obj.SubObject.Value != *newCompObj.SubObject.Value { 508 | t.Errorf("Expected current object subobject value to be '%+v', got '%+v'", *newCompObj.SubObject.Value, *obj.SubObject.Value) 509 | } 510 | } 511 | 512 | func TestStackUpdateOutOfBounds(t *testing.T) { 513 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 514 | s, err := OpenStack(file) 515 | if err != nil { 516 | t.Error(err) 517 | } 518 | defer s.Drop() 519 | 520 | for i := 1; i <= 10; i++ { 521 | if _, err = s.PushString(fmt.Sprintf("value for item %d", i)); err != nil { 522 | t.Error(err) 523 | } 524 | } 525 | 526 | if s.Length() != 10 { 527 | t.Errorf("Expected stack length of 10, got %d", s.Length()) 528 | } 529 | 530 | popItem, err := s.Pop() 531 | if err != nil { 532 | t.Error(err) 533 | } 534 | 535 | if s.Length() != 9 { 536 | t.Errorf("Expected stack length of 9, got %d", s.Length()) 537 | } 538 | 539 | if _, err = s.Update(popItem.ID, []byte(`new value`)); err != ErrOutOfBounds { 540 | t.Errorf("Expected to get stack out of bounds error, got %s", err.Error()) 541 | } 542 | 543 | if _, err = s.Update(popItem.ID-1, []byte(`new value`)); err != nil { 544 | t.Error(err) 545 | } 546 | } 547 | 548 | func TestStackEmpty(t *testing.T) { 549 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 550 | s, err := OpenStack(file) 551 | if err != nil { 552 | t.Error(err) 553 | } 554 | defer s.Drop() 555 | 556 | _, err = s.PushString("value for item") 557 | if err != nil { 558 | t.Error(err) 559 | } 560 | 561 | _, err = s.Pop() 562 | if err != nil { 563 | t.Error(err) 564 | } 565 | 566 | _, err = s.Pop() 567 | if err != ErrEmpty { 568 | t.Errorf("Expected to get empty error, got %s", err.Error()) 569 | } 570 | } 571 | 572 | func TestStackOutOfBounds(t *testing.T) { 573 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 574 | s, err := OpenStack(file) 575 | if err != nil { 576 | t.Error(err) 577 | } 578 | defer s.Drop() 579 | 580 | _, err = s.PushString("value for item") 581 | if err != nil { 582 | t.Error(err) 583 | } 584 | 585 | _, err = s.PeekByOffset(2) 586 | if err != ErrOutOfBounds { 587 | t.Errorf("Expected to get stack out of bounds error, got %s", err.Error()) 588 | } 589 | } 590 | 591 | func BenchmarkStackPush(b *testing.B) { 592 | // Open test database 593 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 594 | s, err := OpenStack(file) 595 | if err != nil { 596 | b.Error(err) 597 | } 598 | defer s.Drop() 599 | 600 | b.ResetTimer() 601 | b.ReportAllocs() 602 | 603 | for n := 0; n < b.N; n++ { 604 | _, _ = s.PushString("value") 605 | } 606 | } 607 | 608 | func BenchmarkStackPop(b *testing.B) { 609 | // Open test database 610 | file := fmt.Sprintf("test_db_%d", time.Now().UnixNano()) 611 | s, err := OpenStack(file) 612 | if err != nil { 613 | b.Error(err) 614 | } 615 | defer s.Drop() 616 | 617 | // Fill with dummy data 618 | for n := 0; n < b.N; n++ { 619 | if _, err = s.PushString("value"); err != nil { 620 | b.Error(err) 621 | } 622 | } 623 | 624 | // Start benchmark 625 | b.ResetTimer() 626 | b.ReportAllocs() 627 | 628 | for n := 0; n < b.N; n++ { 629 | _, _ = s.Pop() 630 | } 631 | } 632 | --------------------------------------------------------------------------------