├── .github └── workflows │ ├── lint.yaml │ └── test.yaml ├── LICENSE ├── README.md ├── benchmark_test.go ├── example_test.go ├── go.mod ├── go.sum ├── queue.go ├── queue_test.go ├── segment.go ├── segment_test.go └── util.go /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: lint 2 | on: [push, pull_request] 3 | jobs: 4 | lint: 5 | runs-on: ubuntu-latest 6 | env: 7 | GOLANGCILINTVERSION: 1.22.2 8 | steps: 9 | - uses: actions/checkout@v1 10 | - name: Download golangci-lint 11 | run: curl -sL https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCILINTVERSION}/golangci-lint-${GOLANGCILINTVERSION}-linux-amd64.tar.gz | tar xz 12 | - name: Run golangci-lint 13 | run: golangci-lint-${GOLANGCILINTVERSION}-linux-amd64/golangci-lint run 14 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | on: [push, pull_request] 3 | jobs: 4 | test: 5 | runs-on: ubuntu-latest 6 | strategy: 7 | matrix: 8 | go: [1.13, 1.11] 9 | env: 10 | GO111MODULE: auto 11 | steps: 12 | - uses: actions/checkout@v1 13 | - name: Setup go 14 | uses: actions/setup-go@v1 15 | with: 16 | go-version: ${{ matrix.go }} 17 | - name: Download dependencies 18 | run: go mod download && go mod verify && go mod graph 19 | - name: Build tests 20 | run: go test -c 21 | - name: Run unit tests 22 | run: go test -race -cover ./... 23 | - name: Test building on windows 24 | env: {GOOS: windows} 25 | run: go test -c 26 | - name: Test building on darwin 27 | env: {GOOS: darwin} 28 | run: go test -c 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Jon Carlson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # dque - a fast embedded durable queue for Go 3 | 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/joncrlsn/dque)](https://goreportcard.com/report/github.com/joncrlsn/dque) 5 | [![GoDoc](https://godoc.org/github.com/joncrlsn/dque?status.svg)](https://godoc.org/github.com/joncrlsn/dque) 6 | 7 | dque is: 8 | 9 | * persistent -- survives program restarts 10 | * scalable -- not limited by your RAM, but by your disk space 11 | * FIFO -- First In First Out 12 | * embedded -- compiled into your Golang program 13 | * synchronized -- safe for concurrent usage 14 | * fast or safe, you choose -- turbo mode lets the OS decide when to write to disk 15 | * has a liberal license -- allows any use, commercial or personal 16 | 17 | I love tools that do one thing well. Hopefully this fits that category. 18 | 19 | I am indebted to Gabor Cselle who, years ago, inspired me with an example of an [in-memory persistent queue written in Java](http://www.gaborcselle.com/open_source/java/persistent_queue.html). I was intrigued by the simplicity of his approach, which became the foundation of the "segment" part of this queue which holds the head and the tail of the queue in memory as well as storing the segment files in between. 20 | 21 | ### performance 22 | 23 | There are two performance modes: safe and turbo 24 | 25 | ##### safe mode 26 | 27 | * safe mode is the default 28 | * forces an fsync to disk every time you enqueue or dequeue an item. 29 | * while this is the safest way to use dque with little risk of data loss, it is also the slowest. 30 | 31 | ##### turbo mode 32 | 33 | * can be enabled/disabled with a call to [DQue.TurboOn()](https://godoc.org/github.com/joncrlsn/dque#DQue.TurboOn) or [DQue.TurboOff()](https://godoc.org/github.com/joncrlsn/dque#DQue.TurboOff) 34 | * lets the OS batch up your changes to disk, which makes it a lot faster. 35 | * also allows you to flush changes to disk at opportune times. See [DQue.TurboSync()](https://godoc.org/github.com/joncrlsn/dque#DQue.TurboSync) 36 | * comes with a risk that a power failure could lose changes. By turning on Turbo mode you accept that risk. 37 | * run the benchmark to see the difference on your hardware. 38 | * there is a todo item to force flush changes to disk after a configurable amount of time to limit risk. 39 | 40 | ### implementation 41 | 42 | * The queue is held in segments of a configurable size. 43 | * The queue is protected against re-opening from other processes. 44 | * Each in-memory segment corresponds with a file on disk. Think of the segment files as a bit like rolling log files. The oldest segment files are eventually deleted, not based on time, but whenever their items have all been dequeued. 45 | * Segment files are only appended to until they fill up. At which point a new segment is created. They are never modified (other than being appended to and deleted when each of their items has been dequeued). 46 | * If there is more than one segment, new items are enqueued to the last segment while dequeued items are taken from the first segment. 47 | * Because the encoding/gob package is used to store the struct to disk: 48 | * Only structs can be stored in the queue. 49 | * Only one type of struct can be stored in each queue. 50 | * Only public fields in a struct will be stored. 51 | * A function is required that returns a pointer to a new struct of the type stored in the queue. This function is used when loading segments into memory from disk. I'd love to find a way to avoid this function. 52 | * Queue segment implementation: 53 | * For nice visuals, see [Gabor Cselle's documentation here](http://www.gaborcselle.com/open_source/java/persistent_queue.html). Note that Gabor's implementation kept the entire queue in memory as well as disk. dque keeps only the head and tail segments in memory. 54 | * Enqueueing an item adds it both to the end of the last segment file and to the in-memory item slice for that segment. 55 | * When a segment reaches its maximum size a new segment is created. 56 | * Dequeueing an item removes it from the beginning of the in-memory slice and appends a 4-byte "delete" marker to the end of the segment file. This allows the item to be left in the file until the number of delete markers matches the number of items, at which point the entire file is deleted. 57 | * When a segment is reconstituted from disk, each "delete" marker found in the file causes a removal of the first element of the in-memory slice. 58 | * When each item in the segment has been dequeued, the segment file is deleted and the next segment is loaded into memory. 59 | 60 | ### example 61 | 62 | See the [full example code here](https://raw.githubusercontent.com/joncrlsn/dque/v2/example_test.go) 63 | 64 | Or a shortened version here: 65 | 66 | ```golang 67 | package dque_test 68 | 69 | import ( 70 | "log" 71 | 72 | "github.com/joncrlsn/dque" 73 | ) 74 | 75 | // Item is what we'll be storing in the queue. It can be any struct 76 | // as long as the fields you want stored are public. 77 | type Item struct { 78 | Name string 79 | Id int 80 | } 81 | 82 | // ItemBuilder creates a new item and returns a pointer to it. 83 | // This is used when we load a segment of the queue from disk. 84 | func ItemBuilder() interface{} { 85 | return &Item{} 86 | } 87 | 88 | func main() { 89 | ExampleDQue_main() 90 | } 91 | 92 | // ExampleQueue_main() show how the queue works 93 | func ExampleDQue_main() { 94 | qName := "item-queue" 95 | qDir := "/tmp" 96 | segmentSize := 50 97 | 98 | // Create a new queue with segment size of 50 99 | q, err := dque.New(qName, qDir, segmentSize, ItemBuilder) 100 | ... 101 | 102 | // Add an item to the queue 103 | err := q.Enqueue(&Item{"Joe", 1}) 104 | ... 105 | 106 | // Properly close a queue 107 | q.Close() 108 | 109 | // You can reconsitute the queue from disk at any time 110 | q, err = dque.Open(qName, qDir, segmentSize, ItemBuilder) 111 | ... 112 | 113 | // Peek at the next item in the queue 114 | var iface interface{} 115 | if iface, err = q.Peek(); err != nil { 116 | if err != dque.ErrEmpty { 117 | log.Fatal("Error peeking at item ", err) 118 | } 119 | } 120 | 121 | // Dequeue the next item in the queue 122 | if iface, err = q.Dequeue(); err != nil { 123 | if err != dque.ErrEmpty { 124 | log.Fatal("Error dequeuing item ", err) 125 | } 126 | } 127 | 128 | // Dequeue the next item in the queue and block until one is available 129 | if iface, err = q.DequeueBlock(); err != nil { 130 | log.Fatal("Error dequeuing item ", err) 131 | } 132 | 133 | // Assert type of the response to an Item pointer so we can work with it 134 | item, ok := iface.(*Item) 135 | if !ok { 136 | log.Fatal("Dequeued object is not an Item pointer") 137 | } 138 | 139 | doSomething(item) 140 | } 141 | 142 | func doSomething(item *Item) { 143 | log.Println("Dequeued", item) 144 | } 145 | ``` 146 | 147 | ### contributors 148 | 149 | * [Neil Isaac](https://github.com/neilisaac) 150 | * [Thomas Kriechbaumer](https://github.com/Kriechi) 151 | 152 | ### todo? (feel free to submit pull requests) 153 | 154 | * add option to enable turbo with a timeout that would ensure you would never lose more than n seconds of changes. 155 | * add Lock() and Unlock() methods so you can peek at the first item and then conditionally dequeue it without worrying that another goroutine has grabbed it out from under you. The use case is when you don't want to actually remove it from the queue until you know you were able to successfully handle it. 156 | * store the segment size in a config file inside the queue. Then it only needs to be specified on dque.New(...) 157 | 158 | ### alternative tools 159 | 160 | * [CurlyQ](https://github.com/mcmathja/curlyq) is a bit heavier (requires Redis) but has more background processing features. 161 | 162 | -------------------------------------------------------------------------------- /benchmark_test.go: -------------------------------------------------------------------------------- 1 | // benchmark_test.go 2 | // 3 | // Benchmarks to see how long each operation takes on average. 4 | // 5 | // Example: go test -bench=. 6 | // 7 | package dque_test 8 | 9 | import ( 10 | "os" 11 | "testing" 12 | 13 | "github.com/joncrlsn/dque" 14 | ) 15 | 16 | // item3 is the thing we'll be storing in the queue 17 | type item3 struct { 18 | Name string 19 | Id int 20 | SomeBool bool 21 | } 22 | 23 | // item3Builder creates a new item and returns a pointer to it. 24 | // This is used when we load a segment of the queue from disk. 25 | func item3Builder() interface{} { 26 | return &item3{} 27 | } 28 | 29 | func BenchmarkEnqueue_Safe(b *testing.B) { 30 | benchmarkEnqueue(b, false /* true=turbo */) 31 | } 32 | func BenchmarkEnqueue_Turbo(b *testing.B) { 33 | benchmarkEnqueue(b, true /* true=turbo */) 34 | } 35 | 36 | func benchmarkEnqueue(b *testing.B, turbo bool) { 37 | 38 | qName := "testBenchEnqueue" 39 | 40 | b.StopTimer() 41 | 42 | // Clean up from a previous run 43 | if err := os.RemoveAll(qName); err != nil { 44 | b.Fatal("Error removing queue directory:", err) 45 | } 46 | 47 | // Create the queue 48 | q, err := dque.New(qName, ".", 100, item3Builder) 49 | if err != nil { 50 | b.Fatal("Error creating new dque:", err) 51 | } 52 | if turbo { 53 | _ = q.TurboOn() 54 | } 55 | b.StartTimer() 56 | 57 | for n := 0; n < b.N; n++ { 58 | err := q.Enqueue(item3{"Short Name", n, true}) 59 | if err != nil { 60 | b.Fatal("Error enqueuing to dque:", err) 61 | } 62 | } 63 | 64 | // Clean up from the run 65 | if err := os.RemoveAll(qName); err != nil { 66 | b.Fatal("Error removing queue directory for BenchmarkDequeue:", err) 67 | } 68 | } 69 | 70 | func BenchmarkDequeue_Safe(b *testing.B) { 71 | benchmarkDequeue(b, false /* true=turbo */) 72 | } 73 | func BenchmarkDequeue_Turbo(b *testing.B) { 74 | benchmarkDequeue(b, true /* true=turbo */) 75 | } 76 | 77 | func benchmarkDequeue(b *testing.B, turbo bool) { 78 | 79 | qName := "testBenchDequeue" 80 | 81 | b.StopTimer() 82 | 83 | // Clean up from a previous run 84 | if err := os.RemoveAll(qName); err != nil { 85 | b.Fatal("Error removing queue directory:", err) 86 | } 87 | 88 | // Create the queue 89 | q, err := dque.New(qName, ".", 100, item3Builder) 90 | if err != nil { 91 | b.Fatal("Error creating new dque", err) 92 | } 93 | var iterations int = 5000 94 | if turbo { 95 | _ = q.TurboOn() 96 | iterations = iterations * 10 97 | } 98 | 99 | for i := 0; i < iterations; i++ { 100 | err := q.Enqueue(item3{"Sorta, kind of, a Big Long Name", i, true}) 101 | if err != nil { 102 | b.Fatal("Error enqueuing to dque:", err) 103 | } 104 | } 105 | b.StartTimer() 106 | 107 | for n := 0; n < b.N; n++ { 108 | _, err := q.Dequeue() 109 | if err != nil { 110 | b.Fatal("Error dequeuing from dque:", err) 111 | } 112 | } 113 | 114 | // Clean up from the run 115 | if err := os.RemoveAll(qName); err != nil { 116 | b.Fatal("Error removing queue directory for BenchmarkDequeue", err) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package dque_test 2 | 3 | // 4 | // Example usage 5 | // Run with: go test -v example_test.go 6 | // 7 | 8 | import ( 9 | "fmt" 10 | "log" 11 | 12 | "github.com/joncrlsn/dque" 13 | ) 14 | 15 | // Item is what we'll be storing in the queue. It can be any struct 16 | // as long as the fields you want stored are public. 17 | type Item struct { 18 | Name string 19 | Id int 20 | } 21 | 22 | // ItemBuilder creates a new item and returns a pointer to it. 23 | // This is used when we load a segment of the queue from disk. 24 | func ItemBuilder() interface{} { 25 | return &Item{} 26 | } 27 | 28 | // ExampleDQue shows how the queue works 29 | func ExampleDQue() { 30 | qName := "item-queue" 31 | qDir := "/tmp" 32 | segmentSize := 50 33 | 34 | // Create a new queue with segment size of 50 35 | q, err := dque.NewOrOpen(qName, qDir, segmentSize, ItemBuilder) 36 | if err != nil { 37 | log.Fatal("Error creating new dque ", err) 38 | } 39 | 40 | // Add an item to the queue 41 | if err := q.Enqueue(&Item{"Joe", 1}); err != nil { 42 | log.Fatal("Error enqueueing item ", err) 43 | } 44 | log.Println("Size should be 1:", q.Size()) 45 | 46 | // Properly close a queue 47 | q.Close() 48 | 49 | // You can reconsitute the queue from disk at any time 50 | q, err = dque.Open(qName, qDir, segmentSize, ItemBuilder) 51 | if err != nil { 52 | log.Fatal("Error opening existing dque ", err) 53 | } 54 | 55 | // Peek at the next item in the queue 56 | var iface interface{} 57 | if iface, err = q.Peek(); err != nil { 58 | if err != dque.ErrEmpty { 59 | log.Fatal("Error peeking at item", err) 60 | } 61 | } 62 | log.Println("Peeked at:", iface) 63 | 64 | // Dequeue the next item in the queue 65 | if iface, err = q.Dequeue(); err != nil && err != dque.ErrEmpty { 66 | log.Fatal("Error dequeuing item:", err) 67 | } 68 | log.Println("Dequeued an interface:", iface) 69 | log.Println("Size should be zero:", q.Size()) 70 | 71 | go func() { 72 | err := q.Enqueue(&Item{"Joe", 1}) 73 | log.Println("Enqueued from goroutine", err == nil) 74 | }() 75 | 76 | // Dequeue the next item in the queue and block until one is available 77 | if iface, err = q.DequeueBlock(); err != nil { 78 | log.Fatal("Error dequeuing item ", err) 79 | } 80 | 81 | // Assert type of the response to an Item pointer so we can work with it 82 | item, ok := iface.(*Item) 83 | if !ok { 84 | log.Fatal("Dequeued object is not an Item pointer") 85 | } 86 | 87 | doSomething(item) 88 | // Output: Dequeued: &{Joe 1} 89 | } 90 | 91 | func doSomething(item *Item) { 92 | fmt.Println("Dequeued:", item) 93 | } 94 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/joncrlsn/dque 2 | 3 | require github.com/pkg/errors v0.9.1 4 | 5 | require ( 6 | github.com/gofrs/flock v0.7.1 7 | github.com/kr/pretty v0.2.0 // indirect 8 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 9 | ) 10 | 11 | go 1.13 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc= 2 | github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= 3 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= 4 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 5 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 6 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 7 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 8 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 9 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 10 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 11 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 12 | -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- 1 | // 2 | // Package dque is a fast embedded durable queue for Go 3 | // 4 | package dque 5 | 6 | // 7 | // Copyright (c) 2018 Jon Carlson. All rights reserved. 8 | // Use of this source code is governed by an MIT-style 9 | // license that can be found in the LICENSE file. 10 | // 11 | 12 | import ( 13 | "strconv" 14 | "sync" 15 | 16 | "github.com/gofrs/flock" 17 | "github.com/pkg/errors" 18 | 19 | "io/ioutil" 20 | "math" 21 | "os" 22 | "path" 23 | "regexp" 24 | ) 25 | 26 | const lockFile = "lock.lock" 27 | 28 | // ErrQueueClosed is the error returned when a queue is closed. 29 | var ErrQueueClosed = errors.New("queue is closed") 30 | 31 | var ( 32 | filePattern *regexp.Regexp 33 | 34 | // ErrEmpty is returned when attempting to dequeue from an empty queue. 35 | ErrEmpty = errors.New("dque is empty") 36 | ) 37 | 38 | func init() { 39 | filePattern, _ = regexp.Compile(`^([0-9]+)\.dque$`) 40 | } 41 | 42 | type config struct { 43 | ItemsPerSegment int 44 | } 45 | 46 | // DQue is the in-memory representation of a queue on disk. You must never have 47 | // two *active* DQue instances pointing at the same path on disk. It is 48 | // acceptable to reconstitute a new instance from disk, but make sure the old 49 | // instance is never enqueued to (or dequeued from) again. 50 | type DQue struct { 51 | Name string 52 | DirPath string 53 | config config 54 | 55 | fullPath string 56 | fileLock *flock.Flock 57 | firstSegment *qSegment 58 | lastSegment *qSegment 59 | builder func() interface{} // builds a structure to load via gob 60 | 61 | mutex sync.Mutex 62 | 63 | emptyCond *sync.Cond 64 | 65 | turbo bool 66 | } 67 | 68 | // New creates a new durable queue 69 | func New(name string, dirPath string, itemsPerSegment int, builder func() interface{}) (*DQue, error) { 70 | 71 | // Validation 72 | if len(name) == 0 { 73 | return nil, errors.New("the queue name requires a value") 74 | } 75 | if len(dirPath) == 0 { 76 | return nil, errors.New("the queue directory requires a value") 77 | } 78 | if !dirExists(dirPath) { 79 | return nil, errors.New("the given queue directory is not valid: " + dirPath) 80 | } 81 | fullPath := path.Join(dirPath, name) 82 | if dirExists(fullPath) { 83 | return nil, errors.New("the given queue directory already exists: " + fullPath + ". Use Open instead") 84 | } 85 | 86 | if err := os.Mkdir(fullPath, 0755); err != nil { 87 | return nil, errors.Wrap(err, "error creating queue directory "+fullPath) 88 | } 89 | 90 | q := DQue{Name: name, DirPath: dirPath} 91 | q.fullPath = fullPath 92 | q.config.ItemsPerSegment = itemsPerSegment 93 | q.builder = builder 94 | q.emptyCond = sync.NewCond(&q.mutex) 95 | 96 | if err := q.lock(); err != nil { 97 | return nil, err 98 | } 99 | 100 | if err := q.load(); err != nil { 101 | er := q.fileLock.Unlock() 102 | if er != nil { 103 | return nil, er 104 | } 105 | return nil, err 106 | } 107 | 108 | return &q, nil 109 | } 110 | 111 | // Open opens an existing durable queue. 112 | func Open(name string, dirPath string, itemsPerSegment int, builder func() interface{}) (*DQue, error) { 113 | 114 | // Validation 115 | if len(name) == 0 { 116 | return nil, errors.New("the queue name requires a value") 117 | } 118 | if len(dirPath) == 0 { 119 | return nil, errors.New("the queue directory requires a value") 120 | } 121 | if !dirExists(dirPath) { 122 | return nil, errors.New("the given queue directory is not valid (" + dirPath + ")") 123 | } 124 | fullPath := path.Join(dirPath, name) 125 | if !dirExists(fullPath) { 126 | return nil, errors.New("the given queue does not exist (" + fullPath + ")") 127 | } 128 | 129 | q := DQue{Name: name, DirPath: dirPath} 130 | q.fullPath = fullPath 131 | q.config.ItemsPerSegment = itemsPerSegment 132 | q.builder = builder 133 | q.emptyCond = sync.NewCond(&q.mutex) 134 | 135 | if err := q.lock(); err != nil { 136 | return nil, err 137 | } 138 | 139 | if err := q.load(); err != nil { 140 | er := q.fileLock.Unlock() 141 | if er != nil { 142 | return nil, er 143 | } 144 | return nil, err 145 | } 146 | 147 | return &q, nil 148 | } 149 | 150 | // NewOrOpen either creates a new queue or opens an existing durable queue. 151 | func NewOrOpen(name string, dirPath string, itemsPerSegment int, builder func() interface{}) (*DQue, error) { 152 | 153 | // Validation 154 | if len(name) == 0 { 155 | return nil, errors.New("the queue name requires a value") 156 | } 157 | if len(dirPath) == 0 { 158 | return nil, errors.New("the queue directory requires a value") 159 | } 160 | if !dirExists(dirPath) { 161 | return nil, errors.New("the given queue directory is not valid (" + dirPath + ")") 162 | } 163 | fullPath := path.Join(dirPath, name) 164 | if dirExists(fullPath) { 165 | return Open(name, dirPath, itemsPerSegment, builder) 166 | } 167 | 168 | return New(name, dirPath, itemsPerSegment, builder) 169 | } 170 | 171 | // Close releases the lock on the queue rendering it unusable for further usage by this instance. 172 | // Close will return an error if it has already been called. 173 | func (q *DQue) Close() error { 174 | // only allow Close while no other function is active 175 | q.mutex.Lock() 176 | defer q.mutex.Unlock() 177 | 178 | if q.fileLock == nil { 179 | return ErrQueueClosed 180 | } 181 | 182 | err := q.fileLock.Close() 183 | if err != nil { 184 | return err 185 | } 186 | 187 | // Finally mark this instance as closed to prevent any further access 188 | q.fileLock = nil 189 | 190 | // Wake-up any waiting goroutines for blocking queue access - they should get a ErrQueueClosed 191 | q.emptyCond.Broadcast() 192 | 193 | // Close the first and last segments' file handles 194 | if err = q.firstSegment.close(); err != nil { 195 | return err 196 | } 197 | if q.firstSegment != q.lastSegment { 198 | if err = q.lastSegment.close(); err != nil { 199 | return err 200 | } 201 | } 202 | 203 | // Safe-guard ourself from accidentally using segments after closing the queue 204 | q.firstSegment = nil 205 | q.lastSegment = nil 206 | 207 | return nil 208 | } 209 | 210 | // Enqueue adds an item to the end of the queue 211 | func (q *DQue) Enqueue(obj interface{}) error { 212 | // This is heavy-handed but its safe 213 | q.mutex.Lock() 214 | defer q.mutex.Unlock() 215 | 216 | if q.fileLock == nil { 217 | return ErrQueueClosed 218 | } 219 | 220 | // If this segment is full then create a new one 221 | if q.lastSegment.sizeOnDisk() >= q.config.ItemsPerSegment { 222 | 223 | // We have filled our last segment to capacity, so create a new one 224 | seg, err := newQueueSegment(q.fullPath, q.lastSegment.number+1, q.turbo, q.builder) 225 | if err != nil { 226 | return errors.Wrapf(err, "error creating new queue segment: %d.", q.lastSegment.number+1) 227 | } 228 | 229 | // If the last segment is not the first segment 230 | // then we need to close the file. 231 | if q.firstSegment != q.lastSegment { 232 | var err = q.lastSegment.close() 233 | if err != nil { 234 | return errors.Wrapf(err, "error closing previous segment file #%d.", q.lastSegment.number) 235 | } 236 | } 237 | 238 | // Replace the last segment with the new one 239 | q.lastSegment = seg 240 | 241 | } 242 | 243 | // Add the object to the last segment 244 | if err := q.lastSegment.add(obj); err != nil { 245 | return errors.Wrap(err, "error adding item to the last segment") 246 | } 247 | 248 | // Wakeup any goroutine that is currently waiting for an item to be enqueued 249 | q.emptyCond.Broadcast() 250 | 251 | return nil 252 | } 253 | 254 | // Dequeue removes and returns the first item in the queue. 255 | // When the queue is empty, nil and dque.ErrEmpty are returned. 256 | func (q *DQue) Dequeue() (interface{}, error) { 257 | // This is heavy-handed but its safe 258 | q.mutex.Lock() 259 | defer q.mutex.Unlock() 260 | 261 | return q.dequeueLocked() 262 | } 263 | 264 | func (q *DQue) dequeueLocked() (interface{}, error) { 265 | if q.fileLock == nil { 266 | return nil, ErrQueueClosed 267 | } 268 | 269 | // Remove the first object from the first segment 270 | obj, err := q.firstSegment.remove() 271 | if err == errEmptySegment { 272 | return nil, ErrEmpty 273 | } 274 | if err != nil { 275 | return nil, errors.Wrap(err, "error removing item from the first segment") 276 | } 277 | 278 | // If this segment is empty and we've reached the max for this segment 279 | // then delete the file and open the next one. 280 | if q.firstSegment.size() == 0 && 281 | q.firstSegment.sizeOnDisk() >= q.config.ItemsPerSegment { 282 | 283 | // Delete the segment file 284 | if err := q.firstSegment.delete(); err != nil { 285 | return obj, errors.Wrap(err, "error deleting queue segment "+q.firstSegment.filePath()+". Queue is in an inconsistent state") 286 | } 287 | 288 | // We have only one segment and it's now empty so destroy it and 289 | // create a new one. 290 | if q.firstSegment.number == q.lastSegment.number { 291 | 292 | // Create the next segment 293 | seg, err := newQueueSegment(q.fullPath, q.firstSegment.number+1, q.turbo, q.builder) 294 | if err != nil { 295 | return obj, errors.Wrap(err, "error creating new segment. Queue is in an inconsistent state") 296 | } 297 | q.firstSegment = seg 298 | q.lastSegment = seg 299 | 300 | } else { 301 | 302 | if q.firstSegment.number+1 == q.lastSegment.number { 303 | // We have 2 segments, moving down to 1 shared segment 304 | q.firstSegment = q.lastSegment 305 | } else { 306 | 307 | // Open the next segment 308 | seg, err := openQueueSegment(q.fullPath, q.firstSegment.number+1, q.turbo, q.builder) 309 | if err != nil { 310 | return obj, errors.Wrap(err, "error creating new segment. Queue is in an inconsistent state") 311 | } 312 | q.firstSegment = seg 313 | } 314 | 315 | } 316 | } 317 | 318 | return obj, nil 319 | } 320 | 321 | // Peek returns the first item in the queue without dequeueing it. 322 | // When the queue is empty, nil and dque.ErrEmpty are returned. 323 | // Do not use this method with multiple dequeueing threads or you may regret it. 324 | func (q *DQue) Peek() (interface{}, error) { 325 | // This is heavy-handed but it is safe 326 | q.mutex.Lock() 327 | defer q.mutex.Unlock() 328 | 329 | return q.peekLocked() 330 | } 331 | 332 | func (q *DQue) peekLocked() (interface{}, error) { 333 | if q.fileLock == nil { 334 | return nil, ErrQueueClosed 335 | } 336 | 337 | // Return the first object from the first segment 338 | obj, err := q.firstSegment.peek() 339 | if err == errEmptySegment { 340 | return nil, ErrEmpty 341 | } 342 | if err != nil { 343 | // In reality this will (i.e. should not) never happen 344 | return nil, errors.Wrap(err, "error getting item from the first segment") 345 | } 346 | 347 | return obj, nil 348 | } 349 | 350 | // DequeueBlock behaves similar to Dequeue, but is a blocking call until an item is available. 351 | func (q *DQue) DequeueBlock() (interface{}, error) { 352 | q.mutex.Lock() 353 | defer q.mutex.Unlock() 354 | for { 355 | obj, err := q.dequeueLocked() 356 | if err == ErrEmpty { 357 | q.emptyCond.Wait() 358 | // Wait() atomically unlocks mutexEmptyCond and suspends execution of the calling goroutine. 359 | // Receiving the signal does not guarantee an item is available, let's loop and check again. 360 | continue 361 | } else if err != nil { 362 | return nil, err 363 | } 364 | return obj, nil 365 | } 366 | } 367 | 368 | // PeekBlock behaves similar to Peek, but is a blocking call until an item is available. 369 | func (q *DQue) PeekBlock() (interface{}, error) { 370 | q.mutex.Lock() 371 | defer q.mutex.Unlock() 372 | for { 373 | obj, err := q.peekLocked() 374 | if err == ErrEmpty { 375 | q.emptyCond.Wait() 376 | // Wait() atomically unlocks mutexEmptyCond and suspends execution of the calling goroutine. 377 | // Receiving the signal does not guarantee an item is available, let's loop and check again. 378 | continue 379 | } else if err != nil { 380 | return nil, err 381 | } 382 | return obj, nil 383 | } 384 | } 385 | 386 | // Size locks things up while calculating so you are guaranteed an accurate 387 | // size... unless you have changed the itemsPerSegment value since the queue 388 | // was last empty. Then it could be wildly inaccurate. 389 | func (q *DQue) Size() int { 390 | if q.fileLock == nil { 391 | return 0 392 | } 393 | 394 | // This is heavy-handed but it is safe 395 | q.mutex.Lock() 396 | defer q.mutex.Unlock() 397 | 398 | return q.SizeUnsafe() 399 | } 400 | 401 | // SizeUnsafe returns the approximate number of items in the queue. Use Size() if 402 | // having the exact size is important to your use-case. 403 | // 404 | // The return value could be wildly inaccurate if the itemsPerSegment value has 405 | // changed since the queue was last empty. 406 | // Also, because this method is not synchronized, the size may change after 407 | // entering this method. 408 | func (q *DQue) SizeUnsafe() int { 409 | if q.fileLock == nil { 410 | return 0 411 | } 412 | if q.firstSegment.number == q.lastSegment.number { 413 | return q.firstSegment.size() 414 | } 415 | numSegmentsBetween := q.lastSegment.number - q.firstSegment.number - 1 416 | return q.firstSegment.size() + (numSegmentsBetween * q.config.ItemsPerSegment) + q.lastSegment.size() 417 | } 418 | 419 | // SegmentNumbers returns the number of both the first last segmment. 420 | // There is likely no use for this information other than testing. 421 | func (q *DQue) SegmentNumbers() (int, int) { 422 | if q.fileLock == nil { 423 | return 0, 0 424 | } 425 | return q.firstSegment.number, q.lastSegment.number 426 | } 427 | 428 | // Turbo returns true if the turbo flag is on. Having turbo on speeds things 429 | // up significantly. 430 | func (q *DQue) Turbo() bool { 431 | return q.turbo 432 | } 433 | 434 | // TurboOn allows the filesystem to decide when to sync file changes to disk. 435 | // Throughput is greatly increased by turning turbo on, however there is some 436 | // risk of losing data if a power-loss occurs. 437 | // If turbo is already on an error is returned 438 | func (q *DQue) TurboOn() error { 439 | // This is heavy-handed but it is safe 440 | q.mutex.Lock() 441 | defer q.mutex.Unlock() 442 | 443 | if q.fileLock == nil { 444 | return ErrQueueClosed 445 | } 446 | 447 | if q.turbo { 448 | return errors.New("DQue.TurboOn() is not valid when turbo is on") 449 | } 450 | q.turbo = true 451 | q.firstSegment.turboOn() 452 | q.lastSegment.turboOn() 453 | return nil 454 | } 455 | 456 | // TurboOff re-enables the "safety" mode that syncs every file change to disk as 457 | // they happen. 458 | // If turbo is already off an error is returned 459 | func (q *DQue) TurboOff() error { 460 | // This is heavy-handed but it is safe 461 | q.mutex.Lock() 462 | defer q.mutex.Unlock() 463 | 464 | if q.fileLock == nil { 465 | return ErrQueueClosed 466 | } 467 | 468 | if !q.turbo { 469 | return errors.New("DQue.TurboOff() is not valid when turbo is off") 470 | } 471 | if err := q.firstSegment.turboOff(); err != nil { 472 | return err 473 | } 474 | if err := q.lastSegment.turboOff(); err != nil { 475 | return err 476 | } 477 | q.turbo = false 478 | return nil 479 | } 480 | 481 | // TurboSync allows you to fsync changes to disk, but only if turbo is on. 482 | // If turbo is off an error is returned 483 | func (q *DQue) TurboSync() error { 484 | // This is heavy-handed but it is safe 485 | q.mutex.Lock() 486 | defer q.mutex.Unlock() 487 | 488 | if q.fileLock == nil { 489 | return ErrQueueClosed 490 | } 491 | if !q.turbo { 492 | return errors.New("DQue.TurboSync() is inappropriate when turbo is off") 493 | } 494 | if err := q.firstSegment.turboSync(); err != nil { 495 | return errors.Wrap(err, "unable to sync changes to disk") 496 | } 497 | if err := q.lastSegment.turboSync(); err != nil { 498 | return errors.Wrap(err, "unable to sync changes to disk") 499 | } 500 | return nil 501 | } 502 | 503 | // load populates the queue from disk 504 | func (q *DQue) load() error { 505 | 506 | // Find all queue files 507 | files, err := ioutil.ReadDir(q.fullPath) 508 | if err != nil { 509 | return errors.Wrap(err, "unable to read files in "+q.fullPath) 510 | } 511 | 512 | // Find the smallest and the largest file numbers 513 | minNum := math.MaxInt32 514 | maxNum := 0 515 | for _, f := range files { 516 | if !f.IsDir() && filePattern.MatchString(f.Name()) { 517 | // Extract number out of the filename 518 | fileNumStr := filePattern.FindStringSubmatch(f.Name())[1] 519 | fileNum, _ := strconv.Atoi(fileNumStr) 520 | if fileNum > maxNum { 521 | maxNum = fileNum 522 | } 523 | if fileNum < minNum { 524 | minNum = fileNum 525 | } 526 | } 527 | } 528 | 529 | // If files were found, set q.firstSegment and q.lastSegment 530 | if maxNum > 0 { 531 | 532 | // We found files 533 | for { 534 | seg, err := openQueueSegment(q.fullPath, minNum, q.turbo, q.builder) 535 | if err != nil { 536 | return errors.Wrap(err, "unable to create queue segment in "+q.fullPath) 537 | } 538 | // Make sure the first segment is not empty or it's not complete (i.e. is current) 539 | if seg.size() > 0 || seg.sizeOnDisk() < q.config.ItemsPerSegment { 540 | q.firstSegment = seg 541 | break 542 | } 543 | // Delete the segment as it's empty and complete 544 | seg.delete() 545 | // Try the next one 546 | minNum++ 547 | } 548 | 549 | if minNum == maxNum { 550 | // We have only one segment so the 551 | // first and last are the same instance (in this case) 552 | q.lastSegment = q.firstSegment 553 | } else { 554 | // We have multiple segments 555 | seg, err := openQueueSegment(q.fullPath, maxNum, q.turbo, q.builder) 556 | if err != nil { 557 | return errors.Wrap(err, "unable to create segment for "+q.fullPath) 558 | } 559 | q.lastSegment = seg 560 | } 561 | 562 | } else { 563 | // We found no files so build a new queue starting with segment 1 564 | seg, err := newQueueSegment(q.fullPath, 1, q.turbo, q.builder) 565 | if err != nil { 566 | return errors.Wrap(err, "unable to create queue segment in "+q.fullPath) 567 | } 568 | 569 | // The first and last are the same instance (in this case) 570 | q.firstSegment = seg 571 | q.lastSegment = seg 572 | } 573 | 574 | return nil 575 | } 576 | 577 | func (q *DQue) lock() error { 578 | l := path.Join(q.DirPath, q.Name, lockFile) 579 | fileLock := flock.New(l) 580 | 581 | locked, err := fileLock.TryLock() 582 | if err != nil { 583 | return err 584 | } 585 | if !locked { 586 | return errors.New("failed to acquire flock") 587 | } 588 | 589 | q.fileLock = fileLock 590 | return nil 591 | } 592 | -------------------------------------------------------------------------------- /queue_test.go: -------------------------------------------------------------------------------- 1 | // queue_test.go 2 | package dque_test 3 | 4 | import ( 5 | "fmt" 6 | "math/rand" 7 | "os" 8 | "path/filepath" 9 | "runtime" 10 | "sync" 11 | "testing" 12 | "time" 13 | 14 | "github.com/joncrlsn/dque" 15 | ) 16 | 17 | // item2 is the thing we'll be storing in the queue 18 | type item2 struct { 19 | Id int 20 | } 21 | 22 | // item2Builder creates a new item and returns a pointer to it. 23 | // This is used when we load a segment of the queue from disk. 24 | func item2Builder() interface{} { 25 | return &item2{} 26 | } 27 | 28 | // Adds 1 and removes 1 in a loop to ensure that when we've filled 29 | // up the first segment that we delete it and move on to the next segment 30 | func TestQueue_AddRemoveLoop(t *testing.T) { 31 | testQueue_AddRemoveLoop(t, true /* true=turbo */) 32 | testQueue_AddRemoveLoop(t, false /* true=turbo */) 33 | } 34 | 35 | func testQueue_AddRemoveLoop(t *testing.T, turbo bool) { 36 | qName := "test1" 37 | if err := os.RemoveAll(qName); err != nil { 38 | t.Fatal("Error removing queue directory", err) 39 | } 40 | 41 | // Create a new queue with segment size of 3 42 | var err error 43 | q := newQ(t, qName, turbo) 44 | 45 | for i := 0; i < 4; i++ { 46 | if err := q.Enqueue(&item2{i}); err != nil { 47 | t.Fatal("Error enqueueing", err) 48 | } 49 | _, err = q.Dequeue() 50 | if err != nil { 51 | t.Fatal("Error dequeueing", err) 52 | } 53 | } 54 | 55 | assert(t, 0 == q.Size(), "Size is not 0") 56 | 57 | firstSegNum, lastSegNum := q.SegmentNumbers() 58 | 59 | // Assert that we have just one segment 60 | assert(t, firstSegNum == lastSegNum, "The first segment must match the last") 61 | 62 | // Assert that the first segment is #2 63 | assert(t, 2 == firstSegNum, "The first segment is not 2") 64 | 65 | // Now reopen the queue and check our assertions again. 66 | q.Close() 67 | q = openQ(t, qName, turbo) 68 | 69 | firstSegNum, lastSegNum = q.SegmentNumbers() 70 | 71 | // Assert that we have just one segment 72 | assert(t, firstSegNum == lastSegNum, "After opening, the first segment must match the second") 73 | 74 | // Assert that the first segment is #2 75 | assert(t, 2 == firstSegNum, "After opening, the first segment is not 2") 76 | 77 | if err := os.RemoveAll(qName); err != nil { 78 | t.Fatal("Error cleaning up the queue directory", err) 79 | } 80 | } 81 | 82 | // Adds 2 and removes 1 in a loop to ensure that when we've filled 83 | // up the first segment that we delete it and move on to the next segment 84 | func TestQueue_Add2Remove1(t *testing.T) { 85 | testQueue_Add2Remove1(t, true /* true=turbo */) 86 | testQueue_Add2Remove1(t, false /* true=turbo */) 87 | } 88 | func testQueue_Add2Remove1(t *testing.T, turbo bool) { 89 | qName := "test1" 90 | if err := os.RemoveAll(qName); err != nil { 91 | t.Fatal("Error removing queue directory", err) 92 | } 93 | 94 | // Create a new queue with segment size of 3 95 | var err error 96 | q := newQ(t, qName, turbo) 97 | 98 | // Add 2 and remove one each loop 99 | for i := 0; i < 4; i = i + 2 { 100 | var item interface{} 101 | if err := q.Enqueue(&item2{i}); err != nil { 102 | t.Fatal("Error enqueueing", err) 103 | } 104 | if err := q.Enqueue(&item2{i + 1}); err != nil { 105 | t.Fatal("Error enqueueing", err) 106 | } 107 | item, err = q.Dequeue() 108 | if err != nil { 109 | t.Fatal("Error dequeueing", err) 110 | } 111 | assert(t, item != nil, "Item is nil") 112 | } 113 | 114 | firstSegNum, lastSegNum := q.SegmentNumbers() 115 | 116 | // Assert that we have more than one segment 117 | assert(t, firstSegNum < lastSegNum, "The first segment cannot match the second") 118 | 119 | // Assert that the first segment is #2 120 | assert(t, 2 == lastSegNum, "The last segment must be 2") 121 | 122 | // Now reopen the queue and check our assertions again. 123 | q.Close() 124 | q = openQ(t, qName, turbo) 125 | 126 | firstSegNum, lastSegNum = q.SegmentNumbers() 127 | 128 | // Assert that we have more than one segment 129 | assert(t, firstSegNum < lastSegNum, "After opening, the first segment can not match the second") 130 | 131 | // Assert that the first segment is #2 132 | assert(t, 2 == lastSegNum, "After opening, the last segment must be 2") 133 | 134 | // Test Peek to make sure the size doesn't change 135 | assert(t, 2 == q.Size(), "Queue size is not 2 before peeking") 136 | obj, err := q.Peek() 137 | if err != nil { 138 | t.Fatal("Error peeking at the queue", err) 139 | } 140 | 141 | assert(t, 2 == q.Size(), "After peaking, aueue size must still be 2") 142 | assert(t, obj != nil, "Peeked object must not be nil.") 143 | 144 | if err := os.RemoveAll(qName); err != nil { 145 | t.Fatal("Error cleaning up the queue directory", err) 146 | } 147 | } 148 | 149 | // Adds 9 and removes 8 150 | func TestQueue_Add9Remove8(t *testing.T) { 151 | testQueue_Add9Remove8(t, true /* true = turbo */) 152 | testQueue_Add9Remove8(t, false /* true = turbo */) 153 | } 154 | 155 | func testQueue_Add9Remove8(t *testing.T, turbo bool) { 156 | qName := "test1" 157 | if err := os.RemoveAll(qName); err != nil { 158 | t.Fatal("Error removing queue directory", err) 159 | } 160 | 161 | // Create new queue with segment size 3 162 | q := newQ(t, qName, turbo) 163 | 164 | // Enqueue 9 items 165 | for i := 0; i < 9; i++ { 166 | if err := q.Enqueue(&item2{i}); err != nil { 167 | t.Fatal("Error enqueueing", err) 168 | } 169 | } 170 | 171 | // Check the Size calculation 172 | assert(t, 9 == q.Size(), "the size is calculated wrong. Should be 9") 173 | 174 | firstSegNum, lastSegNum := q.SegmentNumbers() 175 | 176 | // Assert that the first segment is #1 177 | assert(t, 1 == firstSegNum, "the first segment is not 1") 178 | 179 | // Assert that the last segment is #4 180 | assert(t, 3 == lastSegNum, "the last segment is not 3") 181 | 182 | // Dequeue 8 items 183 | for i := 0; i < 8; i++ { 184 | iface, err := q.Dequeue() 185 | if err != nil { 186 | t.Fatal("Error dequeueing:", err) 187 | } 188 | 189 | // Check the Size calculation 190 | assert(t, 8-i == q.Size(), "the size is calculated wrong.") 191 | item, ok := iface.(item2) 192 | if ok { 193 | fmt.Printf("Dequeued %T %t %#v\n", item, ok, item) 194 | assert(t, i == item.Id, "Unexpected itemId") 195 | } else { 196 | item, ok := iface.(*item2) 197 | assert(t, ok, "Dequeued object is not of type *item2") 198 | assert(t, i == item.Id, "Unexpected itemId") 199 | } 200 | } 201 | 202 | firstSegNum, lastSegNum = q.SegmentNumbers() 203 | 204 | // Assert that we have only one segment 205 | assert(t, firstSegNum == lastSegNum, "The first segment must match the second") 206 | 207 | // Assert that the first segment is #3 208 | assert(t, 3 == firstSegNum, "The last segment is not 3") 209 | 210 | // Now reopen the queue and check our assertions again. 211 | q.Close() 212 | _ = openQ(t, qName, turbo) 213 | 214 | // Assert that we have more than one segment 215 | assert(t, firstSegNum == lastSegNum, "After opening, the first segment must match the second") 216 | 217 | // Assert that the last segment is #3 218 | assert(t, 3 == lastSegNum, "After opening, the last segment is not 3") 219 | 220 | if err := os.RemoveAll(qName); err != nil { 221 | t.Fatal("Error cleaning up the queue directory:", err) 222 | } 223 | } 224 | 225 | func TestQueue_EmptyDequeue(t *testing.T) { 226 | testQueue_EmptyDequeue(t, true /* true=turbo */) 227 | testQueue_EmptyDequeue(t, false /* true=turbo */) 228 | } 229 | func testQueue_EmptyDequeue(t *testing.T, turbo bool) { 230 | qName := "testEmptyDequeue" 231 | if err := os.RemoveAll(qName); err != nil { 232 | t.Fatal("Error removing queue directory:", err) 233 | } 234 | 235 | // Create new queue 236 | q := newQ(t, qName, turbo) 237 | assert(t, 0 == q.Size(), "Expected an empty queue") 238 | 239 | // Dequeue an item from the empty queue 240 | item, err := q.Dequeue() 241 | assert(t, dque.ErrEmpty == err, "Expected an ErrEmpty error") 242 | assert(t, item == nil, "Expected nil because queue is empty") 243 | 244 | if err := os.RemoveAll(qName); err != nil { 245 | t.Fatal("Error cleaning up the queue directory:", err) 246 | } 247 | } 248 | 249 | func TestQueue_NewOrOpen(t *testing.T) { 250 | testQueue_NewOrOpen(t, true /* true=turbo */) 251 | testQueue_NewOrOpen(t, false /* true=turbo */) 252 | } 253 | 254 | func testQueue_NewOrOpen(t *testing.T, turbo bool) { 255 | qName := "testNewOrOpen" 256 | if err := os.RemoveAll(qName); err != nil { 257 | t.Fatal("Error removing queue directory:", err) 258 | } 259 | 260 | // Create new queue with newOrOpen 261 | q := newOrOpenQ(t, qName, turbo) 262 | q.Close() 263 | 264 | // Open the same queue with newOrOpen 265 | q = newOrOpenQ(t, qName, turbo) 266 | q.Close() 267 | 268 | if err := os.RemoveAll(qName); err != nil { 269 | t.Fatal("Error cleaning up the queue directory:", err) 270 | } 271 | } 272 | 273 | func TestQueue_Turbo(t *testing.T) { 274 | qName := "testNewOrOpen" 275 | if err := os.RemoveAll(qName); err != nil { 276 | t.Fatal("Error removing queue directory:", err) 277 | } 278 | 279 | // Create new queue 280 | q := newQ(t, qName, false) 281 | 282 | if err := q.TurboOff(); err == nil { 283 | t.Fatal("Expected an error") 284 | } 285 | 286 | if err := q.TurboSync(); err == nil { 287 | t.Fatal("Expected an error") 288 | } 289 | 290 | if err := q.TurboOn(); err != nil { 291 | t.Fatal("Error turning on turbo:", err) 292 | } 293 | 294 | if err := q.TurboOn(); err == nil { 295 | t.Fatal("Expected an error") 296 | } 297 | 298 | if err := q.TurboSync(); err != nil { 299 | t.Fatal("Error running TurboSync:", err) 300 | } 301 | 302 | // Enqueue 1000 items 303 | start := time.Now() 304 | for i := 0; i < 1000; i++ { 305 | if err := q.Enqueue(&item2{i}); err != nil { 306 | t.Fatal("Error enqueueing:", err) 307 | } 308 | } 309 | elapsedTurbo := time.Since(start) 310 | 311 | assert(t, q.Turbo(), "Expected turbo to be on") 312 | 313 | if err := q.TurboOff(); err != nil { 314 | t.Fatal("Error turning off turbo:", err) 315 | } 316 | 317 | // Enqueue 1000 items 318 | start = time.Now() 319 | for i := 0; i < 1000; i++ { 320 | if err := q.Enqueue(&item2{i}); err != nil { 321 | t.Fatal("Error enqueueing:", err) 322 | } 323 | } 324 | elapsedSafe := time.Since(start) 325 | 326 | assert(t, elapsedTurbo < elapsedSafe/2, "Turbo time (%v) must be faster than safe mode (%v)", elapsedTurbo, elapsedSafe) 327 | 328 | if err := os.RemoveAll(qName); err != nil { 329 | t.Fatal("Error cleaning up the queue directory:", err) 330 | } 331 | } 332 | 333 | func TestQueue_NewFlock(t *testing.T) { 334 | qName := "testFlock" 335 | if err := os.RemoveAll(qName); err != nil { 336 | t.Fatal("Error cleaning up the queue directory:", err) 337 | } 338 | 339 | // New and Close a DQue properly should work 340 | q, err := dque.New(qName, ".", 3, item2Builder) 341 | if err != nil { 342 | t.Fatal("Error creating dque:", err) 343 | } 344 | err = q.Close() 345 | if err != nil { 346 | t.Fatal("Error closing dque:", err) 347 | } 348 | 349 | // Double-open should fail 350 | q, err = dque.Open(qName, ".", 3, item2Builder) 351 | if err != nil { 352 | t.Fatal("Error opening dque:", err) 353 | } 354 | _, err = dque.Open(qName, ".", 3, item2Builder) 355 | if err == nil { 356 | t.Fatal("No error during double-open dque") 357 | } 358 | err = q.Close() 359 | if err != nil { 360 | t.Fatal("Error closing dque:", err) 361 | } 362 | 363 | // Double-close should fail 364 | q, err = dque.Open(qName, ".", 3, item2Builder) 365 | if err != nil { 366 | t.Fatal("Error opening dque:", err) 367 | } 368 | err = q.Close() 369 | if err != nil { 370 | t.Fatal("Error closing dque:", err) 371 | } 372 | err = q.Close() 373 | if err == nil { 374 | t.Fatal("No error during double-closing dque") 375 | } 376 | 377 | // Cleanup 378 | if err := os.RemoveAll(qName); err != nil { 379 | t.Fatal("Error removing queue directory:", err) 380 | } 381 | } 382 | 383 | func TestQueue_UseAfterClose(t *testing.T) { 384 | qName := "testUseAfterClose" 385 | if err := os.RemoveAll(qName); err != nil { 386 | t.Fatal("Error cleaning up the queue directory:", err) 387 | } 388 | 389 | q, err := dque.New(qName, ".", 3, item2Builder) 390 | if err != nil { 391 | t.Fatal("Error creating dque:", err) 392 | } 393 | err = q.Enqueue(&item2{0}) 394 | if err != nil { 395 | t.Fatal("Error enqueing item:", err) 396 | } 397 | err = q.Close() 398 | if err != nil { 399 | t.Fatal("Error closing dque:", err) 400 | } 401 | 402 | queueClosedError := "queue is closed" 403 | 404 | err = q.Close() 405 | assert(t, err.Error() == queueClosedError, "Expected error not found", err) 406 | 407 | err = q.Enqueue(&item2{0}) 408 | assert(t, err.Error() == queueClosedError, "Expected error not found", err) 409 | 410 | _, err = q.Dequeue() 411 | assert(t, err.Error() == queueClosedError, "Expected error not found", err) 412 | 413 | _, err = q.Peek() 414 | assert(t, err.Error() == queueClosedError, "Expected error not found", err) 415 | 416 | s := q.Size() 417 | assert(t, s == 0, "Expected error") 418 | 419 | s = q.SizeUnsafe() 420 | assert(t, s == 0, "Expected error") 421 | 422 | err = q.TurboOn() 423 | assert(t, err.Error() == queueClosedError, "Expected error not found", err) 424 | 425 | err = q.TurboOff() 426 | assert(t, err.Error() == queueClosedError, "Expected error not found", err) 427 | 428 | err = q.TurboSync() 429 | assert(t, err.Error() == queueClosedError, "Expected error not found", err) 430 | 431 | // Cleanup 432 | if err := os.RemoveAll(qName); err != nil { 433 | t.Fatal("Error removing queue directory:", err) 434 | } 435 | } 436 | 437 | func TestQueue_BlockingBehaviour(t *testing.T) { 438 | qName := "testBlocking" 439 | if err := os.RemoveAll(qName); err != nil { 440 | t.Fatal("Error removing queue directory:", err) 441 | } 442 | 443 | q := newQ(t, qName, false) 444 | 445 | go func() { 446 | err := q.Enqueue(&item2{0}) 447 | assert(t, err == nil, "Expected no error") 448 | }() 449 | 450 | x, err := q.PeekBlock() 451 | assert(t, err == nil, "Expected no error") 452 | assert(t, x != nil, "Item is nil") 453 | 454 | x, err = q.DequeueBlock() 455 | assert(t, err == nil, "Expected no error") 456 | assert(t, x != nil, "Item is nil") 457 | 458 | x, err = q.Dequeue() 459 | assert(t, err == dque.ErrEmpty, "Expected error not found") 460 | 461 | timeout := time.After(3 * time.Second) 462 | done := make(chan bool) 463 | go func() { 464 | x, err = q.DequeueBlock() 465 | assert(t, err == nil, "Expected no error") 466 | assert(t, x != nil, "Item is nil") 467 | done <- true 468 | }() 469 | 470 | go func() { 471 | time.Sleep(1 * time.Second) 472 | err := q.Enqueue(&item2{2}) 473 | assert(t, err == nil, "Expected no error") 474 | }() 475 | 476 | select { 477 | case <-timeout: 478 | t.Fatal("Test didn't finish in time") 479 | case <-done: 480 | } 481 | 482 | // Cleanup 483 | if err := os.RemoveAll(qName); err != nil { 484 | t.Fatal("Error removing queue directory:", err) 485 | } 486 | } 487 | 488 | func TestQueue_BlockingWithClose(t *testing.T) { 489 | qName := "testBlockingWithClose" 490 | if err := os.RemoveAll(qName); err != nil { 491 | t.Fatal("Error removing queue directory:", err) 492 | } 493 | 494 | q := newQ(t, qName, false) 495 | 496 | go func() { 497 | time.Sleep(1 * time.Second) 498 | err := q.Close() 499 | assert(t, err == nil, "Expected no error") 500 | }() 501 | 502 | timeout := time.After(3 * time.Second) 503 | done := make(chan bool) 504 | go func() { 505 | // The queue is empty, 506 | // so DequeueBlock should really block and wait, 507 | // until the other goroutine calls Close, 508 | // and the Close should wake-up this DequeueBlock block, 509 | // and return an error because the queue is now closed. 510 | _, err := q.DequeueBlock() 511 | assert(t, err == dque.ErrQueueClosed, "Expected ErrQueueClosed error") 512 | done <- true 513 | }() 514 | 515 | select { 516 | case <-timeout: 517 | t.Fatal("Test didn't finish in time") 518 | case <-done: 519 | } 520 | 521 | // Cleanup 522 | if err := os.RemoveAll(qName); err != nil { 523 | t.Fatal("Error removing queue directory:", err) 524 | } 525 | } 526 | 527 | func TestQueue_BlockingAggresive(t *testing.T) { 528 | rand.Seed(0) // ensure we have reproducible sleeps 529 | 530 | qName := "testBlockingAggresive" 531 | if err := os.RemoveAll(qName); err != nil { 532 | t.Fatal("Error removing queue directory:", err) 533 | } 534 | 535 | q := newQ(t, qName, false) 536 | 537 | numProducers := 5 538 | numItemsPerProducer := 50 539 | numConsumers := 25 540 | 541 | done := make(chan bool) 542 | var wg sync.WaitGroup 543 | wg.Add(numProducers * numItemsPerProducer) 544 | 545 | go func() { 546 | wg.Wait() 547 | q.Close() 548 | done <- true 549 | }() 550 | 551 | // producers 552 | for p := 0; p < numProducers; p++ { 553 | go func(producer int) { 554 | for i := 0; i < numItemsPerProducer; i++ { 555 | s := rand.Intn(150) 556 | time.Sleep(time.Duration(s) * time.Millisecond) 557 | err := q.Enqueue(&item2{i}) 558 | assert(t, err == nil, "Expected no error", err) 559 | fmt.Println("Enqueued item", i, "by producer", producer, "after sleeping", s) 560 | } 561 | }(p) 562 | } 563 | 564 | // consumers 565 | for c := 0; c < numConsumers; c++ { 566 | go func(consumer int) { 567 | for { 568 | x, err := q.DequeueBlock() 569 | if err == dque.ErrQueueClosed { 570 | return 571 | } 572 | assert(t, err == nil, "Expected no error") 573 | fmt.Println("Dequeued item", x, "by consumer", consumer) 574 | wg.Done() 575 | } 576 | }(c) 577 | } 578 | 579 | timeout := time.After(10 * time.Second) 580 | select { 581 | case <-timeout: 582 | t.Fatal("Test didn't finish in time") 583 | case <-done: 584 | } 585 | 586 | // Cleanup 587 | if err := os.RemoveAll(qName); err != nil { 588 | t.Fatal("Error removing queue directory:", err) 589 | } 590 | } 591 | 592 | func newOrOpenQ(t *testing.T, qName string, turbo bool) *dque.DQue { 593 | // Create a new segment with segment size of 3 594 | q, err := dque.NewOrOpen(qName, ".", 3, item2Builder) 595 | if err != nil { 596 | t.Fatal("Error creating or opening dque:", err) 597 | } 598 | 599 | if turbo { 600 | _ = q.TurboOn() 601 | } 602 | return q 603 | } 604 | 605 | func newQ(t *testing.T, qName string, turbo bool) *dque.DQue { 606 | // Create a new segment with segment size of 3 607 | q, err := dque.New(qName, ".", 3, item2Builder) 608 | if err != nil { 609 | t.Fatal("Error creating new dque:", err) 610 | } 611 | if turbo { 612 | _ = q.TurboOn() 613 | } 614 | return q 615 | } 616 | 617 | func openQ(t *testing.T, qName string, turbo bool) *dque.DQue { 618 | // Open an existing segment with segment size of 3 619 | q, err := dque.Open(qName, ".", 3, item2Builder) 620 | if err != nil { 621 | t.Fatal("Error opening dque:", err) 622 | } 623 | if turbo { 624 | _ = q.TurboOn() 625 | } 626 | return q 627 | } 628 | 629 | // assert fails the test if the condition is false. 630 | func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { 631 | if !condition { 632 | _, file, line, _ := runtime.Caller(1) 633 | fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) 634 | tb.FailNow() 635 | } 636 | } 637 | -------------------------------------------------------------------------------- /segment.go: -------------------------------------------------------------------------------- 1 | package dque 2 | 3 | // 4 | // Copyright (c) 2018 Jon Carlson. All rights reserved. 5 | // Use of this source code is governed by an MIT-style 6 | // license that can be found in the LICENSE file. 7 | // 8 | 9 | // 10 | // This is a segment of a memory-efficient FIFO durable queue. Items in the queue must be of the same type. 11 | // 12 | // Each qSegment instance corresponds to a file on disk. 13 | // 14 | // This segment is both persistent and in-memory so there is a memory limit to the size 15 | // (which is why it is just a segment instead of being used for the entire queue). 16 | // 17 | 18 | import ( 19 | "bytes" 20 | "encoding/binary" 21 | "encoding/gob" 22 | "fmt" 23 | "io" 24 | "os" 25 | "path" 26 | "sync" 27 | 28 | "github.com/pkg/errors" 29 | ) 30 | 31 | // ErrCorruptedSegment is returned when a segment file cannot be opened due to inconsistent formatting. 32 | // Recovery may be possible by clearing or deleting the file, then reloading using dque.New(). 33 | type ErrCorruptedSegment struct { 34 | Path string 35 | Err error 36 | } 37 | 38 | // Error returns a string describing ErrCorruptedSegment 39 | func (e ErrCorruptedSegment) Error() string { 40 | return fmt.Sprintf("segment file %s is corrupted: %s", e.Path, e.Err) 41 | } 42 | 43 | // Unwrap returns the wrapped error 44 | func (e ErrCorruptedSegment) Unwrap() error { 45 | return e.Err 46 | } 47 | 48 | // ErrUnableToDecode is returned when an object cannot be decoded. 49 | type ErrUnableToDecode struct { 50 | Path string 51 | Err error 52 | } 53 | 54 | // Error returns a string describing ErrUnableToDecode error 55 | func (e ErrUnableToDecode) Error() string { 56 | return fmt.Sprintf("object in segment file %s cannot be decoded: %s", e.Path, e.Err) 57 | } 58 | 59 | // Unwrap returns the wrapped error 60 | func (e ErrUnableToDecode) Unwrap() error { 61 | return e.Err 62 | } 63 | 64 | var ( 65 | errEmptySegment = errors.New("Segment is empty") 66 | ) 67 | 68 | // qSegment represents a portion (segment) of a persistent queue 69 | type qSegment struct { 70 | dirPath string 71 | number int 72 | objects []interface{} 73 | objectBuilder func() interface{} 74 | file *os.File 75 | mutex sync.Mutex 76 | removeCount int 77 | turbo bool 78 | maybeDirty bool // filesystem changes may not have been flushed to disk 79 | syncCount int64 // for testing 80 | } 81 | 82 | // load reads all objects from the queue file into a slice 83 | // returns ErrCorruptedSegment or ErrUnableToDecode for errors pertaining to file contents. 84 | func (seg *qSegment) load() error { 85 | 86 | // This is heavy-handed but its safe 87 | seg.mutex.Lock() 88 | defer seg.mutex.Unlock() 89 | 90 | // Open the file in read mode 91 | f, err := os.OpenFile(seg.filePath(), os.O_RDONLY, 0644) 92 | if err != nil { 93 | return errors.Wrap(err, "error opening file: "+seg.filePath()) 94 | } 95 | defer f.Close() 96 | seg.file = f 97 | 98 | // Loop until we can load no more 99 | for { 100 | // Read the 4 byte length of the gob 101 | lenBytes := make([]byte, 4) 102 | if n, err := io.ReadFull(seg.file, lenBytes); err != nil { 103 | if err == io.EOF { 104 | return nil 105 | } 106 | return ErrCorruptedSegment{ 107 | Path: seg.filePath(), 108 | Err: errors.Wrapf(err, "error reading object length (read %d/4 bytes)", n), 109 | } 110 | } 111 | 112 | // Convert the bytes into a 32-bit unsigned int 113 | gobLen := binary.LittleEndian.Uint32(lenBytes) 114 | if gobLen == 0 { 115 | // Remove the first item from the in-memory queue 116 | if len(seg.objects) == 0 { 117 | return ErrCorruptedSegment{ 118 | Path: seg.filePath(), 119 | Err: fmt.Errorf("excess deletion records (%d)", seg.removeCount+1), 120 | } 121 | } 122 | seg.objects = seg.objects[1:] 123 | // log.Println("TEMP: Detected delete in load()") 124 | seg.removeCount++ 125 | continue 126 | } 127 | 128 | data := make([]byte, int(gobLen)) 129 | if _, err := io.ReadFull(seg.file, data); err != nil { 130 | return ErrCorruptedSegment{ 131 | Path: seg.filePath(), 132 | Err: errors.Wrap(err, "error reading gob data from file"), 133 | } 134 | } 135 | 136 | // Decode the bytes into an object 137 | object := seg.objectBuilder() 138 | if err := gob.NewDecoder(bytes.NewReader(data)).Decode(object); err != nil { 139 | return ErrUnableToDecode{ 140 | Path: seg.filePath(), 141 | Err: errors.Wrapf(err, "failed to decode %T", object), 142 | } 143 | } 144 | 145 | // Add item to the objects slice 146 | seg.objects = append(seg.objects, object) 147 | 148 | // log.Printf("TEMP: Loaded: %#v\n", object) 149 | } 150 | } 151 | 152 | // peek returns the first item in the segment without removing it. 153 | // If the queue is already empty, the emptySegment error will be returned. 154 | func (seg *qSegment) peek() (interface{}, error) { 155 | 156 | // This is heavy-handed but its safe 157 | seg.mutex.Lock() 158 | defer seg.mutex.Unlock() 159 | 160 | if len(seg.objects) == 0 { 161 | // Queue is empty so return nil object (and emptySegment error) 162 | return nil, errEmptySegment 163 | } 164 | 165 | // Save a reference to the first item in the in-memory queue 166 | object := seg.objects[0] 167 | 168 | return object, nil 169 | } 170 | 171 | // remove removes and returns the first item in the segment and adds 172 | // a zero length marker to the end of the queue file to signify a removal. 173 | // If the queue is already empty, the emptySegment error will be returned. 174 | func (seg *qSegment) remove() (interface{}, error) { 175 | 176 | // This is heavy-handed but its safe 177 | seg.mutex.Lock() 178 | defer seg.mutex.Unlock() 179 | 180 | if len(seg.objects) == 0 { 181 | // Queue is empty so return nil object (and empty_segment error) 182 | return nil, errEmptySegment 183 | } 184 | 185 | // Create a 4-byte length of value zero (this signifies a removal) 186 | deleteLen := 0 187 | deleteLenBytes := make([]byte, 4) 188 | binary.LittleEndian.PutUint32(deleteLenBytes, uint32(deleteLen)) 189 | 190 | // Write the 4-byte length (of zero) first 191 | if _, err := seg.file.Write(deleteLenBytes); err != nil { 192 | return nil, errors.Wrapf(err, "failed to remove item from segment %d", seg.number) 193 | } 194 | 195 | // Save a reference to the first item in the in-memory queue 196 | object := seg.objects[0] 197 | 198 | // Remove the first item from the in-memory queue 199 | seg.objects = seg.objects[1:] 200 | 201 | // Increment the delete count 202 | seg.removeCount++ 203 | 204 | // Possibly force writes to disk 205 | if err := seg._sync(); err != nil { 206 | return nil, err 207 | } 208 | 209 | return object, nil 210 | } 211 | 212 | // Add adds an item to the in-memory queue segment and appends it to the persistent file 213 | func (seg *qSegment) add(object interface{}) error { 214 | 215 | // This is heavy-handed but its safe 216 | seg.mutex.Lock() 217 | defer seg.mutex.Unlock() 218 | 219 | // Encode the struct to a byte buffer 220 | var buff bytes.Buffer 221 | enc := gob.NewEncoder(&buff) 222 | if err := enc.Encode(object); err != nil { 223 | return errors.Wrap(err, "error gob encoding object") 224 | } 225 | 226 | // Count the bytes stored in the byte buffer 227 | // and store the count into a 4-byte byte array 228 | buffLen := len(buff.Bytes()) 229 | buffLenBytes := make([]byte, 4) 230 | binary.LittleEndian.PutUint32(buffLenBytes, uint32(buffLen)) 231 | 232 | // Write the 4-byte buffer length first 233 | if _, err := seg.file.Write(buffLenBytes); err != nil { 234 | return errors.Wrapf(err, "failed to write object length to segment %d", seg.number) 235 | } 236 | 237 | // Then write the buffer bytes 238 | if _, err := seg.file.Write(buff.Bytes()); err != nil { 239 | return errors.Wrapf(err, "failed to write object to segment %d", seg.number) 240 | } 241 | 242 | seg.objects = append(seg.objects, object) 243 | 244 | // Possibly force writes to disk 245 | return seg._sync() 246 | } 247 | 248 | // size returns the number of objects in this segment. 249 | // The size does not include items that have been removed. 250 | func (seg *qSegment) size() int { 251 | 252 | // This is heavy-handed but its safe 253 | seg.mutex.Lock() 254 | defer seg.mutex.Unlock() 255 | 256 | return len(seg.objects) 257 | } 258 | 259 | // sizeOnDisk returns the number of objects in memory plus removed objects. This 260 | // number will match the number of objects still on disk. 261 | // This number is used to keep the file from growing forever when items are 262 | // removed about as fast as they are added. 263 | func (seg *qSegment) sizeOnDisk() int { 264 | 265 | // This is heavy-handed but its safe 266 | seg.mutex.Lock() 267 | defer seg.mutex.Unlock() 268 | 269 | return len(seg.objects) + seg.removeCount 270 | } 271 | 272 | // delete wipes out the queue and its persistent state 273 | func (seg *qSegment) delete() error { 274 | 275 | // This is heavy-handed but its safe 276 | seg.mutex.Lock() 277 | defer seg.mutex.Unlock() 278 | 279 | if err := seg.file.Close(); err != nil { 280 | return errors.Wrap(err, "unable to close the segment file before deleting") 281 | } 282 | 283 | // Delete the storage for this queue 284 | err := os.Remove(seg.filePath()) 285 | if err != nil { 286 | return errors.Wrap(err, "error deleting file: "+seg.filePath()) 287 | } 288 | 289 | // Empty the in-memory slice of objects 290 | seg.objects = seg.objects[:0] 291 | 292 | seg.file = nil 293 | 294 | return nil 295 | } 296 | 297 | func (seg *qSegment) fileName() string { 298 | return fmt.Sprintf("%013d.dque", seg.number) 299 | } 300 | 301 | func (seg *qSegment) filePath() string { 302 | return path.Join(seg.dirPath, seg.fileName()) 303 | } 304 | 305 | // turboOn allows the filesystem to decide when to sync file changes to disk 306 | // Speed is be greatly increased by turning turbo on, however there is some 307 | // risk of losing data should a power-loss occur. 308 | func (seg *qSegment) turboOn() { 309 | seg.turbo = true 310 | } 311 | 312 | // turboOff re-enables the "safety" mode that syncs every file change to disk as 313 | // they happen. 314 | func (seg *qSegment) turboOff() error { 315 | if !seg.turbo { 316 | // turboOff is know to be called twice when the first and last ssegments 317 | // are the same. 318 | return nil 319 | } 320 | if err := seg.turboSync(); err != nil { 321 | return err 322 | } 323 | seg.turbo = false 324 | return nil 325 | } 326 | 327 | // turboSync does an fsync to disk if turbo is on. 328 | func (seg *qSegment) turboSync() error { 329 | if !seg.turbo { 330 | // When the first and last segments are the same, this method 331 | // will be called twice. 332 | return nil 333 | } 334 | if seg.maybeDirty { 335 | if err := seg.file.Sync(); err != nil { 336 | return errors.Wrap(err, "unable to sync file changes.") 337 | } 338 | seg.syncCount++ 339 | seg.maybeDirty = false 340 | } 341 | return nil 342 | } 343 | 344 | // _sync must only be called by the add and remove methods on qSegment. 345 | // Only syncs if turbo is off 346 | func (seg *qSegment) _sync() error { 347 | if seg.turbo { 348 | // We do *not* force a sync if turbo is on 349 | // We just mark it maybe dirty 350 | seg.maybeDirty = true 351 | return nil 352 | } 353 | 354 | if err := seg.file.Sync(); err != nil { 355 | return errors.Wrap(err, "unable to sync file changes in _sync method.") 356 | } 357 | seg.syncCount++ 358 | seg.maybeDirty = false 359 | return nil 360 | } 361 | 362 | // close is used when this is the last segment, but is now full, so we are 363 | // creating a new last segment. 364 | // This should only be called if this segment is not also the first segment. 365 | func (seg *qSegment) close() error { 366 | 367 | if err := seg.file.Close(); err != nil { 368 | return errors.Wrapf(err, "unable to close segment file %s.", seg.fileName()) 369 | } 370 | 371 | return nil 372 | } 373 | 374 | // newQueueSegment creates a new, persistent segment of the queue 375 | func newQueueSegment(dirPath string, number int, turbo bool, builder func() interface{}) (*qSegment, error) { 376 | 377 | seg := qSegment{dirPath: dirPath, number: number, turbo: turbo, objectBuilder: builder} 378 | 379 | if !dirExists(seg.dirPath) { 380 | return nil, errors.New("dirPath is not a valid directory: " + seg.dirPath) 381 | } 382 | 383 | if fileExists(seg.filePath()) { 384 | return nil, errors.New("file already exists: " + seg.filePath()) 385 | } 386 | 387 | // Create the file in append mode 388 | var err error 389 | seg.file, err = os.OpenFile(seg.filePath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 390 | if err != nil { 391 | return nil, errors.Wrapf(err, "error creating file: %s.", seg.filePath()) 392 | } 393 | // Leave the file open for future writes 394 | 395 | return &seg, nil 396 | } 397 | 398 | // openQueueSegment reads an existing persistent segment of the queue into memory 399 | func openQueueSegment(dirPath string, number int, turbo bool, builder func() interface{}) (*qSegment, error) { 400 | 401 | seg := qSegment{dirPath: dirPath, number: number, turbo: turbo, objectBuilder: builder} 402 | 403 | if !dirExists(seg.dirPath) { 404 | return nil, errors.New("dirPath is not a valid directory: " + seg.dirPath) 405 | } 406 | 407 | if !fileExists(seg.filePath()) { 408 | return nil, errors.New("file does not exist: " + seg.filePath()) 409 | } 410 | 411 | // Load the items into memory 412 | if err := seg.load(); err != nil { 413 | return nil, errors.Wrap(err, "unable to load queue segment in "+dirPath) 414 | } 415 | 416 | // Re-open the file in append mode 417 | var err error 418 | seg.file, err = os.OpenFile(seg.filePath(), os.O_APPEND|os.O_WRONLY, 0644) 419 | if err != nil { 420 | return nil, errors.Wrap(err, "error opening file: "+seg.filePath()) 421 | } 422 | // Leave the file open for future writes 423 | 424 | return &seg, nil 425 | } 426 | -------------------------------------------------------------------------------- /segment_test.go: -------------------------------------------------------------------------------- 1 | // segement_test.go 2 | package dque 3 | 4 | // 5 | // White box texting of the aSegment struct and methods. 6 | // 7 | 8 | import ( 9 | "fmt" 10 | "os" 11 | "path/filepath" 12 | "runtime" 13 | "testing" 14 | ) 15 | 16 | // item1 is the thing we'll be storing in the queue 17 | type item1 struct { 18 | Name string 19 | } 20 | 21 | // item1Builder creates a new item and returns a pointer to it. 22 | // This is used when we load a queue from disk. 23 | func item1Builder() interface{} { 24 | return &item1{} 25 | } 26 | 27 | // Test_segment verifies the behavior of one segment. 28 | func TestSegment(t *testing.T) { 29 | testDir := "./TestSegment" 30 | os.RemoveAll(testDir) 31 | if err := os.Mkdir(testDir, 0755); err != nil { 32 | t.Fatalf("Error creating directory from the TestSegment method: %s\n", err) 33 | } 34 | 35 | // Create a new segment of the queue 36 | seg, err := newQueueSegment(testDir, 1, false, item1Builder) 37 | if err != nil { 38 | t.Fatalf("newQueueSegment('%s') failed with '%s'\n", testDir, err.Error()) 39 | } 40 | 41 | // 42 | // Add some items and remove one 43 | // 44 | assert(t, seg.add(&item1{Name: "Number 1"}) == nil, "failed to add item1") 45 | assert(t, 1 == seg.size(), "Expected size of 1") 46 | 47 | assert(t, seg.add(&item1{Name: "Number 2"}) == nil, "failed to add item2") 48 | assert(t, 2 == seg.size(), "Expected size of 2") 49 | _, err = seg.remove() 50 | if err != nil { 51 | t.Fatalf("Remove() failed with '%s'\n", err.Error()) 52 | } 53 | assert(t, 1 == seg.size(), "Expected size of 1") 54 | assert(t, 2 == seg.sizeOnDisk(), "Expected sizeOnDisk of 2") 55 | assert(t, seg.add(&item1{Name: "item3"}) == nil, "failed to add item3") 56 | assert(t, 2 == seg.size(), "Expected size of 2") 57 | _, err = seg.remove() 58 | if err != nil { 59 | t.Fatalf("Remove() failed with '%s'\n", err.Error()) 60 | } 61 | assert(t, 1 == seg.size(), "Expected size of 1") 62 | 63 | // 64 | // Recreate the segment from disk and remove the remaining item 65 | // 66 | seg, err = openQueueSegment(testDir, 1, false, item1Builder) 67 | if err != nil { 68 | t.Fatalf("openQueueSegment('%s') failed with '%s'\n", testDir, err.Error()) 69 | } 70 | assert(t, 1 == seg.size(), "Expected size of 1") 71 | 72 | _, err = seg.remove() 73 | if err != nil { 74 | if err != errEmptySegment { 75 | t.Fatalf("Remove() failed with '%s'\n", err.Error()) 76 | } 77 | } 78 | assert(t, 0 == seg.size(), "Expected size of 0") 79 | 80 | // Cleanup 81 | if err := os.RemoveAll(testDir); err != nil { 82 | t.Fatalf("Error cleaning up directory from the TestSegment method with '%s'\n", err.Error()) 83 | } 84 | } 85 | 86 | // TestSegment_ErrCorruptedSegment tests error handling for corrupted data 87 | func TestSegment_ErrCorruptedSegment(t *testing.T) { 88 | testDir := "./TestSegmentError" 89 | os.RemoveAll(testDir) 90 | defer os.RemoveAll((testDir)) 91 | 92 | if err := os.Mkdir(testDir, 0755); err != nil { 93 | t.Fatalf("Error creating directory in the TestSegment_ErrCorruptedSegment method: %s\n", err) 94 | } 95 | 96 | f, err := os.Create((&qSegment{dirPath: testDir}).filePath()) 97 | if err != nil { 98 | t.Fatal(err) 99 | } 100 | 101 | // expect an 8 byte object, but only write 7 bytes 102 | if _, err := f.Write([]byte{0, 0, 0, 8, 1, 2, 3, 4, 5, 6, 7}); err != nil { 103 | t.Fatal(err) 104 | } 105 | f.Close() 106 | 107 | _, err = openQueueSegment(testDir, 0, false, func() interface{} { return make([]byte, 8) }) 108 | if err == nil { 109 | t.Fatal("expected ErrCorruptedSegment but got nil") 110 | } 111 | // // go >= 1.13: 112 | // var corruptedError ErrCorruptedSegment 113 | // if !errors.As(err, &corruptedError) { 114 | // t.Fatalf("expected ErrCorruptedSegment but got %T: %s", err, err) 115 | // } 116 | corruptedError, ok := unwrapError(unwrapError(err)).(ErrCorruptedSegment) 117 | if !ok { 118 | t.Fatalf("expected ErrCorruptedSegment but got %T: %s", err, err) 119 | } 120 | if corruptedError.Path != "TestSegmentError/0000000000000.dque" { 121 | t.Fatalf("unexpected file path: %s", corruptedError.Path) 122 | } 123 | if corruptedError.Error() != "segment file TestSegmentError/0000000000000.dque is corrupted: error reading gob data from file: unexpected EOF" { 124 | t.Fatalf("wrong error message: %s", corruptedError.Error()) 125 | } 126 | } 127 | 128 | func unwrapError(err error) error { 129 | return err.(interface{ Unwrap() error }).Unwrap() 130 | } 131 | 132 | // TestSegment_Open verifies the behavior of the openSegment function. 133 | func TestSegment_openQueueSegment_failIfNew(t *testing.T) { 134 | testDir := "./TestSegment_Open" 135 | os.RemoveAll(testDir) 136 | if err := os.Mkdir(testDir, 0755); err != nil { 137 | t.Fatalf("Error creating directory in the TestSegment_Open method: %s\n", err) 138 | } 139 | 140 | seg, err := openQueueSegment(testDir, 1, false, item1Builder) 141 | if err == nil { 142 | t.Fatalf("openQueueSegment('%s') should have failed because it should be new\n", testDir) 143 | } 144 | assert(t, seg == nil, "segment after failure must be nil") 145 | 146 | // Cleanup 147 | if err := os.RemoveAll(testDir); err != nil { 148 | t.Fatalf("Error cleaning up directory from the TestSegment_Open method with '%s'\n", err.Error()) 149 | } 150 | } 151 | 152 | // TestSegment_Turbo verifies the behavior of the turboOn() and turboOff() methods. 153 | func TestSegment_Turbo(t *testing.T) { 154 | testDir := "./TestSegment" 155 | os.RemoveAll(testDir) 156 | if err := os.Mkdir(testDir, 0755); err != nil { 157 | t.Fatalf("Error creating directory in the TestSegment_Turbo method: %s\n", err) 158 | } 159 | 160 | seg, err := newQueueSegment(testDir, 10, false, item1Builder) 161 | if err != nil { 162 | t.Fatalf("newQueueSegment('%s') failed\n", testDir) 163 | } 164 | 165 | // turbo is off so expect syncCount to change 166 | assert(t, seg.add(&item1{Name: "Number 1"}) == nil, "failed to add item1") 167 | assert(t, 1 == seg.size(), "Expected size of 1") 168 | assert(t, 1 == seg.syncCount, "syncCount must be 1") 169 | 170 | // Turn on turbo and expect sync count to stay the same. 171 | seg.turboOn() 172 | assert(t, seg.add(&item1{Name: "Number 2"}) == nil, "failed to add item2") 173 | assert(t, 2 == seg.size(), "Expected size of 2") 174 | assert(t, 1 == seg.syncCount, "syncCount must still be 1") 175 | 176 | // Turn off turbo and expect the syncCount to increase when remove is called. 177 | if err = seg.turboOff(); err != nil { 178 | t.Fatalf("Unexpecte error turning off turbo('%s')\n", testDir) 179 | } 180 | 181 | // seg.turboOff() calls seg.turboSync() which increments syncCount 182 | assert(t, 2 == seg.syncCount, "syncCount must be 2 now") 183 | 184 | _, err = seg.remove() 185 | if err != nil { 186 | t.Fatalf("Remove() failed with '%s'\n", err.Error()) 187 | } 188 | // seg.remove() calls seg._sync() which increments syncCount 189 | assert(t, 3 == seg.syncCount, "syncCount must be 3 now") 190 | 191 | // Cleanup 192 | if err := os.RemoveAll(testDir); err != nil { 193 | t.Fatalf("Error cleaning up directory from the TestSegment_Open method with '%s'\n", err.Error()) 194 | } 195 | } 196 | 197 | // assert fails the test if the condition is false. 198 | func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { 199 | if !condition { 200 | _, file, line, _ := runtime.Caller(1) 201 | fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) 202 | tb.FailNow() 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package dque 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | // dirExists returns true or false 8 | func dirExists(path string) bool { 9 | fileInfo, err := os.Stat(path) 10 | if err == nil { 11 | return fileInfo.IsDir() 12 | } 13 | return false 14 | } 15 | 16 | // fileExists returns true or false 17 | func fileExists(path string) bool { 18 | fileInfo, err := os.Stat(path) 19 | if err == nil { 20 | return !fileInfo.IsDir() 21 | } 22 | return false 23 | } 24 | --------------------------------------------------------------------------------