├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── example_test.go ├── qr.go └── qr_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | d/ 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | script: make test 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Harmen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: short test install bench 2 | 3 | short: 4 | go test -short 5 | go vet . 6 | golint . 7 | 8 | test: 9 | go test 10 | 11 | install: test 12 | go install 13 | 14 | bench: 15 | go test -short -bench . 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | In-process queue with disk based overflow. 2 | 3 | 4 | When everything is fine elements flow over Qr.q. This is a simple channel 5 | connecting the producer(s) and the consumer(s). 6 | If that channel is full elements are written to the Qr.planb channel. 7 | swapout() will write all elements from Qr.planb to disk. It makes a new file 8 | every `timeout`. At the same time swapin() will deal with completed files. 9 | swapin() will open the oldest file and write the elements to Qr.q. 10 | 11 | ``` 12 | ---> Enqueue() ------ .q -----> merge() -> .out -> Dequeue() ---> 13 | \ ^ 14 | .planb .confluence 15 | \ / 16 | \--> swapout() swapin() --/ 17 | \ ^ 18 | \--> fs() --/ 19 | ``` 20 | 21 | Gob is used to serialize entries; custom types should be registered using 22 | gob.Register(). 23 | 24 | Same idea as https://github.com/alicebob/q but cleaner, and this queue doesn't care about keeping things ordered. 25 | 26 | 27 | # &c. 28 | 29 | [![Build Status](https://travis-ci.org/alicebob/qr.svg?branch=master)](https://travis-ci.org/alicebob/qr) 30 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package qr_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/alicebob/qr" 6 | ) 7 | 8 | func Example() { 9 | q, err := qr.New( 10 | "/tmp/", 11 | "example", 12 | qr.OptionBuffer(100), 13 | qr.OptionTest("your datatype"), 14 | ) 15 | if err != nil { 16 | panic(err) 17 | } 18 | defer q.Close() 19 | go func() { 20 | for e := range q.Dequeue() { 21 | fmt.Printf("We got: %v\n", e) 22 | } 23 | }() 24 | 25 | // elsewhere: 26 | q.Enqueue("aap") 27 | q.Enqueue("noot") 28 | } 29 | -------------------------------------------------------------------------------- /qr.go: -------------------------------------------------------------------------------- 1 | // Package qr is an in process queue with disk based overflow. Element order is 2 | // not strictly preserved. 3 | // 4 | // When everything is fine elements flow over Qr.q. This is a simple channel 5 | // connecting the producer(s) and the consumer(s). 6 | // If that channel is full elements are written to the Qr.planb channel. 7 | // swapout() will write all elements from Qr.planb to disk. It makes a new file 8 | // every `timeout`. At the same time swapin() will deal with completed files. 9 | // swapin() will open the oldest file and write the elements to Qr.q. 10 | // 11 | // ---> Enqueue() ------ .q -----> merge() -> .out -> Dequeue() ---> 12 | // \ ^ 13 | // .planb .confluence 14 | // \ / 15 | // \--> swapout() swapin() --/ 16 | // \ ^ 17 | // \--> fs() --/ 18 | // 19 | // 20 | // Gob is used to serialize entries; custom types should be registered using 21 | // gob.Register(). 22 | // 23 | // 24 | package qr 25 | 26 | import ( 27 | "encoding/gob" 28 | "errors" 29 | "fmt" 30 | "io" 31 | "log" 32 | "os" 33 | "path/filepath" 34 | "reflect" 35 | "sort" 36 | "strings" 37 | "sync/atomic" 38 | "time" 39 | ) 40 | 41 | const ( 42 | // DefaultTimeout can be changed with OptionTimeout. 43 | DefaultTimeout = 10 * time.Second 44 | // DefaultBuffer can be changed with OptionBuffer. 45 | DefaultBuffer = 1000 46 | 47 | fileExtension = ".qr" 48 | ) 49 | 50 | var ( 51 | // ErrInvalidPrefix is potentially returned by New. 52 | ErrInvalidPrefix = errors.New("invalid prefix") 53 | ) 54 | 55 | // Qr is a disk-based queue. Create one with New(). 56 | type Qr struct { 57 | q chan interface{} // the main channel. 58 | planb chan interface{} // to disk, used when q is full. 59 | confluence chan interface{} // from disk to merge() 60 | out chan interface{} 61 | dir string 62 | prefix string 63 | timeout time.Duration 64 | bufferSize int 65 | logf func(string, ...interface{}) // Printf() style 66 | fileCount int64 // via atomic 67 | } 68 | 69 | // Option is an option to New(), which can change some settings. 70 | type Option func(qr *Qr) error 71 | 72 | // OptionTimeout is an option for New(). It specifies the time after which a queue 73 | // file is closed. Smaller means more files. 74 | func OptionTimeout(t time.Duration) Option { 75 | return func(qr *Qr) error { 76 | qr.timeout = t 77 | return nil 78 | } 79 | } 80 | 81 | // OptionBuffer is an option for New(). It specifies the in-memory size of the 82 | // queue. Smaller means the disk will be used sooner, larger means more memory. 83 | func OptionBuffer(n int) Option { 84 | return func(qr *Qr) error { 85 | qr.bufferSize = n 86 | return nil 87 | } 88 | } 89 | 90 | // OptionLogger is an option for New(). Is sets the logger, the default is 91 | // log.Printf, but glog.Errorf would also work. 92 | func OptionLogger(l func(string, ...interface{})) Option { 93 | return func(qr *Qr) error { 94 | qr.logf = l 95 | return nil 96 | } 97 | } 98 | 99 | // OptionTest is an option for New(). It tests that the given sample item can 100 | // be serialized to disk and deserialized successfully. This verifies that disk 101 | // access works, and that the type can be fully serialized and deserialized 102 | // with gob. The option can be repeated. 103 | func OptionTest(t interface{}) Option { 104 | return func(qr *Qr) error { 105 | return qr.test(t) 106 | } 107 | } 108 | 109 | // New starts a Queue which stores files in /-..qr 110 | // 'prefix' must be a simple ASCII string. 111 | func New(dir, prefix string, options ...Option) (*Qr, error) { 112 | if len(prefix) == 0 || strings.ContainsAny(prefix, ":-/") { 113 | return nil, ErrInvalidPrefix 114 | } 115 | 116 | qr := Qr{ 117 | planb: make(chan interface{}), 118 | confluence: make(chan interface{}), 119 | out: make(chan interface{}), 120 | dir: dir, 121 | prefix: prefix, 122 | timeout: DefaultTimeout, 123 | bufferSize: DefaultBuffer, 124 | logf: log.Printf, 125 | } 126 | for _, cb := range options { 127 | if err := cb(&qr); err != nil { 128 | return nil, err 129 | } 130 | } 131 | 132 | qr.q = make(chan interface{}, qr.bufferSize) 133 | 134 | var ( 135 | filesToDisk = make(chan string) 136 | filesFromDisk = make(chan string) 137 | ) 138 | go qr.merge() 139 | go qr.swapout(filesToDisk) 140 | go qr.fs(filesToDisk, filesFromDisk) 141 | go qr.swapin(filesFromDisk) 142 | for _, f := range qr.findOld() { 143 | filesToDisk <- f 144 | } 145 | return &qr, nil 146 | } 147 | 148 | // Enqueue adds something in the queue. This never blocks, and is safe to be 149 | // called by different goroutines. 150 | func (qr *Qr) Enqueue(e interface{}) { 151 | select { 152 | case qr.q <- e: 153 | default: 154 | qr.planb <- e 155 | } 156 | } 157 | 158 | // Dequeue is the channel where elements come out the queue. It'll be closed 159 | // on Close(). 160 | func (qr *Qr) Dequeue() <-chan interface{} { 161 | return qr.out 162 | } 163 | 164 | // FileCount gives the number of files on disk. Useful to graph to get an idea 165 | // about disk usage. 166 | func (qr *Qr) FileCount() int { 167 | return int(atomic.LoadInt64(&qr.fileCount)) 168 | } 169 | 170 | // Close shuts down all Go routines and closes the Dequeue() channel. It'll 171 | // write all in-flight entries to disk. Calling Enqueue() after Close will 172 | // panic. 173 | func (qr *Qr) Close() { 174 | close(qr.q) 175 | // Closing planb triggers a cascade closing of all go-s and channels. 176 | close(qr.planb) 177 | 178 | // Store the in-flight entries for next time. 179 | filename := qr.batchFilename(0) // special filename 180 | fh, err := os.Create(filename) 181 | if err != nil { 182 | qr.logf("QR create err: %v", err) 183 | return 184 | } 185 | enc := gob.NewEncoder(fh) 186 | count := 0 187 | for e := range qr.out { 188 | count++ 189 | if err = enc.Encode(&e); err != nil { 190 | qr.logf("QR encode err: %v", err) 191 | } 192 | } 193 | fh.Close() 194 | if count == 0 { 195 | // there was nothing to queue 196 | os.Remove(filename) 197 | } 198 | } 199 | 200 | // test tests that the given sample item can be serialized to disk and 201 | // deserialized successfully. This verifies that disk access works, and that 202 | // the type can be fully serialized and deserialized. 203 | func (qr *Qr) test(i interface{}) error { 204 | filename := qr.testBatchFilename() 205 | 206 | f, err := os.Create(filename) 207 | if err != nil { 208 | return fmt.Errorf("create err: %v", err) 209 | } 210 | defer os.Remove(filename) 211 | defer f.Close() 212 | enc := gob.NewEncoder(f) 213 | if err := enc.Encode(&i); err != nil { 214 | return err 215 | } 216 | 217 | if f, err = os.Open(filename); err != nil { 218 | return fmt.Errorf("create err: %v", err) 219 | } 220 | defer f.Close() 221 | dec := gob.NewDecoder(f) 222 | var c interface{} 223 | if err = dec.Decode(&c); err != nil { 224 | return err 225 | } 226 | if !reflect.DeepEqual(i, c) { 227 | return fmt.Errorf("deserialization error: have %#v, want %#v", c, i) 228 | } 229 | return nil 230 | } 231 | 232 | func (qr *Qr) merge() { 233 | defer func() { 234 | for e := range qr.q { 235 | qr.out <- e 236 | } 237 | for e := range qr.confluence { 238 | qr.out <- e 239 | } 240 | close(qr.out) 241 | }() 242 | 243 | // read q and planb, and write them to out 244 | for { 245 | // prefer to read from Q 246 | select { 247 | case e, ok := <-qr.q: 248 | if !ok { 249 | return 250 | } 251 | qr.out <- e 252 | continue 253 | default: 254 | } 255 | 256 | // otherwise try both 257 | select { 258 | case e, ok := <-qr.q: 259 | if !ok { 260 | return 261 | } 262 | qr.out <- e 263 | case e, ok := <-qr.confluence: 264 | if !ok { 265 | return 266 | } 267 | qr.out <- e 268 | } 269 | } 270 | } 271 | 272 | func (qr *Qr) swapout(files chan<- string) { 273 | var ( 274 | enc *gob.Encoder 275 | filename string 276 | fh io.WriteCloser 277 | tc <-chan time.Time 278 | t = time.NewTimer(0) 279 | err error 280 | ) 281 | defer func() { 282 | if enc != nil { 283 | fh.Close() 284 | files <- filename 285 | } 286 | close(files) 287 | t.Stop() 288 | }() 289 | for { 290 | select { 291 | case e, ok := <-qr.planb: 292 | if !ok { 293 | return 294 | } 295 | if enc == nil { 296 | filename = qr.batchFilename(time.Now().UnixNano()) 297 | fh, err = os.Create(filename) 298 | if err != nil { 299 | // TODO: sure we return? 300 | qr.logf("QR create err: %v", err) 301 | return 302 | } 303 | enc = gob.NewEncoder(fh) 304 | t.Reset(qr.timeout) 305 | tc = t.C 306 | } 307 | if err = enc.Encode(&e); err != nil { 308 | qr.logf("QR encode err: %v", err) 309 | } 310 | case <-tc: 311 | fh.Close() 312 | files <- filename 313 | enc = nil 314 | tc = nil 315 | } 316 | } 317 | } 318 | 319 | func (qr *Qr) swapin(files <-chan string) { 320 | defer close(qr.confluence) 321 | for filename := range files { 322 | fh, err := os.Open(filename) 323 | if err != nil { 324 | qr.logf("QR open err: %v", err) 325 | continue 326 | } 327 | os.Remove(filename) 328 | dec := gob.NewDecoder(fh) 329 | for { 330 | var next interface{} 331 | if err = dec.Decode(&next); err != nil { 332 | if err != io.EOF { 333 | qr.logf("QR decode err: %v", err) 334 | } 335 | fh.Close() 336 | break 337 | } 338 | qr.confluence <- next 339 | } 340 | } 341 | } 342 | 343 | func (qr *Qr) fs(in <-chan string, out chan<- string) { 344 | defer close(out) 345 | var ( 346 | filenames []string 347 | checkOut chan<- string 348 | next string 349 | ) 350 | for { 351 | select { 352 | case f, ok := <-in: 353 | if !ok { 354 | return 355 | } 356 | if checkOut == nil { 357 | checkOut = out 358 | next = f 359 | } else { 360 | filenames = append(filenames, f) 361 | } 362 | case checkOut <- next: 363 | if len(filenames) > 0 { 364 | next, filenames = filenames[0], filenames[1:] 365 | } else { 366 | // case disabled since there is no file 367 | checkOut = nil 368 | } 369 | } 370 | atomic.StoreInt64(&qr.fileCount, int64(len(filenames))) 371 | } 372 | } 373 | 374 | func (qr *Qr) batchFilename(n int64) string { 375 | return fmt.Sprintf("%s/%s-%020d%s", 376 | qr.dir, 377 | qr.prefix, 378 | n, 379 | fileExtension, 380 | ) 381 | } 382 | 383 | func (qr *Qr) testBatchFilename() string { 384 | return fmt.Sprintf("%s/%s-test%s", qr.dir, qr.prefix, fileExtension) 385 | } 386 | 387 | // findOld finds .qr files from a previous run. 388 | func (qr *Qr) findOld() []string { 389 | f, err := os.Open(qr.dir) 390 | if err != nil { 391 | return nil 392 | } 393 | defer f.Close() 394 | 395 | names, err := f.Readdirnames(-1) 396 | if err != nil { 397 | return nil 398 | } 399 | 400 | var existing []string 401 | for _, n := range names { 402 | if !strings.HasPrefix(n, qr.prefix+"-") || 403 | !strings.HasSuffix(n, fileExtension) || 404 | strings.HasSuffix(n, "-test"+fileExtension) { 405 | continue 406 | } 407 | existing = append(existing, filepath.Join(qr.dir, n)) 408 | } 409 | 410 | sort.Strings(existing) 411 | 412 | return existing 413 | } 414 | -------------------------------------------------------------------------------- /qr_test.go: -------------------------------------------------------------------------------- 1 | package qr_test 2 | 3 | import ( 4 | "encoding/gob" 5 | "fmt" 6 | "math/rand" 7 | "os" 8 | "strings" 9 | "sync" 10 | "testing" 11 | "time" 12 | 13 | "github.com/alicebob/qr" 14 | ) 15 | 16 | func init() { 17 | rand.Seed(time.Now().Unix()) 18 | } 19 | 20 | func setupDataDir() string { 21 | os.RemoveAll("./d") 22 | if err := os.Mkdir("./d/", 0700); err != nil { 23 | panic(fmt.Sprintf("Can't make ./d/: %v", err)) 24 | } 25 | return "./d" 26 | } 27 | 28 | func TestBasic(t *testing.T) { 29 | d := setupDataDir() 30 | 31 | q, err := qr.New(d, "test") 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | defer q.Close() 36 | for i := 0; i < 1000; i++ { 37 | q.Enqueue(i) 38 | } 39 | 40 | ret := make([]int, 1000) 41 | for i := range ret { 42 | select { 43 | case ii := <-q.Dequeue(): 44 | ret[i] = ii.(int) 45 | case <-time.After(2 * time.Second): 46 | t.Fatalf("q should not be empty") 47 | } 48 | } 49 | 50 | select { 51 | case e := <-q.Dequeue(): 52 | t.Fatalf("q should be empty, got a %#v", e) 53 | default: 54 | // ok 55 | } 56 | } 57 | 58 | func TestBlock(t *testing.T) { 59 | // Read should block until there is something. 60 | d := setupDataDir() 61 | q, err := qr.New(d, "test") 62 | if err != nil { 63 | t.Fatal(err) 64 | } 65 | defer q.Close() 66 | 67 | ready := make(chan struct{}) 68 | 69 | wg := sync.WaitGroup{} 70 | wg.Add(1) 71 | go func() { 72 | defer wg.Done() 73 | ready <- struct{}{} 74 | if got := <-q.Dequeue(); got != "hello world" { 75 | t.Errorf("Want hello, got %#v", got) 76 | } 77 | }() 78 | <-ready 79 | 80 | q.Enqueue("hello world") 81 | 82 | wg.Wait() 83 | } 84 | 85 | func TestBig(t *testing.T) { 86 | // Queue a lot of elements. 87 | if testing.Short() { 88 | t.Skip("skipping test in short mode.") 89 | } 90 | 91 | var ( 92 | d = setupDataDir() 93 | eventCount = 10000 94 | payload = strings.Repeat("0xDEADBEEF", 300) 95 | ) 96 | q, err := qr.New(d, "events", qr.OptionTimeout(10*time.Millisecond)) 97 | if err != nil { 98 | t.Fatal(err) 99 | } 100 | 101 | for i := 0; i < eventCount; i++ { 102 | q.Enqueue(payload) 103 | } 104 | if have, wantMin := q.FileCount(), 1; have < wantMin { 105 | t.Errorf("have %d, want at least %d", have, wantMin) 106 | } 107 | for i := 0; i < eventCount; i++ { 108 | if have, want := <-q.Dequeue(), payload; have != want { 109 | t.Fatalf("Want for %d: have: %#v, want %#v", i, have, want) 110 | } 111 | } 112 | if have, want := q.FileCount(), 0; have != want { 113 | t.Errorf("have %d, want %d", have, want) 114 | } 115 | q.Close() 116 | 117 | if have, want := fileCount(d), 0; have != want { 118 | t.Fatalf("Wrong number of files: have %d, have %d", have, want) 119 | } 120 | } 121 | 122 | func TestAsync(t *testing.T) { 123 | // Random sleep readers and writers. 124 | if testing.Short() { 125 | t.Skip("skipping test in short mode.") 126 | } 127 | 128 | var ( 129 | d = setupDataDir() 130 | eventCount = 10000 131 | payload = strings.Repeat("0xDEADBEEF", 300) 132 | wg = sync.WaitGroup{} 133 | ) 134 | q, err := qr.New(d, "events", qr.OptionTimeout(10*time.Millisecond)) 135 | if err != nil { 136 | t.Fatal(err) 137 | } 138 | 139 | wg.Add(1) 140 | go func() { 141 | defer wg.Done() 142 | for i := 0; i < eventCount; i++ { 143 | q.Enqueue(payload) 144 | time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) 145 | } 146 | }() 147 | 148 | // Reader is a little slower. 149 | wg.Add(1) 150 | go func() { 151 | defer wg.Done() 152 | for i := 0; i < eventCount; i++ { 153 | if have, want := <-q.Dequeue(), payload; have != want { 154 | t.Fatalf("have %#v, want %#v", have, want) 155 | } 156 | time.Sleep(time.Duration(rand.Intn(150)) * time.Microsecond) 157 | } 158 | }() 159 | 160 | wg.Wait() 161 | q.Close() 162 | 163 | if got, want := fileCount(d), 0; got != want { 164 | t.Fatalf("Wrong number of files: got %d, want %d", got, want) 165 | } 166 | } 167 | 168 | func TestMany(t *testing.T) { 169 | // Read and write a lot of messages, as fast as possible. 170 | if testing.Short() { 171 | t.Skip("skipping test in short mode.") 172 | } 173 | 174 | var ( 175 | d = setupDataDir() 176 | eventCount = 1000000 177 | clients = 10 178 | payload = strings.Repeat("0xDEADBEEF", 30) 179 | wg = sync.WaitGroup{} 180 | ) 181 | q, err := qr.New(d, "events", qr.OptionTimeout(100*time.Millisecond)) 182 | if err != nil { 183 | t.Fatal(err) 184 | } 185 | 186 | for i := 0; i < clients; i++ { 187 | wg.Add(1) 188 | go func() { 189 | defer wg.Done() 190 | for j := 0; j < eventCount/clients; j++ { 191 | q.Enqueue(payload) 192 | } 193 | }() 194 | } 195 | wg.Wait() 196 | 197 | wg.Add(1) 198 | go func() { 199 | defer wg.Done() 200 | for i := 0; i < eventCount; i++ { 201 | if got := <-q.Dequeue(); payload != got { 202 | t.Fatalf("Want for %d: %#v, got %#v", i, payload, got) 203 | } 204 | } 205 | }() 206 | wg.Wait() 207 | 208 | q.Close() 209 | 210 | if got, want := fileCount(d), 0; got != want { 211 | t.Fatalf("Wrong number of files: got %d, want %d", got, want) 212 | } 213 | } 214 | 215 | func TestReopen(t *testing.T) { 216 | // Simple reopening. 217 | d := setupDataDir() 218 | q, err := qr.New(d, "events") 219 | if err != nil { 220 | t.Fatal(err) 221 | } 222 | 223 | q.Enqueue("Message 1") 224 | q.Enqueue("Message 2") 225 | q.Close() 226 | 227 | q, err = qr.New(d, "events") 228 | if err != nil { 229 | t.Fatal(err) 230 | } 231 | select { 232 | case <-q.Dequeue(): 233 | case <-time.After(10 * time.Millisecond): 234 | t.Fatalf("nothing to read") 235 | } 236 | <-q.Dequeue() 237 | q.Close() 238 | 239 | if got, want := fileCount(d), 0; got != want { 240 | t.Fatalf("Wrong number of files: got %d, want %d", got, want) 241 | } 242 | } 243 | 244 | func TestReopenBig(t *testing.T) { 245 | // Queue a lot of elements. 246 | if testing.Short() { 247 | t.Skip("skipping test in short mode.") 248 | } 249 | 250 | var ( 251 | d = setupDataDir() 252 | eventCount = 10000 253 | payload = strings.Repeat("0xDEADBEEF", 300) 254 | ) 255 | q, err := qr.New(d, "events", qr.OptionTimeout(10*time.Millisecond)) 256 | if err != nil { 257 | t.Fatal(err) 258 | } 259 | 260 | for i := 0; i < eventCount; i++ { 261 | q.Enqueue(payload) 262 | } 263 | q.Close() 264 | 265 | q, err = qr.New(d, "events") 266 | if err != nil { 267 | t.Fatal(err) 268 | } 269 | for i := 0; i < eventCount; i++ { 270 | if have, want := <-q.Dequeue(), payload; have != want { 271 | t.Fatalf("Want for %d: have: %#v, want %#v", i, have, want) 272 | } 273 | } 274 | q.Close() 275 | 276 | if have, want := fileCount(d), 0; have != want { 277 | t.Fatalf("Wrong number of files: have %d, have %d", have, want) 278 | } 279 | } 280 | 281 | func TestReadOnly(t *testing.T) { 282 | // Only reading doesn't block the close. 283 | d := setupDataDir() 284 | q, err := qr.New(d, "i") 285 | if err != nil { 286 | t.Fatal(err) 287 | } 288 | 289 | select { 290 | case v := <-q.Dequeue(): 291 | t.Fatalf("Impossible read: %v", v) 292 | default: 293 | } 294 | 295 | q.Close() 296 | } 297 | 298 | func TestStruct(t *testing.T) { 299 | d := setupDataDir() 300 | q, err := qr.New(d, "events") 301 | if err != nil { 302 | t.Fatal(err) 303 | } 304 | defer q.Close() 305 | 306 | type s struct { 307 | X string 308 | Y int 309 | } 310 | gob.Register(s{}) 311 | 312 | data := []s{ 313 | {"Event", 1}, 314 | {"alice", 2}, 315 | {"bob", 3}, 316 | } 317 | for _, d := range data { 318 | q.Enqueue(d) 319 | } 320 | for _, want := range data { 321 | if got := <-q.Dequeue(); want != got { 322 | t.Errorf("Want %#v, got %#v", want, got) 323 | } 324 | } 325 | } 326 | 327 | func TestTwoStructs(t *testing.T) { 328 | d := setupDataDir() 329 | q1, err := qr.New(d, "s1") 330 | if err != nil { 331 | t.Fatal(err) 332 | } 333 | q2, err := qr.New(d, "s2") 334 | if err != nil { 335 | t.Fatal(err) 336 | } 337 | defer q1.Close() 338 | defer q2.Close() 339 | 340 | type s1 struct { 341 | X string 342 | Y int 343 | } 344 | gob.Register(s1{}) 345 | 346 | type s2 struct { 347 | A float64 348 | B string 349 | } 350 | gob.Register(s2{}) 351 | 352 | data1 := []s1{ 353 | {"Event", 1}, 354 | {"alice", 2}, 355 | {"bob", 3}, 356 | } 357 | 358 | data2 := []s2{ 359 | {3.14, "pi"}, 360 | {2.72, "e"}, 361 | } 362 | 363 | for _, d1 := range data1 { 364 | q1.Enqueue(d1) 365 | } 366 | for _, d2 := range data2 { 367 | q2.Enqueue(d2) 368 | } 369 | 370 | for _, want := range data1 { 371 | if got := <-q1.Dequeue(); want != got { 372 | t.Errorf("Want %#v, got %#v", want, got) 373 | } 374 | } 375 | for _, want := range data2 { 376 | if got := <-q2.Dequeue(); want != got { 377 | t.Errorf("Want %#v, got %#v", want, got) 378 | } 379 | } 380 | 381 | if got, want := fileCount(d), 0; got != want { 382 | t.Fatalf("Wrong number of files: got %d, want %d", got, want) 383 | } 384 | } 385 | 386 | func TestTest(t *testing.T) { 387 | d := setupDataDir() 388 | q, err := qr.New(d, "xxx", qr.OptionTest("hello")) 389 | if err != nil { 390 | t.Fatal(err) 391 | } 392 | defer q.Close() 393 | 394 | type r struct { 395 | X string 396 | Y int 397 | } 398 | 399 | if _, err := qr.New(d, "xxx", qr.OptionTest(r{"hello", 1})); err == nil { 400 | t.Errorf("should have failed for unregistered struct") 401 | } 402 | 403 | gob.Register(r{}) 404 | if _, err := qr.New(d, "xxx", qr.OptionTest(r{"hello", 1})); err != nil { 405 | t.Fatal(err) 406 | } 407 | } 408 | 409 | func TestInvalidPrefix(t *testing.T) { 410 | // Need a non-nil prefix. 411 | d := setupDataDir() 412 | for prefix, valid := range map[string]bool{ 413 | "": false, 414 | "foobar": true, 415 | "foo/bar": false, 416 | "foo-bar": false, 417 | } { 418 | _, err := qr.New(d, prefix) 419 | if have, want := (err == nil), valid; have != want { 420 | t.Fatalf("prefix: %q, have: %t, want: %t", prefix, have, want) 421 | } 422 | } 423 | } 424 | 425 | // fileCount is a helper to count files in a directory. 426 | func fileCount(dir string) int { 427 | fh, err := os.Open(dir) 428 | if err != nil { 429 | panic(err) 430 | } 431 | defer fh.Close() 432 | 433 | n, err := fh.Readdirnames(-1) 434 | if err != nil { 435 | panic(err) 436 | } 437 | return len(n) 438 | } 439 | --------------------------------------------------------------------------------