├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── queued.go └── queued ├── application.go ├── application_test.go ├── config.go ├── goleveldb_store.go ├── goleveldb_store_test.go ├── handlers.go ├── item.go ├── level_store.go ├── level_store_test.go ├── memory_store.go ├── memory_store_test.go ├── queue.go ├── queue_test.go ├── record.go ├── server.go ├── server_test.go ├── stats.go ├── store.go └── version.go /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | queued.db -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 1.1 3 | script: make test 4 | before_install: 5 | - cd 6 | - wget https://github.com/google/leveldb/archive/v1.10.tar.gz 7 | - tar xzf v1.10.tar.gz 8 | - cd leveldb-1.10 9 | - make 10 | - export CGO_CFLAGS="-I`pwd`/include" 11 | - export CGO_LDFLAGS="-L`pwd`" 12 | - export LD_LIBRARY_PATH="`pwd`" 13 | - cd $TRAVIS_BUILD_DIR 14 | - go get github.com/bmizerany/assert 15 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | from ubuntu:12.04 2 | maintainer Scott Nelson "scott@scttnlsn.com" 3 | 4 | run apt-get update 5 | run apt-get install -y python-software-properties git wget build-essential 6 | 7 | # Go 8 | run add-apt-repository -y ppa:duh/golang 9 | run apt-get update 10 | run apt-get install -y golang 11 | run mkdir /go 12 | env GOPATH /go 13 | 14 | # LevelDB 15 | run wget https://leveldb.googlecode.com/files/leveldb-1.13.0.tar.gz --no-check-certificate 16 | run tar -zxvf leveldb-1.13.0.tar.gz 17 | run cd leveldb-1.13.0; make 18 | run cp -r leveldb-1.13.0/include/leveldb /usr/include/ 19 | run cp leveldb-1.13.0/libleveldb.* /usr/lib/ 20 | 21 | # Queued 22 | run mkdir -p /queued/src 23 | run git clone https://github.com/scttnlsn/queued.git /queued/src 24 | run go get github.com/jmhodges/levigo 25 | run go get github.com/gorilla/mux 26 | run cd /queued/src; make 27 | run cp /queued/src/build/queued /usr/bin/queued 28 | 29 | expose 5353 30 | entrypoint ["/usr/bin/queued", "-db-path=/queued/db"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Scott Nelson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PREFIX=/usr/local 2 | BINDIR=${PREFIX}/bin 3 | 4 | all: build/queued 5 | 6 | build: 7 | mkdir build 8 | 9 | build/queued: build $(wildcard queued.go queued/*.go) 10 | go get -d -tags=${BUILD_TAGS} 11 | go build -o build/queued -tags=${BUILD_TAGS} 12 | 13 | clean: 14 | rm -rf build 15 | 16 | install: build/queued 17 | install -m 755 -d ${BINDIR} 18 | install -m 755 build/queued ${BINDIR}/queued 19 | 20 | uninstall: 21 | rm ${BINDIR}/queued 22 | 23 | test: 24 | go get -d -tags=${BUILD_TAGS} 25 | cd queued; go test -tags=${BUILD_TAGS} 26 | 27 | .PHONY: install uninstall clean all test 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # queued 2 | 3 | Simple HTTP-based queue server 4 | 5 | [![Build Status](https://travis-ci.org/scttnlsn/queued.png?branch=master)](https://travis-ci.org/scttnlsn/queued) 6 | 7 | ## Getting Started 8 | 9 | **Install:** 10 | 11 | Ensure [Go](http://golang.org/) and [LevelDB](https://code.google.com/p/leveldb/) are installed and then run: 12 | 13 | $ make 14 | $ sudo make install 15 | 16 | **Run:** 17 | 18 | $ queued [options] 19 | 20 | ## API 21 | 22 | **Enqueue:** 23 | 24 | $ curl -X POST http://localhost:5353/:queue -d 'foo' 25 | 26 | Append the POSTed data to the end of the specified queue (note that queues are created on-the-fly). The `Location` header will point to the enqueued item and is of the form `http://localhost:5353/:queue/:id`. 27 | 28 | **Dequeue:** 29 | 30 | $ curl -X POST http://localhost:5353/:queue/dequeue 31 | 32 | Dequeue the item currently on the head of the queue. Guaranteed not to return the same item twice unless a completion timeout is specified (see below). The `Location` header will point to the dequeued item and is of the form `http://localhost:5353/:queue/:id`. Queued message data is returned in the response body. 33 | 34 | Dequeue optionally takes `wait` and/or `timeout` query string parameters: 35 | 36 | * `wait=` - block for the specified number of seconds or until there is an item to be read 37 | off the head of the queue 38 | 39 | * `timeout=` - if the item is not completed (see endpoint below) within the specified number of seconds, the item will automatically be re-enqueued (when no timeout is specified the item is automatically completed when dequeued) 40 | 41 | **Get:** 42 | 43 | $ curl -X GET http://localhost:5353/:queue/:id 44 | 45 | Get a specific item. The header `X-Dequeued` will be `true` if the item is currently dequeued and waiting for completion. 46 | 47 | **Complete:** 48 | 49 | $ curl -X DELETE http://localhost:5353/:queue/:id 50 | 51 | Complete the specified item and destroy it (note that only items dequeued with a timeout can be completed). 52 | 53 | **Stats:** 54 | 55 | $ curl -X GET http://localhost:5353/:queue 56 | 57 | Get stats about a given queue. 58 | 59 | ## CLI Options 60 | 61 | * **-auth=""** - HTTP basic auth password required for all requests 62 | * **-db-path="./queued.db"** - the directory in which queue items will be persisted (n/a for memory store) 63 | * **-port=5353** - port on which to listen 64 | * **-store=leveldb** - the backend in which items will be stored (`leveldb` or `memory`) 65 | * **-sync=true** - boolean indicating whether data should be synced to disk after every write (n/a for memory store, see LevelDB's `WriteOptions::sync`) 66 | * **-v** - output the version number 67 | 68 | ## Client Libraries 69 | 70 | * [Node.js](http://github.com/scttnlsn/node-queued) 71 | * [Python](http://github.com/miku/pyqueued) 72 | * [Ruby](http://github.com/scttnlsn/queued-ruby) -------------------------------------------------------------------------------- /queued.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/scttnlsn/queued/queued" 7 | "os" 8 | "runtime" 9 | ) 10 | 11 | var config *queued.Config 12 | 13 | func init() { 14 | config = queued.NewConfig() 15 | 16 | flag.UintVar(&config.Port, "port", 5353, "port on which to listen") 17 | flag.StringVar(&config.Auth, "auth", "", "HTTP basic auth password required for all requests") 18 | flag.StringVar(&config.Store, "store", "leveldb", "the backend in which items will be stored (leveldb or memory)") 19 | flag.StringVar(&config.DbPath, "db-path", "./queued.db", "the directory in which queue items will be persisted (n/a for memory store)") 20 | flag.BoolVar(&config.Sync, "sync", true, "boolean indicating whether data should be synced to disk after every write (n/a for memory store)") 21 | } 22 | 23 | func main() { 24 | version := flag.Bool("v", false, "output the version number") 25 | 26 | flag.Parse() 27 | 28 | if *version { 29 | fmt.Println(queued.Version) 30 | os.Exit(0) 31 | } 32 | 33 | runtime.GOMAXPROCS(runtime.NumCPU()) 34 | 35 | s := queued.NewServer(config) 36 | 37 | err := s.ListenAndServe() 38 | if err != nil { 39 | panic(fmt.Sprintf("main: %v", err)) 40 | } 41 | 42 | fmt.Printf("Listening on http://localhost%s\n", s.Addr) 43 | 44 | shutdown := make(chan bool) 45 | <-shutdown 46 | } 47 | -------------------------------------------------------------------------------- /queued/application.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | type Info struct { 9 | record *Record 10 | dequeued bool 11 | } 12 | 13 | type Application struct { 14 | store Store 15 | queues map[string]*Queue 16 | items map[int]*Item 17 | qmutex sync.Mutex 18 | imutex sync.RWMutex 19 | } 20 | 21 | func NewApplication(store Store) *Application { 22 | app := &Application{ 23 | store: store, 24 | queues: make(map[string]*Queue), 25 | items: make(map[int]*Item), 26 | } 27 | 28 | it := store.Iterator() 29 | record, ok := it.NextRecord() 30 | 31 | for ok { 32 | queue := app.GetQueue(record.Queue) 33 | item := queue.Enqueue(record.Id) 34 | app.items[item.value] = item 35 | 36 | record, ok = it.NextRecord() 37 | } 38 | 39 | return app 40 | } 41 | 42 | func (a *Application) GetQueue(name string) *Queue { 43 | a.qmutex.Lock() 44 | defer a.qmutex.Unlock() 45 | 46 | queue, ok := a.queues[name] 47 | 48 | if !ok { 49 | queue = NewQueue() 50 | a.queues[name] = queue 51 | } 52 | 53 | return queue 54 | } 55 | 56 | func (a *Application) GetItem(id int) (*Item, bool) { 57 | a.imutex.RLock() 58 | defer a.imutex.RUnlock() 59 | 60 | item, ok := a.items[id] 61 | return item, ok 62 | } 63 | 64 | func (a *Application) PutItem(item *Item) { 65 | a.imutex.Lock() 66 | defer a.imutex.Unlock() 67 | 68 | a.items[item.value] = item 69 | } 70 | 71 | func (a *Application) RemoveItem(id int) { 72 | a.imutex.Lock() 73 | defer a.imutex.Unlock() 74 | 75 | delete(a.items, id) 76 | } 77 | 78 | func (a *Application) Enqueue(name string, value []byte, mime string) (*Record, error) { 79 | queue := a.GetQueue(name) 80 | record := NewRecord(value, name) 81 | record.Mime = mime 82 | 83 | err := a.store.Put(record) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | item := queue.Enqueue(record.Id) 89 | a.PutItem(item) 90 | 91 | return record, nil 92 | } 93 | 94 | func (a *Application) Dequeue(name string, wait time.Duration, timeout time.Duration) (*Record, error) { 95 | queue := a.GetQueue(name) 96 | item := queue.Dequeue(wait, timeout) 97 | if item == nil { 98 | return nil, nil 99 | } 100 | 101 | record, err := a.store.Get(item.value) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | if !item.dequeued { 107 | a.Complete(name, item.value) 108 | } 109 | 110 | return record, nil 111 | } 112 | 113 | func (a *Application) Complete(name string, id int) (bool, error) { 114 | item, ok := a.GetItem(id) 115 | 116 | if !ok || !item.dequeued { 117 | return false, nil 118 | } 119 | 120 | err := a.store.Remove(id) 121 | if err != nil { 122 | return false, err 123 | } 124 | 125 | item.Complete() 126 | a.RemoveItem(id) 127 | 128 | return true, nil 129 | } 130 | 131 | func (a *Application) Info(name string, id int) (*Info, error) { 132 | record, err := a.store.Get(id) 133 | if err != nil { 134 | return nil, err 135 | } 136 | if record == nil { 137 | return nil, nil 138 | } 139 | 140 | if record.Queue != name { 141 | return nil, nil 142 | } 143 | 144 | item, ok := a.GetItem(id) 145 | info := &Info{record, ok && item.dequeued} 146 | 147 | return info, nil 148 | } 149 | 150 | func (a *Application) Stats(name string) map[string]int { 151 | queue := a.GetQueue(name) 152 | return queue.Stats() 153 | } 154 | -------------------------------------------------------------------------------- /queued/application_test.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | "time" 7 | 8 | "github.com/bmizerany/assert" 9 | ) 10 | 11 | func TestApplication(t *testing.T) { 12 | store := NewLevelStore("./test1.db", true) 13 | defer store.Drop() 14 | 15 | app := NewApplication(store) 16 | 17 | assert.Equal(t, app.GetQueue("test"), app.GetQueue("test")) 18 | assert.NotEqual(t, app.GetQueue("test"), app.GetQueue("foobar")) 19 | 20 | record, err := app.Enqueue("test", []byte("foo"), "") 21 | 22 | assert.Equal(t, err, nil) 23 | assert.Equal(t, record.Id, 1) 24 | assert.Equal(t, record.Value, []byte("foo")) 25 | assert.Equal(t, record.Queue, "test") 26 | 27 | stats := app.Stats("test") 28 | 29 | assert.Equal(t, stats["enqueued"], 1) 30 | assert.Equal(t, stats["dequeued"], 0) 31 | assert.Equal(t, stats["depth"], 1) 32 | assert.Equal(t, stats["timeouts"], 0) 33 | 34 | info, err := app.Info("test", 1) 35 | 36 | assert.Equal(t, err, nil) 37 | assert.Equal(t, info.record.Value, []byte("foo")) 38 | assert.Equal(t, info.dequeued, false) 39 | 40 | record, err = app.Dequeue("test", NilDuration, NilDuration) 41 | 42 | assert.Equal(t, err, nil) 43 | assert.T(t, record != nil) 44 | assert.Equal(t, record.Id, 1) 45 | assert.Equal(t, record.Value, []byte("foo")) 46 | assert.Equal(t, record.Queue, "test") 47 | 48 | ok, err := app.Complete("test", 1) 49 | assert.Equal(t, err, nil) 50 | assert.Equal(t, ok, false) 51 | 52 | app.Enqueue("test", []byte("bar"), "") 53 | record, err = app.Dequeue("test", NilDuration, time.Millisecond) 54 | 55 | assert.Equal(t, err, nil) 56 | assert.T(t, record != nil) 57 | assert.Equal(t, record.Id, 2) 58 | assert.Equal(t, record.Value, []byte("bar")) 59 | assert.Equal(t, record.Queue, "test") 60 | 61 | ok, err = app.Complete("test", 2) 62 | assert.Equal(t, err, nil) 63 | assert.Equal(t, ok, true) 64 | 65 | ok, err = app.Complete("test", 2) 66 | assert.Equal(t, err, nil) 67 | assert.Equal(t, ok, false) 68 | } 69 | 70 | func TestNewApplication(t *testing.T) { 71 | store := NewLevelStore("./test2.db", true) 72 | defer store.Drop() 73 | 74 | store.Put(NewRecord([]byte("foo"), "test")) 75 | store.Put(NewRecord([]byte("bar"), "test")) 76 | store.Put(NewRecord([]byte("baz"), "another")) 77 | 78 | app := NewApplication(store) 79 | 80 | one, _ := app.Dequeue("test", NilDuration, NilDuration) 81 | assert.Equal(t, one.Id, 1) 82 | assert.Equal(t, one.Value, []byte("foo")) 83 | 84 | two, _ := app.Dequeue("test", NilDuration, NilDuration) 85 | assert.Equal(t, two.Id, 2) 86 | assert.Equal(t, two.Value, []byte("bar")) 87 | 88 | three, _ := app.Dequeue("another", NilDuration, NilDuration) 89 | assert.Equal(t, three.Id, 3) 90 | assert.Equal(t, three.Value, []byte("baz")) 91 | } 92 | 93 | func BenchmarkSmallQueue(b *testing.B) { 94 | store := NewLevelStore("./bench1.db", true) 95 | defer store.Drop() 96 | app := NewApplication(store) 97 | b.ResetTimer() 98 | for i := 0; i < b.N; i++ { 99 | app.Enqueue("test", []byte(strconv.Itoa(i)), "") 100 | } 101 | } 102 | 103 | func BenchmarkSmallDequeue(b *testing.B) { 104 | store := NewLevelStore("./bench2.db", true) 105 | defer store.Drop() 106 | app := NewApplication(store) 107 | for i := 0; i < b.N; i++ { 108 | app.Enqueue("test", []byte(strconv.Itoa(i)), "") 109 | } 110 | b.ResetTimer() 111 | for i := 0; i < b.N; i++ { 112 | app.Dequeue("test", NilDuration, NilDuration) 113 | } 114 | } 115 | 116 | var testValue = []byte(`{ "glossary": { "title": "example glossary", "GlossDiv": { "title": "S", "GlossList": { "GlossEntry": { "ID": "SGML", "SortAs": "SGML", "GlossTerm": "Standard Generalized Markup Language", "Acronym": "SGML", "Abbrev": "ISO 8879:1986", "GlossDef": { "para": "A meta-markup language, used to create markup languages such as DocBook.", "GlossSeeAlso": ["GML", "XML"] }, "GlossSee": "markup" } } } } }`) 117 | 118 | func BenchmarkQueue(b *testing.B) { 119 | store := NewLevelStore("./bench1.db", true) 120 | defer store.Drop() 121 | app := NewApplication(store) 122 | b.ResetTimer() 123 | for i := 0; i < b.N; i++ { 124 | app.Enqueue("test", testValue, "") 125 | } 126 | 127 | } 128 | 129 | func BenchmarkDequeue(b *testing.B) { 130 | store := NewLevelStore("./bench2.db", true) 131 | defer store.Drop() 132 | app := NewApplication(store) 133 | for i := 0; i < b.N; i++ { 134 | app.Enqueue("test", testValue, "") 135 | } 136 | b.ResetTimer() 137 | for i := 0; i < b.N; i++ { 138 | app.Dequeue("test", NilDuration, NilDuration) 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /queued/config.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type Config struct { 8 | Port uint 9 | Auth string 10 | Store string 11 | DbPath string 12 | Sync bool 13 | } 14 | 15 | func NewConfig() *Config { 16 | return &Config{} 17 | } 18 | 19 | func (c *Config) CreateStore() Store { 20 | if c.Store == "leveldb" { 21 | return NewLevelStore(c.DbPath, c.Sync) 22 | } else if c.Store == "memory" { 23 | return NewMemoryStore() 24 | } else { 25 | panic(fmt.Sprintf("queued.Config: Invalid store: %s", c.Store)) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /queued/goleveldb_store.go: -------------------------------------------------------------------------------- 1 | // +build use_goleveldb 2 | 3 | package queued 4 | 5 | import ( 6 | "bytes" 7 | "encoding/gob" 8 | "fmt" 9 | "os" 10 | "strconv" 11 | "sync" 12 | 13 | "github.com/syndtr/goleveldb/leveldb" 14 | "github.com/syndtr/goleveldb/leveldb/errors" 15 | "github.com/syndtr/goleveldb/leveldb/filter" 16 | leveldb_iterator "github.com/syndtr/goleveldb/leveldb/iterator" 17 | "github.com/syndtr/goleveldb/leveldb/opt" 18 | ) 19 | 20 | // Iterator 21 | 22 | type LevelIterator struct { 23 | leveldb_iterator.Iterator 24 | } 25 | 26 | func (it *LevelIterator) NextRecord() (*Record, bool) { 27 | if !it.Valid() { 28 | return nil, false 29 | } 30 | 31 | id, err := strconv.Atoi(string(it.Key())) 32 | if err != nil { 33 | panic(fmt.Sprintf("queued.LevelIterator: Error loading db: %v", err)) 34 | } 35 | 36 | value := it.Value() 37 | if value == nil { 38 | return nil, false 39 | } 40 | 41 | var record Record 42 | buf := bytes.NewBuffer(value) 43 | dec := gob.NewDecoder(buf) 44 | err = dec.Decode(&record) 45 | if err != nil { 46 | panic(fmt.Sprintf("queued.LevelIterator: Error decoding value: %v", err)) 47 | } 48 | 49 | record.Id = id 50 | 51 | it.Next() 52 | return &record, true 53 | } 54 | 55 | // Store 56 | 57 | type LevelStore struct { 58 | path string 59 | sync bool 60 | db *leveldb.DB 61 | id int 62 | mutex sync.Mutex 63 | } 64 | 65 | func NewLevelStore(path string, sync bool) *LevelStore { 66 | opts := &opt.Options{ 67 | Filter: filter.NewBloomFilter(10), 68 | ErrorIfMissing: false, 69 | } 70 | db, err := leveldb.OpenFile(path, opts) 71 | if err != nil { 72 | panic(fmt.Sprintf("queued.LevelStore: Unable to open db: %v", err)) 73 | } 74 | 75 | id := 0 76 | 77 | iter := db.NewIterator(nil, nil) 78 | iter.Last() 79 | if iter.Valid() { 80 | id, err = strconv.Atoi(string(iter.Key())) 81 | if err != nil { 82 | panic(fmt.Sprintf("queued.LevelStore: Error loading db: %v", err)) 83 | } 84 | } 85 | 86 | return &LevelStore{ 87 | id: id, 88 | path: path, 89 | sync: sync, 90 | db: db, 91 | } 92 | } 93 | 94 | func (s *LevelStore) Get(id int) (*Record, error) { 95 | value, err := s.db.Get(key(id), nil) 96 | if err == errors.ErrNotFound { 97 | return nil, nil 98 | } 99 | if err != nil { 100 | return nil, err 101 | } 102 | if value == nil { 103 | return nil, nil 104 | } 105 | 106 | var record Record 107 | buf := bytes.NewBuffer(value) 108 | dec := gob.NewDecoder(buf) 109 | err = dec.Decode(&record) 110 | if err != nil { 111 | panic(fmt.Sprintf("queued.LevelStore: Error decoding value: %v", err)) 112 | } 113 | 114 | record.Id = id 115 | 116 | return &record, nil 117 | } 118 | 119 | func (s *LevelStore) Put(record *Record) error { 120 | s.mutex.Lock() 121 | defer s.mutex.Unlock() 122 | 123 | id := s.id + 1 124 | 125 | var buf bytes.Buffer 126 | enc := gob.NewEncoder(&buf) 127 | err := enc.Encode(record) 128 | if err != nil { 129 | panic(fmt.Sprintf("queued.LevelStore: Error encoding record: %v", err)) 130 | } 131 | 132 | err = s.db.Put(key(id), buf.Bytes(), &opt.WriteOptions{Sync: s.sync}) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | record.Id = id 138 | s.id = id 139 | 140 | return nil 141 | } 142 | 143 | func (s *LevelStore) Remove(id int) error { 144 | s.mutex.Lock() 145 | defer s.mutex.Unlock() 146 | 147 | return s.db.Delete(key(id), &opt.WriteOptions{Sync: s.sync}) 148 | } 149 | 150 | func (s *LevelStore) Close() { 151 | s.db.Close() 152 | } 153 | 154 | func (s *LevelStore) Drop() { 155 | s.Close() 156 | 157 | err := os.RemoveAll(s.path) 158 | if err != nil { 159 | panic(fmt.Sprintf("queued.LevelStore: Error dropping db: %v", err)) 160 | } 161 | } 162 | 163 | func (s *LevelStore) Iterator() Iterator { 164 | it := s.db.NewIterator(nil, nil) 165 | it.First() 166 | return &LevelIterator{it} 167 | } 168 | 169 | // Helpers 170 | 171 | func key(id int) []byte { 172 | return []byte(fmt.Sprintf("%d", id)) 173 | } 174 | -------------------------------------------------------------------------------- /queued/goleveldb_store_test.go: -------------------------------------------------------------------------------- 1 | // +build use_goleveldb 2 | 3 | package queued 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/bmizerany/assert" 9 | ) 10 | 11 | func TestLevelStore(t *testing.T) { 12 | store := NewLevelStore("./test1.db", true) 13 | defer store.Drop() 14 | 15 | assert.Equal(t, store.id, 0) 16 | 17 | record := NewRecord([]byte("foo"), "testqueue") 18 | 19 | err := store.Put(record) 20 | assert.Equal(t, err, nil) 21 | assert.Equal(t, record.Id, 1) 22 | 23 | record, err = store.Get(1) 24 | assert.Equal(t, err, nil) 25 | assert.Equal(t, record.Id, 1) 26 | assert.Equal(t, record.Value, []byte("foo")) 27 | assert.Equal(t, record.Queue, "testqueue") 28 | 29 | err = store.Remove(1) 30 | assert.Equal(t, err, nil) 31 | 32 | record, err = store.Get(1) 33 | assert.Equal(t, err, nil) 34 | assert.T(t, record == nil) 35 | } 36 | 37 | func TestLevelStoreLoad(t *testing.T) { 38 | temp := NewLevelStore("./test2.db", true) 39 | temp.Put(NewRecord([]byte("foo"), "testqueue")) 40 | temp.Put(NewRecord([]byte("bar"), "testqueue")) 41 | temp.Close() 42 | 43 | store := NewLevelStore("./test2.db", true) 44 | defer store.Drop() 45 | 46 | assert.Equal(t, store.id, 2) 47 | } 48 | 49 | func TestLevelStoreIterator(t *testing.T) { 50 | temp := NewLevelStore("./test3.db", true) 51 | temp.Put(NewRecord([]byte("foo"), "testqueue")) 52 | temp.Put(NewRecord([]byte("bar"), "testqueue")) 53 | temp.Close() 54 | 55 | store := NewLevelStore("./test3.db", true) 56 | defer store.Drop() 57 | 58 | it := store.Iterator() 59 | 60 | one, ok := it.NextRecord() 61 | assert.Equal(t, ok, true) 62 | assert.Equal(t, one.Id, 1) 63 | assert.Equal(t, one.Value, []byte("foo")) 64 | assert.Equal(t, one.Queue, "testqueue") 65 | 66 | two, ok := it.NextRecord() 67 | assert.Equal(t, ok, true) 68 | assert.Equal(t, two.Id, 2) 69 | assert.Equal(t, two.Value, []byte("bar")) 70 | assert.Equal(t, two.Queue, "testqueue") 71 | 72 | _, ok = it.NextRecord() 73 | assert.Equal(t, ok, false) 74 | } 75 | -------------------------------------------------------------------------------- /queued/handlers.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/gorilla/mux" 8 | "io/ioutil" 9 | "net/http" 10 | "strconv" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | func (s *Server) EnqueueHandler(w http.ResponseWriter, req *http.Request) { 16 | params := mux.Vars(req) 17 | 18 | value, err := ioutil.ReadAll(req.Body) 19 | if err != nil { 20 | send(w, http.StatusInternalServerError, Json{"error": err.Error()}) 21 | return 22 | } 23 | 24 | mime := req.Header.Get("Content-Type") 25 | record, err := s.App.Enqueue(params["queue"], value, mime) 26 | if err != nil { 27 | send(w, http.StatusInternalServerError, Json{"error": err.Error()}) 28 | return 29 | } 30 | 31 | w.Header().Set("Location", url(req, record)) 32 | w.WriteHeader(http.StatusCreated) 33 | } 34 | 35 | func (s *Server) DequeueHandler(w http.ResponseWriter, req *http.Request) { 36 | params := mux.Vars(req) 37 | 38 | wait, err := Stod(req.URL.Query().Get("wait"), time.Second) 39 | if err != nil { 40 | send(w, http.StatusBadRequest, Json{"error": "Invalid wait parameter"}) 41 | return 42 | } 43 | 44 | timeout, err := Stod(req.URL.Query().Get("timeout"), time.Second) 45 | if err != nil { 46 | send(w, http.StatusBadRequest, Json{"error": "Invalid timeout parameter"}) 47 | return 48 | } 49 | 50 | record, err := s.App.Dequeue(params["queue"], wait, timeout) 51 | if err != nil { 52 | send(w, http.StatusInternalServerError, Json{"error": "Dequeue failed"}) 53 | return 54 | } 55 | 56 | if record != nil { 57 | w.Header().Set("Location", url(req, record)) 58 | w.Header().Set("Content-Type", record.ContentType()) 59 | w.Write(record.Value) 60 | } else { 61 | w.WriteHeader(http.StatusNotFound) 62 | } 63 | } 64 | 65 | func (s *Server) InfoHandler(w http.ResponseWriter, req *http.Request) { 66 | params := mux.Vars(req) 67 | 68 | id, err := strconv.Atoi(params["id"]) 69 | if err != nil { 70 | send(w, http.StatusNotFound, Json{"error": "Item not found"}) 71 | return 72 | } 73 | 74 | info, err := s.App.Info(params["queue"], id) 75 | if err != nil { 76 | send(w, http.StatusInternalServerError, Json{"error": "Failed to read item"}) 77 | return 78 | } 79 | 80 | if info != nil { 81 | dequeued := "false" 82 | if info.dequeued { 83 | dequeued = "true" 84 | } 85 | 86 | w.Header().Set("X-Dequeued", dequeued) 87 | w.Header().Set("Content-Type", info.record.ContentType()) 88 | w.Write(info.record.Value) 89 | } else { 90 | send(w, http.StatusNotFound, Json{"error": "Item not found"}) 91 | } 92 | } 93 | 94 | func (s *Server) StatsHandler(w http.ResponseWriter, req *http.Request) { 95 | params := mux.Vars(req) 96 | stats := s.App.Stats(params["queue"]) 97 | 98 | result := map[string]interface{}{} 99 | for field, value := range stats { 100 | result[field] = value 101 | } 102 | 103 | send(w, http.StatusOK, result) 104 | } 105 | 106 | func (s *Server) CompleteHandler(w http.ResponseWriter, req *http.Request) { 107 | params := mux.Vars(req) 108 | 109 | id, err := strconv.Atoi(params["id"]) 110 | if err != nil { 111 | send(w, http.StatusNotFound, Json{"error": "Item not found"}) 112 | return 113 | } 114 | 115 | ok, err := s.App.Complete(params["queue"], id) 116 | if err != nil { 117 | send(w, http.StatusInternalServerError, Json{"error": "Complete failed"}) 118 | return 119 | } 120 | 121 | if ok { 122 | w.WriteHeader(http.StatusNoContent) 123 | } else { 124 | send(w, http.StatusBadRequest, Json{"error": "Item not dequeued with timeout"}) 125 | } 126 | } 127 | 128 | // Helpers 129 | 130 | type Json map[string]interface{} 131 | 132 | func Stod(val string, scale ...time.Duration) (time.Duration, error) { 133 | duration := NilDuration 134 | 135 | if val != "" { 136 | n, err := strconv.Atoi(val) 137 | 138 | if err != nil { 139 | return duration, err 140 | } else { 141 | duration = time.Duration(n) 142 | 143 | if len(scale) == 1 { 144 | duration *= scale[0] 145 | } 146 | } 147 | } 148 | 149 | return duration, nil 150 | } 151 | 152 | func send(w http.ResponseWriter, code int, data Json) error { 153 | bytes, err := json.Marshal(data) 154 | 155 | if err != nil { 156 | return err 157 | } 158 | 159 | w.Header().Set("Content-Type", "application/json") 160 | w.WriteHeader(code) 161 | w.Write(bytes) 162 | 163 | return nil 164 | } 165 | 166 | func auth(config *Config, next http.Handler) http.Handler { 167 | unauthorized := func(w http.ResponseWriter) { 168 | send(w, http.StatusUnauthorized, Json{"error": "Unauthorized"}) 169 | } 170 | 171 | return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 172 | if config.Auth != "" { 173 | s := strings.SplitN(req.Header.Get("Authorization"), " ", 2) 174 | if len(s) != 2 || s[0] != "Basic" { 175 | unauthorized(w) 176 | return 177 | } 178 | 179 | base, err := base64.StdEncoding.DecodeString(s[1]) 180 | if err != nil { 181 | unauthorized(w) 182 | return 183 | } 184 | 185 | pair := strings.SplitN(string(base), ":", 2) 186 | if len(pair) != 2 { 187 | unauthorized(w) 188 | return 189 | } 190 | 191 | password := pair[1] 192 | if config.Auth != password { 193 | unauthorized(w) 194 | return 195 | } 196 | } 197 | 198 | next.ServeHTTP(w, req) 199 | }) 200 | } 201 | 202 | func url(req *http.Request, record *Record) string { 203 | return fmt.Sprintf("http://%s/%s/%d", req.Host, record.Queue, record.Id) 204 | } 205 | -------------------------------------------------------------------------------- /queued/item.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | type Item struct { 4 | value int 5 | dequeued bool 6 | complete chan bool 7 | } 8 | 9 | func NewItem(value int) *Item { 10 | return &Item{ 11 | value: value, 12 | dequeued: false, 13 | complete: make(chan bool), 14 | } 15 | } 16 | 17 | func (item *Item) Complete() { 18 | if !item.dequeued { 19 | return 20 | } 21 | 22 | item.complete <- true 23 | } 24 | -------------------------------------------------------------------------------- /queued/level_store.go: -------------------------------------------------------------------------------- 1 | // +build !use_goleveldb 2 | 3 | package queued 4 | 5 | import ( 6 | "bytes" 7 | "encoding/gob" 8 | "fmt" 9 | "github.com/jmhodges/levigo" 10 | "os" 11 | "strconv" 12 | "sync" 13 | ) 14 | 15 | // Iterator 16 | 17 | type LevelIterator struct { 18 | *levigo.Iterator 19 | } 20 | 21 | func (it *LevelIterator) NextRecord() (*Record, bool) { 22 | if !it.Valid() { 23 | return nil, false 24 | } 25 | 26 | id, err := strconv.Atoi(string(it.Key())) 27 | if err != nil { 28 | panic(fmt.Sprintf("queued.LevelIterator: Error loading db: %v", err)) 29 | } 30 | 31 | value := it.Value() 32 | if value == nil { 33 | return nil, false 34 | } 35 | 36 | var record Record 37 | buf := bytes.NewBuffer(value) 38 | dec := gob.NewDecoder(buf) 39 | err = dec.Decode(&record) 40 | if err != nil { 41 | panic(fmt.Sprintf("queued.LevelIterator: Error decoding value: %v", err)) 42 | } 43 | 44 | record.Id = id 45 | 46 | it.Next() 47 | return &record, true 48 | } 49 | 50 | // Store 51 | 52 | type LevelStore struct { 53 | path string 54 | sync bool 55 | db *levigo.DB 56 | id int 57 | mutex sync.Mutex 58 | } 59 | 60 | func NewLevelStore(path string, sync bool) *LevelStore { 61 | opts := levigo.NewOptions() 62 | opts.SetCreateIfMissing(true) 63 | defer opts.Close() 64 | 65 | db, err := levigo.Open(path, opts) 66 | if err != nil { 67 | panic(fmt.Sprintf("queued.LevelStore: Unable to open db: %v", err)) 68 | } 69 | 70 | id := 0 71 | 72 | it := db.NewIterator(levigo.NewReadOptions()) 73 | defer it.Close() 74 | 75 | it.SeekToLast() 76 | if it.Valid() { 77 | id, err = strconv.Atoi(string(it.Key())) 78 | if err != nil { 79 | panic(fmt.Sprintf("queued.LevelStore: Error loading db: %v", err)) 80 | } 81 | } 82 | 83 | return &LevelStore{ 84 | id: id, 85 | path: path, 86 | sync: sync, 87 | db: db, 88 | } 89 | } 90 | 91 | func (s *LevelStore) Get(id int) (*Record, error) { 92 | ropts := levigo.NewReadOptions() 93 | defer ropts.Close() 94 | 95 | value, err := s.db.Get(ropts, key(id)) 96 | if err != nil { 97 | return nil, err 98 | } 99 | if value == nil { 100 | return nil, nil 101 | } 102 | 103 | var record Record 104 | buf := bytes.NewBuffer(value) 105 | dec := gob.NewDecoder(buf) 106 | err = dec.Decode(&record) 107 | if err != nil { 108 | panic(fmt.Sprintf("queued.LevelStore: Error decoding value: %v", err)) 109 | } 110 | 111 | record.Id = id 112 | 113 | return &record, nil 114 | } 115 | 116 | func (s *LevelStore) Put(record *Record) error { 117 | s.mutex.Lock() 118 | defer s.mutex.Unlock() 119 | 120 | id := s.id + 1 121 | 122 | var buf bytes.Buffer 123 | enc := gob.NewEncoder(&buf) 124 | err := enc.Encode(record) 125 | if err != nil { 126 | panic(fmt.Sprintf("queued.LevelStore: Error encoding record: %v", err)) 127 | } 128 | 129 | wopts := levigo.NewWriteOptions() 130 | wopts.SetSync(s.sync) 131 | defer wopts.Close() 132 | 133 | err = s.db.Put(wopts, key(id), buf.Bytes()) 134 | if err != nil { 135 | return err 136 | } 137 | 138 | record.Id = id 139 | s.id = id 140 | 141 | return nil 142 | } 143 | 144 | func (s *LevelStore) Remove(id int) error { 145 | s.mutex.Lock() 146 | defer s.mutex.Unlock() 147 | 148 | wopts := levigo.NewWriteOptions() 149 | wopts.SetSync(s.sync) 150 | defer wopts.Close() 151 | 152 | return s.db.Delete(wopts, key(id)) 153 | } 154 | 155 | func (s *LevelStore) Close() { 156 | s.db.Close() 157 | } 158 | 159 | func (s *LevelStore) Drop() { 160 | s.Close() 161 | 162 | err := os.RemoveAll(s.path) 163 | if err != nil { 164 | panic(fmt.Sprintf("queued.LevelStore: Error dropping db: %v", err)) 165 | } 166 | } 167 | 168 | func (s *LevelStore) Iterator() Iterator { 169 | it := s.db.NewIterator(levigo.NewReadOptions()) 170 | it.SeekToFirst() 171 | return &LevelIterator{it} 172 | } 173 | 174 | // Helpers 175 | 176 | func key(id int) []byte { 177 | return []byte(fmt.Sprintf("%d", id)) 178 | } 179 | -------------------------------------------------------------------------------- /queued/level_store_test.go: -------------------------------------------------------------------------------- 1 | // +build !use_goleveldb 2 | 3 | package queued 4 | 5 | import ( 6 | "github.com/bmizerany/assert" 7 | "testing" 8 | ) 9 | 10 | func TestLevelStore(t *testing.T) { 11 | store := NewLevelStore("./test1.db", true) 12 | defer store.Drop() 13 | 14 | assert.Equal(t, store.id, 0) 15 | 16 | record := NewRecord([]byte("foo"), "testqueue") 17 | 18 | err := store.Put(record) 19 | assert.Equal(t, err, nil) 20 | assert.Equal(t, record.Id, 1) 21 | 22 | record, err = store.Get(1) 23 | assert.Equal(t, err, nil) 24 | assert.Equal(t, record.Id, 1) 25 | assert.Equal(t, record.Value, []byte("foo")) 26 | assert.Equal(t, record.Queue, "testqueue") 27 | 28 | err = store.Remove(1) 29 | assert.Equal(t, err, nil) 30 | 31 | record, err = store.Get(1) 32 | assert.Equal(t, err, nil) 33 | assert.T(t, record == nil) 34 | } 35 | 36 | func TestLevelStoreLoad(t *testing.T) { 37 | temp := NewLevelStore("./test2.db", true) 38 | temp.Put(NewRecord([]byte("foo"), "testqueue")) 39 | temp.Put(NewRecord([]byte("bar"), "testqueue")) 40 | temp.Close() 41 | 42 | store := NewLevelStore("./test2.db", true) 43 | defer store.Drop() 44 | 45 | assert.Equal(t, store.id, 2) 46 | } 47 | 48 | func TestLevelStoreIterator(t *testing.T) { 49 | temp := NewLevelStore("./test3.db", true) 50 | temp.Put(NewRecord([]byte("foo"), "testqueue")) 51 | temp.Put(NewRecord([]byte("bar"), "testqueue")) 52 | temp.Close() 53 | 54 | store := NewLevelStore("./test3.db", true) 55 | defer store.Drop() 56 | 57 | it := store.Iterator() 58 | 59 | one, ok := it.NextRecord() 60 | assert.Equal(t, ok, true) 61 | assert.Equal(t, one.Id, 1) 62 | assert.Equal(t, one.Value, []byte("foo")) 63 | assert.Equal(t, one.Queue, "testqueue") 64 | 65 | two, ok := it.NextRecord() 66 | assert.Equal(t, ok, true) 67 | assert.Equal(t, two.Id, 2) 68 | assert.Equal(t, two.Value, []byte("bar")) 69 | assert.Equal(t, two.Queue, "testqueue") 70 | 71 | _, ok = it.NextRecord() 72 | assert.Equal(t, ok, false) 73 | } 74 | -------------------------------------------------------------------------------- /queued/memory_store.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | // Iterator 8 | 9 | type MemoryIterator struct { 10 | } 11 | 12 | func (it *MemoryIterator) NextRecord() (*Record, bool) { 13 | return nil, false 14 | } 15 | 16 | // Store 17 | 18 | type MemoryStore struct { 19 | id int 20 | records map[int]*Record 21 | mutex sync.Mutex 22 | } 23 | 24 | func NewMemoryStore() *MemoryStore { 25 | records := make(map[int]*Record) 26 | 27 | return &MemoryStore{ 28 | id: 0, 29 | records: records, 30 | } 31 | } 32 | 33 | func (s *MemoryStore) Get(id int) (*Record, error) { 34 | s.mutex.Lock() 35 | defer s.mutex.Unlock() 36 | 37 | if record, ok := s.records[id]; ok { 38 | return record, nil 39 | } else { 40 | return nil, nil 41 | } 42 | } 43 | 44 | func (s *MemoryStore) Put(record *Record) error { 45 | s.mutex.Lock() 46 | defer s.mutex.Unlock() 47 | 48 | record.Id = s.id + 1 49 | s.records[record.Id] = record 50 | s.id = record.Id 51 | return nil 52 | } 53 | 54 | func (s *MemoryStore) Remove(id int) error { 55 | s.mutex.Lock() 56 | defer s.mutex.Unlock() 57 | 58 | delete(s.records, id) 59 | return nil 60 | } 61 | 62 | func (s *MemoryStore) Iterator() Iterator { 63 | return &MemoryIterator{} 64 | } 65 | 66 | func (s *MemoryStore) Drop() { 67 | s.records = make(map[int]*Record) 68 | } 69 | -------------------------------------------------------------------------------- /queued/memory_store_test.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "github.com/bmizerany/assert" 5 | "testing" 6 | ) 7 | 8 | func TestMemoryStore(t *testing.T) { 9 | store := NewMemoryStore() 10 | defer store.Drop() 11 | 12 | assert.Equal(t, store.id, 0) 13 | 14 | record := NewRecord([]byte("foo"), "testqueue") 15 | 16 | err := store.Put(record) 17 | assert.Equal(t, err, nil) 18 | assert.Equal(t, record.Id, 1) 19 | 20 | record, err = store.Get(1) 21 | assert.Equal(t, err, nil) 22 | assert.Equal(t, record.Id, 1) 23 | assert.Equal(t, record.Value, []byte("foo")) 24 | assert.Equal(t, record.Queue, "testqueue") 25 | 26 | err = store.Remove(1) 27 | assert.Equal(t, err, nil) 28 | 29 | record, err = store.Get(1) 30 | assert.Equal(t, err, nil) 31 | assert.T(t, record == nil) 32 | } 33 | -------------------------------------------------------------------------------- /queued/queue.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | const NilDuration = time.Duration(-1) 9 | 10 | type Queue struct { 11 | items []*Item 12 | waiting chan *Item 13 | stats *Stats 14 | mutex sync.Mutex 15 | } 16 | 17 | func NewQueue() *Queue { 18 | counters := map[string]int{ 19 | "enqueued": 0, 20 | "dequeued": 0, 21 | "depth": 0, 22 | "timeouts": 0, 23 | } 24 | 25 | return &Queue{ 26 | items: []*Item{}, 27 | waiting: make(chan *Item), 28 | stats: NewStats(counters), 29 | } 30 | } 31 | 32 | func (q *Queue) Enqueue(value int) *Item { 33 | item := NewItem(value) 34 | q.EnqueueItem(item) 35 | return item 36 | } 37 | 38 | func (q *Queue) EnqueueItem(item *Item) { 39 | q.stats.Inc("enqueued") 40 | 41 | select { 42 | case q.waiting <- item: 43 | default: 44 | q.append(item) 45 | } 46 | } 47 | 48 | func (q *Queue) Dequeue(wait time.Duration, timeout time.Duration) *Item { 49 | if item := q.shift(); item != nil { 50 | q.stats.Inc("dequeued") 51 | q.timeout(item, timeout) 52 | return item 53 | } else if wait != NilDuration { 54 | select { 55 | case <-time.After(wait): 56 | return nil 57 | case item := <-q.waiting: 58 | q.stats.Inc("dequeued") 59 | q.timeout(item, timeout) 60 | return item 61 | } 62 | } else { 63 | return nil 64 | } 65 | } 66 | 67 | func (q *Queue) Stats() map[string]int { 68 | return q.stats.Get() 69 | } 70 | 71 | func (q *Queue) shift() *Item { 72 | q.mutex.Lock() 73 | defer q.mutex.Unlock() 74 | 75 | if len(q.items) > 0 { 76 | item := q.items[0] 77 | q.items = q.items[1:] 78 | q.stats.Dec("depth") 79 | return item 80 | } else { 81 | return nil 82 | } 83 | } 84 | 85 | func (q *Queue) append(item *Item) { 86 | q.mutex.Lock() 87 | defer q.mutex.Unlock() 88 | 89 | q.items = append(q.items, item) 90 | q.stats.Inc("depth") 91 | } 92 | 93 | func (q *Queue) timeout(item *Item, timeout time.Duration) { 94 | if timeout != NilDuration { 95 | item.dequeued = true 96 | 97 | go func() { 98 | select { 99 | case <-time.After(timeout): 100 | item.dequeued = false 101 | q.EnqueueItem(item) 102 | q.stats.Inc("timeouts") 103 | case <-item.complete: 104 | item.dequeued = false 105 | return 106 | } 107 | }() 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /queued/queue_test.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "github.com/bmizerany/assert" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestQueue(t *testing.T) { 10 | q := NewQueue() 11 | 12 | q.Enqueue(123) 13 | q.Enqueue(456) 14 | 15 | one := q.Dequeue(NilDuration, NilDuration) 16 | assert.Equal(t, one.value, 123) 17 | 18 | two := q.Dequeue(NilDuration, NilDuration) 19 | assert.Equal(t, two.value, 456) 20 | } 21 | 22 | func TestDequeueWait(t *testing.T) { 23 | q := NewQueue() 24 | 25 | wait := time.Millisecond 26 | 27 | go func() { 28 | time.Sleep(wait) 29 | q.Enqueue(123) 30 | }() 31 | 32 | one := q.Dequeue(NilDuration, NilDuration) 33 | assert.T(t, one == nil) 34 | 35 | two := q.Dequeue(time.Second, NilDuration) 36 | assert.T(t, two != nil) 37 | assert.Equal(t, two.value, 123) 38 | } 39 | 40 | func TestDequeueTimeout(t *testing.T) { 41 | q := NewQueue() 42 | 43 | timeout := time.Millisecond 44 | 45 | q.Enqueue(123) 46 | 47 | one := q.Dequeue(NilDuration, timeout) 48 | assert.T(t, one != nil) 49 | 50 | time.Sleep(timeout * 2) 51 | 52 | two := q.Dequeue(NilDuration, timeout) 53 | assert.T(t, two != nil) 54 | 55 | two.Complete() 56 | time.Sleep(timeout) 57 | 58 | three := q.Dequeue(NilDuration, NilDuration) 59 | assert.T(t, three == nil) 60 | } 61 | 62 | func TestStats(t *testing.T) { 63 | q := NewQueue() 64 | 65 | q.Enqueue(123) 66 | q.Enqueue(456) 67 | 68 | assert.Equal(t, q.Stats()["enqueued"], 2) 69 | assert.Equal(t, q.Stats()["dequeued"], 0) 70 | assert.Equal(t, q.Stats()["depth"], 2) 71 | 72 | one := q.Dequeue(NilDuration, NilDuration) 73 | assert.Equal(t, one.value, 123) 74 | 75 | assert.Equal(t, q.Stats()["enqueued"], 2) 76 | assert.Equal(t, q.Stats()["dequeued"], 1) 77 | assert.Equal(t, q.Stats()["depth"], 1) 78 | 79 | two := q.Dequeue(NilDuration, NilDuration) 80 | assert.Equal(t, two.value, 456) 81 | 82 | assert.Equal(t, q.Stats()["enqueued"], 2) 83 | assert.Equal(t, q.Stats()["dequeued"], 2) 84 | assert.Equal(t, q.Stats()["depth"], 0) 85 | 86 | q.Enqueue(789) 87 | q.Dequeue(NilDuration, time.Millisecond) 88 | time.Sleep(time.Millisecond * 2) 89 | 90 | assert.Equal(t, q.Stats()["enqueued"], 4) 91 | assert.Equal(t, q.Stats()["dequeued"], 3) 92 | assert.Equal(t, q.Stats()["depth"], 1) 93 | assert.Equal(t, q.Stats()["timeouts"], 1) 94 | 95 | three := q.Dequeue(NilDuration, NilDuration) 96 | assert.Equal(t, three.value, 789) 97 | 98 | assert.Equal(t, q.Stats()["enqueued"], 4) 99 | assert.Equal(t, q.Stats()["dequeued"], 4) 100 | assert.Equal(t, q.Stats()["depth"], 0) 101 | assert.Equal(t, q.Stats()["timeouts"], 1) 102 | 103 | four := q.Dequeue(NilDuration, NilDuration) 104 | assert.Equal(t, four, (*Item)(nil)) 105 | 106 | assert.Equal(t, q.Stats()["enqueued"], 4) 107 | assert.Equal(t, q.Stats()["dequeued"], 4) 108 | assert.Equal(t, q.Stats()["depth"], 0) 109 | assert.Equal(t, q.Stats()["timeouts"], 1) 110 | } 111 | -------------------------------------------------------------------------------- /queued/record.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | type Record struct { 4 | Id int 5 | Value []byte 6 | Mime string 7 | Queue string 8 | } 9 | 10 | func NewRecord(value []byte, queue string) *Record { 11 | return &Record{ 12 | Id: 0, 13 | Value: value, 14 | Mime: "", 15 | Queue: queue, 16 | } 17 | } 18 | 19 | func (r *Record) ContentType() string { 20 | if r.Mime == "" { 21 | return "application/octet-stream" 22 | } else { 23 | return r.Mime 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /queued/server.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "fmt" 5 | "github.com/gorilla/mux" 6 | "net" 7 | "net/http" 8 | ) 9 | 10 | type Server struct { 11 | Config *Config 12 | Router *mux.Router 13 | Store Store 14 | App *Application 15 | Addr string 16 | } 17 | 18 | func NewServer(config *Config) *Server { 19 | router := mux.NewRouter() 20 | store := config.CreateStore() 21 | app := NewApplication(store) 22 | addr := fmt.Sprintf(":%d", config.Port) 23 | 24 | s := &Server{config, router, store, app, addr} 25 | 26 | s.HandleFunc("/{queue}", s.EnqueueHandler).Methods("POST") 27 | s.HandleFunc("/{queue}", s.StatsHandler).Methods("GET") 28 | s.HandleFunc("/{queue}/dequeue", s.DequeueHandler).Methods("POST") 29 | s.HandleFunc("/{queue}/{id}", s.InfoHandler).Methods("GET") 30 | s.HandleFunc("/{queue}/{id}", s.CompleteHandler).Methods("DELETE") 31 | 32 | return s 33 | } 34 | 35 | func (s *Server) HandleFunc(route string, fn http.HandlerFunc) *mux.Route { 36 | return s.Router.Handle(route, auth(s.Config, fn)) 37 | } 38 | 39 | func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { 40 | s.Router.ServeHTTP(w, req) 41 | } 42 | 43 | func (s *Server) ListenAndServe() error { 44 | listener, err := net.Listen("tcp", s.Addr) 45 | 46 | if err != nil { 47 | return err 48 | } 49 | 50 | srv := http.Server{Handler: s} 51 | go srv.Serve(listener) 52 | 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /queued/server_test.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "github.com/bmizerany/assert" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func TestServer(t *testing.T) { 12 | s := NewServer(&Config{DbPath: "./server1", Store: "memory"}) 13 | defer s.Store.Drop() 14 | 15 | // Enqueue 16 | body := strings.NewReader("bar") 17 | req, _ := http.NewRequest("POST", "/foo", body) 18 | req.Header.Add("Content-Type", "text/plain") 19 | w := httptest.NewRecorder() 20 | s.ServeHTTP(w, req) 21 | 22 | assert.Equal(t, w.Code, 201) 23 | 24 | // Invalid complete (must dequeue first) 25 | req, _ = http.NewRequest("DELETE", "/foo/1", nil) 26 | w = httptest.NewRecorder() 27 | s.ServeHTTP(w, req) 28 | 29 | assert.Equal(t, w.Code, 400) 30 | 31 | // Info 32 | req, _ = http.NewRequest("GET", "/foo/1", nil) 33 | w = httptest.NewRecorder() 34 | s.ServeHTTP(w, req) 35 | 36 | assert.Equal(t, w.Code, 200) 37 | assert.Equal(t, w.Header().Get("Content-Type"), "text/plain") 38 | 39 | // Dequeue 40 | req, _ = http.NewRequest("POST", "/foo/dequeue?wait=30&timeout=30", nil) 41 | w = httptest.NewRecorder() 42 | s.ServeHTTP(w, req) 43 | 44 | assert.Equal(t, w.Code, 200) 45 | assert.Equal(t, w.Header().Get("Content-Type"), "text/plain") 46 | 47 | // Stats 48 | req, _ = http.NewRequest("GET", "/foo", nil) 49 | w = httptest.NewRecorder() 50 | s.ServeHTTP(w, req) 51 | 52 | assert.Equal(t, w.Code, 200) 53 | 54 | // Complete 55 | req, _ = http.NewRequest("DELETE", "/foo/1", nil) 56 | w = httptest.NewRecorder() 57 | s.ServeHTTP(w, req) 58 | 59 | assert.Equal(t, w.Code, 204) 60 | 61 | // Info not found 62 | req, _ = http.NewRequest("GET", "/foo/1", nil) 63 | w = httptest.NewRecorder() 64 | s.ServeHTTP(w, req) 65 | 66 | assert.Equal(t, w.Code, 404) 67 | } 68 | 69 | func TestServerAuth(t *testing.T) { 70 | s := NewServer(&Config{DbPath: "./server2", Auth: "secret", Store: "memory"}) 71 | defer s.Store.Drop() 72 | 73 | body := strings.NewReader("bar") 74 | 75 | req, _ := http.NewRequest("POST", "/foo", body) 76 | w := httptest.NewRecorder() 77 | s.ServeHTTP(w, req) 78 | 79 | assert.Equal(t, w.Code, 401) 80 | 81 | req, _ = http.NewRequest("POST", "/foo", body) 82 | req.SetBasicAuth("", "secret") 83 | w = httptest.NewRecorder() 84 | s.ServeHTTP(w, req) 85 | 86 | assert.Equal(t, w.Code, 201) 87 | } 88 | -------------------------------------------------------------------------------- /queued/stats.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type Stats struct { 8 | Counters map[string]int 9 | mutex sync.Mutex 10 | } 11 | 12 | func NewStats(counters map[string]int) *Stats { 13 | return &Stats{ 14 | Counters: counters, 15 | } 16 | } 17 | 18 | func (s *Stats) Inc(field string) { 19 | s.mutex.Lock() 20 | defer s.mutex.Unlock() 21 | 22 | s.Counters[field] += 1 23 | } 24 | 25 | func (s *Stats) Dec(field string) { 26 | s.mutex.Lock() 27 | defer s.mutex.Unlock() 28 | 29 | s.Counters[field] -= 1 30 | } 31 | 32 | func (s *Stats) Get() map[string]int { 33 | s.mutex.Lock() 34 | defer s.mutex.Unlock() 35 | 36 | return s.Counters 37 | } 38 | -------------------------------------------------------------------------------- /queued/store.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | type Iterator interface { 4 | NextRecord() (*Record, bool) 5 | } 6 | 7 | type Store interface { 8 | Get(id int) (*Record, error) 9 | Put(record *Record) error 10 | Remove(id int) error 11 | Iterator() Iterator 12 | Drop() 13 | } 14 | -------------------------------------------------------------------------------- /queued/version.go: -------------------------------------------------------------------------------- 1 | package queued 2 | 3 | const Version = "0.1.4" 4 | --------------------------------------------------------------------------------