├── .gitignore ├── util.go ├── queued_item.go ├── qdb.go ├── server.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | journals 3 | goq 4 | db/* 5 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | func QDBKey(id int64) []byte { 8 | return []byte(fmt.Sprintf("%d", id)) 9 | } 10 | -------------------------------------------------------------------------------- /queued_item.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | ) 6 | 7 | type QueuedItem struct { 8 | ID int64 9 | Data []byte 10 | } 11 | 12 | func (qi *QueuedItem) Size() int { 13 | return (binary.Size(qi.ID) + binary.Size(qi.Data)) 14 | } 15 | -------------------------------------------------------------------------------- /qdb.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "strconv" 8 | "sync" 9 | 10 | "github.com/syndtr/goleveldb/leveldb" 11 | "github.com/syndtr/goleveldb/leveldb/opt" 12 | ) 13 | 14 | type QDB struct { 15 | path string 16 | wo opt.WriteOptions 17 | db *leveldb.DB 18 | mutex sync.Mutex 19 | } 20 | 21 | func NewQDB(path string, syncWrites bool) *QDB { 22 | // Open 23 | db, err := leveldb.OpenFile(path, nil) 24 | if err != nil { 25 | panic(fmt.Sprintf("goq: Unable to open db: %v", err)) 26 | } 27 | 28 | log.Println("goq: Starting health check") 29 | 30 | // Health check each record 31 | iter := db.NewIterator(nil, nil) 32 | defer iter.Release() 33 | for iter.Next() { 34 | _, err := strconv.Atoi(string(iter.Key())) 35 | if err != nil { 36 | panic(fmt.Sprintf("goq: Health check failure (key not int): %s, %s, %v", string(iter.Key()), string(iter.Value()), err)) 37 | } 38 | } 39 | 40 | // General 41 | if iter.Error() != nil { 42 | panic(fmt.Sprintf("goq: Error loading db: %v", err)) 43 | } 44 | 45 | log.Println("goq: Health check successful") 46 | 47 | return &QDB{ 48 | path: path, 49 | wo: opt.WriteOptions{Sync: syncWrites}, 50 | db: db, 51 | } 52 | } 53 | 54 | func (self *QDB) Get(id int64) (*QueuedItem, error) { 55 | // Grab from level 56 | value, err := self.db.Get(QDBKey(id), nil) 57 | 58 | // Error retrieving key 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | // Nil value, should never happen 64 | if value == nil { 65 | return nil, nil 66 | } 67 | 68 | return &QueuedItem{id, value}, nil 69 | } 70 | 71 | func (self *QDB) Put(qi *QueuedItem) error { 72 | self.mutex.Lock() 73 | defer self.mutex.Unlock() 74 | 75 | // Put into level 76 | return self.db.Put(QDBKey(qi.ID), qi.Data, &self.wo) 77 | } 78 | 79 | func (self *QDB) Remove(id int64) error { 80 | self.mutex.Lock() 81 | defer self.mutex.Unlock() 82 | 83 | // Delete from level 84 | return self.db.Delete(QDBKey(id), &self.wo) 85 | } 86 | 87 | func (self *QDB) Close() { 88 | self.db.Close() 89 | } 90 | 91 | func (self *QDB) Drop() { 92 | self.Close() 93 | err := os.RemoveAll(self.path) 94 | if err != nil { 95 | panic(fmt.Sprintf("goq: Error removing db from disk: %v", err)) 96 | } 97 | } 98 | 99 | // Abstractions 100 | 101 | func (self *QDB) Next(remove bool) *QueuedItem { 102 | iter := self.db.NewIterator(nil, nil) 103 | defer iter.Release() 104 | 105 | for iter.Next() { 106 | id, err := strconv.Atoi(string(iter.Key())) 107 | if err != nil { 108 | panic(fmt.Sprintf("goq: Key not int: %s, %s, %v", string(iter.Key()), string(iter.Value()), err)) 109 | } 110 | if remove { 111 | self.Remove(int64(id)) 112 | } 113 | return &QueuedItem{int64(id), iter.Value()} 114 | } 115 | 116 | return nil 117 | } 118 | 119 | func (self *QDB) CacheFetch(maxBytes int) ([]*QueuedItem, int) { 120 | iter := self.db.NewIterator(nil, nil) 121 | defer iter.Release() 122 | 123 | // Collect 124 | items := make([]*QueuedItem, 0) 125 | totalSize := 0 126 | for iter.Next() { 127 | id, err := strconv.Atoi(string(iter.Key())) 128 | if err != nil { 129 | panic(fmt.Sprintf("goq: Key not int: %s, %s, %v", string(iter.Key()), string(iter.Value()), err)) 130 | } 131 | qi := QueuedItem{int64(id), iter.Value()} 132 | if totalSize+qi.Size() < maxBytes { 133 | items = append(items, &qi) 134 | } 135 | } 136 | 137 | return items, totalSize 138 | } 139 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | "runtime" 10 | "strconv" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | const ( 16 | VERSION = "1.0.0" 17 | ) 18 | 19 | var ( 20 | address string 21 | port int 22 | syncWrites bool 23 | dbPath string 24 | 25 | db *QDB 26 | 27 | totalEnqueues int 28 | totalDequeues int 29 | totalEmpties int 30 | ) 31 | 32 | func Enqueue(w http.ResponseWriter, req *http.Request) { 33 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 34 | 35 | if req.Method != "POST" { 36 | w.WriteHeader(405) 37 | fmt.Fprint(w, "{success:false,message:\"post request required\"}") 38 | return 39 | } 40 | 41 | data := strings.TrimSpace(req.FormValue("data")) 42 | if len(data) == 0 { 43 | w.WriteHeader(400) 44 | fmt.Fprint(w, "{success:false,message:\"data with length > 0 required\"}") 45 | return 46 | } 47 | 48 | db.Put(&QueuedItem{time.Now().UnixNano(), []byte(data)}) 49 | w.WriteHeader(200) 50 | fmt.Fprint(w, "{success:true,message:\"worked\"}") 51 | 52 | totalEnqueues++ 53 | } 54 | 55 | func Dequeue(w http.ResponseWriter, req *http.Request) { 56 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 57 | 58 | if req.Method != "GET" { 59 | w.WriteHeader(405) 60 | fmt.Fprint(w, "{success:false,message:\"get request required\"}") 61 | return 62 | } 63 | 64 | count, err := strconv.Atoi(strings.TrimSpace(req.FormValue("count"))) 65 | if err != nil { 66 | count = 1 67 | } 68 | 69 | w.WriteHeader(200) 70 | 71 | items := make([]string, 0) 72 | for i := 0; i < count; i++ { 73 | qi := db.Next(true) 74 | if qi == nil { 75 | break 76 | } 77 | items = append(items, string(qi.Data)) 78 | } 79 | 80 | itemsJson, jsonErr := json.Marshal(items) 81 | 82 | if jsonErr != nil { 83 | w.WriteHeader(500) 84 | fmt.Fprint(w, "{success:false,data:[],message:\"internal error\"}") 85 | return 86 | } 87 | 88 | fmt.Fprint(w, fmt.Sprintf("{success:true,data:%s,message:\"worked\"}", string(itemsJson))) 89 | 90 | totalDequeues += len(items) 91 | if len(items) == 0 { 92 | totalEmpties++ 93 | } 94 | } 95 | 96 | func Statistics(w http.ResponseWriter, req *http.Request) { 97 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 98 | w.WriteHeader(200) 99 | fmt.Fprint(w, fmt.Sprintf("{\"enqueues\":%d,\"dequeues\":%d,\"empties\":%d}", totalEnqueues, totalDequeues, totalEmpties)) 100 | } 101 | 102 | func Version(w http.ResponseWriter, req *http.Request) { 103 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 104 | w.WriteHeader(200) 105 | fmt.Fprint(w, fmt.Sprintf("{version:\"%s\"}", VERSION)) 106 | } 107 | 108 | func HealthCheck(w http.ResponseWriter, req *http.Request) { 109 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 110 | w.WriteHeader(200) 111 | fmt.Fprint(w, 1) 112 | } 113 | 114 | func init() { 115 | runtime.GOMAXPROCS(runtime.NumCPU()) 116 | 117 | flag.StringVar(&address, "address", "", "Address to listen on. Default is all.") 118 | flag.IntVar(&port, "port", 11311, "Port to listen on. Default is 11311.") 119 | flag.BoolVar(&syncWrites, "sync", true, "Synchronize database writes") 120 | flag.StringVar(&dbPath, "path", "db", "Database path. Default is db in current directory.") 121 | flag.Parse() 122 | } 123 | 124 | func main() { 125 | log.Printf("Listening on %s:%d\n", address, port) 126 | log.Printf("DB Path: %s\n", dbPath) 127 | 128 | db = NewQDB(dbPath, syncWrites) 129 | 130 | http.HandleFunc("/enqueue", Enqueue) 131 | http.HandleFunc("/dequeue", Dequeue) 132 | http.HandleFunc("/statistics", Statistics) 133 | http.HandleFunc("/version", Version) 134 | http.HandleFunc("/", HealthCheck) 135 | 136 | log.Println("Ready...") 137 | 138 | err := http.ListenAndServe(fmt.Sprintf("%s:%d", address, port), nil) 139 | if err != nil { 140 | panic(fmt.Sprintf("goq failed to launch: %v", err)) 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # goq 2 | 3 | goq is a persistent queue implemented in Go. 4 | 5 | 6 | ## Features 7 | 8 | - Communication over HTTP 9 | - RESTful with JSON responses (trivial to write client libraries) 10 | - Minimal API (`enqueue`, `dequeue`, `statistics`, `version`) 11 | - Basic configuration (`address`, `port`, `sync`, and `path`) 12 | - Operations are journaled for persistence via LevelDB 13 | - Database is health checked during startup, allowing for safe recovery 14 | 15 | 16 | ## Dependencies 17 | 18 | goq only has one external dependency (LevelDB). Before you start installing anything, go ahead and execute following command: 19 | 20 | `go get github.com/syndtr/goleveldb` 21 | 22 | 23 | ## Installation 24 | 25 | 1. Download/clone this repository to your system. 26 | 2. `cd` into the repository and execute `go build`. Note, this will create a binary called `goq`. 27 | 3. After successfully creating the binary, read about how to configure and run your first instance. 28 | 29 | 30 | ## Configuration 31 | 32 | There are only a handful of arguments needed to configure goq. Instead of managing a specific file, these are simply binary arguments/flags. 33 | 34 | These arguments include: 35 | 36 | - **port** - The port that this instance of goq should listen on. Default: 11311. 37 | 38 | - **sync** - Synchronize to LevelDB on every write. Default: true. 39 | 40 | - **path** - The path to the LevelDB database directory. goq will create this directory if needed. Default: ./db. 41 | 42 | goq listen on all addresses by default. If you want to listen on only one address use this parameter: 43 | 44 | - **address** - The address that this instance of goq should listen on. 45 | 46 | 47 | ## Initialization 48 | 49 | Now that you've created a binary and read about the configuration parameters above, you are ready to fire up an instance. Go ahead and execute the following command: 50 | 51 | ./goq -port=11311 -sync=true -journals=/var/log/goq/ 52 | 53 | After you execute the command, you should see your terminal contain the following log information: 54 | 55 | 2014/03/10 13:44:17 Listening on :11311 56 | 2014/03/10 13:44:17 DB Path: /var/log/goq/ 57 | 2014/03/10 13:44:17 goq: Starting health check 58 | 2014/03/10 13:44:17 goq: Health check successful 59 | 2014/03/10 13:44:17 Ready... 60 | 61 | This informs you that a LevelDB instance was created in the specified directory and that goq is listening on the desired port. Learn about goq's API from the documentation below. 62 | 63 | 64 | ## API 65 | 66 | There are only 4 API endpoints of interest, so this should be quick. 67 | 68 | ### POST /enqueue 69 | 70 | To enqueue something, execute the following command from your client: 71 | 72 | POST /enqueue data=>I am the first item! 73 | 74 | The only input parameter required is `data`, which is a string. Depending on your use case, you may need to URL encode this parameter. 75 | 76 | The response you got back should be: 77 | 78 | {success:true,message:"worked"} 79 | 80 | Go ahead and enqueue more items using the above process. 81 | 82 | ### GET /dequeue 83 | 84 | To dequeue data, execute the following from your client: 85 | 86 | GET /dequeue?count=1 87 | 88 | Note, if the `count` query parameter isn't specified, goq will only return one item at a time. 89 | 90 | If you followed the first enqueue command from above, you should see: 91 | 92 | {success:true,data:["I am the first item!"],message:"worked"} 93 | 94 | Your data is returned in the exact format that it was enqueued. If you enqueued anything else, try dequeueing again. If you call dequeue on an empty queue, you will receive the following: 95 | 96 | {success:true,data:[],message:"worked"} 97 | 98 | ### GET /statistics 99 | 100 | To see current goq statistics, execute the following from your client: 101 | 102 | GET /statistics 103 | 104 | You should see a JSON structure of the server's current statistics in a structure that resembles the following: 105 | 106 | {"enqueues":0,"dequeues":0,"empties":0} 107 | 108 | Your values may be different from above if you've enqueued or dequeued other items. `empties` refers to the number of dequeues that were made to an empty queue. 109 | 110 | ### GET /version 111 | 112 | To see the current goq version, execute the following from your client: 113 | 114 | GET /version 115 | 116 | As of the most recent version, you should see: 117 | 118 | {version:"1.0.0"} 119 | 120 | This should be parsed as a string. The version number will become increasingly important as new features are introduced. 121 | 122 | 123 | ## Restarts / Health Checks 124 | 125 | When restarted, goq replays all of the items in LevelDB to ensure consistency. You can restore a goq instance just from its LevelDB directory. Please raise an issue if you come across an issue replaying transactions. 126 | 127 | 128 | ## Contributions 129 | 130 | goq was developed by [Kunal Anand][0]. 131 | 132 | 133 | ## License 134 | 135 | This code is completely free under the MIT License: [http://mit-license.org/][2]. 136 | 137 | 138 | [0]: https://twitter.com/ka 139 | [2]: http://mit-license.org/ 140 | --------------------------------------------------------------------------------