├── .gitignore ├── go.mod ├── UNLICENSE ├── script.go ├── README.md ├── main.go └── database.go /.gitignore: -------------------------------------------------------------------------------- 1 | atomkv 2 | atomkv.exe 3 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/skeeto/atomkv 2 | 3 | go 1.15 4 | -------------------------------------------------------------------------------- /UNLICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /script.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | ) 7 | 8 | func script(w http.ResponseWriter, r *http.Request) { 9 | w.Header().Set("Content-Type", "application/javascript") 10 | io.WriteString(w, `"use strict" 11 | let AtomKV = { 12 | BASE: new URL(document.currentScript.src).origin, 13 | 14 | get: function(key) { 15 | let url = this.BASE + key 16 | return new Promise(function(resolve, reject) { 17 | let xhr = new XMLHttpRequest() 18 | xhr.onload = function() { 19 | if (xhr.status == 404) { 20 | resolve([undefined, -1]) 21 | } else { 22 | let revision = Number(xhr.getResponseHeader('X-Revision')) 23 | resolve([JSON.parse(xhr.responseText), revision]) 24 | } 25 | } 26 | xhr.onerror = function() { 27 | reject(xhr.responseText) 28 | } 29 | xhr.open('GET', url, true) 30 | xhr.send() 31 | }) 32 | }, 33 | 34 | set: function(key, value) { 35 | let url = this.BASE + key 36 | return new Promise(function(resolve, reject) { 37 | let xhr = new XMLHttpRequest() 38 | xhr.onload = function() { 39 | resolve() 40 | } 41 | xhr.onerror = function() { 42 | reject(xhr.responseText) 43 | } 44 | xhr.open('POST', url, true) 45 | xhr.send(JSON.stringify(value)) 46 | }) 47 | }, 48 | 49 | update: function(key, value, revision) { 50 | let url = this.BASE + key 51 | return new Promise(function(resolve, reject) { 52 | let xhr = new XMLHttpRequest() 53 | xhr.onload = function() { 54 | resolve(xhr.status == 200) 55 | } 56 | xhr.onerror = function() { 57 | reject(xhr.responseText) 58 | } 59 | xhr.open('PUT', url, true) 60 | xhr.setRequestHeader('X-Revision', String(revision)) 61 | xhr.send(JSON.stringify(value)) 62 | }) 63 | }, 64 | 65 | subscribe: async function*(keypath) { 66 | let resolve = null 67 | let promise = null 68 | function reset() { 69 | promise = new Promise(function(r) {resolve = r}) 70 | } 71 | reset() 72 | let sse = new EventSource(this.BASE + keypath) 73 | sse.onmessage = function(event) { 74 | let value = JSON.parse(event.data) 75 | let [key, revision] = event.lastEventId.split(/:/) 76 | resolve([key, value, Number(revision)]) 77 | reset() 78 | } 79 | try { 80 | for (;;) { 81 | yield await promise 82 | } 83 | } finally { 84 | sse.close() 85 | } 86 | } 87 | }`) 88 | } 89 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AtomKV 2 | 3 | AtomKV is an in-memory, JSON, key-value service with compare-and-swap 4 | updates and [event streams][es]. It supports [CORS][cors] and hosts its 5 | own JavaScript library, so any page on any domain can use it seamlessly. 6 | 7 | Keys are hierarchical, always begin with a slash, and never end with a 8 | slash. Components between slashes are limited to letters, numbers, dash, 9 | and underscore. Listeners can monitor a specific key or all keys rooted 10 | under a key space. Each key-value has a revision number which starts at 11 | zero and increments by one for each update. 12 | 13 | ## Usage 14 | 15 | $ go build 16 | $ ./atomkv -addr :8000 17 | 18 | ## JavaScript API (high level) 19 | 20 | The JavaScript library is hosted at `/` and defines an `AtomKV` object 21 | with the following functions (with TypeScript-like notation): 22 | 23 | ```ts 24 | /** 25 | * Retrieve the current value for the given key, returning the value and 26 | * the revision number. If the key does not exist, returns [null, -1]. 27 | */ 28 | AtomKV.get = async function(key: string): [any, number] 29 | 30 | /** 31 | * Forcefully assign a key to a JSON-serializable value. 32 | */ 33 | AtomKV.set = async function(key: string, value: any) 34 | 35 | /** 36 | * Attempt to update the value for the specific key, but only if it 37 | * would have the given revision number. Returns true if successful, 38 | * otherwise false. On failure, use .get() to retrieve the current value 39 | * and revision, then try again. 40 | */ 41 | AtomKV.update = async function(key: string, value: any, revision: number): boolean 42 | 43 | /** 44 | * Returns an asynchronous generator of update events for the given key 45 | * path. If the key path ends with a slash, it will monitor the entire 46 | * hierarchy under that path. Each yield returns the key, value, and 47 | * revision number. Typically used in a "for await" statement. 48 | */ 49 | AtomKV.subscribe = async function*(keypath: string): [string, any, number] 50 | ``` 51 | 52 | ### JavaScript examples 53 | 54 | The following example iterates the Fibonacci state at `/fib`. This is 55 | safe for multiple concurrent clients because of the compare-and-swap 56 | semantics of `.update()`. If there's a conflict, the fastest client 57 | succeeds and the others retry. 58 | 59 | ```js 60 | async function init() { 61 | // Acceptably fails if already initialized 62 | AtomKV.update('/fib', [0, 1], 0) 63 | } 64 | 65 | async function increment() { 66 | for (;;) { 67 | let [[a, b], revision] = await AtomKV.get('/fib') 68 | let next = [b, a + b] 69 | if (await AtomKV.update('/fib', next, revision + 1)) { 70 | return next // success 71 | } 72 | } 73 | } 74 | ``` 75 | 76 | This example monitors all key-values under a given "session" ID: 77 | 78 | ```js 79 | async function monitor(session) { 80 | for await (let [key, value, rev] of AtomKV.subscribe(`/${session}/`)) { 81 | console.log(key, value, rev) 82 | } 83 | } 84 | 85 | let SESSION_ID = Math.floor(Math.random() * 0xffffffff).toString(16) 86 | console.log(`SESSIONID = ${SESSION_ID}`) 87 | monitor(SESSION_ID) 88 | ``` 89 | 90 | ## HTTP API (low level) 91 | 92 | There's really just one endpoint, `/`, which responds differently to 93 | different methods. 94 | 95 | ### `GET /` 96 | 97 | Serves the JavaScript library documented above. Include this on your 98 | page using a `script` tag. 99 | 100 | ### `GET /path/to/key` 101 | 102 | Returns the JSON-encoded value for the given key. The `X-Revision` 103 | header indicates the revision number. 104 | 105 | ### `GET /path/to/key` with `Accept: text/event-stream` 106 | 107 | If the `Accept` header indicates `text/event-stream` then the request 108 | will use [Server-sent Events][sse]. If the path ends in a slash, all 109 | keys under that path are monitored. It is not possible to monitor `/`. 110 | 111 | The `data` field for each event is the JSON-encoded value, and the `id` 112 | field is `/path/to/key:revision`. Keys cannot contain a colon, so this 113 | is trivial to parse. 114 | 115 | ### `POST /path/to/key` 116 | 117 | Forcefully overwrite the key with a JSON-encoded value. This always 118 | succeeds for valid keys. 119 | 120 | ### `PUT /path/to/key` 121 | 122 | Attempt to store a new JSON-encoded value under the given key. An 123 | `X-Revision` header *must* be supplied, indicating the assumed new 124 | revision number — one more than the last observed revision. It only 125 | succeeds if the expected revision number matches. 126 | 127 | 128 | [cors]: https://developer.mozilla.org/en-US/docs/Glossary/CORS 129 | [es]: https://developer.mozilla.org/en-US/docs/Web/API/EventSource 130 | [sse]: https://html.spec.whatwg.org/multipage/server-sent-events.html 131 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "log" 11 | "net" 12 | "net/http" 13 | "strconv" 14 | "unicode" 15 | ) 16 | 17 | type handler struct{} 18 | 19 | func validKey(key string) bool { 20 | if len(key) < 2 || key[0] != '/' { 21 | return false 22 | } 23 | tail := key[1:] 24 | for i, r := range tail { 25 | if r == '/' { 26 | if tail[i-1] == '/' { 27 | return false 28 | } 29 | } else if !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '-' && r != '_' { 30 | return false 31 | } 32 | } 33 | return key[len(key)-1] != '/' 34 | } 35 | 36 | func validPath(path string) bool { 37 | if len(path) == 0 { 38 | return false 39 | } 40 | if path[len(path)-1] == '/' { 41 | return validKey(path + "_") 42 | } 43 | return validKey(path) 44 | } 45 | 46 | func (h *handler) get(w http.ResponseWriter, r *http.Request) { 47 | hdr := w.Header() 48 | hdr.Set("Cache-Control", "no-cache") 49 | hdr.Set("Content-Type", "application/json") 50 | 51 | key := r.URL.Path 52 | log.Printf("GET %s %s", r.RemoteAddr, key) 53 | if !validKey(key) { 54 | http.Error(w, "invalid key", 400) 55 | return 56 | } 57 | 58 | db, _ := FromContext(r.Context()) 59 | value, revision, ok := db.Get(key) 60 | if !ok { 61 | http.Error(w, "no such key", 404) 62 | return 63 | } 64 | hdr.Set("X-Revision", strconv.Itoa(revision)) 65 | io.WriteString(w, value) 66 | } 67 | 68 | func normalize(r io.Reader) (string, bool) { 69 | buf, err := ioutil.ReadAll(r) 70 | if err != nil { 71 | return "", false 72 | } 73 | 74 | var data interface{} 75 | if json.Unmarshal(buf, &data) != nil { 76 | return "", false 77 | } 78 | buf, _ = json.Marshal(data) 79 | return string(buf), true 80 | } 81 | 82 | func (h *handler) post(w http.ResponseWriter, r *http.Request) { 83 | w.Header().Set("Content-Type", "application/json") 84 | 85 | key := r.URL.Path 86 | log.Printf("POST %s %s", r.RemoteAddr, key) 87 | if !validKey(key) { 88 | http.Error(w, "invalid key", 400) 89 | return 90 | } 91 | 92 | value, ok := normalize(r.Body) 93 | if !ok { 94 | http.Error(w, "invalid JSON", 400) 95 | return 96 | } 97 | db, _ := FromContext(r.Context()) 98 | db.Set(key, value) 99 | } 100 | 101 | func (h *handler) put(w http.ResponseWriter, r *http.Request) { 102 | w.Header().Set("Content-Type", "application/json") 103 | 104 | key := r.URL.Path 105 | log.Printf("PUT %s %s", r.RemoteAddr, key) 106 | if !validKey(key) { 107 | http.Error(w, "invalid key", 400) 108 | return 109 | } 110 | 111 | xrevision := r.Header.Get("X-Revision") 112 | if xrevision == "" { 113 | http.Error(w, "missing revision", 400) 114 | return 115 | } 116 | revision, err := strconv.Atoi(xrevision) 117 | if err != nil { 118 | http.Error(w, "invalid revision", 400) 119 | return 120 | } 121 | 122 | json, ok := normalize(r.Body) 123 | if !ok { 124 | http.Error(w, "invalid JSON", 400) 125 | return 126 | } 127 | 128 | db, _ := FromContext(r.Context()) 129 | if !db.Update(key, json, revision) { 130 | http.Error(w, "revision conflict", 409) 131 | return 132 | } 133 | } 134 | 135 | func (h *handler) events(w http.ResponseWriter, r *http.Request) { 136 | hdr := w.Header() 137 | hdr.Set("Cache-Control", "no-cache") 138 | hdr.Set("Connection", "keep-alive") 139 | hdr.Set("Content-Type", "text/event-stream") 140 | hdr.Set("Transfer-Encoding", "chunked") 141 | 142 | f, ok := w.(http.Flusher) 143 | if !ok { 144 | http.Error(w, "unsupported", 500) 145 | return 146 | } 147 | 148 | path := r.URL.Path 149 | log.Printf("SSE %s %s", r.RemoteAddr, path) 150 | if !validPath(path) { 151 | http.Error(w, "invalid path/key", 400) 152 | return 153 | } 154 | 155 | ctx := r.Context() 156 | db, _ := FromContext(r.Context()) 157 | ch := db.Subscribe(path) 158 | defer db.Unsubscribe(ch) 159 | for { 160 | select { 161 | case v := <-ch: 162 | _, err := fmt.Fprintf(w, "data:%s\nid:%s:%d\n\n", v.Value, v.Key, v.Revision) 163 | if err != nil { 164 | return 165 | } 166 | f.Flush() 167 | case <-ctx.Done(): 168 | return 169 | } 170 | } 171 | } 172 | 173 | func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 174 | hdr := w.Header() 175 | hdr.Set("Access-Control-Allow-Headers", "*") 176 | hdr.Set("Access-Control-Allow-Methods", "*") 177 | hdr.Set("Access-Control-Allow-Origin", "*") 178 | hdr.Set("Access-Control-Expose-Headers", "X-Revision") 179 | if r.URL.Path == "/" { 180 | script(w, r) 181 | } else { 182 | switch r.Method { 183 | case "GET": 184 | switch r.Header.Get("Accept") { 185 | case "text/event-stream": 186 | h.events(w, r) 187 | default: 188 | h.get(w, r) 189 | } 190 | case "POST": 191 | h.post(w, r) 192 | case "PUT": 193 | h.put(w, r) 194 | case "OPTIONS": 195 | log.Printf("OPTIONS %s", r.RemoteAddr) 196 | } 197 | } 198 | } 199 | 200 | func main() { 201 | addr := flag.String("addr", ":8000", "Server's host address") 202 | flag.Parse() 203 | 204 | db := NewDatabase(0, 0) 205 | ctx := db.NewContext(context.Background()) 206 | s := &http.Server{ 207 | Addr: *addr, 208 | Handler: &handler{}, 209 | BaseContext: func(l net.Listener) context.Context { return ctx }, 210 | } 211 | log.Printf("listening at %s", *addr) 212 | log.Fatal(s.ListenAndServe()) 213 | } 214 | -------------------------------------------------------------------------------- /database.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "time" 7 | ) 8 | 9 | const ( 10 | defaultBuflen = 64 11 | defaultExpiry = time.Hour * 24 * 7 12 | ) 13 | 14 | type Update struct { 15 | Key string 16 | Value string 17 | Revision int 18 | } 19 | 20 | type entry struct { 21 | value string 22 | revision int 23 | } 24 | 25 | type requestGet struct { 26 | key string 27 | resp chan<- struct { 28 | value string 29 | revision int 30 | ok bool 31 | } 32 | } 33 | 34 | type requestSet struct { 35 | key string 36 | value string 37 | } 38 | 39 | type requestUpdate struct { 40 | key string 41 | value string 42 | revision int 43 | resp chan<- bool 44 | } 45 | 46 | type requestSubscribe struct { 47 | path string 48 | ch chan Update 49 | } 50 | 51 | type requestUnsubscribe struct { 52 | ch <-chan Update 53 | } 54 | 55 | type subscriber struct { 56 | path string 57 | ch chan Update 58 | } 59 | 60 | type Database struct { 61 | values map[string]entry 62 | subscriptions map[string]map[chan Update]struct{} 63 | subscribers map[<-chan Update]subscriber 64 | buflen int 65 | expiry time.Duration 66 | 67 | chGet chan requestGet 68 | chSet chan requestSet 69 | chUpdate chan requestUpdate 70 | chDelete chan string 71 | chSubscribe chan requestSubscribe 72 | chUnsubscribe chan requestUnsubscribe 73 | chStop chan struct{} 74 | } 75 | 76 | func NewDatabase(buflen int, expiry time.Duration) *Database { 77 | if buflen == 0 { 78 | buflen = defaultBuflen 79 | } 80 | if expiry == 0 { 81 | expiry = defaultExpiry 82 | } 83 | 84 | database := Database{ 85 | values: make(map[string]entry), 86 | subscriptions: make(map[string]map[chan Update]struct{}), 87 | subscribers: make(map[<-chan Update]subscriber), 88 | buflen: buflen, 89 | expiry: expiry, 90 | 91 | chGet: make(chan requestGet), 92 | chSet: make(chan requestSet), 93 | chUpdate: make(chan requestUpdate), 94 | chDelete: make(chan string), 95 | chSubscribe: make(chan requestSubscribe), 96 | chUnsubscribe: make(chan requestUnsubscribe), 97 | chStop: make(chan struct{}), 98 | } 99 | go database.dispatch() 100 | return &database 101 | } 102 | 103 | func (d *Database) dispatch() { 104 | for { 105 | select { 106 | case r := <-d.chGet: 107 | e, ok := d.get(r.key) 108 | r.resp <- struct { 109 | value string 110 | revision int 111 | ok bool 112 | }{e.value, e.revision, ok} 113 | case r := <-d.chSet: 114 | d.set(r.key, r.value) 115 | case r := <-d.chUpdate: 116 | r.resp <- d.update(r.key, r.value, r.revision) 117 | case key := <-d.chDelete: 118 | delete(d.values, key) 119 | case r := <-d.chSubscribe: 120 | d.subscribe(r.path, r.ch) 121 | case r := <-d.chUnsubscribe: 122 | d.unsubscribe(r.ch) 123 | case <-d.chStop: 124 | return 125 | } 126 | } 127 | } 128 | 129 | func (d *Database) get(key string) (entry, bool) { 130 | e, ok := d.values[key] 131 | return e, ok 132 | } 133 | 134 | func (d *Database) set(key string, value string) { 135 | e, ok := d.values[key] 136 | if ok { 137 | e.revision++ 138 | } else { 139 | go d.expire(key) 140 | } 141 | e.value = value 142 | d.values[key] = e 143 | d.notify(key, e) 144 | } 145 | 146 | func (d *Database) update(key string, value string, revision int) bool { 147 | e, ok := d.values[key] 148 | if !ok { 149 | e.revision = -1 150 | } 151 | if e.revision+1 != revision { 152 | return false 153 | } 154 | if !ok { 155 | go d.expire(key) 156 | } 157 | e = entry{value, revision} 158 | d.values[key] = e 159 | d.notify(key, e) 160 | return true 161 | } 162 | 163 | func (d *Database) expire(key string) { 164 | t := time.NewTimer(d.expiry) 165 | ch := d.Subscribe(key) 166 | for { 167 | select { 168 | case <-ch: 169 | if !t.Stop() { 170 | <-t.C 171 | } 172 | t.Reset(d.expiry) 173 | case <-t.C: 174 | d.Delete(key) 175 | } 176 | } 177 | } 178 | 179 | func (d *Database) subscribe(path string, ch chan Update) { 180 | m, ok := d.subscriptions[path] 181 | if !ok { 182 | m = make(map[chan Update]struct{}) 183 | d.subscriptions[path] = m 184 | } 185 | m[ch] = struct{}{} 186 | d.subscribers[ch] = subscriber{path: path, ch: ch} 187 | } 188 | 189 | func (d *Database) unsubscribe(ch <-chan Update) { 190 | sub := d.subscribers[ch] 191 | delete(d.subscribers, ch) 192 | m := d.subscriptions[sub.path] 193 | delete(m, sub.ch) 194 | if len(m) == 0 { 195 | delete(d.subscriptions, sub.path) 196 | } 197 | } 198 | 199 | func (d *Database) notify(key string, e entry) { 200 | seen := make(map[chan Update]struct{}) 201 | part := key 202 | for { 203 | if m, ok := d.subscriptions[part]; ok { 204 | for s := range m { 205 | if _, ok := seen[s]; !ok { 206 | seen[s] = struct{}{} 207 | select { 208 | case s <- Update{Key: key, Value: e.value, Revision: e.revision}: 209 | default: // drop 210 | } 211 | } 212 | } 213 | } 214 | slash := strings.LastIndexByte(part[:len(part)-1], '/') 215 | if slash == -1 { 216 | break 217 | } 218 | part = part[:slash+1] 219 | } 220 | } 221 | 222 | func validate(key string) { 223 | if len(key) == 0 || key[0] != '/' { 224 | panic("invalid key") 225 | } 226 | } 227 | 228 | func (d *Database) Get(key string) (string, int, bool) { 229 | validate(key) 230 | resp := make(chan struct { 231 | value string 232 | revision int 233 | ok bool 234 | }) 235 | d.chGet <- requestGet{key, resp} 236 | e := <-resp 237 | return e.value, e.revision, e.ok 238 | } 239 | 240 | func (d *Database) Set(key string, value string) { 241 | validate(key) 242 | d.chSet <- requestSet{key: key, value: value} 243 | } 244 | 245 | func (d *Database) Update(key string, value string, revision int) bool { 246 | validate(key) 247 | resp := make(chan bool) 248 | d.chUpdate <- requestUpdate{ 249 | key: key, 250 | value: value, 251 | revision: revision, 252 | resp: resp, 253 | } 254 | return <-resp 255 | } 256 | 257 | func (d *Database) Delete(key string) { 258 | d.chDelete <- key 259 | } 260 | 261 | func (d *Database) Subscribe(path string) <-chan Update { 262 | validate(path) 263 | ch := make(chan Update, d.buflen) 264 | d.chSubscribe <- requestSubscribe{path: path, ch: ch} 265 | return ch 266 | } 267 | 268 | func (d *Database) Unsubscribe(ch <-chan Update) { 269 | d.chUnsubscribe <- requestUnsubscribe{ch: ch} 270 | } 271 | 272 | func (d *Database) Close() { 273 | close(d.chStop) 274 | } 275 | 276 | type key int 277 | 278 | func (d *Database) NewContext(ctx context.Context) context.Context { 279 | return context.WithValue(ctx, key(0), d) 280 | } 281 | 282 | func FromContext(ctx context.Context) (*Database, bool) { 283 | d, ok := ctx.Value(key(0)).(*Database) 284 | return d, ok 285 | } 286 | --------------------------------------------------------------------------------