├── .github └── workflows │ └── go.yml ├── .gitignore ├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── kvass.go └── src ├── cli.go ├── cli_test.go ├── crypto_test.go ├── model.go ├── server.go ├── sqlite_persistance.go ├── sqlite_persistance_test.go └── sync_test.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v3 18 | with: 19 | go-version: 1.18 20 | 21 | - name: Build 22 | run: go build -v ./... 23 | 24 | - name: Test 25 | working-directory: src 26 | run: go test -v ./... 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | kvass 2 | *.json 3 | .DS_Store 4 | *.sqlite 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Max Nagy 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kvass: a personal key-value store 2 | 3 | ![kvass_small](https://user-images.githubusercontent.com/5411096/179968508-5fe1e390-3136-46a6-bb1e-8d329ad231c3.jpeg) 4 | 5 | 6 | ```bash 7 | # simple usage 8 | $ kvass set hello world 9 | $ kvass get hello 10 | world 11 | 12 | # enumerate keys 13 | $ kvass ls 14 | hello 15 | 16 | # store arbitrary files 17 | $ kvass set logo < kvass.jpg 18 | $ kvass get logo > kvass.jpg 19 | 20 | 21 | # Its trivial to set up and operate kvass across multiple devices 22 | $ ssh you@yourserver.com kvass config show 23 | 24 | Encryption Key: 5abf59f5f1a2f3c998a4f592ce081a23e14a68fd8a792259c6ec0fc1e8fb1246 # <- copy this for the next step 25 | ProcessID: 752176921 26 | Remote: (None) 27 | 28 | $ kvass config key 5abf59f5f1a2f3c998a4f592ce081a23e14a68fd8a792259c6ec0fc1e8fb1246 # set the same key for all your devices 29 | $ kvass config remote yourserver.com:8000 # tell kvass where to find the server instance 30 | 31 | # Run "kvass serve" on your server using systemd, screen or the init system of your choice (runit, anyone?). You can specify the interface and port to host at with [--bind]. 32 | 33 | $ kvass serve --bind="0.0.0.0:80" # host on the default HTTP port (which means you can generate cleaner URLs - just set your remote no port) 34 | 35 | # every set will now be broadcasted to the server 36 | $ kvass set "hello from the other side" hello 37 | $ ssh you@yourserver kvass get "hello from the other side" 38 | hello 39 | 40 | # and every get will check the server for updates 41 | $ ssh you@yourserver kvass set hello 👋 42 | $ kvass get hello 43 | 👋 44 | 45 | # Good to know: All communication between the client and server is authenticated and encrypted using AES-256 GCM. 46 | 47 | # remember the file we stored earlier? Let's get a shareable url for it! 48 | $ kvass url logo 49 | http://demo.maxnagy.com:8000/get?q=OQMwTQmFCz6xiWxFxt4Mkw 50 | 51 | # you can also print the corresponding qr code directly to your terminal 52 | kvass qr logo 53 | ``` 54 | ![Screen Shot 2022-07-20 at 13 23 17](https://user-images.githubusercontent.com/5411096/179970204-f1034add-ce07-4f40-b279-0ac25969c069.png) 55 | 56 | ``` 57 | # run kvass without arguments to get a nice cheat sheet of supported commands 58 | $ kvass 59 | kvass [--db=string] 60 | 61 | Description: 62 | kvass - a personal KV store 63 | 64 | Options: 65 | --db the database file to use (default: ~/.kvassdb.sqlite) 66 | 67 | Sub-commands: 68 | kvass ls list keys 69 | kvass get get a value 70 | kvass set set a value 71 | kvass rm remove a key 72 | kvass url show shareable url of an entry 73 | kvass qr print shareable qr code of entry to console 74 | kvass config set config parameters 75 | kvass serve start in server mode [--bind="ip:port" (default: 0.0.0.0:8000)] 76 | ``` 77 | 78 | # Installation 79 | 80 | ```bash 81 | go install github.com/maxmunzel/kvass@latest 82 | ``` 83 | 84 | # How Syncing works 85 | 86 | TL;DR There is a central server running `kvass serve` with clients 87 | connected to it. Key-value pairs overwrite each other based on wall 88 | clock time 99.999% of the time and using Lamport clocks in the 89 | remaining .001% of the time. You can mostly forget about this, as long 90 | as your clocks are mostly in sync. 91 | 92 | Let's dive into the details! 93 | 94 | Each time we `set` or `rm` a key, kvass creates a new `KvEntry` struct and 95 | merges it onto its local state. The local state is a set of `KvEntry`s that 96 | represent a key-value mapping. 97 | Technically, `kvass rm key` is `kvass set key ""`, so they work the same way. 98 | 99 | `KvEntry` is defined as follows: 100 | ```go 101 | type KvEntry struct { 102 | Key string 103 | Value []byte // empty slice means key deleted 104 | UrlToken string // random token used for url 105 | 106 | // The following fields are used for state merging 107 | TimestampUnixMicro int64 108 | ProcessID uint32 // randomly chosen for each node 109 | Counter uint64 // Lamport clock 110 | } 111 | ``` 112 | 113 | ## Lamport Clocks 114 | 115 | Lamport Clocks are a common and easy way to order events in a distributed system. 116 | The [Wikipedia](https://en.wikipedia.org/wiki/Lamport_timestamp) summarizes them nicely: 117 | > The algorithm follows some simple rules: 118 | > 1. A process increments its counter before each local event (e.g., message sending event); 119 | > 2. When a process sends a message, it includes its counter value with the message after executing step 1; 120 | > 3. On receiving a message, the counter of the recipient is updated, if necessary, to the greater of its current counter and the timestamp in the received message. The counter is then incremented by 1 before the message is considered received. 121 | 122 | In kvass, sending a message means `set`ting a key locally and 123 | receiving a message means merging a `KvEntry` into the local state. 124 | 125 | Lamport clocks have a nice property: If an event `a` happens causally 126 | after another event `b`, then it follows, that `a.count` > `b.count`: 127 | 128 | 129 | ``` 130 | node1 set foo=bar -> set foo=baz \ (send KvEntry{Key="foo", Counter=2, Value="baz"} to node2) 131 | count=0 count = 1 count = 2 \ 132 | \ 133 | node2 -> rm foo 134 | count=0 count = 3 135 | ``` 136 | 137 | node2 updated its counter upon receiving node1's update, so the counter values nicely reflect the fact, 138 | that node2 knew about node1's updates to foo before deleting it. This isn't always helpful though: 139 | 140 | ``` 141 | node1 set foo=bar \ (send KvEntry{Key="foo", Counter=1, Value="bar"} to node2) 142 | count=0 count = 1 \ 143 | \ 144 | node2 set foo=baz > (What is foo supposed to be know?) 145 | count=0 count = 1 146 | ``` 147 | 148 | The canonical answer in this case is to resolve conflicts based on node number (`ProcessID`). 149 | This introduces a global, consistent and total order of events. However it may not always reflect 150 | the order in which a user actually performs the events 151 | ``` 152 | node1 set foo=bar -> set foo=baz \ (send KvEntry{Key="foo", Counter=2, Value="baz"} to node2) 153 | count=0 count = 1 count = 2 \ 154 | \ 155 | node2 rm foo -> (foo is not set to "baz" again) 156 | count=0 count = 1 157 | ``` 158 | 159 | In kvass, we therefore use wall-clock time to resolve conflicts: The most recent action of a user is probably the 160 | one he intents to persist, independent of the node he triggered it on. Still, we use Lamport timestamps to handle 161 | identical timestamps and to keep track of which `KvEntry`s we need to exchange between nodes: 162 | 163 | 164 | ``` 165 | node1 set foo=bar -> set foo=baz \ (send KvEntry{Key="foo", Counter=2, Value="baz"} to node2) 166 | count=0 count = 1 count = 2 \ 167 | \ 168 | node2 rm foo -> (foo stays removed, its 169 | count=0 count = 1 count increased to 3) 170 | ``` 171 | 172 | The merging of states is actually trivial now: 173 | ```go 174 | func (s *SqlitePersistance) UpdateOn(entry KvEntry) error { 175 | 176 | oldEntry := getCurrentEntryFromDB(entry.Key) 177 | 178 | // update the remote counter 179 | s.State.RemoteCounter = mathutil.MaxUint64(s.State.RemoteCounter, entry.Counter) 180 | 181 | // select LUB of old and new entry 182 | entry = entry.Max(oldEntry) // returns the entry with the greater (time, counter, - pid) tuple 183 | 184 | // update local counter 185 | newCounter := mathutil.MaxUint64(entry.Counter, s.State.Counter) + 1 186 | s.State.Counter = newCounter 187 | 188 | // set new entries counter 189 | entry.Counter = newCounter 190 | 191 | // write back LUB to db 192 | } 193 | ``` 194 | 195 | `LUB` is CRDT-speak for "least upper bound" ie the smallest entry that is >= the old and new entry. 196 | We convince ourself, that the `KvEntry.Max()` satisfies this LUB property and can therefore derive 197 | that it is also commutative, *idempotent* and associative. 198 | 199 | 200 | 201 | 202 | # Shoutouts 203 | 204 | [Charm skate](https://github.com/charmbracelet/skate) -- the inspiration for this tool 205 | 206 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/maxmunzel/kvass 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/teris-io/cli v1.0.1 7 | modernc.org/mathutil v1.4.1 8 | modernc.org/sqlite v1.17.3 9 | ) 10 | 11 | require ( 12 | github.com/google/uuid v1.3.0 // indirect 13 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect 14 | github.com/lizebang/qrcode-terminal v0.0.0-20180928100242-57607785510a 15 | github.com/mattn/go-isatty v0.0.12 // indirect 16 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect 17 | github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e 18 | golang.org/x/mod v0.3.0 // indirect 19 | golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect 20 | golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 // indirect 21 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 22 | lukechampine.com/uint128 v1.1.1 // indirect 23 | modernc.org/cc/v3 v3.36.0 // indirect 24 | modernc.org/ccgo/v3 v3.16.6 // indirect 25 | modernc.org/libc v1.16.7 // indirect 26 | modernc.org/memory v1.1.1 // indirect 27 | modernc.org/opt v0.1.1 // indirect 28 | modernc.org/strutil v1.1.1 // indirect 29 | modernc.org/token v1.0.0 // indirect 30 | ) 31 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/dawndiy/qrcode-terminal v0.0.0-20201102154902-d62dbb5be992 h1:BixIZNWpKQKifHERX/Q2f6KgaTsiqExz/N1FdmgdslI= 2 | github.com/dawndiy/qrcode-terminal v0.0.0-20201102154902-d62dbb5be992/go.mod h1:mtg39F95eiC0XwS+JesArRKeaPtjbzuNuTcKRGhIVik= 3 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 4 | github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 5 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 6 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 7 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= 8 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= 9 | github.com/lizebang/qrcode-terminal v0.0.0-20180928100242-57607785510a h1:tZbV9V/of5Yb8DdCLlGqoogy0urKwvE1ALloO2uWGvM= 10 | github.com/lizebang/qrcode-terminal v0.0.0-20180928100242-57607785510a/go.mod h1:M2rVKvY6gXYx4D6XgHPjO+OmZhwBWVE5tKYGYvHVAEo= 11 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 12 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 13 | github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 14 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 15 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= 16 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 17 | github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= 18 | github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= 19 | github.com/teris-io/cli v1.0.1 h1:J6jnVHC552uqx7zT+Ux0++tIvLmJQULqxVhCid2u/Gk= 20 | github.com/teris-io/cli v1.0.1/go.mod h1:V9nVD5aZ873RU/tQXLSXO8FieVPQhQvuNohsdsKXsGw= 21 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 22 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 23 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 24 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 25 | golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= 26 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 27 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 28 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 29 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 30 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 31 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 32 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 33 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 34 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 35 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 36 | golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= 37 | golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 38 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 39 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 40 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 41 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 42 | golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 h1:M8tBwCtWD/cZV9DZpFYRUgaymAYAr+aIUTWzDaM3uPs= 43 | golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 44 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 45 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 46 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 47 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 48 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 49 | lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU= 50 | lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= 51 | modernc.org/cc/v3 v3.36.0 h1:0kmRkTmqNidmu3c7BNDSdVHCxXCkWLmWmCIVX4LUboo= 52 | modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= 53 | modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= 54 | modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= 55 | modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= 56 | modernc.org/ccgo/v3 v3.16.6 h1:3l18poV+iUemQ98O3X5OMr97LOqlzis+ytivU4NqGhA= 57 | modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= 58 | modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= 59 | modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= 60 | modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= 61 | modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= 62 | modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= 63 | modernc.org/libc v1.16.7 h1:qzQtHhsZNpVPpeCu+aMIQldXeV1P0vRhSqCL0nOIJOA= 64 | modernc.org/libc v1.16.7/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= 65 | modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= 66 | modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8= 67 | modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= 68 | modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU= 69 | modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= 70 | modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A= 71 | modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= 72 | modernc.org/sqlite v1.17.3 h1:iE+coC5g17LtByDYDWKpR6m2Z9022YrSh3bumwOnIrI= 73 | modernc.org/sqlite v1.17.3/go.mod h1:10hPVYar9C0kfXuTWGz8s0XtB8uAGymUy51ZzStYe3k= 74 | modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs= 75 | modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= 76 | modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= 77 | modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= 78 | modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= 79 | modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= 80 | -------------------------------------------------------------------------------- /kvass.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | kvass "github.com/maxmunzel/kvass/src" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | app := kvass.GetApp() 10 | os.Exit(app.Run(os.Args, os.Stdout)) 11 | } 12 | -------------------------------------------------------------------------------- /src/cli.go: -------------------------------------------------------------------------------- 1 | package kvass 2 | 3 | import ( 4 | qr "github.com/skip2/go-qrcode" 5 | 6 | "bytes" 7 | "encoding/hex" 8 | "fmt" 9 | "github.com/lizebang/qrcode-terminal" 10 | "github.com/teris-io/cli" 11 | "io" 12 | "io/ioutil" 13 | "log" 14 | "math" 15 | "net/url" 16 | "os" 17 | "path" 18 | "strconv" 19 | "strings" 20 | ) 21 | 22 | func getPersistance(options map[string]string) *SqlitePersistance { 23 | dbpath, contains := options["db"] 24 | if !contains { 25 | 26 | defaultFilename := ".kvassdb.sqlite" 27 | home, err := os.UserHomeDir() 28 | if err != nil { 29 | panic(err) 30 | } 31 | dbpath = path.Join(home, defaultFilename) 32 | } 33 | 34 | p, err := NewSqlitePersistance(dbpath) 35 | if err != nil { 36 | panic(err) 37 | } 38 | return p 39 | } 40 | func GetApp() cli.App { 41 | logger := log.New(os.Stderr, "", log.Llongfile|log.LstdFlags) 42 | ls := cli.NewCommand("ls", "list keys"). 43 | WithAction(func(args []string, options map[string]string) int { 44 | p := getPersistance(options) 45 | defer p.Close() 46 | 47 | err := p.GetRemoteUpdates() 48 | if err != nil { 49 | logger.Println("Couldn't get updates from server. ", err) 50 | } 51 | 52 | keys, err := p.GetKeys() 53 | if err != nil { 54 | panic(err) 55 | } 56 | 57 | for _, k := range keys { 58 | fmt.Println(k) 59 | } 60 | 61 | return 0 62 | }) 63 | get := cli.NewCommand("get", "get a value"). 64 | WithArg(cli.NewArg("key", "the key to get")). 65 | WithAction(func(args []string, options map[string]string) int { 66 | key := args[0] 67 | p := getPersistance(options) 68 | defer p.Close() 69 | 70 | err := p.GetRemoteUpdates() 71 | if err != nil { 72 | logger.Println("Couldn't get updates from server. ", err) 73 | } 74 | val, err := p.GetEntry(key) 75 | if err != nil { 76 | panic(err) 77 | } 78 | 79 | if val == nil { 80 | println() 81 | return 0 // no entry 82 | } 83 | 84 | _, err = io.Copy(os.Stdout, bytes.NewBuffer(val.Value)) 85 | if err != nil { 86 | panic(err) 87 | } 88 | return 0 89 | }) 90 | 91 | set := cli.NewCommand("set", "set a value"). 92 | WithArg(cli.NewArg("key", "the key to set")). 93 | WithArg(cli.NewArg("value", "the value to set (ommit for stdin)").AsOptional()). 94 | WithAction(func(args []string, options map[string]string) int { 95 | key := args[0] 96 | 97 | p := getPersistance(options) 98 | defer p.Close() 99 | 100 | var err error 101 | var val []byte 102 | 103 | if len(args) < 2 { 104 | valBytes, err := ioutil.ReadAll(os.Stdin) 105 | val = valBytes 106 | if err != nil { 107 | panic(err) 108 | } 109 | 110 | } else { 111 | val = []byte(args[1] + "\n") 112 | } 113 | 114 | err = Set(p, key, []byte(val)) 115 | if err != nil { 116 | panic(err) 117 | } 118 | 119 | if err = p.Push(); err != nil { 120 | fmt.Println("Could not push changes to server: ", err) 121 | return 1 122 | } 123 | return 0 124 | }) 125 | 126 | rm := cli.NewCommand("rm", "remove a key"). 127 | WithArg(cli.NewArg("key", "the key to remove")). 128 | WithAction(func(args []string, options map[string]string) int { 129 | key := args[0] 130 | 131 | p := getPersistance(options) 132 | defer p.Close() 133 | 134 | err := Delete(p, key) 135 | if err != nil { 136 | panic(err) 137 | } 138 | 139 | if err = p.Push(); err != nil { 140 | fmt.Println("Could not push changes to server: ", err) 141 | return 1 142 | } 143 | return 0 144 | }) 145 | 146 | serve := cli.NewCommand("serve", "start in server mode [--bind=\"ip:port\" (default: 0.0.0.0:8000)]"). 147 | WithOption(cli.NewOption("bind", "bind address (default: \"0.0.0.0:8000\" meaning all interfaces, port 8000)")). 148 | WithAction(func(args []string, options map[string]string) int { 149 | bind, contains := options["bind"] 150 | if !contains { 151 | bind = "0.0.0.0:8000" 152 | } 153 | p := getPersistance(options) 154 | defer p.Close() 155 | RunServer(p, bind) 156 | return 0 157 | }) 158 | 159 | config_show := cli.NewCommand("show", "print current config"). 160 | WithAction(func(args []string, options map[string]string) int { 161 | p := getPersistance(options) 162 | remote := p.State.RemoteHostname 163 | if remote == "" { 164 | remote = "(None)" 165 | } 166 | 167 | fmt.Printf("Encryption Key: \t%v\n", p.State.Key) 168 | fmt.Printf("ProcessID: \t%v\n", p.State.Pid) 169 | fmt.Printf("Remote: \t%v\n", remote) 170 | return 0 171 | }) 172 | 173 | config_key := cli.NewCommand("key", "set encryption key"). 174 | WithArg(cli.NewArg("key", "the hex-encoded enryption key")). 175 | WithAction(func(args []string, options map[string]string) int { 176 | key_hex := args[0] 177 | key, err := hex.DecodeString(strings.TrimSpace(key_hex)) 178 | if err != nil { 179 | fmt.Println("Error, could not decode supplied key.") 180 | return 1 181 | } 182 | 183 | if len(key) != 32 { 184 | fmt.Println("Error, key has to be 32 bytes long.") 185 | return 1 186 | } 187 | 188 | p := getPersistance(options) 189 | p.State.Key = key_hex 190 | err = p.CommitState() 191 | if err != nil { 192 | fmt.Println("Internal error: ", err.Error()) 193 | return 1 194 | } 195 | 196 | return 0 197 | }) 198 | 199 | config_pid := cli.NewCommand("pid", "set process id (lower pid wins in case of conflicts"). 200 | WithArg(cli.NewArg("id", "the new process id.").WithType(cli.TypeInt)). 201 | WithAction(func(args []string, options map[string]string) int { 202 | pid64, err := strconv.ParseInt(args[0], 10, 64) 203 | 204 | if err != nil { 205 | // should never happen, as cli lib does type checking. 206 | panic(err) 207 | } 208 | if pid64 <= 0 || pid64 > math.MaxUint32 { 209 | // fmt.Println("PID has to be in [1,", math.MaxUint32, "] (inclusive).") // does not compile for 32 bit targets -.- 210 | fmt.Println("PID has to be in [1,4294967295] (inclusive).") 211 | return 1 212 | } 213 | pid := uint32(pid64) 214 | 215 | p := getPersistance(options) 216 | p.State.Pid = pid 217 | err = p.CommitState() 218 | if err != nil { 219 | fmt.Println("Internal error: ", err.Error()) 220 | return 1 221 | } 222 | 223 | return 0 224 | }) 225 | 226 | config_remote := cli.NewCommand("remote", "set remote server"). 227 | WithArg(cli.NewArg("host", `example: "1.2.3.4:4242", "" means using no remote`)). 228 | WithAction(func(args []string, options map[string]string) int { 229 | host := strings.TrimSpace(args[0]) 230 | 231 | p := getPersistance(options) 232 | 233 | url, err := url.ParseRequestURI(host) 234 | if err != nil { 235 | p.State.RemoteHostname = "http://" + host 236 | } else { 237 | p.State.RemoteHostname = url.String() 238 | } 239 | 240 | err = p.CommitState() 241 | if err != nil { 242 | fmt.Println("Internal error: ", err.Error()) 243 | return 1 244 | } 245 | 246 | return 0 247 | }) 248 | 249 | config := cli.NewCommand("config", "set config parameters"). 250 | WithCommand(config_show). 251 | WithCommand(config_key). 252 | WithCommand(config_remote). 253 | WithCommand(config_pid) 254 | 255 | url := cli.NewCommand("url", "show shareable url of an entry"). 256 | WithArg(cli.NewArg("key", "the key of your entry")). 257 | WithAction(func(args []string, options map[string]string) int { 258 | key := args[0] 259 | p := getPersistance(options) 260 | entry, err := p.GetEntry(key) 261 | if err != nil { 262 | panic(err) 263 | } 264 | 265 | if entry == nil { 266 | logger.Fatal("Key not found.") 267 | } 268 | 269 | fmt.Println(p.State.RemoteHostname + "/get?q=" + entry.UrlToken) 270 | return 0 271 | }) 272 | qr := cli.NewCommand("qr", "print shareable qr code of entry to console"). 273 | WithArg(cli.NewArg("key", "the key of your entry")). 274 | WithAction(func(args []string, options map[string]string) int { 275 | key := args[0] 276 | p := getPersistance(options) 277 | defer p.Close() 278 | 279 | err := p.GetRemoteUpdates() 280 | if err != nil { 281 | logger.Println("Couldn't get updates from server. ", err) 282 | } 283 | entry, err := p.GetEntry(key) 284 | if err != nil { 285 | panic(err) 286 | } 287 | 288 | if entry == nil { 289 | logger.Fatal("Key not found.") 290 | } 291 | 292 | url := p.State.RemoteHostname + "/get?q=" + entry.UrlToken 293 | 294 | qrcode.QRCode(url, qrcode.BrightBlack, qrcode.BrightWhite, qr.Low) 295 | 296 | return 0 297 | }) 298 | 299 | app := cli.New("- a personal KV store"). 300 | WithOption(cli.NewOption("db", "the database file to use (default: ~/.b.sqlite)")). 301 | WithCommand(ls). 302 | WithCommand(get). 303 | WithCommand(set). 304 | WithCommand(rm). 305 | WithCommand(url). 306 | WithCommand(qr). 307 | WithCommand(config). 308 | WithCommand(serve) 309 | return app 310 | 311 | } 312 | -------------------------------------------------------------------------------- /src/cli_test.go: -------------------------------------------------------------------------------- 1 | package kvass 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | const SERVER_PORT = "34521" 11 | const REMOTE_DB = "test_remote.sqlite" 12 | const LOCAL_DB = "test_local.sqlite" 13 | const KEY = "cd9994e3a0c5ca40c2652ff21ab401d6d3a1a5c5fd5482ae62c03ce32d340b49" 14 | 15 | func TestCli(t *testing.T) { 16 | fail := func(err error) { 17 | if err != nil { 18 | t.FailNow() 19 | } 20 | } 21 | 22 | assertKeyEqualsVal := func(db string, key string, value []byte) { 23 | instance, err := NewSqlitePersistance(db) 24 | fail(err) 25 | 26 | instanceEntry, err := instance.GetEntry(key) 27 | fail(err) 28 | if instanceEntry == nil { 29 | t.Errorf("%s instance did not have a value for key %s.", db, key) 30 | } else if !bytes.Equal(value, instanceEntry.Value) { 31 | t.Errorf("%s instance had unexpected value for key %s. \nExpected: '%s'\nActual: '%s'", db, key, value, instanceEntry.Value) 32 | } 33 | } 34 | os.Remove(LOCAL_DB) 35 | os.Remove(REMOTE_DB) 36 | t.Cleanup(func() { os.Remove(REMOTE_DB) }) 37 | t.Cleanup(func() { os.Remove(LOCAL_DB) }) 38 | 39 | for _, db := range []string{LOCAL_DB, REMOTE_DB} { 40 | GetApp().Run([]string{ 41 | "kvass", 42 | "config", 43 | "key", 44 | "--db=" + db, 45 | KEY, 46 | }, io.Discard) 47 | } 48 | 49 | // TODO set keys 50 | go GetApp().Run([]string{ 51 | "kvass", 52 | "serve", 53 | "--db=" + REMOTE_DB, 54 | "--bind=localhost:" + SERVER_PORT, 55 | }, io.Discard) 56 | 57 | GetApp().Run([]string{ 58 | "kvass", 59 | "config", 60 | "remote", 61 | "--db=" + LOCAL_DB, 62 | "http://localhost:" + SERVER_PORT, 63 | }, io.Discard) 64 | 65 | GetApp().Run([]string{ 66 | "kvass", 67 | "set", 68 | "--db=" + LOCAL_DB, 69 | "testkey", 70 | "testval", 71 | }, io.Discard) 72 | 73 | assertKeyEqualsVal(LOCAL_DB, "testkey", []byte("testval\n")) 74 | assertKeyEqualsVal(REMOTE_DB, "testkey", []byte("testval\n")) 75 | 76 | GetApp().Run([]string{ 77 | "kvass", 78 | "set", 79 | "--db=" + REMOTE_DB, 80 | "testkey", 81 | "OVERWRITTEN", 82 | }, io.Discard) 83 | 84 | GetApp().Run([]string{ 85 | "kvass", 86 | "get", 87 | "--db=" + LOCAL_DB, 88 | "testkey", 89 | }, io.Discard) 90 | 91 | assertKeyEqualsVal(LOCAL_DB, "testkey", []byte("OVERWRITTEN\n")) 92 | assertKeyEqualsVal(REMOTE_DB, "testkey", []byte("OVERWRITTEN\n")) 93 | 94 | //remote, err := NewSqlitePersistance(REMOTE_DB) 95 | //fail(err) 96 | 97 | //remoteEntry, err := remote.GetEntry("testkey") 98 | //fail(err) 99 | 100 | } 101 | -------------------------------------------------------------------------------- /src/crypto_test.go: -------------------------------------------------------------------------------- 1 | package kvass 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func TestCrypto(t *testing.T) { 9 | t.Parallel() 10 | p, err := NewSqlitePersistance(":memory:") 11 | if err != nil { 12 | t.Error(err) 13 | } 14 | text := []byte("Hello Crypto!") 15 | enc, err := p.Encrypt(text) 16 | if err != nil { 17 | t.Error(err) 18 | } 19 | dec, err := p.DecryptData(enc) 20 | if err != nil { 21 | t.Error(err) 22 | } 23 | 24 | if !bytes.Equal(text, dec) { 25 | t.Error("Payload changed!") 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/model.go: -------------------------------------------------------------------------------- 1 | package kvass 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/base64" 6 | "time" 7 | ) 8 | 9 | const ReservedProcessID = 0 10 | 11 | type KvEntry struct { 12 | Key string 13 | Value []byte // empty slice means key deleted 14 | UrlToken string // random token used for url 15 | 16 | // The following fields are used for state merging 17 | TimestampUnixMicro int64 18 | ProcessID uint32 // randomly chosen for each node 19 | Counter uint64 // lamport clock 20 | NeedsToBePushed bool 21 | } 22 | 23 | func (e KvEntry) isGreaterOrEqualThan(other KvEntry) bool { 24 | // compares two KvEntrys based on their (time, counter, - pid) tuple 25 | 26 | if e.TimestampUnixMicro > other.TimestampUnixMicro { 27 | return true 28 | } 29 | if e.TimestampUnixMicro < other.TimestampUnixMicro { 30 | return false 31 | } 32 | if e.Counter > other.Counter { 33 | return true 34 | } 35 | if e.Counter < other.Counter { 36 | return false 37 | } 38 | 39 | if e.ProcessID < other.ProcessID { 40 | return true 41 | } 42 | 43 | if e.ProcessID > other.ProcessID { 44 | return false 45 | } 46 | 47 | // non of the fields differ -> they are equal 48 | return true 49 | } 50 | 51 | func (e KvEntry) Max(other KvEntry) KvEntry { 52 | if e.isGreaterOrEqualThan(other) { 53 | return e 54 | } 55 | return other 56 | 57 | } 58 | 59 | func Delete(p *SqlitePersistance, key string) error { 60 | return Set(p, key, []byte("")) 61 | } 62 | func Set(p *SqlitePersistance, key string, value []byte) error { 63 | pid, err := p.GetProcessID() 64 | if err != nil { 65 | return err 66 | } 67 | t := time.Now().UnixMicro() 68 | 69 | count, err := p.GetCounter() 70 | if err != nil { 71 | return err 72 | } 73 | 74 | url_bytes := make([]byte, 16) 75 | _, err = rand.Read(url_bytes) 76 | if err != nil { 77 | panic(err) 78 | } 79 | 80 | err = p.UpdateOn(KvEntry{ 81 | ProcessID: pid, 82 | TimestampUnixMicro: t, 83 | Key: key, 84 | Value: value, 85 | Counter: count, 86 | UrlToken: base64.RawURLEncoding.EncodeToString(url_bytes), 87 | NeedsToBePushed: true, 88 | }) 89 | return err 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/server.go: -------------------------------------------------------------------------------- 1 | package kvass 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "encoding/json" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "os" 12 | "strings" 13 | ) 14 | 15 | func RunServer(p *SqlitePersistance, bind string) { 16 | logger := log.New(os.Stdout, "", log.LstdFlags) 17 | 18 | http.HandleFunc("/push", func(w http.ResponseWriter, r *http.Request) { 19 | payload, err := ioutil.ReadAll(r.Body) 20 | if err != nil { 21 | http.Error(w, err.Error(), 400) 22 | return 23 | } 24 | payload, err = p.DecryptData(payload) 25 | if err != nil { 26 | http.Error(w, err.Error(), 400) 27 | return 28 | } 29 | 30 | entries := make([]KvEntry, 0) 31 | err = json.Unmarshal(payload, &entries) 32 | if err != nil { 33 | http.Error(w, err.Error(), 400) 34 | return 35 | } 36 | 37 | for _, e := range entries { 38 | p.UpdateOn(e) 39 | } 40 | 41 | }) 42 | http.HandleFunc("/pull", func(w http.ResponseWriter, r *http.Request) { 43 | payload_enc, err := ioutil.ReadAll(r.Body) 44 | if err != nil { 45 | http.Error(w, err.Error(), 400) 46 | return 47 | } 48 | 49 | payload, err := p.DecryptData(payload_enc) 50 | if err != nil { 51 | http.Error(w, err.Error(), 400) 52 | return 53 | } 54 | 55 | updateRequest := UpdateRequest{} 56 | err = json.Unmarshal(payload, &updateRequest) 57 | if err != nil { 58 | http.Error(w, err.Error(), 400) 59 | return 60 | } 61 | 62 | updates, err := p.GetUpdates(updateRequest) 63 | if err != nil { 64 | http.Error(w, err.Error(), 500) 65 | return 66 | } 67 | 68 | response_payload, err := json.MarshalIndent(updates, "", " ") 69 | if err != nil { 70 | http.Error(w, err.Error(), 500) 71 | return 72 | } 73 | response_payload, err = p.Encrypt(response_payload) 74 | 75 | if err != nil { 76 | http.Error(w, err.Error(), 500) 77 | return 78 | } 79 | 80 | w.Write(response_payload) 81 | }) 82 | http.HandleFunc("/get", func(w http.ResponseWriter, r *http.Request) { 83 | file := r.URL.Query().Get("q") 84 | 85 | if file == "" { 86 | http.Error(w, "Please specify file", 400) 87 | return 88 | } 89 | 90 | row := p.db.QueryRow("select key from entries where urltoken = ?;", file) 91 | var key string 92 | err := row.Scan(&key) 93 | if err == sql.ErrNoRows { 94 | http.Error(w, "Unknown File", 404) 95 | return 96 | } 97 | 98 | entry, err := p.GetEntry(key) 99 | if err != nil { 100 | http.Error(w, err.Error(), 500) 101 | } 102 | 103 | if entry == nil { 104 | http.Error(w, "Aaaaand it's gone", 419) // entry was deleted since we got the key 105 | return 106 | } 107 | 108 | if strings.HasSuffix(key, ".html") { 109 | r.Header.Add("Content-Type", "application/html") 110 | } 111 | 112 | io.Copy(w, bytes.NewBuffer(entry.Value)) 113 | 114 | }) 115 | 116 | logger.Printf("Server started and listening on %v\n", bind) 117 | panic(http.ListenAndServe(bind, nil)) 118 | } 119 | -------------------------------------------------------------------------------- /src/sqlite_persistance.go: -------------------------------------------------------------------------------- 1 | package kvass 2 | 3 | import ( 4 | "bytes" 5 | "crypto/aes" 6 | "crypto/cipher" 7 | "crypto/rand" 8 | "database/sql" 9 | "encoding/hex" 10 | "encoding/json" 11 | "errors" 12 | "fmt" 13 | "io" 14 | "io/ioutil" 15 | "math" 16 | "math/big" 17 | "modernc.org/mathutil" 18 | _ "modernc.org/sqlite" 19 | "net/http" 20 | "os" 21 | ) 22 | 23 | type SqliteState struct { 24 | Counter uint64 25 | Pid uint32 26 | Key string 27 | RemoteHostname string 28 | RemoteCounter uint64 29 | SchemaVersion uint32 30 | } 31 | 32 | type SqlitePersistance struct { 33 | path string 34 | db *sql.DB 35 | State SqliteState 36 | } 37 | 38 | func (p *SqlitePersistance) GetRemoteUpdates() (err error) { 39 | if p.State.RemoteHostname == "" { 40 | return nil 41 | } 42 | request, err := json.Marshal(UpdateRequest{ProcessID: p.State.Pid, Counter: p.State.RemoteCounter}) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | request, err = p.Encrypt(request) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | resp, err := http.Post(p.State.RemoteHostname+"/pull", "application/json", bytes.NewReader(request)) 53 | if err != nil { 54 | return err 55 | } 56 | body, err := ioutil.ReadAll(resp.Body) 57 | if err != nil { 58 | return err 59 | } 60 | body, err = p.DecryptData(body) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | updates := make([]KvEntry, 0) 66 | err = json.Unmarshal(body, &updates) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | for _, u := range updates { 72 | u.NeedsToBePushed = false 73 | err = p.UpdateOn(u) 74 | if err != nil { 75 | return err 76 | } 77 | } 78 | return nil 79 | 80 | } 81 | 82 | func (p *SqlitePersistance) Push() error { 83 | // push changes to remote 84 | 85 | host := p.State.RemoteHostname 86 | if host == "" { 87 | return nil 88 | } 89 | updates, err := p.GetUpdates(UpdateRequest{Counter: p.State.RemoteCounter, ProcessID: ReservedProcessID}) 90 | if err != nil { 91 | panic(err) 92 | } 93 | payload, err := json.Marshal(updates) 94 | if err != nil { 95 | panic(err) 96 | } 97 | payload, err = p.Encrypt(payload) 98 | if err != nil { 99 | panic(err) 100 | } 101 | 102 | resp, err := http.DefaultClient.Post(host+"/push", "application/json", bytes.NewReader(payload)) 103 | if err != nil || resp.StatusCode != 200 { 104 | return fmt.Errorf("Error posting update to server: %v", err) 105 | } else { 106 | for _, u := range updates { 107 | p.db.Exec("update entries set NeedsToBePushed = 0 where key = ?", u.Key) 108 | if err != nil { 109 | panic(err) 110 | } 111 | } 112 | } 113 | return nil 114 | } 115 | 116 | func (s *SqlitePersistance) CommitState() error { 117 | // saves the internal state to the sqlite db 118 | state, err := json.MarshalIndent(&s.State, "", " ") 119 | if err != nil { 120 | return err 121 | } 122 | 123 | tx, err := s.db.Begin() 124 | 125 | defer tx.Rollback() 126 | if err != nil { 127 | return err 128 | } 129 | 130 | _, err = tx.Exec("DELETE from state;") 131 | if err != nil { 132 | return err 133 | } 134 | _, err = tx.Exec("INSERT into state values (?);", state) 135 | if err != nil { 136 | return err 137 | } 138 | return tx.Commit() 139 | } 140 | 141 | func (s *SqlitePersistance) Close() error { 142 | return s.db.Close() 143 | } 144 | 145 | func NewSqlitePersistance(path string) (*SqlitePersistance, error) { 146 | db, err := sql.Open("sqlite", path) 147 | if err != nil { 148 | return nil, err 149 | } 150 | 151 | // check if DB has the expected tables 152 | 153 | persistance := &SqlitePersistance{ 154 | path: path, 155 | db: db, 156 | } 157 | 158 | if _, err := os.Stat(path); err == nil { 159 | 160 | // load state and return 161 | row := db.QueryRow("select * from state;") 162 | var state_json []byte 163 | err := row.Scan(&state_json) 164 | if err != nil { 165 | return nil, err 166 | } 167 | 168 | err = json.Unmarshal(state_json, &persistance.State) 169 | if err != nil { 170 | return nil, err 171 | } 172 | 173 | } else { 174 | // init DB 175 | _, err := db.Exec(` 176 | create table if not exists entries (key, value, timestamp, pid, counter, urltoken); 177 | create table if not exists state (state);`) 178 | 179 | if err != nil { 180 | return nil, err 181 | } 182 | pid_, err := rand.Int(rand.Reader, big.NewInt(math.MaxUint32)) 183 | persistance.State.Pid = uint32(pid_.Int64() + 1) 184 | if err != nil { 185 | panic(err) 186 | } 187 | 188 | // generate key 189 | key := make([]byte, 32) 190 | _, err = io.ReadFull(rand.Reader, key) 191 | if err != nil { 192 | return nil, err 193 | } 194 | persistance.State.Key = hex.EncodeToString(key) 195 | 196 | err = persistance.CommitState() 197 | if err != nil { 198 | return nil, err 199 | } 200 | } 201 | 202 | // schema migrations 203 | if persistance.State.SchemaVersion == 0 { 204 | _, err = db.Exec(`alter table entries add column 'NeedsToBePushed' boolean default true;`) 205 | if err != nil { 206 | return nil, err 207 | } 208 | 209 | persistance.State.SchemaVersion = 1 210 | err = persistance.CommitState() 211 | if err != nil { 212 | return nil, err 213 | } 214 | } 215 | return persistance, nil 216 | } 217 | 218 | func (s *SqlitePersistance) DecryptData(data []byte) ([]byte, error) { 219 | key, err := hex.DecodeString(s.State.Key) 220 | if err != nil { 221 | return nil, err 222 | } 223 | 224 | if len(key) != 32 { 225 | return nil, errors.New("Invalid key length.") 226 | } 227 | 228 | block, err := aes.NewCipher(key) 229 | if err != nil { 230 | return nil, err 231 | } 232 | gcm, err := cipher.NewGCM(block) 233 | if err != nil { 234 | return nil, err 235 | } 236 | 237 | if len(data) < gcm.NonceSize() { 238 | return nil, errors.New("Data too short!") 239 | } 240 | 241 | nonce := data[:gcm.NonceSize()] 242 | ciphertext := data[gcm.NonceSize():] 243 | 244 | return gcm.Open(nil, nonce, ciphertext, nil) 245 | } 246 | func (s *SqlitePersistance) Encrypt(data []byte) ([]byte, error) { 247 | key, err := hex.DecodeString(s.State.Key) 248 | if err != nil { 249 | return nil, err 250 | } 251 | 252 | if len(key) != 32 { 253 | return nil, errors.New("Invalid key length.") 254 | } 255 | 256 | block, err := aes.NewCipher(key) 257 | if err != nil { 258 | return nil, err 259 | } 260 | gcm, err := cipher.NewGCM(block) 261 | if err != nil { 262 | return nil, err 263 | } 264 | 265 | nonce := make([]byte, gcm.NonceSize()) 266 | _, err = io.ReadFull(rand.Reader, nonce) 267 | if err != nil { 268 | return nil, err 269 | } 270 | 271 | ciphertext := gcm.Seal(nil, nonce, data, nil) 272 | 273 | result := append(nonce, ciphertext...) 274 | return result, nil 275 | 276 | } 277 | func (s *SqlitePersistance) GetProcessID() (uint32, error) { 278 | return s.State.Pid, nil 279 | } 280 | func (s *SqlitePersistance) GetCounter() (uint64, error) { 281 | return s.State.Counter, nil 282 | } 283 | 284 | type UpdateRequest struct { 285 | Counter uint64 286 | ProcessID uint32 // ignore updates from this pid 287 | } 288 | 289 | func (s *SqlitePersistance) GetUpdates(req UpdateRequest) ([]KvEntry, error) { 290 | result := make([]KvEntry, 0) 291 | rows, err := s.db.Query("select * from entries where counter >= ? and pid != ? and NeedsToBePushed = 1;", req.Counter, req.ProcessID) 292 | if err != nil { 293 | return nil, err 294 | } 295 | defer rows.Close() 296 | 297 | for rows.Next() { 298 | var entry KvEntry 299 | err = rows.Scan(&entry.Key, &entry.Value, &entry.TimestampUnixMicro, &entry.ProcessID, &entry.Counter, &entry.UrlToken, &entry.NeedsToBePushed) 300 | if err != nil { 301 | return nil, err 302 | } 303 | result = append(result, entry) 304 | } 305 | 306 | if err := rows.Err(); err != nil { 307 | return nil, err 308 | } 309 | return result, nil 310 | 311 | } 312 | func (s *SqlitePersistance) UpdateOn(entry KvEntry) error { 313 | // get current entry from db 314 | tx, err := s.db.Begin() 315 | if err != nil { 316 | return err 317 | } 318 | defer tx.Rollback() 319 | var oldEntry KvEntry 320 | row := tx.QueryRow("select * from entries order by timestamp desc, counter desc, pid asc where key = ? limit 1;", entry.Key) 321 | err = row.Scan(&oldEntry.Key, &oldEntry.Value, &oldEntry.TimestampUnixMicro, &oldEntry.ProcessID, &oldEntry.Counter, &oldEntry.UrlToken, &entry.NeedsToBePushed) 322 | if err != nil { 323 | // no result 324 | oldEntry = entry 325 | } 326 | 327 | // if the update came from the remote, update the remote counter 328 | pid, err := s.GetProcessID() 329 | if err != nil { 330 | return err 331 | } 332 | if entry.ProcessID != pid { 333 | s.State.RemoteCounter = mathutil.MaxUint64(s.State.RemoteCounter, entry.Counter) 334 | } 335 | 336 | // select LUB of old and new entry 337 | entry = entry.Max(oldEntry) 338 | 339 | // update local counter 340 | newCounter := mathutil.MaxUint64(entry.Counter, s.State.Counter) + 1 341 | s.State.Counter = newCounter 342 | 343 | // set new entries counter 344 | entry.Counter = newCounter 345 | 346 | // write back LUB to db 347 | _, err = tx.Exec("delete from entries where key = ?;", entry.Key) 348 | if err != nil { 349 | return err 350 | } 351 | 352 | _, err = tx.Exec("insert into entries values (?, ?, ?, ?, ?, ?, ?);", 353 | entry.Key, 354 | entry.Value, 355 | entry.TimestampUnixMicro, 356 | entry.ProcessID, 357 | entry.Counter, 358 | entry.UrlToken, 359 | entry.NeedsToBePushed, 360 | ) 361 | if err != nil { 362 | return err 363 | } 364 | 365 | err = tx.Commit() 366 | if err != nil { 367 | return err 368 | } 369 | return s.CommitState() 370 | 371 | } 372 | func (s *SqlitePersistance) GetKeys() ([]string, error) { 373 | result := make([]string, 0) 374 | 375 | rows, err := s.db.Query("select distinct key from entries where length(value) != 0 order by key asc;") 376 | if err != nil { 377 | return result, err 378 | } 379 | var entry string 380 | 381 | for rows.Next() { 382 | err := rows.Scan(&entry) 383 | if err != nil { 384 | return result, err 385 | } 386 | result = append(result, entry) 387 | } 388 | return result, nil 389 | } 390 | func (s *SqlitePersistance) GetEntry(key string) (*KvEntry, error) { 391 | 392 | row := s.db.QueryRow("select * from entries where key = ? order by timestamp desc, pid desc, counter desc limit 1;", key) 393 | entry := KvEntry{} 394 | err := row.Scan(&entry.Key, &entry.Value, &entry.TimestampUnixMicro, &entry.ProcessID, &entry.Counter, &entry.UrlToken, &entry.NeedsToBePushed) 395 | if err != nil { 396 | if err == sql.ErrNoRows { 397 | return nil, nil 398 | } 399 | return nil, err 400 | } 401 | return &entry, nil 402 | } 403 | -------------------------------------------------------------------------------- /src/sqlite_persistance_test.go: -------------------------------------------------------------------------------- 1 | package kvass 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestUpdates(t *testing.T) { 9 | t.Parallel() 10 | 11 | p, _ := NewSqlitePersistance(":memory:") 12 | for i := 0; i < 100; i += 1 { 13 | err := Set(p, "test", []byte(fmt.Sprint(i))) 14 | if err != nil { 15 | panic(err) 16 | } 17 | Set(p, "foo", []byte("bar")) 18 | } 19 | 20 | val, err := p.GetEntry("test") 21 | if err != nil { 22 | t.Errorf(err.Error()) 23 | } 24 | if string(val.Value) != "99" { 25 | t.Error("key did not properly update") 26 | } 27 | 28 | val, err = p.GetEntry("foo") 29 | if err != nil { 30 | t.Errorf(err.Error()) 31 | } 32 | if string(val.Value) != "bar" { 33 | t.Error("key did not properly update") 34 | } 35 | val, err = p.GetEntry("nonexistent") 36 | if err != nil { 37 | t.Errorf(err.Error()) 38 | } 39 | if val != nil { 40 | t.Error("unset key had unexpected value") 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/sync_test.go: -------------------------------------------------------------------------------- 1 | package kvass 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func TestPush(t *testing.T) { 9 | // create two instances client and remote, set a key on the client, 10 | // apply its updates on the remote and check if the remote also set the 11 | // key. 12 | t.Parallel() 13 | fail := func(err error) { 14 | if err != nil { 15 | t.Error(err) 16 | } 17 | } 18 | client, err := NewSqlitePersistance(":memory:") 19 | defer client.Close() 20 | fail(err) 21 | 22 | remote, err := NewSqlitePersistance(":memory:") 23 | fail(err) 24 | defer remote.Close() 25 | 26 | Set(client, "foo", []byte("bar")) 27 | 28 | updates, err := client.GetUpdates(UpdateRequest{ 29 | Counter: client.State.RemoteCounter, 30 | ProcessID: ReservedProcessID, 31 | }) 32 | fail(err) 33 | 34 | for _, update := range updates { 35 | fail(remote.UpdateOn(update)) 36 | } 37 | 38 | entry, err := remote.GetEntry("foo") 39 | fail(err) 40 | 41 | if !bytes.Equal(entry.Value, []byte("bar")) { 42 | t.Error("Remote did not get the correct value.") 43 | } 44 | 45 | } 46 | --------------------------------------------------------------------------------