├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── tailetc.go └── tailetc_test.go /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Tailscale Inc. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Tailscale Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tailetc 2 | 3 | See docs: https://godoc.org/github.com/tailscale/tailetc 4 | 5 | And blog post https://tailscale.com/blog/an-unlikely-database-migration/. 6 | 7 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tailscale/tailetc 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/google/go-cmp v0.4.0 7 | go.etcd.io/etcd v0.5.0-alpha.5.0.20201125193152-8a03d2e9614b 8 | tailscale.com v1.2.10 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/Masterminds/semver/v3 v3.0.3/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= 5 | github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= 6 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 7 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 8 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 9 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 10 | github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5/go.mod h1:976q2ETgjT2snVCf2ZaBnyBbVoPERGjUz+0sofzEfro= 11 | github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= 12 | github.com/apenwarr/fixconsole v0.0.0-20191012055117-5a9f6489cc29/go.mod h1:JYWahgHer+Z2xbsgHPtaDYVWzeHDminu+YIBWkxpCAY= 13 | github.com/apenwarr/w32 v0.0.0-20190407065021-aa00fece76ab/go.mod h1:nfFtvHn2Hgs9G1u0/J6LHQv//EksNC+7G8vXmd1VTJ8= 14 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 15 | github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= 16 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 17 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 18 | github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= 19 | github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A= 20 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 21 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 22 | github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= 23 | github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= 24 | github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= 25 | github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= 26 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 27 | github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM= 28 | github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 29 | github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI= 30 | github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 31 | github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= 32 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 33 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 34 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 35 | github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= 36 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 37 | github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= 38 | github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 39 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 40 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 41 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 42 | github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= 43 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 44 | github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= 45 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 46 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 47 | github.com/go-multierror/multierror v1.0.2/go.mod h1:U7SZR/D9jHgt2nkSj8XcbCWdmVM2igraCHQ3HC1HiKY= 48 | github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= 49 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 50 | github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 51 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 52 | github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= 53 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 54 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 55 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 56 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 57 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= 58 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 59 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 60 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 61 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 62 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 63 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 64 | github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= 65 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 66 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 67 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 68 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 69 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 70 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 71 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 72 | github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg= 73 | github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= 74 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 75 | github.com/goreleaser/nfpm v1.1.10/go.mod h1:oOcoGRVwvKIODz57NUfiRwFWGfn00NXdgnn6MrYtO5k= 76 | github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8= 77 | github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 78 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= 79 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 80 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= 81 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 82 | github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= 83 | github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 84 | github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 85 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 86 | github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= 87 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 88 | github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= 89 | github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= 90 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 91 | github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= 92 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 93 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 94 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 95 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 96 | github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= 97 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 98 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 99 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 100 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 101 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 102 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 103 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 104 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 105 | github.com/lxn/walk v0.0.0-20191128110447-55ccb3a9f5c1/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= 106 | github.com/lxn/win v0.0.0-20191128105842-2da648fda5b4/go.mod h1:ouWl4wViUNh8tPSIwxTVMuS014WakR1hqvBc2I0bMoA= 107 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 108 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 109 | github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= 110 | github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= 111 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 112 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 113 | github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= 114 | github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= 115 | github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= 116 | github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= 117 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 118 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 119 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 120 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 121 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 122 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 123 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 124 | github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= 125 | github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= 126 | github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= 127 | github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= 128 | github.com/peterbourgon/ff/v2 v2.0.0/go.mod h1:xjwr+t+SjWm4L46fcj/D+Ap+6ME7+HqFzaP22pP5Ggk= 129 | github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= 130 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 131 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 132 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 133 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 134 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 135 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 136 | github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= 137 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 138 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 139 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 140 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= 141 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 142 | github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= 143 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 144 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 145 | github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= 146 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 147 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 148 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 149 | github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I= 150 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 151 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 152 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 153 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 154 | github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= 155 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 156 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 157 | github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= 158 | github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 159 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 160 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 161 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 162 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 163 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 164 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 165 | github.com/tailscale/depaware v0.0.0-20201003033024-5d95aab075be/go.mod h1:jissDaJNHiyV2tFdr3QyNEfsZrax/i2yQiSO+CljThI= 166 | github.com/tailscale/wireguard-go v0.0.0-20201021041318-a6168fd06b3f/go.mod h1:WXq+IkSOJGIgfF1XW+4z4oW+LX/TXzU9DcKlT5EZLi4= 167 | github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8= 168 | github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= 169 | github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 170 | github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM= 171 | github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= 172 | github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 173 | github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= 174 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= 175 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 176 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 177 | go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= 178 | go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 179 | go.etcd.io/etcd v0.5.0-alpha.5.0.20201125193152-8a03d2e9614b h1:PLwvCoe5rvpuo9Un6/hlNRMAfOMVb7zBsOOeKAjV81g= 180 | go.etcd.io/etcd v0.5.0-alpha.5.0.20201125193152-8a03d2e9614b/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= 181 | go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= 182 | go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 183 | go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= 184 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 185 | go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= 186 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 187 | go4.org/mem v0.0.0-20200706164138-185c595c3ecc/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k= 188 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 189 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 190 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 191 | golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 192 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 193 | golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 194 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 195 | golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= 196 | golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 197 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 198 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 199 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 200 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 201 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 202 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 203 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 204 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 205 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 206 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 207 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 208 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 209 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 210 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 211 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 212 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 213 | golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 214 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 215 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 216 | golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 217 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 218 | golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 219 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 220 | golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= 221 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 222 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 223 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 224 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 225 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 226 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 227 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 228 | golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 229 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 230 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 231 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 232 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 233 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 234 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 235 | golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 236 | golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 237 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 238 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 239 | golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 240 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 241 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 242 | golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 243 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 244 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 245 | golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 246 | golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 247 | golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d h1:QQrM/CCYEzTs91GZylDCQjGHudbPTxF/1fvXdVh5lMo= 248 | golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 249 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 250 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 251 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 252 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 253 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 254 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= 255 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 256 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 257 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 258 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 259 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 260 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 261 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 262 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 263 | golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 264 | golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 265 | golang.org/x/tools v0.0.0-20201001230009-b5b87423c93b/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 266 | golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 267 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 268 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 269 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 270 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 271 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 272 | golang.zx2c4.com/wireguard v0.0.20200321-0.20200715051853-507f148e1c42/go.mod h1:GJvYs5O24/ASlwPiRklVnjMx2xQzrOic0DuU6GvYJL4= 273 | golang.zx2c4.com/wireguard/windows v0.1.2-0.20201004085714-dd60d0447f81/go.mod h1:GaK5zcgr5XE98WaRzIDilumDBp5/yP8j2kG/LCDnvAM= 274 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 275 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 276 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 277 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= 278 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 279 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 280 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 281 | google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= 282 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 283 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 284 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 285 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 286 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 287 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 288 | gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= 289 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 290 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 291 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 292 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 293 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 294 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 295 | gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= 296 | gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 297 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 298 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 299 | honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= 300 | inet.af/netaddr v0.0.0-20200810144936-56928fe48a98/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww= 301 | rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= 302 | sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= 303 | sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= 304 | tailscale.com v1.2.10 h1:6/rGwUmZNF7w3RWGlw+3YTyNgxgkDPYqWf1tcLwNtAg= 305 | tailscale.com v1.2.10/go.mod h1:JEJiCce3MHtPCTdX2ahLc4tcnxZ7b5etish1Yt0B6+w= 306 | -------------------------------------------------------------------------------- /tailetc.go: -------------------------------------------------------------------------------- 1 | // Package tailetc implements an total-memory-cache etcd v3 client 2 | // implemented by tailing (watching) an entire etcd. 3 | // 4 | // The client maintains a complete copy of the etcd database in memory, 5 | // with values decoded into Go objects. 6 | // 7 | // 8 | // Transaction Model 9 | // 10 | // It presents a simplified transaction model that assumes that multiple 11 | // writers will not be contending on keys. When you call tx.Get or 12 | // tx.Put, your Tx records the current etcd global revision. 13 | // When you tx.Commit, if some newer revision of any key touched by Tx 14 | // is in etcd then the commit will fail. 15 | // 16 | // Contention failures are reported as ErrTxStale from tx.Commit. 17 | // Failures may be reported sooner as tx.Get or tx.Put errors, but the 18 | // tx error is sticky, that is, if you ignore those errors the eventual 19 | // error from tx.Commit will have ErrTxStale in its error chain. 20 | // 21 | // The Tx.Commit method waits before successfully returning until DB 22 | // has caught up with the etcd global revision of the transaction. 23 | // This ensures that sequential happen in the strict database sequence. 24 | // So if you serve a REST API from a single *etcd.DB instance, then the 25 | // API will behave as users expect it to. 26 | // 27 | // If you know only one thing about this etcd client, know this: 28 | // 29 | // Do not have writer contention on individual keys. 30 | // 31 | // Everything else should follow a programmer's intuition for an 32 | // in-memory map guarded by a RWMutex. 33 | // 34 | // 35 | // Caching 36 | // 37 | // We take advantage of etcd's watch model to maintain a *decoded* 38 | // value cache of the entire database in memory. This means that 39 | // querying a value is extremely cheap. Issuing tx.Get(key) involves 40 | // no more work than holding a mutex read lock, reading from a map, 41 | // and cloning the value. 42 | // 43 | // The etcd watch is then exposed to the user of *etcd.DB via the 44 | // WatchFunc. This lets users maintain in-memory higher-level indexes 45 | // into the etcd database that respond to external commits. 46 | // WatchFunc is called while the database lock is held, so a WatchFunc 47 | // implementation cannot synchronously issue transactions. 48 | // 49 | // 50 | // Object Ownership 51 | // 52 | // The cache in etcd.DB is very careful not to copy objects both into 53 | // and out of the cache so that users of the etcd.DB cannot get pointers 54 | // directly into the memory inside the cache. This means over-copying. 55 | // Users can control precisely how much copying is done, and how, by 56 | // providing a CloneFunc implementation. 57 | // 58 | // The general ownership semantics are: objects are copied as soon as 59 | // they are passed to etcd, and any memory returned by etcd is owned 60 | // by the caller. 61 | package tailetc 62 | 63 | import ( 64 | "context" 65 | "errors" 66 | "expvar" 67 | "fmt" 68 | "log" 69 | "math/rand" 70 | "net/http" 71 | "os" 72 | "path/filepath" 73 | "reflect" 74 | "runtime/pprof" 75 | "sort" 76 | "strings" 77 | "sync" 78 | "sync/atomic" 79 | "time" 80 | 81 | "go.etcd.io/etcd/clientv3" 82 | "go.etcd.io/etcd/embed" 83 | "go.etcd.io/etcd/mvcc/mvccpb" 84 | "go.etcd.io/etcd/pkg/types" 85 | "tailscale.com/syncs" 86 | ) 87 | 88 | // ErrTxStale is reported when another transaction has modified a key 89 | // referenced by this transaction, so it can no longer be applied to 90 | // the database. 91 | var ErrTxStale = errors.New("tx stale") 92 | 93 | // ErrTxClosed is reported when a method is called on a committed or 94 | // canceled Tx. 95 | var ErrTxClosed = errors.New("tx closed") 96 | 97 | // dbMuLockLatency reports the most recent time it took to lock db.Mu. 98 | var dbMuLockLatency = new(expvar.Int) 99 | 100 | func init() { 101 | expvar.Publish("db_mu_lock_latency", dbMuLockLatency) 102 | } 103 | 104 | // DB is a read-write datastore backed by etcd. 105 | type DB struct { 106 | cli *clientv3.Client 107 | opts Options 108 | inMemory bool // entirely in-memory 109 | 110 | done <-chan struct{} 111 | watchCancel func() 112 | shutdownWG sync.WaitGroup // shutdownWG.Add is called under mu when !closing 113 | embedClose func() 114 | 115 | panicOnWrite int32 // testing hook: panic any time a write is requested 116 | 117 | // Mu is the database lock. 118 | // 119 | // Reads are guarded by read locks. 120 | // Transaction commits and background watch updates 121 | // are guarded by write lcoks. 122 | // 123 | // The mutex is exported so that higher-level wrappers that want to 124 | // keep indexes in sync with background updates without problematic 125 | // lock ordering. 126 | Mu sync.RWMutex 127 | 128 | // The following fields are guarded by Mu. 129 | 130 | cache map[string]valueRev // in-memory copy of all etcd key-values 131 | rev rev // rev is the latest known etcd db revision 132 | closing bool // DB.Close called 133 | pending map[rev][]chan struct{} // channels to be closed when rev >= map key 134 | prefixWatchers map[string][]watch // key prefix -> watch funcs 135 | keyWatchers map[string][]watch // key -> watch funcs 136 | } 137 | 138 | // Options are optional settings for a DB. 139 | // 140 | // If one of EncodeFunc, DecodeFunc, and CloneFunc are set they must all be set. 141 | type Options struct { 142 | Logf func(format string, args ...interface{}) 143 | HTTPC *http.Client 144 | // KeyPrefix is a prefix on all etcd keys accessed through this client. 145 | // The value "" means "/", because etcd keys are file-system-like. 146 | KeyPrefix string 147 | // AuthHeader is passed as the "Authorization" header to etcd. 148 | AuthHeader string 149 | // EncodeFunc encodes values for storage. 150 | // If nil, the default encoder produces []byte. 151 | EncodeFunc func(key string, value interface{}) ([]byte, error) 152 | // DecodeFunc decodes values from storage. 153 | // If nil, the default decoder produces []byte. 154 | DecodeFunc func(key string, data []byte) (interface{}, error) 155 | // CloneFunc clones src into dst with no aliased mutable memory. 156 | // The definition of "aliased mutable memory" is left to the user. 157 | // For example, if a user is certain that no values ever passed to or 158 | // read from the etcd package are ever modified, use a no-op CloneFunc. 159 | // If nil, the default requires all values to be []byte. 160 | CloneFunc func(dst interface{}, key string, src interface{}) error 161 | // WatchFunc is called when key-value pairs change in the DB. 162 | // 163 | // When the update is a Tx from this DB, WatchFunc is called after 164 | // the transaction has been successfully applied by the etcd server 165 | // but before the Commit method returns. 166 | // 167 | // The DB.Mu write lock is held for the call, so no transactions 168 | // can be issued from inside WatchFunc. 169 | // 170 | // Entire etcd transactions are single calls to WatchFunc. 171 | // 172 | // The called WatchFunc owns the values passed to it. 173 | WatchFunc func([]KV) 174 | // DeleteAllOnStart deletes all keys when the client is created. 175 | // Used for testing. 176 | DeleteAllOnStart bool 177 | } 178 | 179 | func (opts Options) fillDefaults() (Options, error) { 180 | if opts.HTTPC == nil { 181 | opts.HTTPC = http.DefaultClient 182 | } 183 | if opts.KeyPrefix == "" { 184 | opts.KeyPrefix = "/" 185 | } 186 | if opts.Logf == nil { 187 | opts.Logf = log.Printf 188 | } 189 | if opts.EncodeFunc == nil || opts.DecodeFunc == nil || opts.CloneFunc == nil { 190 | if opts.EncodeFunc != nil || opts.DecodeFunc != nil || opts.CloneFunc != nil { 191 | return opts, fmt.Errorf("etcd: if one of EncodeFunc, DecodeFunc, CloneFunc is set, all must be set") 192 | } 193 | opts.EncodeFunc = func(key string, value interface{}) ([]byte, error) { 194 | if value == nil { 195 | return nil, nil 196 | } 197 | b, isBytes := value.([]byte) 198 | if !isBytes { 199 | return nil, fmt.Errorf("default EncodeFunc requires all values be []byte") 200 | } 201 | b2 := make([]byte, len(b)) 202 | copy(b2, b) 203 | return b2, nil 204 | } 205 | opts.DecodeFunc = func(key string, data []byte) (interface{}, error) { 206 | return data, nil 207 | } 208 | opts.CloneFunc = func(dst interface{}, key string, value interface{}) error { 209 | if value == nil { 210 | return nil 211 | } 212 | b, isBytes := value.([]byte) 213 | if !isBytes { 214 | return fmt.Errorf("default CloneFunc requires all values be []byte") 215 | } 216 | *dst.(*[]byte) = append([]byte(nil), b...) 217 | return nil 218 | } 219 | } 220 | return opts, nil 221 | } 222 | 223 | // KV is a value change for an etcd key. 224 | // Both the old value being replaced and the new value are provided, decoded. 225 | type KV struct { 226 | Key string 227 | OldValue interface{} // nil if there is no old value 228 | Value interface{} // nil if the key has been deleted 229 | } 230 | 231 | // New loads the contents of an etcd prefix range and creates a *DB 232 | // for reading and writing from the prefix range. 233 | // 234 | // The urls parameter is a comma-separated list of etcd HTTP endpoint, 235 | // e.g. "http://1.1.1.1:2379,http://2.2.2.2:2379". 236 | // 237 | // There are two special case values of urls: 238 | // 239 | // If urls is "memory://", the DB does not connect to any etcd server, 240 | // instead all operations are performed on the in-memory cache. 241 | // 242 | // If urls starts with the prefix "file://" then an embedded copy of etcd 243 | // is started and creates the database in a "tailscale.etcd" directory 244 | // under the referenced path. For example, the urls value "file:///data" 245 | // uses an etcd database stored in "/data/tailscale.etcd". 246 | // If only the prefix is provided, that is urls equals "file://", then 247 | // an embedded etcd is started in the current working directory. 248 | func New(ctx context.Context, urls string, opts Options) (db *DB, err error) { 249 | defer func() { 250 | if err != nil { 251 | err = fmt.Errorf("etcd.New: %w", err) 252 | } 253 | }() 254 | 255 | opts, err = opts.fillDefaults() 256 | if err != nil { 257 | return nil, err 258 | } 259 | 260 | db = &DB{ 261 | opts: opts, 262 | cache: map[string]valueRev{}, 263 | pending: map[rev][]chan struct{}{}, 264 | keyWatchers: map[string][]watch{}, 265 | prefixWatchers: map[string][]watch{}, 266 | } 267 | watchCtx, cancel := context.WithCancel(context.Background()) 268 | db.done = watchCtx.Done() 269 | db.watchCancel = cancel 270 | 271 | if urls == "memory://" { 272 | db.inMemory = true 273 | return db, nil 274 | } 275 | var eps []string 276 | if strings.HasPrefix(urls, "file://") { 277 | randURLs, err := types.NewURLs([]string{"http://127.0.0.1:0"}) 278 | if err != nil { 279 | return nil, err 280 | } 281 | 282 | cfg := embed.NewConfig() 283 | cfg.LCUrls = randURLs 284 | cfg.ACUrls = randURLs 285 | cfg.LPUrls = randURLs 286 | cfg.APUrls = randURLs 287 | cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) 288 | cfg.Dir = filepath.Join(strings.TrimPrefix(urls, "file://"), "tailscale.etcd") 289 | cfg.Logger = "zap" // set to avoid data race in the default logger 290 | 291 | if strings.HasPrefix(cfg.Dir, os.TempDir()) { 292 | // Well this is a pickle. 293 | // The tradeoff here is startup time vs. long-running efficiency. 294 | // Etcd does a leader election on startup even in single-node mode, 295 | // and the default value of ElectionMs is 1000 meaning it takes a 296 | // full second to start. That really hurts tests. 297 | // 298 | // But the election timeout must be 5x the heartbeat interval, so 299 | // to do a faster initial election, we have to commit to far more 300 | // frequent heartbeats (which are meaningless in a single-node 301 | // embedded cluster). 302 | // 303 | // So we crank the heartbeats to every 15ms if the data dir is 304 | // in $TMPDIR. The CPU overhead is minimal in tests and the 305 | // wall-time savings are huge. 306 | cfg.TickMs = 15 307 | cfg.ElectionMs = 75 308 | } else { 309 | cfg.TickMs = 50 310 | cfg.ElectionMs = 250 311 | } 312 | 313 | start := time.Now() 314 | e, err := embed.StartEtcd(cfg) 315 | if err != nil { 316 | return nil, fmt.Errorf("embedded server failed to start: %v", err) 317 | } 318 | db.embedClose = e.Close 319 | select { 320 | case <-e.Server.ReadyNotify(): 321 | case <-ctx.Done(): 322 | e.Server.Stop() // trigger a shutdown 323 | return nil, fmt.Errorf("embedded server took too long to start") 324 | } 325 | db.opts.Logf("etcd: embedded server started in %s (election timeout: %dms)", time.Since(start).Round(time.Microsecond), cfg.ElectionMs) 326 | eps = []string{"http://" + e.Clients[0].Addr().String()} 327 | } else { 328 | eps = strings.Split(urls, ",") 329 | } 330 | 331 | db.cli, err = clientv3.New(clientv3.Config{Endpoints: eps}) 332 | if err != nil { 333 | return nil, fmt.Errorf("etcd.New: %v", err) 334 | } 335 | if opts.DeleteAllOnStart { 336 | _, err := db.cli.Delete(ctx, opts.KeyPrefix, clientv3.WithPrefix()) 337 | if err != nil { 338 | db.cli.Close() 339 | return nil, err 340 | } 341 | } 342 | 343 | watchCh := db.cli.Watch(watchCtx, db.opts.KeyPrefix, clientv3.WithPrefix(), clientv3.WithCreatedNotify()) 344 | firstWatchRes := <-watchCh // etcd watch sends a creation event 345 | if len(firstWatchRes.Events) > 0 { 346 | if err := db.watchResult(&firstWatchRes); err != nil { 347 | return nil, fmt.Errorf("first watch: %w", err) 348 | } 349 | } 350 | db.Mu.Lock() 351 | db.rev = rev(firstWatchRes.Header.Revision) 352 | db.Mu.Unlock() 353 | 354 | db.shutdownWG.Add(1) 355 | go func() { 356 | defer db.shutdownWG.Done() 357 | if err := db.watch(watchCh); err != nil { 358 | if watchCtx.Err() == nil { 359 | panic("etcd.watch: " + err.Error()) 360 | } 361 | // otherwise, context was canceled so exit gracefully 362 | db.opts.Logf("etcd.watch: shutdown") 363 | } 364 | }() 365 | 366 | if err := db.loadAll(ctx); err != nil { 367 | return nil, fmt.Errorf("etcd.New: could not load: %w", err) 368 | } 369 | 370 | db.shutdownWG.Add(1) 371 | const watchdogMax = 300 * time.Second 372 | watchdogCh := syncs.Watch(watchCtx, &db.Mu, 30*time.Second, watchdogMax) 373 | go func() { 374 | defer db.shutdownWG.Done() 375 | for d := range watchdogCh { 376 | dbMuLockLatency.Set(int64(d)) 377 | if d == watchdogMax { 378 | buf := new(strings.Builder) 379 | pprof.Lookup("goroutine").WriteTo(buf, 1) 380 | db.opts.Logf("etcd watchdog timeout stack:\n%s", buf.String()) 381 | log.Fatalf("etcd watchdog timeout") 382 | } 383 | } 384 | }() 385 | 386 | return db, nil 387 | } 388 | 389 | // ReadTx create a new read-only transaction. 390 | func (db *DB) ReadTx() *Tx { 391 | return &Tx{ro: true, db: db} 392 | } 393 | 394 | // Tx creates a new database transaction. 395 | func (db *DB) Tx(ctx context.Context) *Tx { 396 | return &Tx{ctx: ctx, db: db} 397 | } 398 | 399 | // Close cancels all transactions and releases all DB resources. 400 | func (db *DB) Close() error { 401 | db.Mu.Lock() 402 | closing := db.closing 403 | db.closing = true 404 | db.Mu.Unlock() 405 | 406 | if closing { 407 | return errors.New("etcd.DB: close already called") 408 | } 409 | 410 | db.watchCancel() 411 | db.shutdownWG.Wait() 412 | 413 | var err error 414 | if db.cli != nil { 415 | err = db.cli.Close() 416 | } 417 | if db.embedClose != nil { 418 | db.embedClose() 419 | } 420 | return err 421 | } 422 | 423 | // A Tx is an etcd transaction. 424 | // 425 | // A Tx holds no resources besides some private memory, so there is 426 | // no notion of closing a transaction or rolling back a transaction. 427 | // A cheap etcd read can be done with: 428 | // 429 | // found, err := db.ReadTx().Get(key, &val) 430 | // 431 | // Tx is not safe for concurrent access. 432 | // For concurrency, create more transactions. 433 | type Tx struct { 434 | // PendingUpdate, if not nil, is called on each Put. 435 | // It can be used by higher-level objects to keep an index 436 | // up-to-date on a transaction state. 437 | // 438 | // The memory passed to PendingUpdate are only valid for the 439 | // duration of the call and must not be modified. 440 | // 441 | // A nil new value means the key has been deleted. 442 | PendingUpdate func(key string, old, new interface{}) 443 | 444 | // Err is any error reported by the transaction during use. 445 | // This can be set used externally to ensure Commit does not fire. 446 | // 447 | // Once Err is set all future calls to Tx methods will return Err. 448 | // On Commit, if Err is not set, it is set to ErrTxClosed 449 | Err error 450 | 451 | ctx context.Context 452 | db *DB 453 | ro bool // readonly 454 | 455 | // maxRev is the maximum revision of this transaction. 456 | // If any key read-from or written-to has a greater rev, 457 | // then this Tx will fail with ErrTxStale. 458 | maxRev rev 459 | 460 | // cmps are keys that was read or written by this tx. 461 | // Tracked so on Commit they can be reported to etcd 462 | // to make sure they didn't change rev. 463 | cmps map[string]struct{} 464 | 465 | // puts are key-values written by this tx. 466 | // The value is cloned before being placed in this map 467 | // so the tx owns the memory (and can pass ownership onto 468 | // the db.cache on commit). 469 | puts map[string]interface{} 470 | } 471 | 472 | // Get retrieves a key-value from the etcd cache into value. 473 | // 474 | // The value must be a pointer to the decoded type of the key, or nil. 475 | // The caller owns the returned value. 476 | // 477 | // No network events are generated. 478 | // 479 | // The first call to Get in a Tx will pin the global revision number 480 | // to the current etcd revision. If a subsequent Get finds a value 481 | // with a greater revision then get will return ErrTxStale. 482 | // This ensures that a Tx has a consistent view of the values it fetches. 483 | func (tx *Tx) Get(key string, value interface{}) (found bool, err error) { 484 | if tx.Err != nil { 485 | return false, tx.Err 486 | } 487 | defer func() { 488 | if err != nil { 489 | err = fmt.Errorf("etcd.Get(%q): %w", key, err) 490 | tx.Err = err 491 | } 492 | }() 493 | 494 | found, kv, err := tx.get(key) 495 | if err != nil { 496 | return false, err 497 | } 498 | if !found { 499 | return false, nil 500 | } 501 | if value != nil { 502 | if err := tx.db.opts.CloneFunc(value, key, kv.value); err != nil { 503 | return false, err 504 | } 505 | } 506 | return true, nil 507 | } 508 | 509 | // UnsafePeek lets the caller see the cached value for a key. 510 | // 511 | // It is vital that the caller does not modify the value, or the DB will 512 | // be corrupted. 513 | func (tx *Tx) UnsafePeek(key string, peekFunc func(v interface{})) (found bool, err error) { 514 | if tx.Err != nil { 515 | return false, tx.Err 516 | } 517 | found, kv, err := tx.get(key) 518 | if err != nil { 519 | return false, err 520 | } 521 | if !found { 522 | return false, nil 523 | } 524 | peekFunc(kv.value) 525 | return true, nil 526 | } 527 | 528 | // GetRange gets a range of KV-pairs from the etcd cache. 529 | // 530 | // The parameter fn is called with batches of matching KV-pairs. 531 | // The passed slice and all the memory it references is owned by fn. 532 | // If fn returns an error then GetRange aborts early and returns the error. 533 | // 534 | // While fn is called GetRange holds either the DB read or write lock, 535 | // so no transactions can be committed from inside the fn callback. 536 | // 537 | // It is possible for the same key to be sent to fn more than once in a 538 | // GetRange call. If this happens, the later key-value pair is a newer 539 | // version that replaces the old value. 540 | // 541 | // When all keys have been read, finalFn is called holding the DB write lock. 542 | // This gives the caller a chance to do something knowing that no key updates 543 | // can happen between reading the range and executing finalFn. 544 | func (db *DB) GetRange(keyPrefix string, fn func([]KV) error, finalFn func()) (err error) { 545 | defer func() { 546 | if err != nil { 547 | err = fmt.Errorf("etcd.GetRange(%q): %w", keyPrefix, err) 548 | } 549 | }() 550 | 551 | // First take a read lock and send all relevant KV-pairs to fn. 552 | // This should include almost all of the KV-space. 553 | var revDone rev 554 | func() { 555 | db.Mu.RLock() 556 | defer db.Mu.RUnlock() 557 | revDone = db.rev 558 | err = db.getRange(keyPrefix, fn, 0) 559 | }() 560 | 561 | if err != nil { 562 | return err 563 | } 564 | 565 | // Now grab a write lock. Find all KV-pairs that have changed since 566 | // we held the read lock and send those to fn. 567 | // 568 | // The double pass is to minimize the time GetRange holds the write lock. 569 | db.Mu.Lock() 570 | defer db.Mu.Unlock() 571 | err = db.getRange(keyPrefix, fn, revDone+1) 572 | if err == nil && finalFn != nil { 573 | finalFn() 574 | } 575 | 576 | return err 577 | } 578 | 579 | // getRange gets all key-values with keyPrefix and passes them to fn. 580 | // 581 | // The DB read lock must be held for the duration of the call. 582 | // 583 | // TODO(crawshaw): This is an inefficient O(N) implementation. 584 | // We can make this in-memory efficient by storing an ordered tree of keys, 585 | // e.g. https://pkg.go.dev/github.com/dghubble/trie?tab=doc#PathTrie 586 | // Or we can factor out the db.load method and use etcd's /range with keys_only=true. 587 | func (db *DB) getRange(keyPrefix string, fn func([]KV) error, min rev) error { 588 | const window = 256 589 | var kvs []KV 590 | 591 | for key, kv := range db.cache { 592 | if !strings.HasPrefix(key, keyPrefix) { 593 | continue 594 | } 595 | if kv.modRev < min { 596 | continue 597 | } 598 | if kv.value == nil { 599 | continue // deleted key 600 | } 601 | cloned, err := db.clone(key, kv.value) 602 | if err != nil { 603 | return fmt.Errorf("clone %s: %w", key, err) 604 | } 605 | kvs = append(kvs, KV{Key: key, Value: cloned}) 606 | if len(kvs) > window { 607 | if err := fn(kvs); err != nil { 608 | return err 609 | } 610 | kvs = nil // passing ownership of kvs to fn 611 | } 612 | } 613 | if len(kvs) > 0 { 614 | return fn(kvs) 615 | } 616 | return nil 617 | } 618 | 619 | // Put adds or replaces a KV-pair in the transaction. 620 | // If a newer value for the key is in the DB this will return ErrTxStale. 621 | // A nil value deletes the key. 622 | func (tx *Tx) Put(key string, value interface{}) error { 623 | if tx.ro { 624 | err := fmt.Errorf("etcd.Put(%q) called on read-only transaction", key) 625 | if tx.Err == nil { 626 | tx.Err = err 627 | } 628 | return err 629 | } 630 | if tx.Err != nil { 631 | return tx.Err 632 | } 633 | _, curVal, err := tx.get(key) 634 | if err != nil { 635 | tx.Err = fmt.Errorf("etcd.Put(%q): %w", key, err) 636 | return tx.Err 637 | } 638 | if tx.puts == nil { 639 | tx.puts = make(map[string]interface{}) 640 | } 641 | var cloned interface{} 642 | if value != nil { 643 | cloned, err = tx.db.clone(key, value) 644 | if err != nil { 645 | tx.Err = fmt.Errorf("etcd.Put(%q): %w", key, err) 646 | return tx.Err 647 | } 648 | } 649 | if tx.PendingUpdate != nil { 650 | tx.PendingUpdate(key, curVal.value, value) 651 | } 652 | tx.puts[key] = cloned 653 | return nil 654 | } 655 | 656 | // Commit commits the transaction to etcd. 657 | // It is an error to call Commit on a read-only transaction. 658 | func (tx *Tx) Commit() (err error) { 659 | if tx.Err != nil { 660 | return fmt.Errorf("etcd.Commit: %w", tx.Err) 661 | } 662 | defer func() { 663 | if err != nil { 664 | err = fmt.Errorf("etcd.Commit: %w", err) 665 | } 666 | if tx.Err == nil { 667 | if err != nil { 668 | tx.Err = err 669 | } else { 670 | tx.Err = ErrTxClosed 671 | } 672 | } 673 | }() 674 | if tx.ro { 675 | return errors.New("tx is read-only") 676 | } 677 | if len(tx.puts) == 0 { 678 | return nil 679 | } 680 | 681 | if atomic.LoadInt32(&tx.db.panicOnWrite) > 0 { 682 | panic("db.PanicOnWrite: db write detected") 683 | } 684 | 685 | if tx.db.inMemory { 686 | return tx.commitInMemory() 687 | } 688 | 689 | tx.db.Mu.RLock() 690 | if tx.db.closing { 691 | tx.db.Mu.RUnlock() 692 | return ErrTxClosed 693 | } 694 | tx.db.shutdownWG.Add(1) 695 | tx.db.Mu.RUnlock() 696 | 697 | defer tx.db.shutdownWG.Done() 698 | ctx, cancel := context.WithCancel(tx.ctx) 699 | defer cancel() 700 | go func() { 701 | select { 702 | case <-tx.db.done: 703 | // db.Close called, cancel commit 704 | case <-ctx.Done(): 705 | // tx.Commit complete or canceled, clean up this goroutine 706 | } 707 | cancel() 708 | }() 709 | 710 | var cmps []clientv3.Cmp 711 | for key := range tx.cmps { 712 | // Here we set the required mod revision of every key 713 | // we ever fetched in the transaction, and require it 714 | // not to have changed since we started. 715 | cmps = append(cmps, clientv3.Compare(clientv3.ModRevision(key), "<", int64(tx.maxRev+1))) 716 | } 717 | var puts []clientv3.Op 718 | for key, val := range tx.puts { 719 | if val == nil { 720 | puts = append(puts, clientv3.OpDelete(key)) 721 | continue 722 | } 723 | data, err := tx.db.opts.EncodeFunc(key, val) 724 | if err != nil { 725 | return fmt.Errorf("EncodeFunc failed for key %q: %v", key, err) 726 | } 727 | puts = append(puts, clientv3.OpPut(key, string(data))) 728 | } 729 | 730 | txn := tx.db.cli.Txn(ctx) 731 | txn = txn.If(cmps...).Then(puts...) 732 | txnRes, err := txn.Commit() 733 | if err != nil { 734 | return err 735 | } 736 | if !txnRes.Succeeded { 737 | if len(tx.puts) == 1 { 738 | var key string 739 | for k := range tx.puts { 740 | key = k 741 | } 742 | return fmt.Errorf("%w: key %s", ErrTxStale, key) 743 | } 744 | return ErrTxStale 745 | } 746 | 747 | txRev := rev(txnRes.Header.Revision) 748 | var done chan struct{} 749 | 750 | tx.db.Mu.Lock() 751 | // TODO(crawshaw): a potential optimization here is to put our 752 | // unaliased, ready to use value objects directly into the cache, 753 | // saving an encode/decode round-trip to the database. 754 | // 755 | // However there is one significant hurdle: once we put the rev 756 | // into the cache, we must increment tx.db.rev or a new Tx that 757 | // attempts to read the value will immediately fail with ErrTxStale. 758 | // But we cannot increment db.rev yet, as there may commits created 759 | // by other clients pending in the server that will come in later. 760 | // 761 | // So instead we must put the objects aside in a limbo, and add 762 | // them to the db.cache in watchResult. We must do this especially 763 | // carefully, as etcd may have chosen to amalgamate our commit with 764 | // some other client's commit, so the incoming txRev may include 765 | // more objects than we committed here. Either way, it is unsafe 766 | // to simply call: 767 | // 768 | // tx.commitCacheLocked(txRev) 769 | // 770 | // This optimization is significant and we should do it. 771 | // It should be easy enough to add a db.pendingCache and extract 772 | // the values from it in watchResult. 773 | if tx.db.rev < txRev { 774 | done = make(chan struct{}) 775 | tx.db.pending[txRev] = append(tx.db.pending[txRev], done) 776 | } 777 | tx.db.Mu.Unlock() 778 | 779 | // Ensure the background watch has caught up to this commit, so that the 780 | // db revision is at or beyond this commit. This means sequential 781 | // commits will always see this one. 782 | if done != nil { 783 | <-done 784 | } 785 | 786 | return nil 787 | } 788 | 789 | // commitCacheLocked pushes the contents of tx into the DB cache. 790 | // db.Mu must be held to call. 791 | func (tx *Tx) commitCacheLocked(modRev rev) { 792 | // Immediately load the new values into the cache. 793 | // The watch will fill in these values shortly if it hasn't already 794 | // (and Tx.Commit will wait for it) but doing it here avoids having 795 | // to call DecodeFunc in the watch goroutine. 796 | var kvs []KV 797 | for key, val := range tx.puts { 798 | kv, exists := tx.db.cache[key] 799 | if exists && modRev <= kv.modRev { 800 | continue 801 | } 802 | tx.db.cache[key] = valueRev{ 803 | value: val, 804 | modRev: modRev, 805 | } 806 | if tx.db.hasWatchLocked() { 807 | var cloned interface{} 808 | if val != nil { 809 | var err error 810 | cloned, err = tx.db.clone(key, val) 811 | if err != nil { 812 | // By this point, we know val is the output of CloneFunc 813 | // called earlier in Tx.Put. That CloneFunc fails on 814 | // the value's second pass through suggests a bug in 815 | // the implementation of CloneFunc. 816 | panic(fmt.Sprintf("etcd tx watch second clone of %q failed: %v", key, err)) 817 | } 818 | } 819 | kvs = append(kvs, KV{Key: key, OldValue: kv.value, Value: cloned}) 820 | } 821 | } 822 | if tx.db.hasWatchLocked() && len(kvs) > 0 { 823 | tx.db.informWatchLocked(kvs) 824 | } 825 | } 826 | 827 | func (tx *Tx) commitInMemory() error { 828 | tx.db.Mu.Lock() 829 | defer tx.db.Mu.Unlock() 830 | 831 | // Check to make sure no other Tx beat us to the punch. 832 | for key := range tx.puts { 833 | kv, exists := tx.db.cache[key] 834 | if !exists { 835 | if tx.puts[key] == nil { 836 | delete(tx.puts, key) // do not process no-op deletes 837 | } 838 | continue 839 | } 840 | if kv.modRev > tx.maxRev { 841 | return fmt.Errorf("%w: key %s", ErrTxStale, key) 842 | } 843 | if tx.puts[key] == nil && kv.value == nil { 844 | delete(tx.puts, key) // do not process no-op deletes 845 | } 846 | } 847 | if len(tx.puts) == 0 { 848 | return nil 849 | } 850 | 851 | tx.db.rev++ 852 | tx.commitCacheLocked(tx.db.rev) 853 | return nil 854 | } 855 | 856 | // valueRev is a decoded database value paired with its etcd mod revision. 857 | type valueRev struct { 858 | value interface{} // decoded value, nil if key was deleted 859 | modRev rev // mod revision of this value 860 | } 861 | 862 | // rev is an etcd mod revision. 863 | // The etcd server assigns a mod revision to every KV and to the whole database. 864 | type rev int64 865 | 866 | // watch issues a long-running watch request against etcd. 867 | // Each transaction is received as a line of JSON. 868 | func (db *DB) watch(ch clientv3.WatchChan) error { 869 | for res := range ch { 870 | if err := res.Err(); err != nil { 871 | return err 872 | } 873 | if err := db.watchResult(&res); err != nil { 874 | return err 875 | } 876 | } 877 | return fmt.Errorf("etcd.watch: [unexpected] watchchan closed") 878 | } 879 | 880 | // watchResult processes a JSON blob from the etcd watch API. 881 | func (db *DB) watchResult(res *clientv3.WatchResponse) error { 882 | type newkv struct { 883 | key string 884 | valueRev valueRev 885 | } 886 | var newkvs []newkv 887 | 888 | for _, ev := range res.Events { 889 | key := string(ev.Kv.Key) 890 | 891 | // As a first pass, we check the cache to see if we can avoid decoding 892 | // the value. This is a performance optimization, it's entirely possible 893 | // the Tx committing these values is still in-flight and will update the 894 | // db.cache momentarily, so it is checked again below under the mutex. 895 | db.Mu.RLock() 896 | kv, exists := db.cache[key] 897 | db.Mu.RUnlock() 898 | 899 | if exists && rev(ev.Kv.ModRevision) <= kv.modRev { 900 | // We already have this value. 901 | continue 902 | } 903 | 904 | if ev.Type == mvccpb.DELETE { 905 | newkvs = append(newkvs, newkv{ 906 | key: key, 907 | valueRev: valueRev{ 908 | modRev: rev(ev.Kv.ModRevision), 909 | }, 910 | }) 911 | continue 912 | } 913 | 914 | v, err := db.opts.DecodeFunc(key, ev.Kv.Value) 915 | if err != nil { 916 | panic(fmt.Sprintf("etcd.watch: bad decoded value for key %q: %v: %q", key, err, string(ev.Kv.Value))) 917 | } 918 | newkvs = append(newkvs, newkv{ 919 | key: key, 920 | valueRev: valueRev{ 921 | value: v, 922 | modRev: rev(ev.Kv.ModRevision), 923 | }, 924 | }) 925 | } 926 | 927 | db.Mu.Lock() 928 | defer db.Mu.Unlock() 929 | 930 | var kvs []KV 931 | for _, newkv := range newkvs { 932 | kv, exists := db.cache[newkv.key] 933 | if exists && newkv.valueRev.modRev <= kv.modRev { 934 | // Value has just been updated by a Tx, keep newer value. 935 | continue 936 | } 937 | if !exists && newkv.valueRev.value == nil { 938 | // Value has been deleted but we never knew about it, ignore. 939 | continue 940 | } 941 | if db.hasWatchLocked() { 942 | var cloned interface{} 943 | if newkv.valueRev.value != nil { 944 | var err error 945 | cloned, err = db.clone(newkv.key, newkv.valueRev.value) 946 | if err != nil { 947 | panic(fmt.Sprintf("etcd.watch clone of %q failed: %v", newkv.key, err)) 948 | } 949 | } 950 | kvs = append(kvs, KV{Key: newkv.key, OldValue: kv.value, Value: cloned}) 951 | } 952 | db.cache[newkv.key] = newkv.valueRev 953 | } 954 | if db.hasWatchLocked() && len(kvs) > 0 { 955 | db.informWatchLocked(kvs) 956 | } 957 | db.rev = rev(res.Header.Revision) 958 | for rev, doneChs := range db.pending { 959 | if rev <= db.rev { 960 | for _, done := range doneChs { 961 | close(done) 962 | } 963 | delete(db.pending, rev) 964 | } 965 | } 966 | 967 | return nil 968 | } 969 | 970 | func (db *DB) clone(key string, val interface{}) (interface{}, error) { 971 | dst := reflect.New(reflect.TypeOf(val)) 972 | if err := db.opts.CloneFunc(dst.Interface(), key, val); err != nil { 973 | return nil, err 974 | } 975 | return dst.Elem().Interface(), nil 976 | } 977 | 978 | // loadAll loads all the keys from etcd into DB using a series of 979 | // paged range requests. 980 | // 981 | // The requests are not pinned to any version. To get a consistent view 982 | // of the DB, a watcher must be started before loadAll is called. 983 | func (db *DB) loadAll(ctx context.Context) error { 984 | const batchSize = 1000 985 | start := time.Now() 986 | db.opts.Logf("etcd.loadAll: loading all KVs with prefix %s", db.opts.KeyPrefix) 987 | opts := []clientv3.OpOption{ 988 | clientv3.WithLimit(batchSize), 989 | clientv3.WithRange(clientv3.GetPrefixRangeEnd(db.opts.KeyPrefix)), 990 | } 991 | 992 | errCh := make(chan error) 993 | countCh := make(chan int) 994 | respSizeCh := make(chan int) 995 | 996 | batches := 0 997 | key := db.opts.KeyPrefix 998 | for { 999 | batches++ 1000 | resp, err := db.cli.Get(ctx, key, opts...) 1001 | if err != nil { 1002 | return err 1003 | } 1004 | go func() { 1005 | count, err := db.load(resp) 1006 | errCh <- err 1007 | countCh <- count 1008 | 1009 | n := 0 1010 | for _, kv := range resp.Kvs { 1011 | n += len(kv.Value) 1012 | } 1013 | respSizeCh <- n 1014 | }() 1015 | if !resp.More { 1016 | break 1017 | } 1018 | key = string(append(resp.Kvs[len(resp.Kvs)-1].Key, 0)) 1019 | } 1020 | db.opts.Logf("etcd.loadAll: KVs read from DB in %s", time.Since(start).Round(time.Millisecond)) 1021 | 1022 | var err error 1023 | loaded := 0 1024 | maxRespSize := 0 1025 | for i := 0; i < batches; i++ { 1026 | if err2 := <-errCh; err == nil { 1027 | err = err2 1028 | } 1029 | loaded += <-countCh 1030 | if n := <-respSizeCh; n > maxRespSize { 1031 | maxRespSize = n 1032 | } 1033 | } 1034 | if err != nil { 1035 | return err 1036 | } 1037 | 1038 | db.opts.Logf("etcd.loadAll: %d KVs read and decoded in %s", loaded, time.Since(start).Round(time.Millisecond)) 1039 | db.opts.Logf("etcd.loadAll: %d batches read, largest batch was %d bytes", batches, maxRespSize) 1040 | return nil 1041 | } 1042 | 1043 | func (db *DB) load(resp *clientv3.GetResponse) (count int, err error) { 1044 | var vals []interface{} 1045 | for _, kv := range resp.Kvs { 1046 | v, err := db.opts.DecodeFunc(string(kv.Key), kv.Value) 1047 | if err != nil { 1048 | return 0, fmt.Errorf("%q: cannot decode: %w", string(kv.Key), err) 1049 | } 1050 | vals = append(vals, v) 1051 | } 1052 | 1053 | db.Mu.Lock() 1054 | defer db.Mu.Unlock() 1055 | 1056 | var kvs []KV 1057 | for i, kv := range resp.Kvs { 1058 | key := string(kv.Key) 1059 | if _, exists := db.cache[key]; exists { 1060 | continue // skip keys already filled by watch 1061 | } 1062 | v := vals[i] 1063 | db.cache[key] = valueRev{ 1064 | value: v, 1065 | modRev: rev(kv.ModRevision), 1066 | } 1067 | if db.opts.WatchFunc != nil { 1068 | cloned, err := db.clone(key, v) 1069 | if err != nil { 1070 | return 0, fmt.Errorf("%q clone of decoded value failed: %w", key, err) 1071 | } 1072 | kvs = append(kvs, KV{Key: key, Value: cloned}) 1073 | } 1074 | } 1075 | 1076 | if len(kvs) > 0 { 1077 | db.opts.WatchFunc(kvs) 1078 | } 1079 | 1080 | return len(kvs), nil 1081 | } 1082 | 1083 | // PanicOnWrite sets whether the db should panic when a write is requested. 1084 | // It is used in tests to ensure that particular actions do not create db writes. 1085 | // Calls to PanicOnWrite may be nested. 1086 | func (db *DB) PanicOnWrite(enable bool) { 1087 | if enable { 1088 | atomic.AddInt32(&db.panicOnWrite, 1) 1089 | } else { 1090 | if atomic.AddInt32(&db.panicOnWrite, -1) < 0 { 1091 | panic("db.PanicOnWrite underflow") 1092 | } 1093 | } 1094 | } 1095 | 1096 | // UnsafeClient exposes the raw underlying etcd client used by the database. 1097 | // Use with extreme care. 1098 | func (db *DB) UnsafeClient() *clientv3.Client { 1099 | return db.cli 1100 | } 1101 | 1102 | func (tx *Tx) get(key string) (bool, valueRev, error) { 1103 | if !strings.HasPrefix(key, tx.db.opts.KeyPrefix) { 1104 | return false, valueRev{}, fmt.Errorf("key does not use prefix %s", tx.db.opts.KeyPrefix) 1105 | } 1106 | 1107 | putValue, isPut := tx.puts[key] 1108 | if isPut { 1109 | if putValue == nil { 1110 | return false, valueRev{}, nil 1111 | } 1112 | v, err := tx.db.clone(key, putValue) 1113 | if err != nil { 1114 | return false, valueRev{}, err 1115 | } 1116 | return true, valueRev{value: v, modRev: tx.maxRev}, nil 1117 | } 1118 | 1119 | tx.db.Mu.RLock() 1120 | kv, ok := tx.db.cache[key] 1121 | if ok && tx.maxRev == 0 { 1122 | tx.maxRev = tx.db.rev 1123 | if kv.modRev > tx.maxRev { 1124 | tx.db.Mu.RUnlock() 1125 | panic(fmt.Sprintf("on new tx kv.modRev %d > tx.maxRev %d", kv.modRev, tx.maxRev)) 1126 | } 1127 | } 1128 | tx.db.Mu.RUnlock() 1129 | 1130 | if tx.maxRev < kv.modRev { 1131 | return false, valueRev{}, ErrTxStale 1132 | } 1133 | if !ok || kv.value == nil { 1134 | return false, valueRev{}, nil 1135 | } 1136 | if !tx.ro { 1137 | if tx.cmps == nil { 1138 | tx.cmps = make(map[string]struct{}) 1139 | } 1140 | tx.cmps[key] = struct{}{} 1141 | } 1142 | return true, kv, nil 1143 | } 1144 | 1145 | func (db *DB) hasWatchLocked() bool { 1146 | return db.opts.WatchFunc != nil || len(db.keyWatchers) > 0 || len(db.prefixWatchers) > 0 1147 | } 1148 | 1149 | func (db *DB) informWatchLocked(kvs []KV) { 1150 | if len(kvs) == 0 { 1151 | panic("informWatchLocked called with no KVs") 1152 | } 1153 | sort.Slice(kvs, func(i, j int) bool { return kvs[i].Key < kvs[j].Key }) 1154 | if db.opts.WatchFunc != nil { 1155 | db.opts.WatchFunc(kvs) 1156 | } 1157 | 1158 | runWatches := func(watches []watch, kvs []KV) []watch { 1159 | if len(watches) == 0 { 1160 | return nil 1161 | } 1162 | if len(watches) > 1 { 1163 | rand.Shuffle(len(watches), func(i, j int) { 1164 | watches[i], watches[j] = watches[j], watches[i] 1165 | }) 1166 | } 1167 | // TODO(crawshaw): implement compaction by swapping last w into 1168 | // current spot, nilling out last function, decrementing i, 1169 | // and shrinking the slice? 1170 | newWatches := watches[:0] 1171 | for _, w := range watches { 1172 | if w.ctx.Err() != nil { 1173 | continue 1174 | } 1175 | w.fn(kvs) 1176 | if w.ctx.Err() != nil { 1177 | continue 1178 | } 1179 | newWatches = append(newWatches, w) 1180 | } 1181 | tail := watches[len(newWatches):] 1182 | for i := range tail { 1183 | // Clear out old funcs so that the backing array 1184 | // does not pin removed watch functions. 1185 | tail[i] = watch{} 1186 | } 1187 | return newWatches 1188 | } 1189 | 1190 | // Process key watchers. 1191 | for _, kv := range kvs { 1192 | db.keyWatchers[kv.Key] = runWatches(db.keyWatchers[kv.Key], []KV{kv}) 1193 | } 1194 | 1195 | // Process prefix watchers. 1196 | deliveries := make(map[string][]KV) 1197 | for _, kv := range kvs { 1198 | for prefix := range db.prefixWatchers { 1199 | if strings.HasPrefix(kv.Key, prefix) { 1200 | deliveries[prefix] = append(deliveries[prefix], kv) 1201 | } 1202 | } 1203 | } 1204 | for prefix, kvs := range deliveries { 1205 | db.prefixWatchers[prefix] = runWatches(db.prefixWatchers[prefix], kvs) 1206 | } 1207 | } 1208 | 1209 | type watch struct { 1210 | ctx context.Context 1211 | fn func(kvs []KV) 1212 | } 1213 | 1214 | // WatchKey registers fn to be called every time a change is made to a key. 1215 | // 1216 | // If there is an existing value of the key it is played through fn 1217 | // before WatchKey returns. 1218 | // 1219 | // The fn function is called holding either the read or write lock. 1220 | // Do not do any operation in fn that tries to take the etcd read or 1221 | // write lock. 1222 | // 1223 | // The watch is de-registered when ctx is Done. 1224 | func (db *DB) WatchKey(ctx context.Context, key string, fn func(old, new interface{})) error { 1225 | w := watch{ 1226 | ctx: ctx, 1227 | fn: func(kvs []KV) { 1228 | if len(kvs) != 1 { 1229 | panic(fmt.Sprintf("WatchKey callback receieved %d kvs", len(kvs))) 1230 | } 1231 | kv := kvs[0] 1232 | if kv.Key != key { 1233 | panic(fmt.Sprintf("WatchKey callback receieved wrong key %q, want %q", kv.Key, key)) 1234 | } 1235 | fn(kv.OldValue, kv.Value) 1236 | }, 1237 | } 1238 | 1239 | // Two passes. First try to send the current value holding the read lock. 1240 | // This sometimes short-cuts the watch so no write lock is necessary. 1241 | // This mostly lets us call fn under the read lock, for better 1242 | // general concurrency. 1243 | 1244 | db.Mu.RLock() 1245 | kv1, kv1ok := db.cache[key] 1246 | if kv1ok { 1247 | cloned, err := db.clone(key, kv1.value) 1248 | if err != nil { 1249 | db.Mu.RUnlock() 1250 | return fmt.Errorf("WatchKey clone %s: %w", key, err) 1251 | } 1252 | fn(nil, cloned) 1253 | } 1254 | db.Mu.RUnlock() 1255 | 1256 | if ctx.Err() != nil { 1257 | return nil // first value was enough, call it quits 1258 | } 1259 | 1260 | db.Mu.Lock() 1261 | defer db.Mu.Unlock() 1262 | if kv2, ok := db.cache[key]; ok && kv2.modRev > kv1.modRev { 1263 | // The value changed between the rlock and the wlock, 1264 | // so send the newer value. 1265 | var cloned1 interface{} 1266 | if kv1ok { 1267 | var err error 1268 | cloned1, err = db.clone(key, kv1.value) 1269 | if err != nil { 1270 | return fmt.Errorf("WatchKey clone %s: %w", key, err) 1271 | } 1272 | } 1273 | cloned2, err := db.clone(key, kv2.value) 1274 | if err != nil { 1275 | return fmt.Errorf("WatchKey clone %s: %w", key, err) 1276 | } 1277 | fn(cloned1, cloned2) 1278 | } 1279 | db.keyWatchers[key] = append(db.keyWatchers[key], w) 1280 | return nil 1281 | } 1282 | 1283 | // WatchPrefix registers fn to be called every time a change is made to 1284 | // a key-value with the prefix keyPrefix. 1285 | // 1286 | // All existing key-values matching keyPrefix are played through fn 1287 | // before WatchPrefix returns. 1288 | // 1289 | // The fn function is called holding either the read or write lock. 1290 | // Do not do any operation in fn that tries to take the etcd read or 1291 | // write lock. 1292 | // 1293 | // The watch is de-registered when ctx is Done. 1294 | // 1295 | // The data structure storing prefix watchers is relatively inefficient 1296 | // at present, so adding large numbers of prefix watchers is expensive. 1297 | func (db *DB) WatchPrefix(ctx context.Context, keyPrefix string, fn func(kvs []KV)) error { 1298 | // errNoContinue is used internally to deregister watch functions 1299 | errNoContinue := errors.New("watch does not continue") 1300 | 1301 | // TODO: reorganize this to more aggressively end the watch when 1302 | // ctx is done. (Probably by changing GetRange to use a ctx.) 1303 | 1304 | // Run all existing key-values through fn. 1305 | fnRange := func(kv []KV) error { 1306 | if ctx.Err() != nil { 1307 | return errNoContinue 1308 | } 1309 | fn(kv) 1310 | return nil 1311 | } 1312 | onSuccess := func() { 1313 | // etcd.Mu write lock is held by GetRange 1314 | db.prefixWatchers[keyPrefix] = append(db.prefixWatchers[keyPrefix], watch{ 1315 | ctx: ctx, 1316 | fn: fn, 1317 | }) 1318 | } 1319 | err := db.GetRange(keyPrefix, fnRange, onSuccess) 1320 | if err != nil && !errors.Is(err, errNoContinue) { 1321 | return fmt.Errorf("cfgdb.AddWatch: %w", err) 1322 | } 1323 | return nil 1324 | 1325 | } 1326 | 1327 | // TODO(crawshaw): type Key string ? 1328 | // TODO(crawshaw): Delete 1329 | // TODO(crawshaw): Watch Delete 1330 | -------------------------------------------------------------------------------- /tailetc_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tailetc 6 | 7 | import ( 8 | "context" 9 | "encoding/json" 10 | "errors" 11 | "fmt" 12 | "reflect" 13 | "sort" 14 | "strings" 15 | "testing" 16 | "time" 17 | 18 | "github.com/google/go-cmp/cmp" 19 | ) 20 | 21 | func etcdURL(tb testing.TB) string { 22 | return "file://" + tb.TempDir() 23 | } 24 | 25 | // person is a database value test object used with the keys "/db/person/". 26 | type person struct { 27 | ID int `json:"id"` 28 | Name string `json:"name"` 29 | LikesIceCream bool `json:"likes_ice_cream"` 30 | } 31 | 32 | var personOptions = Options{ 33 | KeyPrefix: "/db/", 34 | EncodeFunc: func(key string, value interface{}) ([]byte, error) { 35 | if value == nil { 36 | return nil, nil 37 | } 38 | switch { 39 | case strings.HasPrefix(key, "/db/person"): 40 | return json.Marshal(value) 41 | default: 42 | b, isBytes := value.([]byte) 43 | if isBytes { 44 | return b, nil 45 | } 46 | return nil, fmt.Errorf("encodePerson: unknown value type %T", value) 47 | } 48 | }, 49 | DecodeFunc: func(key string, data []byte) (interface{}, error) { 50 | switch { 51 | case strings.HasPrefix(key, "/db/person"): 52 | var p person 53 | if err := json.Unmarshal(data, &p); err != nil { 54 | return nil, err 55 | } 56 | return p, nil 57 | default: 58 | return data, nil 59 | } 60 | }, 61 | CloneFunc: func(dst interface{}, key string, value interface{}) error { 62 | if value == nil { 63 | return nil 64 | } 65 | switch { 66 | case strings.HasPrefix(key, "/db/person"): 67 | *dst.(*person) = value.(person) 68 | default: 69 | b := value.([]byte) 70 | *dst.(*[]byte) = append([]byte(nil), b...) 71 | } 72 | return nil 73 | }, 74 | DeleteAllOnStart: true, 75 | } 76 | 77 | func TestDB(t *testing.T) { 78 | t.Parallel() 79 | ctx := context.Background() 80 | alice := person{ID: 42, Name: "Alice", LikesIceCream: true} 81 | 82 | url := "file://" + t.TempDir() 83 | 84 | t.Run("readwrite", func(t *testing.T) { 85 | opts := personOptions 86 | opts.Logf = t.Logf 87 | db, err := New(ctx, url, opts) 88 | if err != nil { 89 | t.Fatal(err) 90 | } 91 | defer db.Close() 92 | tx := db.Tx(context.Background()) 93 | tx.Put("/db/person/alice", alice) 94 | 95 | // add bob 96 | var gotBob person 97 | if found, err := tx.Get("/db/person/bob", &gotBob); err != nil { 98 | t.Fatal(err) 99 | } else if found { 100 | t.Errorf("expected no bob, got: %v", gotBob) 101 | } 102 | tx.Put("/db/person/bob", person{ID: 43, Name: "Bob"}) 103 | if found, err := tx.Get("/db/person/bob", &gotBob); err != nil { 104 | t.Fatal(err) 105 | } else if !found { 106 | t.Errorf("could not get pending /db/person/bob entry") 107 | } 108 | found, err := tx.UnsafePeek("/db/person/bob", func(v interface{}) { 109 | if gotName := v.(person).Name; gotName != "Bob" { 110 | t.Fatalf("UnsafePeek Name=%s, want Bob", gotName) 111 | } 112 | }) 113 | if !found { 114 | t.Fatal("UnsafePeek could not find key") 115 | } 116 | if err != nil { 117 | t.Fatal(err) 118 | } 119 | tx.Put("/db/person/bob", person{ID: 43, Name: "Bob", LikesIceCream: true}) 120 | if found, err := tx.Get("/db/person/bob", &gotBob); err != nil { 121 | t.Fatal(err) 122 | } else if !found { 123 | t.Errorf("could not get updated pending /db/person/bob entry") 124 | } else if !gotBob.LikesIceCream { 125 | t.Errorf("updated pending /db/person/bob entry LikesIceCream=false, want true") 126 | } 127 | 128 | if err := tx.Commit(); err != nil { 129 | t.Fatal(err) 130 | } 131 | 132 | var gotAlice person 133 | if found, err := db.Tx(context.Background()).Get("/db/person/alice", &gotAlice); err != nil { 134 | t.Fatal(err) 135 | } else if !found { 136 | t.Errorf("/db/person/alice not found") 137 | } else if gotAlice != alice { 138 | t.Errorf("/db/person/alice=%v, want %v", gotAlice, alice) 139 | } 140 | }) 141 | 142 | t.Run("readwrite-newdb", func(t *testing.T) { 143 | opts := personOptions 144 | opts.Logf = t.Logf 145 | opts.DeleteAllOnStart = false // we want to read the prev keys 146 | db, err := New(ctx, url, opts) 147 | if err != nil { 148 | t.Fatal(err) 149 | } 150 | defer db.Close() 151 | var gotAlice person 152 | if found, err := db.Tx(context.Background()).Get("/db/person/alice", &gotAlice); err != nil { 153 | t.Fatal(err) 154 | } else if !found { 155 | t.Errorf("/db/person/alice not found") 156 | } else if gotAlice != alice { 157 | t.Errorf("/db/person/alice=%v, want %v", gotAlice, alice) 158 | } 159 | if _, err := db.ReadTx().Get("/db/person/alice", &gotAlice); err != nil { 160 | t.Fatal(err) 161 | } else if gotAlice != alice { 162 | t.Errorf("/db/person/alice=%v, want %v", gotAlice, alice) 163 | } 164 | }) 165 | 166 | t.Run("newline", func(t *testing.T) { 167 | opts := personOptions 168 | opts.Logf = t.Logf 169 | opts.DeleteAllOnStart = false // we want to read the prev keys 170 | db, err := New(ctx, url, opts) 171 | if err != nil { 172 | t.Fatal(err) 173 | } 174 | defer db.Close() 175 | tx := db.Tx(ctx) 176 | 177 | newline1 := person{Name: "line1\n"} 178 | newline2 := person{Name: "line1\nline2"} 179 | 180 | const key = "/db/person/newline" 181 | var pendingCalls int 182 | var pendingErr error 183 | tx.PendingUpdate = func(k string, old, new interface{}) { 184 | if pendingErr != nil { 185 | return 186 | } 187 | if k != key { 188 | pendingErr = fmt.Errorf("PendingUpdate call %d: key=%q, want %q", pendingCalls, k, key) 189 | } 190 | v, _ := new.(person) 191 | if pendingCalls == 0 && v.Name != newline1.Name { 192 | pendingErr = fmt.Errorf("PendingUpdate call %d: Name=%q, want %q", pendingCalls, v.Name, newline1.Name) 193 | } 194 | if pendingCalls == 1 && v.Name != newline2.Name { 195 | pendingErr = fmt.Errorf("PendingUpdate call %d: Name=%q, want %q", pendingCalls, v.Name, newline2.Name) 196 | } 197 | pendingCalls++ 198 | } 199 | 200 | tx.Put(key, newline1) 201 | tx.Put(key, newline2) 202 | if err := tx.Commit(); err != nil { 203 | t.Fatal(err) 204 | } 205 | tx = db.ReadTx() 206 | var got person 207 | if _, err := tx.Get(key, &got); err != nil { 208 | t.Fatal(err) 209 | } 210 | if got != newline2 { 211 | t.Errorf("Get(%q) = %v, want %v", key, got, newline2) 212 | } 213 | 214 | if pendingErr != nil { 215 | t.Error(pendingErr) 216 | } 217 | }) 218 | } 219 | 220 | func TestStaleTx(t *testing.T) { 221 | t.Parallel() 222 | testStaleTx(t, etcdURL(t)) 223 | } 224 | 225 | func TestStaleTxInMemory(t *testing.T) { 226 | t.Parallel() 227 | testStaleTx(t, "memory://") 228 | } 229 | 230 | func testStaleTx(t *testing.T, url string) { 231 | watchCh := make(chan []KV, 8) 232 | checkWatch := func(want []KV) { 233 | t.Helper() 234 | select { 235 | case got := <-watchCh: 236 | if !reflect.DeepEqual(got, want) { 237 | t.Errorf("Watch=%v, want %v", got, want) 238 | } 239 | case <-time.After(10 * time.Second): 240 | t.Fatal("no watch update") 241 | } 242 | } 243 | checkNoWatch := func() { 244 | t.Helper() 245 | select { 246 | case unexpected := <-watchCh: 247 | t.Errorf("unexpected watch update: %v", unexpected) 248 | default: 249 | } 250 | } 251 | 252 | ctx := context.Background() 253 | opts := personOptions 254 | opts.Logf = t.Logf 255 | opts.WatchFunc = func(kvs []KV) { 256 | watchCh <- kvs 257 | } 258 | db, err := New(ctx, url, opts) 259 | if err != nil { 260 | t.Fatal(err) 261 | } 262 | defer db.Close() 263 | 264 | alice := person{ID: 42, Name: "Alice", LikesIceCream: true} 265 | aliceNoIceCream := person{ID: 42, Name: "Alice", LikesIceCream: false} 266 | 267 | // set key 268 | tx := db.Tx(ctx) 269 | if err := tx.Put("/db/person/alice", alice); err != nil { 270 | t.Fatal(err) 271 | } 272 | if err := tx.Commit(); err != nil { 273 | t.Fatal(err) 274 | } 275 | checkWatch([]KV{{"/db/person/alice", nil, alice}}) 276 | checkNoWatch() 277 | 278 | // no-op replace key 279 | tx = db.Tx(ctx) 280 | if err := tx.Put("/db/person/alice", alice); err != nil { 281 | t.Fatal(err) 282 | } 283 | if err := tx.Commit(); err != nil { 284 | t.Fatal(err) 285 | } 286 | checkWatch([]KV{{"/db/person/alice", alice, alice}}) 287 | checkNoWatch() 288 | 289 | // update key 290 | tx = db.Tx(ctx) 291 | if err := tx.Put("/db/person/alice", aliceNoIceCream); err != nil { 292 | t.Fatal(err) 293 | } 294 | if err := tx.Commit(); err != nil { 295 | t.Fatal(err) 296 | } 297 | checkWatch([]KV{{"/db/person/alice", alice, aliceNoIceCream}}) 298 | checkNoWatch() 299 | 300 | // stale write 301 | tx = db.Tx(ctx) 302 | var gotAlice person 303 | tx.Get("/db/person/alice", &gotAlice) 304 | if err := tx.Put("/db/person/alice", person{Name: "BadAliceTx"}); err != nil { 305 | t.Fatal(err) 306 | } 307 | 308 | tx2 := db.Tx(ctx) 309 | aliceNewID := person{ID: 4242, Name: "Alice", LikesIceCream: true} 310 | if err := tx2.Put("/db/person/alice", aliceNewID); err != nil { 311 | t.Fatal(err) 312 | } 313 | if err := tx2.Commit(); err != nil { 314 | t.Fatal(err) 315 | } 316 | checkWatch([]KV{{"/db/person/alice", aliceNoIceCream, aliceNewID}}) 317 | checkNoWatch() 318 | 319 | if err := tx.Commit(); err == nil || !errors.Is(err, ErrTxStale) { 320 | t.Errorf("err=%v, want ErrTxStale", err) 321 | } 322 | checkNoWatch() 323 | 324 | // stale read 325 | tx = db.Tx(ctx) 326 | if _, err := tx.Get("/db/person/alice", &gotAlice); err != nil { 327 | t.Fatal(err) 328 | } 329 | 330 | tx2 = db.Tx(ctx) 331 | if err := tx2.Put("/db/person/alice", aliceNoIceCream); err != nil { 332 | t.Fatal(err) 333 | } 334 | if err := tx2.Commit(); err != nil { 335 | t.Fatal(err) 336 | } 337 | checkWatch([]KV{{"/db/person/alice", aliceNewID, aliceNoIceCream}}) 338 | 339 | if _, err := tx.Get("/db/person/alice", &gotAlice); err == nil || !errors.Is(err, ErrTxStale) { 340 | t.Errorf("err=%v, want ErrTxStale", err) 341 | } 342 | } 343 | 344 | func TestVariableKeys(t *testing.T) { 345 | t.Parallel() 346 | testVariableKeys(t, etcdURL(t)) 347 | } 348 | 349 | func TestVariableKeysInMemory(t *testing.T) { 350 | t.Parallel() 351 | testVariableKeys(t, "memory://") 352 | } 353 | 354 | func testVariableKeys(t *testing.T, url string) { 355 | watchCh := make(chan []KV, 8) 356 | ctx := context.Background() 357 | opts := personOptions 358 | opts.Logf = t.Logf 359 | opts.WatchFunc = func(kvs []KV) { 360 | watchCh <- kvs 361 | } 362 | db, err := New(ctx, url, opts) 363 | if err != nil { 364 | t.Fatal(err) 365 | } 366 | defer db.Close() 367 | 368 | for i := 0; i < 5; i++ { 369 | for j := 1; j < 5; j++ { 370 | var want []KV 371 | for k := 0; k < j; k++ { 372 | key := fmt.Sprintf("/db/person/k%d", k) 373 | val := person{Name: key} 374 | want = append(want, KV{key, val, val}) 375 | } 376 | tx := db.Tx(ctx) 377 | for _, kv := range want { 378 | if err := tx.Put(kv.Key, kv.Value); err != nil { 379 | t.Fatal(err) 380 | } 381 | } 382 | if err := tx.Commit(); err != nil { 383 | t.Fatal(err) 384 | } 385 | 386 | select { 387 | case got := <-watchCh: 388 | if i > 0 && !reflect.DeepEqual(got, want) { 389 | t.Errorf("i=%d, j=%d, Watch=%v, want %v", i, j, got, want) 390 | } 391 | case <-time.After(10 * time.Second): 392 | t.Fatalf("i=%d, j=%d, no watch update", i, j) 393 | } 394 | select { 395 | case unexpected := <-watchCh: 396 | t.Errorf("i=%d, j=%d, unexpected watch update: %v", i, j, unexpected) 397 | default: 398 | } 399 | } 400 | } 401 | } 402 | 403 | func TestGetRange(t *testing.T) { 404 | t.Parallel() 405 | testGetRange(t, etcdURL(t)) 406 | } 407 | 408 | func TestGetRangeInMemory(t *testing.T) { 409 | t.Parallel() 410 | testGetRange(t, "memory://") 411 | } 412 | 413 | func testGetRange(t *testing.T, url string) { 414 | ctx := context.Background() 415 | db, err := New(ctx, url, Options{Logf: t.Logf, DeleteAllOnStart: true}) 416 | if err != nil { 417 | t.Fatal(err) 418 | } 419 | defer db.Close() 420 | 421 | tx := db.Tx(ctx) 422 | tx.Put("/a/1", []byte("1")) 423 | tx.Put("/a/2", []byte("2")) 424 | tx.Put("/a/3", []byte("3")) 425 | tx.Put("/b/1", []byte("b1")) 426 | tx.Put("/b/2", []byte("b2")) 427 | tx.Put("/b/3", []byte("b3")) 428 | if err := tx.Commit(); err != nil { 429 | t.Fatal(err) 430 | } 431 | 432 | var kvs []KV 433 | fn := func(k []KV) error { 434 | kvs = append(kvs, k...) 435 | return nil 436 | } 437 | if err := db.GetRange("/a/", fn, nil); err != nil { 438 | t.Fatal(err) 439 | } 440 | sort.Slice(kvs, func(i, j int) bool { return kvs[i].Key < kvs[j].Key }) 441 | want := []KV{{"/a/1", nil, []byte("1")}, {"/a/2", nil, []byte("2")}, {"/a/3", nil, []byte("3")}} 442 | if !reflect.DeepEqual(want, kvs) { 443 | t.Errorf(`GetRange("/a/")=%v, want %v`, kvs, want) 444 | } 445 | 446 | kvs = nil 447 | if err := db.GetRange("/b/", fn, nil); err != nil { 448 | t.Fatal(err) 449 | } 450 | sort.Slice(kvs, func(i, j int) bool { return kvs[i].Key < kvs[j].Key }) 451 | want = []KV{{"/b/1", nil, []byte("b1")}, {"/b/2", nil, []byte("b2")}, {"/b/3", nil, []byte("b3")}} 452 | if !reflect.DeepEqual(want, kvs) { 453 | t.Errorf(`GetRange("/b/")=%v, want %v`, kvs, want) 454 | } 455 | } 456 | 457 | func TestDelete(t *testing.T) { 458 | t.Parallel() 459 | testDelete(t, etcdURL(t)) 460 | } 461 | 462 | func TestDeleteInMemory(t *testing.T) { 463 | t.Parallel() 464 | testDelete(t, "memory://") 465 | } 466 | 467 | func testDelete(t *testing.T, url string) { 468 | watch := make(chan []KV, 16) 469 | watchFunc := func(kvs []KV) { 470 | watch <- kvs 471 | } 472 | watchPrefix := make(chan []KV, 16) 473 | watchPrefixFunc := func(kvs []KV) { 474 | watchPrefix <- kvs 475 | } 476 | checkWatch := func(want []KV) { 477 | t.Helper() 478 | timer := time.NewTimer(5 * time.Second) 479 | defer timer.Stop() 480 | var got, gotPrefix []KV 481 | select { 482 | case got = <-watch: 483 | case <-timer.C: 484 | t.Fatalf("timeout waiting for %v", want) 485 | } 486 | select { 487 | case gotPrefix = <-watchPrefix: 488 | case <-timer.C: 489 | t.Fatalf("timeout waiting for prefix watch %v", want) 490 | } 491 | if !cmp.Equal(got, want) { 492 | t.Errorf("watch got: %v,\nwant: %v", got, want) 493 | } 494 | if !cmp.Equal(gotPrefix, want) { 495 | t.Errorf("watch gotPrefix: %v,\nwant: %v", gotPrefix, want) 496 | } 497 | } 498 | 499 | ctx := context.Background() 500 | db, err := New(ctx, url, Options{Logf: t.Logf, DeleteAllOnStart: true, WatchFunc: watchFunc}) 501 | if err != nil { 502 | t.Fatal(err) 503 | } 504 | defer db.Close() 505 | 506 | if err := db.WatchPrefix(ctx, "/", watchPrefixFunc); err != nil { 507 | t.Fatal(err) 508 | } 509 | 510 | checkNoKey := func(tx *Tx, key string) { 511 | t.Helper() 512 | var got []byte 513 | if found, err := tx.Get(key, &got); err != nil { 514 | t.Fatalf("%s: %v", key, err) 515 | } else if found { 516 | t.Errorf("found deleted key: %s", key) 517 | } else if got != nil { 518 | t.Errorf("%s: got=%v, want nil", key, got) 519 | } 520 | } 521 | 522 | var watchKeyCalls int 523 | var watchKeyErr error 524 | watchCtx, watchCancel := context.WithCancel(context.Background()) 525 | err = db.WatchKey(watchCtx, "/b/2", func(old, new interface{}) { 526 | if watchKeyErr != nil { 527 | return 528 | } 529 | watchKeyCalls++ 530 | switch watchKeyCalls { 531 | case 1: 532 | if old != nil || new == nil || string(new.([]byte)) != "b2" { 533 | watchKeyErr = fmt.Errorf("watch key: old=%v, new=%v, want b2", old, new) 534 | } 535 | case 2: 536 | if old == nil || new != nil || string(old.([]byte)) != "b2" { 537 | watchKeyErr = fmt.Errorf("watch key: old=%v, new=%v, want nil", old, new) 538 | } 539 | watchCancel() 540 | } 541 | }) 542 | if err != nil { 543 | t.Fatal(err) 544 | } 545 | 546 | tx := db.Tx(ctx) 547 | tx.Put("/a/1", []byte("1")) 548 | tx.Put("/a/2", []byte("2")) 549 | tx.Put("/a/3", []byte("3")) 550 | tx.Put("/b/1", []byte("b1")) 551 | tx.Put("/b/2", []byte("b2")) 552 | tx.Put("/b/3", []byte("b3")) 553 | tx.Put("/b/3", nil) 554 | checkNoKey(tx, "/b/3") 555 | if err := tx.Commit(); err != nil { 556 | t.Fatal(err) 557 | } 558 | checkNoKey(db.ReadTx(), "/b/3") 559 | checkWatch([]KV{ 560 | {Key: "/a/1", Value: []byte("1")}, 561 | {Key: "/a/2", Value: []byte("2")}, 562 | {Key: "/a/3", Value: []byte("3")}, 563 | {Key: "/b/1", Value: []byte("b1")}, 564 | {Key: "/b/2", Value: []byte("b2")}, 565 | }) 566 | if watchKeyErr != nil { 567 | t.Fatal(err) 568 | } else if watchKeyCalls != 1 { 569 | t.Fatal("missing WatchKey call") 570 | } 571 | 572 | tx = db.Tx(ctx) 573 | pendingUpdate := errors.New("no pending update") 574 | tx.PendingUpdate = func(key string, old, new interface{}) { 575 | if key != "/b/2" { 576 | pendingUpdate = fmt.Errorf("key=%q, want %q", key, "/b/2") 577 | return 578 | } 579 | if !cmp.Equal(old, []byte("b2")) { 580 | pendingUpdate = fmt.Errorf("old=%v, want 'b2'", old) 581 | return 582 | } 583 | if new != nil { 584 | pendingUpdate = fmt.Errorf("new=%v, want nil", new) 585 | return 586 | } 587 | pendingUpdate = nil 588 | } 589 | checkNoKey(tx, "/b/3") 590 | tx.Put("/b/2", nil) 591 | if pendingUpdate != nil { 592 | t.Fatalf("PendingUpdate: %v", pendingUpdate) 593 | } 594 | checkNoKey(tx, "/b/2") 595 | if err := tx.Commit(); err != nil { 596 | t.Fatal(err) 597 | } 598 | checkWatch([]KV{ 599 | {Key: "/b/2", OldValue: []byte("b2"), Value: nil}, 600 | }) 601 | if watchKeyErr != nil { 602 | t.Fatal(err) 603 | } else if watchKeyCalls != 2 { 604 | t.Fatal("missing WatchKey call") 605 | } 606 | 607 | tx = db.Tx(ctx) 608 | tx.Put("/b/2", nil) 609 | if err := tx.Commit(); err != nil { 610 | t.Fatal(err) 611 | } 612 | select { 613 | case got := <-watch: 614 | t.Errorf("unexpected watch result: %v", got) 615 | case got := <-watchPrefix: 616 | t.Errorf("unexpected watchPrefix result: %v", got) 617 | default: 618 | } 619 | if watchKeyCalls != 2 { 620 | t.Fatal("unexpected WatchKey call") 621 | } 622 | } 623 | 624 | func BenchmarkPutOver(b *testing.B) { 625 | ctx := context.Background() 626 | opts := personOptions 627 | opts.Logf = b.Logf 628 | db, err := New(ctx, etcdURL(b), opts) 629 | if err != nil { 630 | b.Fatal(err) 631 | } 632 | defer db.Close() 633 | 634 | alice := person{ID: 42, Name: "Alice MacDuff", LikesIceCream: true} 635 | 636 | b.ResetTimer() 637 | for i := 0; i < b.N; i++ { 638 | tx := db.Tx(ctx) 639 | tx.Put("/db/person/alice", alice) 640 | if err := tx.Commit(); err != nil { 641 | b.Fatal(err) 642 | } 643 | } 644 | } 645 | func BenchmarkPut(b *testing.B) { 646 | ctx := context.Background() 647 | opts := personOptions 648 | opts.Logf = b.Logf 649 | db, err := New(ctx, etcdURL(b), opts) 650 | if err != nil { 651 | b.Fatal(err) 652 | } 653 | defer db.Close() 654 | 655 | alice := person{ID: 42, Name: "Alice MacDuff", LikesIceCream: true} 656 | 657 | b.ResetTimer() 658 | for i := 0; i < b.N; i++ { 659 | tx := db.Tx(ctx) 660 | tx.Put("/db/person/alice", alice) 661 | if err := tx.Commit(); err != nil { 662 | b.Fatal(err) 663 | } 664 | } 665 | } 666 | 667 | func BenchmarkPutX2(b *testing.B) { benchmarkPutX(b, 2) } 668 | func BenchmarkPutX4(b *testing.B) { benchmarkPutX(b, 4) } 669 | func BenchmarkPutX8(b *testing.B) { benchmarkPutX(b, 8) } 670 | 671 | func benchmarkPutX(b *testing.B, x int) { 672 | ctx := context.Background() 673 | opts := personOptions 674 | opts.Logf = b.Logf 675 | db, err := New(ctx, etcdURL(b), opts) 676 | if err != nil { 677 | b.Fatal(err) 678 | } 679 | defer db.Close() 680 | 681 | limit := make(chan struct{}, x) 682 | for i := 0; i < cap(limit); i++ { 683 | limit <- struct{}{} 684 | } 685 | errch := make(chan error) 686 | b.ResetTimer() 687 | for i := 0; i < b.N; i++ { 688 | select { 689 | case <-limit: 690 | case err := <-errch: 691 | b.Fatal(err) 692 | } 693 | go func(i int) { 694 | defer func() { 695 | limit <- struct{}{} 696 | }() 697 | tx := db.Tx(ctx) 698 | tx.Put(fmt.Sprintf("/db/person/k%d", i), person{ID: i, LikesIceCream: true}) 699 | if err := tx.Commit(); err != nil { 700 | errch <- err 701 | } 702 | }(i) 703 | } 704 | for i := 0; i < cap(limit); i++ { 705 | select { 706 | case err := <-errch: 707 | b.Fatal(err) 708 | case <-limit: 709 | } 710 | } 711 | } 712 | --------------------------------------------------------------------------------