├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE.code ├── LICENSE.docs ├── README.md ├── docs ├── compatibility.md └── examples.md ├── libkv.go ├── libkv_test.go ├── script ├── .validate ├── coverage ├── travis_consul.sh ├── travis_etcd.sh ├── travis_redis.sh ├── travis_zk.sh └── validate-gofmt ├── store ├── boltdb │ ├── boltdb.go │ └── boltdb_test.go ├── consul │ ├── consul.go │ └── consul_test.go ├── etcd │ ├── v2 │ │ ├── etcd.go │ │ └── etcd_test.go │ └── v3 │ │ ├── etcd.go │ │ └── etcd_test.go ├── helpers.go ├── mock │ └── mock.go ├── redis │ ├── lua.go │ ├── redis.go │ └── redis_test.go ├── store.go └── zookeeper │ ├── zookeeper.go │ └── zookeeper_test.go └── testutils └── utils.go /.gitignore: -------------------------------------------------------------------------------- 1 | dump.rdb 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.9.1 5 | 6 | sudo: true 7 | 8 | before_install: 9 | # Symlink below is needed for Travis CI to work correctly on personal forks of libkv 10 | - ln -s $HOME/gopath/src/github.com/${TRAVIS_REPO_SLUG///libkv/} $HOME/gopath/src/github.com/docker 11 | - go get golang.org/x/tools/cmd/cover 12 | - go get github.com/mattn/goveralls 13 | - go get github.com/golang/lint/golint 14 | - go get github.com/GeertJohan/fgt 15 | # ca-certificates is needed for wget to work properly 16 | - sudo apt-get install ca-certificates 17 | 18 | before_script: 19 | - script/travis_consul.sh 0.9.3 20 | - script/travis_etcd.sh 3.2.9 21 | - script/travis_zk.sh 3.4.10 22 | - script/travis_redis.sh 4.0.2 23 | 24 | script: 25 | - ./consul agent -server -bootstrap -advertise=127.0.0.1 -data-dir /tmp/consul -config-file=./config.json 1>/dev/null & 26 | - ./etcd/etcd --listen-client-urls 'http://0.0.0.0:4001' --advertise-client-urls 'http://127.0.0.1:4001' >/dev/null 2>&1 & 27 | - ./zk/bin/zkServer.sh start ./zk/conf/zoo.cfg 1> /dev/null 28 | - ./redis/src/redis-server & 29 | - script/validate-gofmt 30 | - go vet ./... 31 | - fgt golint ./... 32 | - go test -v -race ./... 33 | - script/coverage 34 | - goveralls -service=travis-ci -coverprofile=goverage.report 35 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | All contributions are useful, whether it is a simple typo, a more complex change, or 4 | just pointing out an issue. We welcome any contribution so feel free to take part in 5 | the discussions. If you want to contribute to the project, please make sure to review 6 | this document carefully. 7 | 8 | - [Setting up the environment](#setting-up-the-environment) 9 | - [Before submitting a change](#before-submitting-a-change) 10 | - [Your first pull request](#your-first-pull-request) 11 | 12 | ## Working Environment 13 | 14 | ### Prerequisites 15 | 16 | - Git 17 | - Golang 18 | - One or all of the supported datastores (Zookeeper / Consul / Etcd / Redis / BoltDB) 19 | 20 | ### Installing Golang 21 | 22 | Install golang using your favorite package manager on Linux or with the archive 23 | following these [Guidelines](https://golang.org/doc/install). 24 | 25 | An easy way to get started on mac OS is to use [homebrew](https://brew.sh) and type 26 | `brew install go` in a shell. 27 | 28 | In addition to the language runtime, make sure you install these tools locally using 29 | `go get`: 30 | 31 | - **fmt** (to format source code) 32 | - **goimports** (to automatically include and triage imports) 33 | 34 | Once you have a working Go installation, follow the next steps: 35 | 36 | - Get the repository: 37 | 38 | go get -u github.com/abronan/libkv 39 | 40 | - Checkout on a new branch from the master branch to start working on a patch 41 | 42 | git checkout -b mybranch 43 | 44 | ### Local testing of key/value stores 45 | 46 | In addition to installing golang, you will need to install some or all of the key 47 | value stores for testing. 48 | 49 | Refer to each of these stores documentation in order to proceed with installation. 50 | Generally, the tests are using the **default configuration** with the **default port** 51 | to connect to a store and run the test suite. 52 | 53 | To test a change, you can proceed in two ways: 54 | 55 | - You installed a **single key/value** store of your choice: 56 | 57 | - In this case, navigate to the store folder, for example `libkv/store/etcd/v3` and run: 58 | 59 | go test . 60 | 61 | - Finally, test for race conditions using the following command: 62 | 63 | go test -v -race . 64 | 65 | - You installed **all key/value** stores and want to run the whole test suite: 66 | 67 | - At the base of the project directory, run: 68 | 69 | go test ./... 70 | 71 | - To test for race conditions, run: 72 | 73 | go test -v -race ./... 74 | 75 | ### Flush Key/Value pairs and Specific configurations 76 | 77 | Once in a while, you may need to flush key/value pairs from your local store installations: 78 | for example if you stop the tests purposefully with keys still being locked. This section 79 | describes how to easily start distributed backend storage locally and flush the key/value 80 | pairs when needed. 81 | 82 | #### Consul 83 | 84 | To start consul, use the following command: 85 | 86 | consul agent -server -bootstrap -advertise=127.0.0.1 -data-dir /tmp/consul -config-file=/path/to/config.json 87 | 88 | This is pointing to a `config.json` file having the following configuration: 89 | 90 | {"session_ttl_min": "1s"} 91 | 92 | Finally, to flush the key/value pairs: 93 | 94 | rm -rf /tmp/consul 95 | 96 | #### Etcd 97 | 98 | To start etcd, use the following command: 99 | 100 | etcd --data-dir=/tmp/default.etcd --listen-client-urls 'http://0.0.0.0:4001' --advertise-client-urls 'http://localhost:4001' 101 | 102 | To flush key/value pairs: 103 | 104 | rm -rf /tmp/default.etcd 105 | 106 | #### Zookeeper 107 | 108 | To start zookeeper, use: 109 | 110 | zkServer.sh start 111 | 112 | Make sure you modify the `zoo.cfg` file to change the default zookeeper directory for testing. For 113 | example, to have the zookeeper director in the `/tmp` folder, modify the following line: 114 | 115 | dataDir=/tmp/zookeeper 116 | 117 | To flush the key/value pairs: 118 | 119 | rm -rf /tmp/zookeeper 120 | 121 | #### Redis 122 | 123 | To start redis: 124 | 125 | redis-server 126 | 127 | For redis, flushing the key/value pairs is as simple as: 128 | 129 | redis-cli flushall 130 | 131 | #### Convenient scripts 132 | 133 | You can group startup/stop/clean operations for every store with simple scripts: 134 | 135 | - **Start**: 136 | 137 | ``` 138 | #!/bin/bash 139 | 140 | nohup etcd --data-dir=/tmp/default.etcd --listen-client-urls 'http://0.0.0.0:4001' --advertise-client-urls 'http://localhost:4001' &>/dev/null & 141 | nohup consul agent -server -bootstrap -advertise=127.0.0.1 -data-dir /tmp/consul -config-file=/path/to/config.json &>/dev/null & 142 | zkServer start &>/dev/null & 143 | nohup redis-server &>/dev/null & 144 | ``` 145 | 146 | - **Clean**: 147 | 148 | ``` 149 | #!/bin/bash 150 | 151 | rm -rf /tmp/default.etcd 152 | rm -rf /tmp/consul 153 | rm -rf /tmp/zookeeper 154 | redis-cli flushall 155 | ``` 156 | 157 | - **Stop**: 158 | 159 | ``` 160 | #!/bin/bash 161 | 162 | pkill consul 163 | pkill etcd 164 | pkill -f zookeeper 165 | pkill redis 166 | ``` 167 | 168 | ## Before submitting a change 169 | 170 | Make sure you check each of these items before you submit a pull request to avoid 171 | many unnecessary back and forth in github comments (and will help us review and include 172 | the change as soon as possible): 173 | 174 | - **Open an issue** to clearly state the problem. This will be helpful to keep track 175 | of what needs to be fixed. This also helps triaging and prioritising issues. 176 | 177 | - **Run the following command**: `go fmt ./...`, to ensure that your code is properly 178 | formatted. 179 | 180 | - **For non-trivial changes, write a test**: this is to ensure that we don't encounter 181 | any regression in the future. 182 | 183 | - **Write a complete description** for your pull request (avoid using `-m` flag when 184 | committing a change unless it is a trivial one). 185 | 186 | - **Sign-off your commits** using the `-s` flag (you can configure an alias to 187 | `git commit` adding `-s` for convenience). 188 | 189 | - **Squash your commits** if the pull requests includes many commits that are related. 190 | This is to maintain a clean history of the change and better identify faulty commits 191 | if reverting a change is ever needed. We will tell you if squashing your commits is 192 | necessary. 193 | 194 | - **If the change is solving one or more issues listed on the repository**: you can reference 195 | the issue in your comment with `closes #XXX` or `fixes #XXX`. This will automatically close 196 | related issues on merging the change. 197 | 198 | Finally, submit your *Pull Request*. 199 | 200 | ## Your first Pull Request 201 | 202 | You made it to your first Pull Request? It's only the start of the process. 203 | Following steps may include a discussion on the design and tradeoffs of your 204 | proposed solution. 205 | 206 | Additionaly there will be a *code review process* to find out potential bugs. Part 207 | of being a helpful community is to make sure we point out improvements and deliver 208 | actionable items to work towards fixing potential issues. Feel free to ask questions 209 | if you are stuck so we can help you. 210 | 211 | *Don't be discouraged* if your change happens not to be included. All contributions 212 | are helpful in a way. Your PR most certainly made the discussion go forward in many 213 | aspects and helped working towards our common goal of making the project better for 214 | everyone. 215 | 216 | **Welcome!** -------------------------------------------------------------------------------- /LICENSE.code: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | Copyright 2014-2016 Docker, Inc. 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. 192 | -------------------------------------------------------------------------------- /LICENSE.docs: -------------------------------------------------------------------------------- 1 | Attribution-ShareAlike 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More_considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution-ShareAlike 4.0 International Public 58 | License 59 | 60 | By exercising the Licensed Rights (defined below), You accept and agree 61 | to be bound by the terms and conditions of this Creative Commons 62 | Attribution-ShareAlike 4.0 International Public License ("Public 63 | License"). To the extent this Public License may be interpreted as a 64 | contract, You are granted the Licensed Rights in consideration of Your 65 | acceptance of these terms and conditions, and the Licensor grants You 66 | such rights in consideration of benefits the Licensor receives from 67 | making the Licensed Material available under these terms and 68 | conditions. 69 | 70 | 71 | Section 1 -- Definitions. 72 | 73 | a. Adapted Material means material subject to Copyright and Similar 74 | Rights that is derived from or based upon the Licensed Material 75 | and in which the Licensed Material is translated, altered, 76 | arranged, transformed, or otherwise modified in a manner requiring 77 | permission under the Copyright and Similar Rights held by the 78 | Licensor. For purposes of this Public License, where the Licensed 79 | Material is a musical work, performance, or sound recording, 80 | Adapted Material is always produced where the Licensed Material is 81 | synched in timed relation with a moving image. 82 | 83 | b. Adapter's License means the license You apply to Your Copyright 84 | and Similar Rights in Your contributions to Adapted Material in 85 | accordance with the terms and conditions of this Public License. 86 | 87 | c. BY-SA Compatible License means a license listed at 88 | creativecommons.org/compatiblelicenses, approved by Creative 89 | Commons as essentially the equivalent of this Public License. 90 | 91 | d. Copyright and Similar Rights means copyright and/or similar rights 92 | closely related to copyright including, without limitation, 93 | performance, broadcast, sound recording, and Sui Generis Database 94 | Rights, without regard to how the rights are labeled or 95 | categorized. For purposes of this Public License, the rights 96 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 97 | Rights. 98 | 99 | e. Effective Technological Measures means those measures that, in the 100 | absence of proper authority, may not be circumvented under laws 101 | fulfilling obligations under Article 11 of the WIPO Copyright 102 | Treaty adopted on December 20, 1996, and/or similar international 103 | agreements. 104 | 105 | f. Exceptions and Limitations means fair use, fair dealing, and/or 106 | any other exception or limitation to Copyright and Similar Rights 107 | that applies to Your use of the Licensed Material. 108 | 109 | g. License Elements means the license attributes listed in the name 110 | of a Creative Commons Public License. The License Elements of this 111 | Public License are Attribution and ShareAlike. 112 | 113 | h. Licensed Material means the artistic or literary work, database, 114 | or other material to which the Licensor applied this Public 115 | License. 116 | 117 | i. Licensed Rights means the rights granted to You subject to the 118 | terms and conditions of this Public License, which are limited to 119 | all Copyright and Similar Rights that apply to Your use of the 120 | Licensed Material and that the Licensor has authority to license. 121 | 122 | j. Licensor means the individual(s) or entity(ies) granting rights 123 | under this Public License. 124 | 125 | k. Share means to provide material to the public by any means or 126 | process that requires permission under the Licensed Rights, such 127 | as reproduction, public display, public performance, distribution, 128 | dissemination, communication, or importation, and to make material 129 | available to the public including in ways that members of the 130 | public may access the material from a place and at a time 131 | individually chosen by them. 132 | 133 | l. Sui Generis Database Rights means rights other than copyright 134 | resulting from Directive 96/9/EC of the European Parliament and of 135 | the Council of 11 March 1996 on the legal protection of databases, 136 | as amended and/or succeeded, as well as other essentially 137 | equivalent rights anywhere in the world. 138 | 139 | m. You means the individual or entity exercising the Licensed Rights 140 | under this Public License. Your has a corresponding meaning. 141 | 142 | 143 | Section 2 -- Scope. 144 | 145 | a. License grant. 146 | 147 | 1. Subject to the terms and conditions of this Public License, 148 | the Licensor hereby grants You a worldwide, royalty-free, 149 | non-sublicensable, non-exclusive, irrevocable license to 150 | exercise the Licensed Rights in the Licensed Material to: 151 | 152 | a. reproduce and Share the Licensed Material, in whole or 153 | in part; and 154 | 155 | b. produce, reproduce, and Share Adapted Material. 156 | 157 | 2. Exceptions and Limitations. For the avoidance of doubt, where 158 | Exceptions and Limitations apply to Your use, this Public 159 | License does not apply, and You do not need to comply with 160 | its terms and conditions. 161 | 162 | 3. Term. The term of this Public License is specified in Section 163 | 6(a). 164 | 165 | 4. Media and formats; technical modifications allowed. The 166 | Licensor authorizes You to exercise the Licensed Rights in 167 | all media and formats whether now known or hereafter created, 168 | and to make technical modifications necessary to do so. The 169 | Licensor waives and/or agrees not to assert any right or 170 | authority to forbid You from making technical modifications 171 | necessary to exercise the Licensed Rights, including 172 | technical modifications necessary to circumvent Effective 173 | Technological Measures. For purposes of this Public License, 174 | simply making modifications authorized by this Section 2(a) 175 | (4) never produces Adapted Material. 176 | 177 | 5. Downstream recipients. 178 | 179 | a. Offer from the Licensor -- Licensed Material. Every 180 | recipient of the Licensed Material automatically 181 | receives an offer from the Licensor to exercise the 182 | Licensed Rights under the terms and conditions of this 183 | Public License. 184 | 185 | b. Additional offer from the Licensor -- Adapted Material. 186 | Every recipient of Adapted Material from You 187 | automatically receives an offer from the Licensor to 188 | exercise the Licensed Rights in the Adapted Material 189 | under the conditions of the Adapter's License You apply. 190 | 191 | c. No downstream restrictions. You may not offer or impose 192 | any additional or different terms or conditions on, or 193 | apply any Effective Technological Measures to, the 194 | Licensed Material if doing so restricts exercise of the 195 | Licensed Rights by any recipient of the Licensed 196 | Material. 197 | 198 | 6. No endorsement. Nothing in this Public License constitutes or 199 | may be construed as permission to assert or imply that You 200 | are, or that Your use of the Licensed Material is, connected 201 | with, or sponsored, endorsed, or granted official status by, 202 | the Licensor or others designated to receive attribution as 203 | provided in Section 3(a)(1)(A)(i). 204 | 205 | b. Other rights. 206 | 207 | 1. Moral rights, such as the right of integrity, are not 208 | licensed under this Public License, nor are publicity, 209 | privacy, and/or other similar personality rights; however, to 210 | the extent possible, the Licensor waives and/or agrees not to 211 | assert any such rights held by the Licensor to the limited 212 | extent necessary to allow You to exercise the Licensed 213 | Rights, but not otherwise. 214 | 215 | 2. Patent and trademark rights are not licensed under this 216 | Public License. 217 | 218 | 3. To the extent possible, the Licensor waives any right to 219 | collect royalties from You for the exercise of the Licensed 220 | Rights, whether directly or through a collecting society 221 | under any voluntary or waivable statutory or compulsory 222 | licensing scheme. In all other cases the Licensor expressly 223 | reserves any right to collect such royalties. 224 | 225 | 226 | Section 3 -- License Conditions. 227 | 228 | Your exercise of the Licensed Rights is expressly made subject to the 229 | following conditions. 230 | 231 | a. Attribution. 232 | 233 | 1. If You Share the Licensed Material (including in modified 234 | form), You must: 235 | 236 | a. retain the following if it is supplied by the Licensor 237 | with the Licensed Material: 238 | 239 | i. identification of the creator(s) of the Licensed 240 | Material and any others designated to receive 241 | attribution, in any reasonable manner requested by 242 | the Licensor (including by pseudonym if 243 | designated); 244 | 245 | ii. a copyright notice; 246 | 247 | iii. a notice that refers to this Public License; 248 | 249 | iv. a notice that refers to the disclaimer of 250 | warranties; 251 | 252 | v. a URI or hyperlink to the Licensed Material to the 253 | extent reasonably practicable; 254 | 255 | b. indicate if You modified the Licensed Material and 256 | retain an indication of any previous modifications; and 257 | 258 | c. indicate the Licensed Material is licensed under this 259 | Public License, and include the text of, or the URI or 260 | hyperlink to, this Public License. 261 | 262 | 2. You may satisfy the conditions in Section 3(a)(1) in any 263 | reasonable manner based on the medium, means, and context in 264 | which You Share the Licensed Material. For example, it may be 265 | reasonable to satisfy the conditions by providing a URI or 266 | hyperlink to a resource that includes the required 267 | information. 268 | 269 | 3. If requested by the Licensor, You must remove any of the 270 | information required by Section 3(a)(1)(A) to the extent 271 | reasonably practicable. 272 | 273 | b. ShareAlike. 274 | 275 | In addition to the conditions in Section 3(a), if You Share 276 | Adapted Material You produce, the following conditions also apply. 277 | 278 | 1. The Adapter's License You apply must be a Creative Commons 279 | license with the same License Elements, this version or 280 | later, or a BY-SA Compatible License. 281 | 282 | 2. You must include the text of, or the URI or hyperlink to, the 283 | Adapter's License You apply. You may satisfy this condition 284 | in any reasonable manner based on the medium, means, and 285 | context in which You Share Adapted Material. 286 | 287 | 3. You may not offer or impose any additional or different terms 288 | or conditions on, or apply any Effective Technological 289 | Measures to, Adapted Material that restrict exercise of the 290 | rights granted under the Adapter's License You apply. 291 | 292 | 293 | Section 4 -- Sui Generis Database Rights. 294 | 295 | Where the Licensed Rights include Sui Generis Database Rights that 296 | apply to Your use of the Licensed Material: 297 | 298 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 299 | to extract, reuse, reproduce, and Share all or a substantial 300 | portion of the contents of the database; 301 | 302 | b. if You include all or a substantial portion of the database 303 | contents in a database in which You have Sui Generis Database 304 | Rights, then the database in which You have Sui Generis Database 305 | Rights (but not its individual contents) is Adapted Material, 306 | 307 | including for purposes of Section 3(b); and 308 | c. You must comply with the conditions in Section 3(a) if You Share 309 | all or a substantial portion of the contents of the database. 310 | 311 | For the avoidance of doubt, this Section 4 supplements and does not 312 | replace Your obligations under this Public License where the Licensed 313 | Rights include other Copyright and Similar Rights. 314 | 315 | 316 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 317 | 318 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 319 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 320 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 321 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 322 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 323 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 324 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 325 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 326 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 327 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 328 | 329 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 330 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 331 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 332 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 333 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 334 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 335 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 336 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 337 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 338 | 339 | c. The disclaimer of warranties and limitation of liability provided 340 | above shall be interpreted in a manner that, to the extent 341 | possible, most closely approximates an absolute disclaimer and 342 | waiver of all liability. 343 | 344 | 345 | Section 6 -- Term and Termination. 346 | 347 | a. This Public License applies for the term of the Copyright and 348 | Similar Rights licensed here. However, if You fail to comply with 349 | this Public License, then Your rights under this Public License 350 | terminate automatically. 351 | 352 | b. Where Your right to use the Licensed Material has terminated under 353 | Section 6(a), it reinstates: 354 | 355 | 1. automatically as of the date the violation is cured, provided 356 | it is cured within 30 days of Your discovery of the 357 | violation; or 358 | 359 | 2. upon express reinstatement by the Licensor. 360 | 361 | For the avoidance of doubt, this Section 6(b) does not affect any 362 | right the Licensor may have to seek remedies for Your violations 363 | of this Public License. 364 | 365 | c. For the avoidance of doubt, the Licensor may also offer the 366 | Licensed Material under separate terms or conditions or stop 367 | distributing the Licensed Material at any time; however, doing so 368 | will not terminate this Public License. 369 | 370 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 371 | License. 372 | 373 | 374 | Section 7 -- Other Terms and Conditions. 375 | 376 | a. The Licensor shall not be bound by any additional or different 377 | terms or conditions communicated by You unless expressly agreed. 378 | 379 | b. Any arrangements, understandings, or agreements regarding the 380 | Licensed Material not stated herein are separate from and 381 | independent of the terms and conditions of this Public License. 382 | 383 | 384 | Section 8 -- Interpretation. 385 | 386 | a. For the avoidance of doubt, this Public License does not, and 387 | shall not be interpreted to, reduce, limit, restrict, or impose 388 | conditions on any use of the Licensed Material that could lawfully 389 | be made without permission under this Public License. 390 | 391 | b. To the extent possible, if any provision of this Public License is 392 | deemed unenforceable, it shall be automatically reformed to the 393 | minimum extent necessary to make it enforceable. If the provision 394 | cannot be reformed, it shall be severed from this Public License 395 | without affecting the enforceability of the remaining terms and 396 | conditions. 397 | 398 | c. No term or condition of this Public License will be waived and no 399 | failure to comply consented to unless expressly agreed to by the 400 | Licensor. 401 | 402 | d. Nothing in this Public License constitutes or may be interpreted 403 | as a limitation upon, or waiver of, any privileges and immunities 404 | that apply to the Licensor or You, including from the legal 405 | processes of any jurisdiction or authority. 406 | 407 | 408 | ======================================================================= 409 | 410 | Creative Commons is not a party to its public licenses. 411 | Notwithstanding, Creative Commons may elect to apply one of its public 412 | licenses to material it publishes and in those instances will be 413 | considered the "Licensor." Except for the limited purpose of indicating 414 | that material is shared under a Creative Commons public license or as 415 | otherwise permitted by the Creative Commons policies published at 416 | creativecommons.org/policies, Creative Commons does not authorize the 417 | use of the trademark "Creative Commons" or any other trademark or logo 418 | of Creative Commons without its prior written consent including, 419 | without limitation, in connection with any unauthorized modifications 420 | to any of its public licenses or any other arrangements, 421 | understandings, or agreements concerning use of licensed material. For 422 | the avoidance of doubt, this paragraph does not form part of the public 423 | licenses. 424 | 425 | Creative Commons may be contacted at creativecommons.org. 426 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # libkv 2 | 3 | [![GoDoc](https://godoc.org/github.com/docker/libkv?status.png)](https://godoc.org/github.com/abronan/libkv) 4 | [![Build Status](https://travis-ci.org/docker/libkv.svg?branch=master)](https://travis-ci.org/abronan/libkv) 5 | [![Coverage Status](https://coveralls.io/repos/docker/libkv/badge.svg)](https://coveralls.io/r/abronan/libkv) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/abronan/libkv)](https://goreportcard.com/report/github.com/abronan/libkv) 7 | 8 | `libkv` provides a `Go` native library to store metadata using Distributed Key/Value stores (or common databases). 9 | 10 | The goal of `libkv` is to abstract common store operations (Get/Put/List/etc.) for multiple distributed and/or local Key/Value store backends thus using the same self-contained codebase to manage them all. 11 | 12 | This repository is a fork of the [docker/libkv](https://github.com/docker/libkv) project which includes many fixes/additional features and is maintained by an original project maintainer. This project is notably used by [containous/traefik](https://github.com/containous/traefik), [docker/swarm](https://github.com/docker/swarm) and [docker/libnetwork](https://github.com/docker/libnetwork). 13 | 14 | As of now, `libkv` offers support for `Consul`, `Etcd`, `Zookeeper`, `Redis` (**Distributed** store) and `BoltDB` (**Local** store). 15 | 16 | ## Usage 17 | 18 | `libkv` is meant to be used as an abstraction layer over existing distributed Key/Value stores. It is especially useful if you plan to support `consul`, `etcd` and `zookeeper` using the same codebase. 19 | 20 | It is ideal if you plan for something written in Go that should support: 21 | 22 | - A simple metadata storage, distributed or local 23 | - A lightweight discovery service for your nodes 24 | - A distributed lock mechanism 25 | 26 | You can also easily implement a generic *Leader Election* algorithm on top of it (see the [docker/leadership](https://github.com/docker/leadership) repository). 27 | 28 | You can find examples of usage for `libkv` under in [docs/examples.go](https://github.com/abronan/libkv/blob/master/docs/examples.md). Optionally you can also take a look at the `docker/swarm`, `docker/libnetwork` or `containous/traefik` repositories which are using `libkv` for all the use cases listed above. 29 | 30 | ## Supported versions 31 | 32 | `libkv` supports: 33 | - **Consul** versions >= `0.5.1` because it uses Sessions with `Delete` behavior for the use of `TTLs` (mimics zookeeper's Ephemeral node support), If you don't plan to use `TTLs`: you can use Consul version `0.4.0+`. 34 | - **Etcd** versions >= `2.0` with **APIv2** (*deprecated*) and **APIv3** (*recommended*). 35 | - **Zookeeper** versions >= `3.4.5`. Although this might work with previous version but this remains untested as of now. 36 | - **Boltdb**, which shouldn't be subject to any version dependencies. 37 | - **Redis** versions >= `3.2.6`. Although this might work with previous version but this remains untested as of now. 38 | 39 | ## Interface 40 | 41 | A **storage backend** in `libkv` should implement (fully or partially) these interfaces: 42 | 43 | ```go 44 | type Store interface { 45 | Put(key string, value []byte, options *WriteOptions) error 46 | Get(key string, options *ReadOptions) (*KVPair, error) 47 | Delete(key string) error 48 | Exists(key string, options *ReadOptions) (bool, error) 49 | Watch(key string, stopCh <-chan struct{}, options *ReadOptions) (<-chan *KVPair, error) 50 | WatchTree(directory string, stopCh <-chan struct{}, options *ReadOptions) (<-chan []*KVPair, error) 51 | NewLock(key string, options *LockOptions) (Locker, error) 52 | List(directory string, options *ReadOptions) ([]*KVPair, error) 53 | DeleteTree(directory string) error 54 | AtomicPut(key string, value []byte, previous *KVPair, options *WriteOptions) (bool, *KVPair, error) 55 | AtomicDelete(key string, previous *KVPair) (bool, error) 56 | Close() 57 | } 58 | 59 | type Locker interface { 60 | Lock(stopChan chan struct{}) (<-chan struct{}, error) 61 | Unlock() error 62 | } 63 | ``` 64 | 65 | ## Compatibility matrix 66 | 67 | Backend drivers in `libkv` are generally divided between **local drivers** and **distributed drivers**. Distributed backends offer enhanced capabilities like `Watches` and/or distributed `Locks`. 68 | 69 | Local drivers are usually used in complement to the distributed drivers to store informations that only needs to be available locally. 70 | 71 | | Calls | Consul | Etcd | Zookeeper | Redis | BoltDB | 72 | |-----------------------|:----------:|:------:|:-----------:|:----------:|:--------:| 73 | | Put | X | X | X | X | X | 74 | | Get | X | X | X | X | X | 75 | | Delete | X | X | X | X | X | 76 | | Exists | X | X | X | X | X | 77 | | Watch | X | X | X | X | | 78 | | WatchTree | X | X | X | X | | 79 | | NewLock (Lock/Unlock) | X | X | X | X | | 80 | | List | X | X | X | X | X | 81 | | DeleteTree | X | X | X | X | X | 82 | | AtomicPut | X | X | X | X | X | 83 | | AtomicDelete | X | X | X | X | X | 84 | | Close | X | X | X | X | X | 85 | 86 | ## Limitations 87 | 88 | Distributed Key/Value stores often have different concepts for managing and formatting keys and their associated values. Even though `libkv` tries to abstract those stores aiming for some consistency, in some cases it can't be applied easily. 89 | 90 | Please refer to the `docs/compatibility.md` to see what are the special cases for cross-backend compatibility. 91 | 92 | Other than those special cases, you should expect the same experience for basic operations like `Get`/`Put`, etc. 93 | 94 | Calls like `WatchTree` may return different events (or number of events) depending on the backend (for now, `Etcd` and `Consul` will likely return more events than `Zookeeper` that you should triage properly). Although you should be able to use it successfully to watch on events in an interchangeable way (see the **docker/leadership** repository or the **pkg/discovery/kv** package in **docker/docker**). 95 | 96 | For `Redis` backend, it relies on [key space notification](https://redis.io/topics/notifications) to perform WatchXXX/Lock related features. Please read the doc before using this feature. 97 | 98 | ## TLS 99 | 100 | Only `Consul` and `etcd` have support for TLS and you should build and provide your own `config.TLS` object to feed the client. Support is planned for `zookeeper` and `redis`. 101 | 102 | ## Contributing 103 | 104 | Want to contribute to libkv? Take a look at the [Contribution Guidelines](https://github.com/abronan/libkv/blob/master/CONTRIBUTING.md). 105 | 106 | ## Maintainers 107 | 108 | **Alexandre Beslic** 109 | 110 | - [abronan.com](https://abronan.com) 111 | - [@abronan](https://twitter.com/abronan) 112 | 113 | ## Copyright and license 114 | 115 | Copyright © 2014-2016 Docker, Inc. All rights reserved, except as follows. Code is released under the Apache 2.0 license. The README.md file, and files in the "docs" folder are licensed under the Creative Commons Attribution 4.0 International License under the terms and conditions set forth in the file "LICENSE.docs". You may obtain a duplicate copy of the same license, titled CC-BY-SA-4.0, at http://creativecommons.org/licenses/by/4.0/. 116 | -------------------------------------------------------------------------------- /docs/compatibility.md: -------------------------------------------------------------------------------- 1 | # Cross-Backend Compatibility 2 | 3 | The value of `libkv` is not to duplicate the code for programs that should support multiple distributed Key/Value stores such as `Consul`/`etcd`/`zookeeper`, etc. 4 | 5 | This document offers general guidelines for users willing to support those backends with the same codebase using `libkv`. 6 | 7 | ## Etcd versions 8 | 9 | Support for etcd comes with two API versions: **APIv2** and **APIv3**. We recommend you use the new **APIv3** because of incompatibilities of **APIv2** with other stores. Use **APIv2** only if you plan to support older versions of etcd. 10 | 11 | ### Etcd APIv2 pitfalls 12 | 13 | In the case you plan to support etcd with **APIv2**, please be aware of some pitfalls. 14 | 15 | #### Etcd directory/key distinction 16 | 17 | `etcd` with APIv2 makes the distinction between keys and directories. The result with `libkv` is that when using the etcd driver: 18 | 19 | - You cannot store values on directories 20 | - You cannot invoke `WatchTree` (watching on child values), on a regular key 21 | 22 | This is fundamental difference from stores like `Consul` and `zookeeper` which are more permissive and allow the same set of operations on keys and directories (called a Node for zookeeper). 23 | 24 | ### Put 25 | 26 | `etcd` cannot put values on directories, so this puts a major restriction compared to `Consul` and `zookeeper`. 27 | 28 | If you want to support all those three backends, you should make sure to only put data on **leaves**. 29 | 30 | For example: 31 | 32 | ```go 33 | _ := kv.Put("path/to/key/bis", []byte("foo"), nil) 34 | _ := kv.Put("path/to/key", []byte("bar"), nil) 35 | ``` 36 | 37 | Will work on `Consul` and `zookeeper` but fail for `etcd`. This is because the first `Put` in the case of `etcd` will recursively create the directory hierarchy and `path/to/key` is now considered as a directory. Thus, values should always be stored on leaves if the support for all backends is planned. 38 | 39 | #### WatchTree 40 | 41 | When initializing the `WatchTree`, the natural way to do so is through the following code: 42 | 43 | ```go 44 | key := "path/to/key" 45 | if !kv.Exists(key, nil) { 46 | err := kv.Put(key, []byte("data"), nil) 47 | } 48 | events, err := kv.WatchTree(key, nil, nil) 49 | ``` 50 | 51 | The code above will not work across backends and etcd with **APIv2** will fail on the `WatchTree` call. What happens exactly: 52 | 53 | - `Consul` will create a regular `key` because it has no distinction between directories and keys. This is not an issue as we can invoke `WatchTree` on regular keys. 54 | - `zookeeper` is going to create a `node` that can either be a directory or a key during the lifetime of a program but it does not matter as a directory can hold values and be watchable like a regular key. 55 | - `etcd` is going to create a regular `key`. We cannot invoke `WatchTree` on regular keys using etcd. 56 | 57 | To use `WatchTree` with every supported backend, we need to enforce a parameter that is only interpreted with `etcd` **APIv2** and which asks the client to create a `directory` instead of a key. 58 | 59 | ```go 60 | key := "path/to/key" 61 | if !kv.Exists(key, nil) { 62 | // We enforce IsDir = true to make sure etcd creates a directory 63 | err := kv.Put(key, []byte("data"), &store.WriteOptions{IsDir:true}) 64 | } 65 | events, err := kv.WatchTree(key, nil, nil) 66 | ``` 67 | 68 | The code above will work for all backends but make sure not to try to store any value on that path as the call to `Put` will fail for `etcd` (you can only put at `path/to/key/foo`, `path/to/key/bar` for example). 69 | -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | This document contains useful example of usage for `libkv`. It might not be complete but provides with general informations on how to use the client. 4 | 5 | ## Create a store and use Put/Get/Delete 6 | 7 | ```go 8 | package main 9 | 10 | import ( 11 | "fmt" 12 | "time" 13 | "log" 14 | 15 | "github.com/docker/libkv" 16 | "github.com/docker/libkv/store" 17 | "github.com/docker/libkv/store/boltdb" 18 | "github.com/docker/libkv/store/consul" 19 | "github.com/docker/libkv/store/etcd/v3" 20 | "github.com/docker/libkv/store/zookeeper" 21 | "github.com/docker/libkv/store/redis" 22 | ) 23 | 24 | func init() { 25 | // Register consul store to libkv 26 | consul.Register() 27 | 28 | // We can register more backends that are supported by 29 | // libkv if we plan to use these 30 | etcdv3.Register() 31 | zookeeper.Register() 32 | boltdb.Register() 33 | redis.Register() 34 | } 35 | 36 | func main() { 37 | client := "localhost:8500" 38 | 39 | // Initialize a new store with consul 40 | kv, err := libkv.NewStore( 41 | store.CONSUL, // or "consul" 42 | []string{client}, 43 | &store.Config{ 44 | ConnectionTimeout: 10*time.Second, 45 | }, 46 | ) 47 | if err != nil { 48 | log.Fatal("Cannot create store consul") 49 | } 50 | 51 | key := "foo" 52 | err = kv.Put(key, []byte("bar"), nil) 53 | if err != nil { 54 | fmt.Errorf("Error trying to put value at key: %v", key) 55 | } 56 | 57 | pair, err := kv.Get(key, nil) 58 | if err != nil { 59 | fmt.Errorf("Error trying accessing value at key: %v", key) 60 | } 61 | 62 | err = kv.Delete(key) 63 | if err != nil { 64 | fmt.Errorf("Error trying to delete key %v", key) 65 | } 66 | 67 | log.Info("value: ", string(pair.Value)) 68 | } 69 | ``` 70 | 71 | ## List keys 72 | 73 | ```go 74 | // List will list all the keys under `key` if it contains a set of child keys/values 75 | entries, err := kv.List(key, nil) 76 | for _, pair := range entries { 77 | fmt.Printf("key=%v - value=%v", pair.Key, string(pair.Value)) 78 | } 79 | 80 | ``` 81 | 82 | ## Watching for events on a single key (Watch) 83 | 84 | You can use watches to watch modifications on a key. First you need to check if the key exists. If this is not the case, we need to create it using the `Put` function. 85 | 86 | ```go 87 | // Checking on the key before watching 88 | if !kv.Exists(key, nil) { 89 | err := kv.Put(key, []byte("bar"), nil) 90 | if err != nil { 91 | fmt.Errorf("Something went wrong when initializing key %v", key) 92 | } 93 | } 94 | 95 | stopCh := make(<-chan struct{}) 96 | events, err := kv.Watch(key, stopCh, nil) 97 | 98 | for { 99 | select { 100 | case pair := <-events: 101 | // Do something with events 102 | fmt.Printf("value changed on key %v: new value=%v", key, pair.Value) 103 | } 104 | } 105 | 106 | ``` 107 | 108 | ## Watching for events happening on child keys (WatchTree) 109 | 110 | You can use watches to watch modifications on a key. First you need to check if the key exists. If this is not the case, we need to create it using the `Put` function. There is a special step here if you are using etcd **APIv2** and if want your code to work across backends. `etcd` with **APIv2** makes the distinction between directories and keys, we need to make sure that the created key is considered as a directory by enforcing `IsDir` at `true`. 111 | 112 | ```go 113 | // Checking on the key before watching 114 | if !kv.Exists(key, nil) { 115 | // Do not forget `IsDir:true` if you are using etcd APIv2 116 | err := kv.Put(key, []byte("bar"), &store.WriteOptions{IsDir:true}) 117 | if err != nil { 118 | fmt.Errorf("Something went wrong when initializing key %v", key) 119 | } 120 | } 121 | 122 | stopCh := make(<-chan struct{}) 123 | events, err := kv.WatchTree(key, stopCh, nil) 124 | 125 | select { 126 | case pairs := <-events: 127 | // Do something with events 128 | for _, pair := range pairs { 129 | fmt.Printf("value changed on key %v: new value=%v", key, pair.Value) 130 | } 131 | } 132 | 133 | ``` 134 | 135 | ## Distributed Locking, using Lock/Unlock 136 | 137 | ```go 138 | key := "lockKey" 139 | value := []byte("bar") 140 | 141 | // Initialize a distributed lock. TTL is optional, it is here to make sure that 142 | // the lock is released after the program that is holding the lock ends or crashes 143 | lock, err := kv.NewLock(key, &store.LockOptions{Value: value, TTL: 2 * time.Second}) 144 | if err != nil { 145 | fmt.Errorf("something went wrong when trying to initialize the Lock") 146 | } 147 | 148 | // Try to lock the key, the call to Lock() is blocking 149 | _, err := lock.Lock(nil) 150 | if err != nil { 151 | fmt.Errorf("something went wrong when trying to lock key %v", key) 152 | } 153 | 154 | // Get should work because we are holding the key 155 | pair, err := kv.Get(key, nil) 156 | if err != nil { 157 | fmt.Errorf("key %v has value %v", key, pair.Value) 158 | } 159 | 160 | // Unlock the key 161 | err = lock.Unlock() 162 | if err != nil { 163 | fmt.Errorf("something went wrong when trying to unlock key %v", key) 164 | } 165 | ``` 166 | -------------------------------------------------------------------------------- /libkv.go: -------------------------------------------------------------------------------- 1 | package libkv 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | 8 | "github.com/docker/libkv/store" 9 | ) 10 | 11 | // Initialize creates a new Store object, initializing the client 12 | type Initialize func(addrs []string, options *store.Config) (store.Store, error) 13 | 14 | var ( 15 | // Backend initializers 16 | initializers = make(map[store.Backend]Initialize) 17 | 18 | supportedBackend = func() string { 19 | keys := make([]string, 0, len(initializers)) 20 | for k := range initializers { 21 | keys = append(keys, string(k)) 22 | } 23 | sort.Strings(keys) 24 | return strings.Join(keys, ", ") 25 | }() 26 | ) 27 | 28 | // NewStore creates an instance of store 29 | func NewStore(backend store.Backend, addrs []string, options *store.Config) (store.Store, error) { 30 | if init, exists := initializers[backend]; exists { 31 | return init(addrs, options) 32 | } 33 | 34 | return nil, fmt.Errorf("%s %s", store.ErrBackendNotSupported.Error(), supportedBackend) 35 | } 36 | 37 | // AddStore adds a new store backend to libkv 38 | func AddStore(store store.Backend, init Initialize) { 39 | initializers[store] = init 40 | } 41 | -------------------------------------------------------------------------------- /libkv_test.go: -------------------------------------------------------------------------------- 1 | package libkv 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/docker/libkv/store" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestNewStoreUnsupported(t *testing.T) { 12 | client := "localhost:9999" 13 | 14 | kv, err := NewStore( 15 | "unsupported", 16 | []string{client}, 17 | &store.Config{ 18 | ConnectionTimeout: 10 * time.Second, 19 | }, 20 | ) 21 | assert.Error(t, err) 22 | assert.Nil(t, kv) 23 | assert.Equal(t, "Backend storage not supported yet, please choose one of ", err.Error()) 24 | } 25 | -------------------------------------------------------------------------------- /script/.validate: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$VALIDATE_UPSTREAM" ]; then 4 | # this is kind of an expensive check, so let's not do this twice if we 5 | # are running more than one validate bundlescript 6 | 7 | VALIDATE_REPO='https://github.com/docker/libkv.git' 8 | VALIDATE_BRANCH='master' 9 | 10 | if [ "$TRAVIS" = 'true' -a "$TRAVIS_PULL_REQUEST" != 'false' ]; then 11 | VALIDATE_REPO="https://github.com/${TRAVIS_REPO_SLUG}.git" 12 | VALIDATE_BRANCH="${TRAVIS_BRANCH}" 13 | fi 14 | 15 | VALIDATE_HEAD="$(git rev-parse --verify HEAD)" 16 | 17 | git fetch -q "$VALIDATE_REPO" "refs/heads/$VALIDATE_BRANCH" 18 | VALIDATE_UPSTREAM="$(git rev-parse --verify FETCH_HEAD)" 19 | 20 | VALIDATE_COMMIT_LOG="$VALIDATE_UPSTREAM..$VALIDATE_HEAD" 21 | VALIDATE_COMMIT_DIFF="$VALIDATE_UPSTREAM...$VALIDATE_HEAD" 22 | 23 | validate_diff() { 24 | if [ "$VALIDATE_UPSTREAM" != "$VALIDATE_HEAD" ]; then 25 | git diff "$VALIDATE_COMMIT_DIFF" "$@" 26 | fi 27 | } 28 | validate_log() { 29 | if [ "$VALIDATE_UPSTREAM" != "$VALIDATE_HEAD" ]; then 30 | git log "$VALIDATE_COMMIT_LOG" "$@" 31 | fi 32 | } 33 | fi 34 | -------------------------------------------------------------------------------- /script/coverage: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | MODE="mode: count" 4 | ROOT=${TRAVIS_BUILD_DIR:-.}/../../.. 5 | 6 | # Grab the list of packages. 7 | # Exclude the API and CLI from coverage as it will be covered by integration tests. 8 | PACKAGES=`go list ./...` 9 | 10 | # Create the empty coverage file. 11 | echo $MODE > goverage.report 12 | 13 | # Run coverage on every package. 14 | for package in $PACKAGES; do 15 | output="$ROOT/$package/coverage.out" 16 | 17 | go test -test.short -covermode=count -coverprofile=$output $package 18 | if [ -f "$output" ] ; then 19 | cat "$output" | grep -v "$MODE" >> goverage.report 20 | fi 21 | done 22 | -------------------------------------------------------------------------------- /script/travis_consul.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -gt 0 ] ; then 4 | CONSUL_VERSION="$1" 5 | else 6 | CONSUL_VERSION="0.5.2" 7 | fi 8 | 9 | # install consul 10 | wget "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" 11 | unzip "consul_${CONSUL_VERSION}_linux_amd64.zip" 12 | 13 | # make config for minimum ttl 14 | touch config.json 15 | echo "{\"session_ttl_min\": \"1s\"}" >> config.json 16 | 17 | # check 18 | ./consul --version 19 | -------------------------------------------------------------------------------- /script/travis_etcd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -gt 0 ] ; then 4 | ETCD_VERSION="$1" 5 | else 6 | ETCD_VERSION="2.2.0" 7 | fi 8 | 9 | curl -L https://github.com/coreos/etcd/releases/download/v$ETCD_VERSION/etcd-v$ETCD_VERSION-linux-amd64.tar.gz -o etcd-v$ETCD_VERSION-linux-amd64.tar.gz 10 | tar xzvf etcd-v$ETCD_VERSION-linux-amd64.tar.gz 11 | mv etcd-v$ETCD_VERSION-linux-amd64 etcd 12 | -------------------------------------------------------------------------------- /script/travis_redis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -gt 0 ] ; then 4 | REDIS_VERSION="$1" 5 | else 6 | REDIS_VERSION="3.2.6" 7 | fi 8 | 9 | # install redis 10 | 11 | curl -L http://download.redis.io/releases/redis-$REDIS_VERSION.tar.gz -o redis.tar.gz 12 | tar xzf redis.tar.gz && mv redis-$REDIS_VERSION redis && cd redis && make 13 | -------------------------------------------------------------------------------- /script/travis_zk.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -gt 0 ] ; then 4 | ZK_VERSION="$1" 5 | else 6 | ZK_VERSION="3.4.7" 7 | fi 8 | 9 | wget "http://apache.cs.utah.edu/zookeeper/zookeeper-${ZK_VERSION}/zookeeper-${ZK_VERSION}.tar.gz" 10 | tar -xvf "zookeeper-${ZK_VERSION}.tar.gz" 11 | mv zookeeper-$ZK_VERSION zk 12 | mv ./zk/conf/zoo_sample.cfg ./zk/conf/zoo.cfg 13 | -------------------------------------------------------------------------------- /script/validate-gofmt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source "$(dirname "$BASH_SOURCE")/.validate" 4 | 5 | IFS=$'\n' 6 | files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^Godeps/' || true) ) 7 | unset IFS 8 | 9 | badFiles=() 10 | for f in "${files[@]}"; do 11 | # we use "git show" here to validate that what's committed is formatted 12 | if [ "$(git show "$VALIDATE_HEAD:$f" | gofmt -s -l)" ]; then 13 | badFiles+=( "$f" ) 14 | fi 15 | done 16 | 17 | if [ ${#badFiles[@]} -eq 0 ]; then 18 | echo 'Congratulations! All Go source files are properly formatted.' 19 | else 20 | { 21 | echo "These files are not properly gofmt'd:" 22 | for f in "${badFiles[@]}"; do 23 | echo " - $f" 24 | done 25 | echo 26 | echo 'Please reformat the above files using "gofmt -s -w" and commit the result.' 27 | echo 28 | } >&2 29 | false 30 | fi 31 | -------------------------------------------------------------------------------- /store/boltdb/boltdb.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "errors" 7 | "os" 8 | "path/filepath" 9 | "sync" 10 | "sync/atomic" 11 | "time" 12 | 13 | "github.com/coreos/bbolt" 14 | "github.com/docker/libkv" 15 | "github.com/docker/libkv/store" 16 | ) 17 | 18 | var ( 19 | // ErrMultipleEndpointsUnsupported is thrown when multiple endpoints specified for 20 | // BoltDB. Endpoint has to be a local file path 21 | ErrMultipleEndpointsUnsupported = errors.New("boltdb supports one endpoint and should be a file path") 22 | // ErrBoltBucketOptionMissing is thrown when boltBcuket config option is missing 23 | ErrBoltBucketOptionMissing = errors.New("boltBucket config option missing") 24 | ) 25 | 26 | const ( 27 | filePerm os.FileMode = 0644 28 | ) 29 | 30 | //BoltDB type implements the Store interface 31 | type BoltDB struct { 32 | client *bolt.DB 33 | boltBucket []byte 34 | dbIndex uint64 35 | path string 36 | timeout time.Duration 37 | // By default libkv opens and closes the bolt DB connection for every 38 | // get/put operation. This allows multiple apps to use a Bolt DB at the 39 | // same time. 40 | // PersistConnection flag provides an option to override ths behavior. 41 | // ie: open the connection in New and use it till Close is called. 42 | PersistConnection bool 43 | sync.Mutex 44 | } 45 | 46 | const ( 47 | libkvmetadatalen = 8 48 | transientTimeout = time.Duration(10) * time.Second 49 | ) 50 | 51 | // Register registers boltdb to libkv 52 | func Register() { 53 | libkv.AddStore(store.BOLTDB, New) 54 | } 55 | 56 | // New opens a new BoltDB connection to the specified path and bucket 57 | func New(endpoints []string, options *store.Config) (store.Store, error) { 58 | var ( 59 | db *bolt.DB 60 | err error 61 | boltOptions *bolt.Options 62 | timeout = transientTimeout 63 | ) 64 | 65 | if len(endpoints) > 1 { 66 | return nil, ErrMultipleEndpointsUnsupported 67 | } 68 | 69 | if (options == nil) || (len(options.Bucket) == 0) { 70 | return nil, ErrBoltBucketOptionMissing 71 | } 72 | 73 | dir, _ := filepath.Split(endpoints[0]) 74 | if err = os.MkdirAll(dir, 0750); err != nil { 75 | return nil, err 76 | } 77 | 78 | if options.PersistConnection { 79 | boltOptions = &bolt.Options{Timeout: options.ConnectionTimeout} 80 | db, err = bolt.Open(endpoints[0], filePerm, boltOptions) 81 | if err != nil { 82 | return nil, err 83 | } 84 | } 85 | 86 | if options.ConnectionTimeout != 0 { 87 | timeout = options.ConnectionTimeout 88 | } 89 | 90 | b := &BoltDB{ 91 | client: db, 92 | path: endpoints[0], 93 | boltBucket: []byte(options.Bucket), 94 | timeout: timeout, 95 | PersistConnection: options.PersistConnection, 96 | } 97 | 98 | return b, nil 99 | } 100 | 101 | func (b *BoltDB) reset() { 102 | b.path = "" 103 | b.boltBucket = []byte{} 104 | } 105 | 106 | func (b *BoltDB) getDBhandle() (*bolt.DB, error) { 107 | var ( 108 | db *bolt.DB 109 | err error 110 | ) 111 | if !b.PersistConnection { 112 | boltOptions := &bolt.Options{Timeout: b.timeout} 113 | if db, err = bolt.Open(b.path, filePerm, boltOptions); err != nil { 114 | return nil, err 115 | } 116 | b.client = db 117 | } 118 | 119 | return b.client, nil 120 | } 121 | 122 | func (b *BoltDB) releaseDBhandle() { 123 | if !b.PersistConnection { 124 | b.client.Close() 125 | } 126 | } 127 | 128 | // Get the value at "key". BoltDB doesn't provide an inbuilt last modified index with every kv pair. Its implemented by 129 | // by a atomic counter maintained by the libkv and appened to the value passed by the client. 130 | func (b *BoltDB) Get(key string, opts *store.ReadOptions) (*store.KVPair, error) { 131 | var ( 132 | val []byte 133 | db *bolt.DB 134 | err error 135 | ) 136 | b.Lock() 137 | defer b.Unlock() 138 | 139 | if db, err = b.getDBhandle(); err != nil { 140 | return nil, err 141 | } 142 | defer b.releaseDBhandle() 143 | 144 | err = db.View(func(tx *bolt.Tx) error { 145 | bucket := tx.Bucket(b.boltBucket) 146 | if bucket == nil { 147 | return store.ErrKeyNotFound 148 | } 149 | 150 | v := bucket.Get([]byte(key)) 151 | val = make([]byte, len(v)) 152 | copy(val, v) 153 | 154 | return nil 155 | }) 156 | 157 | if len(val) == 0 { 158 | return nil, store.ErrKeyNotFound 159 | } 160 | if err != nil { 161 | return nil, err 162 | } 163 | 164 | dbIndex := binary.LittleEndian.Uint64(val[:libkvmetadatalen]) 165 | val = val[libkvmetadatalen:] 166 | 167 | return &store.KVPair{Key: key, Value: val, LastIndex: (dbIndex)}, nil 168 | } 169 | 170 | //Put the key, value pair. index number metadata is prepended to the value 171 | func (b *BoltDB) Put(key string, value []byte, opts *store.WriteOptions) error { 172 | var ( 173 | dbIndex uint64 174 | db *bolt.DB 175 | err error 176 | ) 177 | b.Lock() 178 | defer b.Unlock() 179 | 180 | dbval := make([]byte, libkvmetadatalen) 181 | 182 | if db, err = b.getDBhandle(); err != nil { 183 | return err 184 | } 185 | defer b.releaseDBhandle() 186 | 187 | err = db.Update(func(tx *bolt.Tx) error { 188 | bucket, err := tx.CreateBucketIfNotExists(b.boltBucket) 189 | if err != nil { 190 | return err 191 | } 192 | 193 | dbIndex = atomic.AddUint64(&b.dbIndex, 1) 194 | binary.LittleEndian.PutUint64(dbval, dbIndex) 195 | dbval = append(dbval, value...) 196 | 197 | err = bucket.Put([]byte(key), dbval) 198 | if err != nil { 199 | return err 200 | } 201 | return nil 202 | }) 203 | return err 204 | } 205 | 206 | //Delete the value for the given key. 207 | func (b *BoltDB) Delete(key string) error { 208 | var ( 209 | db *bolt.DB 210 | err error 211 | ) 212 | b.Lock() 213 | defer b.Unlock() 214 | 215 | if db, err = b.getDBhandle(); err != nil { 216 | return err 217 | } 218 | defer b.releaseDBhandle() 219 | 220 | err = db.Update(func(tx *bolt.Tx) error { 221 | bucket := tx.Bucket(b.boltBucket) 222 | if bucket == nil { 223 | return store.ErrKeyNotFound 224 | } 225 | err := bucket.Delete([]byte(key)) 226 | return err 227 | }) 228 | return err 229 | } 230 | 231 | // Exists checks if the key exists inside the store 232 | func (b *BoltDB) Exists(key string, opts *store.ReadOptions) (bool, error) { 233 | var ( 234 | val []byte 235 | db *bolt.DB 236 | err error 237 | ) 238 | b.Lock() 239 | defer b.Unlock() 240 | 241 | if db, err = b.getDBhandle(); err != nil { 242 | return false, err 243 | } 244 | defer b.releaseDBhandle() 245 | 246 | err = db.View(func(tx *bolt.Tx) error { 247 | bucket := tx.Bucket(b.boltBucket) 248 | if bucket == nil { 249 | return store.ErrKeyNotFound 250 | } 251 | 252 | val = bucket.Get([]byte(key)) 253 | 254 | return nil 255 | }) 256 | 257 | if len(val) == 0 { 258 | return false, err 259 | } 260 | return true, err 261 | } 262 | 263 | // List returns the range of keys starting with the passed in prefix 264 | func (b *BoltDB) List(keyPrefix string, opts *store.ReadOptions) ([]*store.KVPair, error) { 265 | var ( 266 | db *bolt.DB 267 | err error 268 | ) 269 | b.Lock() 270 | defer b.Unlock() 271 | 272 | kv := []*store.KVPair{} 273 | 274 | if db, err = b.getDBhandle(); err != nil { 275 | return nil, err 276 | } 277 | defer b.releaseDBhandle() 278 | hasResult := false 279 | err = db.View(func(tx *bolt.Tx) error { 280 | bucket := tx.Bucket(b.boltBucket) 281 | if bucket == nil { 282 | return store.ErrKeyNotFound 283 | } 284 | 285 | cursor := bucket.Cursor() 286 | prefix := []byte(keyPrefix) 287 | 288 | for key, v := cursor.Seek(prefix); bytes.HasPrefix(key, prefix); key, v = cursor.Next() { 289 | hasResult = true 290 | dbIndex := binary.LittleEndian.Uint64(v[:libkvmetadatalen]) 291 | v = v[libkvmetadatalen:] 292 | val := make([]byte, len(v)) 293 | copy(val, v) 294 | 295 | if string(key) != keyPrefix { 296 | kv = append(kv, &store.KVPair{ 297 | Key: string(key), 298 | Value: val, 299 | LastIndex: dbIndex, 300 | }) 301 | } 302 | } 303 | return nil 304 | }) 305 | if !hasResult { 306 | return nil, store.ErrKeyNotFound 307 | } 308 | return kv, err 309 | } 310 | 311 | // AtomicDelete deletes a value at "key" if the key 312 | // has not been modified in the meantime, throws an 313 | // error if this is the case 314 | func (b *BoltDB) AtomicDelete(key string, previous *store.KVPair) (bool, error) { 315 | var ( 316 | val []byte 317 | db *bolt.DB 318 | err error 319 | ) 320 | b.Lock() 321 | defer b.Unlock() 322 | 323 | if previous == nil { 324 | return false, store.ErrPreviousNotSpecified 325 | } 326 | if db, err = b.getDBhandle(); err != nil { 327 | return false, err 328 | } 329 | defer b.releaseDBhandle() 330 | 331 | err = db.Update(func(tx *bolt.Tx) error { 332 | bucket := tx.Bucket(b.boltBucket) 333 | if bucket == nil { 334 | return store.ErrKeyNotFound 335 | } 336 | 337 | val = bucket.Get([]byte(key)) 338 | if val == nil { 339 | return store.ErrKeyNotFound 340 | } 341 | dbIndex := binary.LittleEndian.Uint64(val[:libkvmetadatalen]) 342 | if dbIndex != previous.LastIndex { 343 | return store.ErrKeyModified 344 | } 345 | err := bucket.Delete([]byte(key)) 346 | return err 347 | }) 348 | if err != nil { 349 | return false, err 350 | } 351 | return true, err 352 | } 353 | 354 | // AtomicPut puts a value at "key" if the key has not been 355 | // modified since the last Put, throws an error if this is the case 356 | func (b *BoltDB) AtomicPut(key string, value []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) { 357 | var ( 358 | val []byte 359 | dbIndex uint64 360 | db *bolt.DB 361 | err error 362 | ) 363 | b.Lock() 364 | defer b.Unlock() 365 | 366 | dbval := make([]byte, libkvmetadatalen) 367 | 368 | if db, err = b.getDBhandle(); err != nil { 369 | return false, nil, err 370 | } 371 | defer b.releaseDBhandle() 372 | 373 | err = db.Update(func(tx *bolt.Tx) error { 374 | var err error 375 | bucket := tx.Bucket(b.boltBucket) 376 | if bucket == nil { 377 | if previous != nil { 378 | return store.ErrKeyNotFound 379 | } 380 | bucket, err = tx.CreateBucket(b.boltBucket) 381 | if err != nil { 382 | return err 383 | } 384 | } 385 | // AtomicPut is equivalent to Put if previous is nil and the Ky 386 | // doesn't exist in the DB. 387 | val = bucket.Get([]byte(key)) 388 | if previous == nil && len(val) != 0 { 389 | return store.ErrKeyExists 390 | } 391 | if previous != nil { 392 | if len(val) == 0 { 393 | return store.ErrKeyNotFound 394 | } 395 | dbIndex = binary.LittleEndian.Uint64(val[:libkvmetadatalen]) 396 | if dbIndex != previous.LastIndex { 397 | return store.ErrKeyModified 398 | } 399 | } 400 | dbIndex = atomic.AddUint64(&b.dbIndex, 1) 401 | binary.LittleEndian.PutUint64(dbval, b.dbIndex) 402 | dbval = append(dbval, value...) 403 | return (bucket.Put([]byte(key), dbval)) 404 | }) 405 | if err != nil { 406 | return false, nil, err 407 | } 408 | 409 | updated := &store.KVPair{ 410 | Key: key, 411 | Value: value, 412 | LastIndex: dbIndex, 413 | } 414 | 415 | return true, updated, nil 416 | } 417 | 418 | // Close the db connection to the BoltDB 419 | func (b *BoltDB) Close() { 420 | b.Lock() 421 | defer b.Unlock() 422 | 423 | if !b.PersistConnection { 424 | b.reset() 425 | } else { 426 | b.client.Close() 427 | } 428 | return 429 | } 430 | 431 | // DeleteTree deletes a range of keys with a given prefix 432 | func (b *BoltDB) DeleteTree(keyPrefix string) error { 433 | var ( 434 | db *bolt.DB 435 | err error 436 | ) 437 | b.Lock() 438 | defer b.Unlock() 439 | 440 | if db, err = b.getDBhandle(); err != nil { 441 | return err 442 | } 443 | defer b.releaseDBhandle() 444 | 445 | err = db.Update(func(tx *bolt.Tx) error { 446 | bucket := tx.Bucket(b.boltBucket) 447 | if bucket == nil { 448 | return store.ErrKeyNotFound 449 | } 450 | 451 | cursor := bucket.Cursor() 452 | prefix := []byte(keyPrefix) 453 | 454 | for key, _ := cursor.Seek(prefix); bytes.HasPrefix(key, prefix); key, _ = cursor.Next() { 455 | _ = bucket.Delete([]byte(key)) 456 | } 457 | return nil 458 | }) 459 | 460 | return err 461 | } 462 | 463 | // NewLock has to implemented at the library level since its not supported by BoltDB 464 | func (b *BoltDB) NewLock(key string, options *store.LockOptions) (store.Locker, error) { 465 | return nil, store.ErrCallNotSupported 466 | } 467 | 468 | // Watch has to implemented at the library level since its not supported by BoltDB 469 | func (b *BoltDB) Watch(key string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan *store.KVPair, error) { 470 | return nil, store.ErrCallNotSupported 471 | } 472 | 473 | // WatchTree has to implemented at the library level since its not supported by BoltDB 474 | func (b *BoltDB) WatchTree(directory string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan []*store.KVPair, error) { 475 | return nil, store.ErrCallNotSupported 476 | } 477 | -------------------------------------------------------------------------------- /store/boltdb/boltdb_test.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/docker/libkv" 9 | "github.com/docker/libkv/store" 10 | "github.com/docker/libkv/testutils" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func makeBoltDBClient(t *testing.T) store.Store { 15 | kv, err := New([]string{"/tmp/not_exist_dir/__boltdbtest"}, &store.Config{Bucket: "boltDBTest"}) 16 | 17 | if err != nil { 18 | t.Fatalf("cannot create store: %v", err) 19 | } 20 | 21 | return kv 22 | } 23 | 24 | func TestRegister(t *testing.T) { 25 | Register() 26 | 27 | kv, err := libkv.NewStore( 28 | store.BOLTDB, 29 | []string{"/tmp/not_exist_dir/__boltdbtest"}, 30 | &store.Config{Bucket: "boltDBTest"}, 31 | ) 32 | assert.NoError(t, err) 33 | assert.NotNil(t, kv) 34 | 35 | if _, ok := kv.(*BoltDB); !ok { 36 | t.Fatal("Error registering and initializing boltDB") 37 | } 38 | 39 | _ = os.Remove("/tmp/not_exist_dir/__boltdbtest") 40 | } 41 | 42 | // TestMultiplePersistConnection tests the second connection to a 43 | // BoltDB fails when one is already open with PersistConnection flag 44 | func TestMultiplePersistConnection(t *testing.T) { 45 | kv, err := libkv.NewStore( 46 | store.BOLTDB, 47 | []string{"/tmp/not_exist_dir/__boltdbtest"}, 48 | &store.Config{ 49 | Bucket: "boltDBTest", 50 | ConnectionTimeout: 1 * time.Second, 51 | PersistConnection: true}, 52 | ) 53 | assert.NoError(t, err) 54 | assert.NotNil(t, kv) 55 | 56 | if _, ok := kv.(*BoltDB); !ok { 57 | t.Fatal("Error registering and initializing boltDB") 58 | } 59 | 60 | // Must fail if multiple boltdb requests are made with a valid timeout 61 | _, err = libkv.NewStore( 62 | store.BOLTDB, 63 | []string{"/tmp/not_exist_dir/__boltdbtest"}, 64 | &store.Config{ 65 | Bucket: "boltDBTest", 66 | ConnectionTimeout: 1 * time.Second, 67 | PersistConnection: true}, 68 | ) 69 | assert.Error(t, err) 70 | 71 | _ = os.Remove("/tmp/not_exist_dir/__boltdbtest") 72 | } 73 | 74 | // TestConcurrentConnection tests simultaneous get/put using 75 | // two handles. 76 | func TestConcurrentConnection(t *testing.T) { 77 | var err error 78 | kv1, err1 := libkv.NewStore( 79 | store.BOLTDB, 80 | []string{"/tmp/__boltdbtest"}, 81 | &store.Config{ 82 | Bucket: "boltDBTest", 83 | ConnectionTimeout: 1 * time.Second}, 84 | ) 85 | assert.NoError(t, err1) 86 | assert.NotNil(t, kv1) 87 | 88 | kv2, err2 := libkv.NewStore( 89 | store.BOLTDB, 90 | []string{"/tmp/__boltdbtest"}, 91 | &store.Config{Bucket: "boltDBTest", 92 | ConnectionTimeout: 1 * time.Second}, 93 | ) 94 | assert.NoError(t, err2) 95 | assert.NotNil(t, kv2) 96 | 97 | key1 := "TestKV1" 98 | value1 := []byte("TestVal1") 99 | err = kv1.Put(key1, value1, nil) 100 | assert.NoError(t, err) 101 | 102 | key2 := "TestKV2" 103 | value2 := []byte("TestVal2") 104 | err = kv2.Put(key2, value2, nil) 105 | assert.NoError(t, err) 106 | 107 | pair1, err1 := kv1.Get(key1, nil) 108 | assert.NoError(t, err1) 109 | if assert.NotNil(t, pair1) { 110 | assert.NotNil(t, pair1.Value) 111 | } 112 | assert.Equal(t, pair1.Value, value1) 113 | 114 | pair2, err2 := kv2.Get(key2, nil) 115 | assert.NoError(t, err2) 116 | if assert.NotNil(t, pair2) { 117 | assert.NotNil(t, pair2.Value) 118 | } 119 | assert.Equal(t, pair2.Value, value2) 120 | 121 | // AtomicPut using kv1 and kv2 should succeed 122 | _, _, err = kv1.AtomicPut(key1, []byte("TestnewVal1"), pair1, nil) 123 | assert.NoError(t, err) 124 | 125 | _, _, err = kv2.AtomicPut(key2, []byte("TestnewVal2"), pair2, nil) 126 | assert.NoError(t, err) 127 | 128 | testutils.RunTestCommon(t, kv1) 129 | testutils.RunTestCommon(t, kv2) 130 | 131 | kv1.Close() 132 | kv2.Close() 133 | 134 | _ = os.Remove("/tmp/__boltdbtest") 135 | } 136 | 137 | func TestBoldDBStore(t *testing.T) { 138 | kv := makeBoltDBClient(t) 139 | 140 | testutils.RunTestCommon(t, kv) 141 | testutils.RunTestAtomic(t, kv) 142 | 143 | _ = os.Remove("/tmp/not_exist_dir/__boltdbtest") 144 | } 145 | -------------------------------------------------------------------------------- /store/consul/consul.go: -------------------------------------------------------------------------------- 1 | package consul 2 | 3 | import ( 4 | "crypto/tls" 5 | "errors" 6 | "net/http" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/docker/libkv" 12 | "github.com/docker/libkv/store" 13 | api "github.com/hashicorp/consul/api" 14 | ) 15 | 16 | const ( 17 | // DefaultWatchWaitTime is how long we block for at a 18 | // time to check if the watched key has changed. This 19 | // affects the minimum time it takes to cancel a watch. 20 | DefaultWatchWaitTime = 15 * time.Second 21 | 22 | // RenewSessionRetryMax is the number of time we should try 23 | // to renew the session before giving up and throwing an error 24 | RenewSessionRetryMax = 5 25 | 26 | // MaxSessionDestroyAttempts is the maximum times we will try 27 | // to explicitly destroy the session attached to a lock after 28 | // the connectivity to the store has been lost 29 | MaxSessionDestroyAttempts = 5 30 | 31 | // defaultLockTTL is the default ttl for the consul lock 32 | defaultLockTTL = 20 * time.Second 33 | ) 34 | 35 | var ( 36 | // ErrMultipleEndpointsUnsupported is thrown when there are 37 | // multiple endpoints specified for Consul 38 | ErrMultipleEndpointsUnsupported = errors.New("consul does not support multiple endpoints") 39 | 40 | // ErrSessionRenew is thrown when the session can't be 41 | // renewed because the Consul version does not support sessions 42 | ErrSessionRenew = errors.New("cannot set or renew session for ttl, unable to operate on sessions") 43 | ) 44 | 45 | // Consul is the receiver type for the 46 | // Store interface 47 | type Consul struct { 48 | sync.Mutex 49 | config *api.Config 50 | client *api.Client 51 | } 52 | 53 | type consulLock struct { 54 | lock *api.Lock 55 | renewCh chan struct{} 56 | } 57 | 58 | // Register registers consul to libkv 59 | func Register() { 60 | libkv.AddStore(store.CONSUL, New) 61 | } 62 | 63 | // New creates a new Consul client given a list 64 | // of endpoints and optional tls config 65 | func New(endpoints []string, options *store.Config) (store.Store, error) { 66 | if len(endpoints) > 1 { 67 | return nil, ErrMultipleEndpointsUnsupported 68 | } 69 | 70 | s := &Consul{} 71 | 72 | // Create Consul client 73 | config := api.DefaultConfig() 74 | s.config = config 75 | config.HttpClient = http.DefaultClient 76 | config.Address = endpoints[0] 77 | 78 | // Set options 79 | if options != nil { 80 | if options.TLS != nil { 81 | s.setTLS(options.TLS) 82 | } 83 | if options.ConnectionTimeout != 0 { 84 | s.setTimeout(options.ConnectionTimeout) 85 | } 86 | } 87 | 88 | // Creates a new client 89 | client, err := api.NewClient(config) 90 | if err != nil { 91 | return nil, err 92 | } 93 | s.client = client 94 | 95 | return s, nil 96 | } 97 | 98 | // SetTLS sets Consul TLS options 99 | func (s *Consul) setTLS(tls *tls.Config) { 100 | s.config.HttpClient.Transport = &http.Transport{ 101 | TLSClientConfig: tls, 102 | } 103 | s.config.Scheme = "https" 104 | } 105 | 106 | // SetTimeout sets the timeout for connecting to Consul 107 | func (s *Consul) setTimeout(time time.Duration) { 108 | s.config.WaitTime = time 109 | } 110 | 111 | // Normalize the key for usage in Consul 112 | func (s *Consul) normalize(key string) string { 113 | key = store.Normalize(key) 114 | return strings.TrimPrefix(key, "/") 115 | } 116 | 117 | func (s *Consul) renewSession(pair *api.KVPair, ttl time.Duration) error { 118 | // Check if there is any previous session with an active TTL 119 | session, err := s.getActiveSession(pair.Key) 120 | if err != nil { 121 | return err 122 | } 123 | 124 | if session == "" { 125 | entry := &api.SessionEntry{ 126 | Behavior: api.SessionBehaviorDelete, // Delete the key when the session expires 127 | TTL: (ttl / 2).String(), // Consul multiplies the TTL by 2x 128 | LockDelay: 1 * time.Millisecond, // Virtually disable lock delay 129 | } 130 | 131 | // Create the key session 132 | session, _, err = s.client.Session().Create(entry, nil) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | lockOpts := &api.LockOptions{ 138 | Key: pair.Key, 139 | Session: session, 140 | } 141 | 142 | // Lock and ignore if lock is held 143 | // It's just a placeholder for the 144 | // ephemeral behavior 145 | lock, _ := s.client.LockOpts(lockOpts) 146 | if lock != nil { 147 | lock.Lock(nil) 148 | } 149 | } 150 | 151 | _, _, err = s.client.Session().Renew(session, nil) 152 | return err 153 | } 154 | 155 | // getActiveSession checks if the key already has 156 | // a session attached 157 | func (s *Consul) getActiveSession(key string) (string, error) { 158 | pair, _, err := s.client.KV().Get(key, nil) 159 | if err != nil { 160 | return "", err 161 | } 162 | if pair != nil && pair.Session != "" { 163 | return pair.Session, nil 164 | } 165 | return "", nil 166 | } 167 | 168 | // Get the value at "key", returns the last modified index 169 | // to use in conjunction to CAS calls 170 | func (s *Consul) Get(key string, opts *store.ReadOptions) (*store.KVPair, error) { 171 | options := &api.QueryOptions{ 172 | AllowStale: false, 173 | RequireConsistent: true, 174 | } 175 | 176 | // Get options 177 | if opts != nil { 178 | options.RequireConsistent = opts.Consistent 179 | } 180 | 181 | pair, meta, err := s.client.KV().Get(s.normalize(key), options) 182 | if err != nil { 183 | return nil, err 184 | } 185 | 186 | // If pair is nil then the key does not exist 187 | if pair == nil { 188 | return nil, store.ErrKeyNotFound 189 | } 190 | 191 | return &store.KVPair{Key: pair.Key, Value: pair.Value, LastIndex: meta.LastIndex}, nil 192 | } 193 | 194 | // Put a value at "key" 195 | func (s *Consul) Put(key string, value []byte, opts *store.WriteOptions) error { 196 | key = s.normalize(key) 197 | 198 | p := &api.KVPair{ 199 | Key: key, 200 | Value: value, 201 | Flags: api.LockFlagValue, 202 | } 203 | 204 | if opts != nil && opts.TTL > 0 { 205 | // Create or renew a session holding a TTL. Operations on sessions 206 | // are not deterministic: creating or renewing a session can fail 207 | for retry := 1; retry <= RenewSessionRetryMax; retry++ { 208 | err := s.renewSession(p, opts.TTL) 209 | if err == nil { 210 | break 211 | } 212 | if retry == RenewSessionRetryMax { 213 | return ErrSessionRenew 214 | } 215 | } 216 | } 217 | 218 | _, err := s.client.KV().Put(p, nil) 219 | return err 220 | } 221 | 222 | // Delete a value at "key" 223 | func (s *Consul) Delete(key string) error { 224 | if _, err := s.Get(key, nil); err != nil { 225 | return err 226 | } 227 | _, err := s.client.KV().Delete(s.normalize(key), nil) 228 | return err 229 | } 230 | 231 | // Exists checks that the key exists inside the store 232 | func (s *Consul) Exists(key string, opts *store.ReadOptions) (bool, error) { 233 | _, err := s.Get(key, opts) 234 | if err != nil { 235 | if err == store.ErrKeyNotFound { 236 | return false, nil 237 | } 238 | return false, err 239 | } 240 | return true, nil 241 | } 242 | 243 | // List child nodes of a given directory 244 | func (s *Consul) List(directory string, opts *store.ReadOptions) ([]*store.KVPair, error) { 245 | options := &api.QueryOptions{ 246 | AllowStale: false, 247 | RequireConsistent: true, 248 | } 249 | 250 | if opts != nil { 251 | if !opts.Consistent { 252 | options.AllowStale = true 253 | options.RequireConsistent = false 254 | } 255 | } 256 | 257 | pairs, _, err := s.client.KV().List(s.normalize(directory), options) 258 | if err != nil { 259 | return nil, err 260 | } 261 | if len(pairs) == 0 { 262 | return nil, store.ErrKeyNotFound 263 | } 264 | 265 | kv := []*store.KVPair{} 266 | 267 | for _, pair := range pairs { 268 | if pair.Key == directory { 269 | continue 270 | } 271 | kv = append(kv, &store.KVPair{ 272 | Key: pair.Key, 273 | Value: pair.Value, 274 | LastIndex: pair.ModifyIndex, 275 | }) 276 | } 277 | 278 | return kv, nil 279 | } 280 | 281 | // DeleteTree deletes a range of keys under a given directory 282 | func (s *Consul) DeleteTree(directory string) error { 283 | if _, err := s.List(directory, nil); err != nil { 284 | return err 285 | } 286 | _, err := s.client.KV().DeleteTree(s.normalize(directory), nil) 287 | return err 288 | } 289 | 290 | // Watch for changes on a "key" 291 | // It returns a channel that will receive changes or pass 292 | // on errors. Upon creation, the current value will first 293 | // be sent to the channel. Providing a non-nil stopCh can 294 | // be used to stop watching. 295 | func (s *Consul) Watch(key string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan *store.KVPair, error) { 296 | kv := s.client.KV() 297 | watchCh := make(chan *store.KVPair) 298 | 299 | go func() { 300 | defer close(watchCh) 301 | 302 | // Use a wait time in order to check if we should quit 303 | // from time to time. 304 | opts := &api.QueryOptions{WaitTime: DefaultWatchWaitTime} 305 | 306 | for { 307 | // Check if we should quit 308 | select { 309 | case <-stopCh: 310 | return 311 | default: 312 | } 313 | 314 | // Get the key 315 | pair, meta, err := kv.Get(key, opts) 316 | if err != nil { 317 | return 318 | } 319 | 320 | // If LastIndex didn't change then it means `Get` returned 321 | // because of the WaitTime and the key didn't changed. 322 | if opts.WaitIndex == meta.LastIndex { 323 | continue 324 | } 325 | opts.WaitIndex = meta.LastIndex 326 | 327 | // Return the value to the channel 328 | if pair != nil { 329 | watchCh <- &store.KVPair{ 330 | Key: pair.Key, 331 | Value: pair.Value, 332 | LastIndex: pair.ModifyIndex, 333 | } 334 | } 335 | } 336 | }() 337 | 338 | return watchCh, nil 339 | } 340 | 341 | // WatchTree watches for changes on a "directory" 342 | // It returns a channel that will receive changes or pass 343 | // on errors. Upon creating a watch, the current childs values 344 | // will be sent to the channel .Providing a non-nil stopCh can 345 | // be used to stop watching. 346 | func (s *Consul) WatchTree(directory string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan []*store.KVPair, error) { 347 | kv := s.client.KV() 348 | watchCh := make(chan []*store.KVPair) 349 | 350 | go func() { 351 | defer close(watchCh) 352 | 353 | // Use a wait time in order to check if we should quit 354 | // from time to time. 355 | opts := &api.QueryOptions{WaitTime: DefaultWatchWaitTime} 356 | for { 357 | // Check if we should quit 358 | select { 359 | case <-stopCh: 360 | return 361 | default: 362 | } 363 | 364 | // Get all the childrens 365 | pairs, meta, err := kv.List(directory, opts) 366 | if err != nil { 367 | return 368 | } 369 | 370 | // If LastIndex didn't change then it means `Get` returned 371 | // because of the WaitTime and the child keys didn't change. 372 | if opts.WaitIndex == meta.LastIndex { 373 | continue 374 | } 375 | opts.WaitIndex = meta.LastIndex 376 | 377 | // Return children KV pairs to the channel 378 | kvpairs := []*store.KVPair{} 379 | for _, pair := range pairs { 380 | if pair.Key == directory { 381 | continue 382 | } 383 | kvpairs = append(kvpairs, &store.KVPair{ 384 | Key: pair.Key, 385 | Value: pair.Value, 386 | LastIndex: pair.ModifyIndex, 387 | }) 388 | } 389 | watchCh <- kvpairs 390 | } 391 | }() 392 | 393 | return watchCh, nil 394 | } 395 | 396 | // NewLock returns a handle to a lock struct which can 397 | // be used to provide mutual exclusion on a key 398 | func (s *Consul) NewLock(key string, options *store.LockOptions) (store.Locker, error) { 399 | lockOpts := &api.LockOptions{ 400 | Key: s.normalize(key), 401 | } 402 | 403 | lock := &consulLock{} 404 | 405 | ttl := defaultLockTTL 406 | 407 | if options != nil { 408 | // Set optional TTL on Lock 409 | if options.TTL != 0 { 410 | ttl = options.TTL 411 | } 412 | // Set optional value on Lock 413 | if options.Value != nil { 414 | lockOpts.Value = options.Value 415 | } 416 | } 417 | 418 | entry := &api.SessionEntry{ 419 | Behavior: api.SessionBehaviorRelease, // Release the lock when the session expires 420 | TTL: (ttl / 2).String(), // Consul multiplies the TTL by 2x 421 | LockDelay: 1 * time.Millisecond, // Virtually disable lock delay 422 | } 423 | 424 | // Create the key session 425 | session, _, err := s.client.Session().Create(entry, nil) 426 | if err != nil { 427 | return nil, err 428 | } 429 | 430 | // Place the session and renew chan on lock 431 | lockOpts.Session = session 432 | lock.renewCh = options.RenewLock 433 | 434 | l, err := s.client.LockOpts(lockOpts) 435 | if err != nil { 436 | return nil, err 437 | } 438 | 439 | // Renew the session ttl lock periodically 440 | s.renewLockSession(entry.TTL, session, options.RenewLock) 441 | 442 | lock.lock = l 443 | return lock, nil 444 | } 445 | 446 | // renewLockSession is used to renew a session Lock, it takes 447 | // a stopRenew chan which is used to explicitly stop the session 448 | // renew process. The renew routine never stops until a signal is 449 | // sent to this channel. If deleting the session fails because the 450 | // connection to the store is lost, it keeps trying to delete the 451 | // session periodically until it can contact the store, this ensures 452 | // that the lock is not maintained indefinitely which ensures liveness 453 | // over safety for the lock when the store becomes unavailable. 454 | func (s *Consul) renewLockSession(initialTTL string, id string, stopRenew chan struct{}) { 455 | sessionDestroyAttempts := 0 456 | ttl, err := time.ParseDuration(initialTTL) 457 | if err != nil { 458 | return 459 | } 460 | go func() { 461 | for { 462 | select { 463 | case <-time.After(ttl / 2): 464 | entry, _, err := s.client.Session().Renew(id, nil) 465 | if err != nil { 466 | // If an error occurs, continue until the 467 | // session gets destroyed explicitly or 468 | // the session ttl times out 469 | continue 470 | } 471 | if entry == nil { 472 | return 473 | } 474 | 475 | // Handle the server updating the TTL 476 | ttl, _ = time.ParseDuration(entry.TTL) 477 | 478 | case <-stopRenew: 479 | // Attempt a session destroy 480 | _, err := s.client.Session().Destroy(id, nil) 481 | if err == nil { 482 | return 483 | } 484 | 485 | // We cannot destroy the session because the store 486 | // is unavailable, wait for the session renew period. 487 | // Give up after 'MaxSessionDestroyAttempts'. 488 | sessionDestroyAttempts++ 489 | 490 | if sessionDestroyAttempts >= MaxSessionDestroyAttempts { 491 | return 492 | } 493 | 494 | time.Sleep(ttl / 2) 495 | } 496 | } 497 | }() 498 | } 499 | 500 | // Lock attempts to acquire the lock and blocks while 501 | // doing so. It returns a channel that is closed if our 502 | // lock is lost or if an error occurs 503 | func (l *consulLock) Lock(stopChan chan struct{}) (<-chan struct{}, error) { 504 | return l.lock.Lock(stopChan) 505 | } 506 | 507 | // Unlock the "key". Calling unlock while 508 | // not holding the lock will throw an error 509 | func (l *consulLock) Unlock() error { 510 | if l.renewCh != nil { 511 | close(l.renewCh) 512 | } 513 | return l.lock.Unlock() 514 | } 515 | 516 | // AtomicPut put a value at "key" if the key has not been 517 | // modified in the meantime, throws an error if this is the case 518 | func (s *Consul) AtomicPut(key string, value []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) { 519 | 520 | p := &api.KVPair{Key: s.normalize(key), Value: value, Flags: api.LockFlagValue} 521 | 522 | if previous == nil { 523 | // Consul interprets ModifyIndex = 0 as new key. 524 | p.ModifyIndex = 0 525 | } else { 526 | p.ModifyIndex = previous.LastIndex 527 | } 528 | 529 | ok, _, err := s.client.KV().CAS(p, nil) 530 | if err != nil { 531 | return false, nil, err 532 | } 533 | if !ok { 534 | if previous == nil { 535 | return false, nil, store.ErrKeyExists 536 | } 537 | return false, nil, store.ErrKeyModified 538 | } 539 | 540 | pair, err := s.Get(key, nil) 541 | if err != nil { 542 | return false, nil, err 543 | } 544 | 545 | return true, pair, nil 546 | } 547 | 548 | // AtomicDelete deletes a value at "key" if the key has not 549 | // been modified in the meantime, throws an error if this is the case 550 | func (s *Consul) AtomicDelete(key string, previous *store.KVPair) (bool, error) { 551 | if previous == nil { 552 | return false, store.ErrPreviousNotSpecified 553 | } 554 | 555 | p := &api.KVPair{Key: s.normalize(key), ModifyIndex: previous.LastIndex, Flags: api.LockFlagValue} 556 | 557 | // Extra Get operation to check on the key 558 | _, err := s.Get(key, nil) 559 | if err != nil && err == store.ErrKeyNotFound { 560 | return false, err 561 | } 562 | 563 | if work, _, err := s.client.KV().DeleteCAS(p, nil); err != nil { 564 | return false, err 565 | } else if !work { 566 | return false, store.ErrKeyModified 567 | } 568 | 569 | return true, nil 570 | } 571 | 572 | // Close closes the client connection 573 | func (s *Consul) Close() { 574 | return 575 | } 576 | -------------------------------------------------------------------------------- /store/consul/consul_test.go: -------------------------------------------------------------------------------- 1 | package consul 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/docker/libkv" 8 | "github.com/docker/libkv/store" 9 | "github.com/docker/libkv/testutils" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var ( 14 | client = "localhost:8500" 15 | ) 16 | 17 | func makeConsulClient(t *testing.T) store.Store { 18 | 19 | kv, err := New( 20 | []string{client}, 21 | &store.Config{ 22 | ConnectionTimeout: 3 * time.Second, 23 | }, 24 | ) 25 | 26 | if err != nil { 27 | t.Fatalf("cannot create store: %v", err) 28 | } 29 | 30 | return kv 31 | } 32 | 33 | func TestRegister(t *testing.T) { 34 | Register() 35 | 36 | kv, err := libkv.NewStore(store.CONSUL, []string{client}, nil) 37 | assert.NoError(t, err) 38 | assert.NotNil(t, kv) 39 | 40 | if _, ok := kv.(*Consul); !ok { 41 | t.Fatal("Error registering and initializing consul") 42 | } 43 | } 44 | 45 | func TestConsulStore(t *testing.T) { 46 | kv := makeConsulClient(t) 47 | lockKV := makeConsulClient(t) 48 | ttlKV := makeConsulClient(t) 49 | 50 | testutils.RunTestCommon(t, kv) 51 | testutils.RunTestAtomic(t, kv) 52 | testutils.RunTestWatch(t, kv) 53 | testutils.RunTestLock(t, kv) 54 | testutils.RunTestLockTTL(t, kv, lockKV) 55 | testutils.RunTestTTL(t, kv, ttlKV) 56 | testutils.RunCleanup(t, kv) 57 | } 58 | 59 | func TestGetActiveSession(t *testing.T) { 60 | kv := makeConsulClient(t) 61 | 62 | consul := kv.(*Consul) 63 | 64 | key := "foo" 65 | value := []byte("bar") 66 | 67 | // Put the first key with the Ephemeral flag 68 | err := kv.Put(key, value, &store.WriteOptions{TTL: 2 * time.Second}) 69 | assert.NoError(t, err) 70 | 71 | // Session should not be empty 72 | session, err := consul.getActiveSession(key) 73 | assert.NoError(t, err) 74 | assert.NotEqual(t, session, "") 75 | 76 | // Delete the key 77 | err = kv.Delete(key) 78 | assert.NoError(t, err) 79 | 80 | // Check the session again, it should return nothing 81 | session, err = consul.getActiveSession(key) 82 | assert.NoError(t, err) 83 | assert.Equal(t, session, "") 84 | } 85 | -------------------------------------------------------------------------------- /store/etcd/v2/etcd.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "crypto/tls" 5 | "errors" 6 | "log" 7 | "net" 8 | "net/http" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | "golang.org/x/net/context" 14 | 15 | etcd "github.com/coreos/etcd/client" 16 | "github.com/docker/libkv" 17 | "github.com/docker/libkv/store" 18 | ) 19 | 20 | const ( 21 | lockSuffix = "___lock" 22 | ) 23 | 24 | var ( 25 | // ErrAbortTryLock is thrown when a user stops trying to seek the lock 26 | // by sending a signal to the stop chan, this is used to verify if the 27 | // operation succeeded 28 | ErrAbortTryLock = errors.New("lock operation aborted") 29 | ) 30 | 31 | // Etcd is the receiver type for the 32 | // Store interface 33 | type Etcd struct { 34 | client etcd.KeysAPI 35 | } 36 | 37 | type etcdLock struct { 38 | lock sync.Mutex 39 | client etcd.KeysAPI 40 | 41 | stopLock chan struct{} 42 | stopRenew chan struct{} 43 | 44 | mutexKey string 45 | writeKey string 46 | value string 47 | last *etcd.Response 48 | ttl time.Duration 49 | } 50 | 51 | const ( 52 | defaultLockTTL = 20 * time.Second 53 | defaultUpdateTime = 5 * time.Second 54 | ) 55 | 56 | // Register registers etcd to libkv 57 | func Register() { 58 | libkv.AddStore(store.ETCD, New) 59 | } 60 | 61 | // New creates a new Etcd client given a list 62 | // of endpoints and an optional tls config 63 | func New(addrs []string, options *store.Config) (store.Store, error) { 64 | s := &Etcd{} 65 | 66 | var ( 67 | entries []string 68 | err error 69 | ) 70 | 71 | entries = store.CreateEndpoints(addrs, "http") 72 | cfg := &etcd.Config{ 73 | Endpoints: entries, 74 | Transport: etcd.DefaultTransport, 75 | HeaderTimeoutPerRequest: 3 * time.Second, 76 | } 77 | 78 | // Set options 79 | if options != nil { 80 | if options.TLS != nil { 81 | setTLS(cfg, options.TLS, addrs) 82 | } 83 | if options.ConnectionTimeout != 0 { 84 | setTimeout(cfg, options.ConnectionTimeout) 85 | } 86 | if options.Username != "" { 87 | setCredentials(cfg, options.Username, options.Password) 88 | } 89 | } 90 | 91 | c, err := etcd.New(*cfg) 92 | if err != nil { 93 | log.Fatal(err) 94 | } 95 | 96 | s.client = etcd.NewKeysAPI(c) 97 | 98 | // Periodic Cluster Sync 99 | if options != nil && options.SyncPeriod != 0 { 100 | go func() { 101 | for { 102 | c.AutoSync(context.Background(), options.SyncPeriod) 103 | } 104 | }() 105 | } 106 | 107 | return s, nil 108 | } 109 | 110 | // SetTLS sets the tls configuration given a tls.Config scheme 111 | func setTLS(cfg *etcd.Config, tls *tls.Config, addrs []string) { 112 | entries := store.CreateEndpoints(addrs, "https") 113 | cfg.Endpoints = entries 114 | 115 | // Set transport 116 | t := http.Transport{ 117 | Dial: (&net.Dialer{ 118 | Timeout: 30 * time.Second, 119 | KeepAlive: 30 * time.Second, 120 | }).Dial, 121 | TLSHandshakeTimeout: 10 * time.Second, 122 | TLSClientConfig: tls, 123 | } 124 | 125 | cfg.Transport = &t 126 | } 127 | 128 | // setTimeout sets the timeout used for connecting to the store 129 | func setTimeout(cfg *etcd.Config, time time.Duration) { 130 | cfg.HeaderTimeoutPerRequest = time 131 | } 132 | 133 | // setCredentials sets the username/password credentials for connecting to Etcd 134 | func setCredentials(cfg *etcd.Config, username, password string) { 135 | cfg.Username = username 136 | cfg.Password = password 137 | } 138 | 139 | // Normalize the key for usage in Etcd 140 | func (s *Etcd) normalize(key string) string { 141 | key = store.Normalize(key) 142 | return strings.TrimPrefix(key, "/") 143 | } 144 | 145 | // keyNotFound checks on the error returned by the KeysAPI 146 | // to verify if the key exists in the store or not 147 | func keyNotFound(err error) bool { 148 | if err != nil { 149 | if etcdError, ok := err.(etcd.Error); ok { 150 | if etcdError.Code == etcd.ErrorCodeKeyNotFound || 151 | etcdError.Code == etcd.ErrorCodeNotFile || 152 | etcdError.Code == etcd.ErrorCodeNotDir { 153 | return true 154 | } 155 | } 156 | } 157 | return false 158 | } 159 | 160 | // Get the value at "key", returns the last modified 161 | // index to use in conjunction to Atomic calls 162 | func (s *Etcd) Get(key string, opts *store.ReadOptions) (pair *store.KVPair, err error) { 163 | getOpts := &etcd.GetOptions{ 164 | Quorum: true, 165 | } 166 | 167 | // Get options 168 | if opts != nil { 169 | getOpts.Quorum = opts.Consistent 170 | } 171 | 172 | result, err := s.client.Get(context.Background(), s.normalize(key), getOpts) 173 | if err != nil { 174 | if keyNotFound(err) { 175 | return nil, store.ErrKeyNotFound 176 | } 177 | return nil, err 178 | } 179 | 180 | pair = &store.KVPair{ 181 | Key: key, 182 | Value: []byte(result.Node.Value), 183 | LastIndex: result.Node.ModifiedIndex, 184 | } 185 | 186 | return pair, nil 187 | } 188 | 189 | // Put a value at "key" 190 | func (s *Etcd) Put(key string, value []byte, opts *store.WriteOptions) error { 191 | setOpts := &etcd.SetOptions{} 192 | 193 | // Set options 194 | if opts != nil { 195 | setOpts.Dir = opts.IsDir 196 | setOpts.TTL = opts.TTL 197 | } 198 | 199 | _, err := s.client.Set(context.Background(), s.normalize(key), string(value), setOpts) 200 | return err 201 | } 202 | 203 | // Delete a value at "key" 204 | func (s *Etcd) Delete(key string) error { 205 | opts := &etcd.DeleteOptions{ 206 | Recursive: false, 207 | } 208 | 209 | _, err := s.client.Delete(context.Background(), s.normalize(key), opts) 210 | if keyNotFound(err) { 211 | return store.ErrKeyNotFound 212 | } 213 | return err 214 | } 215 | 216 | // Exists checks if the key exists inside the store 217 | func (s *Etcd) Exists(key string, opts *store.ReadOptions) (bool, error) { 218 | _, err := s.Get(key, opts) 219 | if err != nil { 220 | if err == store.ErrKeyNotFound { 221 | return false, nil 222 | } 223 | return false, err 224 | } 225 | return true, nil 226 | } 227 | 228 | // Watch for changes on a "key" 229 | // It returns a channel that will receive changes or pass 230 | // on errors. Upon creation, the current value will first 231 | // be sent to the channel. Providing a non-nil stopCh can 232 | // be used to stop watching. 233 | func (s *Etcd) Watch(key string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan *store.KVPair, error) { 234 | wopts := &etcd.WatcherOptions{Recursive: false} 235 | watcher := s.client.Watcher(s.normalize(key), wopts) 236 | 237 | // watchCh is sending back events to the caller 238 | watchCh := make(chan *store.KVPair) 239 | 240 | // Get the current value 241 | pair, err := s.Get(key, opts) 242 | if err != nil { 243 | return nil, err 244 | } 245 | 246 | go func() { 247 | defer close(watchCh) 248 | 249 | // Push the current value through the channel. 250 | watchCh <- pair 251 | 252 | for { 253 | // Check if the watch was stopped by the caller 254 | select { 255 | case <-stopCh: 256 | return 257 | default: 258 | } 259 | 260 | result, err := watcher.Next(context.Background()) 261 | 262 | if err != nil { 263 | return 264 | } 265 | 266 | watchCh <- &store.KVPair{ 267 | Key: key, 268 | Value: []byte(result.Node.Value), 269 | LastIndex: result.Node.ModifiedIndex, 270 | } 271 | } 272 | }() 273 | 274 | return watchCh, nil 275 | } 276 | 277 | // WatchTree watches for changes on a "directory" 278 | // It returns a channel that will receive changes or pass 279 | // on errors. Upon creating a watch, the current childs values 280 | // will be sent to the channel. Providing a non-nil stopCh can 281 | // be used to stop watching. 282 | func (s *Etcd) WatchTree(directory string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan []*store.KVPair, error) { 283 | watchOpts := &etcd.WatcherOptions{Recursive: true} 284 | watcher := s.client.Watcher(s.normalize(directory), watchOpts) 285 | 286 | // watchCh is sending back events to the caller 287 | watchCh := make(chan []*store.KVPair) 288 | 289 | // List current children 290 | list, err := s.List(directory, opts) 291 | if err != nil { 292 | return nil, err 293 | } 294 | 295 | go func() { 296 | defer close(watchCh) 297 | 298 | // Push the current value through the channel. 299 | watchCh <- list 300 | 301 | for { 302 | // Check if the watch was stopped by the caller 303 | select { 304 | case <-stopCh: 305 | return 306 | default: 307 | } 308 | 309 | _, err := watcher.Next(context.Background()) 310 | 311 | if err != nil { 312 | return 313 | } 314 | 315 | list, err = s.List(directory, opts) 316 | if err != nil { 317 | return 318 | } 319 | 320 | watchCh <- list 321 | } 322 | }() 323 | 324 | return watchCh, nil 325 | } 326 | 327 | // AtomicPut puts a value at "key" if the key has not been 328 | // modified in the meantime, throws an error if this is the case 329 | func (s *Etcd) AtomicPut(key string, value []byte, previous *store.KVPair, opts *store.WriteOptions) (bool, *store.KVPair, error) { 330 | var ( 331 | meta *etcd.Response 332 | err error 333 | ) 334 | 335 | setOpts := &etcd.SetOptions{} 336 | 337 | if previous != nil { 338 | setOpts.PrevExist = etcd.PrevExist 339 | setOpts.PrevIndex = previous.LastIndex 340 | if previous.Value != nil { 341 | setOpts.PrevValue = string(previous.Value) 342 | } 343 | } else { 344 | setOpts.PrevExist = etcd.PrevNoExist 345 | } 346 | 347 | if opts != nil { 348 | if opts.TTL > 0 { 349 | setOpts.TTL = opts.TTL 350 | } 351 | } 352 | 353 | meta, err = s.client.Set(context.Background(), s.normalize(key), string(value), setOpts) 354 | if err != nil { 355 | if etcdError, ok := err.(etcd.Error); ok { 356 | // Compare failed 357 | if etcdError.Code == etcd.ErrorCodeTestFailed { 358 | return false, nil, store.ErrKeyModified 359 | } 360 | // Node exists error (when PrevNoExist) 361 | if etcdError.Code == etcd.ErrorCodeNodeExist { 362 | return false, nil, store.ErrKeyExists 363 | } 364 | } 365 | return false, nil, err 366 | } 367 | 368 | updated := &store.KVPair{ 369 | Key: key, 370 | Value: value, 371 | LastIndex: meta.Node.ModifiedIndex, 372 | } 373 | 374 | return true, updated, nil 375 | } 376 | 377 | // AtomicDelete deletes a value at "key" if the key 378 | // has not been modified in the meantime, throws an 379 | // error if this is the case 380 | func (s *Etcd) AtomicDelete(key string, previous *store.KVPair) (bool, error) { 381 | if previous == nil { 382 | return false, store.ErrPreviousNotSpecified 383 | } 384 | 385 | delOpts := &etcd.DeleteOptions{} 386 | 387 | if previous != nil { 388 | delOpts.PrevIndex = previous.LastIndex 389 | if previous.Value != nil { 390 | delOpts.PrevValue = string(previous.Value) 391 | } 392 | } 393 | 394 | _, err := s.client.Delete(context.Background(), s.normalize(key), delOpts) 395 | if err != nil { 396 | if etcdError, ok := err.(etcd.Error); ok { 397 | // Key Not Found 398 | if etcdError.Code == etcd.ErrorCodeKeyNotFound { 399 | return false, store.ErrKeyNotFound 400 | } 401 | // Compare failed 402 | if etcdError.Code == etcd.ErrorCodeTestFailed { 403 | return false, store.ErrKeyModified 404 | } 405 | } 406 | return false, err 407 | } 408 | 409 | return true, nil 410 | } 411 | 412 | // List child nodes of a given directory 413 | func (s *Etcd) List(directory string, opts *store.ReadOptions) ([]*store.KVPair, error) { 414 | getOpts := &etcd.GetOptions{ 415 | Quorum: true, 416 | Recursive: true, 417 | Sort: true, 418 | } 419 | 420 | // Get options 421 | if opts != nil { 422 | getOpts.Quorum = opts.Consistent 423 | } 424 | 425 | resp, err := s.client.Get(context.Background(), s.normalize(directory), getOpts) 426 | if err != nil { 427 | if keyNotFound(err) { 428 | return nil, store.ErrKeyNotFound 429 | } 430 | return nil, err 431 | } 432 | 433 | kv := []*store.KVPair{} 434 | for _, n := range resp.Node.Nodes { 435 | if n.Key == directory { 436 | continue 437 | } 438 | 439 | // Etcd v2 seems to stop listing child keys at directories even 440 | // with the "Recursive" option. If the child is a directory, 441 | // we call `List` recursively to go through the whole set. 442 | if n.Dir { 443 | pairs, err := s.List(n.Key, opts) 444 | if err != nil { 445 | return nil, err 446 | } 447 | kv = append(kv, pairs...) 448 | } 449 | 450 | // Filter out etcd mutex side keys with `___lock` suffix 451 | if strings.Contains(string(n.Key), lockSuffix) { 452 | continue 453 | } 454 | 455 | kv = append(kv, &store.KVPair{ 456 | Key: n.Key, 457 | Value: []byte(n.Value), 458 | LastIndex: n.ModifiedIndex, 459 | }) 460 | } 461 | return kv, nil 462 | } 463 | 464 | // DeleteTree deletes a range of keys under a given directory 465 | func (s *Etcd) DeleteTree(directory string) error { 466 | delOpts := &etcd.DeleteOptions{ 467 | Recursive: true, 468 | } 469 | 470 | _, err := s.client.Delete(context.Background(), s.normalize(directory), delOpts) 471 | if keyNotFound(err) { 472 | return store.ErrKeyNotFound 473 | } 474 | return err 475 | } 476 | 477 | // NewLock returns a handle to a lock struct which can 478 | // be used to provide mutual exclusion on a key 479 | func (s *Etcd) NewLock(key string, options *store.LockOptions) (lock store.Locker, err error) { 480 | var value string 481 | ttl := defaultLockTTL 482 | renewCh := make(chan struct{}) 483 | 484 | // Apply options on Lock 485 | if options != nil { 486 | if options.Value != nil { 487 | value = string(options.Value) 488 | } 489 | if options.TTL != 0 { 490 | ttl = options.TTL 491 | } 492 | if options.RenewLock != nil { 493 | renewCh = options.RenewLock 494 | } 495 | } 496 | 497 | // Create lock object 498 | lock = &etcdLock{ 499 | client: s.client, 500 | stopRenew: renewCh, 501 | mutexKey: s.normalize(key + lockSuffix), 502 | writeKey: s.normalize(key), 503 | value: value, 504 | ttl: ttl, 505 | } 506 | 507 | return lock, nil 508 | } 509 | 510 | // Lock attempts to acquire the lock and blocks while 511 | // doing so. It returns a channel that is closed if our 512 | // lock is lost or if an error occurs 513 | func (l *etcdLock) Lock(stopChan chan struct{}) (<-chan struct{}, error) { 514 | l.lock.Lock() 515 | defer l.lock.Unlock() 516 | 517 | // Lock holder channel 518 | lockHeld := make(chan struct{}) 519 | stopLocking := l.stopRenew 520 | 521 | setOpts := &etcd.SetOptions{ 522 | TTL: l.ttl, 523 | } 524 | 525 | for { 526 | setOpts.PrevExist = etcd.PrevNoExist 527 | resp, err := l.client.Set(context.Background(), l.mutexKey, "", setOpts) 528 | if err != nil { 529 | if etcdError, ok := err.(etcd.Error); ok { 530 | if etcdError.Code != etcd.ErrorCodeNodeExist { 531 | return nil, err 532 | } 533 | setOpts.PrevIndex = ^uint64(0) 534 | } 535 | } else { 536 | setOpts.PrevIndex = resp.Node.ModifiedIndex 537 | } 538 | 539 | setOpts.PrevExist = etcd.PrevExist 540 | l.last, err = l.client.Set(context.Background(), l.mutexKey, "", setOpts) 541 | 542 | if err == nil { 543 | // Leader section 544 | l.stopLock = stopLocking 545 | go l.holdLock(l.mutexKey, lockHeld, stopLocking) 546 | 547 | // We are holding the lock, set the write key 548 | _, err = l.client.Set(context.Background(), l.writeKey, l.value, nil) 549 | if err != nil { 550 | return nil, err 551 | } 552 | 553 | break 554 | } else { 555 | // If this is a legitimate error, return 556 | if etcdError, ok := err.(etcd.Error); ok { 557 | if etcdError.Code != etcd.ErrorCodeTestFailed { 558 | return nil, err 559 | } 560 | } 561 | 562 | // Seeker section 563 | errorCh := make(chan error) 564 | chWStop := make(chan bool) 565 | free := make(chan bool) 566 | 567 | go l.waitLock(l.mutexKey, errorCh, chWStop, free) 568 | 569 | // Wait for the key to be available or for 570 | // a signal to stop trying to lock the key 571 | select { 572 | case <-free: 573 | break 574 | case err := <-errorCh: 575 | return nil, err 576 | case <-stopChan: 577 | return nil, ErrAbortTryLock 578 | } 579 | 580 | // Delete or Expire event occurred 581 | // Retry 582 | } 583 | } 584 | 585 | return lockHeld, nil 586 | } 587 | 588 | // Hold the lock as long as we can 589 | // Updates the key ttl periodically until we receive 590 | // an explicit stop signal from the Unlock method 591 | func (l *etcdLock) holdLock(key string, lockHeld chan struct{}, stopLocking <-chan struct{}) { 592 | defer close(lockHeld) 593 | 594 | update := time.NewTicker(l.ttl / 3) 595 | defer update.Stop() 596 | 597 | var err error 598 | setOpts := &etcd.SetOptions{TTL: l.ttl} 599 | 600 | for { 601 | select { 602 | case <-update.C: 603 | setOpts.PrevIndex = l.last.Node.ModifiedIndex 604 | l.last, err = l.client.Set(context.Background(), key, "", setOpts) 605 | if err != nil { 606 | return 607 | } 608 | 609 | case <-stopLocking: 610 | return 611 | } 612 | } 613 | } 614 | 615 | // WaitLock simply waits for the key to be available for creation 616 | func (l *etcdLock) waitLock(key string, errorCh chan error, stopWatchCh chan bool, free chan<- bool) { 617 | opts := &etcd.WatcherOptions{Recursive: false} 618 | watcher := l.client.Watcher(key, opts) 619 | 620 | for { 621 | event, err := watcher.Next(context.Background()) 622 | if err != nil { 623 | errorCh <- err 624 | return 625 | } 626 | if event.Action == "delete" || event.Action == "compareAndDelete" || event.Action == "expire" { 627 | free <- true 628 | return 629 | } 630 | } 631 | } 632 | 633 | // Unlock the "key". Calling unlock while 634 | // not holding the lock will throw an error 635 | func (l *etcdLock) Unlock() error { 636 | l.lock.Lock() 637 | defer l.lock.Unlock() 638 | 639 | if l.stopLock != nil { 640 | l.stopLock <- struct{}{} 641 | } 642 | if l.last != nil { 643 | delOpts := &etcd.DeleteOptions{ 644 | PrevIndex: l.last.Node.ModifiedIndex, 645 | } 646 | _, err := l.client.Delete(context.Background(), l.mutexKey, delOpts) 647 | if err != nil { 648 | return err 649 | } 650 | } 651 | return nil 652 | } 653 | 654 | // Close closes the client connection 655 | func (s *Etcd) Close() { 656 | return 657 | } 658 | -------------------------------------------------------------------------------- /store/etcd/v2/etcd_test.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/docker/libkv" 8 | "github.com/docker/libkv/store" 9 | "github.com/docker/libkv/testutils" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var ( 14 | client = "localhost:4001" 15 | ) 16 | 17 | func makeEtcdClient(t *testing.T) store.Store { 18 | kv, err := New( 19 | []string{client}, 20 | &store.Config{ 21 | ConnectionTimeout: 3 * time.Second, 22 | Username: "test", 23 | Password: "very-secure", 24 | }, 25 | ) 26 | 27 | if err != nil { 28 | t.Fatalf("cannot create store: %v", err) 29 | } 30 | 31 | return kv 32 | } 33 | 34 | func TestRegister(t *testing.T) { 35 | Register() 36 | 37 | kv, err := libkv.NewStore(store.ETCD, []string{client}, nil) 38 | assert.NoError(t, err) 39 | assert.NotNil(t, kv) 40 | 41 | if _, ok := kv.(*Etcd); !ok { 42 | t.Fatal("Error registering and initializing etcd") 43 | } 44 | } 45 | 46 | func TestEtcdStore(t *testing.T) { 47 | kv := makeEtcdClient(t) 48 | lockKV := makeEtcdClient(t) 49 | ttlKV := makeEtcdClient(t) 50 | 51 | testutils.RunTestCommon(t, kv) 52 | testutils.RunTestAtomic(t, kv) 53 | testutils.RunTestWatch(t, kv) 54 | testutils.RunTestLock(t, kv) 55 | testutils.RunTestLockTTL(t, kv, lockKV) 56 | testutils.RunTestListLock(t, kv) 57 | testutils.RunTestTTL(t, kv, ttlKV) 58 | testutils.RunCleanup(t, kv) 59 | } 60 | -------------------------------------------------------------------------------- /store/etcd/v3/etcd.go: -------------------------------------------------------------------------------- 1 | package etcdv3 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "strings" 7 | "sync" 8 | "time" 9 | 10 | etcd "github.com/coreos/etcd/clientv3" 11 | "github.com/coreos/etcd/clientv3/concurrency" 12 | "github.com/docker/libkv" 13 | "github.com/docker/libkv/store" 14 | ) 15 | 16 | const ( 17 | defaultLockTTL = 20 * time.Second 18 | etcdDefaultTimeout = 5 * time.Second 19 | lockSuffix = "___lock" 20 | ) 21 | 22 | // EtcdV3 is the receiver type for the 23 | // Store interface 24 | type EtcdV3 struct { 25 | client *etcd.Client 26 | } 27 | 28 | type etcdLock struct { 29 | lock sync.Mutex 30 | store *EtcdV3 31 | 32 | mutex *concurrency.Mutex 33 | session *concurrency.Session 34 | 35 | mutexKey string // mutexKey is the key to write appended with a "_lock" suffix 36 | writeKey string // writeKey is the actual key to update protected by the mutexKey 37 | value string 38 | ttl time.Duration 39 | } 40 | 41 | // Register registers etcd to libkv 42 | func Register() { 43 | libkv.AddStore(store.ETCDV3, New) 44 | } 45 | 46 | // New creates a new Etcd client given a list 47 | // of endpoints and an optional tls config 48 | func New(addrs []string, options *store.Config) (store.Store, error) { 49 | s := &EtcdV3{} 50 | 51 | var ( 52 | entries []string 53 | err error 54 | ) 55 | 56 | entries = store.CreateEndpoints(addrs, "http") 57 | cfg := &etcd.Config{ 58 | Endpoints: entries, 59 | } 60 | 61 | // Set options 62 | if options != nil { 63 | if options.TLS != nil { 64 | setTLS(cfg, options.TLS, addrs) 65 | } 66 | if options.ConnectionTimeout != 0 { 67 | setTimeout(cfg, options.ConnectionTimeout) 68 | } 69 | if options.Username != "" { 70 | setCredentials(cfg, options.Username, options.Password) 71 | } 72 | if options.SyncPeriod != 0 { 73 | cfg.AutoSyncInterval = options.SyncPeriod 74 | } 75 | } 76 | 77 | s.client, err = etcd.New(*cfg) 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | return s, nil 83 | } 84 | 85 | // setTLS sets the tls configuration given a tls.Config scheme 86 | func setTLS(cfg *etcd.Config, tls *tls.Config, addrs []string) { 87 | entries := store.CreateEndpoints(addrs, "https") 88 | cfg.Endpoints = entries 89 | cfg.TLS = tls 90 | } 91 | 92 | // setTimeout sets the timeout used for connecting to the store 93 | func setTimeout(cfg *etcd.Config, time time.Duration) { 94 | cfg.DialTimeout = time 95 | } 96 | 97 | // setCredentials sets the username/password credentials for connecting to Etcd 98 | func setCredentials(cfg *etcd.Config, username, password string) { 99 | cfg.Username = username 100 | cfg.Password = password 101 | } 102 | 103 | // Normalize the key for usage in Etcd 104 | func (s *EtcdV3) normalize(key string) string { 105 | key = store.Normalize(key) 106 | return strings.TrimPrefix(key, "/") 107 | } 108 | 109 | // Get the value at "key", returns the last modified 110 | // index to use in conjunction to Atomic calls 111 | func (s *EtcdV3) Get(key string, opts *store.ReadOptions) (pair *store.KVPair, err error) { 112 | ctx, cancel := context.WithTimeout(context.Background(), etcdDefaultTimeout) 113 | 114 | var result *etcd.GetResponse 115 | 116 | if opts != nil && !opts.Consistent { 117 | result, err = s.client.KV.Get(ctx, s.normalize(key), etcd.WithSerializable()) 118 | } else { 119 | result, err = s.client.KV.Get(ctx, s.normalize(key)) 120 | } 121 | 122 | cancel() 123 | 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | if result.Count == 0 { 129 | return nil, store.ErrKeyNotFound 130 | } 131 | 132 | kvs := []*store.KVPair{} 133 | 134 | for _, pair := range result.Kvs { 135 | kvs = append(kvs, &store.KVPair{ 136 | Key: string(pair.Key), 137 | Value: []byte(pair.Value), 138 | LastIndex: uint64(pair.ModRevision), 139 | }) 140 | } 141 | 142 | return kvs[0], nil 143 | } 144 | 145 | // Put a value at "key" 146 | func (s *EtcdV3) Put(key string, value []byte, opts *store.WriteOptions) (err error) { 147 | ctx, cancel := context.WithTimeout(context.Background(), etcdDefaultTimeout) 148 | pr := s.client.Txn(ctx) 149 | 150 | if opts != nil && opts.TTL > 0 { 151 | lease := etcd.NewLease(s.client) 152 | resp, err := lease.Grant(context.Background(), int64(opts.TTL/time.Second)) 153 | if err != nil { 154 | cancel() 155 | return err 156 | } 157 | pr.Then(etcd.OpPut(key, string(value), etcd.WithLease(resp.ID))) 158 | } else { 159 | pr.Then(etcd.OpPut(key, string(value))) 160 | } 161 | 162 | _, err = pr.Commit() 163 | cancel() 164 | if err != nil { 165 | return err 166 | } 167 | 168 | return nil 169 | } 170 | 171 | // Delete a value at "key" 172 | func (s *EtcdV3) Delete(key string) error { 173 | resp, err := s.client.KV.Delete(context.Background(), s.normalize(key)) 174 | if resp.Deleted == 0 { 175 | return store.ErrKeyNotFound 176 | } 177 | return err 178 | } 179 | 180 | // Exists checks if the key exists inside the store 181 | func (s *EtcdV3) Exists(key string, opts *store.ReadOptions) (bool, error) { 182 | _, err := s.Get(key, opts) 183 | if err != nil { 184 | if err == store.ErrKeyNotFound { 185 | return false, nil 186 | } 187 | return false, err 188 | } 189 | return true, nil 190 | } 191 | 192 | // Watch for changes on a "key" 193 | // It returns a channel that will receive changes or pass 194 | // on errors. Upon creation, the current value will first 195 | // be sent to the channel. Providing a non-nil stopCh can 196 | // be used to stop watching. 197 | func (s *EtcdV3) Watch(key string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan *store.KVPair, error) { 198 | wc := etcd.NewWatcher(s.client) 199 | 200 | // respCh is sending back events to the caller 201 | respCh := make(chan *store.KVPair) 202 | 203 | // Get the current value 204 | pair, err := s.Get(key, opts) 205 | if err != nil { 206 | return nil, err 207 | } 208 | 209 | go func() { 210 | defer wc.Close() 211 | defer close(respCh) 212 | 213 | // Push the current value through the channel. 214 | respCh <- pair 215 | 216 | watchCh := wc.Watch(context.Background(), s.normalize(key)) 217 | 218 | for resp := range watchCh { 219 | // Check if the watch was stopped by the caller 220 | select { 221 | case <-stopCh: 222 | return 223 | default: 224 | } 225 | 226 | for _, ev := range resp.Events { 227 | respCh <- &store.KVPair{ 228 | Key: key, 229 | Value: []byte(ev.Kv.Value), 230 | LastIndex: uint64(ev.Kv.ModRevision), 231 | } 232 | } 233 | } 234 | }() 235 | 236 | return respCh, nil 237 | } 238 | 239 | // WatchTree watches for changes on a "directory" 240 | // It returns a channel that will receive changes or pass 241 | // on errors. Upon creating a watch, the current childs values 242 | // will be sent to the channel. Providing a non-nil stopCh can 243 | // be used to stop watching. 244 | func (s *EtcdV3) WatchTree(directory string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan []*store.KVPair, error) { 245 | wc := etcd.NewWatcher(s.client) 246 | 247 | // respCh is sending back events to the caller 248 | respCh := make(chan []*store.KVPair) 249 | 250 | // Get the current value 251 | rev, pairs, err := s.list(directory, opts) 252 | if err != nil { 253 | return nil, err 254 | } 255 | 256 | go func() { 257 | defer wc.Close() 258 | defer close(respCh) 259 | 260 | // Push the current value through the channel. 261 | respCh <- pairs 262 | 263 | rev++ 264 | watchCh := wc.Watch(context.Background(), s.normalize(directory), etcd.WithPrefix(), etcd.WithRev(rev)) 265 | 266 | for resp := range watchCh { 267 | // Check if the watch was stopped by the caller 268 | select { 269 | case <-stopCh: 270 | return 271 | default: 272 | } 273 | 274 | list := make([]*store.KVPair, len(resp.Events)) 275 | 276 | for i, ev := range resp.Events { 277 | list[i] = &store.KVPair{ 278 | Key: string(ev.Kv.Key), 279 | Value: []byte(ev.Kv.Value), 280 | LastIndex: uint64(ev.Kv.ModRevision), 281 | } 282 | } 283 | 284 | respCh <- list 285 | } 286 | }() 287 | 288 | return respCh, nil 289 | } 290 | 291 | // AtomicPut puts a value at "key" if the key has not been 292 | // modified in the meantime, throws an error if this is the case 293 | func (s *EtcdV3) AtomicPut(key string, value []byte, previous *store.KVPair, opts *store.WriteOptions) (bool, *store.KVPair, error) { 294 | var cmp etcd.Cmp 295 | var testIndex bool 296 | 297 | if previous != nil { 298 | // We compare on the last modified index 299 | testIndex = true 300 | cmp = etcd.Compare(etcd.ModRevision(key), "=", int64(previous.LastIndex)) 301 | } else { 302 | // Previous key is not given, thus we want the key not to exist 303 | testIndex = false 304 | cmp = etcd.Compare(etcd.CreateRevision(key), "=", 0) 305 | } 306 | 307 | ctx, cancel := context.WithTimeout(context.Background(), etcdDefaultTimeout) 308 | pr := s.client.Txn(ctx).If(cmp) 309 | 310 | // We set the TTL if given 311 | if opts != nil && opts.TTL > 0 { 312 | lease := etcd.NewLease(s.client) 313 | resp, err := lease.Grant(context.Background(), int64(opts.TTL/time.Second)) 314 | if err != nil { 315 | cancel() 316 | return false, nil, err 317 | } 318 | pr.Then(etcd.OpPut(key, string(value), etcd.WithLease(resp.ID))) 319 | } else { 320 | pr.Then(etcd.OpPut(key, string(value))) 321 | } 322 | 323 | txn, err := pr.Commit() 324 | cancel() 325 | if err != nil { 326 | return false, nil, err 327 | } 328 | 329 | if !txn.Succeeded { 330 | if testIndex { 331 | return false, nil, store.ErrKeyModified 332 | } 333 | return false, nil, store.ErrKeyExists 334 | } 335 | 336 | updated := &store.KVPair{ 337 | Key: key, 338 | Value: value, 339 | LastIndex: uint64(txn.Header.Revision), 340 | } 341 | 342 | return true, updated, nil 343 | } 344 | 345 | // AtomicDelete deletes a value at "key" if the key 346 | // has not been modified in the meantime, throws an 347 | // error if this is the case 348 | func (s *EtcdV3) AtomicDelete(key string, previous *store.KVPair) (bool, error) { 349 | if previous == nil { 350 | return false, store.ErrPreviousNotSpecified 351 | } 352 | 353 | // We compare on the last modified index 354 | cmp := etcd.Compare(etcd.ModRevision(key), "=", int64(previous.LastIndex)) 355 | 356 | ctx, cancel := context.WithTimeout(context.Background(), etcdDefaultTimeout) 357 | txn, err := s.client.Txn(ctx). 358 | If(cmp). 359 | Then(etcd.OpDelete(key)). 360 | Commit() 361 | cancel() 362 | 363 | if err != nil { 364 | return false, err 365 | } 366 | 367 | if len(txn.Responses) == 0 { 368 | return false, store.ErrKeyNotFound 369 | } 370 | 371 | if !txn.Succeeded { 372 | return false, store.ErrKeyModified 373 | } 374 | 375 | return true, nil 376 | } 377 | 378 | // List child nodes of a given directory 379 | func (s *EtcdV3) List(directory string, opts *store.ReadOptions) ([]*store.KVPair, error) { 380 | _, kv, err := s.list(directory, opts) 381 | return kv, err 382 | } 383 | 384 | // DeleteTree deletes a range of keys under a given directory 385 | func (s *EtcdV3) DeleteTree(directory string) error { 386 | ctx, cancel := context.WithTimeout(context.Background(), etcdDefaultTimeout) 387 | resp, err := s.client.KV.Delete(ctx, s.normalize(directory), etcd.WithPrefix()) 388 | cancel() 389 | if resp.Deleted == 0 { 390 | return store.ErrKeyNotFound 391 | } 392 | return err 393 | } 394 | 395 | // NewLock returns a handle to a lock struct which can 396 | // be used to provide mutual exclusion on a key 397 | func (s *EtcdV3) NewLock(key string, options *store.LockOptions) (lock store.Locker, err error) { 398 | var value string 399 | ttl := defaultLockTTL 400 | renewCh := make(chan struct{}) 401 | 402 | // Apply options on Lock 403 | if options != nil { 404 | if options.Value != nil { 405 | value = string(options.Value) 406 | } 407 | if options.TTL != 0 { 408 | ttl = options.TTL 409 | } 410 | if options.RenewLock != nil { 411 | renewCh = options.RenewLock 412 | } 413 | } 414 | 415 | // Create Session for Mutex 416 | session, err := concurrency.NewSession(s.client, concurrency.WithTTL(int(ttl/time.Second))) 417 | if err != nil { 418 | return nil, err 419 | } 420 | 421 | go func() { 422 | <-renewCh 423 | session.Close() 424 | return 425 | }() 426 | 427 | // A Mutex is a simple key that can only be held by a single process. 428 | // An etcd mutex behaves like a Zookeeper lock: a side key is created with 429 | // a suffix (such as "_lock") and represents the mutex. Thus we have a pair 430 | // composed of the key to protect with a lock: "/key", and a side key that 431 | // acts as the lock: "/key_lock" 432 | mutexKey := s.normalize(key + lockSuffix) 433 | writeKey := s.normalize(key) 434 | 435 | // Create lock object 436 | lock = &etcdLock{ 437 | store: s, 438 | mutex: concurrency.NewMutex(session, mutexKey), 439 | session: session, 440 | mutexKey: mutexKey, 441 | writeKey: writeKey, 442 | value: value, 443 | ttl: ttl, 444 | } 445 | 446 | return lock, nil 447 | } 448 | 449 | // Lock attempts to acquire the lock and blocks while 450 | // doing so. It returns a channel that is closed if our 451 | // lock is lost or if an error occurs 452 | func (l *etcdLock) Lock(stopChan chan struct{}) (<-chan struct{}, error) { 453 | l.lock.Lock() 454 | defer l.lock.Unlock() 455 | 456 | ctx, cancel := context.WithCancel(context.Background()) 457 | go func() { 458 | <-stopChan 459 | cancel() 460 | }() 461 | err := l.mutex.Lock(ctx) 462 | if err != nil { 463 | if err == context.Canceled { 464 | return nil, nil 465 | } 466 | return nil, err 467 | } 468 | 469 | err = l.store.Put(l.writeKey, []byte(l.value), nil) 470 | if err != nil { 471 | return nil, err 472 | } 473 | 474 | return l.session.Done(), nil 475 | } 476 | 477 | // Unlock the "key". Calling unlock while 478 | // not holding the lock will throw an error 479 | func (l *etcdLock) Unlock() error { 480 | l.lock.Lock() 481 | defer l.lock.Unlock() 482 | 483 | return l.mutex.Unlock(context.Background()) 484 | } 485 | 486 | // Close closes the client connection 487 | func (s *EtcdV3) Close() { 488 | s.client.Close() 489 | } 490 | 491 | // list child nodes of a given directory and return revision number 492 | func (s *EtcdV3) list(directory string, opts *store.ReadOptions) (int64, []*store.KVPair, error) { 493 | ctx, cancel := context.WithTimeout(context.Background(), etcdDefaultTimeout) 494 | 495 | var resp *etcd.GetResponse 496 | var err error 497 | 498 | if opts != nil && !opts.Consistent { 499 | resp, err = s.client.KV.Get(ctx, s.normalize(directory), etcd.WithSerializable(), etcd.WithPrefix(), etcd.WithSort(etcd.SortByKey, etcd.SortDescend)) 500 | } else { 501 | resp, err = s.client.KV.Get(ctx, s.normalize(directory), etcd.WithPrefix(), etcd.WithSort(etcd.SortByKey, etcd.SortDescend)) 502 | } 503 | 504 | cancel() 505 | 506 | if err != nil { 507 | return 0, nil, err 508 | } 509 | 510 | if resp.Count == 0 { 511 | return 0, nil, store.ErrKeyNotFound 512 | } 513 | 514 | kv := []*store.KVPair{} 515 | 516 | for _, n := range resp.Kvs { 517 | if string(n.Key) == directory { 518 | continue 519 | } 520 | 521 | // Filter out etcd mutex side keys with `___lock` suffix 522 | if strings.Contains(string(n.Key), lockSuffix) { 523 | continue 524 | } 525 | 526 | kv = append(kv, &store.KVPair{ 527 | Key: string(n.Key), 528 | Value: []byte(n.Value), 529 | LastIndex: uint64(n.ModRevision), 530 | }) 531 | } 532 | 533 | return resp.Header.Revision, kv, nil 534 | } 535 | -------------------------------------------------------------------------------- /store/etcd/v3/etcd_test.go: -------------------------------------------------------------------------------- 1 | package etcdv3 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/docker/libkv" 8 | "github.com/docker/libkv/store" 9 | "github.com/docker/libkv/testutils" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var ( 14 | client = "localhost:4001" 15 | ) 16 | 17 | func makeEtcdV3Client(t *testing.T) store.Store { 18 | kv, err := New( 19 | []string{client}, 20 | &store.Config{ 21 | ConnectionTimeout: 3 * time.Second, 22 | Username: "test", 23 | Password: "very-secure", 24 | }, 25 | ) 26 | 27 | if err != nil { 28 | t.Fatalf("cannot create store: %v", err) 29 | } 30 | 31 | return kv 32 | } 33 | 34 | func TestRegister(t *testing.T) { 35 | Register() 36 | 37 | kv, err := libkv.NewStore(store.ETCDV3, []string{client}, nil) 38 | assert.NoError(t, err) 39 | assert.NotNil(t, kv) 40 | 41 | if _, ok := kv.(*EtcdV3); !ok { 42 | t.Fatal("Error registering and initializing etcd with v3 client") 43 | } 44 | } 45 | 46 | func TestEtcdV3Store(t *testing.T) { 47 | kv := makeEtcdV3Client(t) 48 | lockKV := makeEtcdV3Client(t) 49 | ttlKV := makeEtcdV3Client(t) 50 | 51 | testutils.RunTestCommon(t, kv) 52 | testutils.RunTestAtomic(t, kv) 53 | testutils.RunTestWatch(t, kv) 54 | testutils.RunTestLock(t, kv) 55 | testutils.RunTestLockTTL(t, kv, lockKV) 56 | testutils.RunTestListLock(t, kv) 57 | testutils.RunTestTTL(t, kv, ttlKV) 58 | testutils.RunCleanup(t, kv) 59 | } 60 | -------------------------------------------------------------------------------- /store/helpers.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | // CreateEndpoints creates a list of endpoints given the right scheme 8 | func CreateEndpoints(addrs []string, scheme string) (entries []string) { 9 | for _, addr := range addrs { 10 | entries = append(entries, scheme+"://"+addr) 11 | } 12 | return entries 13 | } 14 | 15 | // Normalize the key for each store to the form: 16 | // 17 | // /path/to/key 18 | // 19 | func Normalize(key string) string { 20 | return "/" + join(SplitKey(key)) 21 | } 22 | 23 | // GetDirectory gets the full directory part of 24 | // the key to the form: 25 | // 26 | // /path/to/ 27 | // 28 | func GetDirectory(key string) string { 29 | parts := SplitKey(key) 30 | parts = parts[:len(parts)-1] 31 | return "/" + join(parts) 32 | } 33 | 34 | // SplitKey splits the key to extract path informations 35 | func SplitKey(key string) (path []string) { 36 | if strings.Contains(key, "/") { 37 | path = strings.Split(key, "/") 38 | } else { 39 | path = []string{key} 40 | } 41 | return path 42 | } 43 | 44 | // join the path parts with '/' 45 | func join(parts []string) string { 46 | return strings.Join(parts, "/") 47 | } 48 | -------------------------------------------------------------------------------- /store/mock/mock.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "github.com/docker/libkv/store" 5 | "github.com/stretchr/testify/mock" 6 | ) 7 | 8 | // Mock store. Mocks all Store functions using testify.Mock 9 | type Mock struct { 10 | mock.Mock 11 | 12 | // Endpoints passed to InitializeMock 13 | Endpoints []string 14 | 15 | // Options passed to InitializeMock 16 | Options *store.Config 17 | } 18 | 19 | // New creates a Mock store 20 | func New(endpoints []string, options *store.Config) (store.Store, error) { 21 | s := &Mock{} 22 | s.Endpoints = endpoints 23 | s.Options = options 24 | return s, nil 25 | } 26 | 27 | // Put mock 28 | func (s *Mock) Put(key string, value []byte, opts *store.WriteOptions) error { 29 | args := s.Mock.Called(key, value, opts) 30 | return args.Error(0) 31 | } 32 | 33 | // Get mock 34 | func (s *Mock) Get(key string, opts *store.ReadOptions) (*store.KVPair, error) { 35 | args := s.Mock.Called(key, opts) 36 | return args.Get(0).(*store.KVPair), args.Error(1) 37 | } 38 | 39 | // Delete mock 40 | func (s *Mock) Delete(key string) error { 41 | args := s.Mock.Called(key) 42 | return args.Error(0) 43 | } 44 | 45 | // Exists mock 46 | func (s *Mock) Exists(key string, opts *store.ReadOptions) (bool, error) { 47 | args := s.Mock.Called(key, opts) 48 | return args.Bool(0), args.Error(1) 49 | } 50 | 51 | // Watch mock 52 | func (s *Mock) Watch(key string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan *store.KVPair, error) { 53 | args := s.Mock.Called(key, stopCh, opts) 54 | return args.Get(0).(<-chan *store.KVPair), args.Error(1) 55 | } 56 | 57 | // WatchTree mock 58 | func (s *Mock) WatchTree(prefix string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan []*store.KVPair, error) { 59 | args := s.Mock.Called(prefix, stopCh, opts) 60 | return args.Get(0).(chan []*store.KVPair), args.Error(1) 61 | } 62 | 63 | // NewLock mock 64 | func (s *Mock) NewLock(key string, options *store.LockOptions) (store.Locker, error) { 65 | args := s.Mock.Called(key, options) 66 | return args.Get(0).(store.Locker), args.Error(1) 67 | } 68 | 69 | // List mock 70 | func (s *Mock) List(prefix string, opts *store.ReadOptions) ([]*store.KVPair, error) { 71 | args := s.Mock.Called(prefix, opts) 72 | return args.Get(0).([]*store.KVPair), args.Error(1) 73 | } 74 | 75 | // DeleteTree mock 76 | func (s *Mock) DeleteTree(prefix string) error { 77 | args := s.Mock.Called(prefix) 78 | return args.Error(0) 79 | } 80 | 81 | // AtomicPut mock 82 | func (s *Mock) AtomicPut(key string, value []byte, previous *store.KVPair, opts *store.WriteOptions) (bool, *store.KVPair, error) { 83 | args := s.Mock.Called(key, value, previous, opts) 84 | return args.Bool(0), args.Get(1).(*store.KVPair), args.Error(2) 85 | } 86 | 87 | // AtomicDelete mock 88 | func (s *Mock) AtomicDelete(key string, previous *store.KVPair) (bool, error) { 89 | args := s.Mock.Called(key, previous) 90 | return args.Bool(0), args.Error(1) 91 | } 92 | 93 | // Lock mock implementation of Locker 94 | type Lock struct { 95 | mock.Mock 96 | } 97 | 98 | // Lock mock 99 | func (l *Lock) Lock(stopCh chan struct{}) (<-chan struct{}, error) { 100 | args := l.Mock.Called(stopCh) 101 | return args.Get(0).(<-chan struct{}), args.Error(1) 102 | } 103 | 104 | // Unlock mock 105 | func (l *Lock) Unlock() error { 106 | args := l.Mock.Called() 107 | return args.Error(0) 108 | } 109 | 110 | // Close mock 111 | func (s *Mock) Close() { 112 | return 113 | } 114 | -------------------------------------------------------------------------------- /store/redis/lua.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | const ( 4 | cmdCAS = "cas" 5 | cmdCAD = "cad" 6 | ) 7 | 8 | func luaScript() string { 9 | return luaScriptStr 10 | } 11 | 12 | const luaScriptStr = ` 13 | -- This lua script implements CAS based commands using lua and redis commands. 14 | 15 | if #KEYS > 0 then error('No Keys should be provided') end 16 | if #ARGV <= 0 then error('ARGV should be provided') end 17 | 18 | local command_name = assert(table.remove(ARGV, 1), 'Must provide a command') 19 | 20 | local decode = function(val) 21 | return cjson.decode(val) 22 | end 23 | 24 | local encode = function(val) 25 | return cjson.encode(val) 26 | end 27 | 28 | local exists = function(key) 29 | return redis.call('exists', key) == 1 30 | end 31 | 32 | local get = function(key) 33 | return redis.call('get', key) 34 | end 35 | 36 | local setex = function(key, val, ex) 37 | if ex == "0" then 38 | return redis.call('set', key, val) 39 | end 40 | return redis.call('set', key, val, 'ex', ex) 41 | end 42 | 43 | local del = function(key) 44 | return redis.call('del', key) 45 | end 46 | 47 | -- cas is compare-and-swap function which compare the old value's signature 48 | -- if they are the same, then swap with new val 49 | -- noted that $old and $new are json formatted strings 50 | -- and key is keyed with 'lastIndex' 51 | local lastIndex = "LastIndex" 52 | local cas = function(key, old, new, ttl) 53 | if not exists(key) then 54 | error("redis: key is not found") 55 | end 56 | local decodedOrig = decode(get(key)) 57 | local decodedOld = decode(old) 58 | if decodedOrig[lastIndex] == decodedOld[lastIndex] then 59 | setex(key, new, ttl) 60 | return "OK" 61 | else 62 | error("redis: value has been changed") 63 | end 64 | end 65 | 66 | -- cad is compare-and-del function which compare the old value's signature 67 | -- if they are the same, then the key will be deleted 68 | -- noted that $old is a json formatted string 69 | -- and key is keyed with 'lastIndex' 70 | local cad = function(key, old) 71 | if not exists(key) then 72 | error("redis: key is not found") 73 | end 74 | local decodedOrig = decode(get(key)) 75 | local decodedOld = decode(old) 76 | if decodedOrig[lastIndex] == decodedOld[lastIndex] then 77 | del(key) 78 | return "OK" 79 | else 80 | error("redis: value has been changed") 81 | end 82 | end 83 | 84 | -- Launcher exposes interfaces which be called by passing the arguments. 85 | local Launcher = { 86 | cas = cas, 87 | cad = cad 88 | } 89 | 90 | local command = assert(Launcher[command_name], 'Unknown command ' .. command_name) 91 | return command(unpack(ARGV)) 92 | ` 93 | -------------------------------------------------------------------------------- /store/redis/redis.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "strings" 9 | "time" 10 | 11 | "github.com/docker/libkv" 12 | "github.com/docker/libkv/store" 13 | 14 | "gopkg.in/redis.v5" 15 | ) 16 | 17 | var ( 18 | // ErrMultipleEndpointsUnsupported is thrown when there are 19 | // multiple endpoints specified for Redis 20 | ErrMultipleEndpointsUnsupported = errors.New("redis: does not support multiple endpoints") 21 | 22 | // ErrTLSUnsupported is thrown when tls config is given 23 | ErrTLSUnsupported = errors.New("redis does not support tls") 24 | 25 | // ErrAbortTryLock is thrown when a user stops trying to seek the lock 26 | // by sending a signal to the stop chan, this is used to verify if the 27 | // operation succeeded 28 | ErrAbortTryLock = errors.New("redis: lock operation aborted") 29 | ) 30 | 31 | // Register registers Redis to libkv 32 | func Register() { 33 | libkv.AddStore(store.REDIS, New) 34 | } 35 | 36 | // New creates a new Redis client given a list 37 | // of endpoints and optional tls config 38 | func New(endpoints []string, options *store.Config) (store.Store, error) { 39 | var password string 40 | if len(endpoints) > 1 { 41 | return nil, ErrMultipleEndpointsUnsupported 42 | } 43 | if options != nil && options.TLS != nil { 44 | return nil, ErrTLSUnsupported 45 | } 46 | if options != nil && options.Password != "" { 47 | password = options.Password 48 | } 49 | return newRedis(endpoints, password) 50 | } 51 | 52 | func newRedis(endpoints []string, password string) (*Redis, error) { 53 | // TODO: use *redis.ClusterClient if we support miltiple endpoints 54 | client := redis.NewClient(&redis.Options{ 55 | Addr: endpoints[0], 56 | DialTimeout: 5 * time.Second, 57 | ReadTimeout: 30 * time.Second, 58 | WriteTimeout: 30 * time.Second, 59 | Password: password, 60 | }) 61 | 62 | return &Redis{ 63 | client: client, 64 | script: redis.NewScript(luaScript()), 65 | codec: defaultCodec{}, 66 | }, nil 67 | } 68 | 69 | type defaultCodec struct{} 70 | 71 | func (c defaultCodec) encode(kv *store.KVPair) (string, error) { 72 | b, err := json.Marshal(kv) 73 | return string(b), err 74 | } 75 | 76 | func (c defaultCodec) decode(b string, kv *store.KVPair) error { 77 | return json.Unmarshal([]byte(b), kv) 78 | } 79 | 80 | // Redis implements libkv.Store interface with redis backend 81 | type Redis struct { 82 | client *redis.Client 83 | script *redis.Script 84 | codec defaultCodec 85 | } 86 | 87 | const ( 88 | noExpiration = time.Duration(0) 89 | defaultLockTTL = 60 * time.Second 90 | ) 91 | 92 | // Put a value at the specified key 93 | func (r *Redis) Put(key string, value []byte, options *store.WriteOptions) error { 94 | expirationAfter := noExpiration 95 | if options != nil && options.TTL != 0 { 96 | expirationAfter = options.TTL 97 | } 98 | 99 | return r.setTTL(normalize(key), &store.KVPair{ 100 | Key: key, 101 | Value: value, 102 | LastIndex: sequenceNum(), 103 | }, expirationAfter) 104 | } 105 | 106 | func (r *Redis) setTTL(key string, val *store.KVPair, ttl time.Duration) error { 107 | valStr, err := r.codec.encode(val) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | return r.client.Set(key, valStr, ttl).Err() 113 | } 114 | 115 | // Get a value given its key 116 | func (r *Redis) Get(key string, opts *store.ReadOptions) (*store.KVPair, error) { 117 | return r.get(normalize(key)) 118 | } 119 | 120 | func (r *Redis) get(key string) (*store.KVPair, error) { 121 | reply, err := r.client.Get(key).Bytes() 122 | if err != nil { 123 | if err == redis.Nil { 124 | return nil, store.ErrKeyNotFound 125 | } 126 | return nil, err 127 | } 128 | val := store.KVPair{} 129 | if err := r.codec.decode(string(reply), &val); err != nil { 130 | return nil, err 131 | } 132 | return &val, nil 133 | } 134 | 135 | // Delete the value at the specified key 136 | func (r *Redis) Delete(key string) error { 137 | return r.client.Del(normalize(key)).Err() 138 | } 139 | 140 | // Exists verify if a Key exists in the store 141 | func (r *Redis) Exists(key string, opts *store.ReadOptions) (bool, error) { 142 | return r.client.Exists(normalize(key)).Result() 143 | } 144 | 145 | // Watch for changes on a key 146 | // glitch: we use notified-then-retrieve to retrieve *store.KVPair. 147 | // so the responses may sometimes inaccurate 148 | func (r *Redis) Watch(key string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan *store.KVPair, error) { 149 | watchCh := make(chan *store.KVPair) 150 | nKey := normalize(key) 151 | 152 | get := getter(func() (interface{}, error) { 153 | pair, err := r.get(nKey) 154 | if err != nil { 155 | return nil, err 156 | } 157 | return pair, nil 158 | }) 159 | 160 | push := pusher(func(v interface{}) { 161 | if val, ok := v.(*store.KVPair); ok { 162 | watchCh <- val 163 | } 164 | }) 165 | 166 | sub, err := newSubscribe(r.client, regexWatch(nKey, false)) 167 | if err != nil { 168 | return nil, err 169 | } 170 | 171 | go func(sub *subscribe, stopCh <-chan struct{}, get getter, push pusher) { 172 | defer sub.Close() 173 | 174 | msgCh := sub.Receive(stopCh) 175 | if err := watchLoop(msgCh, stopCh, get, push); err != nil { 176 | log.Printf("watchLoop in Watch err:%v\n", err) 177 | } 178 | }(sub, stopCh, get, push) 179 | 180 | return watchCh, nil 181 | } 182 | 183 | func regexWatch(key string, withChildren bool) string { 184 | var regex string 185 | if withChildren { 186 | regex = fmt.Sprintf("__keyspace*:%s*", key) 187 | // for all database and keys with $key prefix 188 | } else { 189 | regex = fmt.Sprintf("__keyspace*:%s", key) 190 | // for all database and keys with $key 191 | } 192 | return regex 193 | } 194 | 195 | // getter defines a func type which retrieves data from remote storage 196 | type getter func() (interface{}, error) 197 | 198 | // pusher defines a func type which pushes data blob into watch channel 199 | type pusher func(interface{}) 200 | 201 | func watchLoop(msgCh chan *redis.Message, stopCh <-chan struct{}, get getter, push pusher) error { 202 | 203 | // deliver the original data before we setup any events 204 | pair, err := get() 205 | if err != nil { 206 | return err 207 | } 208 | push(pair) 209 | 210 | for range msgCh { 211 | // retrieve and send back 212 | pair, err := get() 213 | if err != nil { 214 | return err 215 | } 216 | push(pair) 217 | } 218 | 219 | return nil 220 | } 221 | 222 | type subscribe struct { 223 | pubsub *redis.PubSub 224 | closeCh chan struct{} 225 | } 226 | 227 | func newSubscribe(client *redis.Client, regex string) (*subscribe, error) { 228 | ch, err := client.PSubscribe(regex) 229 | if err != nil { 230 | return nil, err 231 | } 232 | return &subscribe{ 233 | pubsub: ch, 234 | closeCh: make(chan struct{}), 235 | }, nil 236 | } 237 | 238 | func (s *subscribe) Close() error { 239 | close(s.closeCh) 240 | return s.pubsub.Close() 241 | } 242 | 243 | func (s *subscribe) Receive(stopCh <-chan struct{}) chan *redis.Message { 244 | msgCh := make(chan *redis.Message) 245 | go s.receiveLoop(msgCh, stopCh) 246 | return msgCh 247 | } 248 | 249 | func (s *subscribe) receiveLoop(msgCh chan *redis.Message, stopCh <-chan struct{}) { 250 | defer close(msgCh) 251 | 252 | for { 253 | select { 254 | case <-s.closeCh: 255 | return 256 | case <-stopCh: 257 | return 258 | default: 259 | msg, err := s.pubsub.ReceiveMessage() 260 | if err != nil { 261 | return 262 | } 263 | if msg != nil { 264 | msgCh <- msg 265 | } 266 | } 267 | } 268 | } 269 | 270 | // WatchTree watches for changes on child nodes under 271 | // a given directory 272 | func (r *Redis) WatchTree(directory string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan []*store.KVPair, error) { 273 | watchCh := make(chan []*store.KVPair) 274 | nKey := normalize(directory) 275 | 276 | get := getter(func() (interface{}, error) { 277 | pair, err := r.list(nKey) 278 | if err != nil { 279 | return nil, err 280 | } 281 | return pair, nil 282 | }) 283 | 284 | push := pusher(func(v interface{}) { 285 | if _, ok := v.([]*store.KVPair); !ok { 286 | return 287 | } 288 | watchCh <- v.([]*store.KVPair) 289 | }) 290 | 291 | sub, err := newSubscribe(r.client, regexWatch(nKey, true)) 292 | if err != nil { 293 | return nil, err 294 | } 295 | 296 | go func(sub *subscribe, stopCh <-chan struct{}, get getter, push pusher) { 297 | defer sub.Close() 298 | 299 | msgCh := sub.Receive(stopCh) 300 | if err := watchLoop(msgCh, stopCh, get, push); err != nil { 301 | log.Printf("watchLoop in WatchTree err:%v\n", err) 302 | } 303 | }(sub, stopCh, get, push) 304 | 305 | return watchCh, nil 306 | } 307 | 308 | // NewLock creates a lock for a given key. 309 | // The returned Locker is not held and must be acquired 310 | // with `.Lock`. The Value is optional. 311 | func (r *Redis) NewLock(key string, options *store.LockOptions) (store.Locker, error) { 312 | var ( 313 | value []byte 314 | ttl = defaultLockTTL 315 | ) 316 | 317 | if options != nil && options.TTL != 0 { 318 | ttl = options.TTL 319 | } 320 | if options != nil && len(options.Value) != 0 { 321 | value = options.Value 322 | } 323 | 324 | return &redisLock{ 325 | redis: r, 326 | last: nil, 327 | key: key, 328 | value: value, 329 | ttl: ttl, 330 | unlockCh: make(chan struct{}), 331 | }, nil 332 | } 333 | 334 | type redisLock struct { 335 | redis *Redis 336 | last *store.KVPair 337 | unlockCh chan struct{} 338 | 339 | key string 340 | value []byte 341 | ttl time.Duration 342 | } 343 | 344 | func (l *redisLock) Lock(stopCh chan struct{}) (<-chan struct{}, error) { 345 | lockHeld := make(chan struct{}) 346 | 347 | success, err := l.tryLock(lockHeld, stopCh) 348 | if err != nil { 349 | return nil, err 350 | } 351 | if success { 352 | return lockHeld, nil 353 | } 354 | 355 | // wait for changes on the key 356 | watch, err := l.redis.Watch(l.key, stopCh, nil) 357 | if err != nil { 358 | return nil, err 359 | } 360 | 361 | for { 362 | select { 363 | case <-stopCh: 364 | return nil, ErrAbortTryLock 365 | case <-watch: 366 | success, err := l.tryLock(lockHeld, stopCh) 367 | if err != nil { 368 | return nil, err 369 | } 370 | if success { 371 | return lockHeld, nil 372 | } 373 | } 374 | } 375 | } 376 | 377 | // tryLock return true, nil when it acquired and hold the lock 378 | // and return false, nil when it can't lock now, 379 | // and return false, err if any unespected error happened underlying 380 | func (l *redisLock) tryLock(lockHeld, stopChan chan struct{}) (bool, error) { 381 | success, new, err := l.redis.AtomicPut( 382 | l.key, 383 | l.value, 384 | l.last, 385 | &store.WriteOptions{ 386 | TTL: l.ttl, 387 | }) 388 | if success { 389 | l.last = new 390 | // keep holding 391 | go l.holdLock(lockHeld, stopChan) 392 | return true, nil 393 | } 394 | if err != nil && (err == store.ErrKeyNotFound || err == store.ErrKeyModified || err == store.ErrKeyExists) { 395 | return false, nil 396 | } 397 | return false, err 398 | } 399 | 400 | func (l *redisLock) holdLock(lockHeld, stopChan chan struct{}) { 401 | defer close(lockHeld) 402 | 403 | hold := func() error { 404 | _, new, err := l.redis.AtomicPut( 405 | l.key, 406 | l.value, 407 | l.last, 408 | &store.WriteOptions{ 409 | TTL: l.ttl, 410 | }) 411 | if err == nil { 412 | l.last = new 413 | } 414 | return err 415 | } 416 | 417 | heartbeat := time.NewTicker(l.ttl / 3) 418 | defer heartbeat.Stop() 419 | 420 | for { 421 | select { 422 | case <-heartbeat.C: 423 | if err := hold(); err != nil { 424 | return 425 | } 426 | case <-l.unlockCh: 427 | return 428 | case <-stopChan: 429 | return 430 | } 431 | } 432 | } 433 | 434 | func (l *redisLock) Unlock() error { 435 | l.unlockCh <- struct{}{} 436 | 437 | _, err := l.redis.AtomicDelete(l.key, l.last) 438 | if err != nil { 439 | return err 440 | } 441 | l.last = nil 442 | 443 | return err 444 | } 445 | 446 | // List the content of a given prefix 447 | func (r *Redis) List(directory string, opts *store.ReadOptions) ([]*store.KVPair, error) { 448 | return r.list(normalize(directory)) 449 | } 450 | 451 | func (r *Redis) list(directory string) ([]*store.KVPair, error) { 452 | 453 | var allKeys []string 454 | regex := scanRegex(directory) // for all keyed with $directory 455 | allKeys, err := r.keys(regex) 456 | if err != nil { 457 | return nil, err 458 | } 459 | // TODO: need to handle when #key is too large 460 | return r.mget(directory, allKeys...) 461 | } 462 | 463 | func (r *Redis) keys(regex string) ([]string, error) { 464 | const ( 465 | startCursor = 0 466 | endCursor = 0 467 | defaultCount = 10 468 | ) 469 | 470 | var allKeys []string 471 | 472 | keys, nextCursor, err := r.client.Scan(startCursor, regex, defaultCount).Result() 473 | if err != nil { 474 | return nil, err 475 | } 476 | allKeys = append(allKeys, keys...) 477 | for nextCursor != endCursor { 478 | keys, nextCursor, err = r.client.Scan(nextCursor, regex, defaultCount).Result() 479 | if err != nil { 480 | return nil, err 481 | } 482 | 483 | allKeys = append(allKeys, keys...) 484 | } 485 | if len(allKeys) == 0 { 486 | return nil, store.ErrKeyNotFound 487 | } 488 | return allKeys, nil 489 | } 490 | 491 | // mget values given their keys 492 | func (r *Redis) mget(directory string, keys ...string) ([]*store.KVPair, error) { 493 | replies, err := r.client.MGet(keys...).Result() 494 | if err != nil { 495 | return nil, err 496 | } 497 | 498 | pairs := []*store.KVPair{} 499 | for _, reply := range replies { 500 | var sreply string 501 | if _, ok := reply.(string); ok { 502 | sreply = reply.(string) 503 | } 504 | if sreply == "" { 505 | // empty reply 506 | continue 507 | } 508 | 509 | newkv := &store.KVPair{} 510 | if err := r.codec.decode(sreply, newkv); err != nil { 511 | return nil, err 512 | } 513 | if normalize(newkv.Key) != directory { 514 | pairs = append(pairs, newkv) 515 | } 516 | } 517 | return pairs, nil 518 | } 519 | 520 | // DeleteTree deletes a range of keys under a given directory 521 | // glitch: we list all available keys first and then delete them all 522 | // it costs two operations on redis, so is not atomicity. 523 | func (r *Redis) DeleteTree(directory string) error { 524 | var allKeys []string 525 | regex := scanRegex(normalize(directory)) // for all keyed with $directory 526 | allKeys, err := r.keys(regex) 527 | if err != nil { 528 | return err 529 | } 530 | return r.client.Del(allKeys...).Err() 531 | } 532 | 533 | // AtomicPut is an atomic CAS operation on a single value. 534 | // Pass previous = nil to create a new key. 535 | // we introduced script on this page, so atomicity is guaranteed 536 | func (r *Redis) AtomicPut(key string, value []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) { 537 | expirationAfter := noExpiration 538 | if options != nil && options.TTL != 0 { 539 | expirationAfter = options.TTL 540 | } 541 | 542 | newKV := &store.KVPair{ 543 | Key: key, 544 | Value: value, 545 | LastIndex: sequenceNum(), 546 | } 547 | nKey := normalize(key) 548 | 549 | // if previous == nil, set directly 550 | if previous == nil { 551 | if err := r.setNX(nKey, newKV, expirationAfter); err != nil { 552 | return false, nil, err 553 | } 554 | return true, newKV, nil 555 | } 556 | 557 | if err := r.cas( 558 | nKey, 559 | previous, 560 | newKV, 561 | formatSec(expirationAfter), 562 | ); err != nil { 563 | return false, nil, err 564 | } 565 | return true, newKV, nil 566 | } 567 | 568 | func (r *Redis) setNX(key string, val *store.KVPair, expirationAfter time.Duration) error { 569 | valBlob, err := r.codec.encode(val) 570 | if err != nil { 571 | return err 572 | } 573 | 574 | if !r.client.SetNX(key, valBlob, expirationAfter).Val() { 575 | return store.ErrKeyExists 576 | } 577 | return nil 578 | } 579 | 580 | func (r *Redis) cas(key string, old, new *store.KVPair, secInStr string) error { 581 | newVal, err := r.codec.encode(new) 582 | if err != nil { 583 | return err 584 | } 585 | 586 | oldVal, err := r.codec.encode(old) 587 | if err != nil { 588 | return err 589 | } 590 | 591 | return r.runScript( 592 | cmdCAS, 593 | key, 594 | oldVal, 595 | newVal, 596 | secInStr, 597 | ) 598 | } 599 | 600 | // AtomicDelete is an atomic delete operation on a single value 601 | // the value will be deleted if previous matched the one stored in db 602 | func (r *Redis) AtomicDelete(key string, previous *store.KVPair) (bool, error) { 603 | if err := r.cad(normalize(key), previous); err != nil { 604 | return false, err 605 | } 606 | return true, nil 607 | } 608 | 609 | func (r *Redis) cad(key string, old *store.KVPair) error { 610 | oldVal, err := r.codec.encode(old) 611 | if err != nil { 612 | return err 613 | } 614 | 615 | return r.runScript( 616 | cmdCAD, 617 | key, 618 | oldVal, 619 | ) 620 | } 621 | 622 | // Close the store connection 623 | func (r *Redis) Close() { 624 | r.client.Close() 625 | } 626 | 627 | func scanRegex(directory string) string { 628 | return fmt.Sprintf("%s*", directory) 629 | } 630 | 631 | func (r *Redis) runScript(args ...interface{}) error { 632 | err := r.script.Run( 633 | r.client, 634 | nil, 635 | args..., 636 | ).Err() 637 | if err != nil && strings.Contains(err.Error(), "redis: key is not found") { 638 | return store.ErrKeyNotFound 639 | } 640 | if err != nil && strings.Contains(err.Error(), "redis: value has been changed") { 641 | return store.ErrKeyModified 642 | } 643 | return err 644 | } 645 | 646 | func normalize(key string) string { 647 | return store.Normalize(key) 648 | } 649 | 650 | func formatSec(dur time.Duration) string { 651 | return fmt.Sprintf("%d", int(dur/time.Second)) 652 | } 653 | 654 | func sequenceNum() uint64 { 655 | // TODO: use uuid if we concerns collision probability of this number 656 | return uint64(time.Now().Nanosecond()) 657 | } 658 | -------------------------------------------------------------------------------- /store/redis/redis_test.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/docker/libkv" 7 | "github.com/docker/libkv/store" 8 | "github.com/docker/libkv/testutils" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | var ( 13 | client = "localhost:6379" 14 | ) 15 | 16 | func makeRedisClient(t *testing.T) store.Store { 17 | kv, err := newRedis([]string{client}, "") 18 | if err != nil { 19 | t.Fatalf("cannot create store: %v", err) 20 | } 21 | 22 | // NOTE: please turn on redis's notification 23 | // before you using watch/watchtree/lock related features 24 | kv.client.ConfigSet("notify-keyspace-events", "KA") 25 | 26 | return kv 27 | } 28 | 29 | func TestRegister(t *testing.T) { 30 | Register() 31 | 32 | kv, err := libkv.NewStore(store.REDIS, []string{client}, nil) 33 | assert.NoError(t, err) 34 | assert.NotNil(t, kv) 35 | 36 | if _, ok := kv.(*Redis); !ok { 37 | t.Fatal("Error registering and initializing redis") 38 | } 39 | } 40 | 41 | func TestRedisStore(t *testing.T) { 42 | kv := makeRedisClient(t) 43 | lockTTL := makeRedisClient(t) 44 | kvTTL := makeRedisClient(t) 45 | 46 | testutils.RunTestCommon(t, kv) 47 | testutils.RunTestAtomic(t, kv) 48 | testutils.RunTestWatch(t, kv) 49 | testutils.RunTestLock(t, kv) 50 | testutils.RunTestLockTTL(t, kv, lockTTL) 51 | testutils.RunTestTTL(t, kv, kvTTL) 52 | testutils.RunCleanup(t, kv) 53 | } 54 | -------------------------------------------------------------------------------- /store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "crypto/tls" 5 | "errors" 6 | "time" 7 | ) 8 | 9 | // Backend represents a KV Store Backend 10 | type Backend string 11 | 12 | const ( 13 | // CONSUL backend 14 | CONSUL Backend = "consul" 15 | // ETCD backend with v2 client (backward compatibility) 16 | ETCD Backend = "etcd" 17 | // ETCDV3 backend with v3 client 18 | ETCDV3 Backend = "etcdv3" 19 | // ZK backend 20 | ZK Backend = "zk" 21 | // BOLTDB backend 22 | BOLTDB Backend = "boltdb" 23 | // REDIS backend 24 | REDIS Backend = "redis" 25 | ) 26 | 27 | var ( 28 | // ErrBackendNotSupported is thrown when the backend k/v store is not supported by libkv 29 | ErrBackendNotSupported = errors.New("Backend storage not supported yet, please choose one of") 30 | // ErrCallNotSupported is thrown when a method is not implemented/supported by the current backend 31 | ErrCallNotSupported = errors.New("The current call is not supported with this backend") 32 | // ErrNotReachable is thrown when the API cannot be reached for issuing common store operations 33 | ErrNotReachable = errors.New("Api not reachable") 34 | // ErrCannotLock is thrown when there is an error acquiring a lock on a key 35 | ErrCannotLock = errors.New("Error acquiring the lock") 36 | // ErrKeyModified is thrown during an atomic operation if the index does not match the one in the store 37 | ErrKeyModified = errors.New("Unable to complete atomic operation, key modified") 38 | // ErrKeyNotFound is thrown when the key is not found in the store during a Get operation 39 | ErrKeyNotFound = errors.New("Key not found in store") 40 | // ErrPreviousNotSpecified is thrown when the previous value is not specified for an atomic operation 41 | ErrPreviousNotSpecified = errors.New("Previous K/V pair should be provided for the Atomic operation") 42 | // ErrKeyExists is thrown when the previous value exists in the case of an AtomicPut 43 | ErrKeyExists = errors.New("Previous K/V pair exists, cannot complete Atomic operation") 44 | ) 45 | 46 | // Config contains the options for a storage client 47 | type Config struct { 48 | ClientTLS *ClientTLSConfig 49 | TLS *tls.Config 50 | ConnectionTimeout time.Duration 51 | SyncPeriod time.Duration 52 | Bucket string 53 | PersistConnection bool 54 | Username string 55 | Password string 56 | } 57 | 58 | // ClientTLSConfig contains data for a Client TLS configuration in the form 59 | // the etcd client wants it. Eventually we'll adapt it for ZK and Consul. 60 | type ClientTLSConfig struct { 61 | CertFile string 62 | KeyFile string 63 | CACertFile string 64 | } 65 | 66 | // Store represents the backend K/V storage 67 | // Each store should support every call listed 68 | // here. Or it couldn't be implemented as a K/V 69 | // backend for libkv 70 | type Store interface { 71 | // Put a value at the specified key 72 | Put(key string, value []byte, options *WriteOptions) error 73 | 74 | // Get a value given its key 75 | Get(key string, options *ReadOptions) (*KVPair, error) 76 | 77 | // Delete the value at the specified key 78 | Delete(key string) error 79 | 80 | // Verify if a Key exists in the store 81 | Exists(key string, options *ReadOptions) (bool, error) 82 | 83 | // Watch for changes on a key 84 | Watch(key string, stopCh <-chan struct{}, options *ReadOptions) (<-chan *KVPair, error) 85 | 86 | // WatchTree watches for changes on child nodes under 87 | // a given directory 88 | WatchTree(directory string, stopCh <-chan struct{}, options *ReadOptions) (<-chan []*KVPair, error) 89 | 90 | // NewLock creates a lock for a given key. 91 | // The returned Locker is not held and must be acquired 92 | // with `.Lock`. The Value is optional. 93 | NewLock(key string, options *LockOptions) (Locker, error) 94 | 95 | // List the content of a given prefix 96 | List(directory string, options *ReadOptions) ([]*KVPair, error) 97 | 98 | // DeleteTree deletes a range of keys under a given directory 99 | DeleteTree(directory string) error 100 | 101 | // Atomic CAS operation on a single value. 102 | // Pass previous = nil to create a new key. 103 | AtomicPut(key string, value []byte, previous *KVPair, options *WriteOptions) (bool, *KVPair, error) 104 | 105 | // Atomic delete of a single value 106 | AtomicDelete(key string, previous *KVPair) (bool, error) 107 | 108 | // Close the store connection 109 | Close() 110 | } 111 | 112 | // KVPair represents {Key, Value, Lastindex} tuple 113 | type KVPair struct { 114 | Key string 115 | Value []byte 116 | LastIndex uint64 117 | } 118 | 119 | // WriteOptions contains optional request parameters 120 | type WriteOptions struct { 121 | IsDir bool 122 | TTL time.Duration 123 | } 124 | 125 | // ReadOptions contains optional request parameters 126 | type ReadOptions struct { 127 | // Consistent defines if the behavior of a Get operation is 128 | // linearizable or not. Linearizability allows us to 'see' 129 | // objects based on a real-time total order as opposed to 130 | // an arbitrary order or with stale values ('inconsistent' 131 | // scenario). 132 | Consistent bool 133 | } 134 | 135 | // LockOptions contains optional request parameters 136 | type LockOptions struct { 137 | Value []byte // Optional, value to associate with the lock 138 | TTL time.Duration // Optional, expiration ttl associated with the lock 139 | RenewLock chan struct{} // Optional, chan used to control and stop the session ttl renewal for the lock 140 | } 141 | 142 | // Locker provides locking mechanism on top of the store. 143 | // Similar to `sync.Lock` except it may return errors. 144 | type Locker interface { 145 | Lock(stopChan chan struct{}) (<-chan struct{}, error) 146 | Unlock() error 147 | } 148 | -------------------------------------------------------------------------------- /store/zookeeper/zookeeper.go: -------------------------------------------------------------------------------- 1 | package zookeeper 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/docker/libkv" 8 | "github.com/docker/libkv/store" 9 | zk "github.com/samuel/go-zookeeper/zk" 10 | ) 11 | 12 | const ( 13 | // SOH control character 14 | SOH = "\x01" 15 | 16 | defaultTimeout = 10 * time.Second 17 | 18 | syncRetryLimit = 5 19 | ) 20 | 21 | // Zookeeper is the receiver type for 22 | // the Store interface 23 | type Zookeeper struct { 24 | timeout time.Duration 25 | client *zk.Conn 26 | } 27 | 28 | type zookeeperLock struct { 29 | client *zk.Conn 30 | lock *zk.Lock 31 | key string 32 | value []byte 33 | } 34 | 35 | // Register registers zookeeper to libkv 36 | func Register() { 37 | libkv.AddStore(store.ZK, New) 38 | } 39 | 40 | // New creates a new Zookeeper client given a 41 | // list of endpoints and an optional tls config 42 | func New(endpoints []string, options *store.Config) (store.Store, error) { 43 | s := &Zookeeper{} 44 | s.timeout = defaultTimeout 45 | 46 | // Set options 47 | if options != nil { 48 | if options.ConnectionTimeout != 0 { 49 | s.setTimeout(options.ConnectionTimeout) 50 | } 51 | } 52 | 53 | // Connect to Zookeeper 54 | conn, _, err := zk.Connect(endpoints, s.timeout) 55 | if err != nil { 56 | return nil, err 57 | } 58 | s.client = conn 59 | 60 | return s, nil 61 | } 62 | 63 | // setTimeout sets the timeout for connecting to Zookeeper 64 | func (s *Zookeeper) setTimeout(time time.Duration) { 65 | s.timeout = time 66 | } 67 | 68 | // Get the value at "key", returns the last modified index 69 | // to use in conjunction to Atomic calls 70 | func (s *Zookeeper) Get(key string, opts *store.ReadOptions) (pair *store.KVPair, err error) { 71 | 72 | resp, meta, err := s.get(key) 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | pair = &store.KVPair{ 78 | Key: key, 79 | Value: resp, 80 | LastIndex: uint64(meta.Version), 81 | } 82 | 83 | return pair, nil 84 | } 85 | 86 | // createFullPath creates the entire path for a directory 87 | // that does not exist and sets the value of the last 88 | // znode to data 89 | func (s *Zookeeper) createFullPath(path []string, data []byte, ephemeral bool) error { 90 | for i := 1; i <= len(path); i++ { 91 | newpath := "/" + strings.Join(path[:i], "/") 92 | 93 | if i == len(path) { 94 | flag := 0 95 | if ephemeral { 96 | flag = zk.FlagEphemeral 97 | } 98 | _, err := s.client.Create(newpath, data, int32(flag), zk.WorldACL(zk.PermAll)) 99 | return err 100 | } 101 | 102 | _, err := s.client.Create(newpath, []byte{}, 0, zk.WorldACL(zk.PermAll)) 103 | if err != nil { 104 | // Skip if node already exists 105 | if err != zk.ErrNodeExists { 106 | return err 107 | } 108 | } 109 | } 110 | return nil 111 | } 112 | 113 | // Put a value at "key" 114 | func (s *Zookeeper) Put(key string, value []byte, opts *store.WriteOptions) error { 115 | fkey := s.normalize(key) 116 | 117 | exists, err := s.Exists(key, nil) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | if !exists { 123 | if opts != nil && opts.TTL > 0 { 124 | s.createFullPath(store.SplitKey(strings.TrimSuffix(key, "/")), value, true) 125 | } else { 126 | s.createFullPath(store.SplitKey(strings.TrimSuffix(key, "/")), value, false) 127 | } 128 | } else { 129 | _, err = s.client.Set(fkey, value, -1) 130 | } 131 | 132 | return err 133 | } 134 | 135 | // Delete a value at "key" 136 | func (s *Zookeeper) Delete(key string) error { 137 | err := s.client.Delete(s.normalize(key), -1) 138 | if err == zk.ErrNoNode { 139 | return store.ErrKeyNotFound 140 | } 141 | return err 142 | } 143 | 144 | // Exists checks if the key exists inside the store 145 | func (s *Zookeeper) Exists(key string, opts *store.ReadOptions) (bool, error) { 146 | exists, _, err := s.client.Exists(s.normalize(key)) 147 | if err != nil { 148 | return false, err 149 | } 150 | return exists, nil 151 | } 152 | 153 | // Watch for changes on a "key" 154 | // It returns a channel that will receive changes or pass 155 | // on errors. Upon creation, the current value will first 156 | // be sent to the channel. Providing a non-nil stopCh can 157 | // be used to stop watching. 158 | func (s *Zookeeper) Watch(key string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan *store.KVPair, error) { 159 | // Catch zk notifications and fire changes into the channel. 160 | watchCh := make(chan *store.KVPair) 161 | go func() { 162 | defer close(watchCh) 163 | 164 | var fireEvt = true 165 | for { 166 | resp, meta, eventCh, err := s.getW(key) 167 | if err != nil { 168 | return 169 | } 170 | if fireEvt { 171 | watchCh <- &store.KVPair{ 172 | Key: key, 173 | Value: resp, 174 | LastIndex: uint64(meta.Version), 175 | } 176 | } 177 | select { 178 | case e := <-eventCh: 179 | // Only fire an event if the data in the node changed. 180 | // Simply reset the watch if this is any other event 181 | // (e.g. a session event). 182 | fireEvt = e.Type == zk.EventNodeDataChanged 183 | case <-stopCh: 184 | // There is no way to stop GetW so just quit 185 | return 186 | } 187 | } 188 | }() 189 | 190 | return watchCh, nil 191 | } 192 | 193 | // WatchTree watches for changes on a "directory" 194 | // It returns a channel that will receive changes or pass 195 | // on errors. Upon creating a watch, the current childs values 196 | // will be sent to the channel .Providing a non-nil stopCh can 197 | // be used to stop watching. 198 | func (s *Zookeeper) WatchTree(directory string, stopCh <-chan struct{}, opts *store.ReadOptions) (<-chan []*store.KVPair, error) { 199 | // Catch zk notifications and fire changes into the channel. 200 | watchCh := make(chan []*store.KVPair) 201 | go func() { 202 | defer close(watchCh) 203 | 204 | var fireEvt = true 205 | for { 206 | WATCH: 207 | keys, _, eventCh, err := s.client.ChildrenW(s.normalize(directory)) 208 | if err != nil { 209 | return 210 | } 211 | if fireEvt { 212 | kvs, err := s.getListWithPath(directory, keys, opts) 213 | if err != nil { 214 | // Failed to get values for one or more of the keys, 215 | // the list may be out of date so try again. 216 | goto WATCH 217 | } 218 | watchCh <- kvs 219 | } 220 | select { 221 | case e := <-eventCh: 222 | // Only fire an event if the children have changed. 223 | // Simply reset the watch if this is any other event 224 | // (e.g. a session event). 225 | fireEvt = e.Type == zk.EventNodeChildrenChanged 226 | case <-stopCh: 227 | // There is no way to stop ChildrenW so just quit 228 | return 229 | } 230 | } 231 | }() 232 | 233 | return watchCh, nil 234 | } 235 | 236 | // listChildren lists the direct children of a directory 237 | func (s *Zookeeper) listChildren(directory string) ([]string, error) { 238 | children, _, err := s.client.Children(s.normalize(directory)) 239 | if err != nil { 240 | if err == zk.ErrNoNode { 241 | return nil, store.ErrKeyNotFound 242 | } 243 | return nil, err 244 | } 245 | return children, nil 246 | } 247 | 248 | // listChildrenRecursive lists the children of a directory as well as 249 | // all the descending childs from sub-folders in a recursive fashion. 250 | func (s *Zookeeper) listChildrenRecursive(list *[]string, directory string) error { 251 | children, err := s.listChildren(directory) 252 | if err != nil { 253 | return err 254 | } 255 | 256 | // We reached a leaf. 257 | if len(children) == 0 { 258 | return nil 259 | } 260 | 261 | for _, c := range children { 262 | c = strings.TrimSuffix(directory, "/") + "/" + c 263 | err := s.listChildrenRecursive(list, c) 264 | if err != nil && err != zk.ErrNoChildrenForEphemerals { 265 | return err 266 | } 267 | *list = append(*list, c) 268 | } 269 | 270 | return nil 271 | } 272 | 273 | // List child nodes of a given directory 274 | func (s *Zookeeper) List(directory string, opts *store.ReadOptions) ([]*store.KVPair, error) { 275 | children := make([]string, 0) 276 | err := s.listChildrenRecursive(&children, directory) 277 | if err != nil { 278 | return nil, err 279 | } 280 | 281 | kvs, err := s.getList(children, opts) 282 | if err != nil { 283 | // If node is not found: List is out of date, retry 284 | if err == store.ErrKeyNotFound { 285 | return s.List(directory, opts) 286 | } 287 | return nil, err 288 | } 289 | 290 | return kvs, nil 291 | } 292 | 293 | // DeleteTree deletes a range of keys under a given directory 294 | func (s *Zookeeper) DeleteTree(directory string) error { 295 | children, err := s.listChildren(directory) 296 | if err != nil { 297 | return err 298 | } 299 | 300 | var reqs []interface{} 301 | 302 | for _, c := range children { 303 | reqs = append(reqs, &zk.DeleteRequest{ 304 | Path: s.normalize(directory + "/" + c), 305 | Version: -1, 306 | }) 307 | } 308 | 309 | _, err = s.client.Multi(reqs...) 310 | return err 311 | } 312 | 313 | // AtomicPut put a value at "key" if the key has not been 314 | // modified in the meantime, throws an error if this is the case 315 | func (s *Zookeeper) AtomicPut(key string, value []byte, previous *store.KVPair, _ *store.WriteOptions) (bool, *store.KVPair, error) { 316 | var lastIndex uint64 317 | 318 | if previous != nil { 319 | meta, err := s.client.Set(s.normalize(key), value, int32(previous.LastIndex)) 320 | if err != nil { 321 | // Compare Failed 322 | if err == zk.ErrBadVersion { 323 | return false, nil, store.ErrKeyModified 324 | } 325 | return false, nil, err 326 | } 327 | lastIndex = uint64(meta.Version) 328 | } else { 329 | // Interpret previous == nil as create operation. 330 | _, err := s.client.Create(s.normalize(key), value, 0, zk.WorldACL(zk.PermAll)) 331 | if err != nil { 332 | // Directory does not exist 333 | if err == zk.ErrNoNode { 334 | 335 | // Create the directory 336 | parts := store.SplitKey(strings.TrimSuffix(key, "/")) 337 | parts = parts[:len(parts)-1] 338 | if err = s.createFullPath(parts, []byte{}, false); err != nil { 339 | // Failed to create the directory. 340 | return false, nil, err 341 | } 342 | 343 | // Create the node 344 | if _, err := s.client.Create(s.normalize(key), value, 0, zk.WorldACL(zk.PermAll)); err != nil { 345 | // Node exist error (when previous nil) 346 | if err == zk.ErrNodeExists { 347 | return false, nil, store.ErrKeyExists 348 | } 349 | return false, nil, err 350 | } 351 | 352 | } else { 353 | // Node Exists error (when previous nil) 354 | if err == zk.ErrNodeExists { 355 | return false, nil, store.ErrKeyExists 356 | } 357 | 358 | // Unhandled error 359 | return false, nil, err 360 | } 361 | } 362 | lastIndex = 0 // Newly created nodes have version 0. 363 | } 364 | 365 | pair := &store.KVPair{ 366 | Key: key, 367 | Value: value, 368 | LastIndex: lastIndex, 369 | } 370 | 371 | return true, pair, nil 372 | } 373 | 374 | // AtomicDelete deletes a value at "key" if the key 375 | // has not been modified in the meantime, throws an 376 | // error if this is the case 377 | func (s *Zookeeper) AtomicDelete(key string, previous *store.KVPair) (bool, error) { 378 | if previous == nil { 379 | return false, store.ErrPreviousNotSpecified 380 | } 381 | 382 | err := s.client.Delete(s.normalize(key), int32(previous.LastIndex)) 383 | if err != nil { 384 | // Key not found 385 | if err == zk.ErrNoNode { 386 | return false, store.ErrKeyNotFound 387 | } 388 | // Compare failed 389 | if err == zk.ErrBadVersion { 390 | return false, store.ErrKeyModified 391 | } 392 | // General store error 393 | return false, err 394 | } 395 | return true, nil 396 | } 397 | 398 | // NewLock returns a handle to a lock struct which can 399 | // be used to provide mutual exclusion on a key 400 | func (s *Zookeeper) NewLock(key string, options *store.LockOptions) (lock store.Locker, err error) { 401 | value := []byte("") 402 | 403 | // Apply options 404 | if options != nil { 405 | if options.Value != nil { 406 | value = options.Value 407 | } 408 | } 409 | 410 | lock = &zookeeperLock{ 411 | client: s.client, 412 | key: s.normalize(key), 413 | value: value, 414 | lock: zk.NewLock(s.client, s.normalize(key), zk.WorldACL(zk.PermAll)), 415 | } 416 | 417 | return lock, err 418 | } 419 | 420 | // Lock attempts to acquire the lock and blocks while 421 | // doing so. It returns a channel that is closed if our 422 | // lock is lost or if an error occurs 423 | func (l *zookeeperLock) Lock(stopChan chan struct{}) (<-chan struct{}, error) { 424 | err := l.lock.Lock() 425 | 426 | lostCh := make(chan struct{}) 427 | if err == nil { 428 | // We hold the lock, we can set our value 429 | _, err = l.client.Set(l.key, l.value, -1) 430 | if err == nil { 431 | go l.monitorLock(stopChan, lostCh) 432 | } 433 | } 434 | 435 | return lostCh, err 436 | } 437 | 438 | // Unlock the "key". Calling unlock while 439 | // not holding the lock will throw an error 440 | func (l *zookeeperLock) Unlock() error { 441 | return l.lock.Unlock() 442 | } 443 | 444 | // Close closes the client connection 445 | func (s *Zookeeper) Close() { 446 | s.client.Close() 447 | } 448 | 449 | // Normalize the key for usage in Zookeeper 450 | func (s *Zookeeper) normalize(key string) string { 451 | key = store.Normalize(key) 452 | return strings.TrimSuffix(key, "/") 453 | } 454 | 455 | func (l *zookeeperLock) monitorLock(stopCh <-chan struct{}, lostCh chan struct{}) { 456 | defer close(lostCh) 457 | 458 | for { 459 | _, _, eventCh, err := l.client.GetW(l.key) 460 | if err != nil { 461 | // We failed to set watch, relinquish the lock 462 | return 463 | } 464 | select { 465 | case e := <-eventCh: 466 | if e.Type == zk.EventNotWatching || 467 | (e.Type == zk.EventSession && e.State == zk.StateExpired) { 468 | // Either the session has been closed and our watch has been 469 | // invalidated or the session has expired. 470 | return 471 | } else if e.Type == zk.EventNodeDataChanged { 472 | // Somemone else has written to the lock node and believes 473 | // that they have the lock. 474 | return 475 | } 476 | case <-stopCh: 477 | // The caller has requested that we relinquish our lock 478 | return 479 | } 480 | } 481 | } 482 | 483 | func (s *Zookeeper) get(key string) ([]byte, *zk.Stat, error) { 484 | var resp []byte 485 | var meta *zk.Stat 486 | var err error 487 | 488 | // To guard against older versions of libkv 489 | // creating and writing to znodes non-atomically, 490 | // We try to resync few times if we read SOH or 491 | // an empty string 492 | for i := 0; i <= syncRetryLimit; i++ { 493 | resp, meta, err = s.client.Get(s.normalize(key)) 494 | 495 | if err != nil { 496 | if err == zk.ErrNoNode { 497 | return nil, nil, store.ErrKeyNotFound 498 | } 499 | return nil, nil, err 500 | } 501 | 502 | if string(resp) != SOH && string(resp) != "" { 503 | return resp, meta, nil 504 | } 505 | 506 | if i < syncRetryLimit { 507 | if _, err = s.client.Sync(s.normalize(key)); err != nil { 508 | return nil, nil, err 509 | } 510 | } 511 | } 512 | return resp, meta, nil 513 | } 514 | 515 | func (s *Zookeeper) getW(key string) ([]byte, *zk.Stat, <-chan zk.Event, error) { 516 | var resp []byte 517 | var meta *zk.Stat 518 | var ech <-chan zk.Event 519 | var err error 520 | 521 | // To guard against older versions of libkv 522 | // creating and writing to znodes non-atomically, 523 | // We try to resync few times if we read SOH or 524 | // an empty string 525 | for i := 0; i <= syncRetryLimit; i++ { 526 | resp, meta, ech, err = s.client.GetW(s.normalize(key)) 527 | 528 | if err != nil { 529 | if err == zk.ErrNoNode { 530 | return nil, nil, nil, store.ErrKeyNotFound 531 | } 532 | return nil, nil, nil, err 533 | } 534 | 535 | if string(resp) != SOH && string(resp) != "" { 536 | return resp, meta, ech, nil 537 | } 538 | 539 | if i < syncRetryLimit { 540 | if _, err = s.client.Sync(s.normalize(key)); err != nil { 541 | return nil, nil, nil, err 542 | } 543 | } 544 | } 545 | return resp, meta, ech, nil 546 | } 547 | 548 | // getListWithPath gets the key/value pairs for a list of keys under 549 | // a given path. 550 | // 551 | // This is generally used when we get a list of child keys which 552 | // are stripped out of their path (for example when using ChildrenW). 553 | func (s *Zookeeper) getListWithPath(path string, keys []string, opts *store.ReadOptions) ([]*store.KVPair, error) { 554 | kvs := []*store.KVPair{} 555 | 556 | for _, key := range keys { 557 | pair, err := s.Get(strings.TrimSuffix(path, "/")+s.normalize(key), opts) 558 | if err != nil { 559 | return nil, err 560 | } 561 | 562 | kvs = append(kvs, &store.KVPair{ 563 | Key: key, 564 | Value: pair.Value, 565 | LastIndex: pair.LastIndex, 566 | }) 567 | } 568 | 569 | return kvs, nil 570 | } 571 | 572 | // getList returns key/value pairs from a list of keys. 573 | // 574 | // This is generally used when we have a full list of keys with 575 | // their full path included. 576 | func (s *Zookeeper) getList(keys []string, opts *store.ReadOptions) ([]*store.KVPair, error) { 577 | kvs := []*store.KVPair{} 578 | 579 | for _, key := range keys { 580 | pair, err := s.Get(strings.TrimSuffix(key, "/"), nil) 581 | if err != nil { 582 | return nil, err 583 | } 584 | 585 | kvs = append(kvs, &store.KVPair{ 586 | Key: key, 587 | Value: pair.Value, 588 | LastIndex: pair.LastIndex, 589 | }) 590 | } 591 | 592 | return kvs, nil 593 | } 594 | -------------------------------------------------------------------------------- /store/zookeeper/zookeeper_test.go: -------------------------------------------------------------------------------- 1 | package zookeeper 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/docker/libkv" 8 | "github.com/docker/libkv/store" 9 | "github.com/docker/libkv/testutils" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var ( 14 | client = "localhost:2181" 15 | ) 16 | 17 | func makeZkClient(t *testing.T) store.Store { 18 | kv, err := New( 19 | []string{client}, 20 | &store.Config{ 21 | ConnectionTimeout: 3 * time.Second, 22 | }, 23 | ) 24 | 25 | if err != nil { 26 | t.Fatalf("cannot create store: %v", err) 27 | } 28 | 29 | return kv 30 | } 31 | 32 | func TestRegister(t *testing.T) { 33 | Register() 34 | 35 | kv, err := libkv.NewStore(store.ZK, []string{client}, nil) 36 | assert.NoError(t, err) 37 | assert.NotNil(t, kv) 38 | 39 | if _, ok := kv.(*Zookeeper); !ok { 40 | t.Fatal("Error registering and initializing zookeeper") 41 | } 42 | } 43 | 44 | func TestZkStore(t *testing.T) { 45 | kv := makeZkClient(t) 46 | ttlKV := makeZkClient(t) 47 | 48 | testutils.RunTestCommon(t, kv) 49 | testutils.RunTestAtomic(t, kv) 50 | testutils.RunTestWatch(t, kv) 51 | testutils.RunTestLock(t, kv) 52 | testutils.RunTestTTL(t, kv, ttlKV) 53 | testutils.RunCleanup(t, kv) 54 | } 55 | -------------------------------------------------------------------------------- /testutils/utils.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | "testing" 8 | "time" 9 | 10 | "github.com/docker/libkv/store" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | // RunTestCommon tests the minimal required APIs which 15 | // should be supported by all K/V backends 16 | func RunTestCommon(t *testing.T, kv store.Store) { 17 | testPutGetDeleteExists(t, kv) 18 | testList(t, kv) 19 | testDeleteTree(t, kv) 20 | } 21 | 22 | // RunTestListLock tests the list output for mutexes 23 | // and checks that internal side keys are not listed 24 | func RunTestListLock(t *testing.T, kv store.Store) { 25 | testListLockKey(t, kv) 26 | } 27 | 28 | // RunTestAtomic tests the Atomic operations by the K/V 29 | // backends 30 | func RunTestAtomic(t *testing.T, kv store.Store) { 31 | testAtomicPut(t, kv) 32 | testAtomicPutCreate(t, kv) 33 | testAtomicPutWithSlashSuffixKey(t, kv) 34 | testAtomicDelete(t, kv) 35 | } 36 | 37 | // RunTestWatch tests the watch/monitor APIs supported 38 | // by the K/V backends. 39 | func RunTestWatch(t *testing.T, kv store.Store) { 40 | testWatch(t, kv) 41 | testWatchTree(t, kv) 42 | } 43 | 44 | // RunTestLock tests the KV pair Lock/Unlock APIs supported 45 | // by the K/V backends. 46 | func RunTestLock(t *testing.T, kv store.Store) { 47 | testLockUnlock(t, kv) 48 | } 49 | 50 | // RunTestLockTTL tests the KV pair Lock with TTL APIs supported 51 | // by the K/V backends. 52 | func RunTestLockTTL(t *testing.T, kv store.Store, backup store.Store) { 53 | testLockTTL(t, kv, backup) 54 | } 55 | 56 | // RunTestTTL tests the TTL functionality of the K/V backend. 57 | func RunTestTTL(t *testing.T, kv store.Store, backup store.Store) { 58 | testPutTTL(t, kv, backup) 59 | } 60 | 61 | func checkPairNotNil(t *testing.T, pair *store.KVPair) { 62 | if assert.NotNil(t, pair) { 63 | if !assert.NotNil(t, pair.Value) { 64 | t.Fatal("test failure, value is nil") 65 | } 66 | } else { 67 | t.Fatal("test failure, pair is nil") 68 | } 69 | } 70 | 71 | func testPutGetDeleteExists(t *testing.T, kv store.Store) { 72 | // Get a not exist key should return ErrKeyNotFound 73 | pair, err := kv.Get("testPutGetDelete_not_exist_key", nil) 74 | assert.Equal(t, store.ErrKeyNotFound, err) 75 | 76 | value := []byte("bar") 77 | for _, key := range []string{ 78 | "testPutGetDeleteExists", 79 | "testPutGetDeleteExists/", 80 | "testPutGetDeleteExists/testbar/", 81 | "testPutGetDeleteExists/testbar/testfoobar", 82 | } { 83 | 84 | // Put the key 85 | err = kv.Put(key, value, nil) 86 | assert.NoError(t, err) 87 | 88 | // Get should return the value and an incremented index 89 | pair, err = kv.Get(key, nil) 90 | assert.NoError(t, err) 91 | checkPairNotNil(t, pair) 92 | assert.Equal(t, pair.Value, value) 93 | assert.NotEqual(t, pair.LastIndex, 0) 94 | 95 | // Exists should return true 96 | exists, err := kv.Exists(key, nil) 97 | assert.NoError(t, err) 98 | assert.True(t, exists) 99 | 100 | // Delete the key 101 | err = kv.Delete(key) 102 | assert.NoError(t, err) 103 | 104 | // Get should fail 105 | pair, err = kv.Get(key, nil) 106 | assert.Error(t, err) 107 | assert.Nil(t, pair) 108 | assert.Nil(t, pair) 109 | 110 | // Exists should return false 111 | exists, err = kv.Exists(key, nil) 112 | assert.NoError(t, err) 113 | assert.False(t, exists) 114 | } 115 | } 116 | 117 | func testWatch(t *testing.T, kv store.Store) { 118 | key := "testWatch" 119 | value := []byte("world") 120 | newValue := []byte("world!") 121 | 122 | // Put the key 123 | err := kv.Put(key, value, nil) 124 | assert.NoError(t, err) 125 | 126 | stopCh := make(<-chan struct{}) 127 | events, err := kv.Watch(key, stopCh, nil) 128 | assert.NoError(t, err) 129 | assert.NotNil(t, events) 130 | 131 | // Update loop 132 | go func() { 133 | timeout := time.After(1 * time.Second) 134 | tick := time.Tick(250 * time.Millisecond) 135 | for { 136 | select { 137 | case <-timeout: 138 | return 139 | case <-tick: 140 | err := kv.Put(key, newValue, nil) 141 | if assert.NoError(t, err) { 142 | continue 143 | } 144 | return 145 | } 146 | } 147 | }() 148 | 149 | // Check for updates 150 | eventCount := 1 151 | for { 152 | select { 153 | case event := <-events: 154 | assert.NotNil(t, event) 155 | if eventCount == 1 { 156 | assert.Equal(t, event.Key, key) 157 | assert.Equal(t, event.Value, value) 158 | } else { 159 | assert.Equal(t, event.Key, key) 160 | assert.Equal(t, event.Value, newValue) 161 | } 162 | eventCount++ 163 | // We received all the events we wanted to check 164 | if eventCount >= 4 { 165 | return 166 | } 167 | case <-time.After(4 * time.Second): 168 | t.Fatal("Timeout reached") 169 | return 170 | } 171 | } 172 | } 173 | 174 | func testWatchTree(t *testing.T, kv store.Store) { 175 | dir := "testWatchTree" 176 | 177 | node1 := "testWatchTree/node1" 178 | value1 := []byte("node1") 179 | 180 | node2 := "testWatchTree/node2" 181 | value2 := []byte("node2") 182 | 183 | node3 := "testWatchTree/node3" 184 | value3 := []byte("node3") 185 | 186 | err := kv.Put(node1, value1, nil) 187 | assert.NoError(t, err) 188 | err = kv.Put(node2, value2, nil) 189 | assert.NoError(t, err) 190 | err = kv.Put(node3, value3, nil) 191 | assert.NoError(t, err) 192 | 193 | stopCh := make(<-chan struct{}) 194 | events, err := kv.WatchTree(dir, stopCh, nil) 195 | assert.NoError(t, err) 196 | assert.NotNil(t, events) 197 | 198 | // Update loop 199 | go func() { 200 | timeout := time.After(500 * time.Millisecond) 201 | for { 202 | select { 203 | case <-timeout: 204 | err := kv.Delete(node3) 205 | assert.NoError(t, err) 206 | return 207 | } 208 | } 209 | }() 210 | 211 | // Check for updates 212 | eventCount := 1 213 | for { 214 | select { 215 | case event := <-events: 216 | assert.NotNil(t, event) 217 | // We received the Delete event on a child node 218 | // Exit test successfully 219 | if eventCount == 2 { 220 | return 221 | } 222 | eventCount++ 223 | case <-time.After(4 * time.Second): 224 | t.Fatal("Timeout reached") 225 | return 226 | } 227 | } 228 | } 229 | 230 | func testAtomicPut(t *testing.T, kv store.Store) { 231 | key := "testAtomicPut" 232 | value := []byte("world") 233 | 234 | // Put the key 235 | err := kv.Put(key, value, nil) 236 | assert.NoError(t, err) 237 | 238 | // Get should return the value and an incremented index 239 | pair, err := kv.Get(key, nil) 240 | assert.NoError(t, err) 241 | checkPairNotNil(t, pair) 242 | assert.Equal(t, pair.Value, value) 243 | assert.NotEqual(t, pair.LastIndex, 0) 244 | 245 | // This CAS should fail: previous exists. 246 | success, _, err := kv.AtomicPut(key, []byte("WORLD"), nil, nil) 247 | assert.Error(t, err) 248 | assert.False(t, success) 249 | 250 | // This CAS should succeed 251 | success, _, err = kv.AtomicPut(key, []byte("WORLD"), pair, nil) 252 | assert.NoError(t, err) 253 | assert.True(t, success) 254 | 255 | // This CAS should fail, key has wrong index. 256 | pair.LastIndex = 6744 257 | success, _, err = kv.AtomicPut(key, []byte("WORLDWORLD"), pair, nil) 258 | assert.Equal(t, err, store.ErrKeyModified) 259 | assert.False(t, success) 260 | } 261 | 262 | func testAtomicPutCreate(t *testing.T, kv store.Store) { 263 | // Use a key in a new directory to ensure Stores will create directories 264 | // that don't yet exist. 265 | key := "testAtomicPutCreate/create" 266 | value := []byte("putcreate") 267 | 268 | // AtomicPut the key, previous = nil indicates create. 269 | success, _, err := kv.AtomicPut(key, value, nil, nil) 270 | assert.NoError(t, err) 271 | assert.True(t, success) 272 | 273 | // Get should return the value and an incremented index 274 | pair, err := kv.Get(key, nil) 275 | assert.NoError(t, err) 276 | checkPairNotNil(t, pair) 277 | assert.Equal(t, pair.Value, value) 278 | 279 | // Attempting to create again should fail. 280 | success, _, err = kv.AtomicPut(key, value, nil, nil) 281 | assert.Equal(t, err, store.ErrKeyExists) 282 | assert.False(t, success) 283 | 284 | // This CAS should succeed, since it has the value from Get() 285 | success, _, err = kv.AtomicPut(key, []byte("PUTCREATE"), pair, nil) 286 | assert.NoError(t, err) 287 | assert.True(t, success) 288 | } 289 | 290 | func testAtomicPutWithSlashSuffixKey(t *testing.T, kv store.Store) { 291 | k1 := "testAtomicPutWithSlashSuffixKey/key/" 292 | success, _, err := kv.AtomicPut(k1, []byte{}, nil, nil) 293 | assert.Nil(t, err) 294 | assert.True(t, success) 295 | } 296 | 297 | func testAtomicDelete(t *testing.T, kv store.Store) { 298 | key := "testAtomicDelete" 299 | value := []byte("world") 300 | 301 | // Put the key 302 | err := kv.Put(key, value, nil) 303 | assert.NoError(t, err) 304 | 305 | // Get should return the value and an incremented index 306 | pair, err := kv.Get(key, nil) 307 | assert.NoError(t, err) 308 | checkPairNotNil(t, pair) 309 | assert.Equal(t, pair.Value, value) 310 | assert.NotEqual(t, pair.LastIndex, 0) 311 | 312 | tempIndex := pair.LastIndex 313 | 314 | // AtomicDelete should fail 315 | pair.LastIndex = 6744 316 | success, err := kv.AtomicDelete(key, pair) 317 | assert.Error(t, err) 318 | assert.False(t, success) 319 | 320 | // AtomicDelete should succeed 321 | pair.LastIndex = tempIndex 322 | success, err = kv.AtomicDelete(key, pair) 323 | assert.NoError(t, err) 324 | assert.True(t, success) 325 | 326 | // Delete a non-existent key; should fail 327 | success, err = kv.AtomicDelete(key, pair) 328 | assert.Equal(t, err, store.ErrKeyNotFound) 329 | assert.False(t, success) 330 | } 331 | 332 | func testLockUnlock(t *testing.T, kv store.Store) { 333 | key := "testLockUnlock" 334 | value := []byte("bar") 335 | 336 | // We should be able to create a new lock on key 337 | lock, err := kv.NewLock(key, &store.LockOptions{Value: value, TTL: 2 * time.Second}) 338 | assert.NoError(t, err) 339 | assert.NotNil(t, lock) 340 | 341 | // Lock should successfully succeed or block 342 | lockChan, err := lock.Lock(nil) 343 | assert.NoError(t, err) 344 | assert.NotNil(t, lockChan) 345 | 346 | // Get should work 347 | pair, err := kv.Get(key, nil) 348 | assert.NoError(t, err) 349 | checkPairNotNil(t, pair) 350 | assert.Equal(t, pair.Value, value) 351 | assert.NotEqual(t, pair.LastIndex, 0) 352 | 353 | // Unlock should succeed 354 | err = lock.Unlock() 355 | assert.NoError(t, err) 356 | 357 | // Lock should succeed again 358 | lockChan, err = lock.Lock(nil) 359 | assert.NoError(t, err) 360 | assert.NotNil(t, lockChan) 361 | 362 | // Get should work 363 | pair, err = kv.Get(key, nil) 364 | assert.NoError(t, err) 365 | checkPairNotNil(t, pair) 366 | assert.Equal(t, pair.Value, value) 367 | assert.NotEqual(t, pair.LastIndex, 0) 368 | 369 | err = lock.Unlock() 370 | assert.NoError(t, err) 371 | } 372 | 373 | func testLockTTL(t *testing.T, kv store.Store, otherConn store.Store) { 374 | key := "testLockTTL" 375 | value := []byte("bar") 376 | 377 | renewCh := make(chan struct{}) 378 | 379 | // We should be able to create a new lock on key 380 | lock, err := otherConn.NewLock(key, &store.LockOptions{ 381 | Value: value, 382 | TTL: 2 * time.Second, 383 | RenewLock: renewCh, 384 | }) 385 | assert.NoError(t, err) 386 | assert.NotNil(t, lock) 387 | 388 | // Lock should successfully succeed 389 | lockChan, err := lock.Lock(nil) 390 | assert.NoError(t, err) 391 | assert.NotNil(t, lockChan) 392 | 393 | // Get should work 394 | pair, err := otherConn.Get(key, nil) 395 | assert.NoError(t, err) 396 | checkPairNotNil(t, pair) 397 | assert.Equal(t, pair.Value, value) 398 | assert.NotEqual(t, pair.LastIndex, 0) 399 | 400 | time.Sleep(3 * time.Second) 401 | 402 | done := make(chan struct{}) 403 | stop := make(chan struct{}) 404 | 405 | value = []byte("foobar") 406 | 407 | // Create a new lock with another connection 408 | lock, err = kv.NewLock( 409 | key, 410 | &store.LockOptions{ 411 | Value: value, 412 | TTL: 3 * time.Second, 413 | }, 414 | ) 415 | assert.NoError(t, err) 416 | assert.NotNil(t, lock) 417 | 418 | // Lock should block, the session on the lock 419 | // is still active and renewed periodically 420 | go func(<-chan struct{}) { 421 | _, _ = lock.Lock(stop) 422 | done <- struct{}{} 423 | }(done) 424 | 425 | select { 426 | case _ = <-done: 427 | t.Fatal("Lock succeeded on a key that is supposed to be locked by another client") 428 | case <-time.After(4 * time.Second): 429 | // Stop requesting the lock as we are blocked as expected 430 | stop <- struct{}{} 431 | break 432 | } 433 | 434 | // Close the connection 435 | otherConn.Close() 436 | 437 | // Force stop the session renewal for the lock 438 | close(renewCh) 439 | 440 | // Let the session on the lock expire 441 | time.Sleep(3 * time.Second) 442 | locked := make(chan struct{}) 443 | 444 | // Lock should now succeed for the other client 445 | go func(<-chan struct{}) { 446 | lockChan, err = lock.Lock(nil) 447 | assert.NoError(t, err) 448 | assert.NotNil(t, lockChan) 449 | locked <- struct{}{} 450 | }(locked) 451 | 452 | select { 453 | case _ = <-locked: 454 | break 455 | case <-time.After(4 * time.Second): 456 | t.Fatal("Unable to take the lock, timed out") 457 | } 458 | 459 | // Get should work with the new value 460 | pair, err = kv.Get(key, nil) 461 | assert.NoError(t, err) 462 | checkPairNotNil(t, pair) 463 | assert.Equal(t, pair.Value, value) 464 | assert.NotEqual(t, pair.LastIndex, 0) 465 | 466 | err = lock.Unlock() 467 | assert.NoError(t, err) 468 | } 469 | 470 | func testPutTTL(t *testing.T, kv store.Store, otherConn store.Store) { 471 | firstKey := "testPutTTL" 472 | firstValue := []byte("foo") 473 | 474 | secondKey := "second" 475 | secondValue := []byte("bar") 476 | 477 | // Put the first key with the Ephemeral flag 478 | err := otherConn.Put(firstKey, firstValue, &store.WriteOptions{TTL: 2 * time.Second}) 479 | assert.NoError(t, err) 480 | 481 | // Put a second key with the Ephemeral flag 482 | err = otherConn.Put(secondKey, secondValue, &store.WriteOptions{TTL: 2 * time.Second}) 483 | assert.NoError(t, err) 484 | 485 | // Get on firstKey should work 486 | pair, err := kv.Get(firstKey, nil) 487 | assert.NoError(t, err) 488 | checkPairNotNil(t, pair) 489 | 490 | // Get on secondKey should work 491 | pair, err = kv.Get(secondKey, nil) 492 | assert.NoError(t, err) 493 | checkPairNotNil(t, pair) 494 | 495 | // Close the connection 496 | otherConn.Close() 497 | 498 | // Let the session expire 499 | time.Sleep(3 * time.Second) 500 | 501 | // Get on firstKey shouldn't work 502 | pair, err = kv.Get(firstKey, nil) 503 | assert.Error(t, err) 504 | assert.Nil(t, pair) 505 | 506 | // Get on secondKey shouldn't work 507 | pair, err = kv.Get(secondKey, nil) 508 | assert.Error(t, err) 509 | assert.Nil(t, pair) 510 | } 511 | 512 | func testList(t *testing.T, kv store.Store) { 513 | parentKey := "testList" 514 | childKey := "testList/child" 515 | subfolderKey := "testList/subfolder" 516 | 517 | // Put the parent key 518 | err := kv.Put(parentKey, nil, &store.WriteOptions{IsDir: true}) 519 | assert.NoError(t, err) 520 | 521 | // Put the first child key 522 | err = kv.Put(childKey, []byte("first"), nil) 523 | assert.NoError(t, err) 524 | 525 | // Put the second child key which is also a directory 526 | err = kv.Put(subfolderKey, []byte("second"), &store.WriteOptions{IsDir: true}) 527 | assert.NoError(t, err) 528 | 529 | // Put child keys under secondKey 530 | for i := 1; i <= 3; i++ { 531 | key := "testList/subfolder/key" + strconv.Itoa(i) 532 | err := kv.Put(key, []byte("value"), nil) 533 | assert.NoError(t, err) 534 | } 535 | 536 | // List should work and return five child entries 537 | for _, parent := range []string{parentKey, parentKey + "/"} { 538 | pairs, err := kv.List(parent, nil) 539 | assert.NoError(t, err) 540 | if assert.NotNil(t, pairs) { 541 | assert.Equal(t, 5, len(pairs)) 542 | } 543 | } 544 | 545 | // List on childKey should return 0 keys 546 | pairs, err := kv.List(childKey, nil) 547 | assert.NoError(t, err) 548 | if assert.NotNil(t, pairs) { 549 | assert.Equal(t, 0, len(pairs)) 550 | } 551 | 552 | // List on subfolderKey should return 3 keys without the directory 553 | pairs, err = kv.List(subfolderKey, nil) 554 | assert.NoError(t, err) 555 | if assert.NotNil(t, pairs) { 556 | assert.Equal(t, 3, len(pairs)) 557 | } 558 | 559 | // List should fail: the key does not exist 560 | pairs, err = kv.List("idontexist", nil) 561 | assert.Equal(t, store.ErrKeyNotFound, err) 562 | assert.Nil(t, pairs) 563 | } 564 | 565 | func testListLockKey(t *testing.T, kv store.Store) { 566 | listKey := "testListLockSide" 567 | 568 | err := kv.Put(listKey, []byte("val"), &store.WriteOptions{IsDir: true}) 569 | assert.NoError(t, err) 570 | 571 | err = kv.Put(listKey+"/subfolder", []byte("val"), &store.WriteOptions{IsDir: true}) 572 | assert.NoError(t, err) 573 | 574 | // Put keys under subfolder. 575 | for i := 1; i <= 3; i++ { 576 | key := listKey + "/subfolder/key" + strconv.Itoa(i) 577 | err := kv.Put(key, []byte("val"), nil) 578 | assert.NoError(t, err) 579 | 580 | // We lock the child key 581 | lock, err := kv.NewLock(key, &store.LockOptions{Value: []byte("locked"), TTL: 2 * time.Second}) 582 | assert.NoError(t, err) 583 | assert.NotNil(t, lock) 584 | 585 | lockChan, err := lock.Lock(nil) 586 | assert.NoError(t, err) 587 | assert.NotNil(t, lockChan) 588 | } 589 | 590 | // List children of the root directory (`listKey`), this should 591 | // not output any `___lock` entries and must contain 4 results. 592 | pairs, err := kv.List(listKey, nil) 593 | assert.NoError(t, err) 594 | assert.NotNil(t, pairs) 595 | assert.Equal(t, 4, len(pairs)) 596 | 597 | for _, pair := range pairs { 598 | if strings.Contains(string(pair.Key), "___lock") { 599 | assert.FailNow(t, "tesListLockKey: found a key containing lock suffix '___lock'") 600 | } 601 | } 602 | } 603 | 604 | func testDeleteTree(t *testing.T, kv store.Store) { 605 | prefix := "testDeleteTree" 606 | 607 | firstKey := "testDeleteTree/first" 608 | firstValue := []byte("first") 609 | 610 | secondKey := "testDeleteTree/second" 611 | secondValue := []byte("second") 612 | 613 | // Put the first key 614 | err := kv.Put(firstKey, firstValue, nil) 615 | assert.NoError(t, err) 616 | 617 | // Put the second key 618 | err = kv.Put(secondKey, secondValue, nil) 619 | assert.NoError(t, err) 620 | 621 | // Get should work on the first Key 622 | pair, err := kv.Get(firstKey, nil) 623 | assert.NoError(t, err) 624 | checkPairNotNil(t, pair) 625 | assert.Equal(t, pair.Value, firstValue) 626 | assert.NotEqual(t, pair.LastIndex, 0) 627 | 628 | // Get should work on the second Key 629 | pair, err = kv.Get(secondKey, nil) 630 | assert.NoError(t, err) 631 | checkPairNotNil(t, pair) 632 | assert.Equal(t, pair.Value, secondValue) 633 | assert.NotEqual(t, pair.LastIndex, 0) 634 | 635 | // Delete Values under directory `nodes` 636 | err = kv.DeleteTree(prefix) 637 | assert.NoError(t, err) 638 | 639 | // Get should fail on both keys 640 | pair, err = kv.Get(firstKey, nil) 641 | assert.Error(t, err) 642 | assert.Nil(t, pair) 643 | 644 | pair, err = kv.Get(secondKey, nil) 645 | assert.Error(t, err) 646 | assert.Nil(t, pair) 647 | } 648 | 649 | // RunCleanup cleans up keys introduced by the tests 650 | func RunCleanup(t *testing.T, kv store.Store) { 651 | for _, key := range []string{ 652 | "testAtomicPutWithSlashSuffixKey", 653 | "testPutGetDeleteExists", 654 | "testWatch", 655 | "testWatchTree", 656 | "testAtomicPut", 657 | "testAtomicPutCreate", 658 | "testAtomicDelete", 659 | "testLockUnlock", 660 | "testLockTTL", 661 | "testPutTTL", 662 | "testList/subfolder", 663 | "testList", 664 | "testListLockSide/subfolder", 665 | "testListLockSide", 666 | "testDeleteTree", 667 | } { 668 | err := kv.DeleteTree(key) 669 | assert.True(t, err == nil || err == store.ErrKeyNotFound, fmt.Sprintf("failed to delete tree key %s: %v", key, err)) 670 | err = kv.Delete(key) 671 | assert.True(t, err == nil || err == store.ErrKeyNotFound, fmt.Sprintf("failed to delete key %s: %v", key, err)) 672 | } 673 | } 674 | --------------------------------------------------------------------------------