├── .github ├── dependabot.yml └── workflows │ ├── jepsen.yml │ └── test.yml ├── .gitignore ├── .travis.yml ├── Dockerfile-erlang ├── LICENSE ├── Makefile ├── README.md ├── ci ├── jepsen-test.sh ├── provision-jepsen-controller.sh └── ra-jepsen-aws.tf ├── erlang.mk ├── jepsen ├── .gitignore ├── docker │ ├── docker-compose.yml │ ├── provision.sh │ └── shared │ │ ├── init-control.sh │ │ └── init-node.sh └── jepsen.rakvstore │ ├── .gitignore │ ├── .mvn │ └── wrapper │ │ ├── maven-wrapper.jar │ │ └── maven-wrapper.properties │ ├── README.md │ ├── mvnw │ ├── mvnw.cmd │ ├── pom.xml │ ├── project.clj │ └── src │ ├── main │ ├── clojure │ │ └── jepsen │ │ │ ├── rakvstore.clj │ │ │ └── set.clj │ └── java │ │ └── com │ │ └── rabbitmq │ │ └── jepsen │ │ ├── RaNodeDownException.java │ │ ├── RaTimeoutException.java │ │ └── Utils.java │ └── test │ └── java │ └── com │ └── rabbitmq │ └── jepsen │ └── UtilsTest.java ├── rebar.config ├── rel ├── sys.config └── vm.args ├── relx.config ├── src ├── ra_kv_store.erl ├── ra_kv_store_app.erl ├── ra_kv_store_handler.erl └── ra_kv_store_sup.erl └── test ├── http_SUITE.erl ├── replay_SUITE.erl └── store_SUITE.erl /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Please see the documentation for all configuration options: 2 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 3 | version: 2 4 | updates: 5 | - package-ecosystem: "github-actions" 6 | directory: "/" 7 | schedule: 8 | interval: "daily" 9 | target-branch: "main" 10 | - package-ecosystem: "maven" 11 | directory: "/jepsen/jepsen.rakvstore" 12 | schedule: 13 | interval: "daily" 14 | target-branch: "main" 15 | ignore: 16 | - dependency-name: "ch.qos.logback:logback-classic" 17 | versions: [ "[1.3,)" ] 18 | - dependency-name: "org.slf4j:slf4j-api" 19 | versions: [ "[2.0,)" ] 20 | - dependency-name: "org.clojure:clojure" 21 | versions: [ "[1.11,)"] 22 | -------------------------------------------------------------------------------- /.github/workflows/jepsen.yml: -------------------------------------------------------------------------------- 1 | name: Jepsen Tests for RA 2 | 3 | on: 4 | workflow_dispatch: 5 | repository_dispatch: 6 | types: [ra_change] 7 | 8 | concurrency: 9 | group: jepsen 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-24.04 14 | container: 15 | image: pivotalrabbitmq/jepsen 16 | steps: 17 | - uses: actions/checkout@v4 18 | - name: Jepsen tests 19 | run: ci/jepsen-test.sh 20 | env: 21 | AWS_CONFIG: ${{ secrets.AWS_CONFIG }} 22 | AWS_CREDENTIALS: ${{ secrets.AWS_CREDENTIALS }} 23 | - name: Stop cluster 24 | if: always() 25 | working-directory: terraform-state 26 | run: | 27 | terraform init 28 | terraform destroy -auto-approve 29 | aws ec2 delete-key-pair --no-cli-pager --key-name jepsen-ra-kv-store-key 30 | - name: Delete configuration 31 | if: always() 32 | run: | 33 | rm -rf ~/.aws 34 | rm -rf terraform-state 35 | rm -rf terraform.tfstate 36 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | workflow_dispatch: 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-22.04 12 | container: 13 | image: pivotalrabbitmq/jepsen 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Test RA KV store 17 | run: make tests 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .eunit 2 | deps 3 | *.o 4 | *.beam 5 | *.plt 6 | erl_crash.dump 7 | ebin/*.beam 8 | rel/example_project 9 | .concrete/DEV_MODE 10 | .rebar 11 | erl_crash.dump 12 | .sw? 13 | .*.sw? 14 | *.beam 15 | /.erlang.mk/ 16 | /cover/ 17 | /deps/ 18 | /ebin/ 19 | /doc/ 20 | !/ebin/ra.app 21 | /logs/ 22 | /plugins/ 23 | /xrefr 24 | elvis 25 | callgrind* 26 | ct.coverdata 27 | test/ct.cover.spec 28 | _build 29 | 30 | *.plt 31 | *.d 32 | _rel/ 33 | *.iml 34 | 35 | .vagrant 36 | *.tar.gz 37 | 38 | .lsp/* 39 | .clj-kondo/* -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | sudo: required 3 | 4 | language: minimal 5 | 6 | notifications: 7 | email: 8 | - acogoluegnes@pivotal.io 9 | - knilsson@pivotal.io 10 | addons: 11 | apt: 12 | sources: 13 | - sourceline: deb https://packages.erlang-solutions.com/ubuntu xenial contrib 14 | key_url: https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc 15 | packages: 16 | - esl-erlang=1:21.3.8.1-1 17 | branches: 18 | only: 19 | - master 20 | 21 | script: make tests 22 | 23 | -------------------------------------------------------------------------------- /Dockerfile-erlang: -------------------------------------------------------------------------------- 1 | FROM buildpack-deps:bookworm 2 | 3 | ENV OTP_VERSION="26.2.5.6" \ 4 | REBAR3_VERSION="3.24.0" 5 | 6 | LABEL org.opencontainers.image.version=$OTP_VERSION 7 | 8 | # We'll install the build dependencies for erlang-odbc along with the erlang 9 | # build process: 10 | RUN set -xe \ 11 | && OTP_DOWNLOAD_URL="https://github.com/erlang/otp/archive/OTP-${OTP_VERSION}.tar.gz" \ 12 | && OTP_DOWNLOAD_SHA256="371e59b98de59822e45fdbe50c18c8d8dd4c872990e7aaaba8a819e167186d03" \ 13 | && runtimeDeps='libodbc1 \ 14 | libsctp1 \ 15 | libwxgtk3.2-1 \ 16 | libwxgtk-webview3.2-1' \ 17 | && buildDeps='unixodbc-dev \ 18 | libsctp-dev \ 19 | libwxgtk-webview3.2-dev' \ 20 | && apt-get update \ 21 | && apt-get install -y --no-install-recommends $runtimeDeps \ 22 | && apt-get install -y --no-install-recommends $buildDeps \ 23 | && curl -fSL -o otp-src.tar.gz "$OTP_DOWNLOAD_URL" \ 24 | && echo "$OTP_DOWNLOAD_SHA256 otp-src.tar.gz" | sha256sum -c - \ 25 | && export ERL_TOP="/usr/src/otp_src_${OTP_VERSION%%@*}" \ 26 | && mkdir -vp $ERL_TOP \ 27 | && tar -xzf otp-src.tar.gz -C $ERL_TOP --strip-components=1 \ 28 | && rm otp-src.tar.gz \ 29 | && ( cd $ERL_TOP \ 30 | && ./otp_build autoconf \ 31 | && gnuArch="$(dpkg-architecture --query DEB_HOST_GNU_TYPE)" \ 32 | && ./configure --build="$gnuArch" \ 33 | && make -j$(nproc) \ 34 | && make -j$(nproc) docs DOC_TARGETS=chunks \ 35 | && make install install-docs DOC_TARGETS=chunks ) \ 36 | && find /usr/local -name examples | xargs rm -rf \ 37 | && apt-get purge -y --auto-remove $buildDeps \ 38 | && rm -rf $ERL_TOP /var/lib/apt/lists/* 39 | 40 | CMD ["erl"] 41 | 42 | # extra useful tools here: rebar & rebar3 43 | 44 | ENV REBAR_VERSION="2.6.4" 45 | 46 | RUN set -xe \ 47 | && REBAR_DOWNLOAD_URL="https://github.com/rebar/rebar/archive/${REBAR_VERSION}.tar.gz" \ 48 | && REBAR_DOWNLOAD_SHA256="577246bafa2eb2b2c3f1d0c157408650446884555bf87901508ce71d5cc0bd07" \ 49 | && mkdir -p /usr/src/rebar-src \ 50 | && curl -fSL -o rebar-src.tar.gz "$REBAR_DOWNLOAD_URL" \ 51 | && echo "$REBAR_DOWNLOAD_SHA256 rebar-src.tar.gz" | sha256sum -c - \ 52 | && tar -xzf rebar-src.tar.gz -C /usr/src/rebar-src --strip-components=1 \ 53 | && rm rebar-src.tar.gz \ 54 | && cd /usr/src/rebar-src \ 55 | && ./bootstrap \ 56 | && install -v ./rebar /usr/local/bin/ \ 57 | && rm -rf /usr/src/rebar-src 58 | 59 | RUN set -xe \ 60 | && REBAR3_DOWNLOAD_URL="https://github.com/erlang/rebar3/archive/${REBAR3_VERSION}.tar.gz" \ 61 | && REBAR3_DOWNLOAD_SHA256="391b0eaa2825bb427fef1e55a0d166493059175f57a33b00346b84a20398216c" \ 62 | && mkdir -p /usr/src/rebar3-src \ 63 | && curl -fSL -o rebar3-src.tar.gz "$REBAR3_DOWNLOAD_URL" \ 64 | && echo "$REBAR3_DOWNLOAD_SHA256 rebar3-src.tar.gz" | sha256sum -c - \ 65 | && tar -xzf rebar3-src.tar.gz -C /usr/src/rebar3-src --strip-components=1 \ 66 | && rm rebar3-src.tar.gz \ 67 | && cd /usr/src/rebar3-src \ 68 | && HOME=$PWD ./bootstrap \ 69 | && install -v ./rebar3 /usr/local/bin/ \ 70 | && rm -rf /usr/src/rebar3-src 71 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | https://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2023 Broadcom. All Rights Reserved. 190 | The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | https://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT = ra_kv_store 2 | PROJECT_DESCRIPTION = Experimental raft-based key/value store 3 | PROJECT_VERSION = 0.1.0 4 | PROJECT_MOD = ra_kv_store_app 5 | 6 | ERLANG_VERSION_FOR_DOCKER_IMAGE ?= 26.2.5.6 7 | 8 | define PROJECT_ENV 9 | [ 10 | {port, 8080}, 11 | {nodes, [{ra_kv1, 'ra_kv_store@127.0.0.1'}]}, 12 | {server_reference, ra_kv1}, 13 | {release_cursor_every, -1}, 14 | {node_reconnection_interval, 10000}, 15 | {restart_ra_cluster, false} 16 | ] 17 | endef 18 | 19 | dep_ra = git https://github.com/rabbitmq/ra.git main 20 | DEPS = ra cowboy cowlib ranch 21 | dep_cowboy_commit = 2.13.0 22 | dep_cowlib = git https://github.com/ninenines/cowlib 2.14.0 23 | dep_ranch = git https://github.com/ninenines/ranch 2.2.0 24 | 25 | DEP_PLUGINS = cowboy 26 | BUILD_DEPS = relx 27 | 28 | clean-rel: 29 | rm -rf _rel 30 | 31 | clean-deps: 32 | rm -rf deps 33 | 34 | rel-docker: clean-rel clean-deps 35 | docker run -it --rm --name erlang-inst1 -v "$(PWD)":/usr/src/ra_kv_store -w /usr/src/ra_kv_store rabbitmqdevenv/erlang-dev-bookworm make rel 36 | 37 | rel-jepsen: rel-docker 38 | cp _rel/ra_kv_store_release/*.tar.gz jepsen/jepsen.rakvstore/ 39 | 40 | rel-jepsen-local: rel 41 | cp _rel/ra_kv_store_release/*.tar.gz jepsen/jepsen.rakvstore/ 42 | 43 | 44 | .PHONY: erlang-docker-image 45 | erlang-docker-image: ## Build Erlang Docker (for local development) 46 | @docker build \ 47 | --file Dockerfile-erlang \ 48 | --tag rabbitmqdevenv/erlang-dev-bookworm:$(ERLANG_VERSION_FOR_DOCKER_IMAGE) \ 49 | --tag rabbitmqdevenv/erlang-dev-bookworm:latest \ 50 | . 51 | 52 | .PHONY: push-erlang-docker-image 53 | push-erlang-docker-image: erlang-docker-image ## Push Erlang Docker image 54 | @docker push rabbitmqdevenv/erlang-dev-bookworm:$(ERLANG_VERSION_FOR_DOCKER_IMAGE) 55 | @docker push rabbitmqdevenv/erlang-dev-bookworm:latest 56 | 57 | include erlang.mk 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Raft-based Key/Value Store 2 | 3 | This is a tiny key/value store based on [Ra](https://github.com/rabbitmq/ra) developed to be used in certain integration tests. 4 | It is not meant to be a general purpose project. 5 | 6 | # Usage 7 | 8 | Start the HTTP API: 9 | 10 | ``` 11 | $ make run 12 | ``` 13 | 14 | Use [HTTPie](https://httpie.org/) to play with the KV store. Retrieving a value: 15 | ``` 16 | $ http GET http://localhost:8080/1 17 | HTTP/1.1 404 Not Found 18 | content-length: 9 19 | content-type: text/plain 20 | date: Wed, 13 Jun 2018 07:12:15 GMT 21 | server: Cowboy 22 | 23 | undefined 24 | ``` 25 | 26 | Setting a value: 27 | ``` 28 | $ http -f PUT http://localhost:8080/1 value=1 29 | HTTP/1.1 204 No Content 30 | date: Wed, 13 Jun 2018 07:12:28 GMT 31 | server: Cowboy 32 | 33 | ``` 34 | 35 | Getting the value back: 36 | ``` 37 | $ http GET http://localhost:8080/1 38 | HTTP/1.1 200 OK 39 | content-length: 1 40 | content-type: text/plain 41 | date: Wed, 13 Jun 2018 07:12:34 GMT 42 | server: Cowboy 43 | 44 | 1 45 | 46 | ``` 47 | 48 | [Comparing-and-swapping](https://en.wikipedia.org/wiki/Compare-and-swap) a value, success case: 49 | ``` 50 | $ http -f PUT http://localhost:8080/1 value=2 expected=1 51 | HTTP/1.1 204 No Content 52 | date: Wed, 13 Jun 2018 07:13:02 GMT 53 | server: Cowboy 54 | 55 | ``` 56 | 57 | [Comparing-and-swapping](https://en.wikipedia.org/wiki/Compare-and-swap) a value, failure case: 58 | ``` 59 | $ http -f PUT http://localhost:8080/1 value=2 expected=1 60 | HTTP/1.1 409 Conflict 61 | content-length: 1 62 | date: Wed, 13 Jun 2018 07:13:08 GMT 63 | server: Cowboy 64 | 65 | 2 66 | ``` 67 | 68 | # Jepsen test 69 | 70 | See the [readme](jepsen/jepsen.rakvstore/README.md). 71 | 72 | # License 73 | 74 | RA KV Store is [Apache 2.0 licensed](https://www.apache.org/licenses/LICENSE-2.0.html). 75 | 76 | Copyright 2018-2023 Broadcom. All Rights Reserved. 77 | The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 78 | -------------------------------------------------------------------------------- /ci/jepsen-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | set +x 6 | # create SSH key that will be used to connect to the Jepsen VMs 7 | ssh-keygen -t rsa -m pem -f jepsen-bot -C jepsen-bot -N '' 8 | 9 | mkdir -p ~/.aws 10 | echo "$AWS_CONFIG" > ~/.aws/config 11 | echo "$AWS_CREDENTIALS" > ~/.aws/credentials 12 | set -x 13 | 14 | # destroy existing resources in they already exist 15 | set +e 16 | aws ec2 terminate-instances --no-cli-pager --instance-ids $(aws ec2 describe-instances --query 'Reservations[].Instances[].InstanceId' --filters "Name=tag:Name,Values=JepsenRaKvStore" --output text) 17 | aws ec2 delete-key-pair --no-cli-pager --key-name jepsen-ra-kv-store-key 18 | set -e 19 | 20 | # copy Terraform configuration file in current directory 21 | cp ./ci/ra-jepsen-aws.tf . 22 | 23 | # initialize Terraform (get plugins and so) 24 | terraform init 25 | 26 | # spin up the VMs 27 | terraform apply -auto-approve 28 | 29 | mkdir terraform-state 30 | 31 | # save Terraform state and configuration to clean up even if the task fails 32 | cp jepsen-bot terraform-state 33 | cp jepsen-bot.pub terraform-state 34 | cp -r .terraform terraform-state 35 | cp terraform.tfstate terraform-state 36 | cp ra-jepsen-aws.tf terraform-state 37 | 38 | # get the Jepsen controller IP 39 | CONTROLLER_IP=$(terraform output -raw controller_ip) 40 | JEPSEN_USER="admin" 41 | # install dependencies and compile the RA KV store on the Jepsen controller 42 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$CONTROLLER_IP 'bash -s' < ci/provision-jepsen-controller.sh 43 | # makes sure the Jepsen test command line works 44 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$CONTROLLER_IP "source ~/.profile ; cd ~/ra_kv_store/jepsen/jepsen.rakvstore/ ; lein run test --help" 45 | 46 | # add the worker hostnames to /etc/hosts 47 | WORKERS_HOSTS_ENTRIES=$(terraform output -raw workers_hosts_entries) 48 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$CONTROLLER_IP "echo '$WORKERS_HOSTS_ENTRIES' | sudo tee --append /etc/hosts" 49 | 50 | # copy the RA KV store distribution on all the Jepsen workers 51 | WORKERS=( $(terraform output -raw workers_hostname) ) 52 | for worker in "${WORKERS[@]}" 53 | do 54 | ssh -o StrictHostKeyChecking=no -i jepsen-bot \ 55 | $JEPSEN_USER@$CONTROLLER_IP \ 56 | "scp -o StrictHostKeyChecking=no -i ~/jepsen-bot ~/ra_kv_store/jepsen/jepsen.rakvstore/ra_kv_store_release-1.tar.gz $JEPSEN_USER@$worker:/tmp/ra_kv_store_release-1.tar.gz" 57 | done 58 | 59 | # create directory for broker /var directory archiving on all the Jepsen workers 60 | WORKERS_IP=( $(terraform output -raw workers_ip) ) 61 | for worker_ip in "${WORKERS_IP[@]}" 62 | do 63 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$worker_ip "mkdir /tmp/ra-kv-store-var" 64 | done 65 | 66 | # miscellaneous configuration on all the Jepsen workers 67 | for worker_ip in "${WORKERS_IP[@]}" 68 | do 69 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$worker_ip "sudo apt-get update" 70 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$worker_ip "echo '$WORKERS_HOSTS_ENTRIES' | sudo tee --append /etc/hosts" 71 | done 72 | 73 | # build up some fixed parameters for the Jepsen tests 74 | NODES="" 75 | for worker in "${WORKERS[@]}" 76 | do 77 | NODES="$NODES --node $worker" 78 | done 79 | 80 | SOURCE_AND_CD="source ~/.profile ; cd ~/ra_kv_store/jepsen/jepsen.rakvstore/" 81 | CREDENTIALS="--username $JEPSEN_USER --ssh-private-key ~/jepsen-bot" 82 | 83 | JEPSEN_TESTS_PARAMETERS=( 84 | "--workload set --nemesis random-partition-halves --time-limit 180 --concurrency 10 --rate 50 --erlang-net-ticktime 7 --disruption-duration 25" 85 | "--workload set --nemesis partition-halves --time-limit 180 --concurrency 10 --rate 50 --erlang-net-ticktime 7 --disruption-duration 25" 86 | "--workload set --nemesis partition-majorities-ring --time-limit 180 --concurrency 10 --rate 50 --erlang-net-ticktime 7 --disruption-duration 25" 87 | "--workload set --nemesis partition-random-node --time-limit 180 --concurrency 10 --rate 50 --erlang-net-ticktime 7 --disruption-duration 25" 88 | "--workload set --nemesis kill-erlang-process --time-limit 120 --concurrency 10 --rate 50 --time-before-disruption 3 --disruption-duration 3 --release-cursor-every 10 --random-nodes 2" 89 | "--workload set --nemesis kill-erlang-vm --time-limit 120 --concurrency 10 --rate 50 --time-before-disruption 10 --disruption-duration 3 --release-cursor-every 10 --random-nodes 2" 90 | "--workload register --nemesis random-partition-halves --time-limit 180 --concurrency 10 --rate 40 --ops-per-key 25 --time-before-disruption 15 --disruption-duration 25 --erlang-net-ticktime 10" 91 | "--workload register --nemesis kill-erlang-vm --time-limit 120 --concurrency 10 --rate 30 --ops-per-key 10 --time-before-disruption 5 --disruption-duration 20 --erlang-net-ticktime 6 --release-cursor-every 10 --wal-max-size-bytes 524288 --random-nodes 2" 92 | "--workload register --nemesis kill-erlang-process --time-limit 180 --concurrency 10 --rate 40 --ops-per-key 25 --time-before-disruption 3 --disruption-duration 3 --erlang-net-ticktime 10 --release-cursor-every 10 --wal-max-size-bytes 524288 --random-nodes 2" 93 | "--workload register --nemesis combined --time-limit 120 --concurrency 10 --rate 40 --ops-per-key 10 --time-before-disruption 15 --disruption-duration 20 --erlang-net-ticktime 6 --release-cursor-every 10 --wal-max-size-bytes 524288 --random-nodes 2" 94 | "--workload set --nemesis combined --time-limit 240 --concurrency 20 --rate 40 --time-before-disruption 25 --disruption-duration 30 --erlang-net-ticktime 10 --release-cursor-every 10 --wal-max-size-bytes 524288 --random-nodes 1" 95 | ) 96 | 97 | TESTS_COUNT=${#JEPSEN_TESTS_PARAMETERS[@]} 98 | TEST_INDEX=1 99 | 100 | failure=false 101 | 102 | set +e 103 | 104 | for jepsen_test_parameter in "${JEPSEN_TESTS_PARAMETERS[@]}" 105 | do 106 | n=1 107 | until [ $n -ge 5 ] 108 | do 109 | echo "Running Jepsen test $TEST_INDEX / $TESTS_COUNT, attempt $n ($(date))" 110 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$CONTROLLER_IP "$SOURCE_AND_CD ; lein run test $NODES $CREDENTIALS $jepsen_test_parameter --erlang-distribution-url file:///tmp/ra_kv_store_release-1.tar.gz" >/dev/null 111 | run_exit_code=$? 112 | for worker_ip in "${WORKERS_IP[@]}" 113 | do 114 | SAVE_VAR_DIRECTORY="/tmp/ra-kv-store-var/test-$TEST_INDEX-attempt-$n" 115 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$worker_ip \ 116 | "mkdir $SAVE_VAR_DIRECTORY ; cp -R /tmp/ra_kv_store/* $SAVE_VAR_DIRECTORY" 117 | done 118 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$CONTROLLER_IP "$SOURCE_AND_CD ; head -n 50 store/current/jepsen.log" 119 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$CONTROLLER_IP "$SOURCE_AND_CD ; tail -n 50 store/current/jepsen.log" 120 | 121 | if [ $run_exit_code -eq 0 ]; then 122 | # run returned 0, but checking the logs for some corner cases 123 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$CONTROLLER_IP "$SOURCE_AND_CD ; grep -q 'Set was never read' ./store/latest/jepsen.log" 124 | if [ $? -eq 0 ]; then 125 | # Could not read the final data structure, see if we can retry 126 | if [ $n -ge 4 ]; then 127 | # It was the last attempt 128 | echo "Test $TEST_INDEX / $TESTS_COUNT failed several times with unexpected errors or inappropriate results, moving on" 129 | # We mark this run as failed 130 | failure=true 131 | break 132 | else 133 | echo "Final data structure could not be read, retrying" 134 | fi 135 | else 136 | # run succeeded, moving on 137 | break 138 | fi 139 | else 140 | echo "Test has failed, checking whether it is an unexpected error or not" 141 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$CONTROLLER_IP "$SOURCE_AND_CD ; grep -q 'Analysis invalid' ./store/latest/jepsen.log" 142 | if [ $? -eq 0 ]; then 143 | echo "Test $TEST_INDEX / $TESTS_COUNT failed, moving on" 144 | failure=true 145 | break 146 | else 147 | if [ $n -ge 4 ]; then 148 | # It was the last attempt 149 | echo "Test $TEST_INDEX / $TESTS_COUNT failed several times with unexpected errors, moving on" 150 | # We mark this run as failed 151 | failure=true 152 | break 153 | fi 154 | echo "Unexpected error, retrying" 155 | fi 156 | fi 157 | n=$[$n+1] 158 | done 159 | ((TEST_INDEX++)) 160 | done 161 | 162 | the_date=$(date '+%Y%m%d-%H%M%S') 163 | archive_name="ra-jepsen-$the_date-jepsen-logs" 164 | archive_file="$archive_name.tar.gz" 165 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$CONTROLLER_IP "$SOURCE_AND_CD ; tar -zcf - store --transform='s/^store/${archive_name}/'" > $archive_file 166 | aws s3 cp $archive_file s3://jepsen-tests-logs/ --quiet 167 | 168 | WORKER_INDEX=0 169 | for worker_ip in "${WORKERS_IP[@]}" 170 | do 171 | var_archive_name="ra-jepsen-$the_date-var-node-$WORKER_INDEX" 172 | var_archive_file="$var_archive_name.tar.gz" 173 | ssh -o StrictHostKeyChecking=no -i jepsen-bot $JEPSEN_USER@$worker_ip \ 174 | "cd /tmp ; tar -zcf - ra-kv-store-var --transform='s/^ra-kv-store-var/${var_archive_name}/'" > $var_archive_file 175 | aws s3 cp $var_archive_file s3://jepsen-tests-logs/ --quiet 176 | ((WORKER_INDEX++)) 177 | done 178 | 179 | echo "Download logs: aws s3 cp s3://jepsen-tests-logs/ . --recursive --exclude '*' --include 'ra-jepsen-$the_date*'" 180 | 181 | if [ "$failure" = true ]; then 182 | exit 1 183 | else 184 | exit 0 185 | fi 186 | -------------------------------------------------------------------------------- /ci/provision-jepsen-controller.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | # script to provision and configure the Jepsen controller 6 | # this controller will compile and package the RA KV store 7 | # and then will run the Jepsen tests 8 | 9 | sudo apt-get update 10 | 11 | sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -V --fix-missing --no-install-recommends \ 12 | apt-transport-https \ 13 | wget \ 14 | ca-certificates \ 15 | gnupg \ 16 | curl 17 | 18 | echo "deb https://ppa1.novemberain.com/rabbitmq/rabbitmq-erlang/deb/debian bookworm main" | sudo tee --append /etc/apt/sources.list.d/rabbitmq-erlang.list 19 | echo "deb https://ppa2.novemberain.com/rabbitmq/rabbitmq-erlang/deb/debian bookworm main" | sudo tee --append /etc/apt/sources.list.d/rabbitmq-erlang.list 20 | wget https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-erlang.E495BB49CC4BBE5B.key 21 | sudo apt-key add cloudsmith.rabbitmq-erlang.E495BB49CC4BBE5B.key 22 | 23 | export ERLANG_VERSION="1:26*" 24 | sudo mkdir -p /etc/apt/preferences.d/ 25 | sudo tee --append /etc/apt/preferences.d/erlang <> ~/.ssh/config 63 | 64 | set +e 65 | # get, compile, and package the RA KV store 66 | git clone https://github.com/rabbitmq/ra-kv-store.git ~/ra_kv_store 67 | make -C ~/ra_kv_store rel-jepsen-local 68 | git --no-pager -C ~/ra_kv_store/deps/ra log -1 --pretty=short 69 | set -e 70 | -------------------------------------------------------------------------------- /ci/ra-jepsen-aws.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | profile = "default" 3 | } 4 | 5 | data "aws_ami" "debian" { 6 | most_recent = true 7 | owners = ["136693071363"] # Debian 8 | 9 | filter { 10 | name = "name" 11 | values = ["debian-12-amd64-*"] 12 | } 13 | 14 | filter { 15 | name = "virtualization-type" 16 | values = ["hvm"] 17 | } 18 | } 19 | 20 | resource "aws_key_pair" "ssh_key" { 21 | key_name = "jepsen-ra-kv-store-key" 22 | public_key = file("jepsen-bot.pub") 23 | } 24 | 25 | resource "aws_instance" "jepsen_controller" { 26 | ami = data.aws_ami.debian.id 27 | instance_type = "m7i.xlarge" 28 | associate_public_ip_address = true 29 | 30 | key_name = aws_key_pair.ssh_key.key_name 31 | security_groups = ["jepsen"] 32 | tags = { 33 | Name = "JepsenRaKvStore" 34 | } 35 | 36 | private_dns_name_options { 37 | enable_resource_name_dns_a_record = true 38 | hostname_type = "ip-name" 39 | } 40 | 41 | provisioner "file" { 42 | source = "jepsen-bot" 43 | destination = "/home/admin/jepsen-bot" 44 | connection { 45 | type = "ssh" 46 | user = "admin" 47 | private_key = file("jepsen-bot") 48 | host = aws_instance.jepsen_controller.public_ip 49 | } 50 | } 51 | 52 | 53 | } 54 | 55 | resource "aws_instance" "jepsen_worker" { 56 | count = 5 57 | ami = data.aws_ami.debian.id 58 | instance_type = "t3.small" 59 | associate_public_ip_address = true 60 | 61 | key_name = aws_key_pair.ssh_key.key_name 62 | security_groups = ["jepsen"] 63 | tags = { 64 | Name = "JepsenRaKvStore" 65 | } 66 | 67 | private_dns_name_options { 68 | enable_resource_name_dns_a_record = true 69 | hostname_type = "ip-name" 70 | } 71 | 72 | } 73 | 74 | output "controller_ip" { 75 | value = aws_instance.jepsen_controller.public_ip 76 | } 77 | 78 | output "workers_ip" { 79 | value = join(" ", aws_instance.jepsen_worker.*.public_ip) 80 | } 81 | 82 | output "workers_hostname" { 83 | value = join(" ", [for worker in aws_instance.jepsen_worker : split(".", worker.private_dns)[0]]) 84 | } 85 | 86 | output "workers_hosts_entries" { 87 | value = join("\n", [for worker in aws_instance.jepsen_worker : join(" ", [worker.private_ip, split(".", worker.private_dns)[0]])]) 88 | } 89 | -------------------------------------------------------------------------------- /jepsen/.gitignore: -------------------------------------------------------------------------------- 1 | .cake 2 | .lein-repl-history 3 | pom.xml 4 | pom.xml.asc 5 | repl-port 6 | *.jar 7 | *.tar 8 | *.tar.bz2 9 | *.war 10 | *.deb 11 | *~ 12 | .*.swp 13 | *.log 14 | log.txt 15 | lib 16 | classes 17 | build 18 | .lein-deps-sum 19 | .lein-failures 20 | /site 21 | site/** 22 | bench/** 23 | */target/** 24 | */checkouts/** 25 | */store/** 26 | */report/** 27 | */timeline.html 28 | */resources/datomic/download-key 29 | */resources/datomic/riak-transactor.properties 30 | /.nrepl-port 31 | .idea 32 | jepsen/doc 33 | -------------------------------------------------------------------------------- /jepsen/docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | x-node-common: &node-common 2 | networks: 3 | - jepsen 4 | image: debian:bookworm 5 | tty: true 6 | volumes: 7 | - ../jepsen.rakvstore:/root/jepsen.rakvstore 8 | - ./shared:/root/shared 9 | cap_add: 10 | - NET_ADMIN 11 | 12 | services: 13 | control: 14 | networks: 15 | - jepsen 16 | hostname: control 17 | container_name: jepsen-control 18 | image: debian:bookworm 19 | tty: true 20 | volumes: 21 | - ../jepsen.rakvstore:/root/jepsen.rakvstore 22 | - ./shared:/root/shared 23 | n1: 24 | <<: *node-common 25 | hostname: n1 26 | container_name: jepsen-n1 27 | n2: 28 | <<: *node-common 29 | hostname: n2 30 | container_name: jepsen-n2 31 | n3: 32 | <<: *node-common 33 | hostname: n3 34 | container_name: jepsen-n3 35 | 36 | 37 | networks: 38 | jepsen: 39 | -------------------------------------------------------------------------------- /jepsen/docker/provision.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker compose exec --no-TTY control /root/shared/init-control.sh & 4 | docker compose exec --no-TTY n1 /root/shared/init-node.sh & 5 | docker compose exec --no-TTY n2 /root/shared/init-node.sh & 6 | docker compose exec --no-TTY n3 /root/shared/init-node.sh & 7 | 8 | wait 9 | -------------------------------------------------------------------------------- /jepsen/docker/shared/init-control.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | apt update 4 | apt-get install -y -V --fix-missing --no-install-recommends \ 5 | apt-transport-https \ 6 | wget \ 7 | ca-certificates \ 8 | gnupg \ 9 | curl 10 | 11 | # Jepsen dependencies 12 | apt install -y -V --fix-missing --no-install-recommends \ 13 | libjna-java gnuplot graphviz openssh-client git 14 | 15 | # Java 16 | export JAVA_PATH="/usr/lib/jdk-21" 17 | JAVA_URL="https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.5%2B11/OpenJDK21U-jdk_x64_linux_hotspot_21.0.5_11.tar.gz" 18 | wget --progress dot:giga --output-document "$JAVA_PATH.tar.gz" $JAVA_URL 19 | 20 | mkdir -p $JAVA_PATH 21 | tar --extract --file "$JAVA_PATH.tar.gz" --directory "$JAVA_PATH" --strip-components 1 22 | rm "$JAVA_PATH.tar.gz" 23 | ln -s "$JAVA_PATH/bin/java" /usr/bin/java 24 | 25 | # Leiningen 26 | wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein 27 | mv lein /usr/bin/lein 28 | chmod u+x /usr/bin/lein 29 | lein -v 30 | -------------------------------------------------------------------------------- /jepsen/docker/shared/init-node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | apt update 4 | apt install -y -V --fix-missing --no-install-recommends \ 5 | apt-transport-https \ 6 | wget \ 7 | ca-certificates \ 8 | gnupg \ 9 | curl 10 | 11 | apt install -y openssh-server sudo 12 | /etc/init.d/ssh start 13 | 14 | mkdir -p ~/.ssh/ 15 | cat /root/shared/jepsen-bot.pub > ~/.ssh/authorized_keys 16 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | 13 | .lsp/ 14 | .clj-kondo/ -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/ra-kv-store/72e1691b45d6ec09bc117c6ceab7e82209501c82/jepsen/jepsen.rakvstore/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip 18 | wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar 19 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/README.md: -------------------------------------------------------------------------------- 1 | # Jepsen Test for Raft-based Key/Value Store 2 | 3 | This work is based on [Jepsen tutorial](https://github.com/jepsen-io/jepsen/blob/master/doc/tutorial/index.md). 4 | 5 | ## Building the Erlang release 6 | 7 | You must be in the project **root directory** to build the Erlang release. The Erlang 8 | release needs to be built on Linux-based system, as this is where it executes in Jepsen tests. 9 | 10 | The easiest is to use a one-time Docker container to compile the release 11 | with the appropriate configuration: 12 | 13 | ```shell 14 | git clone https://github.com/rabbitmq/ra-kv-store.git ra_kv_store 15 | cd ra_kv_store 16 | make rel-jepsen 17 | ``` 18 | 19 | Then copy the release file in the Jepsen test directory: 20 | 21 | ```shell 22 | cp _rel/ra_kv_store_release/ra_kv_store_release-1.tar.gz jepsen/jepsen.rakvstore 23 | ``` 24 | 25 | ## Running Jepsen tests with Docker 26 | 27 | **Make sure the Erlang release file `ra_kv_store_release-1.tar.gz` is in the 28 | `jepsen/jepsen.rakvstore/` directory.** 29 | 30 | `cd` into the `jepsen/docker` directory and start the containers: 31 | 32 | ```shell 33 | cd jepsen/docker 34 | ssh-keygen -t rsa -m pem -f shared/jepsen-bot -C jepsen-bot -N '' 35 | docker compose up --detach 36 | ./provision.sh 37 | ``` 38 | 39 | Connect to the Jepsen control container: 40 | 41 | ```shell 42 | docker exec -it jepsen-control bash 43 | ``` 44 | 45 | Inside the control container, `cd` into the RA KV Store test directory and launch a test: 46 | 47 | ```shell 48 | cd /root/jepsen.rakvstore 49 | lein run test --nodes n1,n2,n3 --ssh-private-key /root/shared/jepsen-bot --time-limit 15 --concurrency 10 --rate 10 --workload set --nemesis random-partition-halves 50 | ``` 51 | 52 | The execution should finish with something like the following: 53 | 54 | ``` 55 | INFO [2024-10-21 12:38:19,023] jepsen test runner - jepsen.core {:perf {:latency-graph {:valid? true}, 56 | :rate-graph {:valid? true}, 57 | :valid? true}, 58 | :workload {:ok-count 144, 59 | :valid? true, 60 | :lost-count 0, 61 | :lost "#{}", 62 | :acknowledged-count 144, 63 | :recovered "#{}", 64 | :ok "#{0..53 55..56 58..59 61..146}", 65 | :attempt-count 147, 66 | :unexpected "#{}", 67 | :unexpected-count 0, 68 | :recovered-count 0}, 69 | :valid? true} 70 | 71 | 72 | Everything looks good! ヽ(‘ー`)ノ 73 | ``` 74 | 75 | Here is how to shut down and delete the containers: 76 | 77 | ```shell 78 | docker compose down 79 | ``` 80 | 81 | You can list all the options of the tests: 82 | ``` 83 | $ lein run test --help 84 | ``` 85 | 86 | Most common options to use: 87 | * `--time-limit`: how long the test should run for (in seconds) 88 | * `--concurrency`: number of workers to run 89 | * `--rate`: number of requests per thread 90 | * `--workload`: the type of test, can be `register` or `set` 91 | * `--nemesis`: the type of nemesis, can be `random-partition-halves`, 92 | `partition-halves`, `partition-majorities-ring`, `random-node` 93 | `kill-erlang-vm`, `kill-erlang-process`, or `combined` (see below for the `combined` nemesis) 94 | * `--network-partition-nemesis`: the type of network partition nemesis to use with the 95 | `combined` nemesis. Can be `random-partition-halves`, `partition-halves`, 96 | `partition-majorities-ring` or `random-node`. Default is `random-partition-halves`. 97 | * `--erlang-net-ticktime`: Erlang net tick time (in seconds) 98 | * `--release-cursor-every`: release RA cursor every n operations 99 | * `--wal-max-size-bytes`: maximum size of RA Write Ahead Log, default is 134217728 (128 MB) 100 | * `--random-nodes`: number of nodes disrupted by Erlang VM and Erlang process killing nemesises 101 | * `--time-before-disruption`: time before the nemesis kicks in (in seconds) 102 | * `--disruption-duration`: duration of disruption (in seconds) 103 | 104 | The `register` workload tests the RA KV Store cluster as a set of registers, identified 105 | by a key. The supported operations are read, write, and compare-and-set. The `set` workload 106 | tests the RA KV Store cluster as a set (a collection of distinct elements stored in the 107 | same key). The `register` workload is more general but also more "expensive" (understand 108 | more resource-consuming when it comes to find bugs) than the `set` test. 109 | 110 | The `combined` nemesis creates a network partition and randomly kills the Erlang VM or Erlang 111 | processes on random nodes (`--random-nodes` option) during the partition. The network partition 112 | strategy can be chosen with the `--network-partition-nemesis` option when using the `combined` nemesis, 113 | the default being `random-partition-halves`. 114 | 115 | ## Client 116 | 117 | Jepsen tests access the KV store through a Java-based client. This client handles HTTP connections, requests, 118 | and responses, as well appropriate logic for the tests (error handling, value swapping). The client is 119 | compiled as part of the Jepsen test. The client and its corresponding unit tests can be modified and the 120 | changes can be checked by running Maven: 121 | 122 | ``` 123 | ./mvnw clean test 124 | ``` 125 | 126 | ## License 127 | 128 | RA KV Store is [Apache 2.0 licensed](https://www.apache.org/licenses/LICENSE-2.0.html). 129 | 130 | Copyright 2018-2024 Broadcom. All Rights Reserved. 131 | The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 132 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Apache Maven Wrapper startup batch script, version 3.2.0 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | # e.g. to debug Maven itself, use 32 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | # ---------------------------------------------------------------------------- 35 | 36 | if [ -z "$MAVEN_SKIP_RC" ] ; then 37 | 38 | if [ -f /usr/local/etc/mavenrc ] ; then 39 | . /usr/local/etc/mavenrc 40 | fi 41 | 42 | if [ -f /etc/mavenrc ] ; then 43 | . /etc/mavenrc 44 | fi 45 | 46 | if [ -f "$HOME/.mavenrc" ] ; then 47 | . "$HOME/.mavenrc" 48 | fi 49 | 50 | fi 51 | 52 | # OS specific support. $var _must_ be set to either true or false. 53 | cygwin=false; 54 | darwin=false; 55 | mingw=false 56 | case "$(uname)" in 57 | CYGWIN*) cygwin=true ;; 58 | MINGW*) mingw=true;; 59 | Darwin*) darwin=true 60 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 61 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 62 | if [ -z "$JAVA_HOME" ]; then 63 | if [ -x "/usr/libexec/java_home" ]; then 64 | JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME 65 | else 66 | JAVA_HOME="/Library/Java/Home"; export JAVA_HOME 67 | fi 68 | fi 69 | ;; 70 | esac 71 | 72 | if [ -z "$JAVA_HOME" ] ; then 73 | if [ -r /etc/gentoo-release ] ; then 74 | JAVA_HOME=$(java-config --jre-home) 75 | fi 76 | fi 77 | 78 | # For Cygwin, ensure paths are in UNIX format before anything is touched 79 | if $cygwin ; then 80 | [ -n "$JAVA_HOME" ] && 81 | JAVA_HOME=$(cygpath --unix "$JAVA_HOME") 82 | [ -n "$CLASSPATH" ] && 83 | CLASSPATH=$(cygpath --path --unix "$CLASSPATH") 84 | fi 85 | 86 | # For Mingw, ensure paths are in UNIX format before anything is touched 87 | if $mingw ; then 88 | [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && 89 | JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" 90 | fi 91 | 92 | if [ -z "$JAVA_HOME" ]; then 93 | javaExecutable="$(which javac)" 94 | if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then 95 | # readlink(1) is not available as standard on Solaris 10. 96 | readLink=$(which readlink) 97 | if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then 98 | if $darwin ; then 99 | javaHome="$(dirname "\"$javaExecutable\"")" 100 | javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" 101 | else 102 | javaExecutable="$(readlink -f "\"$javaExecutable\"")" 103 | fi 104 | javaHome="$(dirname "\"$javaExecutable\"")" 105 | javaHome=$(expr "$javaHome" : '\(.*\)/bin') 106 | JAVA_HOME="$javaHome" 107 | export JAVA_HOME 108 | fi 109 | fi 110 | fi 111 | 112 | if [ -z "$JAVACMD" ] ; then 113 | if [ -n "$JAVA_HOME" ] ; then 114 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 115 | # IBM's JDK on AIX uses strange locations for the executables 116 | JAVACMD="$JAVA_HOME/jre/sh/java" 117 | else 118 | JAVACMD="$JAVA_HOME/bin/java" 119 | fi 120 | else 121 | JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" 122 | fi 123 | fi 124 | 125 | if [ ! -x "$JAVACMD" ] ; then 126 | echo "Error: JAVA_HOME is not defined correctly." >&2 127 | echo " We cannot execute $JAVACMD" >&2 128 | exit 1 129 | fi 130 | 131 | if [ -z "$JAVA_HOME" ] ; then 132 | echo "Warning: JAVA_HOME environment variable is not set." 133 | fi 134 | 135 | # traverses directory structure from process work directory to filesystem root 136 | # first directory with .mvn subdirectory is considered project base directory 137 | find_maven_basedir() { 138 | if [ -z "$1" ] 139 | then 140 | echo "Path not specified to find_maven_basedir" 141 | return 1 142 | fi 143 | 144 | basedir="$1" 145 | wdir="$1" 146 | while [ "$wdir" != '/' ] ; do 147 | if [ -d "$wdir"/.mvn ] ; then 148 | basedir=$wdir 149 | break 150 | fi 151 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 152 | if [ -d "${wdir}" ]; then 153 | wdir=$(cd "$wdir/.." || exit 1; pwd) 154 | fi 155 | # end of workaround 156 | done 157 | printf '%s' "$(cd "$basedir" || exit 1; pwd)" 158 | } 159 | 160 | # concatenates all lines of a file 161 | concat_lines() { 162 | if [ -f "$1" ]; then 163 | # Remove \r in case we run on Windows within Git Bash 164 | # and check out the repository with auto CRLF management 165 | # enabled. Otherwise, we may read lines that are delimited with 166 | # \r\n and produce $'-Xarg\r' rather than -Xarg due to word 167 | # splitting rules. 168 | tr -s '\r\n' ' ' < "$1" 169 | fi 170 | } 171 | 172 | log() { 173 | if [ "$MVNW_VERBOSE" = true ]; then 174 | printf '%s\n' "$1" 175 | fi 176 | } 177 | 178 | BASE_DIR=$(find_maven_basedir "$(dirname "$0")") 179 | if [ -z "$BASE_DIR" ]; then 180 | exit 1; 181 | fi 182 | 183 | MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR 184 | log "$MAVEN_PROJECTBASEDIR" 185 | 186 | ########################################################################################## 187 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 188 | # This allows using the maven wrapper in projects that prohibit checking in binary data. 189 | ########################################################################################## 190 | wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" 191 | if [ -r "$wrapperJarPath" ]; then 192 | log "Found $wrapperJarPath" 193 | else 194 | log "Couldn't find $wrapperJarPath, downloading it ..." 195 | 196 | if [ -n "$MVNW_REPOURL" ]; then 197 | wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" 198 | else 199 | wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" 200 | fi 201 | while IFS="=" read -r key value; do 202 | # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) 203 | safeValue=$(echo "$value" | tr -d '\r') 204 | case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; 205 | esac 206 | done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" 207 | log "Downloading from: $wrapperUrl" 208 | 209 | if $cygwin; then 210 | wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") 211 | fi 212 | 213 | if command -v wget > /dev/null; then 214 | log "Found wget ... using wget" 215 | [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" 216 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 217 | wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" 218 | else 219 | wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" 220 | fi 221 | elif command -v curl > /dev/null; then 222 | log "Found curl ... using curl" 223 | [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" 224 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 225 | curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" 226 | else 227 | curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" 228 | fi 229 | else 230 | log "Falling back to using Java to download" 231 | javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" 232 | javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" 233 | # For Cygwin, switch paths to Windows format before running javac 234 | if $cygwin; then 235 | javaSource=$(cygpath --path --windows "$javaSource") 236 | javaClass=$(cygpath --path --windows "$javaClass") 237 | fi 238 | if [ -e "$javaSource" ]; then 239 | if [ ! -e "$javaClass" ]; then 240 | log " - Compiling MavenWrapperDownloader.java ..." 241 | ("$JAVA_HOME/bin/javac" "$javaSource") 242 | fi 243 | if [ -e "$javaClass" ]; then 244 | log " - Running MavenWrapperDownloader.java ..." 245 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" 246 | fi 247 | fi 248 | fi 249 | fi 250 | ########################################################################################## 251 | # End of extension 252 | ########################################################################################## 253 | 254 | # If specified, validate the SHA-256 sum of the Maven wrapper jar file 255 | wrapperSha256Sum="" 256 | while IFS="=" read -r key value; do 257 | case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; 258 | esac 259 | done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" 260 | if [ -n "$wrapperSha256Sum" ]; then 261 | wrapperSha256Result=false 262 | if command -v sha256sum > /dev/null; then 263 | if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then 264 | wrapperSha256Result=true 265 | fi 266 | elif command -v shasum > /dev/null; then 267 | if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then 268 | wrapperSha256Result=true 269 | fi 270 | else 271 | echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." 272 | echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." 273 | exit 1 274 | fi 275 | if [ $wrapperSha256Result = false ]; then 276 | echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 277 | echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 278 | echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 279 | exit 1 280 | fi 281 | fi 282 | 283 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 284 | 285 | # For Cygwin, switch paths to Windows format before running java 286 | if $cygwin; then 287 | [ -n "$JAVA_HOME" ] && 288 | JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") 289 | [ -n "$CLASSPATH" ] && 290 | CLASSPATH=$(cygpath --path --windows "$CLASSPATH") 291 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 292 | MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") 293 | fi 294 | 295 | # Provide a "standardized" way to retrieve the CLI args that will 296 | # work with both Windows and non-Windows executions. 297 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" 298 | export MAVEN_CMD_LINE_ARGS 299 | 300 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 301 | 302 | # shellcheck disable=SC2086 # safe args 303 | exec "$JAVACMD" \ 304 | $MAVEN_OPTS \ 305 | $MAVEN_DEBUG_OPTS \ 306 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 307 | "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 308 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 309 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Apache Maven Wrapper startup batch script, version 3.2.0 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 28 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending 29 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 30 | @REM e.g. to debug Maven itself, use 31 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 32 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 33 | @REM ---------------------------------------------------------------------------- 34 | 35 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 36 | @echo off 37 | @REM set title of command window 38 | title %0 39 | @REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' 40 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 41 | 42 | @REM set %HOME% to equivalent of $HOME 43 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 44 | 45 | @REM Execute a user defined script before this one 46 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 47 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 48 | if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* 49 | if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* 50 | :skipRcPre 51 | 52 | @setlocal 53 | 54 | set ERROR_CODE=0 55 | 56 | @REM To isolate internal variables from possible post scripts, we use another setlocal 57 | @setlocal 58 | 59 | @REM ==== START VALIDATION ==== 60 | if not "%JAVA_HOME%" == "" goto OkJHome 61 | 62 | echo. 63 | echo Error: JAVA_HOME not found in your environment. >&2 64 | echo Please set the JAVA_HOME variable in your environment to match the >&2 65 | echo location of your Java installation. >&2 66 | echo. 67 | goto error 68 | 69 | :OkJHome 70 | if exist "%JAVA_HOME%\bin\java.exe" goto init 71 | 72 | echo. 73 | echo Error: JAVA_HOME is set to an invalid directory. >&2 74 | echo JAVA_HOME = "%JAVA_HOME%" >&2 75 | echo Please set the JAVA_HOME variable in your environment to match the >&2 76 | echo location of your Java installation. >&2 77 | echo. 78 | goto error 79 | 80 | @REM ==== END VALIDATION ==== 81 | 82 | :init 83 | 84 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 85 | @REM Fallback to current working directory if not found. 86 | 87 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 88 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 89 | 90 | set EXEC_DIR=%CD% 91 | set WDIR=%EXEC_DIR% 92 | :findBaseDir 93 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 94 | cd .. 95 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 96 | set WDIR=%CD% 97 | goto findBaseDir 98 | 99 | :baseDirFound 100 | set MAVEN_PROJECTBASEDIR=%WDIR% 101 | cd "%EXEC_DIR%" 102 | goto endDetectBaseDir 103 | 104 | :baseDirNotFound 105 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 106 | cd "%EXEC_DIR%" 107 | 108 | :endDetectBaseDir 109 | 110 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 111 | 112 | @setlocal EnableExtensions EnableDelayedExpansion 113 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 114 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 115 | 116 | :endReadAdditionalConfig 117 | 118 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 119 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 120 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 121 | 122 | set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" 123 | 124 | FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( 125 | IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B 126 | ) 127 | 128 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 129 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 130 | if exist %WRAPPER_JAR% ( 131 | if "%MVNW_VERBOSE%" == "true" ( 132 | echo Found %WRAPPER_JAR% 133 | ) 134 | ) else ( 135 | if not "%MVNW_REPOURL%" == "" ( 136 | SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" 137 | ) 138 | if "%MVNW_VERBOSE%" == "true" ( 139 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 140 | echo Downloading from: %WRAPPER_URL% 141 | ) 142 | 143 | powershell -Command "&{"^ 144 | "$webclient = new-object System.Net.WebClient;"^ 145 | "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ 146 | "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ 147 | "}"^ 148 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^ 149 | "}" 150 | if "%MVNW_VERBOSE%" == "true" ( 151 | echo Finished downloading %WRAPPER_JAR% 152 | ) 153 | ) 154 | @REM End of extension 155 | 156 | @REM If specified, validate the SHA-256 sum of the Maven wrapper jar file 157 | SET WRAPPER_SHA_256_SUM="" 158 | FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( 159 | IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B 160 | ) 161 | IF NOT %WRAPPER_SHA_256_SUM%=="" ( 162 | powershell -Command "&{"^ 163 | "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^ 164 | "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^ 165 | " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^ 166 | " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^ 167 | " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^ 168 | " exit 1;"^ 169 | "}"^ 170 | "}" 171 | if ERRORLEVEL 1 goto error 172 | ) 173 | 174 | @REM Provide a "standardized" way to retrieve the CLI args that will 175 | @REM work with both Windows and non-Windows executions. 176 | set MAVEN_CMD_LINE_ARGS=%* 177 | 178 | %MAVEN_JAVA_EXE% ^ 179 | %JVM_CONFIG_MAVEN_PROPS% ^ 180 | %MAVEN_OPTS% ^ 181 | %MAVEN_DEBUG_OPTS% ^ 182 | -classpath %WRAPPER_JAR% ^ 183 | "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ 184 | %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 185 | if ERRORLEVEL 1 goto error 186 | goto end 187 | 188 | :error 189 | set ERROR_CODE=1 190 | 191 | :end 192 | @endlocal & set ERROR_CODE=%ERROR_CODE% 193 | 194 | if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost 195 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 196 | if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" 197 | if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" 198 | :skipRcPost 199 | 200 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 201 | if "%MAVEN_BATCH_PAUSE%"=="on" pause 202 | 203 | if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% 204 | 205 | cmd /C exit /B %ERROR_CODE% 206 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.rabbitmq 8 | rabbitmq-ra-jepsen 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 1.8 13 | 1.8 14 | 1.12.0 15 | 1.2.13 16 | 5.13.1 17 | 3.27.3 18 | 1.7.36 19 | 20 | 21 | 22 | 23 | 24 | org.clojure 25 | clojure 26 | ${clojure.version} 27 | 28 | 29 | 30 | org.slf4j 31 | slf4j-api 32 | ${slf4j.version} 33 | 34 | 35 | 36 | org.junit.jupiter 37 | junit-jupiter 38 | ${junit.version} 39 | test 40 | 41 | 42 | 43 | org.assertj 44 | assertj-core 45 | ${assertj.version} 46 | test 47 | 48 | 49 | 50 | ch.qos.logback 51 | logback-classic 52 | ${logback.version} 53 | test 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | src/main/resources 63 | 64 | 65 | 66 | 67 | 68 | maven-surefire-plugin 69 | 3.5.3 70 | 71 | 72 | maven-failsafe-plugin 73 | 3.5.3 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/project.clj: -------------------------------------------------------------------------------- 1 | (defproject jepsen.rakvstore "0.1.0-SNAPSHOT" 2 | :description "Jepsen for raft-based key/value store" 3 | :url "https://github.com/rabbitmq/ra-kv-store/tree/master/jepsen.rakvstore" 4 | :source-paths ["src/main/clojure"] 5 | :java-source-paths ["src/main/java"] 6 | :jvm-opts ["-Xmx12g"] 7 | :license {:name "Apache 2.0 License" 8 | :url "https://www.apache.org/licenses/LICENSE-2.0.html"} 9 | :main jepsen.rakvstore 10 | :dependencies [[org.clojure/clojure "1.12.0"] 11 | [jepsen "0.3.9"]] 12 | :exclusions [org.slf4j/log4j-over-slf4j 13 | log4j/log4j] 14 | 15 | ) 16 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/src/main/clojure/jepsen/rakvstore.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 2 | ;; 3 | ;; Licensed under the Apache License, Version 2.0 (the "License"); 4 | ;; you may not use this file except in compliance with the License. 5 | ;; You may obtain a copy of the License at 6 | ;; 7 | ;; https://www.apache.org/licenses/LICENSE-2.0 8 | ;; 9 | ;; Unless required by applicable law or agreed to in writing, software 10 | ;; distributed under the License is distributed on an "AS IS" BASIS, 11 | ;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ;; See the License for the specific language governing permissions and 13 | ;; limitations under the License. 14 | ;; 15 | 16 | (ns jepsen.rakvstore 17 | (:require [clojure.tools.logging :refer :all] 18 | [clojure.string :as str] 19 | [knossos.model :as model] 20 | [slingshot.slingshot :refer [try+]] 21 | [jepsen [cli :as cli] 22 | [checker :as checker] 23 | [control :as c] 24 | [db :as db] 25 | [client :as client] 26 | [nemesis :as nemesis] 27 | [independent :as independent] 28 | [generator :as gen] 29 | [util :as util] 30 | [tests :as tests]] 31 | [jepsen.set :as set] 32 | [jepsen.checker.timeline :as timeline] 33 | [jepsen.control.util :as cu] 34 | [jepsen.os.debian :as debian])) 35 | 36 | 37 | (defn r [_ _] {:type :invoke, :f :read, :value nil}) 38 | (defn w [_ _] {:type :invoke, :f :write, :value (rand-int 5)}) 39 | (defn cas [_ _] {:type :invoke, :f :cas, :value [(rand-int 5) (rand-int 5)]}) 40 | 41 | (defn parse-long-nil 42 | "Parses a string to a Long. Passes through `nil`." 43 | [s] 44 | (when s (parse-long s))) 45 | 46 | (defrecord Client [conn] 47 | client/Client 48 | (open! [this test node] 49 | (assoc this :conn (com.rabbitmq.jepsen.Utils/createClient node))) 50 | 51 | (setup! [this test]) 52 | 53 | (invoke! [_ test op] 54 | (let [[k v] (:value op)] 55 | (try+ 56 | (case (:f op) 57 | :read (assoc op :type :ok, :value (independent/tuple k (parse-long-nil (com.rabbitmq.jepsen.Utils/get conn k)))) 58 | :write (do (let [result (com.rabbitmq.jepsen.Utils/write conn k v)] 59 | (assoc op :type :ok :error (str (com.rabbitmq.jepsen.Utils/node conn) " " (.getHeaders result))))) 60 | :cas (let [[old new] v] 61 | (do (let [result (com.rabbitmq.jepsen.Utils/cas conn k old new)] 62 | (assoc op :type (if (.isOk result) 63 | :ok 64 | :fail) :error (str (com.rabbitmq.jepsen.Utils/node conn) " " (.getHeaders result))) 65 | )))) 66 | (catch com.rabbitmq.jepsen.RaTimeoutException _ 67 | (assoc op 68 | :type :info 69 | :error :timeout)) 70 | (catch com.rabbitmq.jepsen.RaNodeDownException _ 71 | (assoc op 72 | :type :info 73 | :error (str :nodedown " " (com.rabbitmq.jepsen.Utils/node conn)))) 74 | (catch java.lang.Exception _ 75 | (assoc op 76 | :type (if (= :read (:f op)) :fail :info) 77 | :error :exception)) 78 | )) 79 | 80 | ) 81 | (teardown! [this test]) 82 | 83 | (close! [_ test])) 84 | 85 | (def dir "/opt/rakvstore") 86 | (def log-dir "/opt/rakvstore/log") 87 | (def status-file "/opt/rakvstore/log/status.dump") 88 | (def configurationFile "/opt/rakvstore/releases/1/sys.config") 89 | (def vmArgsFile "/opt/rakvstore/releases/1/vm.args") 90 | (def env-variables "ERL_CRASH_DUMP=/opt/rakvstore/log/erl_crash.dump RUN_ERL_LOG_MAXSIZE=1000000 RUN_ERL_LOG_GENERATIONS=100") 91 | (def binary "/opt/rakvstore/bin/ra_kv_store_release") 92 | 93 | (defn db 94 | "RA KV Store." 95 | [] 96 | (reify db/DB 97 | (setup! [_ test node] 98 | (info node "installing RA KV Store") 99 | (c/su 100 | (c/exec :rm :-rf "/tmp/ra_kv_store") 101 | (c/exec :rm :-rf dir) 102 | (c/exec :mkdir :-p log-dir) 103 | (cu/install-archive! (str (test :erlang-distribution-url)) dir) 104 | (let [configuration (com.rabbitmq.jepsen.Utils/configuration test node)] 105 | (c/exec :echo configuration :| :tee configurationFile) 106 | ) 107 | (let [vmArgs (com.rabbitmq.jepsen.Utils/vmArgs)] 108 | (c/exec :echo vmArgs :| :tee vmArgsFile) 109 | ) 110 | (c/exec* "chmod u+x /opt/rakvstore/erts*/bin/*") 111 | (info node "starting RA server" binary) 112 | (c/exec* env-variables binary "daemon") 113 | (Thread/sleep 5000) 114 | ) 115 | ) 116 | (teardown! [_ test node] 117 | (info node "tearing down RA KV Store") 118 | (c/su 119 | (c/exec :mkdir :-p log-dir) 120 | (if (not= "" (try 121 | (c/exec :pgrep :beam) 122 | (catch RuntimeException _ ""))) 123 | (c/exec* env-variables binary "stop") 124 | (do (info node "RA KV Store already stopped") 125 | )) 126 | ; (c/exec :rm :-rf dir) 127 | ) 128 | ) 129 | db/LogFiles 130 | (log-files [_ test node] 131 | (c/su 132 | (c/exec* (str "chmod o+r " log-dir "/*")) 133 | (if (not= "" (try 134 | (c/exec :pgrep :beam) 135 | (catch RuntimeException _ ""))) 136 | (do 137 | (let [ 138 | ra-node-id (com.rabbitmq.jepsen.Utils/raNodeId node -1) 139 | erlang-eval-status (str "eval \"sys:get_status(" ra-node-id ").\"") 140 | erlang-eval-counters (str "eval \"ra_counters:overview().\"") 141 | ] 142 | (c/exec* binary erlang-eval-status ">>" status-file) 143 | (c/exec* binary erlang-eval-counters ">>" status-file) 144 | )) 145 | (do (info node "RA KV Store stopped, cannot get status") 146 | )) 147 | ) 148 | (conj (jepsen.control.util/ls-full log-dir)) 149 | ))) 150 | 151 | (defn register-workload 152 | "Tests linearizable reads, writes, and compare-and-set operations on 153 | independent keys." 154 | [opts] 155 | {:client (Client. nil) 156 | :checker (independent/checker 157 | (checker/compose 158 | {:linear (checker/linearizable {:model (model/cas-register)}) 159 | :timeline (timeline/html)})) 160 | :generator (independent/concurrent-generator 161 | 10 162 | (repeatedly #(rand-int 75)) 163 | (fn [k] 164 | (->> (gen/mix [r w cas]) 165 | (gen/limit (:ops-per-key opts)))))}) 166 | 167 | (defn start-erlang-vm! 168 | "Start RA KV Store on node." 169 | [test node] 170 | (c/su 171 | (if (not= "" (try 172 | (c/exec :pgrep :beam) 173 | (catch RuntimeException _ ""))) 174 | (info node "RA KV Store already running.") 175 | (do (info node "Starting RA KV Store...") 176 | (c/exec* env-variables binary "start -ra_kv_store restart_ra_cluster 'true'") 177 | (info node "RA KV Store started")))) 178 | :started) 179 | 180 | (defn kill-erlang-vm! 181 | "Kills RA KV store on node." 182 | [test node] 183 | (util/meh (c/su (c/exec* "killall -9 beam.smp"))) 184 | (info node "RA KV Store killed.") 185 | :killed) 186 | 187 | (defn start-erlang-process! 188 | "Start for Erlang process killer (no-op)." 189 | [test node] 190 | (info node "Called start of Erlang process killer (no-op)") 191 | :started) 192 | 193 | (defn kill-erlang-process! 194 | "Kills a random RA Erlang process" 195 | [test node] 196 | (let [ 197 | ; FIXME looks like the ra_log_segment_writer isn't killed (doesn't show up in the logs) 198 | ;erlangProcess (rand-nth (list "ra_log_wal" "ra_log_snapshot_writer" "ra_log_segment_writer")) 199 | erlangProcess (rand-nth (list "ra_log_wal" "ra_log_segment_writer")) 200 | erlangEval (str "eval 'exit(whereis(" erlangProcess "), kill).'") 201 | ] 202 | (c/su 203 | (info node "Killing" erlangProcess "Erlang process") 204 | (c/exec* binary erlangEval) 205 | ) 206 | 207 | (info node erlangProcess "Erlang process killed" erlangEval)) 208 | :killed) 209 | 210 | (defn kill-erlang-vm-nemesis 211 | "A nemesis that kills the Erlang VM on (a) random node(s)" 212 | [n] 213 | (nemesis/node-start-stopper 214 | (fn [nodes] ((comp (partial take n) shuffle) nodes)) 215 | kill-erlang-vm! 216 | start-erlang-vm!) 217 | ) 218 | 219 | (defn kill-erlang-process-nemesis 220 | "A nemesis that kills a random RA log process on (a) random node(s)" 221 | [n] 222 | (nemesis/node-start-stopper 223 | (fn [nodes] ((comp (partial take n) shuffle) nodes)) 224 | kill-erlang-process! 225 | start-erlang-process!) 226 | ) 227 | 228 | 229 | (def nemesises 230 | "A map of nemesis names." 231 | {"kill-erlang-vm" "" 232 | "kill-erlang-process" "" 233 | "random-partition-halves" "" 234 | "partition-halves" "" 235 | "partition-majorities-ring" "" 236 | "partition-random-node" "" 237 | "combined" "" 238 | }) 239 | 240 | (def network-partition-nemesises 241 | "A map of network partition nemesis names" 242 | {"random-partition-halves" "" 243 | "partition-halves" "" 244 | "partition-majorities-ring" "" 245 | "partition-random-node" "" 246 | }) 247 | 248 | (defn init-network-partition-nemesis 249 | "Returns appropriate network partition nemesis" 250 | [opts] 251 | (case (:network-partition-nemesis opts) 252 | "random-partition-halves" (nemesis/partition-random-halves) 253 | "partition-halves" (nemesis/partition-halves) 254 | "partition-majorities-ring" (nemesis/partition-majorities-ring) 255 | "partition-random-node" (nemesis/partition-random-node) 256 | ) 257 | ) 258 | 259 | (defn init-nemesis 260 | "Returns appropriate nemesis" 261 | [opts] 262 | (case (:nemesis opts) 263 | "kill-erlang-vm" (kill-erlang-vm-nemesis (:random-nodes opts)) 264 | "kill-erlang-process" (kill-erlang-process-nemesis (:random-nodes opts)) 265 | "random-partition-halves" (nemesis/partition-random-halves) 266 | "partition-halves" (nemesis/partition-halves) 267 | "partition-majorities-ring" (nemesis/partition-majorities-ring) 268 | "partition-random-node" (nemesis/partition-random-node) 269 | "combined" (nemesis/compose {{:split-start :start 270 | :split-stop :stop} (init-network-partition-nemesis opts) 271 | {:kill-erlang-vm-start :start 272 | :kill-erlang-vm-stop :stop} (kill-erlang-vm-nemesis (:random-nodes opts)) 273 | {:kill-erlang-process-start :start 274 | :kill-erlang-process-stop :stop} (kill-erlang-process-nemesis (:random-nodes opts)) 275 | } 276 | ) 277 | ) 278 | ) 279 | 280 | (def workloads 281 | "A map of workload names to functions that construct workloads, given opts." 282 | {"set" set/workload 283 | "register" register-workload}) 284 | 285 | (defn combined-nemesis-generator 286 | "Nemesis generator that triggers Erlang VM/process killing during network partition." 287 | [opts] 288 | { 289 | :generator (cycle [ 290 | (gen/sleep (:time-before-disruption opts)) 291 | {:type :info :f :split-start} 292 | (gen/stagger (/ (:disruption-duration opts) 8) 293 | (gen/mix 294 | [ 295 | {:type :info, :f :kill-erlang-vm-start} 296 | {:type :info, :f :kill-erlang-process-start} 297 | ] 298 | )) 299 | (gen/stagger (/ (:disruption-duration opts) 6) 300 | (gen/phases {:type :info :f :kill-erlang-vm-stop} 301 | {:type :info :f :kill-erlang-process-stop})) 302 | (gen/sleep (:disruption-duration opts)) 303 | {:type :info :f :split-stop} 304 | ]) 305 | :stop-generator [(gen/once {:type :info, :f :kill-erlang-vm-stop}) 306 | (gen/nemesis (gen/once {:type :info, :f :kill-erlang-process-stop})) 307 | (gen/nemesis (gen/once {:type :info, :f :split-stop}))] 308 | 309 | }) 310 | 311 | (defn single-nemesis-generator 312 | "Nemesis with single disruption." 313 | [opts] 314 | { 315 | :generator (cycle [ 316 | (gen/sleep (:time-before-disruption opts)) 317 | {:type :info :f :start} 318 | (gen/sleep (:disruption-duration opts)) 319 | {:type :info :f :stop} 320 | ]) 321 | :stop-generator (gen/once {:type :info, :f :stop}) 322 | }) 323 | 324 | (def nemesis-generators 325 | "Map of nemesis types to nemesis generator." 326 | {"combined" combined-nemesis-generator 327 | "kill-erlang-vm" single-nemesis-generator 328 | "kill-erlang-process" single-nemesis-generator 329 | "random-partition-halves" single-nemesis-generator 330 | "partition-halves" single-nemesis-generator 331 | "partition-majorities-ring" single-nemesis-generator 332 | "partition-random-node" single-nemesis-generator 333 | }) 334 | 335 | (def cli-opts 336 | "Additional command line options." 337 | [["-r" "--rate HZ" "Approximate number of requests per second, per thread." 338 | :default 10 339 | :parse-fn read-string 340 | :validate [#(and (number? %) (pos? %)) "Must be a positive number"]] 341 | [nil "--ops-per-key NUM" "Maximum number of operations on any given key." 342 | :default 100 343 | :parse-fn parse-long 344 | :validate [pos? "Must be a positive integer."]] 345 | ["-w" "--workload NAME" "What workload should we run?" 346 | :missing (str "--workload " (cli/one-of workloads)) 347 | :validate [workloads (cli/one-of workloads)]] 348 | [nil "--nemesis NAME" "What nemesis should we use?" 349 | :missing (str "--nemesis " (cli/one-of nemesises)) 350 | :validate [nemesises (cli/one-of nemesises)]] 351 | [nil "--network-partition-nemesis NAME" "What network partition nemesis should we use (only for combined nemesis)? Default is random-partition-halves" 352 | :default "random-partition-halves" 353 | :missing (str "--network-partition-nemesis " (cli/one-of network-partition-nemesises)) 354 | :validate [network-partition-nemesises (cli/one-of network-partition-nemesises)]] 355 | [nil "--random-nodes NUM" "Number of nodes disrupted by Erlang VM and Erlang process killing nemesises" 356 | :default 1 357 | :parse-fn parse-long 358 | :validate [pos? "Must be a positive integer."]] 359 | [nil "--erlang-net-ticktime NUM" "Erlang net tick time in seconds (https://www.rabbitmq.com/nettick.html)." 360 | :default -1 361 | :parse-fn parse-long 362 | :validate [pos? "Must be a positive integer."]] 363 | [nil "--disruption-duration NUM" "Duration of disruption (in seconds)" 364 | :default 5 365 | :parse-fn parse-long 366 | :validate [pos? "Must be a positive integer."]] 367 | [nil "--time-before-disruption NUM" "Time before the nemesis kicks in (in seconds)" 368 | :default 5 369 | :parse-fn parse-long 370 | :validate [pos? "Must be a positive integer."]] 371 | [nil "--release-cursor-every NUM" "Release RA cursor every n operations." 372 | :default -1 373 | :parse-fn parse-long 374 | :validate [pos? "Must be a positive integer."]] 375 | [nil "--wal-max-size-bytes NUM" "Maximum size of RA Write Ahead Log, default is 134217728 (128 MB)." 376 | :default 134217728 377 | :parse-fn parse-long 378 | :validate [pos? "Must be a positive integer."]] 379 | [nil "--erlang-distribution-url URL" "URL to retrieve the Erlang distribution archive" 380 | :default "file:///root/jepsen.rakvstore/ra_kv_store_release-1.tar.gz" 381 | :parse-fn read-string] 382 | ]) 383 | 384 | 385 | 386 | (defn rakvstore-test 387 | "Given an options map from the command line runner (e.g. :nodes, :ssh, 388 | :concurrency ...), constructs a test map. Special options: 389 | 390 | :rate Approximate number of requests per second, per thread. 391 | :ops-per-key Maximum number of operations allowed on any given key. 392 | :workload Type of workload. 393 | :nemesis Type of nemesis. 394 | :erlang-net-ticktime Erlang net tick time. 395 | :release-cursor-every Release RA cursor every n operations." 396 | [opts] 397 | (let [ 398 | workload ((get workloads (:workload opts)) opts) 399 | nemesis (init-nemesis opts) 400 | nemesis-generator ((get nemesis-generators (:nemesis opts)) opts) 401 | ] 402 | (merge tests/noop-test 403 | opts 404 | {:pure-generators true 405 | :name (str (name (:workload opts))) 406 | :os debian/os 407 | :db (db) 408 | :checker (checker/compose 409 | {:perf (checker/perf) 410 | :workload (:checker workload)}) 411 | :client (:client workload) 412 | :nemesis nemesis 413 | :generator (gen/phases 414 | (->> (:generator workload) 415 | (gen/stagger (/ (:rate opts))) 416 | (gen/nemesis 417 | (:generator nemesis-generator) 418 | ) 419 | (gen/time-limit (:time-limit opts)) 420 | ) 421 | (gen/log "Healing cluster") 422 | (gen/nemesis (:stop-generator nemesis-generator)) 423 | (gen/log "Waiting for recovery") 424 | (gen/sleep 10) 425 | (gen/clients (:final-generator workload)))} 426 | ))) 427 | 428 | (defn -main 429 | "Handles command line arguments. Can either run a test, or a web server for 430 | browsing results." 431 | [& args] 432 | (cli/run! (cli/single-test-cmd {:test-fn rakvstore-test, 433 | :opt-spec cli-opts}) 434 | args)) 435 | 436 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/src/main/clojure/jepsen/set.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 2 | ;; 3 | ;; Licensed under the Apache License, Version 2.0 (the "License"); 4 | ;; you may not use this file except in compliance with the License. 5 | ;; You may obtain a copy of the License at 6 | ;; 7 | ;; https://www.apache.org/licenses/LICENSE-2.0 8 | ;; 9 | ;; Unless required by applicable law or agreed to in writing, software 10 | ;; distributed under the License is distributed on an "AS IS" BASIS, 11 | ;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ;; See the License for the specific language governing permissions and 13 | ;; limitations under the License. 14 | ;; 15 | 16 | (ns jepsen.set 17 | (:require [jepsen 18 | [checker :as checker] 19 | [client :as client] 20 | [generator :as gen]] 21 | [slingshot.slingshot :refer [try+]])) 22 | 23 | 24 | (defrecord SetClient [k conn] 25 | client/Client 26 | (open! [this test node] 27 | 28 | (assoc this :conn (com.rabbitmq.jepsen.Utils/createClient node)) 29 | ) 30 | (setup! [this test]) 31 | ; for some reasons, creating an empty set here makes the RA KV Store fail 32 | ; (step 8 of the tutorial actually initializes the set here) 33 | ; solution is to handle a null set in the addToSet function 34 | (invoke! [_ test op] 35 | (try+ 36 | (case (:f op) 37 | :read (assoc op 38 | :type :ok, 39 | :value (read-string (com.rabbitmq.jepsen.Utils/getSet conn k))) 40 | 41 | :add (do (let [result (com.rabbitmq.jepsen.Utils/addToSet conn k (:value op))] 42 | (assoc op :type :ok :error (str (com.rabbitmq.jepsen.Utils/node conn) " " (.getHeaders result))))) 43 | ) 44 | 45 | (catch com.rabbitmq.jepsen.RaTimeoutException rtex 46 | (assoc op 47 | :type :info 48 | :error (str :timeout " " (com.rabbitmq.jepsen.Utils/node conn) " " (.getHeaders rtex)))) 49 | (catch com.rabbitmq.jepsen.RaNodeDownException _ 50 | (assoc op 51 | :type :info 52 | :error (str :nodedown " " (com.rabbitmq.jepsen.Utils/node conn)))) 53 | (catch java.lang.Exception _ 54 | (assoc op 55 | :type (if (= :read (:f op)) :fail :info) 56 | :error :exception)) 57 | 58 | )) 59 | (teardown! [_ test]) 60 | 61 | (close! [_ test]) 62 | ) 63 | 64 | (defn workload 65 | "A generator, client, and checker for a set test." 66 | [opts] 67 | {:client (SetClient. "a-set" nil) 68 | :checker (checker/set) 69 | :generator (->> (range) 70 | (map (fn [x] {:type :invoke, :f :add, :value x}))) 71 | :final-generator (gen/once {:type :invoke, :f :read, :value nil})}) 72 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/src/main/java/com/rabbitmq/jepsen/RaNodeDownException.java: -------------------------------------------------------------------------------- 1 | package com.rabbitmq.jepsen; 2 | 3 | /** 4 | * 5 | */ 6 | public class RaNodeDownException extends RuntimeException { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/src/main/java/com/rabbitmq/jepsen/RaTimeoutException.java: -------------------------------------------------------------------------------- 1 | package com.rabbitmq.jepsen; 2 | 3 | import java.util.LinkedHashMap; 4 | import java.util.Map; 5 | 6 | /** 7 | * 8 | */ 9 | public class RaTimeoutException extends RuntimeException { 10 | 11 | private final Map headers; 12 | 13 | public RaTimeoutException() { 14 | this(new LinkedHashMap<>()); 15 | } 16 | 17 | public RaTimeoutException(Map headers) { 18 | this.headers = headers; 19 | } 20 | 21 | public Map getHeaders() { 22 | return headers; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/src/main/java/com/rabbitmq/jepsen/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.rabbitmq.jepsen; 18 | 19 | import java.io.BufferedReader; 20 | import java.io.FileNotFoundException; 21 | import java.io.IOException; 22 | import java.io.InputStream; 23 | import java.io.InputStreamReader; 24 | import java.io.OutputStreamWriter; 25 | import java.net.ConnectException; 26 | import java.net.HttpURLConnection; 27 | import java.net.URI; 28 | import java.net.URL; 29 | import java.util.*; 30 | import java.util.concurrent.*; 31 | import java.util.concurrent.atomic.AtomicBoolean; 32 | import java.util.concurrent.atomic.AtomicInteger; 33 | import java.util.concurrent.atomic.AtomicLong; 34 | import java.util.concurrent.atomic.AtomicReference; 35 | import java.util.function.Predicate; 36 | import java.util.function.Supplier; 37 | import java.util.stream.Collectors; 38 | import org.slf4j.Logger; 39 | import org.slf4j.LoggerFactory; 40 | 41 | 42 | /** */ 43 | @SuppressWarnings("unchecked") 44 | public class Utils { 45 | 46 | private static final int HTTP_REQUEST_TIMEOUT = 600_000; 47 | 48 | private static final Logger LOGGER = LoggerFactory.getLogger("jepsen.utils.client"); 49 | 50 | // static JepsenTestLog LOG = new DefaultJepsenTestLog(); 51 | static JepsenTestLog LOG = new NoOpJepsenTestLog(); 52 | 53 | private static final Map NODE_TO_ERLANG_NODE = new ConcurrentHashMap<>(); 54 | 55 | public static String configuration(Map test, Object currentNode) { 56 | List nodesObj = (List) get(test, ":nodes"); 57 | String erlangNetTickTime = 58 | get(test, ":erlang-net-ticktime") != null ? erlangNetTickTime(test) : ""; 59 | Object releaseCursorEvery = get(test, ":release-cursor-every"); 60 | releaseCursorEvery = 61 | releaseCursorEvery == null ? -1L : Long.parseLong(releaseCursorEvery.toString()); 62 | Object walMaxSizeBytes = get(test, ":wal-max-size-bytes"); 63 | Long megaBytes128 = 134217728L; 64 | walMaxSizeBytes = 65 | walMaxSizeBytes == null ? megaBytes128 : Long.parseLong(walMaxSizeBytes.toString()); 66 | 67 | List nodes = nodesObj.stream().map(Object::toString).sorted().collect(Collectors.toList()); 68 | String node = currentNode.toString(); 69 | String nodeIndex = String.valueOf(nodes.indexOf(node) + 1); 70 | List erlangNodes = nodesObj.stream().map(n -> raNodeId(n, nodes.indexOf(n.toString()))).collect(Collectors.toList()); 71 | 72 | String configuration = 73 | String.format( 74 | "[\n" 75 | + erlangNetTickTime 76 | + " {ra, [\n" 77 | + " {data_dir, \"/tmp/ra_kv_store\"},\n" 78 | + " {wal_max_size_bytes, %d}\n" 79 | + " ]},\n" 80 | + " {ra_kv_store, [\n" 81 | + " {port, 8080},\n" 82 | + " {nodes, [%s]},\n" 83 | + " {server_reference, ra_kv%s},\n" 84 | + " {release_cursor_every, %d}\n" 85 | + " ]}\n" 86 | + "].", 87 | walMaxSizeBytes, 88 | String.join(", ", erlangNodes), 89 | nodeIndex, 90 | releaseCursorEvery); 91 | 92 | return configuration; 93 | } 94 | 95 | public static String raNodeId(Object n, int index) { 96 | String node = n.toString(); 97 | if (index == -1) { 98 | // we don't have enough context, but the node name should have been computed already 99 | return NODE_TO_ERLANG_NODE.get(node); 100 | } else { 101 | String erlangNode = String.format("{ra_kv%d, 'kv@%s'}", index + 1, node); 102 | NODE_TO_ERLANG_NODE.putIfAbsent(node, erlangNode); 103 | return erlangNode; 104 | } 105 | } 106 | 107 | static String erlangNetTickTime(Map test) { 108 | Object valueObj = get(test, ":erlang-net-ticktime"); 109 | String value = valueObj == null ? "" : valueObj.toString(); 110 | long tickTime = Long.parseLong(value); 111 | if (tickTime >= 0) { 112 | return String.format("{kernel, [{net_ticktime, %d}]},\n", tickTime); 113 | } else { 114 | return ""; 115 | } 116 | } 117 | 118 | public static String vmArgs() { 119 | return "-sname kv\n" + "-setcookie ra_kv_store"; 120 | } 121 | 122 | public static Client createClient(Object node) { 123 | return new Client(node.toString()); 124 | } 125 | 126 | public static Object get(Client client, Object key) throws Exception { 127 | return client.get(key); 128 | } 129 | 130 | public static Response write(Client client, Object key, Object value) throws Exception { 131 | return client.write(key, value); 132 | } 133 | 134 | public static Response cas(Client client, Object key, Object oldValue, Object newValue) 135 | throws Exception { 136 | return client.cas(key, oldValue, newValue); 137 | } 138 | 139 | public static Response addToSet(Client client, Object key, Object value) throws Exception { 140 | return client.addToSet(key, value); 141 | } 142 | 143 | public static String getSet(Client client, Object key) throws Exception { 144 | try { 145 | return client.getSet(key); 146 | } catch (Exception e) { 147 | LOGGER.warn("Error while getting set on node {}", client.node, e); 148 | throw e; 149 | } 150 | } 151 | 152 | public static String node(Client client) { 153 | return client.node; 154 | } 155 | 156 | static Object get(Map map, String keyStringValue) { 157 | for (Map.Entry entry : map.entrySet()) { 158 | if (keyStringValue.equals(entry.getKey().toString())) { 159 | return entry.getValue(); 160 | } 161 | } 162 | return null; 163 | } 164 | 165 | private static Map raHeaders(HttpURLConnection c) { 166 | Map headers = new LinkedHashMap<>(); 167 | for (Map.Entry> entry : c.getHeaderFields().entrySet()) { 168 | if (entry.getKey() != null && entry.getKey().startsWith("ra_")) { 169 | headers.put(entry.getKey().replaceFirst("ra_", ""), String.join(",", entry.getValue())); 170 | } 171 | } 172 | return headers; 173 | } 174 | 175 | private static void logGetResponse(String node, Object key, Response response) { 176 | try { 177 | LOGGER.info("read [" + key + " " + response.getBody() + "] from " + node + " " + response.getHeaders()); 178 | } catch (Exception e) { 179 | LOGGER.warn("Error while logging RA headers: " + e.getMessage()); 180 | } 181 | } 182 | 183 | interface JepsenTestLog { 184 | 185 | RequestAttempt requestAttempt(Object node, Object value); 186 | 187 | void step(RequestAttempt attempt, Supplier message) throws InterruptedException; 188 | 189 | void attempt(RequestAttempt attempt); 190 | 191 | void success(RequestAttempt attempt); 192 | 193 | void alreadyInSet(RequestAttempt attempt); 194 | 195 | CasRequest casRequest(Object node, Object expectedValue, Object newValue); 196 | 197 | void statusCode(CasRequest request, int status); 198 | 199 | void dump(); 200 | } 201 | 202 | static class NoOpJepsenTestLog implements JepsenTestLog { 203 | 204 | @Override 205 | public RequestAttempt requestAttempt(Object node, Object value) { 206 | return null; 207 | } 208 | 209 | @Override 210 | public void step(RequestAttempt attempt, Supplier message) 211 | throws InterruptedException {} 212 | 213 | @Override 214 | public void attempt(RequestAttempt attempt) {} 215 | 216 | @Override 217 | public void success(RequestAttempt attempt) {} 218 | 219 | @Override 220 | public void alreadyInSet(RequestAttempt attempt) {} 221 | 222 | @Override 223 | public CasRequest casRequest(Object node, Object expectedValue, Object newValue) { 224 | return null; 225 | } 226 | 227 | @Override 228 | public void statusCode(CasRequest request, int status) {} 229 | 230 | @Override 231 | public void dump() {} 232 | } 233 | 234 | public static class Client { 235 | 236 | private final String node; 237 | 238 | public Client(String node) { 239 | this.node = node; 240 | } 241 | 242 | V request(Callable call) throws Exception { 243 | try { 244 | return call.call(); 245 | } catch (ConnectException e) { 246 | throw new RaNodeDownException(); 247 | } 248 | } 249 | 250 | private Response get(Object key, boolean log) throws Exception { 251 | return request( 252 | () -> { 253 | URL url = new URI(String.format("http://%s:8080/%s", this.node, key.toString())).toURL(); 254 | HttpURLConnection conn = null; 255 | try { 256 | conn = (HttpURLConnection) url.openConnection(); 257 | conn.setRequestMethod("GET"); 258 | conn.setConnectTimeout(HTTP_REQUEST_TIMEOUT); 259 | conn.setReadTimeout(HTTP_REQUEST_TIMEOUT); 260 | try { 261 | String body = body(conn.getInputStream()); 262 | Response response = new Response<>(true, raHeaders(conn), body); 263 | if (log) { 264 | logGetResponse(node, key, response); 265 | } 266 | return response; 267 | } catch (FileNotFoundException e) { 268 | LOGGER.warn("FileNotFoundException on set GET operation: {}", e.getMessage()); 269 | return null; 270 | } catch (Exception e) { 271 | int responseCode = conn.getResponseCode(); 272 | String responseBody = body(conn.getErrorStream()); 273 | if (responseCode == 503 && "RA timeout".equals(responseBody)) { 274 | throw new RaTimeoutException(raHeaders(conn)); 275 | } else { 276 | throw e; 277 | } 278 | } 279 | } finally { 280 | if (conn != null) { 281 | conn.disconnect(); 282 | } 283 | } 284 | }); 285 | } 286 | 287 | String get(Object key) throws Exception { 288 | Response response = getWithResponse(key); 289 | return response == null ? null : response.getBody(); 290 | } 291 | 292 | Response getWithResponse(Object key) throws Exception { 293 | return get(key, true); 294 | } 295 | 296 | Response write(Object key, Object value) throws Exception { 297 | return request( 298 | () -> { 299 | URL url = new URI(String.format("http://%s:8080/%s", this.node, key.toString())).toURL(); 300 | HttpURLConnection conn = null; 301 | Response response = null; 302 | try { 303 | conn = (HttpURLConnection) url.openConnection(); 304 | conn.setRequestMethod("PUT"); 305 | conn.setDoOutput(true); 306 | conn.setConnectTimeout(HTTP_REQUEST_TIMEOUT); 307 | conn.setReadTimeout(HTTP_REQUEST_TIMEOUT); 308 | try (OutputStreamWriter out = new OutputStreamWriter(conn.getOutputStream())) { 309 | out.write("value=" + value.toString()); 310 | } 311 | conn.getInputStream(); 312 | response = new Response(true, raHeaders(conn)); 313 | } catch (Exception e) { 314 | int responseCode = conn.getResponseCode(); 315 | String responseBody = body(conn.getErrorStream()); 316 | if (responseCode == 503 && "RA timeout".equals(responseBody)) { 317 | throw new RaTimeoutException(raHeaders(conn)); 318 | } else { 319 | throw e; 320 | } 321 | } finally { 322 | if (conn != null) { 323 | conn.disconnect(); 324 | } 325 | } 326 | return response; 327 | }); 328 | } 329 | 330 | Response cas(Object key, Object oldValue, Object newValue) throws Exception { 331 | return request( 332 | () -> { 333 | CasRequest request = LOG.casRequest(node, oldValue, newValue); 334 | URL url = new URI(String.format("http://%s:8080/%s", this.node, key.toString())).toURL(); 335 | HttpURLConnection conn = null; 336 | try { 337 | conn = (HttpURLConnection) url.openConnection(); 338 | conn.setRequestMethod("PUT"); 339 | conn.setDoOutput(true); 340 | conn.setConnectTimeout(HTTP_REQUEST_TIMEOUT); 341 | conn.setReadTimeout(HTTP_REQUEST_TIMEOUT); 342 | try (OutputStreamWriter out = new OutputStreamWriter(conn.getOutputStream())) { 343 | out.write( 344 | String.format( 345 | "value=%s&expected=%s", newValue.toString(), oldValue.toString())); 346 | } 347 | conn.getInputStream(); 348 | LOG.statusCode(request, conn.getResponseCode()); 349 | return new Response(true, raHeaders(conn)); 350 | } catch (Exception e) { 351 | int responseCode = conn.getResponseCode(); 352 | String responseBody = body(conn.getErrorStream()); 353 | LOG.statusCode(request, responseCode); 354 | if (responseCode == 409) { 355 | return new Response(false, raHeaders(conn)); 356 | } else if (responseCode == 503 && "RA timeout".equals(responseBody)) { 357 | throw new RaTimeoutException(raHeaders(conn)); 358 | } else { 359 | throw e; 360 | } 361 | } finally { 362 | if (conn != null) { 363 | conn.disconnect(); 364 | } 365 | LOG.dump(); 366 | } 367 | }); 368 | } 369 | 370 | /** 371 | * Add a value to a set. The uniqueness of the value in the set is enforced thanks to the CAS 372 | * operation (the initial value of the set is checked when trying to update the set with the new 373 | * value). 374 | * 375 | * @param key 376 | * @param value 377 | * @throws Exception 378 | */ 379 | public Response addToSet(Object key, Object value) throws Exception { 380 | AtomicReference response = new AtomicReference<>(); 381 | request( 382 | () -> { 383 | RequestAttempt requestAttempt = LOG.requestAttempt(node, value); 384 | final AtomicBoolean result = new AtomicBoolean(); 385 | retry( 386 | () -> { 387 | LOG.step(requestAttempt, () -> "in retry loop"); 388 | // currentValue is "" 389 | Response currentResponse = get(key, false); 390 | String currentValue = currentResponse == null ? null : currentResponse.getBody(); 391 | if (currentValue == null || currentValue.isEmpty()) { 392 | LOG.step(requestAttempt, () -> "no value in the set"); 393 | LOG.attempt(requestAttempt); 394 | LOG.step(requestAttempt, () -> "sending cas operation for empty set"); 395 | try { 396 | Response casResponse = cas(key.toString(), "", value.toString()); 397 | result.set(casResponse.isOk()); 398 | response.set(casResponse); 399 | } catch (RaTimeoutException e) { 400 | LOG.step( 401 | requestAttempt, 402 | () -> "cas operation timed out, result isn't indeterminate"); 403 | throw e; 404 | } 405 | LOG.step( 406 | requestAttempt, 407 | () -> "cas operation returned " + result + " for empty set"); 408 | if (result.get()) { 409 | LOG.success(requestAttempt); 410 | } 411 | LOG.step(requestAttempt, () -> "returning " + result.get()); 412 | return result.get(); 413 | } 414 | // currentValue is "1 2 3 4 5" 415 | String valueAsString = value.toString(); 416 | LOG.step(requestAttempt, () -> "checking value to add is not already in the set"); 417 | for (String valueInSet : currentValue.split(" ")) { 418 | if (valueAsString.equals(valueInSet)) { 419 | LOG.alreadyInSet(requestAttempt); 420 | LOG.step(requestAttempt, () -> "value already in the set, returning true"); 421 | // already in the set, nothing to do 422 | Response casResponse = new Response(true, Collections.emptyMap()); 423 | result.set(casResponse.isOk()); 424 | response.set(casResponse); 425 | return result.get(); 426 | } 427 | } 428 | LOG.attempt(requestAttempt); 429 | LOG.step(requestAttempt, () -> "value not already in the set"); 430 | LOG.step(requestAttempt, () -> "sending cas option"); 431 | try { 432 | Response casResponse = 433 | cas(key, currentValue, currentValue + " " + valueAsString); 434 | result.set(casResponse.isOk()); 435 | response.set(casResponse); 436 | } catch (RaTimeoutException e) { 437 | LOG.step( 438 | requestAttempt, 439 | () -> "cas operation timed out, result isn't indeterminate"); 440 | throw e; 441 | } 442 | LOG.step(requestAttempt, () -> "cas operation returned " + result.get()); 443 | if (result.get()) { 444 | LOG.success(requestAttempt); 445 | } 446 | LOG.step(requestAttempt, () -> "returning " + result.get()); 447 | return result.get(); 448 | }); 449 | return null; 450 | }); 451 | return response.get(); 452 | } 453 | 454 | private void retry(Callable operation) throws Exception { 455 | boolean done = operation.call(); 456 | while (!done) { 457 | Thread.sleep( 458 | 100); // same as in 459 | // https://github.com/aphyr/verschlimmbesserung/blob/498fd20ca39c52f6f4506dc281d6af9920791342/src/verschlimmbesserung/core.clj#L41-L44 460 | done = operation.call(); 461 | } 462 | } 463 | 464 | private T retry(Callable operation, Predicate retryPredicate) throws Exception { 465 | int attemptCount = 1; 466 | while (attemptCount <= 3) { 467 | try { 468 | return operation.call(); 469 | } catch (Exception e) { 470 | if (retryPredicate.test(e)) { 471 | attemptCount++; 472 | LOGGER.info("Operation failed with '{}' exception, retrying...", e.getClass().getSimpleName()); 473 | Thread.sleep(1000); 474 | } else { 475 | throw e; 476 | } 477 | } 478 | } 479 | throw new RuntimeException("Operation failed after " + (attemptCount - 1) + " attempts"); 480 | } 481 | 482 | private String body(InputStream inputStream) throws IOException { 483 | if (inputStream != null) { 484 | StringBuilder content = new StringBuilder(); 485 | try (BufferedReader in = new BufferedReader(new InputStreamReader(inputStream))) { 486 | String inputLine; 487 | while ((inputLine = in.readLine()) != null) { 488 | content.append(inputLine); 489 | } 490 | } 491 | return content.toString(); 492 | } else { 493 | return ""; 494 | } 495 | } 496 | 497 | /** 498 | * Return value wrapped in #{ }. This allows parsing it as a Clojure set. 499 | * 500 | * @param key 501 | * @return 502 | * @throws Exception 503 | */ 504 | public String getSet(Object key) throws Exception { 505 | LOG.dump(); 506 | return retry(() -> "#{" + get(key, true).getBody() + "}", e -> e instanceof RaTimeoutException); 507 | } 508 | } 509 | 510 | static class RequestAttempt { 511 | 512 | final String node; 513 | final String value; 514 | final AtomicLong attempts = new AtomicLong(0); 515 | final AtomicLong successes = new AtomicLong(0); 516 | final AtomicLong alreadyInSet = new AtomicLong(0); 517 | final BlockingQueue steps = new ArrayBlockingQueue<>(100); 518 | 519 | RequestAttempt(String node, String value) { 520 | this.node = node; 521 | this.value = value; 522 | } 523 | 524 | void markAttempt() { 525 | this.attempts.incrementAndGet(); 526 | } 527 | 528 | void markSuccess() { 529 | this.successes.incrementAndGet(); 530 | } 531 | 532 | void markAlreadyInSet() { 533 | this.alreadyInSet.incrementAndGet(); 534 | } 535 | 536 | void step(String step) throws InterruptedException { 537 | this.steps.offer(step, 1, TimeUnit.SECONDS); 538 | } 539 | 540 | @Override 541 | public String toString() { 542 | return "RequestAttempt{" 543 | + "node='" 544 | + node 545 | + '\'' 546 | + ", value='" 547 | + value 548 | + '\'' 549 | + ", attempts=" 550 | + attempts 551 | + ", successes=" 552 | + successes 553 | + ", alreadyInSet=" 554 | + alreadyInSet 555 | + ", steps=" 556 | + steps 557 | + '}'; 558 | } 559 | } 560 | 561 | static class DefaultJepsenTestLog implements JepsenTestLog { 562 | 563 | // increase the capacity of those queues for long tests, when log is enabled 564 | final BlockingQueue attempts = new ArrayBlockingQueue<>(100_000); 565 | final BlockingQueue casRequests = new ArrayBlockingQueue<>(100_000); 566 | 567 | @Override 568 | public RequestAttempt requestAttempt(Object node, Object value) { 569 | RequestAttempt requestAttempt = new RequestAttempt(node.toString(), value.toString()); 570 | attempts.add(requestAttempt); 571 | return requestAttempt; 572 | } 573 | 574 | @Override 575 | public void step(RequestAttempt attempt, Supplier message) throws InterruptedException { 576 | attempt.step(message.get()); 577 | } 578 | 579 | @Override 580 | public void attempt(RequestAttempt attempt) { 581 | attempt.markAttempt(); 582 | } 583 | 584 | @Override 585 | public void success(RequestAttempt attempt) { 586 | attempt.markSuccess(); 587 | } 588 | 589 | @Override 590 | public void alreadyInSet(RequestAttempt attempt) { 591 | attempt.markAlreadyInSet(); 592 | } 593 | 594 | @Override 595 | public CasRequest casRequest(Object node, Object expectedValue, Object newValue) { 596 | CasRequest request = 597 | new CasRequest(node.toString(), expectedValue.toString(), newValue.toString()); 598 | casRequests.add(request); 599 | return request; 600 | } 601 | 602 | @Override 603 | public void statusCode(CasRequest request, int status) { 604 | request.statusCode(status); 605 | } 606 | 607 | @Override 608 | public void dump() { 609 | System.out.println("REQUESTS: " + attempts); 610 | System.out.println("CAS: " + casRequests); 611 | } 612 | } 613 | 614 | static class CasRequest { 615 | 616 | private final String node, expectedValue, newValue; 617 | 618 | private final AtomicInteger statusCode = new AtomicInteger(-1); 619 | 620 | CasRequest(String node, String expectedValue, String newValue) { 621 | this.node = node; 622 | this.expectedValue = expectedValue; 623 | this.newValue = newValue; 624 | } 625 | 626 | void statusCode(int status) { 627 | statusCode.set(status); 628 | } 629 | 630 | @Override 631 | public String toString() { 632 | return "CasRequest{" 633 | + "node='" 634 | + node 635 | + '\'' 636 | + ", expectedValue='" 637 | + expectedValue 638 | + '\'' 639 | + ", newValue='" 640 | + newValue 641 | + '\'' 642 | + ", statusCode=" 643 | + statusCode 644 | + '}'; 645 | } 646 | } 647 | 648 | public static class Response { 649 | 650 | private final T body; 651 | private final Map headers; 652 | private final boolean ok; 653 | 654 | public Response(boolean ok, Map headers) { 655 | this(ok, headers, null); 656 | } 657 | 658 | public Response(boolean ok, Map headers, T body) { 659 | this.headers = headers; 660 | this.ok = ok; 661 | this.body = body; 662 | } 663 | 664 | public T getBody() { 665 | return body; 666 | } 667 | 668 | public Map getHeaders() { 669 | return headers; 670 | } 671 | 672 | public boolean isOk() { 673 | return ok; 674 | } 675 | 676 | @Override 677 | public String toString() { 678 | return "Response{" + "body=" + body + ", headers=" + headers + ", ok=" + ok + '}'; 679 | } 680 | } 681 | } 682 | -------------------------------------------------------------------------------- /jepsen/jepsen.rakvstore/src/test/java/com/rabbitmq/jepsen/UtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.rabbitmq.jepsen; 18 | 19 | import static com.rabbitmq.jepsen.Utils.erlangNetTickTime; 20 | import static java.util.Arrays.asList; 21 | import static java.util.Collections.singletonMap; 22 | import static org.assertj.core.api.Assertions.assertThat; 23 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 24 | 25 | import java.util.HashMap; 26 | import java.util.LinkedHashSet; 27 | import java.util.List; 28 | import java.util.Map; 29 | import java.util.Random; 30 | import java.util.Set; 31 | import java.util.concurrent.ConcurrentHashMap; 32 | import java.util.concurrent.CopyOnWriteArrayList; 33 | import java.util.concurrent.ExecutorService; 34 | import java.util.concurrent.Executors; 35 | import java.util.concurrent.TimeUnit; 36 | import java.util.concurrent.atomic.AtomicBoolean; 37 | import java.util.stream.Collectors; 38 | import java.util.stream.IntStream; 39 | import org.junit.jupiter.api.BeforeEach; 40 | import org.junit.jupiter.api.Test; 41 | 42 | /** */ 43 | public class UtilsTest { 44 | 45 | private static final String KEY = "foo"; 46 | Utils.Client client; 47 | 48 | @BeforeEach 49 | public void init() throws Exception { 50 | client = Utils.createClient("localhost"); 51 | client.write(KEY, ""); 52 | } 53 | 54 | @Test 55 | public void configurationFile() { 56 | Map test = new HashMap<>(); 57 | test.put(":nodes", asList("192.168.33.10")); 58 | String configuration = Utils.configuration(test, "192.168.33.10"); 59 | 60 | assertThat(configuration) 61 | .isEqualTo( 62 | "[\n" 63 | + " {ra, [\n" 64 | + " {data_dir, \"/tmp/ra_kv_store\"},\n" 65 | + " {wal_max_size_bytes, 134217728}\n" 66 | + " ]},\n" 67 | + " {ra_kv_store, [\n" 68 | + " {port, 8080},\n" 69 | + " {nodes, [{ra_kv1, 'kv@192.168.33.10'}]},\n" 70 | + " {server_reference, ra_kv1},\n" 71 | + " {release_cursor_every, -1}\n" 72 | + " ]}\n" 73 | + "]."); 74 | 75 | test = new HashMap<>(); 76 | test.put(":nodes", asList("192.168.33.10", "192.168.33.11", "192.168.33.12")); 77 | configuration = Utils.configuration(test, "192.168.33.10"); 78 | 79 | assertThat(configuration) 80 | .isEqualTo( 81 | "[\n" 82 | + " {ra, [\n" 83 | + " {data_dir, \"/tmp/ra_kv_store\"},\n" 84 | + " {wal_max_size_bytes, 134217728}\n" 85 | + " ]},\n" 86 | + " {ra_kv_store, [\n" 87 | + " {port, 8080},\n" 88 | + " {nodes, [{ra_kv1, 'kv@192.168.33.10'}, {ra_kv2, 'kv@192.168.33.11'}, {ra_kv3, 'kv@192.168.33.12'}]},\n" 89 | + " {server_reference, ra_kv1},\n" 90 | + " {release_cursor_every, -1}\n" 91 | + " ]}\n" 92 | + "].", 93 | configuration); 94 | 95 | test = new HashMap<>(); 96 | test.put(":nodes", asList("n1", "n2", "n3")); 97 | test.put(":release-cursor-every", "10"); 98 | configuration = Utils.configuration(test, "n2"); 99 | 100 | assertThat(configuration) 101 | .isEqualTo( 102 | "[\n" 103 | + " {ra, [\n" 104 | + " {data_dir, \"/tmp/ra_kv_store\"},\n" 105 | + " {wal_max_size_bytes, 134217728}\n" 106 | + " ]},\n" 107 | + " {ra_kv_store, [\n" 108 | + " {port, 8080},\n" 109 | + " {nodes, [{ra_kv1, 'kv@n1'}, {ra_kv2, 'kv@n2'}, {ra_kv3, 'kv@n3'}]},\n" 110 | + " {server_reference, ra_kv2},\n" 111 | + " {release_cursor_every, 10}\n" 112 | + " ]}\n" 113 | + "].", 114 | configuration); 115 | 116 | test = new HashMap<>(); 117 | test.put(":nodes", asList("n1", "n2", "n3")); 118 | test.put(":erlang-net-ticktime", "-1"); 119 | configuration = Utils.configuration(test, "n2"); 120 | assertThat(configuration) 121 | .isEqualTo( 122 | "[\n" 123 | + " {ra, [\n" 124 | + " {data_dir, \"/tmp/ra_kv_store\"},\n" 125 | + " {wal_max_size_bytes, 134217728}\n" 126 | + " ]},\n" 127 | + " {ra_kv_store, [\n" 128 | + " {port, 8080},\n" 129 | + " {nodes, [{ra_kv1, 'kv@n1'}, {ra_kv2, 'kv@n2'}, {ra_kv3, 'kv@n3'}]},\n" 130 | + " {server_reference, ra_kv2},\n" 131 | + " {release_cursor_every, -1}\n" 132 | + " ]}\n" 133 | + "].", 134 | configuration); 135 | 136 | test = new HashMap<>(); 137 | test.put(":nodes", asList("n1", "n2", "n3")); 138 | test.put(":erlang-net-ticktime", "15"); 139 | configuration = Utils.configuration(test, "n2"); 140 | assertThat(configuration) 141 | .isEqualTo( 142 | "[\n" 143 | + "{kernel, [{net_ticktime, 15}]},\n" 144 | + " {ra, [\n" 145 | + " {data_dir, \"/tmp/ra_kv_store\"},\n" 146 | + " {wal_max_size_bytes, 134217728}\n" 147 | + " ]},\n" 148 | + " {ra_kv_store, [\n" 149 | + " {port, 8080},\n" 150 | + " {nodes, [{ra_kv1, 'kv@n1'}, {ra_kv2, 'kv@n2'}, {ra_kv3, 'kv@n3'}]},\n" 151 | + " {server_reference, ra_kv2},\n" 152 | + " {release_cursor_every, -1}\n" 153 | + " ]}\n" 154 | + "].", 155 | configuration); 156 | 157 | test = new HashMap<>(); 158 | test.put(":nodes", asList("n1", "n2", "n3")); 159 | test.put(":erlang-net-ticktime", "-1"); 160 | test.put(":wal-max-size-bytes", "8388608"); // 8 MB 161 | configuration = Utils.configuration(test, "n2"); 162 | assertThat(configuration) 163 | .isEqualTo( 164 | "[\n" 165 | + " {ra, [\n" 166 | + " {data_dir, \"/tmp/ra_kv_store\"},\n" 167 | + " {wal_max_size_bytes, 8388608}\n" 168 | + " ]},\n" 169 | + " {ra_kv_store, [\n" 170 | + " {port, 8080},\n" 171 | + " {nodes, [{ra_kv1, 'kv@n1'}, {ra_kv2, 'kv@n2'}, {ra_kv3, 'kv@n3'}]},\n" 172 | + " {server_reference, ra_kv2},\n" 173 | + " {release_cursor_every, -1}\n" 174 | + " ]}\n" 175 | + "].", 176 | configuration); 177 | } 178 | 179 | @Test 180 | public void configurationNetTickTime() { 181 | assertThat(erlangNetTickTime(singletonMap(":erlang-net-ticktime", "-1"))).isEmpty(); 182 | assertThat(erlangNetTickTime(singletonMap(":erlang-net-ticktime", "0"))) 183 | .isEqualTo("{kernel, [{net_ticktime, 0}]},\n"); 184 | assertThat(erlangNetTickTime(singletonMap(":erlang-net-ticktime", "120"))) 185 | .isEqualTo("{kernel, [{net_ticktime, 120}]},\n"); 186 | assertThatThrownBy(() -> erlangNetTickTime(singletonMap(":erlang-net-ticktime", "dummy"))) 187 | .isInstanceOf(NumberFormatException.class); 188 | } 189 | 190 | @Test 191 | public void raNodeId() { 192 | assertThat(Utils.raNodeId("n1", 0)).isEqualTo("{ra_kv1, 'kv@n1'}"); 193 | assertThat(Utils.raNodeId("n2", 1)).isEqualTo("{ra_kv2, 'kv@n2'}"); 194 | } 195 | 196 | @Test 197 | public void writeGet() throws Exception { 198 | assertThat(client.get(KEY)).isNull(); 199 | Utils.Response response = Utils.write(client, KEY, "23"); 200 | assertThat(response.isOk()).isTrue(); 201 | assertThat(response.getHeaders()).isNotEmpty(); 202 | assertThat(response.getHeaders().keySet().toString().equals("[term, leader, index]")).isTrue(); 203 | response = client.getWithResponse(KEY); 204 | assertThat(response.getBody()).isEqualTo("23"); 205 | assertThat(response.getHeaders()).isNotEmpty(); 206 | assertThat(response.getHeaders().keySet().toString()).isEqualTo("[term, leader, index]"); 207 | } 208 | 209 | @Test 210 | public void cas() throws Exception { 211 | Utils.write(client, KEY, "1"); 212 | assertThat(client.get(KEY)).isEqualTo("1"); 213 | Utils.Response response = client.cas(KEY, "1", "2"); 214 | assertThat(response.isOk()).isTrue(); 215 | assertThat(response.getHeaders()).isNotEmpty(); 216 | assertThat(response.getHeaders().keySet().toString()).isEqualTo("[term, leader, index]"); 217 | assertThat(client.get(KEY)).isEqualTo("2"); 218 | response = client.cas(KEY, "1", "2"); 219 | assertThat(response.getHeaders().keySet().toString()).isEqualTo("[term, leader, index]"); 220 | assertThat(response.isOk()).isFalse(); 221 | assertThat(client.get(KEY)).isEqualTo("2"); 222 | } 223 | 224 | @Test 225 | public void casWithNull() throws Exception { 226 | Utils.Response response = client.cas(KEY, "2", "1"); 227 | assertThat(response.isOk()).isFalse(); 228 | response = client.cas(KEY, "", "1"); 229 | assertThat(response.isOk()).isTrue(); 230 | assertThat(response.getHeaders()).isNotEmpty(); 231 | assertThat(response.getHeaders().keySet().toString()).isEqualTo("[term, leader, index]"); 232 | assertThat(client.get(KEY)).isEqualTo("1"); 233 | response = client.cas(KEY, "", "2"); 234 | assertThat(response.isOk()).isFalse(); 235 | assertThat(response.getHeaders()).isNotEmpty(); 236 | assertThat(response.getHeaders().keySet().toString()).isEqualTo("[term, leader, index]"); 237 | assertThat(client.get(KEY)).isEqualTo("1"); 238 | } 239 | 240 | @Test 241 | public void addToSet() throws Exception { 242 | Utils.Client client = Utils.createClient("localhost"); 243 | client.write(KEY, ""); 244 | Utils.addToSet(client, KEY, "2"); 245 | assertThat(Utils.get(client, KEY)).isEqualTo("2"); 246 | Utils.addToSet(client, KEY, "2"); 247 | assertThat(Utils.get(client, KEY)).isEqualTo("2"); 248 | Utils.addToSet(client, KEY, "3"); 249 | assertThat(Utils.get(client, KEY)).isEqualTo("2 3"); 250 | Utils.addToSet(client, KEY, "3"); 251 | assertThat(Utils.get(client, KEY)).isEqualTo("2 3"); 252 | Utils.addToSet(client, KEY, "2"); 253 | assertThat(Utils.get(client, KEY)).isEqualTo("2 3"); 254 | Utils.addToSet(client, KEY, "1"); 255 | assertThat(Utils.get(client, KEY)).isEqualTo("2 3 1"); 256 | } 257 | 258 | @Test 259 | public void getSet() throws Exception { 260 | Utils.Client client = Utils.createClient("localhost"); 261 | client.write(KEY, ""); 262 | Utils.addToSet(client, KEY, "1"); 263 | Utils.addToSet(client, KEY, "2"); 264 | Utils.addToSet(client, KEY, "3"); 265 | assertThat(client.getSet(KEY)).isEqualTo("#{1 2 3}"); 266 | } 267 | 268 | /** 269 | * Test the addToSet logic. This is a simplified and local version of the set workload in the 270 | * Jepsen tests. 271 | * 272 | * @throws Exception 273 | */ 274 | @Test 275 | public void addToSetWithCas() throws Exception { 276 | Utils.Client client = Utils.createClient("localhost"); 277 | client.write(KEY, ""); 278 | final int concurrency = 10; 279 | ExecutorService executorService = Executors.newFixedThreadPool(concurrency); 280 | List values = new CopyOnWriteArrayList<>(); 281 | IntStream.range(1, 100).forEach(i -> values.add(i + "")); 282 | Map referenceMap = new ConcurrentHashMap<>(); 283 | Random random = new Random(); 284 | long start = System.currentTimeMillis(); 285 | AtomicBoolean someResponseIsNull = new AtomicBoolean(false); 286 | while ((System.currentTimeMillis() - start) < 10_000) { 287 | IntStream.range(0, concurrency) 288 | .forEach( 289 | i -> { 290 | executorService.submit( 291 | () -> { 292 | String value = values.get(random.nextInt(values.size())); 293 | referenceMap.put(value, ""); 294 | try { 295 | Utils.Response response = client.addToSet(KEY, value); 296 | if (response == null) { 297 | someResponseIsNull.set(true); 298 | } 299 | } catch (Exception e) { 300 | e.printStackTrace(); 301 | } 302 | }); 303 | }); 304 | Thread.sleep(50L); 305 | } 306 | executorService.shutdown(); 307 | assertThat(executorService.awaitTermination(5, TimeUnit.SECONDS)).isTrue(); 308 | assertThat(someResponseIsNull).isFalse(); 309 | 310 | Set referenceSet = referenceMap.keySet(); 311 | 312 | List setInDbAsList = 313 | asList(client.get(KEY).split(" ")).stream() 314 | .map(value -> Integer.parseInt(value)) 315 | .sorted() 316 | .collect(Collectors.toList()); 317 | Set setInDb = new LinkedHashSet<>(setInDbAsList); 318 | assertThat(setInDb).hasSameSizeAs(setInDbAsList); 319 | List referenceSetAsList = 320 | referenceSet.stream() 321 | .map(value -> Integer.parseInt(value)) 322 | .sorted() 323 | .collect(Collectors.toList()); 324 | Set referenceSetWithInt = new LinkedHashSet<>(referenceSetAsList); 325 | 326 | LinkedHashSet lost = new LinkedHashSet<>(referenceSetWithInt); 327 | lost.removeAll(setInDb); 328 | assertThat(lost).describedAs("There should be no lost values").isEmpty(); 329 | assertThat(setInDb.toString()).isEqualTo(referenceSetWithInt.toString()); 330 | } 331 | } 332 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {plugins, [rebar3_format]}. 2 | 3 | {format, [ 4 | {files, ["src/*.erl", "test/*.erl"]}, 5 | {formatter, default_formatter}, 6 | {options, #{ 7 | paper => 80, 8 | ribbon => 70, 9 | inline_attributes => {when_under, 1}, 10 | inline_items => {when_under, 4} 11 | }} 12 | ]}. -------------------------------------------------------------------------------- /rel/sys.config: -------------------------------------------------------------------------------- 1 | [ 2 | {ra, [{data_dir, "/tmp/ra-release"}]} 3 | ]. -------------------------------------------------------------------------------- /rel/vm.args: -------------------------------------------------------------------------------- 1 | -name ra_kv_store@127.0.0.1 2 | -setcookie ra_kv_store 3 | -heart -------------------------------------------------------------------------------- /relx.config: -------------------------------------------------------------------------------- 1 | {release, {ra_kv_store_release, "1"}, [ra_kv_store, sasl, runtime_tools]}. 2 | {extended_start_script, true}. 3 | {sys_config, "rel/sys.config"}. 4 | {vm_args, "rel/vm.args"}. 5 | {dev_mode, false}. 6 | {include_erts, true}. 7 | -------------------------------------------------------------------------------- /src/ra_kv_store.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% https://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | %% 15 | 16 | -module(ra_kv_store). 17 | 18 | -behaviour(ra_machine). 19 | 20 | -record(state, 21 | {store = #{} :: #{term() => term()}, index :: term(), 22 | term :: term()}). 23 | 24 | -export([init/1, 25 | apply/3, 26 | write/3, 27 | read/2, 28 | cas/4]). 29 | 30 | -define(OP_TIMEOUT, 100). 31 | 32 | write(ServerReference, Key, Value) -> 33 | Cmd = {write, Key, Value}, 34 | case ra:process_command(ServerReference, Cmd, ?OP_TIMEOUT) of 35 | {ok, {Index, Term}, LeaderRaNodeId} -> 36 | {ok, {{index, Index}, {term, Term}, {leader, LeaderRaNodeId}}}; 37 | {timeout, _} -> 38 | timeout 39 | end. 40 | 41 | read(ServerReference, Key) -> 42 | case ra:consistent_query(ServerReference, 43 | fun(#state{store = Store, 44 | index = Index, 45 | term = Term}) -> 46 | {maps:get(Key, Store, undefined), Index, Term} 47 | end, 48 | ?OP_TIMEOUT) 49 | of 50 | {ok, {V, Idx, T}, {Leader, _}} -> 51 | {{read, V}, {index, Idx}, {term, T}, {leader, Leader}}; 52 | {timeout, _} -> 53 | logger:info("Read operation failed because of timeout~n"), 54 | timeout; 55 | {error, nodedown} -> 56 | logger:info("Read operation failed because node is down~n"), 57 | error; 58 | R -> 59 | logger:warning("Unexpected result for read operation: ~p~n", [R]), 60 | error 61 | end. 62 | 63 | cas(ServerReference, Key, ExpectedValue, NewValue) -> 64 | Cmd = {cas, Key, ExpectedValue, NewValue}, 65 | case ra:process_command(ServerReference, Cmd, ?OP_TIMEOUT) of 66 | {ok, {{read, ReadValue}, {index, Index}, {term, Term}}, 67 | LeaderRaNodeId} -> 68 | {ok, 69 | {{read, ReadValue}, 70 | {index, Index}, 71 | {term, Term}, 72 | {leader, LeaderRaNodeId}}}; 73 | {timeout, _} -> 74 | timeout 75 | end. 76 | 77 | init(_Config) -> 78 | #state{}. 79 | 80 | apply(#{index := Index, term := Term} = _Metadata, 81 | {write, Key, Value}, #state{store = Store0} = State0) -> 82 | Store1 = maps:put(Key, Value, Store0), 83 | State1 = 84 | State0#state{store = Store1, 85 | index = Index, 86 | term = Term}, 87 | SideEffects = side_effects(Index, State1), 88 | %% return the index and term here as a result 89 | {State1, {Index, Term}, SideEffects}; 90 | apply(#{index := Index, term := Term} = _Metadata, 91 | {cas, Key, ExpectedValue, NewValue}, 92 | #state{store = Store0} = State0) -> 93 | {Store1, ReadValue} = 94 | case maps:get(Key, Store0, undefined) of 95 | ExpectedValue -> 96 | {maps:put(Key, NewValue, Store0), ExpectedValue}; 97 | ValueInStore -> 98 | {Store0, ValueInStore} 99 | end, 100 | State1 = 101 | State0#state{store = Store1, 102 | index = Index, 103 | term = Term}, 104 | SideEffects = side_effects(Index, State1), 105 | {State1, {{read, ReadValue}, {index, Index}, {term, Term}}, 106 | SideEffects}. 107 | 108 | side_effects(RaftIndex, MachineState) -> 109 | case application:get_env(ra_kv_store, release_cursor_every) of 110 | undefined -> 111 | []; 112 | {ok, NegativeOrZero} when NegativeOrZero =< 0 -> 113 | []; 114 | {ok, Every} -> 115 | case release_cursor(RaftIndex, Every) of 116 | release_cursor -> 117 | [{release_cursor, RaftIndex, MachineState}]; 118 | _ -> 119 | [] 120 | end 121 | end. 122 | 123 | release_cursor(Index, Every) -> 124 | case Index rem Every of 125 | 0 -> 126 | release_cursor; 127 | _ -> 128 | do_not_release_cursor 129 | end. 130 | -------------------------------------------------------------------------------- /src/ra_kv_store_app.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% https://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | %% 15 | 16 | -module(ra_kv_store_app). 17 | 18 | -behaviour(application). 19 | 20 | -export([start/2, 21 | connect_nodes/1, 22 | connect_node/1]). 23 | -export([stop/1]). 24 | 25 | wait_for_nodes([]) -> 26 | logger:info("All erlang nodes connected~n", []), 27 | ok; 28 | wait_for_nodes([Node | Rem] = AllNodes) -> 29 | case net_kernel:connect_node(Node) of 30 | true -> 31 | %% we're connected, great 32 | wait_for_nodes(Rem); 33 | false -> 34 | logger:info("Could not connect ~w. Sleeping...~n", [Node]), 35 | %% we could not connect, sleep a bit and recurse 36 | timer:sleep(1000), 37 | wait_for_nodes(AllNodes) 38 | end. 39 | 40 | start(_Type, _Args) -> 41 | {ok, Servers} = application:get_env(ra_kv_store, nodes), 42 | ServerName = element(1, hd(Servers)), 43 | Nodes = [N || {_, N} <- Servers], 44 | {ok, ServerReference} = 45 | application:get_env(ra_kv_store, server_reference), 46 | logger:set_primary_config(level, all), 47 | logger:add_handler_filter(default, ra_domain, 48 | {fun logger_filters:domain/2, 49 | {log, equal, [ra]}}), 50 | ClusterId = <<"ra_kv_store">>, 51 | Config = #{}, 52 | Machine = {module, ra_kv_store, Config}, 53 | {ok, _} = application:ensure_all_started(ra), 54 | SysCfg = #{server_recovery_strategy => registered}, 55 | Cfg = maps:merge(SysCfg, ra_system:default_config()), 56 | {ok, _} = ra_system:start(Cfg), 57 | [N | _] = lists:usort(Nodes), 58 | case N == node() of 59 | true -> 60 | %% wait for all nodes to come online 61 | ok = wait_for_nodes(Nodes), 62 | %% only the smallest node declares a cluster 63 | timer:sleep(2000), 64 | MaybePid = whereis(ServerName), 65 | case ra_directory:uid_of(default, ServerName) of 66 | undefined -> 67 | {ok, Started, Failed} = 68 | ra:start_cluster(default, ClusterId, Machine, Servers), 69 | case length(Started) == length(Servers) of 70 | true -> 71 | %% all started 72 | ok; 73 | false -> 74 | logger:info("RA cluster failures ~w", [Failed]), 75 | ok 76 | end; 77 | _Uid when is_pid(MaybePid) -> 78 | %% already started 79 | ok; 80 | _ -> 81 | %% registered but needs restart 82 | ok = ra:restart_server(default, {ServerName, node()}) 83 | end; 84 | false -> 85 | ok 86 | end, 87 | 88 | % to make sure nodes are always connected 89 | {ok, ReconnectInterval} = 90 | application:get_env(ra_kv_store, node_reconnection_interval), 91 | {ok, _} = 92 | timer:apply_interval(ReconnectInterval, 93 | ?MODULE, 94 | connect_nodes, 95 | [Servers]), 96 | 97 | Dispatch = 98 | cowboy_router:compile([{'_', 99 | [{"/:key", ra_kv_store_handler, 100 | [{server_reference, ServerReference}]}]}]), 101 | 102 | {ok, Port} = application:get_env(ra_kv_store, port), 103 | 104 | {ok, _} = 105 | cowboy:start_clear(kv_store_http_listener, [{port, Port}], 106 | #{env => #{dispatch => Dispatch}}), 107 | ra_kv_store_sup:start_link(). 108 | 109 | stop(_State) -> 110 | ok. 111 | 112 | connect_nodes(Nodes) -> 113 | logger:info("Reconnecting nodes ~p~n", [Nodes]), 114 | lists:foreach(fun connect_node/1, Nodes). 115 | 116 | connect_node({_, Node}) -> 117 | net_kernel:connect_node(Node). 118 | -------------------------------------------------------------------------------- /src/ra_kv_store_handler.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% https://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | %% 15 | 16 | -module(ra_kv_store_handler). 17 | 18 | -behavior(cowboy_handler). 19 | 20 | -export([init/2]). 21 | 22 | -define(READ_BODY_OPTIONS, #{length => 640000}). 23 | 24 | init(Req0 = #{method := <<"GET">>}, State) -> 25 | ServerReference = proplists:get_value(server_reference, State), 26 | Key = cowboy_req:binding(key, Req0), 27 | Value = ra_kv_store:read(ServerReference, Key), 28 | Req = case Value of 29 | timeout -> 30 | logger:info("Timeout on GET, returning 503~n"), 31 | cowboy_req:reply(503, #{}, "RA timeout", Req0); 32 | error -> 33 | logger:info("Error on GET, returning 503~n"), 34 | cowboy_req:reply(503, #{}, "RA error", Req0); 35 | {{read, undefined}, 36 | {index, Index}, 37 | {term, Term}, 38 | {leader, Leader}} -> 39 | cowboy_req:reply(404, 40 | #{"content-type" => <<"text/plain">>, 41 | "ra_index" => to_list(Index), 42 | "ra_term" => to_list(Term), 43 | "ra_leader" => to_list(Leader)}, 44 | <<"undefined">>, 45 | Req0); 46 | {{read, Read}, {index, Index}, {term, Term}, {leader, Leader}} -> 47 | cowboy_req:reply(200, 48 | #{"content-type" => <<"text/plain">>, 49 | "ra_index" => to_list(Index), 50 | "ra_term" => to_list(Term), 51 | "ra_leader" => to_list(Leader)}, 52 | Read, 53 | Req0) 54 | end, 55 | {ok, Req, State}; 56 | init(Req0 = #{method := <<"PUT">>}, State) -> 57 | ServerReference = proplists:get_value(server_reference, State), 58 | Key = cowboy_req:binding(key, Req0), 59 | {ok, KeyValues, Req1} = 60 | cowboy_req:read_urlencoded_body(Req0, ?READ_BODY_OPTIONS), 61 | Value = 62 | case proplists:get_value(<<"value">>, KeyValues) of 63 | <<"">> -> 64 | undefined; 65 | NotNullValue -> 66 | NotNullValue 67 | end, 68 | Expected = 69 | case proplists:get_value(<<"expected">>, KeyValues, not_present) of 70 | <<"">> -> 71 | undefined; 72 | NotNullExpectedValue -> 73 | NotNullExpectedValue 74 | end, 75 | 76 | Req = case Expected of 77 | not_present -> 78 | case ra_kv_store:write(ServerReference, Key, Value) of 79 | {ok, 80 | {{index, Index}, {term, Term}, 81 | {leader, LeaderRaNodeId}}} -> 82 | cowboy_req:reply(204, 83 | #{"ra_index" => to_list(Index), 84 | "ra_term" => to_list(Term), 85 | "ra_leader" => 86 | to_list(LeaderRaNodeId)}, 87 | Req1); 88 | timeout -> 89 | cowboy_req:reply(503, #{}, "RA timeout", Req1) 90 | end; 91 | Expected -> 92 | case ra_kv_store:cas(ServerReference, Key, Expected, Value) of 93 | {ok, 94 | {{read, Expected}, 95 | {index, Index}, 96 | {term, Term}, 97 | {leader, LeaderRaNodeId}}} -> 98 | cowboy_req:reply(204, 99 | #{"ra_index" => to_list(Index), 100 | "ra_term" => to_list(Term), 101 | "ra_leader" => 102 | to_list(LeaderRaNodeId)}, 103 | Req1); 104 | timeout -> 105 | cowboy_req:reply(503, #{}, "RA timeout", Req1); 106 | {ok, 107 | {{read, ReadValue}, 108 | {index, Index}, 109 | {term, Term}, 110 | {leader, LeaderRaNodeId}}} -> 111 | cowboy_req:reply(409, 112 | #{"ra_index" => to_list(Index), 113 | "ra_term" => to_list(Term), 114 | "ra_leader" => 115 | to_list(LeaderRaNodeId)}, 116 | to_list(ReadValue), 117 | Req1) 118 | end 119 | end, 120 | {ok, Req, State}; 121 | init(Req0, State) -> 122 | Req = cowboy_req:reply(405, #{<<"allow">> => <<"GET,PUT">>}, Req0), 123 | {ok, Req, State}. 124 | 125 | -spec to_list(Val :: integer() | list() | binary() | atom()) -> 126 | list(). 127 | to_list(Val) when is_list(Val) -> 128 | Val; 129 | to_list(Val) when is_atom(Val) -> 130 | atom_to_list(Val); 131 | to_list(Val) when is_binary(Val) -> 132 | binary_to_list(Val); 133 | to_list(Val) when is_integer(Val) -> 134 | integer_to_list(Val); 135 | to_list({Leader, _}) when is_atom(Leader) -> 136 | atom_to_list(Leader); 137 | to_list(Val) -> 138 | lists:flatten( 139 | io_lib:format("~w", [Val])). 140 | -------------------------------------------------------------------------------- /src/ra_kv_store_sup.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% https://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | %% 15 | 16 | -module(ra_kv_store_sup). 17 | 18 | -behaviour(supervisor). 19 | 20 | -export([start_link/0]). 21 | -export([init/1]). 22 | 23 | start_link() -> 24 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 25 | 26 | init([]) -> 27 | Procs = [], 28 | {ok, {{one_for_one, 1, 5}, Procs}}. 29 | -------------------------------------------------------------------------------- /test/http_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% https://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | %% 15 | 16 | -module(http_SUITE). 17 | 18 | -include_lib("common_test/include/ct.hrl"). 19 | 20 | -compile(export_all). 21 | 22 | all() -> 23 | [http_handler]. 24 | 25 | group() -> 26 | []. 27 | 28 | %% ------------------------------------------------------------------- 29 | %% Testsuite setup/teardown. 30 | %% ------------------------------------------------------------------- 31 | 32 | init_per_suite(Config) -> 33 | application:load(ra), 34 | logger:set_primary_config(level, all), 35 | WorkDirectory = proplists:get_value(priv_dir, Config), 36 | ok = 37 | application:set_env(ra, data_dir, filename:join(WorkDirectory, "ra")), 38 | Config. 39 | 40 | end_per_suite(Config) -> 41 | application:stop(ra), 42 | Config. 43 | 44 | %% ------------------------------------------------------------------- 45 | %% Testcases. 46 | %% ------------------------------------------------------------------- 47 | 48 | http_handler(_Config) -> 49 | Nodes = [{ra_kv1, node()}, {ra_kv2r, node()}, {ra_kv3, node()}], 50 | ClusterId = <<"ra_kv_store_http_handler">>, 51 | Config = #{}, 52 | Machine = {module, ra_kv_store, Config}, 53 | application:ensure_all_started(ra), 54 | ra_system:start_default(), 55 | {ok, _, _} = ra:start_cluster(default, ClusterId, Machine, Nodes), 56 | {ok, _, {Leader, _}} = ra:members(hd(Nodes)), 57 | 58 | application:ensure_all_started(cowboy), 59 | 60 | Dispatch = 61 | cowboy_router:compile([{'_', 62 | [{"/:key", ra_kv_store_handler, 63 | [{server_reference, Leader}]}]}]), 64 | 65 | {ok, Socket} = gen_tcp:listen(0, []), 66 | {ok, Port} = inet:port(Socket), 67 | gen_tcp:close(Socket), 68 | 69 | {ok, _} = 70 | cowboy:start_clear(kv_store_http_listener, [{port, Port}], 71 | #{env => #{dispatch => Dispatch}}), 72 | 73 | ok = inets:start(), 74 | 75 | Url = io_lib:format("http://localhost:~p/~p", [Port, 1]), 76 | 77 | {ok, {{_, 404, _}, _, _}} = httpc:request(get, {Url, []}, [], []), 78 | 79 | {ok, {{_, 204, _}, Headers1, _}} = 80 | httpc:request(put, {Url, [], [], "value=1"}, [], []), 81 | 82 | LeaderAsList = atom_to_list(Leader), 83 | 84 | "2" = proplists:get_value("ra_index", Headers1), 85 | "1" = proplists:get_value("ra_term", Headers1), 86 | LeaderAsList = proplists:get_value("ra_leader", Headers1), 87 | 88 | {ok, {{_, 200, _}, HeadersGet, "1"}} = 89 | httpc:request(get, {Url, []}, [], []), 90 | 91 | "2" = proplists:get_value("ra_index", HeadersGet), 92 | "1" = proplists:get_value("ra_term", HeadersGet), 93 | LeaderAsList = proplists:get_value("ra_leader", HeadersGet), 94 | 95 | {ok, {{_, 204, _}, Headers2, _}} = 96 | httpc:request(put, {Url, [], [], "value=2&expected=1"}, [], []), 97 | 98 | "3" = proplists:get_value("ra_index", Headers2), 99 | "1" = proplists:get_value("ra_term", Headers2), 100 | LeaderAsList = proplists:get_value("ra_leader", Headers2), 101 | 102 | {ok, {{_, 409, _}, Headers3, "2"}} = 103 | httpc:request(put, {Url, [], [], "value=99&expected=1"}, [], []), 104 | 105 | "4" = proplists:get_value("ra_index", Headers3), 106 | "1" = proplists:get_value("ra_term", Headers3), 107 | LeaderAsList = proplists:get_value("ra_leader", Headers3), 108 | 109 | {ok, {{_, 204, _}, _, _}} = 110 | httpc:request(put, {Url, [], [], "value=3&expected=2"}, [], []), 111 | 112 | %% test with empty values meant to reset a key 113 | {ok, {{_, 204, _}, _, _}} = 114 | httpc:request(put, {Url, [], [], "value=&expected=3"}, [], []), 115 | 116 | {ok, {{_, 404, _}, _, _}} = httpc:request(get, {Url, []}, [], []), 117 | 118 | {ok, {{_, 204, _}, _, _}} = 119 | httpc:request(put, {Url, [], [], "value=1&expected="}, [], []), 120 | 121 | {ok, {{_, 200, _}, _, "1"}} = httpc:request(get, {Url, []}, [], []), 122 | 123 | %% ensure expected value in CAS can be empty 124 | {ok, {{_, 204, _}, _, _}} = 125 | httpc:request(put, {Url, [], [], "value="}, [], []), 126 | 127 | {ok, {{_, 404, _}, _, _}} = httpc:request(get, {Url, []}, [], []), 128 | 129 | {ok, {{_, 409, _}, _, _}} = 130 | httpc:request(put, {Url, [], [], "value=1&expected=2"}, [], []), 131 | 132 | {ok, {{_, 204, _}, _, _}} = 133 | httpc:request(put, {Url, [], [], "value=1&expected="}, [], []), 134 | 135 | {ok, {{_, 200, _}, _, "1"}} = httpc:request(get, {Url, []}, [], []), 136 | 137 | {ok, {{_, 409, _}, _, _}} = 138 | httpc:request(put, {Url, [], [], "value=2&expected="}, [], []), 139 | 140 | {ok, {{_, 200, _}, _, "1"}} = httpc:request(get, {Url, []}, [], []), 141 | 142 | ok. 143 | -------------------------------------------------------------------------------- /test/replay_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% https://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | %% 15 | 16 | -module(replay_SUITE). 17 | 18 | -include_lib("common_test/include/ct.hrl"). 19 | 20 | -compile(export_all). 21 | 22 | all() -> 23 | [replay]. 24 | 25 | group() -> 26 | []. 27 | 28 | %% ------------------------------------------------------------------- 29 | %% Testsuite setup/teardown. 30 | %% ------------------------------------------------------------------- 31 | 32 | init_per_suite(Config) -> 33 | application:load(ra), 34 | WorkDirectory = proplists:get_value(priv_dir, Config), 35 | ok = 36 | application:set_env(ra, data_dir, filename:join(WorkDirectory, "ra")), 37 | ok = application:set_env(ra_kv_store, release_cursor_every, 1), 38 | Config. 39 | 40 | end_per_suite(Config) -> 41 | application:stop(ra), 42 | Config. 43 | 44 | %% ------------------------------------------------------------------- 45 | %% Testcases. 46 | %% ------------------------------------------------------------------- 47 | 48 | replay(_Config) -> 49 | Nodes = [{ra_kv1, node()}, {ra_kv2, node()}, {ra_kv3, node()}], 50 | ClusterId = <<"ra_kv_store">>, 51 | Config = #{}, 52 | Machine = {module, ra_kv_store, Config}, 53 | application:ensure_all_started(ra), 54 | ra_system:start_default(), 55 | {ok, _, _} = ra:start_cluster(default, ClusterId, Machine, Nodes), 56 | {ok, _} = ra_kv_store:write(ra_kv1, 1, 2), 57 | {ok, {{read, 2}, _, _, _}} = ra_kv_store:cas(ra_kv1, 1, 2, 4), 58 | {ok, {{read, 4}, _, _, _}} = ra_kv_store:cas(ra_kv1, 1, 3, 6), 59 | 60 | {ok, {{read, undefined}, _, _, _}} = 61 | ra_kv_store:cas(ra_kv1, 2, undefined, 1), 62 | {ok, {{read, 1}, _, _, _}} = ra_kv_store:cas(ra_kv1, 2, undefined, 3), 63 | 64 | {ok, RaDataDir} = application:get_env(ra, data_dir), 65 | NodeDirectory = filename:join([RaDataDir, node()]), 66 | {ok, RaFiles} = file:list_dir(NodeDirectory), 67 | WalSuffix = "1.wal", 68 | [WalFilename] = lists:filter(fun(F) -> string:find(F, WalSuffix) =:= WalSuffix end, RaFiles), 69 | 70 | WalFile = filename:join([NodeDirectory, WalFilename]), 71 | 72 | InitialState = ra_kv_store:init(Config), 73 | {_, Store, _, _} = 74 | ra_dbg:replay_log(WalFile, ra_kv_store, InitialState), 75 | Store = #{1 => 4, 2 => 1}, 76 | ok. 77 | -------------------------------------------------------------------------------- /test/store_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% https://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | %% 15 | 16 | -module(store_SUITE). 17 | 18 | -include_lib("common_test/include/ct.hrl"). 19 | 20 | -compile(export_all). 21 | 22 | all() -> 23 | [kv_store]. 24 | 25 | group() -> 26 | []. 27 | 28 | %% ------------------------------------------------------------------- 29 | %% Testsuite setup/teardown. 30 | %% ------------------------------------------------------------------- 31 | 32 | init_per_suite(Config) -> 33 | application:load(ra), 34 | WorkDirectory = proplists:get_value(priv_dir, Config), 35 | ok = 36 | application:set_env(ra, data_dir, filename:join(WorkDirectory, "ra")), 37 | ok = application:set_env(ra_kv_store, release_cursor_every, 1), 38 | Config. 39 | 40 | end_per_suite(Config) -> 41 | application:stop(ra), 42 | Config. 43 | 44 | %% ------------------------------------------------------------------- 45 | %% Testcases. 46 | %% ------------------------------------------------------------------- 47 | 48 | kv_store(_Config) -> 49 | Nodes = [{ra_kv1, node()}, {ra_kv2, node()}, {ra_kv3, node()}], 50 | ClusterId = <<"ra_kv_store">>, 51 | Config = #{}, 52 | Machine = {module, ra_kv_store, Config}, 53 | application:ensure_all_started(ra), 54 | ra_system:start_default(), 55 | {ok, _, _} = ra:start_cluster(default, ClusterId, Machine, Nodes), 56 | {ok, _} = ra_kv_store:write(ra_kv1, 1, 2), 57 | {{read, 2}, _, _, _} = ra_kv_store:read(ra_kv1, 1), 58 | {ok, {{read, 2}, _, _, _}} = ra_kv_store:cas(ra_kv1, 1, 2, 4), 59 | {ok, {{read, 4}, _, _, _}} = ra_kv_store:cas(ra_kv1, 1, 3, 6), 60 | {{read, 4}, _, _, _} = ra_kv_store:read(ra_kv1, 1), 61 | 62 | {ok, {{read, undefined}, _, _, _}} = 63 | ra_kv_store:cas(ra_kv1, 2, undefined, 1), 64 | {{read, 1}, _, _, _} = ra_kv_store:read(ra_kv1, 2), 65 | {ok, {{read, 1}, _, _, _}} = ra_kv_store:cas(ra_kv1, 2, undefined, 3), 66 | {{read, 1}, _, _, _} = ra_kv_store:read(ra_kv1, 2), 67 | ok. 68 | --------------------------------------------------------------------------------