├── .github └── workflows │ ├── push-rockspec.yml │ └── test.yml ├── .luacheckrc ├── .luacov ├── Dockerfile.build ├── Dockerfile.test ├── Makefile ├── README.md ├── config-dev-1.rockspec ├── config-scm-1.rockspec ├── config.lua ├── config └── etcd.lua ├── override-config-dev-1.rockspec ├── rockspecs ├── config-scm-1.rockspec ├── config-scm-2.rockspec ├── config-scm-3.rockspec ├── config-scm-4.rockspec └── config-scm-5.rockspec ├── run_test_in_docker.sh ├── spec ├── 01_single_test.lua ├── 02_cluster_master_test.lua ├── helper.lua └── mock │ └── single │ ├── conf.lua │ └── init.lua ├── test ├── Dockerfile ├── Makefile ├── app │ ├── conf.lua │ └── init.lua ├── docker-compose.yml ├── instance.etcd.yaml └── net │ ├── Makefile │ └── README.md └── test_peek.lua /.github/workflows/push-rockspec.yml: -------------------------------------------------------------------------------- 1 | name: Create and push rockspec for moonlibs/config 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | env: 9 | ROCK_NAME: config 10 | 11 | jobs: 12 | pack-and-push-tagged-rockspec: 13 | runs-on: ubuntu-latest 14 | if: startsWith(github.ref, 'refs/tags') 15 | steps: 16 | - uses: actions/checkout@master 17 | - uses: tarantool/setup-tarantool@master 18 | with: 19 | tarantool-version: '2.10.7' 20 | 21 | # https://stackoverflow.com/questions/58177786/get-the-current-pushed-tag-in-github-actions 22 | - name: Set env 23 | run: echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV 24 | 25 | - run: tarantoolctl rocks new_version --tag=${{ env.TAG }} rockspecs/config-scm-5.rockspec ${{ env.TAG }} "git+https://github.com/${{ github.repository }}.git" 26 | - run: tarantoolctl rocks --server https://moonlibs.github.io/rocks install ${{ env.ROCK_NAME }}-${{ env.TAG }}-1.rockspec 27 | - run: tarantoolctl rocks pack ${{ env.ROCK_NAME }}-${{ env.TAG }}-1.rockspec 28 | 29 | - run: tarantoolctl rocks new_version --tag=${{ env.TAG }} override-config-dev-1.rockspec ${{ env.TAG }} "git+https://github.com/${{ github.repository }}.git" 30 | - run: tarantoolctl rocks --server https://moonlibs.github.io/rocks install override-config-${{ env.TAG }}-1.rockspec 31 | - run: tarantoolctl rocks pack override-config-${{ env.TAG }}-1.rockspec 32 | # Install native lua with luarocks 33 | - uses: leafo/gh-actions-lua@v10 34 | with: 35 | luaVersion: "5.1.5" 36 | - uses: leafo/gh-actions-luarocks@v4 37 | with: 38 | luarocksVersion: "3.8.0" 39 | - uses: unfor19/install-aws-cli-action@v1.0.3 40 | - run: mkdir .build && cp ${{env.ROCK_NAME}}-dev-1.rockspec ${{env.ROCK_NAME}}-${{env.TAG}}-1.rockspec ${{env.ROCK_NAME}}-${{env.TAG}}-1.src.rock .build/ 41 | - name: rebuild and publish s3 luarocks server 42 | env: 43 | AWS_ACCESS_KEY_ID: ${{ secrets.MOONLIBS_S3_ACCESS_KEY_ID }} 44 | AWS_SECRET_ACCESS_KEY: ${{ secrets.MOONLIBS_S3_SECRET_KEY }} 45 | AWS_EC2_METADATA_DISABLED: true 46 | run: | 47 | cd .build && aws s3 sync s3://moonlibs/ ./ && luarocks-admin make_manifest . && aws s3 sync --acl public-read ./ s3://moonlibs/; 48 | - uses: "marvinpinto/action-automatic-releases@latest" 49 | with: 50 | repo_token: "${{ secrets.GITHUB_TOKEN }}" 51 | prerelease: false 52 | files: | 53 | README.md 54 | ${{env.ROCK_NAME}}-dev-1.rockspec 55 | ${{env.ROCK_NAME}}-${{env.TAG}}-1.rockspec 56 | ${{env.ROCK_NAME}}-${{env.TAG}}-1.src.rock 57 | override-${{env.ROCK_NAME}}-dev-1.rockspec 58 | override-${{env.ROCK_NAME}}-${{env.TAG}}-1.rockspec 59 | override-${{env.ROCK_NAME}}-${{env.TAG}}-1.src.rock 60 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run unit tests 2 | 3 | on: 4 | push: 5 | 6 | env: 7 | ROCK_NAME: config 8 | 9 | jobs: 10 | run-luacheck: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@master 14 | - uses: tarantool/setup-tarantool@master 15 | with: 16 | tarantool-version: '2.10.7' 17 | - name: install luacheck 0.26.0 18 | run: tarantoolctl rocks install luacheck 0.26.0 19 | - name: run luacheck 20 | run: .rocks/bin/luacheck . 21 | run-unit-tests: 22 | runs-on: ubuntu-latest 23 | strategy: 24 | fail-fast: false 25 | matrix: 26 | version: ["1.10.15", "2.8.4", "2.10.6", "2.10.7-gc64-amd64", "2.11.0", "2.11.1"] 27 | steps: 28 | - uses: actions/checkout@master 29 | - uses: docker/setup-buildx-action@v2 30 | - name: run test suite for ${{matrix.version}} 31 | run: make test-${{matrix.version}} 32 | - name: rename luacov.stats.out 33 | run: mv luacov.stats.out luacov.stats.out-${{matrix.version}} 34 | - uses: actions/upload-artifact@master 35 | with: 36 | name: luacov.stats.out-${{matrix.version}} 37 | path: luacov.stats.out-${{matrix.version}} 38 | run-coverage-report: 39 | runs-on: ubuntu-latest 40 | needs: ["run-unit-tests"] 41 | steps: 42 | - uses: actions/checkout@master 43 | - uses: tarantool/setup-tarantool@master 44 | with: 45 | tarantool-version: '2.10.7' 46 | - name: install luacov-coveralls 0.2.3 47 | run: tarantoolctl rocks install --server=https://luarocks.org luacov-coveralls 0.2.3 48 | - name: install luacov-console 1.2.0 49 | run: tarantoolctl rocks --server http://moonlibs.github.io/rocks install luacov-console 1.2.0 50 | - name: Download run artifacts 51 | uses: actions/download-artifact@v4 52 | with: 53 | pattern: luacov.stats.out-* 54 | merge-multiple: true 55 | - name: debug 56 | run: ls -la . 57 | - name: merge luacov.stats.out 58 | run: cat luacov.stats.out-* | >luacov.stats.out tarantool -e 'm={} for k in io.lines() do local vs=io.read():split(" ") vs[#vs]=nil local r = m[k] if r then for i, v in pairs(vs) do r[i]=r[i]+v end else m[k]=vs end end; for k, v in pairs(m) do print(k) print(table.concat(v, " ")) end' 59 | - name: prepare coverage report 60 | run: .rocks/bin/luacov-console . && .rocks/bin/luacov-console -s 61 | - name: publish coveralls report 62 | env: 63 | COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} 64 | run: .rocks/bin/luacov-coveralls -v 65 | -------------------------------------------------------------------------------- /.luacheckrc: -------------------------------------------------------------------------------- 1 | std = "tarantool" 2 | max_line_length = 200 3 | codes = true 4 | include_files = {"config.lua", "config/"} 5 | read_globals = {"config"} 6 | ignore = {"212"} 7 | -------------------------------------------------------------------------------- /.luacov: -------------------------------------------------------------------------------- 1 | runreport = false 2 | deletestats = false 3 | 4 | exclude = { 5 | "spec/", 6 | "test/", 7 | "test_peek", 8 | "%.rocks/", 9 | "builtin/", 10 | } 11 | 12 | pathcorrect = { 13 | { "^/source/config/", "" }, 14 | } 15 | 16 | coveralls = { 17 | root = "/", 18 | debug = true, 19 | pathcorrect = { 20 | { "^/home/runner/work/config/config/", "" }, 21 | { "^/source/config", "" }, 22 | }, 23 | } -------------------------------------------------------------------------------- /Dockerfile.build: -------------------------------------------------------------------------------- 1 | FROM tarantool/tarantool:2.10 as builder 2 | RUN apk add -u git cmake make gcc musl-dev curl wget 3 | WORKDIR /root 4 | RUN tarantoolctl rocks install luatest scm-1 5 | RUN tarantoolctl rocks install luacov-console 1.1.0 6 | RUN tarantoolctl rocks --server https://moonlibs.github.io/rocks install package-reload scm-1 7 | -------------------------------------------------------------------------------- /Dockerfile.test: -------------------------------------------------------------------------------- 1 | ARG IMAGE=1.10.14 2 | FROM moonlibs-config-test-builder:latest as builder 3 | 4 | FROM tarantool/tarantool:${IMAGE} 5 | 6 | WORKDIR /root 7 | COPY --from=builder /root/.rocks /root/.rocks 8 | WORKDIR /opt/tarantool 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY := all test 2 | 3 | run-etcd: 4 | make -C test run-compose-etcd 5 | 6 | config-test-builder: 7 | docker build -t moonlibs-config-test-builder:latest -f Dockerfile.build . 8 | 9 | config-test-%: config-test-builder run-etcd 10 | docker build -t $(@) --build-arg IMAGE=$(subst config-test-,,$@) -f Dockerfile.test . 11 | 12 | test-%: config-test-% 13 | docker run --rm --name $(<) \ 14 | --net tt_net \ 15 | -e TT_ETCD_ENDPOINTS="http://etcd0:2379,http://etcd1:2379,http://etcd2:2379" \ 16 | -v $$(pwd):/source/config \ 17 | -v $$(pwd)/data:/tmp/ \ 18 | --workdir /source/config \ 19 | --entrypoint '' \ 20 | $(<) \ 21 | ./run_test_in_docker.sh 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Config 2 | 3 | Module to make proper initialization and configuration of tarantool instance. 4 | 5 | It can be used with or without ETCD. 6 | 7 | Only ETCD APIv2 now supported. 8 | 9 | - [Config](#config) 10 | - [Status](#status) 11 | - [Installation](#installation) 12 | - [Configuration](#configuration) 13 | - [Example of `conf.lua`](#example-of-conflua) 14 | - [Usage in `init.lua`](#usage-in-initlua) 15 | - [Usage](#usage) 16 | - [Topologies](#topologies) 17 | - [Single-shard topology](#single-shard-topology) 18 | - [Example of `init.lua`](#example-of-initlua) 19 | - [Example of `/etc/userdb/conf.lua`](#example-of-etcuserdbconflua) 20 | - [Example of ETCD configuration (`etcd.cluster.master`)](#example-of-etcd-configuration-etcdclustermaster) 21 | - [Configuration precedence](#configuration-precedence) 22 | - [Fencing configuration](#fencing-configuration) 23 | - [Fencing algorithm](#fencing-algorithm) 24 | - [Operations during disaster](#operations-during-disaster) 25 | - [Multi-proxy topology (etcd.instance.single)](#multi-proxy-topology-etcdinstancesingle) 26 | - [Example of proxy `init.lua`](#example-of-proxy-initlua) 27 | - [Example of `/etc/proxy/conf.lua`](#example-of-etcproxyconflua) 28 | - [Example of ETCD configuration (`etcd.instance.single`)](#example-of-etcd-configuration-etcdinstancesingle) 29 | - [Multi-shard topology for custom sharding (`etcd.cluster.master`)](#multi-shard-topology-for-custom-sharding-etcdclustermaster) 30 | - [Multi-shard topology for vshard-based applications (`etcd.cluster.vshard`)](#multi-shard-topology-for-vshard-based-applications-etcdclustervshard) 31 | - [Example of ETCD configuration for vshard-based applications (`etcd.cluster.vshard`)](#example-of-etcd-configuration-for-vshard-based-applications-etcdclustervshard) 32 | - [Example of vshard-based init.lua (`etcd.cluster.vshard`)](#example-of-vshard-based-initlua-etcdclustervshard) 33 | - [VShard Maintenance](#vshard-maintenance) 34 | 35 | ## Status 36 | 37 | Ready for production use. 38 | 39 | Latest stable release: `config 0.7.2`. 40 | 41 | ## Installation 42 | 43 | ```bash 44 | tarantoolctl rocks --server=https://moonlibs.org install config 0.7.2 45 | ``` 46 | 47 | Starting with Tarantool 2.10.0 you may add configuration of moonlibs.org into `config-5.1.lua` 48 | 49 | ```bash 50 | $ cat .rocks/config-5.1.lua 51 | rocks_servers = { 52 | "https://moonlibs.org", 53 | "http://moonlibs.github.io/rocks", 54 | "http://rocks.tarantool.org/", 55 | "http://luarocks.org/repositories/rocks" 56 | } 57 | ``` 58 | 59 | ## Configuration 60 | 61 | To configure tarantool instance you must deploy `conf.lua` file. 62 | 63 | ### Example of `conf.lua` 64 | 65 | Typically conf.lua should be located in `/etc//conf.lua`. 66 | 67 | ```lua 68 | assert(instance_name, "instance_name must be defined") 69 | etcd = { 70 | instance_name = instance_name, 71 | prefix = '/etcd/path/to/application/etcd', 72 | endpoints = { 73 | "https://etcd1:2379", 74 | "https://etcd2:2379", 75 | "https://etcd3:2379", 76 | }, 77 | timeout = 3, 78 | boolean_auto = true, 79 | print_config = true, 80 | login = 'etcd-username', 81 | password = 'etcd-password', 82 | } 83 | 84 | -- This options will be passed as is to box.cfg 85 | box = { 86 | pid_file = '/var/run/tarantool/'..instance_name..'.pid', 87 | memtx_dir = '/var/lib/tarantool/snaps/' .. instance_name, 88 | wal_dir = '/var/lib/tarantool/xlogs/' .. instance_name, 89 | log_nonblock = false, 90 | } 91 | 92 | --- You may hardcode options for your application in `app` section 93 | app = { 94 | 95 | } 96 | ``` 97 | 98 | ### Usage in `init.lua` 99 | 100 | ```lua 101 | 102 | local instance_name = os.getenv('TT_INSTANCE_NAME') 103 | 104 | require 'config' { 105 | mkdir = true, 106 | instance_name = instance_name, 107 | file = '/etc//conf.lua', 108 | master_selection_policy = 'etcd.cluster.master', 109 | } 110 | 111 | print("Tarantool bootstrapped") 112 | ``` 113 | 114 | ## Usage 115 | 116 | Module config is used both for bootstrap and configuration of your Tarantool application. 117 | 118 | In application you may access config options using following syntax 119 | 120 | ```lua 121 | local DEFAULT_TIMEOUT = 3 122 | 123 | --- If app/http/timeout is defined in config (ETCD or conf.lua) then it will be returned 124 | --- otherwise value of DEFAULT_TIMEUOT will be returned 125 | local http_timeout = config.get('app.http.timeout', DEFAULT_TIMEOUT) 126 | 127 | --- If app/is_enabled is not defined then `nil` will be returned. 128 | local is_enabled = config.get('app.is_enabled') 129 | ``` 130 | 131 | ## Topologies 132 | 133 | `moonlibs/config` supports different types of Tarantool topologies. 134 | 135 | All of them make sence when application is configured using ETCD. 136 | 137 | To distinguish application topology option `master_selection_policy` is used. 138 | 139 | ### Single-shard topology 140 | 141 | In most cases you need single shard topology. It means, that your application has single master and many replicas. 142 | 143 | Shard will be configured with full-mesh topology. Read more about full-mesh topology on [Tarantool website](https://www.tarantool.io/en/doc/latest/concepts/replication/repl_architecture/). 144 | 145 | Each instance of application must have unique name. For example: 146 | 147 | - `userdb_001` 148 | - `userdb_002` 149 | - `userdb_003` 150 | 151 | Typically instance name **should not** contain `master` or `replica` word in it. 152 | 153 | #### Example of `init.lua` 154 | 155 | ```lua 156 | --- variable instance_name must be derived somehow for each tarantool instance 157 | --- For example from name of the file. or from environment variable 158 | require 'config' { 159 | mkdir = true, 160 | instance_name = instance_name, 161 | file = '/etc/userdb/conf.lua', 162 | master_selection_policy = 'etcd.cluster.master', 163 | } 164 | ``` 165 | 166 | #### Example of `/etc/userdb/conf.lua` 167 | 168 | ```lua 169 | assert(instance_name, "instance_name must be defined") 170 | etcd = { 171 | instance_name = instance_name, 172 | prefix = '/tarantool/userdb', 173 | endpoints = { 174 | "https://etcd1:2379", 175 | "https://etcd2:2379", 176 | "https://etcd3:2379", 177 | }, 178 | timeout = 3, 179 | boolean_auto = true, 180 | print_config = true, 181 | } 182 | 183 | -- This options will be passed as is to box.cfg 184 | box = { 185 | pid_file = '/var/run/tarantool/'..instance_name..'.pid', 186 | memtx_dir = '/var/lib/tarantool/snaps/' .. instance_name, 187 | wal_dir = '/var/lib/tarantool/xlogs/' .. instance_name, 188 | log_nonblock = false, 189 | } 190 | ``` 191 | 192 | #### Example of ETCD configuration (`etcd.cluster.master`) 193 | 194 | ```yaml 195 | tarantool: 196 | userdb: 197 | clusters: 198 | userdb: 199 | master: userdb_001 200 | replicaset_uuid: 045e12d8-0001-0000-0000-000000000000 201 | common: 202 | box: 203 | log_level: 5 204 | memtx_memory: 268435456 205 | instances: 206 | userdb_001: 207 | cluster: userdb 208 | box: 209 | instance_uuid: 045e12d8-0000-0001-0000-000000000000 210 | listen: 10.0.1.11:3301 211 | userdb_002: 212 | cluster: userdb 213 | box: 214 | instance_uuid: 045e12d8-0000-0002-0000-000000000000 215 | listen: 10.0.1.12:3302 216 | userdb_003: 217 | cluster: userdb 218 | box: 219 | instance_uuid: 045e12d8-0000-0003-0000-000000000000 220 | listen: 10.0.1.13:3303 221 | ``` 222 | 223 | `/tarantool/userdb` -- is root path for application configuration 224 | 225 | `/tarantool/userdb/common` -- is common configuration for each instance of application. 226 | 227 | `/tarantool/userdb/common/box` -- is section to configure box.cfg parameters. See more on [Tarantool website](https://www.tarantool.io/en/doc/latest/reference/configuration). 228 | 229 | `/tarantool/userdb/clusters` section contains list of shards. For single-shard application it is good to single shard it as application itself. 230 | 231 | `/tarantool/userdb/instances` section contains instance-specific configuration. It must contain `/box/{listen,instance_uuid}` and `cluster` options. 232 | 233 | ##### Configuration precedence 234 | 235 | - /etc/app-name/conf.lua 236 | - ETCD:/instances/ 237 | - ETCD:/common/ 238 | - config.get default value 239 | 240 | #### Fencing configuration 241 | 242 | `etcd.cluster.master` topology supports auto fencing mechanism. 243 | 244 | Auto fencing is implemented via background fiber which waits for changes on `/clusters/` directory. 245 | 246 | `fencing_pause` must be less than `fencing_timeout`. The `fencing_timeout - fencing_pause` time slot is used to retry ETCD (at least 2 attempts with sleeps in between) to ensure that ETCD is really down. 247 | 248 | You may increase `fencing_pause` if you see that too many requests is made to ETCD. 249 | If `fencing_timeout - fencing_pause` is too small (less than `etcd/timeout`) then you may face False-Positive fencing when ETCD was slightly down. 250 | 251 | Increasing `fencing_timeout` will decrease possibility of False Positive fencing. 252 | 253 | You leader will be RW at least `fencing_timeout` if everyone else is down. 254 | 255 | There are 4 parameters to configure: 256 | 257 | | Parameter | Description | Default Value | 258 | |----------------------------------|---------------------------------------|---------------------| 259 | | `etcd/fencing_enabled` | Trigger to enable/disable fencing | `false` | 260 | | `etcd/fencing_timeout` | Fencing timeout | `10` (seconds) | 261 | | `etcd/fencing_pause` | Fencing pause | `fencing_timeout/2` | 262 | | `etcd/fencing_check_replication` | Respect replication when ETCD is down | `false` | 263 | 264 | Example of enabled fencing: 265 | 266 | ```yaml 267 | tarantool: 268 | userdb: 269 | common: 270 | etcd: 271 | fencing_enabled: true 272 | ``` 273 | 274 | Fencing also can be enabled in `conf.lua`: 275 | 276 | ```lua 277 | etcd = { 278 | endpoints = {"http://etcd1:2379", "http://etcd2:2379", "http://etcd3:2379"}, 279 | prefix = "/tarantool/userdb", 280 | timeout = 3, 281 | fencing_enabled = true, 282 | } 283 | ``` 284 | 285 | #### Fencing algorithm 286 | 287 | Fencing can be enabled only for topology `etcd.cluster.master` and only if `etcd/fencing_enabled` is `true` (default: `false`). 288 | 289 | Fencing algorithm is the following: 290 | 291 | 0. Wait until instance became `rw`. 292 | 1. Wait randomized `(fencing_timeout - fencing_pause) / 10`. 293 | 2. Recheck ETCD `/clusters/` in `fencing_pause`. 294 | 3. Depends on response: 295 | 1. [ETCD is ok] => provision self to be `rw` for next `fencing_timeout` seconds. Go to `1.` 296 | 2. [ETCD is down] => execute `box.cfg{read_only=true}` if `etcd/fencing_check_replication` is disabled. Go to `0.` 297 | 3. [ETCD has another master, and switching in progress] => do nothing. Go to `1.` 298 | 4. [ETCD has another master, and switching is not in progress] => execute `box.cfg{read_only=true}`. Go to `0.` 299 | 300 | **Note:** to request ETCD Quorum Reads are used. So it is safe to use it in split brain. 301 | 302 | ### Operations during disaster 303 | 304 | When ETCD failed to determine leader cluster can't reload it's configuration (or restart). 305 | 306 | During disaster, you must recovery ETCD elections mechanism first. 307 | 308 | If it is not possible, and you need to reload configuration or restart instance you may switch to manual configuration mode. 309 | 310 | To do that set `reduce_listing_quorum` to `true` in `conf.lua` config. 311 | 312 | After ETCD becames operation you **must** remove this option from local configuration. 313 | 314 | ```lua 315 | etcd = { 316 | reduce_listing_quorum = true, 317 | } 318 | ``` 319 | 320 | Enabling this option automatically disables `fencing`. 321 | 322 | Be extra careful to not allow Tarantool Master-Master setup. (Since ETCD may have different configuration on it's replicas). 323 | 324 | ### Multi-proxy topology (etcd.instance.single) 325 | 326 | `moonlibs/config` supports multi proxy topology. This topology is usefull when you need to have many stateless tarantool proxies or totally independent masters. 327 | 328 | Each instance **should** have unique name. For example: 329 | 330 | - proxy_001 331 | - proxy_002 332 | - proxy_003 333 | - proxy_004 334 | - proxy_005 335 | 336 | #### Example of proxy `init.lua` 337 | 338 | ```lua 339 | --- variable instance_name must be derived somehow for each tarantool instance 340 | --- For example from name of the file. or from environment variable 341 | require 'config' { 342 | mkdir = true, 343 | instance_name = instance_name, 344 | file = '/etc/proxy/conf.lua', 345 | master_selection_policy = 'etcd.instance.single', 346 | } 347 | ``` 348 | 349 | #### Example of `/etc/proxy/conf.lua` 350 | 351 | ```lua 352 | assert(instance_name, "instance_name must be defined") 353 | etcd = { 354 | instance_name = instance_name, 355 | prefix = '/tarantool/proxy', 356 | endpoints = { 357 | "https://etcd1:2379", 358 | "https://etcd2:2379", 359 | "https://etcd3:2379", 360 | }, 361 | timeout = 3, 362 | boolean_auto = true, 363 | print_config = true, 364 | } 365 | 366 | -- This options will be passed as is to box.cfg 367 | box = { 368 | pid_file = '/var/run/tarantool/'..instance_name..'.pid', 369 | memtx_dir = '/var/lib/tarantool/snaps/' .. instance_name, 370 | wal_dir = '/var/lib/tarantool/xlogs/' .. instance_name, 371 | log_nonblock = false, 372 | } 373 | ``` 374 | 375 | #### Example of ETCD configuration (`etcd.instance.single`) 376 | 377 | ```yaml 378 | tarantool: 379 | proxy: 380 | common: 381 | box: 382 | log_level: 5 383 | memtx_memory: 33554432 384 | instances: 385 | proxy_001: 386 | box: 387 | instance_uuid: 01712087-0000-0001-0000-000000000000 388 | listen: 10.0.2.12:7101 389 | proxy_002: 390 | box: 391 | instance_uuid: 01712087-0000-0002-0000-000000000000 392 | listen: 10.0.2.13:7102 393 | proxy_003: 394 | box: 395 | instance_uuid: 01712087-0000-0003-0000-000000000000 396 | listen: 10.0.2.11:7103 397 | ``` 398 | 399 | The etcd configuration is the same as `etcd.cluster.master` except that `/tarantool/proxy/clusters` is not defined. 400 | 401 | Also `/tarantool/proxy/instances//cluster` **must not** be defined. 402 | 403 | ### Multi-shard topology for custom sharding (`etcd.cluster.master`) 404 | 405 | `etcd.cluster.master` can be used for multi-shard topologies as well. 406 | 407 | Multi-shard means that application consists of several replicasets. Each replicaset has single master and several replicas. 408 | 409 | `conf.lua` and `init.lua` files remains exactly the same. But configuration of ETCD slightly changes: 410 | 411 | ```yaml 412 | tarantool: 413 | notifications: 414 | clusters: 415 | notifications_002: 416 | master: notifications_002_01 417 | replicaset_uuid: 11079f9c-0002-0000-0000-000000000000 418 | notifications_001: 419 | master: notifications_001_01 420 | replicaset_uuid: 11079f9c-0001-0000-0000-000000000000 421 | common: 422 | box: 423 | log_level: 5 424 | memtx_memory: 268435456 425 | instances: 426 | notifications_001_01: 427 | cluster: notifications_001 428 | box: 429 | instance_uuid: 11079f9c-0001-0001-0000-000000000000 430 | listen: 10.0.3.11:4011 431 | notifications_001_02: 432 | cluster: notifications_001 433 | box: 434 | instance_uuid: 11079f9c-0001-0002-0000-000000000000 435 | listen: 10.0.3.12:4012 436 | notifications_002_01: 437 | cluster: notifications_002 438 | box: 439 | instance_uuid: 11079f9c-0002-0001-0000-000000000000 440 | listen: 10.0.3.11:4021 441 | notifications_002_02: 442 | cluster: notifications_002 443 | box: 444 | instance_uuid: 11079f9c-0002-0002-0000-000000000000 445 | listen: 10.0.3.12:4022 446 | ``` 447 | 448 | This configuration describes configuration of application `notifications` with 2 replicasets `notifications_001` and `notifications_002`. 449 | 450 | Shard `notifications_001` contains 2 nodes: 451 | 452 | - `notifications_001_01` - described as master 453 | - `notifications_001_02` 454 | 455 | Shard `notifications_002` contains 2 nodes: 456 | 457 | - `notifications_002_01` - described as master 458 | - `notifications_002_02` 459 | 460 | ### Multi-shard topology for vshard-based applications (`etcd.cluster.vshard`) 461 | 462 | In most cases for multi-shard applications it is better to use module [tarantool/vshard](https://www.tarantool.io/en/doc/latest/concepts/sharding). 463 | 464 | vshard required to be properly configured. Each instance of the cluster must contain the same view of cluster topology. 465 | 466 | vshard application has 2 groups of instances: storages (data nodes) and routers (stateless proxy nodes). 467 | 468 | #### Example of ETCD configuration for vshard-based applications (`etcd.cluster.vshard`) 469 | 470 | ```yaml 471 | tarantool: 472 | profile: 473 | common: 474 | vshard: 475 | bucket_count: 30000 476 | box: 477 | log_level: 5 478 | replication_connect_quorum: 2 479 | clusters: 480 | profile_001: 481 | master: profile_001_01 482 | replicaset_uuid: 17120f91-0001-0000-0000-000000000000 483 | profile_002: 484 | master: profile_002_01 485 | replicaset_uuid: 17120f91-0002-0000-0000-000000000000 486 | instances: 487 | profile_001_01: 488 | cluster: profile_001 489 | box: 490 | instance_uuid: 17120f91-0001-0001-0000-000000000000 491 | listen: 10.0.4.11:4011 492 | profile_001_02: 493 | cluster: profile_001 494 | box: 495 | instance_uuid: 17120f91-0001-0002-0000-000000000000 496 | listen: 10.0.4.12:4012 497 | profile_002_01: 498 | cluster: profile_002 499 | box: 500 | instance_uuid: 17120f91-0002-0001-0000-000000000000 501 | listen: 10.0.4.11:4021 502 | profile_002_02: 503 | cluster: profile_002 504 | box: 505 | instance_uuid: 17120f91-0002-0002-0000-000000000000 506 | listen: 10.0.4.12:4022 507 | router_001: 508 | router: true 509 | box: 510 | instance_uuid: 12047e12-0000-0001-0000-000000000000 511 | listen: 10.0.5.12:7001 512 | router_002: 513 | router: true 514 | box: 515 | instance_uuid: 12047e12-0000-0002-0000-000000000000 516 | listen: 10.0.5.13:7002 517 | router_003: 518 | router: true 519 | box: 520 | instance_uuid: 12047e12-0000-0003-0000-000000000000 521 | listen: 10.0.5.11:7003 522 | ``` 523 | 524 | #### Example of vshard-based init.lua (`etcd.cluster.vshard`) 525 | 526 | The code of simultanious bootstrap is tricky, and short safe version of it listed below 527 | 528 | ```lua 529 | local fun = require 'fun' 530 | --- variable instance_name must be derived somehow for each tarantool instance 531 | --- For example from name of the file. or from environment variable 532 | require 'config' { 533 | mkdir = true, 534 | instance_name = instance_name, 535 | file = '/etc/profile/conf.lua', 536 | master_selection_policy = 'etcd.cluster.vshard', 537 | on_load = function(conf, cfg) 538 | -- on_load is called each time right after fetching data from ETCD 539 | local all_cfg = conf.etcd:get_all() 540 | 541 | -- Construct vshard/sharding table from ETCD 542 | cfg.sharding = fun.iter(all_cfg.clusters) 543 | :map(function(shard_name, shard_info) 544 | return shard_info.replicaset_uuid, { 545 | replicas = fun.iter(all_cfg.instances) 546 | :grep(function(instance_name, instance_info) 547 | return instance_info.cluster == shard_name 548 | end) 549 | :map(function(instance_name, instance_info) 550 | return instance_info.box.instance_uuid, { 551 | name = instance_name, 552 | uri = 'guest:@'..instance_info.box.listen, 553 | master = instance_name == shard_info.master, 554 | } 555 | end) 556 | :tomap() 557 | } 558 | end) 559 | :tomap() 560 | end, 561 | on_after_cfg = function(conf, cfg) 562 | -- on_after_cfg is called once after returning from box.cfg (Tarantool is already online) 563 | if cfg.cluster then 564 | vshard.storage.cfg({ 565 | sharding = cfg.sharding, 566 | bucket_count = config.get('vshard.bucket_count'), 567 | }, box.info.uuid) 568 | end 569 | if cfg.router then 570 | vshard.router.cfg({ 571 | sharding = cfg.sharding, 572 | bucket_count = config.get('vshard.bucket_count'), 573 | }) 574 | end 575 | end, 576 | } 577 | ``` 578 | 579 | #### VShard Maintenance 580 | 581 | By default vshard does not support master auto discovery. If you switch master in any replicaset you have to reconfigure routers as well. 582 | 583 | With vshard topology it is strongly recommended to use [package.reload](https://github.com/moonlibs/package-reload). Module must be required before first require of `config`. 584 | 585 | ```lua 586 | require 'package.reload' 587 | -- .... 588 | require 'config' { 589 | -- ... 590 | } 591 | -- ... 592 | ``` 593 | 594 | It is good to use [switchover](https://gitlab.com/ochaton/switchover) to maintenance sharded applications. 595 | 596 | To get used to vshard please read getting started of it [Sharding with Vshard](https://www.tarantool.io/en/doc/latest/book/admin/vshard_admin/#vshard-install) 597 | -------------------------------------------------------------------------------- /config-dev-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "config" 2 | version = "dev-1" 3 | source = { 4 | url = "git+https://github.com/moonlibs/config", 5 | branch = "master" 6 | } 7 | description = { 8 | summary = "Package for loading external lua config", 9 | homepage = "https://github.com/moonlibs/config.git", 10 | license = "BSD" 11 | } 12 | dependencies = { 13 | "lua >= 5.1" 14 | } 15 | build = { 16 | type = "builtin", 17 | modules = { 18 | config = "config.lua", 19 | ["config.etcd"] = "config/etcd.lua" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /config-scm-1.rockspec: -------------------------------------------------------------------------------- 1 | --[[ 2 | *** This is legacy rockspec 3 | *** Please, for this version refer instead to 4 | *** rockspecs/config-scm-1.rockspec 5 | ]] 6 | 7 | package = 'config' 8 | version = 'scm-1' 9 | source = { 10 | url = 'git+https://github.com/moonlibs/config.git', 11 | branch = 'v1', 12 | } 13 | description = { 14 | summary = "Package for loading external lua config", 15 | homepage = 'https://github.com/moonlibs/config.git', 16 | license = 'BSD', 17 | } 18 | dependencies = { 19 | 'lua >= 5.1' 20 | } 21 | build = { 22 | type = 'builtin', 23 | modules = { 24 | ['config'] = 'config.lua' 25 | } 26 | } 27 | 28 | -- vim: syntax=lua 29 | -------------------------------------------------------------------------------- /config.lua: -------------------------------------------------------------------------------- 1 | ---@diagnostic disable: inject-field 2 | local log = require 'log' 3 | if log.new then 4 | log = log.new('moonlibs.config') 5 | end 6 | local fio = require 'fio' 7 | local json = require 'json'.new() 8 | local yaml = require 'yaml'.new() 9 | local digest = require 'digest' 10 | local fiber = require 'fiber' 11 | local clock = require 'clock' 12 | json.cfg{ encode_invalid_as_nil = true } 13 | yaml.cfg{ encode_use_tostring = true } 14 | 15 | ---Retrieves all upvalues of given function and returns them as kv-map 16 | ---@param fun fun() 17 | ---@return table variables 18 | local function lookaround(fun) 19 | local vars = {} 20 | local i = 1 21 | while true do 22 | local n,v = debug.getupvalue(fun,i) 23 | if not n then break end 24 | vars[n] = v 25 | i = i + 1 26 | end 27 | 28 | return vars 29 | end 30 | 31 | ---@private 32 | ---@class moonlibs.config.reflect_internals 33 | ---@field dynamic_cfg table 34 | ---@field default_cfg table 35 | ---@field upgrade_cfg? fun(cfg: table, translate_cfg: table): table 36 | ---@field template_cfg? table 37 | ---@field translate_cfg? table 38 | ---@field log? table 39 | 40 | 41 | ---Unwraps box.cfg and retrieves dynamic_cfg, default_cfg tables 42 | ---@return moonlibs.config.reflect_internals 43 | local function reflect_internals() 44 | local peek = { 45 | dynamic_cfg = {}; 46 | default_cfg = {}; 47 | upgrade_cfg = true; 48 | translate_cfg = true; 49 | template_cfg = true; 50 | log = true; 51 | } 52 | 53 | local steps = {} 54 | local peekf = box.cfg 55 | local allow_unwrap = true 56 | while true do 57 | local prevf = peekf 58 | local mt = debug.getmetatable(peekf) 59 | if type(peekf) == 'function' then 60 | -- pass 61 | table.insert(steps,"func") 62 | elseif mt and mt.__call then 63 | peekf = mt.__call 64 | table.insert(steps,"mt_call") 65 | else 66 | error(string.format("Neither function nor callable argument %s after steps: %s", peekf, table.concat(steps, ", "))) 67 | end 68 | 69 | local vars = lookaround(peekf) 70 | if type(vars.default_cfg) == 'table' then 71 | for k in pairs(vars.default_cfg) do 72 | peek.default_cfg[k] = vars.default_cfg[k] 73 | end 74 | end 75 | if allow_unwrap and (vars.orig_cfg or vars.origin_cfg) then 76 | -- It's a wrap of tarantoolctl/tt, unwrap and repeat 77 | peekf = (vars.orig_cfg or vars.origin_cfg) 78 | allow_unwrap = false 79 | table.insert(steps,"ctl-orig") 80 | elseif vars.dynamic_cfg then 81 | log.info("Found config by steps: %s", table.concat(steps, ", ")) 82 | for k in pairs(vars.dynamic_cfg) do 83 | peek.dynamic_cfg[k] = true 84 | end 85 | for k in pairs(peek) do 86 | if peek[k] == true then 87 | if vars[k] ~= nil then 88 | peek[k] = vars[k] 89 | else 90 | peek[k] = nil 91 | end 92 | end 93 | end 94 | break 95 | elseif vars.lock and vars.f and type(vars.f) == 'function' then 96 | peekf = vars.f 97 | table.insert(steps,"lock-unwrap") 98 | elseif vars.old_call and type(vars.old_call) == 'function' then 99 | peekf = vars.old_call 100 | table.insert(steps,"ctl-oldcall") 101 | elseif vars.orig_cfg_call and type(vars.orig_cfg_call) == 'function' then 102 | peekf = vars.orig_cfg_call 103 | table.insert(steps,"ctl-orig_cfg_call") 104 | elseif vars.load_cfg_apply_dynamic then 105 | table.insert(steps,"load_cfg_apply_dynamic") 106 | for k in pairs(peek) do 107 | if peek[k] == true then 108 | if vars[k] ~= nil then 109 | peek[k] = vars[k] 110 | end 111 | end 112 | end 113 | peekf = vars.load_cfg_apply_dynamic 114 | elseif vars.dynamic_cfg_modules then 115 | -- print(yaml.encode(vars.dynamic_cfg_modules)) 116 | log.info("Found config by steps: %s", table.concat(steps, ", ")) 117 | for k, v in pairs(vars.dynamic_cfg_modules) do 118 | peek.dynamic_cfg[k] = true 119 | for op in pairs(v.options) do 120 | peek.dynamic_cfg[op] = true 121 | end 122 | end 123 | break; 124 | elseif vars.reload_cfg then 125 | table.insert(steps,"reload_cfg") 126 | peekf = vars.reload_cfg 127 | elseif vars.reconfig_modules then 128 | table.insert(steps,"reconfig_modules") 129 | for k in pairs(peek) do 130 | if peek[k] == true then 131 | if vars[k] ~= nil then 132 | peek[k] = vars[k] 133 | end 134 | end 135 | end 136 | peekf = vars.reconfig_modules 137 | elseif vars.orig_call and vars.wrapper_impl and type(vars.orig_call) == 'function' and type(vars.wrapper_impl) == 'function' then 138 | peekf = vars.orig_call 139 | table.insert(steps,"queue-cfg_call") 140 | else 141 | for k,v in pairs(vars) do log.info("var %s=%s",k,v) end 142 | error(string.format("Bad vars for %s after steps: %s", peekf, table.concat(steps, ", "))) 143 | end 144 | 145 | if prevf == peekf then 146 | error(string.format("Recursion for %s after steps: %s", peekf, table.concat(steps, ", "))) 147 | end 148 | end 149 | return peek 150 | end 151 | 152 | local load_cfg = reflect_internals() 153 | 154 | ---Filters only valid keys from given cfg 155 | --- 156 | ---Edits given cfg and returns only clear config 157 | ---@param cfg table 158 | ---@return table 159 | local function prepare_box_cfg(cfg) 160 | -- 1. take config, if have upgrade, upgrade it 161 | if load_cfg.upgrade_cfg then 162 | cfg = load_cfg.upgrade_cfg(cfg, load_cfg.translate_cfg) 163 | end 164 | 165 | -- 2. check non-dynamic, and wipe them out 166 | if type(box.cfg) ~= 'function' then 167 | for key, val in pairs(cfg) do 168 | if load_cfg.dynamic_cfg[key] == nil and box.cfg[key] ~= val then 169 | local warn = string.format( 170 | "Can't change option '%s' dynamically from '%s' to '%s'", 171 | key,box.cfg[key],val 172 | ) 173 | log.warn("%s",warn) 174 | print(warn) 175 | cfg[key] = nil 176 | end 177 | end 178 | end 179 | 180 | return cfg 181 | end 182 | 183 | local readonly_mt = { 184 | __index = function(_,k) return rawget(_,k) end; 185 | __newindex = function(_,k) 186 | error("Modification of readonly key "..tostring(k),2) 187 | end; 188 | __serialize = function(_) 189 | local t = {} 190 | for k,v in pairs(_) do 191 | t[k]=v 192 | end 193 | return t 194 | end; 195 | } 196 | 197 | local function flatten (t,prefix,result) 198 | prefix = prefix or '' 199 | local protect = not result 200 | result = result or {} 201 | for k,v in pairs(t) do 202 | if type(v) == 'table' then 203 | flatten(v, prefix..k..'.',result) 204 | end 205 | result[prefix..k] = v 206 | end 207 | if protect then 208 | return setmetatable(result,readonly_mt) 209 | end 210 | return result 211 | end 212 | 213 | local function get_opt() 214 | local take = false 215 | local key 216 | for _,v in ipairs(arg) do 217 | if take then 218 | if key == 'config' or key == 'c' then 219 | return v 220 | end 221 | else 222 | if string.sub( v, 1, 2) == "--" then 223 | local x = string.find( v, "=", 1, true ) 224 | if x then 225 | key = string.sub( v, 3, x-1 ) 226 | -- print("have key=") 227 | if key == 'config' then 228 | return string.sub( v, x+1 ) 229 | end 230 | else 231 | -- print("have key, turn take") 232 | key = string.sub( v, 3 ) 233 | take = true 234 | end 235 | elseif string.sub( v, 1, 1 ) == "-" then 236 | if string.len(v) == 2 then 237 | key = string.sub(v,2,2) 238 | take = true 239 | else 240 | key = string.sub(v,2,2) 241 | if key == 'c' then 242 | return string.sub( v, 3 ) 243 | end 244 | end 245 | end 246 | end 247 | end 248 | end 249 | 250 | local function deep_merge(dst,src,keep) 251 | -- TODO: think of cyclic 252 | if not src or not dst then error("Call to deepmerge with bad args",2) end 253 | for k,v in pairs(src) do 254 | if type(v) == 'table' then 255 | if not dst[k] then dst[k] = {} end 256 | deep_merge(dst[k],src[k],keep) 257 | else 258 | if dst[k] == nil or not keep then 259 | dst[k] = src[k] 260 | end 261 | end 262 | end 263 | end 264 | 265 | local function deep_copy(src) 266 | local t = {} 267 | deep_merge(t, src) 268 | return t 269 | end 270 | 271 | local function is_array(a) 272 | local len = 0 273 | for k in pairs(a) do 274 | len = len + 1 275 | if type(k) ~= 'number' then 276 | return false 277 | end 278 | end 279 | return #a == len 280 | end 281 | 282 | --[[ 283 | returns config diff 284 | 1. deleted values returned as box.NULL 285 | 2. arrays is replaced completely 286 | 3. nil means no diff (and not stored in tables) 287 | ]] 288 | 289 | local function value_diff(old,new) 290 | if type(old) ~= type(new) then 291 | return new 292 | elseif type(old) == 'table' then 293 | if new == old then return end 294 | 295 | if is_array(old) then 296 | if #new ~= #old then return new end 297 | for i = 1,#old do 298 | local diff = value_diff(old[i], new[i]) 299 | if diff ~= nil then 300 | return new 301 | end 302 | end 303 | else 304 | local diff = {} 305 | for k in pairs(old) do 306 | if new[ k ] == nil then 307 | diff[k] = box.NULL 308 | else 309 | local vdiff = value_diff(old[k], new[k]) 310 | if vdiff ~= nil then 311 | diff[k] = vdiff 312 | end 313 | end 314 | end 315 | for k in pairs(new) do 316 | if old[ k ] == nil then 317 | diff[k] = new[k] 318 | end 319 | end 320 | if next(diff) then 321 | return diff 322 | end 323 | end 324 | else 325 | if old ~= new then 326 | return new 327 | end 328 | end 329 | -- no diff 330 | end 331 | 332 | local function toboolean(v) 333 | if v then 334 | if type(v) == 'boolean' then return v end 335 | v = tostring(v):lower() 336 | local n = tonumber(v) 337 | if n then return n ~= 0 end 338 | if v == 'true' or v == 'yes' then 339 | return true 340 | end 341 | end 342 | return false 343 | end 344 | 345 | ---@type table 346 | local master_selection_policies; 347 | master_selection_policies = { 348 | ['etcd.instance.single'] = function(M, instance_name, common_cfg, instance_cfg, cluster_cfg, local_cfg) 349 | local cfg = {} 350 | deep_merge(cfg, common_cfg) 351 | deep_merge(cfg, instance_cfg) 352 | 353 | if cluster_cfg then 354 | error("Cluster config should not exist for single instance config") 355 | end 356 | 357 | deep_merge(cfg, local_cfg) 358 | 359 | if cfg.box.read_only == nil then 360 | log.info("Instance have no read_only option, set read_only=false") 361 | cfg.box.read_only = false 362 | end 363 | 364 | if cfg.box.instance_uuid and not cfg.box.replicaset_uuid then 365 | cfg.box.replicaset_uuid = cfg.box.instance_uuid 366 | end 367 | 368 | log.info("Using policy etcd.instance.single, read_only=%s",cfg.box.read_only) 369 | return cfg 370 | end; 371 | ['etcd.instance.read_only'] = function(M, instance_name, common_cfg, instance_cfg, cluster_cfg, local_cfg) 372 | local cfg = {} 373 | deep_merge(cfg, common_cfg) 374 | deep_merge(cfg, instance_cfg) 375 | 376 | if cluster_cfg then 377 | log.info("cluster=%s",json.encode(cluster_cfg)) 378 | assert(cluster_cfg.replicaset_uuid,"Need cluster uuid") 379 | cfg.box.replicaset_uuid = cluster_cfg.replicaset_uuid 380 | end 381 | 382 | deep_merge(cfg, local_cfg) 383 | 384 | if M.default_read_only and cfg.box.read_only == nil then 385 | log.info("Instance have no read_only option, set read_only=true") 386 | cfg.box.read_only = true 387 | end 388 | 389 | log.info("Using policy etcd.instance.read_only, read_only=%s",cfg.box.read_only) 390 | return cfg 391 | end; 392 | ['etcd.cluster.master'] = function(M, instance_name, common_cfg, instance_cfg, cluster_cfg, local_cfg) 393 | log.info("Using policy etcd.cluster.master") 394 | local cfg = {} 395 | deep_merge(cfg, common_cfg) 396 | deep_merge(cfg, instance_cfg) 397 | 398 | assert(cluster_cfg.replicaset_uuid,"Need cluster uuid") 399 | cfg.box.replicaset_uuid = cluster_cfg.replicaset_uuid 400 | 401 | if cfg.box.read_only ~= nil then 402 | log.info("Ignore box.read_only=%s value due to config policy",cfg.box.read_only) 403 | end 404 | if cluster_cfg.master then 405 | if cluster_cfg.master == instance_name then 406 | log.info("Instance is declared as cluster master, set read_only=false") 407 | cfg.box.read_only = false 408 | if cfg.box.bootstrap_strategy ~= 'auto' then 409 | cfg.box.replication_connect_quorum = 1 410 | cfg.box.replication_connect_timeout = 1 411 | end 412 | else 413 | log.info("Cluster has another master %s, not me %s, set read_only=true", cluster_cfg.master, instance_name) 414 | cfg.box.read_only = true 415 | end 416 | else 417 | log.info("Cluster have no declared master, set read_only=true") 418 | cfg.box.read_only = true 419 | end 420 | 421 | deep_merge(cfg, local_cfg) 422 | 423 | return cfg 424 | end; 425 | ['etcd.cluster.vshard'] = function(M, instance_name, common_cfg, instance_cfg, cluster_cfg, local_cfg) 426 | log.info("Using policy etcd.cluster.vshard") 427 | if instance_cfg.cluster then 428 | return master_selection_policies['etcd.cluster.master'](M, instance_name, common_cfg, instance_cfg, cluster_cfg, local_cfg) 429 | else 430 | return master_selection_policies['etcd.instance.single'](M, instance_name, common_cfg, instance_cfg, cluster_cfg, local_cfg) 431 | end 432 | end; 433 | ['etcd.cluster.raft'] = function(M, instance_name, common_cfg, instance_cfg, cluster_cfg, local_cfg) 434 | log.info("Using policy etcd.cluster.raft") 435 | local cfg = {} 436 | deep_merge(cfg, common_cfg) 437 | deep_merge(cfg, instance_cfg) 438 | 439 | assert(cluster_cfg.replicaset_uuid,"Need cluster uuid") 440 | cfg.box.replicaset_uuid = cluster_cfg.replicaset_uuid 441 | 442 | if not cfg.box.election_mode then 443 | cfg.box.election_mode = M.default_election_mode 444 | end 445 | 446 | -- TODO: anonymous replica 447 | if cfg.box.election_mode == 'off' then 448 | log.info("Force box.read_only=true for election_mode=off") 449 | cfg.box.read_only = true 450 | end 451 | 452 | if not cfg.box.replication_synchro_quorum then 453 | cfg.box.replication_synchro_quorum = M.default_synchro_quorum 454 | end 455 | 456 | if cfg.box.election_mode == "candidate" then 457 | cfg.box.read_only = false 458 | end 459 | 460 | deep_merge(cfg, local_cfg) 461 | 462 | return cfg 463 | end; 464 | } 465 | 466 | local function cast_types(c) 467 | if c then 468 | for k,v in pairs(c) do 469 | if load_cfg.template_cfg[k] == 'boolean' and type(v) == 'string' then 470 | c[k] = c[k] == 'true' 471 | end 472 | end 473 | end 474 | end 475 | 476 | local function gen_instance_uuid(instance_name) 477 | local k,d1,d2 = instance_name:match("^([A-Za-z_]+)_(%d+)_(%d+)$") 478 | if k then 479 | return string.format( 480 | "%08s-%04d-%04d-%04d-%012x", 481 | digest.sha1_hex(k .. "_instance"):sub(1,8), 482 | d1,d2,0,0 483 | ) 484 | end 485 | 486 | k,d1 = instance_name:match("^([A-Za-z_]+)_(%d+)$") 487 | if k then 488 | return string.format( 489 | "%08s-%04d-%04d-%04d-%012d", 490 | digest.sha1_hex(k):sub(1,8), 491 | 0,0,0,d1 492 | ) 493 | end 494 | error("Can't generate uuid for instance "..instance_name, 2) 495 | end 496 | 497 | local function gen_cluster_uuid(cluster_name) 498 | local k,d1 = cluster_name:match("^([A-Za-z_]+)_(%d+)$") 499 | if k then 500 | return string.format( 501 | "%08s-%04d-%04d-%04d-%012d", 502 | digest.sha1_hex(k .. "_shard"):sub(1,8), 503 | d1,0,0,0 504 | ) 505 | end 506 | error("Can't generate uuid for cluster "..cluster_name, 2) 507 | end 508 | 509 | ---@class moonlibs.config.opts.etcd:moonlibs.config.etcd.opts 510 | ---@field instance_name string Mandatory name of the instance 511 | ---@field prefix string Mandatory prefix inside etcd tree 512 | ---@field uuid? 'auto' When auto config generates replicaset_uuid and instance_uuid for nodes 513 | ---@field fixed? table Optional ETCD tree 514 | 515 | ---Loads configuration from etcd and evaluate master_selection_policy 516 | ---@param M moonlibs.config 517 | ---@param etcd_conf moonlibs.config.opts.etcd 518 | ---@param local_cfg table 519 | ---@return table 520 | local function etcd_load( M, etcd_conf, local_cfg ) 521 | 522 | local etcd 523 | local instance_name = assert(etcd_conf.instance_name,"etcd.instance_name is required") 524 | local prefix = assert(etcd_conf.prefix,"etcd.prefix is required") 525 | 526 | if etcd_conf.fixed then 527 | etcd = setmetatable({ data = etcd_conf.fixed },{__index = { 528 | discovery = function() end; 529 | list = function(e,k) 530 | if k:sub(1,#prefix) == prefix then 531 | k = k:sub(#prefix + 1) 532 | end 533 | local v = e.data 534 | for key in k:gmatch("([^/]+)") do 535 | if type(v) ~= "table" then return end 536 | v = v[key] 537 | end 538 | return v 539 | end; 540 | }}) 541 | else 542 | etcd = require 'config.etcd' (etcd_conf) 543 | end 544 | M.etcd = etcd 545 | 546 | function M.etcd.get_common(e) 547 | local common_cfg = e:list(prefix .. "/common") 548 | assert(common_cfg.box,"no box config in etcd common tree") 549 | cast_types(common_cfg.box) 550 | return common_cfg 551 | end 552 | 553 | function M.etcd.get_instances(e) 554 | local all_instances_cfg = e:list(prefix .. "/instances") 555 | for inst_name,inst_cfg in pairs(all_instances_cfg) do 556 | cast_types(inst_cfg.box) 557 | if etcd_conf.uuid == 'auto' and not inst_cfg.box.instance_uuid then 558 | inst_cfg.box.instance_uuid = gen_instance_uuid(inst_name) 559 | end 560 | end 561 | return all_instances_cfg 562 | end 563 | 564 | function M.etcd.get_clusters(e) 565 | local all_clusters_cfg = e:list(prefix .. "/clusters") or etcd:list(prefix .. "/shards") 566 | for cluster_name,cluster_cfg in pairs(all_clusters_cfg) do 567 | cast_types(cluster_cfg) 568 | if etcd_conf.uuid == 'auto' and not cluster_cfg.replicaset_uuid then 569 | cluster_cfg.replicaset_uuid = gen_cluster_uuid(cluster_name) 570 | end 571 | end 572 | return all_clusters_cfg 573 | end 574 | 575 | function M.etcd.get_all(e) 576 | local all_cfg = e:list(prefix) 577 | cast_types(all_cfg.common.box) 578 | for inst_name,inst_cfg in pairs(all_cfg.instances) do 579 | cast_types(inst_cfg.box) 580 | if etcd_conf.uuid == 'auto' and not inst_cfg.box.instance_uuid then 581 | inst_cfg.box.instance_uuid = gen_instance_uuid(inst_name) 582 | end 583 | end 584 | for cluster_name,cluster_cfg in pairs(all_cfg.clusters or all_cfg.shards or {}) do 585 | cast_types(cluster_cfg) 586 | if etcd_conf.uuid == 'auto' and not cluster_cfg.replicaset_uuid then 587 | cluster_cfg.replicaset_uuid = gen_cluster_uuid(cluster_name) 588 | end 589 | end 590 | return all_cfg 591 | end 592 | 593 | etcd:discovery() 594 | 595 | local all_cfg = etcd:get_all() 596 | if etcd_conf.print_config then 597 | print("Loaded config from etcd",yaml.encode(all_cfg)) 598 | end 599 | local common_cfg = all_cfg.common 600 | local all_instances_cfg = all_cfg.instances 601 | 602 | local instance_cfg = all_instances_cfg[instance_name] 603 | assert(instance_cfg,"Instance name "..instance_name.." is not known to etcd") 604 | 605 | local all_clusters_cfg = all_cfg.clusters or all_cfg.shards 606 | 607 | local master_selection_policy 608 | local cluster_cfg 609 | if instance_cfg.cluster or local_cfg.cluster then 610 | cluster_cfg = all_clusters_cfg[ (instance_cfg.cluster or local_cfg.cluster) ] 611 | assert(cluster_cfg,"Cluster section required"); 612 | assert(cluster_cfg.replicaset_uuid,"Need cluster uuid") 613 | master_selection_policy = M.master_selection_policy or 'etcd.instance.read_only' 614 | elseif instance_cfg.router then 615 | -- TODO 616 | master_selection_policy = M.master_selection_policy or 'etcd.instance.single' 617 | else 618 | master_selection_policy = M.master_selection_policy or 'etcd.instance.single' 619 | end 620 | 621 | local master_policy = master_selection_policies[ master_selection_policy ] 622 | if not master_policy then 623 | error(string.format("Unknown master_selection_policy: %s",M.master_selection_policy),0) 624 | end 625 | 626 | local cfg = master_policy(M, instance_name, common_cfg, instance_cfg, cluster_cfg, local_cfg) 627 | 628 | local members = {} 629 | for _,v in pairs(all_instances_cfg) do 630 | if v.cluster == cfg.cluster then -- and k ~= instance_name then 631 | if not toboolean(v.disabled) then 632 | table.insert(members,v) 633 | else 634 | log.warn("Member '%s' from cluster '%s' listening on %s is disabled", instance_name, v.cluster, v.box.listen) 635 | end 636 | end 637 | end 638 | 639 | if cfg.cluster then 640 | --if cfg.box.read_only then 641 | local repl = {} 642 | for _,member in pairs(members) do 643 | if member.box.remote_addr then 644 | table.insert(repl, member.box.remote_addr) 645 | else 646 | table.insert(repl, member.box.listen) 647 | end 648 | end 649 | table.sort(repl, function(a,b) 650 | local ha,pa = a:match('^([^:]+):(.+)') 651 | local hb,pb = a:match('^([^:]+):(.+)') 652 | if pa and pb then 653 | if pa < pb then return true end 654 | if ha < hb then return true end 655 | end 656 | return a < b 657 | end) 658 | if cfg.box.replication then 659 | print( 660 | "Start instance ",cfg.box.listen, 661 | " with locally overriden replication:",table.concat(cfg.box.replication,", "), 662 | " instead of etcd's:", table.concat(repl,", ") 663 | ) 664 | else 665 | cfg.box.replication = repl 666 | print( 667 | "Start instance "..cfg.box.listen, 668 | " with replication:"..table.concat(cfg.box.replication,", "), 669 | string.format("timeout: %s, quorum: %s, lag: %s", 670 | cfg.box.replication_connect_timeout 671 | or ('def:%s'):format(load_cfg.default_cfg.replication_connect_timeout or 30), 672 | cfg.box.replication_connect_quorum or 'def:full', 673 | cfg.box.replication_sync_lag 674 | or ('def:%s'):format(load_cfg.default_cfg.replication_sync_lag or 10) 675 | ) 676 | ) 677 | end 678 | --end 679 | end 680 | -- print(yaml.encode(cfg)) 681 | 682 | return cfg 683 | end 684 | 685 | local function is_replication_changed (old_conf, new_conf) 686 | if type(old_conf) == 'table' and type(new_conf) == 'table' then 687 | local changed_replicas = {} 688 | for _, replica in pairs(old_conf) do 689 | changed_replicas[replica] = true 690 | end 691 | 692 | for _, replica in pairs(new_conf) do 693 | if changed_replicas[replica] then 694 | changed_replicas[replica] = nil 695 | else 696 | return true 697 | end 698 | end 699 | 700 | -- if we have some changed_replicas left, then we definitely need to reconnect 701 | return not not next(changed_replicas) 702 | else 703 | return old_conf ~= new_conf 704 | end 705 | end 706 | 707 | local function optimal_rcq(upstreams) 708 | local n_ups = #(upstreams or {}) 709 | local rcq 710 | if n_ups == 0 then 711 | rcq = 0 712 | else 713 | rcq = 1+math.floor(n_ups/2) 714 | end 715 | return rcq 716 | end 717 | 718 | local function do_cfg(boxcfg, cfg) 719 | for key, val in pairs(cfg) do 720 | if load_cfg.template_cfg[key] == nil then 721 | local warn = string.format("Dropping non-boxcfg option '%s' given '%s'",key,val) 722 | log.warn("%s",warn) 723 | print(warn) 724 | cfg[key] = nil 725 | end 726 | end 727 | 728 | if type(box.cfg) ~= 'function' then 729 | for key, val in pairs(cfg) do 730 | if load_cfg.dynamic_cfg[key] == nil and box.cfg[key] ~= val then 731 | local warn = string.format( 732 | "Dropping dynamic option '%s' previous value '%s' new value '%s'", 733 | key,box.cfg[key],val 734 | ) 735 | log.warn("%s",warn) 736 | print(warn) 737 | cfg[key] = nil 738 | end 739 | end 740 | end 741 | 742 | log.info("Just before first box.cfg %s", yaml.encode(cfg)) 743 | -- Some boxcfg loves to rewrite passing table. We pass a copy of configuration 744 | boxcfg(deep_copy(cfg)) 745 | end 746 | 747 | 748 | ---@class moonlibs.config.opts 749 | ---@field bypass_non_dynamic? boolean (default: true) drops every changed non-dynamic option on reconfiguration 750 | ---@field tidy_load? boolean (default: true) recoveries tarantool with read_only=true 751 | ---@field mkdir? boolean (default: false) should moonlibs/config create memtx_dir and wal_dir 752 | ---@field etcd? moonlibs.config.opts.etcd [legacy] configuration of etcd 753 | ---@field default_replication_connect_timeout? number (default: 1.1) default RCT in seconds 754 | ---@field default_election_mode? election_mode (default: candidate) option is respected only when etcd.cluster.raft is used 755 | ---@field default_synchro_quorum? string|number (default: 'N/2+1') option is respected only when etcd.cluster.raft is used 756 | ---@field default_read_only? boolean (default: false) option is respected only when etcd.instance.read_only is used (deprecated) 757 | ---@field master_selection_policy? 'etcd.cluster.master'|'etcd.cluster.vshard'|'etcd.cluster.raft'|'etcd.instance.single' master selection policy 758 | ---@field strict_mode? boolean (default: false) stricts config retrievals. if key is not found config.get will raise an exception 759 | ---@field strict? boolean (default: false) stricts config retrievals. if key is not found config.get will raise an exception 760 | ---@field default? table (default: nil) globally default options for config.get 761 | ---@field on_load? fun(conf: moonlibs.config, cfg: table) callback which is called every time config is loaded from file and ETCD 762 | ---@field load? fun(conf: moonlibs.config, cfg: table): table do not use this callback 763 | ---@field on_before_cfg? fun(conf: moonlibs.config, cfg: table) callback is called right before running box.cfg (but after on_load) 764 | ---@field boxcfg? fun(cfg: table) [legacy] when provided this function will be called instead box.cfg. tidy_load and everything else will not be used. 765 | ---@field wrap_box_cfg? fun(cfg: table) callback is called instead box.cfg. But tidy_load is respected. Use this, if you need to proxy every option to box.cfg on application side 766 | ---@field on_after_cfg? fun(conf: moonlibs.config, cfg: table) callback which is called after full tarantool configuration 767 | 768 | ---@class moonlibs.config: moonlibs.config.opts 769 | ---@field etcd moonlibs.config.etcd 770 | ---@field public _load_cfg table 771 | ---@field public _flat table 772 | ---@field public _fencing_f? Fiber 773 | ---@field public _enforced_ro? boolean 774 | ---@operator call(moonlibs.config.opts): moonlibs.config 775 | 776 | ---@type moonlibs.config 777 | local M 778 | M = setmetatable({ 779 | _VERSION = '0.7.2', 780 | console = {}; 781 | ---Retrieves value from config 782 | ---@overload fun(k: string, def: any?): any? 783 | ---@param self moonlibs.config 784 | ---@param k string path inside config 785 | ---@param def? any optional default value 786 | ---@return any? 787 | get = function(self,k,def) 788 | if self ~= M then 789 | def = k 790 | k = self 791 | end 792 | if M._flat[k] ~= nil then 793 | return M._flat[k] 794 | elseif def ~= nil then 795 | return def 796 | else 797 | if M.strict_mode then 798 | error(string.format("no %s found in config", k)) 799 | else 800 | return 801 | end 802 | end 803 | end, 804 | enforce_ro = function() 805 | if not M._ro_enforcable then 806 | return false, 'cannot enforce readonly' 807 | end 808 | M._enforced_ro = true 809 | return true, { 810 | info_ro = box.info.ro, 811 | cfg_ro = box.cfg.read_only, 812 | enforce_ro = M._enforced_ro, 813 | } 814 | end, 815 | _load_cfg = load_cfg, 816 | },{ 817 | ---Reinitiates moonlibs.config 818 | ---@param args moonlibs.config.opts 819 | ---@return moonlibs.config 820 | __call = function(_, args) 821 | -- args MUST belong to us, because of modification 822 | local file 823 | if type(args) == 'string' then 824 | file = args 825 | args = {} 826 | elseif type(args) == 'table' then 827 | args = deep_copy(args) 828 | file = args.file 829 | else 830 | args = {} 831 | end 832 | if args.bypass_non_dynamic == nil then 833 | args.bypass_non_dynamic = true 834 | end 835 | if args.tidy_load == nil then 836 | args.tidy_load = true 837 | end 838 | M.default_replication_connect_timeout = args.default_replication_connect_timeout or 1.1 839 | M.default_election_mode = args.default_election_mode or 'candidate' 840 | M.default_synchro_quorum = args.default_synchro_quorum or 'N/2+1' 841 | M.default_read_only = args.default_read_only or false 842 | M.master_selection_policy = args.master_selection_policy 843 | M.default = args.default 844 | M.strict_mode = args.strict_mode or args.strict or false 845 | -- print("config", "loading ",file, json.encode(args)) 846 | if not file then 847 | file = get_opt() 848 | -- todo: maybe etcd? 849 | if not file then error("Neither config call option given not -c|--config option passed",2) end 850 | end 851 | 852 | print(string.format("Loading config %s %s", file, json.encode(args))) 853 | 854 | local function load_config() 855 | 856 | local methods = {} 857 | function methods.merge(dst, src, keep) 858 | if src ~= nil then 859 | deep_merge( dst, src, keep ) 860 | end 861 | return dst 862 | end 863 | 864 | function methods.include(path, opts) 865 | path = fio.pathjoin(fio.dirname(file), path) 866 | opts = opts or { if_exists = false } 867 | if not fio.path.exists(path) then 868 | if opts.if_exists then 869 | return 870 | end 871 | error("Not found include file `"..path.."'", 2) 872 | end 873 | local f,e = loadfile(path) 874 | if not f then error(e,2) end 875 | setfenv(f, getfenv(2)) 876 | local ret = f() 877 | if ret ~= nil then 878 | print("Return value from "..path.." is ignored") 879 | end 880 | end 881 | 882 | function methods.print(...) 883 | local p = {...} 884 | for i = 1, select('#', ...) do 885 | if type(p[i]) == 'table' 886 | and not debug.getmetatable(p[i]) 887 | then 888 | p[i] = json.encode(p[i]) 889 | end 890 | end 891 | print(unpack(p)) 892 | end 893 | 894 | local f,e = loadfile(file) 895 | if not f then error(e,2) end 896 | local cfg = setmetatable({}, { 897 | __index = setmetatable(methods, { 898 | __index = setmetatable(args,{ __index = _G }) 899 | }) 900 | }) 901 | setfenv(f, cfg) 902 | local ret = f() 903 | if ret ~= nil then 904 | print("Return value from "..file.." is ignored") 905 | end 906 | setmetatable(cfg,nil) 907 | setmetatable(args,nil) 908 | deep_merge(cfg,args.default or {},'keep') 909 | 910 | -- subject to change, just a PoC 911 | local etcd_conf = args.etcd or cfg.etcd 912 | -- we can enforce ro during recovery only if we have etcd config 913 | M._ro_enforcable = M._ro_enforcable and etcd_conf ~= nil 914 | if etcd_conf then 915 | local s = clock.time() 916 | cfg = etcd_load(M, etcd_conf, cfg) 917 | log.info("etcd_load took %.3fs", clock.time()-s) 918 | end 919 | 920 | if args.load then 921 | cfg = args.load(M, cfg) 922 | end 923 | 924 | if not cfg.box then 925 | error("No box.* config given", 2) 926 | end 927 | 928 | if cfg.box.remote_addr then 929 | cfg.box.remote_addr = nil 930 | end 931 | 932 | if args.bypass_non_dynamic then 933 | cfg.box = prepare_box_cfg(cfg.box) 934 | end 935 | 936 | deep_merge(cfg,{ 937 | sys = deep_copy(args) 938 | }) 939 | cfg.sys.boxcfg = nil 940 | cfg.sys.on_load = nil 941 | 942 | -- latest modifications and fixups 943 | if args.on_load then 944 | args.on_load(M,cfg) 945 | end 946 | return cfg 947 | end 948 | 949 | -- We cannot enforce ro if any of theese conditions not satisfied 950 | -- Tarantool must be bootstraping with tidy_load and do not overwraps personal boxcfg 951 | M._ro_enforcable = args.boxcfg == nil and args.tidy_load and type(box.cfg) == 'function' 952 | local cfg = load_config() --[[@as table]] 953 | 954 | M._flat = flatten(cfg) 955 | 956 | if args.on_before_cfg then 957 | args.on_before_cfg(M,cfg) 958 | end 959 | 960 | if args.mkdir then 961 | if not ( fio.path and fio.mkdir ) then 962 | error(string.format("Tarantool version %s is too old for mkdir: fio.path is not supported", _TARANTOOL),2) 963 | end 964 | for _,key in pairs({"work_dir", "wal_dir", "snap_dir", "memtx_dir", "vinyl_dir"}) do 965 | local v = cfg.box[key] 966 | if v and not fio.path.exists(v) then 967 | local r,e = fio.mktree(v) 968 | if not r then error(string.format("Failed to create path '%s' for %s: %s",v,key,e),2) end 969 | end 970 | end 971 | local v = cfg.box.pid_file 972 | if v then 973 | v = fio.dirname(v); 974 | if v and not fio.path.exists(v) then 975 | local r,e = fio.mktree(v) 976 | if not r then error(string.format("Failed to create path '%s' for pid_file: %s",v,e),2) end 977 | end 978 | end 979 | end 980 | 981 | -- The code below is very hard to understand and quite hard to fix when any bugs occurs. 982 | -- First, you must remember that this part of code is executed several times in very different environments: 983 | -- 1) Tarantool may be started with tarantool .lua and this part is required from the script 984 | -- 2) Tarantool may be started under tarantoolctl (such as tarantoolctl start .lua) then box.cfg will be wrapped 985 | -- by tarantoolctl itself, and it be returned back to table box.cfg after first successfull execution 986 | -- 3) Tarantool may be started inside docker container and default docker-entrypoint.lua also rewraps box.cfg 987 | -- 4) User might want to overwrite box.cfg with his function via args.boxcfg. Though, this method is not recommended 988 | -- it is possible in some environments 989 | -- 5) User might want to "wrap" box.cfg with his own middleware via (args.wrap_box_cfg). It is more recommended, because 990 | -- full algorithm of tidy_load and ro-enforcing is preserved for the user. 991 | 992 | -- Moreover, first run of box.cfg in the life of the process allows to specify static box.cfg options, such as pid_file, log 993 | -- and many others. 994 | -- But, second reconfiguration of box.cfg (due to reload, or reconfiguration in fencing must never touch static options) 995 | -- Part of this is fixed in `do_cfg` method of this codebase. 996 | 997 | -- Because many wrappers in docker-entrypoint.lua and tarantoolctl LOVES to perform non-redoable actions inside box.cfg and 998 | -- switch box.cfg back to builtin tarantool box.cfg, following code MUST NEVER cache value of box.cfg 999 | 1000 | if args.boxcfg then 1001 | do_cfg(args.boxcfg, cfg.box) 1002 | else 1003 | local boxcfg 1004 | if args.wrap_box_cfg then 1005 | boxcfg = args.wrap_box_cfg 1006 | end 1007 | if type(box.cfg) == 'function' then 1008 | if M.etcd then 1009 | if args.tidy_load then 1010 | local snap_dir = cfg.box.snap_dir or cfg.box.memtx_dir 1011 | if not snap_dir then 1012 | if cfg.box.work_dir then 1013 | snap_dir = cfg.box.work_dir 1014 | else 1015 | snap_dir = "." 1016 | end 1017 | end 1018 | local bootstrapped 1019 | for _,v in pairs(fio.glob(snap_dir..'/*.snap')) do 1020 | bootstrapped = v 1021 | end 1022 | 1023 | if bootstrapped then 1024 | print("Have etcd, use tidy load") 1025 | local ro = cfg.box.read_only 1026 | cfg.box.read_only = true 1027 | if cfg.box.bootstrap_strategy ~= 'auto' then 1028 | if not ro then 1029 | -- Only if node should be master 1030 | cfg.box.replication_connect_quorum = 1 1031 | cfg.box.replication_connect_timeout = M.default_replication_connect_timeout 1032 | elseif not cfg.box.replication_connect_quorum then 1033 | -- For replica tune up to N/2+1 1034 | cfg.box.replication_connect_quorum = optimal_rcq(cfg.box.replication) 1035 | end 1036 | end 1037 | log.info("Start tidy loading with ro=true%s rcq=%s rct=%s (snap=%s)", 1038 | ro ~= true and string.format(' (would be %s)',ro) or '', 1039 | cfg.box.replication_connect_quorum, cfg.box.replication_connect_timeout, 1040 | bootstrapped 1041 | ) 1042 | else 1043 | -- not bootstraped yet cluster 1044 | 1045 | -- if cfg.box.bootstrap_strategy == 'auto' then -- ≥ Tarantool 2.11 1046 | -- local ro = cfg.box.read_only 1047 | -- local is_candidate = cfg.box.election_mode == 'candidate' 1048 | -- if not ro and not is_candidate then 1049 | -- -- master but not Raft/candidate 1050 | -- -- we decrease replication for master, 1051 | -- -- to allow him bootstrap himself 1052 | -- cfg.box.replication = {cfg.box.remote_addr or cfg.box.listen} 1053 | -- end 1054 | if cfg.box.bootstrap_strategy ~= 'auto' then -- < Tarantool 2.11 1055 | if cfg.box.replication_connect_quorum == nil then 1056 | cfg.box.replication_connect_quorum = optimal_rcq(cfg.box.replication) 1057 | end 1058 | end 1059 | 1060 | log.info("Start non-bootstrapped tidy loading with ro=%s rcq=%s rct=%s (dir=%s)", 1061 | cfg.box.read_only, cfg.box.replication_connect_quorum, 1062 | cfg.box.replication_connect_timeout, snap_dir 1063 | ) 1064 | end 1065 | end 1066 | 1067 | do_cfg(boxcfg or box.cfg, cfg.box) 1068 | 1069 | log.info("Reloading config after start") 1070 | 1071 | local new_cfg = load_config() 1072 | if M._enforced_ro then 1073 | log.info("Enforcing RO (should be ro=%s) because told to", new_cfg.box.read_only) 1074 | new_cfg.box.read_only = true 1075 | end 1076 | M._enforced_ro = nil 1077 | M._ro_enforcable = false 1078 | local diff_box = value_diff(cfg.box, new_cfg.box) 1079 | 1080 | -- since load_config loads config also for reloading it removes non-dynamic options 1081 | -- therefore, they would be absent, but should not be passed. remove them 1082 | if diff_box then 1083 | for key in pairs(diff_box) do 1084 | if load_cfg.dynamic_cfg[key] == nil then 1085 | diff_box[key] = nil 1086 | end 1087 | end 1088 | if not next(diff_box) then 1089 | diff_box = nil 1090 | end 1091 | end 1092 | 1093 | if diff_box then 1094 | log.info("Reconfigure after load with %s",require'json'.encode(diff_box)) 1095 | do_cfg(boxcfg or box.cfg, diff_box) 1096 | else 1097 | log.info("Config is actual after load") 1098 | end 1099 | 1100 | M._flat = flatten(new_cfg) 1101 | else 1102 | do_cfg(boxcfg or box.cfg, cfg.box) 1103 | end 1104 | else 1105 | local replication = cfg.box.replication_source or cfg.box.replication 1106 | local box_replication = box.cfg.replication_source or box.cfg.replication 1107 | 1108 | if not is_replication_changed(replication, box_replication) then 1109 | local r = cfg.box.replication 1110 | local rs = cfg.box.replication_source 1111 | cfg.box.replication = nil 1112 | cfg.box.replication_source = nil 1113 | 1114 | do_cfg(boxcfg or box.cfg, cfg.box) 1115 | 1116 | cfg.box.replication = r 1117 | cfg.box.replication_source = rs 1118 | else 1119 | do_cfg(boxcfg or box.cfg, cfg.box) 1120 | end 1121 | end 1122 | end 1123 | 1124 | if args.on_after_cfg then 1125 | args.on_after_cfg(M,cfg) 1126 | end 1127 | -- print(string.format("Box configured")) 1128 | 1129 | local msp = config.get('sys.master_selection_policy') 1130 | if type(cfg.etcd) == 'table' 1131 | and config.get('etcd.fencing_enabled') 1132 | and (msp == 'etcd.cluster.master' or msp == 'etcd.cluster.vshard') 1133 | and type(cfg.cluster) == 'string' and cfg.cluster ~= '' 1134 | and config.get('etcd.reduce_listing_quorum') ~= true 1135 | then 1136 | M._fencing_f = fiber.create(function() 1137 | fiber.name('config/fencing') 1138 | fiber.yield() -- yield execution 1139 | local function in_my_gen() fiber.testcancel() return config._fencing_f == fiber.self() end 1140 | assert(cfg.cluster, "cfg.cluster must be defined") 1141 | 1142 | local watch_path = fio.pathjoin( 1143 | config.get('etcd.prefix'), 1144 | 'clusters', 1145 | cfg.cluster 1146 | ) 1147 | 1148 | local my_name = assert(config.get('sys.instance_name'), "instance_name is not defined") 1149 | local fencing_timeout = config.get('etcd.fencing_timeout', 10) 1150 | local fencing_pause = config.get('etcd.fencing_pause', fencing_timeout/2) 1151 | assert(fencing_pause < fencing_timeout, "fencing_pause must be < fencing_timeout") 1152 | local fencing_check_replication = config.get('etcd.fencing_check_replication') 1153 | if type(fencing_check_replication) == 'string' then 1154 | fencing_check_replication = fencing_check_replication == 'true' 1155 | else 1156 | fencing_check_replication = fencing_check_replication == true 1157 | end 1158 | 1159 | local etcd_cluster, watch_index 1160 | 1161 | local function refresh_list(opts) 1162 | local s = fiber.time() 1163 | local result, resp = config.etcd:list(watch_path, opts) 1164 | local elapsed = fiber.time()-s 1165 | 1166 | log.verbose("[fencing] list(%s) => %s in %.3fs %s", 1167 | watch_path, resp.status, elapsed, json.encode(resp.body)) 1168 | 1169 | if resp.status == 200 then 1170 | etcd_cluster = result 1171 | if type(resp.headers) == 'table' 1172 | and tonumber(resp.headers['x-etcd-index']) 1173 | and tonumber(resp.headers['x-etcd-index']) >= (tonumber(watch_index) or 0) 1174 | then 1175 | watch_index = (tonumber(resp.headers['x-etcd-index']) or -1) + 1 1176 | end 1177 | end 1178 | return etcd_cluster, watch_index 1179 | end 1180 | 1181 | local function fencing_check(deadline) 1182 | -- we can only allow half of the time till deadline 1183 | local timeout = math.min((deadline-fiber.time())*0.5, fencing_pause) 1184 | log.verbose("[wait] timeout:%.3fs FP:%.3fs", timeout, fencing_pause) 1185 | 1186 | local check_started = fiber.time() 1187 | local pcall_ok, err_or_resolution, new_cluster = pcall(function() 1188 | local started = fiber.time() 1189 | local n_endpoints = #config.etcd.endpoints 1190 | local not_timed_out, response = config.etcd:wait(watch_path, { 1191 | index = watch_index, 1192 | timeout = timeout/n_endpoints, 1193 | }) 1194 | local logger 1195 | if not_timed_out then 1196 | if tonumber(response.status) and tonumber(response.status) >= 400 then 1197 | logger = log.error 1198 | else 1199 | logger = log.info 1200 | end 1201 | else 1202 | logger = log.verbose 1203 | end 1204 | logger("[fencing] wait(%s,index=%s,timeout=%.3fs) => %s (ind:%s) %s took %.3fs", 1205 | watch_path, watch_index, timeout, 1206 | response.status, (response.headers or {})['x-etcd-index'], 1207 | json.encode(response.body), fiber.time()-started) 1208 | 1209 | -- http timed out / or network drop - we'll never know 1210 | if not not_timed_out then return 'timeout' end 1211 | local res = json.decode(response.body) 1212 | 1213 | if type(response.headers) == 'table' 1214 | and tonumber(response.headers['x-etcd-index']) 1215 | and tonumber(response.headers['x-etcd-index']) >= watch_index 1216 | then 1217 | watch_index = (tonumber(response.headers['x-etcd-index']) or -1) + 1 1218 | end 1219 | 1220 | if res.node then 1221 | local node = {} 1222 | config.etcd:recursive_extract(watch_path, res.node, node) 1223 | log.info("[fencing] watch index changed: %s => %s", watch_path, json.encode(node)) 1224 | if not node.master then node = nil end 1225 | return 'changed', node 1226 | end 1227 | end) 1228 | 1229 | log.verbose("[wait] took:%.3fs exp:%.3fs", fiber.time()-check_started, timeout) 1230 | if not in_my_gen() then return end 1231 | 1232 | if not pcall_ok then 1233 | log.warn("ETCD watch failed: %s", err_or_resolution) 1234 | end 1235 | 1236 | if err_or_resolution ~= 'changed' then 1237 | new_cluster = nil 1238 | end 1239 | 1240 | if not new_cluster then 1241 | local list_started = fiber.time() 1242 | log.verbose("[listing] left:%.3fs", deadline-fiber.time()) 1243 | repeat 1244 | local ok, e_cluster = pcall(refresh_list, {deadline = deadline}) 1245 | if ok and e_cluster then 1246 | new_cluster = e_cluster 1247 | break 1248 | end 1249 | 1250 | if not in_my_gen() then return end 1251 | -- we can only sleep 50% till deadline will be reached 1252 | local sleep = math.min(fencing_pause, 0.5*(deadline - fiber.time())) 1253 | fiber.sleep(sleep) 1254 | until fiber.time() > deadline 1255 | log.verbose("[list] took:%.3fs left:%.3fs", 1256 | fiber.time()-list_started, deadline-fiber.time()) 1257 | end 1258 | 1259 | if not in_my_gen() then return end 1260 | 1261 | if type(new_cluster) ~= 'table' then -- ETCD is down 1262 | log.warn('[fencing] ETCD %s is not discovered in etcd during %.2fs %s', 1263 | watch_path, fiber.time()-check_started, new_cluster) 1264 | 1265 | if not fencing_check_replication then 1266 | -- ETCD is down, we do not know what is happening 1267 | return nil 1268 | end 1269 | 1270 | -- In proper fencing we must step down immediately as soon as we discover 1271 | -- that coordinator is down. But in real world there are some circumstances 1272 | -- when coordinator can be down for several seconds if someone crashes network 1273 | -- or ETCD itself. 1274 | -- We propose that it is safe to not step down as soon as we are connected to all 1275 | -- replicas in replicaset (etcd.cluster.master is fullmesh topology). 1276 | -- We do not check downstreams here, because downstreams cannot lead to collisions. 1277 | -- If at least 1 upstream is not in status follow 1278 | -- (Tarantool replication checks with tcp-healthchecks once in box.cfg.replication_timeout) 1279 | -- We immediately stepdown. 1280 | for _, ru in pairs(box.info.replication) do 1281 | if ru.id ~= box.info.id and ru.upstream then 1282 | if ru.upstream.status ~= "follow" then 1283 | log.warn("[fencing] upstream %s is not followed by me %s:%s (idle: %s, lag:%s)", 1284 | ru.upstream.peer, ru.upstream.status, ru.upstream.message, 1285 | ru.upstream.idle, ru.upstream.lag 1286 | ) 1287 | return nil 1288 | end 1289 | end 1290 | end 1291 | 1292 | log.warn('[fencing] ETCD is down but all upstreams are followed by me. Continuing leadership') 1293 | return true 1294 | elseif new_cluster.master == my_name then 1295 | -- The most commmon branch. We are registered as the leader. 1296 | return true 1297 | elseif new_cluster.switchover then -- new_cluster.master ~= my_name 1298 | -- Another instance is the leader in ETCD. But we could be the one 1299 | -- who is going to be the next (cluster is under switching right now). 1300 | -- It is almost impossible to get this path in production. But the only one 1301 | -- protection we have is `fencing_pause` and `fencing_timeout`. 1302 | -- So, we will do nothing until ETCD mutex is present 1303 | log.warn('[fencing] It seems that cluster is under switchover right now %s', json.encode(new_cluster)) 1304 | -- Note: this node was rw (otherwise we would not execute fencing_check at all) 1305 | -- During normal switch registered leader is RO (because we are RW, and we are not the leader) 1306 | -- And in the next step coordinator will update leader info in ETCD. 1307 | -- so this condition seems to be unreachable for every node 1308 | return nil 1309 | else 1310 | log.warn('[fencing] ETCD %s/master is %s not us. Stepping down', watch_path, new_cluster.master) 1311 | -- ETCD is up, master is not us => we must step down immediately 1312 | return false 1313 | end 1314 | end 1315 | 1316 | -- Main fencing loop 1317 | -- It is executed on every replica in the shard 1318 | -- if instance is ro then it will wait until instance became rw 1319 | while in_my_gen() do 1320 | -- Wait until instance became rw loop 1321 | while box.info.ro and in_my_gen() do 1322 | -- this is just fancy sleep. 1323 | -- if node became rw in less than 3 seconds we will check it immediately 1324 | pcall(box.ctl.wait_rw, 3) 1325 | end 1326 | 1327 | -- after waiting to be rw we will step into fencing-loop 1328 | -- we must check that we are still in our code generation 1329 | -- to proceed 1330 | if not in_my_gen() then return end 1331 | 1332 | --- Initial Load of etcd_cluster and watch_index 1333 | local attempt = 0 1334 | while in_my_gen() do 1335 | local ok, err = pcall(refresh_list) 1336 | if not in_my_gen() then return end 1337 | 1338 | if ok then break end 1339 | attempt = attempt + 1 1340 | log.warn("[fencing] initial list failed: %s (attempts: %s)", err, attempt) 1341 | 1342 | fiber.sleep(math.random(math.max(0.5, fencing_pause-0.5), fencing_pause+0.5)) 1343 | end 1344 | 1345 | -- we yield to get next ev_run before get fiber.time() 1346 | fiber.sleep(0) 1347 | if not in_my_gen() then return end 1348 | log.info("etcd_cluster is %s (index: %s)", json.encode(etcd_cluster), watch_index) 1349 | 1350 | 1351 | -- we will not step down until deadline. 1352 | local deadline = fiber.time()+fencing_timeout 1353 | repeat 1354 | -- Before ETCD check we better pause 1355 | -- we do a little bit randomized sleep to not spam ETCD 1356 | local hard_limit = deadline-fiber.time() 1357 | local soft_limit = fencing_timeout-fencing_pause 1358 | local rand_sleep = math.random()*0.1*math.min(hard_limit, soft_limit) 1359 | log.verbose("[sleep] hard:%.3fs soft:%.3fs sleep:%.3fs", hard_limit, soft_limit, rand_sleep) 1360 | fiber.sleep(rand_sleep) 1361 | -- After each yield we have to check that we are still in our generation 1362 | if not in_my_gen() then return end 1363 | 1364 | -- some one makes us readonly. There no need to check ETCD 1365 | -- we break from this loop immediately 1366 | if box.info.ro then break end 1367 | 1368 | -- fencing_check(deadline) if it returns true, 1369 | -- then we update leadership leasing 1370 | local verdict = fencing_check(deadline) 1371 | log.verbose("[verdict:%s] Leasing ft:%.3fs up:%.3fs left:%.3fs", 1372 | verdict == true and "ok" 1373 | or verdict == false and "step" 1374 | or "unknown", 1375 | fencing_timeout, 1376 | verdict and (fiber.time()+fencing_timeout-deadline) or 0, 1377 | deadline - fiber.time() 1378 | ) 1379 | if verdict == false then 1380 | -- immediate stepdown 1381 | break 1382 | elseif verdict then 1383 | -- update deadline. 1384 | if deadline <= fiber.time() then 1385 | log.warn("[fencing] deadline was overflowed deadline:%s, now:%s", 1386 | deadline, fiber.time() 1387 | ) 1388 | end 1389 | deadline = fiber.time()+fencing_timeout 1390 | end 1391 | if not in_my_gen() then return end 1392 | 1393 | if deadline <= fiber.time() then 1394 | log.warn("[fencing] deadline has not been upgraded deadline:%s, now:%s", 1395 | deadline, fiber.time() 1396 | ) 1397 | end 1398 | until box.info.ro or fiber.time() > deadline 1399 | 1400 | -- We have left deadline-loop. It means that fencing is required 1401 | if not box.info.ro then 1402 | log.warn('[fencing] Performing self fencing (box.cfg{read_only=true})') 1403 | box.cfg{read_only=true} 1404 | end 1405 | end 1406 | end) 1407 | end 1408 | 1409 | return M 1410 | end 1411 | }) 1412 | rawset(_G,'config',M) 1413 | 1414 | return M 1415 | -------------------------------------------------------------------------------- /config/etcd.lua: -------------------------------------------------------------------------------- 1 | local json = require 'json' 2 | local log = require 'log' 3 | if log.new then 4 | log = log.new('moonlibs.config') 5 | end 6 | local fiber = require 'fiber' 7 | local clock = require 'clock' 8 | 9 | local http_client = require 'http.client' 10 | local digest = require 'digest' 11 | 12 | ---@class moonlibs.config.etcd.opts 13 | ---@field endpoints? string[] (default: {'http://127.0.0.1:4001','http://127.0.0.1:2379'}) list of clientURLs to etcd 14 | ---@field timeout? number (default: 1) timeout of request to each node to etcd 15 | ---@field boolean_auto? boolean (default: false) when true each string value `true`, `false` is converted to boolean value 16 | ---@field print_config? boolean (default: false) when true loaded configuration from etcd is printed out 17 | ---@field discover_endpoints? boolean (default: true) when false connector does not automatically discovers etcd endpoints 18 | ---@field reduce_listing_quorum? boolean (default: false) when true connector does not request etcd:list with quorum 19 | ---@field login? string allows to specify username for each request (Basic-auth) 20 | ---@field password? string allows to specify password for each request (Basic-auth) 21 | 22 | ---@class moonlibs.config.etcd 23 | ---@field endpoints string[] (default: {'http://127.0.0.1:4001','http://127.0.0.1:2379'}) list of clientURLs to etcd 24 | ---@field client http 25 | ---@field timeout number (default: 1) timeout of request to each node to etcd 26 | ---@field boolean_auto? boolean (default: false) when true each string value `true`, `false` is converted to boolean value 27 | ---@field print_config? boolean (default: false) when true loaded configuration from etcd is printed out 28 | ---@field discover_endpoints boolean (default: true) when false connector does not automatically discovers etcd endpoints 29 | ---@field reduce_listing_quorum? boolean (default: false) when true connector does not request etcd:list with quorum 30 | ---@field authorization? string Authorization header for Basic-auth (is set only when login is present) 31 | ---@field headers? table headers which are provided on each request 32 | local M = {} 33 | 34 | M.err = {} 35 | M.EcodeKeyNotFound = 100; M.err[M.EcodeKeyNotFound] = "Key not found" 36 | M.EcodeTestFailed = 101; M.err[M.EcodeTestFailed] = "Compare failed" 37 | M.EcodeNotFile = 102; M.err[M.EcodeNotFile] = "Not a file" 38 | M.EcodeNotDir = 104; M.err[M.EcodeNotDir] = "Not a directory" 39 | M.EcodeNodeExist = 105; M.err[M.EcodeNodeExist] = "Key already exists" 40 | M.EcodeRootROnly = 107; M.err[M.EcodeRootROnly] = "Root is read only" 41 | M.EcodeDirNotEmpty = 108; M.err[M.EcodeDirNotEmpty] = "Directory not empty" 42 | M.EcodePrevValueRequired = 201; M.err[M.EcodePrevValueRequired] = "PrevValue is Required in POST form" 43 | M.EcodeTTLNaN = 202; M.err[M.EcodeTTLNaN] = "The given TTL in POST form is not a number" 44 | M.EcodeIndexNaN = 203; M.err[M.EcodeIndexNaN] = "The given index in POST form is not a number" 45 | M.EcodeInvalidField = 209; M.err[M.EcodeInvalidField] = "Invalid field" 46 | M.EcodeInvalidForm = 210; M.err[M.EcodeInvalidForm] = "Invalid POST form" 47 | M.EcodeRaftInternal = 300; M.err[M.EcodeRaftInternal] = "Raft Internal Error" 48 | M.EcodeLeaderElect = 301; M.err[M.EcodeLeaderElect] = "During Leader Election" 49 | M.EcodeWatcherCleared = 400; M.err[M.EcodeWatcherCleared] = "watcher is cleared due to etcd recovery" 50 | M.EcodeEventIndexCleared = 401; M.err[M.EcodeEventIndexCleared] = "The event in requested index is outdated and cleared" 51 | 52 | function M.errstr(code) 53 | return M.err[ tonumber(code) ] or string.format("Unknown error %s",code) 54 | end 55 | 56 | ---Creates new etcd connector 57 | ---@param mod moonlibs.config.etcd 58 | ---@param options moonlibs.config.etcd.opts 59 | ---@return moonlibs.config.etcd 60 | function M.new(mod,options) 61 | local self = setmetatable({},{__index=mod}) 62 | self.endpoints = options.endpoints or {'http://127.0.0.1:4001','http://127.0.0.1:2379'} 63 | -- self.prefix = options.prefix or '' 64 | self.timeout = options.timeout or 1 65 | self.client = http_client -- .new() - it fix for 1.6 also client: -> clent. 66 | self.boolean_auto = options.boolean_auto 67 | self.print_config = options.print_config 68 | self.discover_endpoints = options.discover_endpoints == nil and true or options.discover_endpoints 69 | if options.reduce_listing_quorum == true then 70 | self.reduce_listing_quorum = true 71 | end 72 | if options.login then 73 | self.authorization = "Basic "..digest.base64_encode(options.login..":"..(options.password or "")) 74 | self.headers = { authorization = self.authorization } 75 | end 76 | return self 77 | end 78 | 79 | setmetatable(M,{ __call = M.new }) 80 | 81 | ---Discovers every ETCD endpoint by requesting clientURLs (/v2/members) 82 | --- ?: make it parallel 83 | function M:discovery() 84 | local start_at = clock.time() 85 | local timeout = self.timeout or 1 86 | local new_endpoints = {} 87 | local tried = {} 88 | for _,e in pairs(self.endpoints) do 89 | local uri = e .. "/v2/members" 90 | local x = self.client.request("GET",uri,'',{timeout = timeout; headers = self.headers}) 91 | if x and x.status == 200 then 92 | if x.headers['content-type'] == 'application/json' then 93 | local data = json.decode( x.body ) 94 | local hash_endpoints = {} 95 | for _,m in pairs(data.members) do 96 | -- print(yaml.encode(m)) 97 | for _, u in pairs(m.clientURLs) do 98 | hash_endpoints[u] = true 99 | end 100 | end 101 | for k in pairs(hash_endpoints) do 102 | table.insert(new_endpoints,k) 103 | end 104 | if #new_endpoints > 0 then 105 | break 106 | end 107 | else 108 | table.insert(tried, e..": bad reply") 109 | end 110 | elseif x and x.status == 404 then 111 | table.insert(tried, e..": no /v2/members: possible run without --enable-v2?") 112 | else 113 | table.insert(tried, e..": "..(x and x.status)) 114 | end 115 | end 116 | if #new_endpoints == 0 then 117 | error("Failed to discover members "..table.concat(tried,", ")..(" in %.3fs"):format(clock.time()-start_at),2) 118 | end 119 | if self.discover_endpoints then 120 | self.endpoints = new_endpoints 121 | table.insert(self.endpoints,table.remove(self.endpoints,1)) 122 | log.info("discovered etcd endpoints "..table.concat(self.endpoints,", ")..(" in %.3fs"):format(clock.time()-start_at)) 123 | else 124 | log.info("hardcoded etcd endpoints "..table.concat(self.endpoints,", ")..(" in %.3fs"):format(clock.time()-start_at)) 125 | end 126 | self.current = math.random(#self.endpoints) 127 | end 128 | 129 | ---@class moonlibs.etcd.request.opts 130 | ---@field deadline? number deadline of request (in seconds, fractional) 131 | ---@field timeout? number timeout of request to each node (in seconds, fractional) 132 | ---@field body? string request body (for PUT) 133 | 134 | ---Performs etcd request 135 | ---@param method 'PUT'|'GET'|'DELETE'|'HEAD' http_method 136 | ---@param path string etcd path after /v2/ 137 | ---@param args? moonlibs.etcd.request.opts 138 | ---@return table, HTTPResponse 139 | function M:request(method, path, args ) 140 | -- path must be prefixed outside 141 | -- TODO: auth 142 | local query = {} 143 | if args then 144 | for k,v in pairs(args) do 145 | if #query > 0 then table.insert(query,'&') end 146 | table.insert(query, k) 147 | table.insert(query, '=') 148 | table.insert(query, tostring(v)) 149 | end 150 | else 151 | args = {} 152 | end 153 | local qs 154 | if #query > 0 then qs = '?'..table.concat(query) else qs = '' end 155 | local body = args and args.body or '' 156 | local lasterror, lastresponse 157 | local deadline = args.deadline 158 | 159 | local len = #self.endpoints 160 | for i = 0, len - 1 do 161 | local cur = self.current + i 162 | if cur > len then 163 | cur = cur % len 164 | end 165 | local uri = string.format("%s/v2/%s%s", self.endpoints[cur], path, qs ) 166 | -- print("[debug] "..uri) 167 | local request_timeout = args.timeout or self.timeout or 1 168 | if deadline then 169 | request_timeout = math.min(deadline-fiber.time(), request_timeout) 170 | end 171 | local s = clock.time() 172 | local x = self.client.request(method,uri,body,{timeout = request_timeout; headers = self.headers}) 173 | lastresponse = x 174 | local status,reply = pcall(json.decode,x and x.body) 175 | local logger = log.verbose 176 | if x.status >= 500 then 177 | logger = log.error 178 | end 179 | logger("%s %s (to:%.3fs) finished with %s%s %s (in %.3fs)", 180 | method, uri, request_timeout, x.status, 181 | status and reply and reply.errorCode and (reply.message or M.err[reply.errorCode] or reply.errorCode), 182 | (x.headers or {})['X-Etcd-Index'], 183 | clock.time()-s 184 | ) 185 | 186 | -- 408 for timeout 187 | if x.status < 500 and x.status ~= 408 then 188 | if status then 189 | self.current = cur 190 | return reply, lastresponse 191 | else 192 | -- passthru 193 | lasterror = { errorCode = 500, message = x.reason } 194 | end 195 | else 196 | if status then 197 | lasterror = reply 198 | else 199 | lasterror = { errorCode = 500, message = x.reason } 200 | end 201 | end 202 | 203 | if deadline and deadline < fiber.time() then 204 | break 205 | end 206 | end 207 | return lasterror, lastresponse 208 | end 209 | 210 | function M:recursive_extract(cut, node, storage) 211 | local _storage 212 | if not storage then _storage = {} else _storage = storage end 213 | 214 | local key 215 | if string.sub(node.key,1,#cut) == cut then 216 | key = string.sub(node.key,#cut+2) 217 | else 218 | key = node.key 219 | end 220 | 221 | if node.dir then 222 | _storage[key] = {} 223 | for _,v in pairs(node.nodes) do 224 | self:recursive_extract(node.key, v, _storage[key]) 225 | end 226 | else 227 | -- ex: {"createdIndex":108,"modifiedIndex":108,"key":".../cluster","value":"instance_001"} 228 | if self.boolean_auto then 229 | if node.value == 'true' then 230 | node.value = true 231 | elseif node.value == 'false' then 232 | node.value = false 233 | end 234 | end 235 | local num = tonumber(node.value) 236 | if num then 237 | _storage[ key ] = num 238 | else 239 | _storage[ key ] = node.value 240 | end 241 | -- TODO: remember index 242 | -- print("key",key, node.value, json.encode(node)) 243 | end 244 | 245 | if not storage then return _storage[''] end 246 | end 247 | 248 | ---@class moonlibs.config.etcd.list.opts:moonlibs.etcd.request.opts 249 | ---@field recursive? boolean (default: true) should listing be recursive 250 | ---@field quorum? boolean (default: not reduce_listing_quorum) when true requests quorum read 251 | 252 | ---Performs listing by given path 253 | ---@param keyspath string path inside etcd 254 | ---@param opts moonlibs.config.etcd.list.opts 255 | ---@return unknown 256 | ---@return HTTPResponse 257 | function M:list(keyspath, opts) 258 | if type(opts) ~= 'table' then 259 | opts = {} 260 | end 261 | if opts.recursive == nil then 262 | opts.recursive = true 263 | end 264 | if opts.quorum == nil then 265 | opts.quorum = not self.reduce_listing_quorum 266 | end 267 | local res, response = self:request("GET","keys"..keyspath, opts) 268 | -- print(yaml.encode(res)) 269 | if res.node then 270 | local result = self:recursive_extract(keyspath,res.node) 271 | -- todo: make it with metatable 272 | -- print(yaml.encode(result)) 273 | return result, response 274 | -- for _,n in pairs(res.node) do 275 | -- print() 276 | -- end 277 | else 278 | error(json.encode(res),2) 279 | end 280 | end 281 | 282 | ---@class moonlibs.config.etcd.wait.opts 283 | ---@field timeout? number (default: etcd.timeout) timeout for each node to await changes 284 | ---@field index number etcd-index that should be awaited 285 | 286 | ---Awaits any change in subtree recursively 287 | ---@param keyspath string 288 | ---@param args moonlibs.config.etcd.wait.opts 289 | ---@return boolean not_timed_out, HTTPResponse 290 | function M:wait(keyspath, args) 291 | args = args or {} 292 | local _, response = self:request("GET","keys"..keyspath, { 293 | wait = true, 294 | recursive = true, 295 | timeout = args.timeout, 296 | waitIndex = args.index, 297 | }) 298 | return response.status ~= 408, response 299 | end 300 | 301 | return M 302 | -------------------------------------------------------------------------------- /override-config-dev-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "override-config" 2 | version = "dev-1" 3 | source = { 4 | url = "git+https://github.com/moonlibs/config", 5 | branch = "master" 6 | } 7 | description = { 8 | summary = "Package for loading external lua config (override)", 9 | homepage = "https://github.com/moonlibs/config.git", 10 | license = "BSD" 11 | } 12 | dependencies = { 13 | "lua >= 5.1" 14 | } 15 | build = { 16 | type = "builtin", 17 | modules = { 18 | ["override.config"] = "config.lua", 19 | ["override.config.etcd"] = "config/etcd.lua" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /rockspecs/config-scm-1.rockspec: -------------------------------------------------------------------------------- 1 | package = 'config' 2 | version = 'scm-1' 3 | source = { 4 | url = 'git+https://github.com/moonlibs/config.git', 5 | branch = 'v1', 6 | } 7 | description = { 8 | summary = "Package for loading external lua config", 9 | homepage = 'https://github.com/moonlibs/config.git', 10 | license = 'BSD', 11 | } 12 | dependencies = { 13 | 'lua >= 5.1' 14 | } 15 | build = { 16 | type = 'builtin', 17 | modules = { 18 | ['config'] = 'config.lua' 19 | } 20 | } 21 | 22 | -- vim: syntax=lua 23 | -------------------------------------------------------------------------------- /rockspecs/config-scm-2.rockspec: -------------------------------------------------------------------------------- 1 | package = 'config' 2 | version = 'scm-2' 3 | source = { 4 | url = 'git+https://github.com/moonlibs/config.git', 5 | branch = 'v2', 6 | } 7 | description = { 8 | summary = "Package for loading external lua config", 9 | homepage = 'https://github.com/moonlibs/config.git', 10 | license = 'BSD', 11 | } 12 | dependencies = { 13 | 'lua >= 5.1' 14 | } 15 | build = { 16 | type = 'builtin', 17 | modules = { 18 | ['config'] = 'config.lua'; 19 | ['config.etcd'] = 'config/etcd.lua'; 20 | } 21 | } 22 | 23 | -- vim: syntax=lua 24 | -------------------------------------------------------------------------------- /rockspecs/config-scm-3.rockspec: -------------------------------------------------------------------------------- 1 | package = 'config' 2 | version = 'scm-3' 3 | source = { 4 | url = 'git+https://github.com/moonlibs/config.git', 5 | branch = 'v3', 6 | } 7 | description = { 8 | summary = "Package for loading external lua config", 9 | homepage = 'https://github.com/moonlibs/config.git', 10 | license = 'BSD', 11 | } 12 | dependencies = { 13 | 'lua >= 5.1' 14 | } 15 | build = { 16 | type = 'builtin', 17 | modules = { 18 | ['config'] = 'config.lua'; 19 | ['config.etcd'] = 'config/etcd.lua'; 20 | } 21 | } 22 | 23 | -- vim: syntax=lua 24 | -------------------------------------------------------------------------------- /rockspecs/config-scm-4.rockspec: -------------------------------------------------------------------------------- 1 | package = 'config' 2 | version = 'scm-4' 3 | source = { 4 | url = 'git+https://github.com/moonlibs/config.git', 5 | branch = 'v4', 6 | } 7 | description = { 8 | summary = "Package for loading external lua config", 9 | homepage = 'https://github.com/moonlibs/config.git', 10 | license = 'BSD', 11 | } 12 | dependencies = { 13 | 'lua >= 5.1' 14 | } 15 | build = { 16 | type = 'builtin', 17 | modules = { 18 | ['config'] = 'config.lua'; 19 | ['config.etcd'] = 'config/etcd.lua'; 20 | } 21 | } 22 | 23 | -- vim: syntax=lua 24 | -------------------------------------------------------------------------------- /rockspecs/config-scm-5.rockspec: -------------------------------------------------------------------------------- 1 | package = 'config' 2 | version = 'scm-5' 3 | source = { 4 | url = 'git+https://github.com/moonlibs/config.git', 5 | branch = 'master', 6 | } 7 | description = { 8 | summary = "Package for loading external lua config", 9 | homepage = 'https://github.com/moonlibs/config.git', 10 | license = 'BSD', 11 | } 12 | dependencies = { 13 | 'lua >= 5.1' 14 | } 15 | build = { 16 | type = 'builtin', 17 | modules = { 18 | ['config'] = 'config.lua'; 19 | ['config.etcd'] = 'config/etcd.lua'; 20 | } 21 | } 22 | 23 | -- vim: syntax=lua 24 | -------------------------------------------------------------------------------- /run_test_in_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pwd 4 | rm -rf /root/.cache/ 5 | cp -ar /root/.rocks /source/config/ 6 | /source/config/.rocks/bin/luatest --coverage -v spec/ 7 | -------------------------------------------------------------------------------- /spec/01_single_test.lua: -------------------------------------------------------------------------------- 1 | local t = require 'luatest' --[[@as luatest]] 2 | local uri = require 'uri' 3 | 4 | ---@class test.config.single:luatest.group 5 | local g = t.group('single', { 6 | { instances = {single = '127.0.0.1:3301'}, run = {'single'} }, 7 | { 8 | instances = {single_01 = '127.0.0.1:3301', single_02 = '127.0.0.1:3302'}, 9 | run = {'single_01', 'single_02'} 10 | }, 11 | { 12 | instances = {single_01 = '127.0.0.1:3301', single_02 = '127.0.0.1:3302'}, 13 | run = {'single_01'} 14 | }, 15 | }) 16 | 17 | local this_file = debug.getinfo(1, "S").source:sub(2) 18 | local fio = require 'fio' 19 | 20 | local root = fio.dirname(this_file) 21 | local init_lua = fio.pathjoin(root, 'mock', 'single', 'init.lua') 22 | 23 | local base_env 24 | 25 | local h = require 'spec.helper' 26 | local test_ctx = {} 27 | 28 | g.before_each(function(cg) 29 | local working_dir = h.create_workdir() 30 | base_env = { 31 | TT_ETCD_PREFIX = '/apps/single', 32 | TT_CONFIG = fio.pathjoin(root, 'mock', 'single', 'conf.lua'), 33 | TT_MASTER_SELECTION_POLICY = 'etcd.instance.single', 34 | TT_ETCD_ENDPOINTS = os.getenv('TT_ETCD_ENDPOINTS') or "http://127.0.0.1:2379", 35 | } 36 | base_env.TT_WAL_DIR = working_dir 37 | base_env.TT_MEMTX_DIR = working_dir 38 | base_env.TT_WORK_DIR = working_dir 39 | 40 | local base_config = { 41 | apps = { 42 | single = { 43 | common = { box = { log_level = 1 } }, 44 | } 45 | } 46 | } 47 | h.clear_etcd() 48 | 49 | local params = cg.params 50 | 51 | local etcd_config = table.deepcopy(base_config) 52 | etcd_config.apps.single.instances = {} 53 | for instance_name, listen_uri in pairs(params.instances) do 54 | etcd_config.apps.single.instances[instance_name] = { box = { listen = listen_uri } } 55 | end 56 | 57 | local ctx = { tts = {}, env = base_env, etcd_config = etcd_config, params = cg.params } 58 | test_ctx[cg.name] = ctx 59 | 60 | h.upload_to_etcd(etcd_config) 61 | end) 62 | 63 | g.after_each(function() 64 | for _, info in pairs(test_ctx) do 65 | for _, tt in pairs(info.tts) do 66 | tt.server:stop() 67 | end 68 | h.clean_directory(info.env.TT_WAL_DIR) 69 | h.clean_directory(info.env.TT_MEMTX_DIR) 70 | h.clean_directory(info.env.TT_WORK_DIR) 71 | end 72 | 73 | h.clear_etcd() 74 | end) 75 | 76 | function g.test_run_instances(cg) 77 | local ctx = test_ctx[cg.name] 78 | 79 | -- Start tarantools 80 | h.start_all_tarantools(ctx, init_lua, root, ctx.etcd_config.apps.single.instances) 81 | 82 | for _, tt in ipairs(ctx.tts) do 83 | tt.server:connect_net_box() 84 | local box_cfg = tt.server:get_box_cfg() 85 | t.assert_covers(box_cfg, { 86 | log_level = ctx.etcd_config.apps.single.common.box.log_level, 87 | listen = ctx.etcd_config.apps.single.instances[tt.name].box.listen, 88 | read_only = false, 89 | }, 'box.cfg is correct') 90 | 91 | local conn = tt.server --[[@as luatest.server]] 92 | local ret = conn:exec(function() 93 | local r = table.deepcopy(config.get('sys')) 94 | for k, v in pairs(r) do 95 | if type(v) == 'function' then 96 | r[k] = nil 97 | end 98 | end 99 | return r 100 | end) 101 | 102 | t.assert_covers(ret, { 103 | instance_name = tt.name, 104 | master_selection_policy = 'etcd.instance.single', 105 | file = base_env.TT_CONFIG, 106 | }, 'get("sys") has correct fields') 107 | end 108 | 109 | -- restart tarantools 110 | for _, tt in ipairs(ctx.tts) do 111 | local conn = tt.server --[[@as luatest.server]] 112 | h.restart_tarantool(conn) 113 | 114 | local box_cfg = tt.server:get_box_cfg() 115 | t.assert_covers(box_cfg, { 116 | log_level = ctx.etcd_config.apps.single.common.box.log_level, 117 | listen = ctx.etcd_config.apps.single.instances[tt.name].box.listen, 118 | read_only = false, 119 | }, 'box.cfg is correct after restart') 120 | 121 | local ret = conn:exec(function() 122 | local r = table.deepcopy(config.get('sys')) 123 | for k, v in pairs(r) do 124 | if type(v) == 'function' then 125 | r[k] = nil 126 | end 127 | end 128 | return r 129 | end) 130 | 131 | t.assert_covers(ret, { 132 | instance_name = tt.name, 133 | master_selection_policy = 'etcd.instance.single', 134 | file = base_env.TT_CONFIG, 135 | }, 'get("sys") has correct fields after restart') 136 | end 137 | end 138 | -------------------------------------------------------------------------------- /spec/02_cluster_master_test.lua: -------------------------------------------------------------------------------- 1 | local t = require 'luatest' --[[@as luatest]] 2 | local uuid = require 'uuid' 3 | local fiber = require 'fiber' 4 | 5 | ---@class test.config.master:luatest.group 6 | local g = t.group('master', { 7 | { 8 | cluster = 'single', 9 | master = 'first_01', 10 | instances = {first_01 = '127.0.0.1:3301', first_02 = '127.0.0.1:3302'}, 11 | run = {'first_01', 'first_02'} 12 | }, 13 | { 14 | cluster = 'single', 15 | master = 'second_01', 16 | instances = {second_01 = '127.0.0.1:3301', second_02 = '127.0.0.1:3302'}, 17 | run = {'second_01'} 18 | }, 19 | { 20 | cluster = 'single', 21 | master = 'third_01', 22 | instances = {third_01 = '127.0.0.1:3301', third_02 = '127.0.0.1:3302',third_03='127.0.0.1:3303'}, 23 | run = {'third_03','third_02','third_01'} 24 | }, 25 | }) 26 | 27 | local this_file = debug.getinfo(1, "S").source:sub(2) 28 | local fio = require 'fio' 29 | 30 | local root = fio.dirname(this_file) 31 | local init_lua = fio.pathjoin(root, 'mock', 'single', 'init.lua') 32 | 33 | local base_env 34 | 35 | local h = require 'spec.helper' 36 | 37 | ---@class moonlibs.config.test.tarantool 38 | ---@field server luatest.server 39 | ---@field net_box_port number 40 | ---@field env table 41 | ---@field name string 42 | 43 | ---@class moonlibs.config.test.context 44 | ---@field tts moonlibs.config.test.tarantool[] 45 | ---@field env table 46 | ---@field etcd_config table 47 | ---@field params table 48 | 49 | ---@type table 50 | local test_ctx = {} 51 | 52 | g.before_each(function(cg) 53 | local working_dir = h.create_workdir() 54 | base_env = { 55 | TT_ETCD_PREFIX = '/apps/single', 56 | TT_CONFIG = fio.pathjoin(root, 'mock', 'single', 'conf.lua'), 57 | TT_MASTER_SELECTION_POLICY = 'etcd.cluster.master', 58 | TT_ETCD_ENDPOINTS = os.getenv('TT_ETCD_ENDPOINTS') or "http://127.0.0.1:2379", 59 | } 60 | 61 | base_env.TT_WAL_DIR = working_dir 62 | base_env.TT_MEMTX_DIR = working_dir 63 | 64 | local base_config = { 65 | apps = { 66 | single = { 67 | common = { 68 | etcd = { fencing_enabled = true }, 69 | box = { log_level = 5 }, 70 | }, 71 | clusters = { 72 | single = { 73 | master = cg.params.master, 74 | replicaset_uuid = uuid.str(), 75 | } 76 | }, 77 | } 78 | }, 79 | } 80 | h.clear_etcd() 81 | 82 | local etcd_config = table.deepcopy(base_config) 83 | etcd_config.apps.single.instances = {} 84 | for instance_name, listen_uri in pairs(cg.params.instances) do 85 | etcd_config.apps.single.instances[instance_name] = { 86 | box = { listen = listen_uri }, 87 | cluster = cg.params.cluster, 88 | } 89 | end 90 | 91 | local this_ctx = { tts = {}, env = base_env, etcd_config = etcd_config, params = cg.params } 92 | test_ctx[cg.name] = this_ctx 93 | 94 | h.upload_to_etcd(etcd_config) 95 | end) 96 | 97 | g.after_each(function() 98 | for _, info in pairs(test_ctx) do 99 | for _, tt in pairs(info.tts) do 100 | tt.server:stop() 101 | end 102 | h.clean_directory(info.env.TT_WAL_DIR) 103 | h.clean_directory(info.env.TT_MEMTX_DIR) 104 | end 105 | 106 | h.clear_etcd() 107 | end) 108 | 109 | function g.test_run_instances(cg) 110 | local ctx = test_ctx[cg.name] 111 | 112 | -- Start tarantools 113 | h.start_all_tarantools(ctx, init_lua, root, ctx.etcd_config.apps.single.instances) 114 | 115 | -- Check configuration 116 | for _, tnt in ipairs(ctx.tts) do 117 | tnt.server:connect_net_box() 118 | local box_cfg = tnt.server:get_box_cfg() 119 | t.assert_covers(box_cfg, { 120 | log_level = ctx.etcd_config.apps.single.common.box.log_level, 121 | listen = ctx.etcd_config.apps.single.instances[tnt.name].box.listen, 122 | read_only = ctx.etcd_config.apps.single.clusters.single.master ~= tnt.name, 123 | }, 'box.cfg is correct') 124 | 125 | local conn = tnt.server --[[@as luatest.server]] 126 | local ret = conn:exec(function() 127 | local r = table.deepcopy(config.get('sys')) 128 | for k, v in pairs(r) do 129 | if type(v) == 'function' then 130 | r[k] = nil 131 | end 132 | end 133 | return r 134 | end) 135 | 136 | t.assert_covers(ret, { 137 | instance_name = tnt.name, 138 | master_selection_policy = 'etcd.cluster.master', 139 | file = base_env.TT_CONFIG, 140 | }, 'get("sys") has correct fields') 141 | end 142 | 143 | -- restart+check configuration 144 | for _, tt in ipairs(ctx.tts) do 145 | h.restart_tarantool(tt.server) 146 | 147 | local box_cfg = tt.server:get_box_cfg() 148 | t.assert_covers(box_cfg, { 149 | log_level = ctx.etcd_config.apps.single.common.box.log_level, 150 | listen = ctx.etcd_config.apps.single.instances[tt.name].box.listen, 151 | read_only = ctx.etcd_config.apps.single.clusters.single.master ~= tt.name, 152 | }, 'box.cfg is correct after restart') 153 | 154 | local ret = tt.server:exec(function() 155 | local r = table.deepcopy(config.get('sys')) 156 | for k, v in pairs(r) do 157 | if type(v) == 'function' then 158 | r[k] = nil 159 | end 160 | end 161 | return r 162 | end) 163 | 164 | t.assert_covers(ret, { 165 | instance_name = tt.name, 166 | master_selection_policy = 'etcd.cluster.master', 167 | file = base_env.TT_CONFIG, 168 | }, 'get("sys") has correct fields after restart') 169 | end 170 | end 171 | 172 | function g.test_reload(cg) 173 | local ctx = test_ctx[cg.name] 174 | 175 | -- Start tarantools 176 | h.start_all_tarantools(ctx, init_lua, root, ctx.etcd_config.apps.single.instances) 177 | 178 | -- reload+check configuration 179 | for _, tt in ipairs(ctx.tts) do 180 | h.reload_tarantool(tt.server) 181 | 182 | local box_cfg = tt.server:get_box_cfg() 183 | t.assert_covers(box_cfg, { 184 | log_level = ctx.etcd_config.apps.single.common.box.log_level, 185 | listen = ctx.etcd_config.apps.single.instances[tt.name].box.listen, 186 | read_only = ctx.etcd_config.apps.single.clusters.single.master ~= tt.name, 187 | }, 'box.cfg is correct after restart') 188 | 189 | local ret = tt.server:exec(function() 190 | local r = table.deepcopy(config.get('sys')) 191 | for k, v in pairs(r) do 192 | if type(v) == 'function' then 193 | r[k] = nil 194 | end 195 | end 196 | return r 197 | end) 198 | 199 | t.assert_covers(ret, { 200 | instance_name = tt.name, 201 | master_selection_policy = 'etcd.cluster.master', 202 | file = base_env.TT_CONFIG, 203 | }, 'get("sys") has correct fields after restart') 204 | end 205 | end 206 | 207 | function g.test_fencing(cg) 208 | local ctx = test_ctx[cg.name] 209 | t.skip_if(not ctx.etcd_config.apps.single.common.etcd.fencing_enabled, "fencing disabled") 210 | 211 | -- Start tarantools 212 | h.start_all_tarantools(ctx, init_lua, root, ctx.etcd_config.apps.single.instances) 213 | 214 | -- Check configuration 215 | for _, tnt in ipairs(ctx.tts) do 216 | tnt.server:connect_net_box() 217 | local box_cfg = tnt.server:get_box_cfg() 218 | t.assert_covers(box_cfg, { 219 | log_level = ctx.etcd_config.apps.single.common.box.log_level, 220 | listen = ctx.etcd_config.apps.single.instances[tnt.name].box.listen, 221 | read_only = ctx.etcd_config.apps.single.clusters.single.master ~= tnt.name, 222 | }, 'box.cfg is correct') 223 | 224 | local conn = tnt.server --[[@as luatest.server]] 225 | local ret = conn:exec(function() 226 | local r = table.deepcopy(config.get('sys')) 227 | for k, v in pairs(r) do 228 | if type(v) == 'function' then 229 | r[k] = nil 230 | end 231 | end 232 | return r 233 | end) 234 | 235 | t.assert_covers(ret, { 236 | instance_name = tnt.name, 237 | master_selection_policy = 'etcd.cluster.master', 238 | file = base_env.TT_CONFIG, 239 | }, 'get("sys") has correct fields') 240 | end 241 | 242 | local master_name = ctx.params.master 243 | 244 | ---@type moonlibs.config.test.tarantool 245 | local master 246 | for _, tt in ipairs(ctx.tts) do 247 | if tt.name == master_name then 248 | master = tt 249 | break 250 | end 251 | end 252 | 253 | t.assert(master, 'master is not connected') 254 | 255 | local ret = master.server:exec(function() 256 | return { cfg_ro = box.cfg.read_only, ro = box.info.ro } 257 | end) 258 | 259 | t.assert_equals(ret.cfg_ro, false, 'box.cfg.read_only == false (before fencing)') 260 | t.assert_equals(ret.ro, false, 'box.info.ro == false (before fencing)') 261 | 262 | ctx.etcd_config.apps.single.clusters.single.master = 'not_exists' 263 | h.upload_to_etcd(ctx.etcd_config) 264 | 265 | local fencing_cfg = ctx.etcd_config.apps.single.common.etcd 266 | local fencing_timeout = fencing_cfg.fencing_timeout or 10 267 | local fencing_pause = fencing_cfg.fencing_pause or fencing_timeout/2 268 | 269 | t.helpers.retrying({ 270 | timeout = fencing_pause, 271 | delay = 0.1, 272 | }, function () 273 | local ret = master.server:exec(function() 274 | return { cfg_ro = box.cfg.read_only, ro = box.info.ro } 275 | end) 276 | assert(ret.cfg_ro, "cfg.read_only must be true") 277 | assert(ret.ro, "info.ro must be true") 278 | end) 279 | 280 | local ret = master.server:exec(function() 281 | return { cfg_ro = box.cfg.read_only, ro = box.info.ro } 282 | end) 283 | 284 | t.assert_equals(ret.cfg_ro, true, 'box.cfg.read_only == true') 285 | t.assert_equals(ret.ro, true, 'box.info.ro == true') 286 | 287 | ctx.etcd_config.apps.single.clusters.single.master = master_name 288 | h.upload_to_etcd(ctx.etcd_config) 289 | 290 | local deadline = 2*fencing_timeout+fiber.time() 291 | while fiber.time() < deadline do 292 | local ret2 = master.server:exec(function() 293 | return { cfg_ro = box.cfg.read_only, ro = box.info.ro } 294 | end) 295 | 296 | t.assert_equals(ret2.cfg_ro, true, 'box.cfg.read_only == true (double check)') 297 | t.assert_equals(ret2.ro, true, 'box.info.ro == true (double check)') 298 | end 299 | end 300 | -------------------------------------------------------------------------------- /spec/helper.lua: -------------------------------------------------------------------------------- 1 | local h = {} 2 | local t = require 'luatest' --[[@as luatest]] 3 | local fio = require 'fio' 4 | local log = require 'log' 5 | local uri = require 'uri' 6 | local fun = require 'fun' 7 | local clock = require 'clock' 8 | local fiber = require 'fiber' 9 | local json = require 'json' 10 | 11 | ---Creates temporary working directory 12 | ---@return string 13 | function h.create_workdir() 14 | local tempdir = fio.tempdir() 15 | assert(fio.mktree(tempdir)) 16 | 17 | return tempdir 18 | end 19 | 20 | ---Removes all data in the path 21 | ---@param path string 22 | function h.clean_directory(path) 23 | if fio.path.is_dir(path) then 24 | assert(fio.rmtree(path)) 25 | elseif fio.path.is_file(path) then 26 | assert(fio.unlink(path)) 27 | end 28 | end 29 | 30 | ---@param tree table 31 | ---@param path string? 32 | ---@param ret table? 33 | ---@return table 34 | local function flatten(tree, path, ret) 35 | path = path or '' 36 | ---@type table 37 | ret = ret or {} 38 | for key, sub in pairs(tree) do 39 | local base_type = type(sub) 40 | if base_type ~= 'table' then 41 | ret[path..'/'..key] = tostring(sub) 42 | else 43 | flatten(sub, path .. '/' .. key, ret) 44 | end 45 | end 46 | return ret 47 | end 48 | 49 | ---comment 50 | ---@return EtcdCfg 51 | function h.get_etcd() 52 | local endpoints = (os.getenv('TT_ETCD_ENDPOINTS') or "http://127.0.0.1:2379") 53 | :gsub(",+", ",") 54 | :gsub(',$','') 55 | :split(',') 56 | 57 | local etcd = require 'config.etcd':new{ 58 | endpoints = endpoints, 59 | prefix = '/', 60 | debug = true, 61 | } 62 | 63 | t.helpers.retrying({ timeout = 5 }, function() etcd:discovery() end) 64 | 65 | return etcd 66 | end 67 | 68 | function h.clear_etcd() 69 | local etcd = h.get_etcd() 70 | 71 | local _, res = etcd:request('DELETE', 'keys/apps', { recursive = true, dir = true, force = true }) 72 | log.info("clear_etcd(%s) => %s:%s", '/apps', res.status, res.reason) 73 | assert(res.status >= 200 and(res.status < 300 or res.status == 404), ("%s %s"):format(res.status, res.body)) 74 | end 75 | 76 | function h.upload_to_etcd(tree) 77 | local etcd = h.get_etcd() 78 | 79 | local flat = flatten(tree) 80 | local keys = fun.totable(flat) 81 | table.sort(keys) 82 | for _, key in ipairs(keys) do 83 | do 84 | local _, res = etcd:request('PUT', 'keys'..key, { value = flat[key], quorum = true }) 85 | assert(res.status < 300 and res.status >= 200, res.reason) 86 | end 87 | end 88 | 89 | local key = keys[1]:match('^(/[^/]+)') 90 | log.info("list(%s): => %s", key, json.encode(etcd:list(key))) 91 | end 92 | 93 | ---Starts new tarantool server 94 | ---@param opts luatest.server.options 95 | ---@return luatest.server 96 | function h.start_tarantool(opts) 97 | log.info("starting tarantool %s", json.encode(opts)) 98 | local srv = t.Server:new(opts) 99 | srv:start() 100 | 101 | local process = srv.process 102 | 103 | local deadline = clock.time() + 30 104 | while clock.time() < deadline do 105 | fiber.sleep(0.1) 106 | if process:is_alive() then break end 107 | end 108 | return srv 109 | end 110 | 111 | function h.start_all_tarantools(ctx, init_lua, root, instances) 112 | for _, name in ipairs(ctx.params.run) do 113 | local env = table.deepcopy(ctx.env) 114 | env.TT_INSTANCE_NAME = name 115 | local net_box_port = tonumber(uri.parse(instances[name].box.listen).service) 116 | 117 | local tt = h.start_tarantool({ 118 | alias = name, 119 | env = env, 120 | command = init_lua, 121 | args = {}, 122 | net_box_port = net_box_port, 123 | workdir = root, 124 | }) 125 | 126 | table.insert(ctx.tts, { 127 | server = tt, 128 | net_box_port = net_box_port, 129 | env = env, 130 | name = name, 131 | }) 132 | end 133 | 134 | for _, tt in ipairs(ctx.tts) do 135 | h.wait_tarantool(tt.server) 136 | end 137 | end 138 | 139 | ---@param srv luatest.server 140 | function h.wait_tarantool(srv) 141 | t.helpers.retrying({ timeout = 30, delay = 0.1 }, function () 142 | srv:connect_net_box() 143 | srv:call('box.info') 144 | end) 145 | end 146 | 147 | ---@param server luatest.server 148 | function h.restart_tarantool(server) 149 | server:stop() 150 | local deadline = clock.time() + 15 151 | 152 | fiber.sleep(3) 153 | server:start() 154 | 155 | while clock.time() < deadline do 156 | fiber.sleep(3) 157 | assert(server.process:is_alive(), "tarantool is dead") 158 | 159 | if pcall(function() server:connect_net_box() end) then 160 | break 161 | end 162 | 163 | end 164 | end 165 | 166 | ---@param server luatest.server 167 | function h.reload_tarantool(server) 168 | server:exec(function() 169 | package.reload() 170 | end) 171 | end 172 | 173 | 174 | return h 175 | -------------------------------------------------------------------------------- /spec/mock/single/conf.lua: -------------------------------------------------------------------------------- 1 | local tt_etcd_endpoints = assert(os.getenv('TT_ETCD_ENDPOINTS')) 2 | local endpoints = tt_etcd_endpoints:gsub(',+', ','):gsub(',$',''):split(',') 3 | 4 | etcd = { 5 | instance_name = instance_name, 6 | endpoints = endpoints, 7 | prefix = os.getenv('TT_ETCD_PREFIX'), 8 | } 9 | 10 | box = { 11 | wal_dir = os.getenv('TT_WAL_DIR') ..'/' .. instance_name, 12 | memtx_dir = os.getenv('TT_MEMTX_DIR') .. '/' .. instance_name, 13 | } 14 | -------------------------------------------------------------------------------- /spec/mock/single/init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | require 'package.reload' 3 | require 'config' { 4 | mkdir = true, 5 | print_config = true, 6 | instance_name = os.getenv('TT_INSTANCE_NAME'), 7 | file = os.getenv('TT_CONFIG'), 8 | master_selection_policy = os.getenv('TT_MASTER_SELECTION_POLICY'), 9 | on_after_cfg = function() 10 | if not box.info.ro then 11 | box.schema.user.grant('guest', 'super', nil, nil, { if_not_exists = true }) 12 | end 13 | end, 14 | } 15 | -------------------------------------------------------------------------------- /test/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tarantool/tarantool:2.11.1 2 | RUN apk add --no-cache -u iproute2 make bind-tools 3 | 4 | WORKDIR /opt/tarantool 5 | RUN tarantoolctl rocks --global --server http://moonlibs.github.io/rocks install package-reload scm-1 6 | 7 | CMD ["tarantool" "/opt/tarantool/init.lua"] 8 | -------------------------------------------------------------------------------- /test/Makefile: -------------------------------------------------------------------------------- 1 | 2 | run-compose-etcd: 3 | docker compose up -d --remove-orphans --build etcd0 etcd1 etcd2 4 | 5 | run-compose: run-compose-etcd 6 | docker compose up -d --remove-orphans --build 7 | -------------------------------------------------------------------------------- /test/app/conf.lua: -------------------------------------------------------------------------------- 1 | etcd = { 2 | instance_name = os.getenv("TT_INSTANCE_NAME"), 3 | prefix = '/instance', 4 | endpoints = {"http://etcd0:2379","http://etcd1:2379","http://etcd2:2379"}, 5 | fencing_enabled = false, 6 | timeout = 2, 7 | login = 'username', 8 | password = 'password', 9 | } 10 | -------------------------------------------------------------------------------- /test/app/init.lua: -------------------------------------------------------------------------------- 1 | local fiber = require "fiber" 2 | 3 | require 'package.reload' 4 | 5 | require 'config' { 6 | mkdir = true, 7 | instance_name = os.getenv("TT_INSTANCE_NAME"), 8 | file = 'conf.lua', 9 | master_selection_policy = 'etcd.cluster.master', 10 | 11 | on_after_cfg = function() 12 | if not box.info.ro then 13 | box.schema.user.grant('guest', 'super', nil, nil, { if_not_exists = true }) 14 | 15 | box.schema.space.create('T', {if_not_exists = true}) 16 | box.space.T:create_index('I', { if_not_exists = true }) 17 | end 18 | end, 19 | } 20 | 21 | fiber.create(function() 22 | fiber.name('pusher') 23 | 24 | while true do 25 | repeat 26 | pcall(box.ctl.wait_rw, 3) 27 | fiber.testcancel() 28 | until not box.info.ro 29 | 30 | local fibers = {} 31 | for _ = 1, 10 do 32 | local f = fiber.create(function() 33 | fiber.self():set_joinable(true) 34 | for _ = 1, 10 do 35 | box.space.T:insert{box.space.T:len(), box.info.id, box.info.vclock} 36 | end 37 | fiber.sleep(0.001) 38 | end) 39 | table.insert(fibers, f) 40 | end 41 | 42 | for _, f in ipairs(fibers) do 43 | f:join() 44 | end 45 | end 46 | end) 47 | 48 | -------------------------------------------------------------------------------- /test/docker-compose.yml: -------------------------------------------------------------------------------- 1 | x-env: &env 2 | ETCDCTL_API: 2 3 | ETCD_ENABLE_V2: true 4 | ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380 5 | ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 6 | ETCD_INITIAL_CLUSTER_TOKEN: etcd-cluster 7 | ETCD_INITIAL_CLUSTER: etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380 8 | 9 | x-etcd: &etcd 10 | image: quay.io/coreos/etcd:v3.5.16 11 | networks: 12 | - tarantool 13 | 14 | # x-tt: &tt 15 | # build: . 16 | # volumes: 17 | # - $PWD/../:/opt/tarantool/.rocks/share/tarantool:ro 18 | # - $PWD/app:/opt/tarantool 19 | # - $PWD/net:/opt/tarantool/net:ro 20 | # depends_on: 21 | # etcd0: 22 | # condition: service_started 23 | # etcd1: 24 | # condition: service_started 25 | # etcd2: 26 | # condition: service_started 27 | # privileged: true 28 | # networks: 29 | # - tarantool 30 | # command: ["/bin/sh", "-c", "sleep 5 && tarantool /opt/tarantool/init.lua"] 31 | 32 | networks: 33 | tarantool: 34 | name: tt_net 35 | driver: bridge 36 | 37 | services: 38 | etcd0: 39 | <<: *etcd 40 | container_name: etcd0 41 | environment: 42 | <<: *env 43 | ETCD_NAME: etcd0 44 | ETCD_ADVERTISE_CLIENT_URLS: http://etcd0:2379 45 | ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd0:2380 46 | etcd1: 47 | <<: *etcd 48 | container_name: etcd1 49 | environment: 50 | <<: *env 51 | ETCD_NAME: etcd1 52 | ETCD_ADVERTISE_CLIENT_URLS: http://etcd1:2379 53 | ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd1:2380 54 | etcd2: 55 | <<: *etcd 56 | container_name: etcd2 57 | environment: 58 | <<: *env 59 | ETCD_NAME: etcd2 60 | ETCD_ADVERTISE_CLIENT_URLS: http://etcd2:2379 61 | ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd2:2380 62 | 63 | # etcd_load: 64 | # image: registry.gitlab.com/ochaton/switchover:2.7.0.20 65 | # networks: 66 | # - tarantool 67 | # volumes: 68 | # - $PWD/instance.etcd.yaml:/instance.etcd.yaml:ro 69 | # depends_on: 70 | # etcd0: 71 | # condition: service_started 72 | # etcd1: 73 | # condition: service_started 74 | # etcd2: 75 | # condition: service_started 76 | # entrypoint: [''] 77 | # command: ["/bin/sh", "-c", "sleep 3 && switchover -v -e http://etcd0:2379,http://etcd1:2379,http://etcd2:2379 etcd load / /instance.etcd.yaml"] 78 | # instance_01: 79 | # <<: *tt 80 | # container_name: instance_01 81 | # environment: 82 | # TT_INSTANCE_NAME: instance_01 83 | # instance_02: 84 | # <<: *tt 85 | # container_name: instance_02 86 | # environment: 87 | # TT_INSTANCE_NAME: instance_02 88 | -------------------------------------------------------------------------------- /test/instance.etcd.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | instance: 3 | clusters: 4 | instance: 5 | master: instance_01 6 | replicaset_uuid: 91157a11-0001-0000-0000-000000000000 7 | common: 8 | etcd: 9 | fencing_timeout: 10 10 | fencing_pause: 5 11 | box: 12 | bootstrap_strategy: auto 13 | log_level: 5 14 | replication_connect_timeout: 3 15 | listen: 0.0.0.0:3301 16 | memtx_memory: 268435456 17 | memtx_dir: /var/lib/tarantool/snaps/ 18 | wal_dir: /var/lib/tarantool/xlogs/ 19 | instances: 20 | instance_01: 21 | cluster: instance 22 | box: 23 | instance_uuid: 91157a11-0000-0001-0000-000000000000 24 | remote_addr: instance_01:3301 25 | instance_02: 26 | cluster: instance 27 | box: 28 | instance_uuid: 91157a11-0000-0002-0000-000000000000 29 | remote_addr: instance_02:3301 30 | ... 31 | -------------------------------------------------------------------------------- /test/net/Makefile: -------------------------------------------------------------------------------- 1 | setup: 2 | tc qdisc add dev eth0 root handle 1: prio 3 | tc qdisc add dev eth0 parent 1:3 handle 10: netem loss 100% 4 | 5 | offline-dport-%: 6 | tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip dport $* 0xffff flowid 1:3 7 | 8 | offline-dst-%: 9 | tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip dst $(shell host -T4 $* | cut -f 4 -d' ') flowid 1:3 10 | 11 | online: 12 | tc filter del dev eth0 parent 1: protocol ip pref 1 u32 13 | 14 | filter: 15 | tc -s -d filter show dev eth0 16 | 17 | qdisc: 18 | tc -d -s qdisc show dev eth0 19 | 20 | clear: 21 | tc fliter del dev eth0 parent 1: 22 | tc qdisc del dev eth0 root 23 | -------------------------------------------------------------------------------- /test/net/README.md: -------------------------------------------------------------------------------- 1 | # Split-Brain test toolchain 2 | 3 | ## Run 4 | 5 | ```bash 6 | $ pwd 7 | config/test 8 | 9 | $ docker compose up --build 10 | ``` 11 | 12 | ## Prepare 13 | 14 | ### Prepare instance_01 15 | 16 | ```bash 17 | docker exec -it instance_001 /bin/sh 18 | 19 | # make setup must be executed only once per container 20 | /opt/tarantool $ make -C net setup 21 | ``` 22 | 23 | ### Prepare instance_02 24 | 25 | ```bash 26 | docker exec -it instance_002 /bin/sh 27 | 28 | # make setup must be executed only once per container 29 | /opt/tarantool $ make -C net setup 30 | ``` 31 | 32 | ## Make online 33 | 34 | ```bash 35 | docker exec -it instance_01 /bin/sh 36 | 37 | /opt/tarantool $ make -C net online 38 | ``` 39 | 40 | ## Isolation 41 | 42 | ### Isolate instance_01 against instance_02 43 | 44 | ```bash 45 | docker exec -it instance_01 /bin/sh 46 | 47 | /opt/tarantool $ make -C net offline-dst-instance_02 48 | ``` 49 | 50 | ### Isolate instance_01 against etcd 51 | 52 | ```bash 53 | docker exec -it instance_01 /bin/sh 54 | 55 | /opt/tarantool $ make -C net offline-dst-etcd 56 | ``` 57 | 58 | ### Total instance_01 isolation 59 | 60 | ```bash 61 | docker exec -it instance_01 /bin/sh 62 | 63 | /opt/tarantool $ make -C net offline-dst-instance_02 64 | /opt/tarantool $ make -C net offline-dst-etcd 65 | ``` 66 | 67 | ### Split brain instance_01 / instance_02 68 | 69 | ```bash 70 | docker exec -it instance_01 /bin/sh 71 | 72 | /opt/tarantool $ make -C net offline-dst-instance_02 73 | /opt/tarantool $ make -C net offline-dst-autofailover-2 74 | ``` 75 | 76 | ```bash 77 | docker exec -it instance_02 /bin/sh 78 | 79 | /opt/tarantool $ make -C net offline-dst-instance_01 80 | /opt/tarantool $ make -C net offline-dst-autofailover-1 81 | ``` 82 | -------------------------------------------------------------------------------- /test_peek.lua: -------------------------------------------------------------------------------- 1 | local log = require 'log' 2 | local json = require 'json' 3 | local yaml = require 'yaml' 4 | local fiber = require 'fiber' 5 | json.cfg{ encode_invalid_as_nil = true } 6 | yaml.cfg{ encode_invalid_as_nil = true } 7 | 8 | local under_tarantoolctl = fiber.name() == 'tarantoolctl' 9 | if rawget(_G,'TARANTOOLCTL') == nil then 10 | local from_env = os.getenv('TARANTOOLCTL') 11 | if from_env ~= nil then 12 | TARANTOOLCTL = from_env 13 | else 14 | TARANTOOLCTL = fiber.name() == 'tarantoolctl' 15 | end 16 | end 17 | log.info("TARANTOOL = %s; TARANTOOLCTL = %s", _TARANTOOL, TARANTOOLCTL) 18 | 19 | local function lookaround(fun) 20 | local vars = {} 21 | local i = 1 22 | while true do 23 | local n,v = debug.getupvalue(fun,i) 24 | if not n then break end 25 | vars[n] = v 26 | i = i + 1 27 | end 28 | i = 1 29 | 30 | return vars, i - 1 31 | end 32 | 33 | function peek_vars() 34 | local peek = { 35 | dynamic_cfg = true; 36 | upgrade_cfg = true; 37 | translate_cfg = true; 38 | -- log = true; 39 | } 40 | 41 | local steps = {} 42 | local peekf = box.cfg 43 | local allow_lock_unwrap = true 44 | local allow_ctl_unwrap = true 45 | while true do 46 | local prevf = peekf 47 | local mt = debug.getmetatable(peekf) 48 | if type(peekf) == 'function' then 49 | -- pass 50 | table.insert(steps,"func") 51 | elseif mt and mt.__call then 52 | peekf = mt.__call 53 | table.insert(steps,"mt_call") 54 | else 55 | error(string.format("Neither function nor callable argument %s after steps: %s", peekf, table.concat(steps, ", "))) 56 | end 57 | 58 | local vars, count = lookaround(peekf) 59 | if allow_ctl_unwrap and vars.orig_cfg then 60 | -- It's a wrap of tarantoolctl, unwrap and repeat 61 | peekf = vars.orig_cfg 62 | allow_ctl_unwrap = false 63 | table.insert(steps,"ctl-orig") 64 | elseif not vars.dynamic_cfg and vars.lock and vars.f and type(vars.f) == 'function' then 65 | peekf = vars.f 66 | table.insert(steps,"lock-unwrap") 67 | elseif not vars.dynamic_cfg and vars.old_call and type(vars.old_call) == 'function' then 68 | peekf = vars.old_call 69 | table.insert(steps,"ctl-oldcall") 70 | elseif vars.dynamic_cfg then 71 | log.info("Found config by steps: %s", table.concat(steps, ", ")) 72 | for k,v in pairs(peek) do 73 | if vars[k] ~= nil then 74 | peek[k] = vars[k] 75 | else 76 | peek[k] = nil 77 | end 78 | end 79 | break 80 | else 81 | for k,v in pairs(vars) do log.info("var %s=%s",k,v) end 82 | error(string.format("Bad vars for %s after steps: %s", peekf, table.concat(steps, ", "))) 83 | end 84 | if prevf == peekf then 85 | error(string.format("Recursion for %s after steps: %s", peekf, table.concat(steps, ", "))) 86 | end 87 | end 88 | return peek 89 | end 90 | 91 | do 92 | local peek = peek_vars() 93 | assert(type(peek.dynamic_cfg)=='table',"have dynamic_cfg") 94 | log.info("1st run: %s",json.encode(peek)) 95 | end 96 | 97 | box.cfg{ log_level = 2 } 98 | 99 | do 100 | print("Do run 2") 101 | local peek = peek_vars() 102 | print("2nd run: ",json.encode(peek)) 103 | log.error("2nd run: %s",json.encode(peek)) 104 | assert(type(peek.dynamic_cfg)=='table',"have dynamic_cfg") 105 | end 106 | 107 | if not TARANTOOLCTL then 108 | os.exit() 109 | end 110 | --------------------------------------------------------------------------------