├── .gitattributes ├── .gitignore ├── .lua-format ├── .travis.yml ├── Makefile ├── README.markdown ├── dist.ini ├── lib └── resty │ └── upstream │ └── healthcheck.lua ├── t ├── cert │ ├── localhost.crt │ ├── localhost.csr │ └── localhost.key ├── https.t ├── lib │ └── ljson.lua ├── prometheus_down.t ├── prometheus_empty.t ├── prometheus_mixed.t ├── prometheus_up.t └── sanity.t └── valgrind.suppress /.gitattributes: -------------------------------------------------------------------------------- 1 | *.t linguist-language=Text 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.swo 3 | *~ 4 | go 5 | t/servroot/ 6 | reindex 7 | nginx 8 | ctags 9 | tags 10 | a.lua 11 | -------------------------------------------------------------------------------- /.lua-format: -------------------------------------------------------------------------------- 1 | column_limit: 80 2 | keep_simple_function_one_line: false 3 | keep_simple_control_block_one_line: false 4 | align_table_field: false 5 | break_after_operator: false 6 | break_after_functioncall_lp: false 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: focal 3 | 4 | branches: 5 | only: 6 | - "master" 7 | 8 | os: linux 9 | 10 | language: c 11 | 12 | compiler: 13 | - gcc 14 | 15 | addons: 16 | apt: 17 | packages: 18 | - axel 19 | 20 | cache: 21 | apt: true 22 | directories: 23 | - download-cache 24 | 25 | env: 26 | global: 27 | - JOBS=3 28 | - NGX_BUILD_JOBS=$JOBS 29 | - LUAJIT_PREFIX=/opt/luajit21 30 | - LUAJIT_LIB=$LUAJIT_PREFIX/lib 31 | - LUAJIT_INC=$LUAJIT_PREFIX/include/luajit-2.1 32 | - LUA_INCLUDE_DIR=$LUAJIT_INC 33 | - LUA_CMODULE_DIR=/lib 34 | - OPENSSL_PREFIX=/opt/ssl 35 | - OPENSSL_LIB=$OPENSSL_PREFIX/lib 36 | - OPENSSL_INC=$OPENSSL_PREFIX/include 37 | - OPENSSL_VER=1.1.1w 38 | - LD_LIBRARY_PATH=$LUAJIT_LIB:$LD_LIBRARY_PATH 39 | - TEST_NGINX_SLEEP=0.006 40 | matrix: 41 | - NGINX_VERSION=1.27.1 42 | 43 | install: 44 | - if [ ! -d download-cache ]; then mkdir download-cache; fi 45 | - if [ ! -f download-cache/openssl-$OPENSSL_VER.tar.gz ]; then wget -O download-cache/openssl-$OPENSSL_VER.tar.gz https://www.openssl.org/source/openssl-$OPENSSL_VER.tar.gz; fi 46 | - cpanm --sudo --notest Test::Nginx > build.log 2>&1 || (cat build.log && exit 1) 47 | - git clone https://github.com/openresty/openresty.git ../openresty 48 | - git clone https://github.com/openresty/openresty-devel-utils.git ../openresty-devel-utils 49 | - git clone https://github.com/simpl/ngx_devel_kit.git ../ndk-nginx-module 50 | - git clone https://github.com/openresty/lua-nginx-module.git ../lua-nginx-module 51 | - git clone https://github.com/openresty/lua-resty-core.git ../lua-resty-core 52 | - git clone https://github.com/openresty/lua-resty-lrucache.git ../lua-resty-lrucache 53 | - git clone https://github.com/openresty/lua-upstream-nginx-module.git ../lua-upstream-nginx-module 54 | - git clone https://github.com/openresty/echo-nginx-module.git ../echo-nginx-module 55 | - git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx 56 | - git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git 57 | - git clone https://github.com/openresty/mockeagain.git 58 | 59 | script: 60 | - cd luajit2/ 61 | - make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS='-DLUA_USE_APICHECK -DLUA_USE_ASSERT' > build.log 2>&1 || (cat build.log && exit 1) 62 | - sudo make install PREFIX=$LUAJIT_PREFIX > build.log 2>&1 || (cat build.log && exit 1) 63 | - cd .. 64 | - tar zxf download-cache/openssl-$OPENSSL_VER.tar.gz 65 | - cd openssl-$OPENSSL_VER/ 66 | - ./config shared --prefix=$OPENSSL_PREFIX -DPURIFY > build.log 2>&1 || (cat build.log && exit 1) 67 | - make -j$JOBS > build.log 2>&1 || (cat build.log && exit 1) 68 | - sudo make PATH=$PATH install_sw > build.log 2>&1 || (cat build.log && exit 1) 69 | - cd ../mockeagain/ && make CC=$CC -j$JOBS && cd .. 70 | - export PATH=$PWD/work/nginx/sbin:$PWD/../openresty-devel-utils:$PATH 71 | - export LD_PRELOAD=$PWD/mockeagain/mockeagain.so 72 | - export LD_LIBRARY_PATH=$PWD/mockeagain:$LD_LIBRARY_PATH 73 | - export TEST_NGINX_RESOLVER=8.8.4.4 74 | - export NGX_BUILD_CC=$CC 75 | - ngx-build $NGINX_VERSION --without-pcre2 --with-ipv6 --with-http_realip_module --with-http_ssl_module --with-cc-opt="-I$OPENSSL_INC" --with-ld-opt="-L$OPENSSL_LIB -Wl,-rpath,$OPENSSL_LIB" --add-module=../ndk-nginx-module --add-module=../lua-nginx-module --add-module=../lua-upstream-nginx-module --add-module=../echo-nginx-module --with-debug > build.log 2>&1 || (cat build.log && exit 1) 76 | - nginx -V 77 | - ldd `which nginx`|grep -E 'luajit|ssl|pcre' 78 | - prove -r t 79 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | OPENRESTY_PREFIX=/usr/local/openresty 2 | 3 | PREFIX ?= /usr/local 4 | LUA_INCLUDE_DIR ?= $(PREFIX)/include 5 | LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION) 6 | INSTALL ?= install 7 | 8 | .PHONY: all test install 9 | 10 | all: ; 11 | 12 | install: all 13 | $(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/resty/upstream/ 14 | $(INSTALL) lib/resty/upstream/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/resty/upstream/ 15 | 16 | test: all 17 | PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH prove -I../test-nginx/lib -r t 18 | 19 | lint: 20 | @find lib -name "*.lua" -type f | sort | xargs -I{} sh -c 'lua-format -i {}' 21 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | Name 2 | ==== 3 | 4 | lua-resty-upstream-healthcheck - Health-checker for Nginx upstream servers 5 | 6 | Table of Contents 7 | ================= 8 | 9 | - [Name](#name) 10 | - [Table of Contents](#table-of-contents) 11 | - [Status](#status) 12 | - [Synopsis](#synopsis) 13 | - [Description](#description) 14 | - [Methods](#methods) 15 | - [spawn_checker](#spawn_checker) 16 | - [status_page](#status_page) 17 | - [Multiple Upstreams](#multiple-upstreams) 18 | - [Installation](#installation) 19 | - [TODO](#todo) 20 | - [Community](#community) 21 | - [Contributing](#contributing) 22 | - [English Mailing List](#english-mailing-list) 23 | - [Chinese Mailing List](#chinese-mailing-list) 24 | - [Bugs and Patches](#bugs-and-patches) 25 | - [Author](#author) 26 | - [Copyright and License](#copyright-and-license) 27 | - [See Also](#see-also) 28 | 29 | Status 30 | ====== 31 | 32 | This library is still under early development but is already production ready. 33 | 34 | Synopsis 35 | ======== 36 | 37 | ```nginx 38 | http { 39 | lua_package_path "/path/to/lua-resty-upstream-healthcheck/lib/?.lua;;"; 40 | 41 | # sample upstream block: 42 | upstream foo.com { 43 | server 127.0.0.1:12354; 44 | server 127.0.0.1:12355; 45 | server 127.0.0.1:12356 backup; 46 | } 47 | 48 | # the size depends on the number of servers in upstream {}: 49 | lua_shared_dict healthcheck 1m; 50 | 51 | lua_socket_log_errors off; 52 | 53 | init_worker_by_lua_block { 54 | local hc = require "resty.upstream.healthcheck" 55 | 56 | local ok, err = hc.spawn_checker{ 57 | shm = "healthcheck", -- defined by "lua_shared_dict" 58 | upstream = "foo.com", -- defined by "upstream" 59 | type = "http", -- support "http" and "https" 60 | 61 | http_req = "GET /status HTTP/1.0\r\nHost: foo.com\r\n\r\n", 62 | -- raw HTTP request for checking 63 | 64 | port = nil, -- the check port, it can be different than the original backend server port, default means the same as the original backend server 65 | interval = 2000, -- run the check cycle every 2 sec 66 | timeout = 1000, -- 1 sec is the timeout for network operations 67 | fall = 3, -- # of successive failures before turning a peer down 68 | rise = 2, -- # of successive successes before turning a peer up 69 | valid_statuses = {200, 302}, -- a list valid HTTP status code 70 | concurrency = 10, -- concurrency level for test requests 71 | -- ssl_verify = true, -- https type only, verify ssl certificate or not, default true 72 | -- host = foo.com, -- https type only, host name in ssl handshake, default nil 73 | } 74 | if not ok then 75 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 76 | return 77 | end 78 | 79 | -- Just call hc.spawn_checker() for more times here if you have 80 | -- more upstream groups to monitor. One call for one upstream group. 81 | -- They can all share the same shm zone without conflicts but they 82 | -- need a bigger shm zone for obvious reasons. 83 | } 84 | 85 | server { 86 | ... 87 | 88 | # status page for all the peers: 89 | location = /status { 90 | access_log off; 91 | allow 127.0.0.1; 92 | deny all; 93 | 94 | default_type text/plain; 95 | content_by_lua_block { 96 | local hc = require "resty.upstream.healthcheck" 97 | ngx.say("Nginx Worker PID: ", ngx.worker.pid()) 98 | ngx.print(hc.status_page()) 99 | } 100 | } 101 | 102 | # status page for all the peers (prometheus format): 103 | location = /metrics { 104 | access_log off; 105 | default_type text/plain; 106 | content_by_lua_block { 107 | local hc = require "resty.upstream.healthcheck" 108 | st , err = hc.prometheus_status_page() 109 | if not st then 110 | ngx.say(err) 111 | return 112 | end 113 | ngx.print(st) 114 | } 115 | } 116 | } 117 | } 118 | ``` 119 | 120 | Description 121 | =========== 122 | 123 | This library performs healthcheck for server peers defined in NGINX `upstream` groups specified by names. 124 | 125 | [Back to TOC](#table-of-contents) 126 | 127 | Methods 128 | ======= 129 | 130 | spawn_checker 131 | ------------- 132 | **syntax:** `ok, err = healthcheck.spawn_checker(options)` 133 | 134 | **context:** *init_worker_by_lua** 135 | 136 | Spawns background timer-based "light threads" to perform periodic healthchecks on 137 | the specified NGINX upstream group with the specified shm storage. 138 | 139 | The healthchecker does not need any client traffic to function. The checks are performed actively 140 | and periodically. 141 | 142 | This method call is asynchronous and returns immediately. 143 | 144 | Returns true on success, or `nil` and a string describing an error otherwise. 145 | 146 | [Back to TOC](#table-of-contents) 147 | 148 | status_page 149 | ----------- 150 | **syntax:** `str, err = healthcheck.status_page()` 151 | 152 | **context:** *any* 153 | 154 | Generates a detailed status report for all the upstreams defined in the current NGINX server. 155 | 156 | One typical output is 157 | 158 | ``` 159 | Upstream foo.com 160 | Primary Peers 161 | 127.0.0.1:12354 UP 162 | 127.0.0.1:12355 DOWN 163 | Backup Peers 164 | 127.0.0.1:12356 UP 165 | 166 | Upstream bar.com 167 | Primary Peers 168 | 127.0.0.1:12354 UP 169 | 127.0.0.1:12355 DOWN 170 | 127.0.0.1:12357 DOWN 171 | Backup Peers 172 | 127.0.0.1:12356 UP 173 | ``` 174 | 175 | If an upstream has no health checkers, then it will be marked by `(NO checkers)`, as in 176 | 177 | ``` 178 | Upstream foo.com (NO checkers) 179 | Primary Peers 180 | 127.0.0.1:12354 UP 181 | 127.0.0.1:12355 UP 182 | Backup Peers 183 | 127.0.0.1:12356 UP 184 | ``` 185 | 186 | If you indeed have spawned a healthchecker in `init_worker_by_lua*`, then you should really 187 | check out the NGINX error log file to see if there is any fatal errors aborting the healthchecker threads. 188 | 189 | [Back to TOC](#table-of-contents) 190 | 191 | Multiple Upstreams 192 | ================== 193 | 194 | One can perform healthchecks on multiple `upstream` groups by calling the [spawn_checker](#spawn_checker) method 195 | multiple times in the `init_worker_by_lua*` handler. For example, 196 | 197 | ```nginx 198 | upstream foo { 199 | ... 200 | } 201 | 202 | upstream bar { 203 | ... 204 | } 205 | 206 | lua_shared_dict healthcheck 1m; 207 | 208 | lua_socket_log_errors off; 209 | 210 | init_worker_by_lua_block { 211 | local hc = require "resty.upstream.healthcheck" 212 | 213 | local ok, err = hc.spawn_checker{ 214 | shm = "healthcheck", 215 | upstream = "foo", 216 | ... 217 | } 218 | 219 | ... 220 | 221 | ok, err = hc.spawn_checker{ 222 | shm = "healthcheck", 223 | upstream = "bar", 224 | ... 225 | } 226 | } 227 | ``` 228 | 229 | Different upstreams' healthcheckers use different keys (by always prefixing the keys with the 230 | upstream name), so sharing a single `lua_shared_dict` among multiple checkers should not have 231 | any issues at all. But you need to compensate the size of the shared dict for multiple users (i.e., multiple checkers). 232 | If you have many upstreams (thousands or even more), then it is more optimal to use separate shm zones 233 | for each (group) of the upstreams. 234 | 235 | [Back to TOC](#table-of-contents) 236 | 237 | Installation 238 | ============ 239 | 240 | If you are using [OpenResty](http://openresty.org) 1.9.3.2 or later, then you should already have this library (and all of its dependencies) installed by default (and this is also the recommended way of using this library). Otherwise continue reading: 241 | 242 | You need to compile both the [ngx_lua](https://github.com/openresty/lua-nginx-module) and [ngx_lua_upstream](https://github.com/openresty/lua-upstream-nginx-module) modules into your Nginx. 243 | 244 | The latest git master branch of [ngx_lua](https://github.com/openresty/lua-nginx-module) is required. 245 | 246 | You need to configure 247 | the [lua_package_path](https://github.com/openresty/lua-nginx-module#lua_package_path) directive to 248 | add the path of your `lua-resty-upstream-healthcheck` source tree to [ngx_lua](https://github.com/openresty/lua-nginx-module)'s Lua module search path, as in 249 | 250 | ```nginx 251 | # nginx.conf 252 | http { 253 | lua_package_path "/path/to/lua-resty-upstream-healthcheck/lib/?.lua;;"; 254 | ... 255 | } 256 | ``` 257 | 258 | [Back to TOC](#table-of-contents) 259 | 260 | TODO 261 | ==== 262 | 263 | [Back to TOC](#table-of-contents) 264 | 265 | Community 266 | ========= 267 | 268 | [Back to TOC](#table-of-contents) 269 | 270 | Contributing 271 | -------------------- 272 | 273 | Use `make lint` to lint the code before you open a PR. This uses the widely used [LuaFormatter](https://github.com/Koihik/LuaFormatter). 274 | 275 | The code style is described in the [`.lua-format`](.lua-format) file.\ 276 | If you are using VS Code, you can install the wrapper for that formatter by clicking [here](vscode:extension/Koihik.vscode-lua-format). 277 | 278 | [Back to TOC](#table-of-contents) 279 | 280 | English Mailing List 281 | -------------------- 282 | 283 | The [openresty-en](https://groups.google.com/group/openresty-en) mailing list is for English speakers. 284 | 285 | [Back to TOC](#table-of-contents) 286 | 287 | Chinese Mailing List 288 | -------------------- 289 | 290 | The [openresty](https://groups.google.com/group/openresty) mailing list is for Chinese speakers. 291 | 292 | [Back to TOC](#table-of-contents) 293 | 294 | Bugs and Patches 295 | ================ 296 | 297 | Please report bugs or submit patches by 298 | 299 | 1. creating a ticket on the [GitHub Issue Tracker](http://github.com/openresty/lua-resty-upstream-healthcheck/issues), 300 | 1. or posting to the [OpenResty community](#community). 301 | 302 | [Back to TOC](#table-of-contents) 303 | 304 | Author 305 | ====== 306 | 307 | Yichun "agentzh" Zhang (章亦春) , OpenResty Inc. 308 | 309 | [Back to TOC](#table-of-contents) 310 | 311 | Copyright and License 312 | ===================== 313 | 314 | This module is licensed under the BSD license. 315 | 316 | Copyright (C) 2014-2017, by Yichun "agentzh" Zhang, OpenResty Inc. 317 | 318 | All rights reserved. 319 | 320 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 321 | 322 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 323 | 324 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 325 | 326 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 327 | 328 | [Back to TOC](#table-of-contents) 329 | 330 | See Also 331 | ======== 332 | * the ngx_lua module: https://github.com/openresty/lua-nginx-module 333 | * the ngx_lua_upstream module: https://github.com/openresty/lua-upstream-nginx-module 334 | * OpenResty: http://openresty.org 335 | 336 | [Back to TOC](#table-of-contents) 337 | 338 | -------------------------------------------------------------------------------- /dist.ini: -------------------------------------------------------------------------------- 1 | name=lua-resty-upstream-healthcheck 2 | abstract=Lua health-checker for Nginx upstream servers 3 | author=Yichun "agentzh" Zhang (agentzh) 4 | is_original=yes 5 | license=2bsd 6 | lib_dir=lib 7 | doc_dir=lib 8 | repo_link=https://github.com/openresty/lua-resty-upstream-healthcheck 9 | main_module=lib/resty/upstream/healthcheck.lua 10 | -------------------------------------------------------------------------------- /lib/resty/upstream/healthcheck.lua: -------------------------------------------------------------------------------- 1 | local stream_sock = ngx.socket.tcp 2 | local log = ngx.log 3 | local ERR = ngx.ERR 4 | local WARN = ngx.WARN 5 | local DEBUG = ngx.DEBUG 6 | local ngx = ngx 7 | local error = error 8 | local string = string 9 | local sub = string.sub 10 | local re_find = ngx.re.find 11 | local new_timer = ngx.timer.at 12 | local shared = ngx.shared 13 | local debug_mode = ngx.config.debug 14 | local concat = table.concat 15 | local tonumber = tonumber 16 | local tostring = tostring 17 | local ipairs = ipairs 18 | local ceil = math.ceil 19 | local spawn = ngx.thread.spawn 20 | local wait = ngx.thread.wait 21 | local pcall = pcall 22 | local setmetatable = setmetatable 23 | 24 | -- LuaFormatter off 25 | local _M = { 26 | _VERSION = '0.08' 27 | } 28 | 29 | if not ngx.config 30 | or not ngx.config.ngx_lua_version 31 | or ngx.config.ngx_lua_version < 9005 32 | then 33 | error("ngx_lua 0.9.5+ required") 34 | end 35 | -- LuaFormatter on 36 | 37 | local ok, upstream = pcall(require, "ngx.upstream") 38 | if not ok then 39 | error("ngx_upstream_lua module required") 40 | end 41 | 42 | local ok, new_tab = pcall(require, "table.new") 43 | if not ok or type(new_tab) ~= "function" then 44 | new_tab = function(narr, nrec) 45 | return {} 46 | end 47 | end 48 | 49 | local set_peer_down = upstream.set_peer_down 50 | local get_primary_peers = upstream.get_primary_peers 51 | local get_backup_peers = upstream.get_backup_peers 52 | local get_upstreams = upstream.get_upstreams 53 | 54 | local upstream_checker_statuses = {} 55 | 56 | local function warn(...) 57 | log(WARN, "healthcheck: ", ...) 58 | end 59 | 60 | local function errlog(...) 61 | log(ERR, "healthcheck: ", ...) 62 | end 63 | 64 | local function debug(...) 65 | -- print("debug mode: ", debug_mode) 66 | if debug_mode then 67 | log(DEBUG, "healthcheck: ", ...) 68 | end 69 | end 70 | 71 | local function gen_peer_key(prefix, u, is_backup, id) 72 | if is_backup then 73 | return prefix .. u .. ":b" .. id 74 | end 75 | return prefix .. u .. ":p" .. id 76 | end 77 | 78 | local function set_peer_down_globally(ctx, is_backup, id, value) 79 | local u = ctx.upstream 80 | local dict = ctx.dict 81 | local ok, err = set_peer_down(u, is_backup, id, value) 82 | if not ok then 83 | errlog("failed to set peer down: ", err) 84 | end 85 | 86 | if not ctx.new_version then 87 | ctx.new_version = true 88 | end 89 | 90 | local key = gen_peer_key("d:", u, is_backup, id) 91 | local ok, err = dict:set(key, value) 92 | if not ok then 93 | errlog("failed to set peer down state: ", err) 94 | end 95 | end 96 | 97 | local function peer_fail(ctx, is_backup, id, peer) 98 | debug("peer ", peer.name, " was checked to be not ok") 99 | 100 | local u = ctx.upstream 101 | local dict = ctx.dict 102 | 103 | local key = gen_peer_key("nok:", u, is_backup, id) 104 | local fails, err = dict:get(key) 105 | if not fails then 106 | if err then 107 | errlog("failed to get peer nok key: ", err) 108 | return 109 | end 110 | fails = 1 111 | 112 | -- below may have a race condition, but it is fine for our 113 | -- purpose here. 114 | local ok, err = dict:set(key, 1) 115 | if not ok then 116 | errlog("failed to set peer nok key: ", err) 117 | end 118 | else 119 | fails = fails + 1 120 | local ok, err = dict:incr(key, 1) 121 | if not ok then 122 | errlog("failed to incr peer nok key: ", err) 123 | end 124 | end 125 | 126 | if fails == 1 then 127 | key = gen_peer_key("ok:", u, is_backup, id) 128 | local succ, err = dict:get(key) 129 | if not succ or succ == 0 then 130 | if err then 131 | errlog("failed to get peer ok key: ", err) 132 | return 133 | end 134 | else 135 | local ok, err = dict:set(key, 0) 136 | if not ok then 137 | errlog("failed to set peer ok key: ", err) 138 | end 139 | end 140 | end 141 | 142 | -- print("ctx fall: ", ctx.fall, ", peer down: ", peer.down, 143 | -- ", fails: ", fails) 144 | 145 | if not peer.down and fails >= ctx.fall then 146 | warn("peer ", peer.name, " is turned down after ", fails, " failure(s)") 147 | peer.down = true 148 | set_peer_down_globally(ctx, is_backup, id, true) 149 | end 150 | end 151 | 152 | local function peer_ok(ctx, is_backup, id, peer) 153 | debug("peer ", peer.name, " was checked to be ok") 154 | 155 | local u = ctx.upstream 156 | local dict = ctx.dict 157 | 158 | local key = gen_peer_key("ok:", u, is_backup, id) 159 | local succ, err = dict:get(key) 160 | if not succ then 161 | if err then 162 | errlog("failed to get peer ok key: ", err) 163 | return 164 | end 165 | succ = 1 166 | 167 | -- below may have a race condition, but it is fine for our 168 | -- purpose here. 169 | local ok, err = dict:set(key, 1) 170 | if not ok then 171 | errlog("failed to set peer ok key: ", err) 172 | end 173 | else 174 | succ = succ + 1 175 | local ok, err = dict:incr(key, 1) 176 | if not ok then 177 | errlog("failed to incr peer ok key: ", err) 178 | end 179 | end 180 | 181 | if succ == 1 then 182 | key = gen_peer_key("nok:", u, is_backup, id) 183 | local fails, err = dict:get(key) 184 | if not fails or fails == 0 then 185 | if err then 186 | errlog("failed to get peer nok key: ", err) 187 | return 188 | end 189 | else 190 | local ok, err = dict:set(key, 0) 191 | if not ok then 192 | errlog("failed to set peer nok key: ", err) 193 | end 194 | end 195 | end 196 | 197 | if peer.down and succ >= ctx.rise then 198 | warn("peer ", peer.name, " is turned up after ", succ, " success(es)") 199 | peer.down = nil 200 | set_peer_down_globally(ctx, is_backup, id, nil) 201 | end 202 | end 203 | 204 | -- shortcut error function for check_peer() 205 | local function peer_error(ctx, is_backup, id, peer, ...) 206 | if not peer.down then 207 | errlog(...) 208 | end 209 | peer_fail(ctx, is_backup, id, peer) 210 | end 211 | 212 | local function check_peer(ctx, id, peer, is_backup) 213 | local ok 214 | local name = peer.name 215 | local statuses = ctx.statuses 216 | local req = ctx.http_req 217 | 218 | local sock, err = stream_sock() 219 | if not sock then 220 | errlog("failed to create stream socket: ", err) 221 | return 222 | end 223 | 224 | sock:settimeout(ctx.timeout) 225 | 226 | if peer.host then 227 | -- print("peer port: ", peer.port) 228 | ok, err = sock:connect(peer.host, peer.port) 229 | else 230 | ok, err = sock:connect(name) 231 | end 232 | if not ok then 233 | if not peer.down then 234 | errlog("failed to connect to ", name, ": ", err) 235 | end 236 | return peer_fail(ctx, is_backup, id, peer) 237 | end 238 | 239 | if ctx.type == "https" then 240 | ok, err = sock:sslhandshake(nil, ctx.host, ctx.ssl_verify) 241 | if not ok then 242 | sock:close() 243 | return peer_error(ctx, is_backup, id, peer, 244 | "failed to ssl handshake to ", name, ": ", err) 245 | end 246 | end 247 | 248 | local bytes, err = sock:send(req) 249 | if not bytes then 250 | return peer_error(ctx, is_backup, id, peer, 251 | "failed to send request to ", name, ": ", err) 252 | end 253 | 254 | local status_line, err = sock:receive() 255 | if not status_line then 256 | peer_error(ctx, is_backup, id, peer, 257 | "failed to receive status line from ", name, ": ", err) 258 | if err == "timeout" then 259 | sock:close() -- timeout errors do not close the socket. 260 | end 261 | return 262 | end 263 | 264 | if statuses then 265 | local from, to, err = re_find(status_line, [[^HTTP/\d+\.\d+\s+(\d+)]], 266 | "joi", nil, 1) 267 | if err then 268 | errlog("failed to parse status line: ", err) 269 | end 270 | 271 | if not from then 272 | peer_error(ctx, is_backup, id, peer, "bad status line from ", name, 273 | ": ", status_line) 274 | sock:close() 275 | return 276 | end 277 | 278 | local status = tonumber(sub(status_line, from, to)) 279 | if not statuses[status] then 280 | peer_error(ctx, is_backup, id, peer, "bad status code from ", name, 281 | ": ", status) 282 | sock:close() 283 | return 284 | end 285 | end 286 | 287 | peer_ok(ctx, is_backup, id, peer) 288 | sock:close() 289 | end 290 | 291 | local function check_peer_range(ctx, from, to, peers, is_backup) 292 | for i = from, to do 293 | check_peer(ctx, i - 1, peers[i], is_backup) 294 | end 295 | end 296 | 297 | local function check_peers(ctx, peers, is_backup) 298 | local n = #peers 299 | if n == 0 then 300 | return 301 | end 302 | 303 | local concur = ctx.concurrency 304 | if concur <= 1 then 305 | for i = 1, n do 306 | check_peer(ctx, i - 1, peers[i], is_backup) 307 | end 308 | else 309 | local threads 310 | local nthr 311 | 312 | if n <= concur then 313 | nthr = n - 1 314 | threads = new_tab(nthr, 0) 315 | for i = 1, nthr do 316 | 317 | if debug_mode then 318 | debug("spawn a thread checking ", 319 | is_backup and "backup" or "primary", " peer ", i - 1) 320 | end 321 | 322 | threads[i] = spawn(check_peer, ctx, i - 1, peers[i], is_backup) 323 | end 324 | -- use the current "light thread" to run the last task 325 | if debug_mode then 326 | debug("check ", is_backup and "backup" or "primary", " peer ", 327 | n - 1) 328 | end 329 | check_peer(ctx, n - 1, peers[n], is_backup) 330 | 331 | else 332 | local group_size = ceil(n / concur) 333 | nthr = ceil(n / group_size) - 1 334 | 335 | threads = new_tab(nthr, 0) 336 | local from = 1 337 | local rest = n 338 | for i = 1, nthr do 339 | local to 340 | if rest >= group_size then 341 | rest = rest - group_size 342 | to = from + group_size - 1 343 | else 344 | rest = 0 345 | to = from + rest - 1 346 | end 347 | 348 | if debug_mode then 349 | debug("spawn a thread checking ", 350 | is_backup and "backup" or "primary", " peers ", 351 | from - 1, " to ", to - 1) 352 | end 353 | 354 | threads[i] = spawn(check_peer_range, ctx, from, to, peers, 355 | is_backup) 356 | from = from + group_size 357 | if rest == 0 then 358 | break 359 | end 360 | end 361 | if rest > 0 then 362 | local to = from + rest - 1 363 | 364 | if debug_mode then 365 | debug("check ", is_backup and "backup" or "primary", 366 | " peers ", from - 1, " to ", to - 1) 367 | end 368 | 369 | check_peer_range(ctx, from, to, peers, is_backup) 370 | end 371 | end 372 | 373 | if nthr and nthr > 0 then 374 | for i = 1, nthr do 375 | local t = threads[i] 376 | if t then 377 | wait(t) 378 | end 379 | end 380 | end 381 | end 382 | end 383 | 384 | local function upgrade_peers_version(ctx, peers, is_backup) 385 | local dict = ctx.dict 386 | local u = ctx.upstream 387 | local n = #peers 388 | for i = 1, n do 389 | local peer = peers[i] 390 | local id = i - 1 391 | local key = gen_peer_key("d:", u, is_backup, id) 392 | local down = false 393 | local res, err = dict:get(key) 394 | if not res then 395 | if err then 396 | errlog("failed to get peer down state: ", err) 397 | end 398 | else 399 | down = true 400 | end 401 | if (peer.down and not down) or (not peer.down and down) then 402 | local ok, err = set_peer_down(u, is_backup, id, down) 403 | if not ok then 404 | errlog("failed to set peer down: ", err) 405 | else 406 | -- update our cache too 407 | peer.down = down 408 | end 409 | end 410 | end 411 | end 412 | 413 | local function check_peers_updates(ctx) 414 | local dict = ctx.dict 415 | local u = ctx.upstream 416 | local key = "v:" .. u 417 | local ver, err = dict:get(key) 418 | if not ver then 419 | if err then 420 | errlog("failed to get peers version: ", err) 421 | return 422 | end 423 | 424 | if ctx.version > 0 then 425 | ctx.new_version = true 426 | end 427 | 428 | elseif ctx.version < ver then 429 | debug("upgrading peers version to ", ver) 430 | upgrade_peers_version(ctx, ctx.primary_peers, false); 431 | upgrade_peers_version(ctx, ctx.backup_peers, true); 432 | ctx.version = ver 433 | end 434 | end 435 | 436 | local function get_lock(ctx) 437 | local dict = ctx.dict 438 | local key = "l:" .. ctx.upstream 439 | 440 | -- the lock is held for the whole interval to prevent multiple 441 | -- worker processes from sending the test request simultaneously. 442 | -- here we substract the lock expiration time by 1ms to prevent 443 | -- a race condition with the next timer event. 444 | local ok, err = dict:add(key, true, ctx.interval - 0.001) 445 | if not ok then 446 | if err == "exists" then 447 | return nil 448 | end 449 | errlog("failed to add key \"", key, "\": ", err) 450 | return nil 451 | end 452 | return true 453 | end 454 | 455 | local function do_check(ctx) 456 | debug("healthcheck: run a check cycle") 457 | 458 | check_peers_updates(ctx) 459 | 460 | if get_lock(ctx) then 461 | check_peers(ctx, ctx.primary_peers, false) 462 | check_peers(ctx, ctx.backup_peers, true) 463 | end 464 | 465 | if ctx.new_version then 466 | local key = "v:" .. ctx.upstream 467 | local dict = ctx.dict 468 | 469 | if debug_mode then 470 | debug("publishing peers version ", ctx.version + 1) 471 | end 472 | 473 | dict:add(key, 0) 474 | local new_ver, err = dict:incr(key, 1) 475 | if not new_ver then 476 | errlog("failed to publish new peers version: ", err) 477 | end 478 | 479 | ctx.version = new_ver 480 | ctx.new_version = nil 481 | end 482 | end 483 | 484 | local function update_upstream_checker_status(upstream, success) 485 | local cnt = upstream_checker_statuses[upstream] 486 | if not cnt then 487 | cnt = 0 488 | end 489 | 490 | if success then 491 | cnt = cnt + 1 492 | else 493 | cnt = cnt - 1 494 | end 495 | 496 | upstream_checker_statuses[upstream] = cnt 497 | end 498 | 499 | local check 500 | check = function(premature, ctx) 501 | if premature then 502 | return 503 | end 504 | 505 | local ok, err = pcall(do_check, ctx) 506 | if not ok then 507 | errlog("failed to run healthcheck cycle: ", err) 508 | end 509 | 510 | local ok, err = new_timer(ctx.interval, check, ctx) 511 | if not ok then 512 | if err ~= "process exiting" then 513 | errlog("failed to create timer: ", err) 514 | end 515 | 516 | update_upstream_checker_status(ctx.upstream, false) 517 | return 518 | end 519 | end 520 | 521 | local function preprocess_peers(peers, port) 522 | local n = #peers 523 | for i = 1, n do 524 | local p = peers[i] 525 | local name = p.name 526 | 527 | if name then 528 | local from, to, err = re_find(name, [[^(.*):\d+$]], "jo", nil, 1) 529 | if from then 530 | p.host = sub(name, 1, to) 531 | p.port = port or tonumber(sub(name, to + 2)) 532 | end 533 | end 534 | end 535 | return peers 536 | end 537 | 538 | function _M.spawn_checker(opts) 539 | local typ = opts.type 540 | if not typ then 541 | return nil, "\"type\" option required" 542 | end 543 | 544 | if typ ~= "http" and typ ~= "https" then 545 | return nil, "only \"http\" and \"https\" type are supported right now" 546 | end 547 | 548 | local ssl_verify = opts.ssl_verify 549 | if ssl_verify == nil then 550 | ssl_verify = true 551 | end 552 | 553 | local http_req = opts.http_req 554 | if not http_req then 555 | return nil, "\"http_req\" option required" 556 | end 557 | 558 | local timeout = opts.timeout 559 | if not timeout then 560 | timeout = 1000 561 | end 562 | 563 | local interval = opts.interval 564 | if not interval then 565 | interval = 1 566 | 567 | else 568 | interval = interval / 1000 569 | if interval < 0.002 then -- minimum 2ms 570 | interval = 0.002 571 | end 572 | end 573 | 574 | local valid_statuses = opts.valid_statuses 575 | local statuses 576 | if valid_statuses then 577 | statuses = new_tab(0, #valid_statuses) 578 | for _, status in ipairs(valid_statuses) do 579 | -- print("found good status ", status) 580 | statuses[status] = true 581 | end 582 | end 583 | 584 | -- debug("interval: ", interval) 585 | 586 | local concur = opts.concurrency 587 | if not concur then 588 | concur = 1 589 | end 590 | 591 | local fall = opts.fall 592 | if not fall then 593 | fall = 5 594 | end 595 | 596 | local rise = opts.rise 597 | if not rise then 598 | rise = 2 599 | end 600 | 601 | local shm = opts.shm 602 | if not shm then 603 | return nil, "\"shm\" option required" 604 | end 605 | 606 | local dict = shared[shm] 607 | if not dict then 608 | return nil, "shm \"" .. tostring(shm) .. "\" not found" 609 | end 610 | 611 | local u = opts.upstream 612 | if not u then 613 | return nil, "no upstream specified" 614 | end 615 | 616 | local ppeers, err = get_primary_peers(u) 617 | if not ppeers then 618 | return nil, "failed to get primary peers: " .. err 619 | end 620 | 621 | local bpeers, err = get_backup_peers(u) 622 | if not bpeers then 623 | return nil, "failed to get backup peers: " .. err 624 | end 625 | 626 | local ctx = { 627 | upstream = u, 628 | primary_peers = preprocess_peers(ppeers, opts.port), 629 | backup_peers = preprocess_peers(bpeers, opts.port), 630 | http_req = http_req, 631 | timeout = timeout, 632 | interval = interval, 633 | dict = dict, 634 | fall = fall, 635 | rise = rise, 636 | statuses = statuses, 637 | version = 0, 638 | concurrency = concur, 639 | type = typ, 640 | host = opts.host, 641 | ssl_verify = ssl_verify 642 | } 643 | 644 | if debug_mode and opts.no_timer then 645 | check(nil, ctx) 646 | 647 | else 648 | local ok, err = new_timer(0, check, ctx) 649 | if not ok then 650 | return nil, "failed to create timer: " .. err 651 | end 652 | end 653 | 654 | update_upstream_checker_status(u, true) 655 | 656 | return true 657 | end 658 | 659 | local new_status_meta = { 660 | __add = function(self, rhs) 661 | -- debug("new_status_meta:__add: rhs: ", rhs) 662 | self.statuses[self.idx] = rhs 663 | self.idx = self.idx + 1 664 | end 665 | } 666 | new_status_meta.__index = new_status_meta 667 | 668 | function new_status_meta:add(rhs) 669 | self:__add(rhs) 670 | end 671 | 672 | local function new_status_table(n) 673 | local tab = {statuses = new_tab(n * 90, 0), idx = 1} 674 | return setmetatable(tab, new_status_meta) 675 | end 676 | 677 | -- combined upstream status adding functions 678 | 679 | local function add_upstream_prometheus_status_line(tab, u, st) 680 | tab:add('nginx_upstream_status_info{name="') 681 | tab:add(u) 682 | tab:add('",status="') 683 | tab:add(st) 684 | tab:add('\n') 685 | end 686 | 687 | local function add_upstream_up_prometheus_status(tab, u) 688 | add_upstream_prometheus_status_line(tab, u, 'UP"} 1'); 689 | add_upstream_prometheus_status_line(tab, u, 'DOWN"} 0'); 690 | add_upstream_prometheus_status_line(tab, u, 'UNKNOWN"} 0'); 691 | end 692 | 693 | local function add_upstream_down_prometheus_status(tab, u) 694 | add_upstream_prometheus_status_line(tab, u, 'UP"} 0'); 695 | add_upstream_prometheus_status_line(tab, u, 'DOWN"} 1'); 696 | add_upstream_prometheus_status_line(tab, u, 'UNKNOWN"} 0'); 697 | end 698 | 699 | local function add_upstream_unknown_prometheus_status(tab, u) 700 | add_upstream_prometheus_status_line(tab, u, 'UP"} 0'); 701 | add_upstream_prometheus_status_line(tab, u, 'DOWN"} 0'); 702 | add_upstream_prometheus_status_line(tab, u, 'UNKNOWN"} 1'); 703 | end 704 | 705 | -- peer status generator functions 706 | 707 | local function gen_peer_prometheus_status(tab, u, p, r, s, n) 708 | tab:add("nginx_upstream_status_info{name=\"") 709 | tab:add(u) 710 | tab:add("\",endpoint=\"") 711 | tab:add(p) 712 | tab:add("\",status=\"") 713 | tab:add(s) 714 | tab:add("\",role=\"") 715 | tab:add(r) 716 | tab:add("\"} ") 717 | tab:add(n) 718 | tab:add("\n") 719 | end 720 | 721 | -- combined peer status adding function 722 | 723 | local function add_peer_status(tab, u, p, r) 724 | gen_peer_prometheus_status(tab, u, p.name, r, "UP", not p.down and 1 or 0) 725 | gen_peer_prometheus_status(tab, u, p.name, r, "DOWN", p.down and 1 or 0) 726 | end 727 | 728 | local function add_peer_prometheus_status(tab, u, p, r) 729 | gen_peer_prometheus_status(tab, u, p.name, r, "UP", not p.down and 1 or 0) 730 | gen_peer_prometheus_status(tab, u, p.name, r, "DOWN", p.down and 1 or 0) 731 | end 732 | 733 | local function add_peers_info(tab, u, peers, role) 734 | local npeers = #peers 735 | for i = 1, npeers do 736 | local peer = peers[i] 737 | tab:add(" ") 738 | tab:add(peer.name) 739 | if peer.down then 740 | tab:add(" DOWN\n") 741 | else 742 | tab:add(" UP\n") 743 | end 744 | end 745 | end 746 | 747 | local function add_peers_prometheus_info(tab, u, peers, role) 748 | local npeers = #peers 749 | local found_up_peer = false 750 | for i = 1, npeers do 751 | add_peer_prometheus_status(tab, u, peers[i], role) 752 | if not peers[i].down then 753 | found_up_peer = true 754 | end 755 | end 756 | return found_up_peer 757 | end 758 | 759 | function _M.prometheus_status_page() 760 | -- generate an prometheus metrics 761 | -- # HELP nginx_upstream_status_info The running staus of nginx upstream 762 | -- # TYPE nginx_upstream_status_info gauge 763 | -- nginx_upstream_status_info{name="",endpoint="",status="",role=""} num 764 | 765 | local us, err = get_upstreams() 766 | if not us then 767 | return nil, "failed to get upstream names: " .. err 768 | end 769 | 770 | local n = #us 771 | 772 | local stats_tab = new_status_table(n) 773 | 774 | stats_tab:add( 775 | "# HELP nginx_upstream_status_info The running status of nginx upstream\n") 776 | stats_tab:add("# TYPE nginx_upstream_status_info gauge\n") 777 | 778 | for i = 1, n do 779 | local u = us[i] 780 | local ncheckers = upstream_checker_statuses[u] 781 | if not ncheckers or ncheckers == 0 then 782 | add_upstream_unknown_prometheus_status(stats_tab, u) 783 | goto continue 784 | end 785 | 786 | local peers, err = get_primary_peers(u) 787 | if not peers then 788 | add_upstream_down_prometheus_status(stats_tab, u) 789 | else 790 | local peers, err = get_primary_peers(u) 791 | local found_up_peer = false 792 | 793 | if peers then 794 | if add_peers_prometheus_info(stats_tab, u, peers, "PRIMARY") then 795 | found_up_peer = true 796 | end 797 | end 798 | 799 | peers, err = get_backup_peers(u) 800 | if peers then 801 | if add_peers_prometheus_info(stats_tab, u, peers, "BACKUP") then 802 | found_up_peer = true 803 | end 804 | end 805 | 806 | if found_up_peer then 807 | add_upstream_up_prometheus_status(stats_tab, u) 808 | else 809 | add_upstream_down_prometheus_status(stats_tab, u) 810 | end 811 | end 812 | ::continue:: 813 | end 814 | 815 | return concat(stats_tab.statuses) 816 | end 817 | 818 | function _M.status_page() 819 | -- generate an HTML page 820 | local us, err = get_upstreams() 821 | if not us then 822 | return "failed to get upstream names: " .. err 823 | end 824 | 825 | local n = #us 826 | local stats_tab = new_status_table(n) 827 | 828 | for i = 1, n do 829 | if i > 1 then 830 | stats_tab:add("\n") 831 | end 832 | 833 | local u = us[i] 834 | 835 | stats_tab:add("Upstream ") 836 | stats_tab:add(u) 837 | 838 | local ncheckers = upstream_checker_statuses[u] 839 | if not ncheckers or ncheckers == 0 then 840 | stats_tab:add(" (NO checkers)") 841 | end 842 | 843 | stats_tab:add("\n Primary Peers\n") 844 | 845 | local peers, err = get_primary_peers(u) 846 | if not peers then 847 | return 848 | "failed to get primary peers in upstream " .. u .. ": " .. err 849 | end 850 | 851 | add_peers_info(stats_tab, u, peers, "PRIMARY") 852 | 853 | stats_tab:add(" Backup Peers\n") 854 | 855 | peers, err = get_backup_peers(u) 856 | if not peers then 857 | return "failed to get backup peers in upstream " .. u .. ": " .. err 858 | end 859 | 860 | add_peers_info(stats_tab, u, peers, "BACKUP") 861 | end 862 | return concat(stats_tab.statuses) 863 | end 864 | 865 | return _M 866 | -------------------------------------------------------------------------------- /t/cert/localhost.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDSjCCAjICCQC+LTeXRR0JYjANBgkqhkiG9w0BAQsFADBmMQswCQYDVQQGEwJD 3 | TjEQMA4GA1UECAwHQmVpamluZzEQMA4GA1UEBwwHQmVpamluZzESMBAGA1UECgwJ 4 | T3BlblJlc3R5MQswCQYDVQQLDAJIQzESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTIx 5 | MDgyMTEzNTU1NVoYDzIxMjEwNzI4MTM1NTU1WjBmMQswCQYDVQQGEwJDTjEQMA4G 6 | A1UECAwHQmVpamluZzEQMA4GA1UEBwwHQmVpamluZzESMBAGA1UECgwJT3BlblJl 7 | c3R5MQswCQYDVQQLDAJIQzESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG 8 | 9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtutm6K84hWDrMEFZ6sUC8fisdoHaHv6YD6fc 9 | tQNfYIGOK0zPQLiezpHYaAAk3BkZ0Si9MVmoUbMqNTZvxkYiCcSWTwKgkbjY9W9u 10 | QNA+klXSR81tpBQCrkendJ11cKgtyN88Y4WwoJrQGJzn7byP41ySgIVNcdgoulRf 11 | wBafj9EutQqtliC5FzNtxcvOQP8ctZIYP+fnPEDdUczKgRnXsvrpapXPiCE3NAUM 12 | WPIRU4AX1yE/q9vimk57t8aydBQoM3/by9TlrD6+y80MDVIrhhfP2pyhT3eF6QrT 13 | eEhzp0Ab6NanDdgg/9VAGnEiNXvfWbDiYQm9KgHyvP7qoFc+CQIDAQABMA0GCSqG 14 | SIb3DQEBCwUAA4IBAQBgq4gozmT2eO6WXhH4jTC0M34ssCjXi+dgO1+Vmf5xrvT+ 15 | w52HibB+nx2UYA4RUimyILuP3mEaNUdzNRQjFadVC1gDa3855PRwCK+Cjxkk8/Cw 16 | S1rADzgLxr8YpW3E3xSXHSS9vhZ7LCtuSnmpVxxM75/ReXl3/3CjDicwq9nvsDXO 17 | CjT5TvgsFVTs8ix+RLl7tlsuC13Wh7jXA7u1ypsHQuR71lCh83f8nh0W/9xdlw2V 18 | cbHlyTKVsK/0YIsc6XhyYQEzBC6OXByuwK5zYubA37mY4WwsuykXXbH1XNKKj919 19 | GzLqOKxs/8YfFwIrcXJTuer9kjbJ/FdTjL2HCLGk 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /t/cert/localhost.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIICqzCCAZMCAQAwZjELMAkGA1UEBhMCQ04xEDAOBgNVBAgMB0JlaWppbmcxEDAO 3 | BgNVBAcMB0JlaWppbmcxEjAQBgNVBAoMCU9wZW5SZXN0eTELMAkGA1UECwwCSEMx 4 | EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC 5 | ggEBALbrZuivOIVg6zBBWerFAvH4rHaB2h7+mA+n3LUDX2CBjitMz0C4ns6R2GgA 6 | JNwZGdEovTFZqFGzKjU2b8ZGIgnElk8CoJG42PVvbkDQPpJV0kfNbaQUAq5Hp3Sd 7 | dXCoLcjfPGOFsKCa0Bic5+28j+NckoCFTXHYKLpUX8AWn4/RLrUKrZYguRczbcXL 8 | zkD/HLWSGD/n5zxA3VHMyoEZ17L66WqVz4ghNzQFDFjyEVOAF9chP6vb4ppOe7fG 9 | snQUKDN/28vU5aw+vsvNDA1SK4YXz9qcoU93hekK03hIc6dAG+jWpw3YIP/VQBpx 10 | IjV731mw4mEJvSoB8rz+6qBXPgkCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4IBAQA/ 11 | CTfXtiHYWJTn+0vfTFvEQwiIcQP6N9ws3+TbTqAP9GBl7yE6Q4KlNVxDNjqkOgfG 12 | DIPJqAeTN+7QMzVsaItJ+qBgzgk/UDj7qJq9NhLBSINbyRAJWaI40d1I/wLPMrST 13 | Xl6mXVCFWOivgMtSmTEhN5TKj8yFfNkcayAzfb31DdPqS4TSCf/FPFMH1E7O5mrn 14 | B8xbWBgvKx+xG+dTziE+9WFTwautQmV5do+mP4vh39hVY6gM/RDucGnVz6LNbx2k 15 | XeLsa94Ut0M3t9U3knJoEsA1pkDtTGWSAyb01kYOYT0PqWdhH2wtehMfBdx/Z8Xt 16 | FcG+JTJHN8QVUvFmJELP 17 | -----END CERTIFICATE REQUEST----- 18 | -------------------------------------------------------------------------------- /t/cert/localhost.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEogIBAAKCAQEAtutm6K84hWDrMEFZ6sUC8fisdoHaHv6YD6fctQNfYIGOK0zP 3 | QLiezpHYaAAk3BkZ0Si9MVmoUbMqNTZvxkYiCcSWTwKgkbjY9W9uQNA+klXSR81t 4 | pBQCrkendJ11cKgtyN88Y4WwoJrQGJzn7byP41ySgIVNcdgoulRfwBafj9EutQqt 5 | liC5FzNtxcvOQP8ctZIYP+fnPEDdUczKgRnXsvrpapXPiCE3NAUMWPIRU4AX1yE/ 6 | q9vimk57t8aydBQoM3/by9TlrD6+y80MDVIrhhfP2pyhT3eF6QrTeEhzp0Ab6Nan 7 | Ddgg/9VAGnEiNXvfWbDiYQm9KgHyvP7qoFc+CQIDAQABAoIBAFV62FJwX7dkvUOF 8 | pqFhg6jUDrQqpmXJkAHw7eDhPJb6tBdkA445OG1MFpczSgZY7ImjwhXn0hKr6VOI 9 | pt8/MvThm2Q81BCFE05OznUXW8ZCuMu5Ij2E5GPRNF34n0MQNKUkPTI87XeAITL/ 10 | UBr2/T7Aqe3JXZGP9Chu+XfompQ34E5XeP+OR0LrQCSx3GOZ8z1p11xN6zJXa/Cd 11 | S643q8E4q3NoaWsKNpi76WcWNnhL8ErFW09yP52TFf9d4ysL8wNsopqjUY82r6q8 12 | li8itgi80k+ewvFE6Q3TawjWvCsOD9nnKwsqw0c5SSSOGKT4tB/hO23EY6u2XVzK 13 | jbQTpokCgYEA5LUFneb33Jb29jJnP4EPRt3BNSd4OlvaTzweNq/pLIIA7whCvfBM 14 | +hQKZ6r6f8ARjSTxaylwL20kB8uURTlOcUO32OqCieyrRbB4lmUPj9HYMUnGFPpe 15 | Q52xLrwGG5xPULG6er6z3enBaiG5BfdgErQqLfQoMibVo8Q7cdxc20MCgYEAzL+T 16 | DR4nwEm3ERRJhA4KKKsexNoI3Wf+YkigpiTZ/K7ohN+1lHb7S21tsbYLhEfuQtH4 17 | aOxundzY+UbkYieSKAVvDjRGtFTlyMWh2vx+ajDFvNrSeyVv+LrSFpTj/zB0FoeD 18 | pgMYCaCQLg8gnr/tjHqM0AShVaX0RzAzV+BWPsMCgYAZs7I25Y7zTk/gqRdrTpCt 19 | 8RvWYmIjhNEK2IG4uZols1Jximcdu1SbQgdNUSynkKkoSH+NAHIoEkbbTWiTQvZq 20 | yiZuDSSwZVV2FnfmxuKx0e5O07CcCOrxBFa2HtE85xsOwXpocuf5x1xCFhoZ7ovw 21 | bijTUz31LOITbmkTyaTxywKBgDYkSoxjdvuAAqq/hJko5ULPzTkcts9lWn0+20xT 22 | 3ljVH6NTTL7Pn6/YZfNiQjVemACPaXLFYpLX/YAdYPMbp6hxl2ZYKIIzGPg4Wo5/ 23 | yiXmc8N20cXCppNNQ5S5Fnk6pNf4SbWyh76z+KxVT7jq47QMDARN1SPC2I+ijVEl 24 | f2jlAoGAT9Czp0+j4njn2QM+c5lmIMwbEvF4AB8J1Ymgy1BbgQ0kJp9TSiNIZ0dV 25 | u0UUEudmGMzzLSM6B1zNtttb8E+RzH8JxXE9IJENHGRvMHa8X9+EAyTYGO4zLLyi 26 | +B/fcW0uUqCTiJHMPM/b0RMAA7jHH/9W7MEyYbcDqC2tZIdOQCE= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /t/https.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 t fdm=marker: 2 | use lib 'lib'; 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | #worker_connections(1014); 7 | #master_process_enabled(1); 8 | #log_level('warn'); 9 | 10 | #repeat_each(2); 11 | 12 | plan tests => repeat_each() * (blocks() * 2); 13 | 14 | my $pwd = cwd(); 15 | 16 | our $HttpConfig = <<_EOC_; 17 | lua_socket_log_errors off; 18 | lua_package_path "$pwd/../lua-resty-lock/?.lua;$pwd/lib/?.lua;$pwd/t/lib/?.lua;;"; 19 | _EOC_ 20 | 21 | #no_diff(); 22 | no_long_string(); 23 | run_tests(); 24 | 25 | __DATA__ 26 | 27 | === TEST 1: https health check (good case), status ignored by default 28 | --- http_config eval 29 | "$::HttpConfig" 30 | . q{ 31 | upstream foo.com { 32 | server 127.0.0.1:12354; 33 | server 127.0.0.1:12355; 34 | server 127.0.0.1:12356 backup; 35 | } 36 | 37 | server { 38 | listen 12354 ssl; 39 | 40 | server_name localhost; 41 | 42 | ssl_certificate ../../cert/localhost.crt; 43 | ssl_certificate_key ../../cert/localhost.key; 44 | 45 | location /status { 46 | return 200; 47 | } 48 | } 49 | 50 | server { 51 | listen 12355 ssl; 52 | 53 | server_name localhost; 54 | 55 | ssl_certificate ../../cert/localhost.crt; 56 | ssl_certificate_key ../../cert/localhost.key; 57 | 58 | location /status { 59 | return 200; 60 | } 61 | } 62 | 63 | server { 64 | listen 12356 ssl; 65 | 66 | server_name localhost; 67 | 68 | ssl_certificate ../../cert/localhost.crt; 69 | ssl_certificate_key ../../cert/localhost.key; 70 | 71 | location /status { 72 | return 200; 73 | } 74 | } 75 | 76 | lua_shared_dict healthcheck 1m; 77 | 78 | init_worker_by_lua ' 79 | ngx.shared.healthcheck:flush_all() 80 | local hc = require "resty.upstream.healthcheck" 81 | local ok, err = hc.spawn_checker{ 82 | shm = "healthcheck", 83 | upstream = "foo.com", 84 | type = "https", 85 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 86 | ssl_verify = false, 87 | host = "localhost", 88 | interval = 100, -- 100ms 89 | fall = 2, 90 | } 91 | if not ok then 92 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 93 | return 94 | end 95 | '; 96 | } 97 | --- config 98 | location = /t { 99 | access_log off; 100 | content_by_lua ' 101 | ngx.sleep(0.52) 102 | 103 | local hc = require "resty.upstream.healthcheck" 104 | ngx.print(hc.status_page()) 105 | 106 | for i = 1, 2 do 107 | local res = ngx.location.capture("/proxy") 108 | ngx.say("upstream addr: ", res.header["X-Foo"]) 109 | end 110 | '; 111 | } 112 | 113 | location = /proxy { 114 | proxy_pass http://foo.com/; 115 | header_filter_by_lua ' 116 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 117 | '; 118 | } 119 | 120 | --- request 121 | GET /t 122 | 123 | --- response_body 124 | Upstream foo.com 125 | Primary Peers 126 | 127.0.0.1:12354 UP 127 | 127.0.0.1:12355 UP 128 | Backup Peers 129 | 127.0.0.1:12356 UP 130 | upstream addr: 127.0.0.1:12354 131 | upstream addr: 127.0.0.1:12355 132 | --- timeout: 6 133 | -------------------------------------------------------------------------------- /t/lib/ljson.lua: -------------------------------------------------------------------------------- 1 | local ngx_null = ngx.null 2 | local tostring = tostring 3 | local gsub = string.gsub 4 | local sort = table.sort 5 | local pairs = pairs 6 | local ipairs = ipairs 7 | local concat = table.concat 8 | 9 | local ok, new_tab = pcall(require, "table.new") 10 | if not ok then 11 | new_tab = function (_, _) return {} end 12 | end 13 | 14 | local _M = {} 15 | 16 | local metachars = { 17 | ['\t'] = '\\t', 18 | ["\\"] = "\\\\", 19 | ['"'] = '\\"', 20 | ['\r'] = '\\r', 21 | ['\n'] = '\\n', 22 | } 23 | 24 | local function encode_str(s) 25 | -- XXX we will rewrite this when string.buffer is implemented 26 | -- in LuaJIT 2.1 because string.gsub cannot be JIT compiled. 27 | return gsub(s, '["\\\r\n\t]', metachars) 28 | end 29 | 30 | local function is_arr(t) 31 | local exp = 1 32 | for k, _ in pairs(t) do 33 | if k ~= exp then 34 | return nil 35 | end 36 | exp = exp + 1 37 | end 38 | return exp - 1 39 | end 40 | 41 | local encode 42 | 43 | encode = function (v) 44 | if v == nil or v == ngx_null then 45 | return "null" 46 | end 47 | 48 | local typ = type(v) 49 | if typ == 'string' then 50 | return '"' .. encode_str(v) .. '"' 51 | end 52 | 53 | if typ == 'number' or typ == 'boolean' then 54 | return tostring(v) 55 | end 56 | 57 | if typ == 'table' then 58 | local n = is_arr(v) 59 | if n then 60 | local bits = new_tab(n, 0) 61 | for i, elem in ipairs(v) do 62 | bits[i] = encode(elem) 63 | end 64 | return "[" .. concat(bits, ",") .. "]" 65 | end 66 | 67 | local keys = {} 68 | local i = 0 69 | for key, _ in pairs(v) do 70 | i = i + 1 71 | keys[i] = key 72 | end 73 | sort(keys) 74 | 75 | local bits = new_tab(0, i) 76 | i = 0 77 | for _, key in ipairs(keys) do 78 | i = i + 1 79 | bits[i] = encode(key) .. ":" .. encode(v[key]) 80 | end 81 | return "{" .. concat(bits, ",") .. "}" 82 | end 83 | 84 | return '"<' .. typ .. '>"' 85 | end 86 | _M.encode = encode 87 | 88 | return _M 89 | -------------------------------------------------------------------------------- /t/prometheus_down.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et fdm=marker: 2 | use lib 'lib'; 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | #worker_connections(1014); 7 | #master_process_enabled(1); 8 | #log_level('warn'); 9 | 10 | #repeat_each(2); 11 | 12 | plan tests => repeat_each() * (blocks() * 2); 13 | 14 | my $pwd = cwd(); 15 | 16 | our $HttpConfig = <<_EOC_; 17 | lua_socket_log_errors off; 18 | lua_package_path "$pwd/../lua-resty-lock/?.lua;$pwd/lib/?.lua;$pwd/t/lib/?.lua;;"; 19 | _EOC_ 20 | 21 | #no_diff(); 22 | no_long_string(); 23 | run_tests(); 24 | 25 | __DATA__ 26 | 27 | === TEST 1: prometheus format status page (everything is down) 28 | --- http_config eval 29 | "$::HttpConfig" 30 | . q{ 31 | upstream foo.com { 32 | server 127.0.0.1:12354; 33 | server 127.0.0.1:12355; 34 | server 127.0.0.1:12356 backup; 35 | } 36 | 37 | server { 38 | listen 12354; 39 | location = /status { 40 | return 500; 41 | } 42 | } 43 | 44 | server { 45 | listen 12355; 46 | location = /status { 47 | return 500; 48 | } 49 | } 50 | 51 | server { 52 | listen 12356; 53 | location = /status { 54 | return 500; 55 | } 56 | } 57 | 58 | lua_shared_dict healthcheck 1m; 59 | init_worker_by_lua ' 60 | ngx.shared.healthcheck:flush_all() 61 | local hc = require "resty.upstream.healthcheck" 62 | local ok, err = hc.spawn_checker{ 63 | shm = "healthcheck", 64 | upstream = "foo.com", 65 | type = "http", 66 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 67 | interval = 100, -- 100ms 68 | fall = 2, 69 | valid_statuses = {200} 70 | } 71 | if not ok then 72 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 73 | return 74 | end 75 | '; 76 | } 77 | --- config 78 | location = /t { 79 | access_log off; 80 | content_by_lua ' 81 | ngx.sleep(0.52) 82 | local hc = require "resty.upstream.healthcheck" 83 | local st , err = hc.prometheus_status_page() 84 | if not st then 85 | ngx.say(err) 86 | return 87 | end 88 | ngx.print(st) 89 | '; 90 | } 91 | --- request 92 | GET /t 93 | 94 | --- response_body 95 | # HELP nginx_upstream_status_info The running status of nginx upstream 96 | # TYPE nginx_upstream_status_info gauge 97 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12354",status="UP",role="PRIMARY"} 0 98 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12354",status="DOWN",role="PRIMARY"} 1 99 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12355",status="UP",role="PRIMARY"} 0 100 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12355",status="DOWN",role="PRIMARY"} 1 101 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12356",status="UP",role="BACKUP"} 0 102 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12356",status="DOWN",role="BACKUP"} 1 103 | nginx_upstream_status_info{name="foo.com",status="UP"} 0 104 | nginx_upstream_status_info{name="foo.com",status="DOWN"} 1 105 | nginx_upstream_status_info{name="foo.com",status="UNKNOWN"} 0 106 | -------------------------------------------------------------------------------- /t/prometheus_empty.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et fdm=marker: 2 | use lib 'lib'; 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | #worker_connections(1014); 7 | #master_process_enabled(1); 8 | #log_level('warn'); 9 | 10 | #repeat_each(2); 11 | 12 | plan tests => repeat_each() * (blocks() * 2); 13 | 14 | my $pwd = cwd(); 15 | 16 | our $HttpConfig = <<_EOC_; 17 | lua_socket_log_errors off; 18 | lua_package_path "$pwd/../lua-resty-lock/?.lua;$pwd/lib/?.lua;$pwd/t/lib/?.lua;;"; 19 | _EOC_ 20 | 21 | #no_diff(); 22 | no_long_string(); 23 | run_tests(); 24 | 25 | __DATA__ 26 | 27 | === TEST 1: prometheus format status page (no peers) 28 | --- http_config eval 29 | "$::HttpConfig" 30 | . q{ 31 | lua_shared_dict healthcheck 1m; 32 | init_worker_by_lua ' 33 | ngx.config.debug = 1 34 | ngx.shared.healthcheck:flush_all() 35 | local hc = require "resty.upstream.healthcheck" 36 | local ok, err = hc.spawn_checker{ 37 | shm = "healthcheck", 38 | upstream = "not_found", 39 | type = "http", 40 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 41 | interval = 100, -- 100ms 42 | fall = 2, 43 | valid_statuses = {200} 44 | } 45 | if not ok then 46 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 47 | return 48 | end 49 | '; 50 | } 51 | --- config 52 | location = /t { 53 | access_log off; 54 | content_by_lua ' 55 | ngx.sleep(0.52) 56 | local hc = require "resty.upstream.healthcheck" 57 | local st , err = hc.prometheus_status_page() 58 | if not st then 59 | ngx.say(err) 60 | return 61 | end 62 | ngx.print(st) 63 | '; 64 | } 65 | --- request 66 | GET /t 67 | 68 | --- response_body 69 | # HELP nginx_upstream_status_info The running status of nginx upstream 70 | # TYPE nginx_upstream_status_info gauge 71 | -------------------------------------------------------------------------------- /t/prometheus_mixed.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et fdm=marker: 2 | use lib 'lib'; 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | #worker_connections(1014); 7 | #master_process_enabled(1); 8 | #log_level('warn'); 9 | 10 | #repeat_each(2); 11 | 12 | plan tests => repeat_each() * (blocks() * 2); 13 | 14 | my $pwd = cwd(); 15 | 16 | our $HttpConfig = <<_EOC_; 17 | lua_socket_log_errors off; 18 | lua_package_path "$pwd/../lua-resty-lock/?.lua;$pwd/lib/?.lua;$pwd/t/lib/?.lua;;"; 19 | _EOC_ 20 | 21 | #no_diff(); 22 | no_long_string(); 23 | run_tests(); 24 | 25 | __DATA__ 26 | 27 | === TEST 1: prometheus format status page (mixed down and up) 28 | --- http_config eval 29 | "$::HttpConfig" 30 | . q{ 31 | upstream unknown.com { 32 | server 127.0.0.1:12366; 33 | } 34 | 35 | upstream foo.com { 36 | server 127.0.0.1:12354; 37 | server 127.0.0.1:12355; 38 | server 127.0.0.1:12356 backup; 39 | server 127.0.0.1:12357; 40 | } 41 | 42 | server { 43 | listen 12354; 44 | location = /status { 45 | return 500; 46 | } 47 | } 48 | 49 | server { 50 | listen 12355; 51 | location = /status { 52 | return 200; 53 | } 54 | } 55 | 56 | server { 57 | listen 12356; 58 | location = /status { 59 | return 500; 60 | } 61 | } 62 | 63 | server { 64 | listen 12357; 65 | location = /status { 66 | return 200; 67 | } 68 | } 69 | 70 | server { 71 | listen 12366; 72 | location = /status { 73 | return 200; 74 | } 75 | } 76 | 77 | lua_shared_dict healthcheck 1m; 78 | init_worker_by_lua ' 79 | ngx.config.debug = 1 80 | ngx.shared.healthcheck:flush_all() 81 | local hc = require "resty.upstream.healthcheck" 82 | local ok, err = hc.spawn_checker{ 83 | shm = "healthcheck", 84 | upstream = "foo.com", 85 | type = "http", 86 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 87 | interval = 100, -- 100ms 88 | fall = 2, 89 | valid_statuses = {200} 90 | } 91 | if not ok then 92 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 93 | return 94 | end 95 | '; 96 | } 97 | --- config 98 | location = /t { 99 | access_log off; 100 | content_by_lua ' 101 | ngx.sleep(0.52) 102 | local hc = require "resty.upstream.healthcheck" 103 | local st, err = hc.prometheus_status_page() 104 | if not st then 105 | ngx.say(err) 106 | return 107 | end 108 | ngx.print(st) 109 | '; 110 | } 111 | --- request 112 | GET /t 113 | 114 | --- response_body 115 | # HELP nginx_upstream_status_info The running status of nginx upstream 116 | # TYPE nginx_upstream_status_info gauge 117 | nginx_upstream_status_info{name="unknown.com",status="UP"} 0 118 | nginx_upstream_status_info{name="unknown.com",status="DOWN"} 0 119 | nginx_upstream_status_info{name="unknown.com",status="UNKNOWN"} 1 120 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12354",status="UP",role="PRIMARY"} 0 121 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12354",status="DOWN",role="PRIMARY"} 1 122 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12355",status="UP",role="PRIMARY"} 1 123 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12355",status="DOWN",role="PRIMARY"} 0 124 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12357",status="UP",role="PRIMARY"} 1 125 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12357",status="DOWN",role="PRIMARY"} 0 126 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12356",status="UP",role="BACKUP"} 0 127 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12356",status="DOWN",role="BACKUP"} 1 128 | nginx_upstream_status_info{name="foo.com",status="UP"} 1 129 | nginx_upstream_status_info{name="foo.com",status="DOWN"} 0 130 | nginx_upstream_status_info{name="foo.com",status="UNKNOWN"} 0 131 | -------------------------------------------------------------------------------- /t/prometheus_up.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et fdm=marker: 2 | use lib 'lib'; 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | #worker_connections(1014); 7 | #master_process_enabled(1); 8 | #log_level('warn'); 9 | 10 | #repeat_each(2); 11 | 12 | plan tests => repeat_each() * (blocks() * 2); 13 | 14 | my $pwd = cwd(); 15 | 16 | our $HttpConfig = <<_EOC_; 17 | lua_socket_log_errors off; 18 | lua_package_path "$pwd/../lua-resty-lock/?.lua;$pwd/lib/?.lua;$pwd/t/lib/?.lua;;"; 19 | _EOC_ 20 | 21 | #no_diff(); 22 | no_long_string(); 23 | run_tests(); 24 | 25 | __DATA__ 26 | 27 | === TEST 1: prometheus format status page (everything is healthy) 28 | --- http_config eval 29 | "$::HttpConfig" 30 | . q{ 31 | upstream foo.com { 32 | server 127.0.0.1:12354; 33 | server 127.0.0.1:12355; 34 | server 127.0.0.1:12356 backup; 35 | } 36 | 37 | server { 38 | listen 12354; 39 | location = /status { 40 | return 200; 41 | } 42 | } 43 | 44 | server { 45 | listen 12355; 46 | location = /status { 47 | return 200; 48 | } 49 | } 50 | 51 | server { 52 | listen 12356; 53 | location = /status { 54 | return 200; 55 | } 56 | } 57 | 58 | lua_shared_dict healthcheck 1m; 59 | init_worker_by_lua ' 60 | ngx.config.debug = 1 61 | ngx.shared.healthcheck:flush_all() 62 | local hc = require "resty.upstream.healthcheck" 63 | local ok, err = hc.spawn_checker{ 64 | shm = "healthcheck", 65 | upstream = "foo.com", 66 | type = "http", 67 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 68 | interval = 100, -- 100ms 69 | fall = 2, 70 | valid_statuses = {200} 71 | } 72 | if not ok then 73 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 74 | return 75 | end 76 | '; 77 | } 78 | --- config 79 | location = /t { 80 | access_log off; 81 | content_by_lua ' 82 | ngx.sleep(0.52) 83 | local hc = require "resty.upstream.healthcheck" 84 | local st , err = hc.prometheus_status_page() 85 | if not st then 86 | ngx.say(err) 87 | return 88 | end 89 | ngx.print(st) 90 | '; 91 | } 92 | --- request 93 | GET /t 94 | 95 | --- response_body 96 | # HELP nginx_upstream_status_info The running status of nginx upstream 97 | # TYPE nginx_upstream_status_info gauge 98 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12354",status="UP",role="PRIMARY"} 1 99 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12354",status="DOWN",role="PRIMARY"} 0 100 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12355",status="UP",role="PRIMARY"} 1 101 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12355",status="DOWN",role="PRIMARY"} 0 102 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12356",status="UP",role="BACKUP"} 1 103 | nginx_upstream_status_info{name="foo.com",endpoint="127.0.0.1:12356",status="DOWN",role="BACKUP"} 0 104 | nginx_upstream_status_info{name="foo.com",status="UP"} 1 105 | nginx_upstream_status_info{name="foo.com",status="DOWN"} 0 106 | nginx_upstream_status_info{name="foo.com",status="UNKNOWN"} 0 107 | -------------------------------------------------------------------------------- /t/sanity.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et fdm=marker: 2 | use lib 'lib'; 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | #worker_connections(1014); 7 | #master_process_enabled(1); 8 | #log_level('warn'); 9 | 10 | #repeat_each(2); 11 | 12 | plan tests => repeat_each() * (blocks() * 6 + 11); 13 | 14 | my $pwd = cwd(); 15 | 16 | our $HttpConfig = <<_EOC_; 17 | lua_socket_log_errors off; 18 | lua_package_path "$pwd/../lua-resty-lock/?.lua;$pwd/lib/?.lua;$pwd/t/lib/?.lua;;"; 19 | _EOC_ 20 | 21 | #no_diff(); 22 | no_long_string(); 23 | run_tests(); 24 | 25 | __DATA__ 26 | 27 | === TEST 1: health check (good case), status ignored by default 28 | --- http_config eval 29 | "$::HttpConfig" 30 | . q{ 31 | upstream foo.com { 32 | server 127.0.0.1:12354; 33 | server 127.0.0.1:12355; 34 | server 127.0.0.1:12356 backup; 35 | } 36 | 37 | server { 38 | listen 12354; 39 | location = /status { 40 | return 200; 41 | } 42 | } 43 | 44 | server { 45 | listen 12355; 46 | location = /status { 47 | return 404; 48 | } 49 | } 50 | 51 | server { 52 | listen 12356; 53 | location = /status { 54 | return 503; 55 | } 56 | } 57 | 58 | lua_shared_dict healthcheck 1m; 59 | init_worker_by_lua ' 60 | ngx.config.debug = 1 61 | ngx.shared.healthcheck:flush_all() 62 | local hc = require "resty.upstream.healthcheck" 63 | local ok, err = hc.spawn_checker{ 64 | shm = "healthcheck", 65 | upstream = "foo.com", 66 | type = "http", 67 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 68 | interval = 100, -- 100ms 69 | fall = 2, 70 | } 71 | if not ok then 72 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 73 | return 74 | end 75 | '; 76 | } 77 | --- config 78 | location = /t { 79 | access_log off; 80 | content_by_lua ' 81 | ngx.sleep(0.52) 82 | 83 | local hc = require "resty.upstream.healthcheck" 84 | ngx.print(hc.status_page()) 85 | 86 | for i = 1, 2 do 87 | local res = ngx.location.capture("/proxy") 88 | ngx.say("upstream addr: ", res.header["X-Foo"]) 89 | end 90 | '; 91 | } 92 | 93 | location = /proxy { 94 | proxy_pass http://foo.com/; 95 | header_filter_by_lua ' 96 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 97 | '; 98 | } 99 | --- request 100 | GET /t 101 | 102 | --- response_body 103 | Upstream foo.com 104 | Primary Peers 105 | 127.0.0.1:12354 UP 106 | 127.0.0.1:12355 UP 107 | Backup Peers 108 | 127.0.0.1:12356 UP 109 | upstream addr: 127.0.0.1:12354 110 | upstream addr: 127.0.0.1:12355 111 | 112 | --- no_error_log 113 | [error] 114 | [alert] 115 | [warn] 116 | was checked to be not ok 117 | failed to run healthcheck cycle 118 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|publishing peers version \d+|upgrading peers version to \d+/ 119 | --- grep_error_log_out eval 120 | qr/^healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 121 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 122 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 123 | (?:healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 124 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 125 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 126 | ){3,5}$/ 127 | --- timeout: 6 128 | 129 | 130 | 131 | === TEST 2: health check (bad case), no listening port in the backup peer 132 | --- http_config eval 133 | "$::HttpConfig" 134 | . q{ 135 | upstream foo.com { 136 | server 127.0.0.1:12354; 137 | server 127.0.0.1:12355; 138 | server 127.0.0.1:12356 backup; 139 | } 140 | 141 | server { 142 | listen 12354; 143 | location = /status { 144 | return 200; 145 | } 146 | } 147 | 148 | server { 149 | listen 12355; 150 | location = /status { 151 | return 404; 152 | } 153 | } 154 | 155 | lua_shared_dict healthcheck 1m; 156 | init_worker_by_lua ' 157 | ngx.config.debug = 1 158 | ngx.shared.healthcheck:flush_all() 159 | local hc = require "resty.upstream.healthcheck" 160 | local ok, err = hc.spawn_checker{ 161 | shm = "healthcheck", 162 | upstream = "foo.com", 163 | type = "http", 164 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 165 | interval = 100, -- 100ms 166 | fall = 2, 167 | } 168 | if not ok then 169 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 170 | return 171 | end 172 | '; 173 | } 174 | --- config 175 | location = /t { 176 | access_log off; 177 | content_by_lua ' 178 | ngx.sleep(0.52) 179 | 180 | local hc = require "resty.upstream.healthcheck" 181 | ngx.print(hc.status_page()) 182 | 183 | for i = 1, 2 do 184 | local res = ngx.location.capture("/proxy") 185 | ngx.say("upstream addr: ", res.header["X-Foo"]) 186 | end 187 | '; 188 | } 189 | 190 | location = /proxy { 191 | proxy_pass http://foo.com/; 192 | header_filter_by_lua ' 193 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 194 | '; 195 | } 196 | --- request 197 | GET /t 198 | 199 | --- response_body 200 | Upstream foo.com 201 | Primary Peers 202 | 127.0.0.1:12354 UP 203 | 127.0.0.1:12355 UP 204 | Backup Peers 205 | 127.0.0.1:12356 DOWN 206 | upstream addr: 127.0.0.1:12354 207 | upstream addr: 127.0.0.1:12355 208 | 209 | --- no_error_log 210 | [alert] 211 | failed to run healthcheck cycle 212 | --- error_log 213 | healthcheck: failed to connect to 127.0.0.1:12356: connection refused 214 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|warn\(\): .*(?=,)|publishing peers version \d+|upgrading peers version to \d+/ 215 | --- grep_error_log_out eval 216 | qr/^healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 217 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 218 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be not ok 219 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 220 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 221 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be not ok 222 | warn\(\): healthcheck: peer 127\.0\.0\.1:12356 is turned down after 2 failure\(s\) 223 | publishing peers version 1 224 | (?:healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 225 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 226 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be not ok 227 | ){2,4}$/ 228 | --- timeout: 6 229 | 230 | 231 | 232 | === TEST 3: health check (bad case), no listening port in a primary peer 233 | --- http_config eval 234 | "$::HttpConfig" 235 | . q{ 236 | upstream foo.com { 237 | server 127.0.0.1:12354; 238 | server 127.0.0.1:12355; 239 | server 127.0.0.1:12356 backup; 240 | } 241 | 242 | server { 243 | listen 12354; 244 | location = /status { 245 | return 200; 246 | } 247 | } 248 | 249 | server { 250 | listen 12356; 251 | location = /status { 252 | return 404; 253 | } 254 | } 255 | 256 | lua_shared_dict healthcheck 1m; 257 | init_worker_by_lua ' 258 | ngx.config.debug = 1 259 | ngx.shared.healthcheck:flush_all() 260 | local hc = require "resty.upstream.healthcheck" 261 | local ok, err = hc.spawn_checker{ 262 | shm = "healthcheck", 263 | upstream = "foo.com", 264 | type = "http", 265 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 266 | interval = 100, -- 100ms 267 | fall = 2, 268 | } 269 | if not ok then 270 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 271 | return 272 | end 273 | '; 274 | } 275 | --- config 276 | location = /t { 277 | access_log off; 278 | content_by_lua ' 279 | ngx.sleep(0.52) 280 | 281 | local hc = require "resty.upstream.healthcheck" 282 | ngx.print(hc.status_page()) 283 | 284 | for i = 1, 2 do 285 | local res = ngx.location.capture("/proxy") 286 | ngx.say("upstream addr: ", res.header["X-Foo"]) 287 | end 288 | '; 289 | } 290 | 291 | location = /proxy { 292 | proxy_pass http://foo.com/; 293 | header_filter_by_lua ' 294 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 295 | '; 296 | } 297 | --- request 298 | GET /t 299 | 300 | --- response_body 301 | Upstream foo.com 302 | Primary Peers 303 | 127.0.0.1:12354 UP 304 | 127.0.0.1:12355 DOWN 305 | Backup Peers 306 | 127.0.0.1:12356 UP 307 | upstream addr: 127.0.0.1:12354 308 | upstream addr: 127.0.0.1:12354 309 | 310 | --- no_error_log 311 | [alert] 312 | failed to run healthcheck cycle 313 | --- error_log 314 | healthcheck: failed to connect to 127.0.0.1:12355: connection refused 315 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|warn\(\): .*(?=,)|upgrading peers version to \d+/ 316 | --- grep_error_log_out eval 317 | qr/^healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 318 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be not ok 319 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 320 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 321 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be not ok 322 | warn\(\): healthcheck: peer 127\.0\.0\.1:12355 is turned down after 2 failure\(s\) 323 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 324 | (?:healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 325 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be not ok 326 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 327 | ){2,4}$/ 328 | --- timeout: 6 329 | 330 | 331 | 332 | === TEST 4: health check (bad case), bad status 333 | --- http_config eval 334 | "$::HttpConfig" 335 | . q{ 336 | upstream foo.com { 337 | server 127.0.0.1:12354; 338 | server 127.0.0.1:12355; 339 | server 127.0.0.1:12356 backup; 340 | } 341 | 342 | server { 343 | listen 12354; 344 | location = /status { 345 | return 200; 346 | } 347 | } 348 | 349 | server { 350 | listen 12355; 351 | location = /status { 352 | return 404; 353 | } 354 | } 355 | 356 | server { 357 | listen 12356; 358 | location = /status { 359 | return 503; 360 | } 361 | } 362 | 363 | lua_shared_dict healthcheck 1m; 364 | init_worker_by_lua ' 365 | ngx.config.debug = 1 366 | ngx.shared.healthcheck:flush_all() 367 | local hc = require "resty.upstream.healthcheck" 368 | local ok, err = hc.spawn_checker{ 369 | shm = "healthcheck", 370 | upstream = "foo.com", 371 | type = "http", 372 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 373 | interval = 100, -- 100ms 374 | fall = 2, 375 | valid_statuses = {200, 503}, 376 | } 377 | if not ok then 378 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 379 | return 380 | end 381 | '; 382 | } 383 | --- config 384 | location = /t { 385 | access_log off; 386 | content_by_lua ' 387 | ngx.sleep(0.52) 388 | 389 | local hc = require "resty.upstream.healthcheck" 390 | ngx.print(hc.status_page()) 391 | 392 | for i = 1, 2 do 393 | local res = ngx.location.capture("/proxy") 394 | ngx.say("upstream addr: ", res.header["X-Foo"]) 395 | end 396 | '; 397 | } 398 | 399 | location = /proxy { 400 | proxy_pass http://foo.com/; 401 | header_filter_by_lua ' 402 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 403 | '; 404 | } 405 | --- request 406 | GET /t 407 | 408 | --- response_body 409 | Upstream foo.com 410 | Primary Peers 411 | 127.0.0.1:12354 UP 412 | 127.0.0.1:12355 DOWN 413 | Backup Peers 414 | 127.0.0.1:12356 UP 415 | upstream addr: 127.0.0.1:12354 416 | upstream addr: 127.0.0.1:12354 417 | --- no_error_log 418 | [alert] 419 | failed to run healthcheck cycle 420 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|warn\(\): .*(?=,)|healthcheck: bad status code from .*(?=,)|upgrading peers version to \d+/ 421 | --- grep_error_log_out eval 422 | qr/^healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 423 | healthcheck: bad status code from 127\.0\.0\.1:12355: 404 424 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be not ok 425 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 426 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 427 | healthcheck: bad status code from 127\.0\.0\.1:12355: 404 428 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be not ok 429 | warn\(\): healthcheck: peer 127\.0\.0\.1:12355 is turned down after 2 failure\(s\) 430 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 431 | (?:healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 432 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be not ok 433 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 434 | ){1,4}$/ 435 | --- timeout: 6 436 | 437 | 438 | 439 | === TEST 5: health check (bad case), timed out 440 | --- http_config eval 441 | "$::HttpConfig" 442 | . q{ 443 | upstream foo.com { 444 | server 127.0.0.1:12354; 445 | server 127.0.0.1:12355; 446 | server 127.0.0.1:12356 backup; 447 | } 448 | 449 | server { 450 | listen 12354; 451 | location = /status { 452 | echo_sleep 0.5; 453 | echo ok; 454 | } 455 | } 456 | 457 | server { 458 | listen 12355; 459 | location = /status { 460 | return 404; 461 | } 462 | } 463 | 464 | server { 465 | listen 12356; 466 | location = /status { 467 | return 503; 468 | } 469 | } 470 | 471 | lua_shared_dict healthcheck 1m; 472 | init_worker_by_lua ' 473 | ngx.config.debug = 1 474 | ngx.shared.healthcheck:flush_all() 475 | local hc = require "resty.upstream.healthcheck" 476 | local ok, err = hc.spawn_checker{ 477 | shm = "healthcheck", 478 | upstream = "foo.com", 479 | type = "http", 480 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 481 | interval = 100, -- 100ms 482 | timeout = 100, -- 100ms 483 | fall = 2, 484 | } 485 | if not ok then 486 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 487 | return 488 | end 489 | '; 490 | } 491 | --- config 492 | location = /t { 493 | access_log off; 494 | content_by_lua ' 495 | ngx.sleep(0.52) 496 | 497 | local hc = require "resty.upstream.healthcheck" 498 | ngx.print(hc.status_page()) 499 | 500 | for i = 1, 2 do 501 | local res = ngx.location.capture("/proxy") 502 | ngx.say("upstream addr: ", res.header["X-Foo"]) 503 | end 504 | '; 505 | } 506 | 507 | location = /proxy { 508 | proxy_pass http://foo.com/; 509 | header_filter_by_lua ' 510 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 511 | '; 512 | } 513 | 514 | --- request 515 | GET /t 516 | 517 | --- response_body 518 | Upstream foo.com 519 | Primary Peers 520 | 127.0.0.1:12354 DOWN 521 | 127.0.0.1:12355 UP 522 | Backup Peers 523 | 127.0.0.1:12356 UP 524 | upstream addr: 127.0.0.1:12355 525 | upstream addr: 127.0.0.1:12355 526 | --- no_error_log 527 | [alert] 528 | failed to run healthcheck cycle 529 | --- error_log 530 | healthcheck: failed to receive status line from 127.0.0.1:12354: timeout 531 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|warn\(\): .*(?=,)|healthcheck: bad status code from .*(?=,)|upgrading peers version to \d+/ 532 | --- grep_error_log_out eval 533 | qr/^healthcheck: peer 127\.0\.0\.1:12354 was checked to be not ok 534 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 535 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 536 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be not ok 537 | warn\(\): healthcheck: peer 127\.0\.0\.1:12354 is turned down after 2 failure\(s\) 538 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 539 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 540 | (?:healthcheck: peer 127\.0\.0\.1:12354 was checked to be not ok 541 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 542 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 543 | ){0,2}$/ 544 | 545 | 546 | 547 | === TEST 6: health check (bad case), bad status, and then rise again 548 | --- http_config eval 549 | "$::HttpConfig" 550 | . q{ 551 | upstream foo.com { 552 | server 127.0.0.1:12354; 553 | server 127.0.0.1:12355; 554 | server 127.0.0.1:12356 backup; 555 | } 556 | 557 | server { 558 | listen 12354; 559 | location = /status { 560 | return 200; 561 | } 562 | } 563 | 564 | server { 565 | listen 12355; 566 | location = /status { 567 | content_by_lua ' 568 | local cnt = package.loaded.cnt 569 | if not cnt then 570 | cnt = 0 571 | end 572 | cnt = cnt + 1 573 | package.loaded.cnt = cnt 574 | if cnt >= 3 then 575 | return ngx.exit(200) 576 | end 577 | return ngx.exit(403) 578 | '; 579 | } 580 | } 581 | 582 | server { 583 | listen 12356; 584 | location = /status { 585 | return 503; 586 | } 587 | } 588 | 589 | lua_shared_dict healthcheck 1m; 590 | init_worker_by_lua ' 591 | ngx.config.debug = 1 592 | ngx.shared.healthcheck:flush_all() 593 | local hc = require "resty.upstream.healthcheck" 594 | local ok, err = hc.spawn_checker{ 595 | shm = "healthcheck", 596 | upstream = "foo.com", 597 | type = "http", 598 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 599 | interval = 100, -- 100ms 600 | fall = 1, 601 | rise = 2, 602 | valid_statuses = {200, 503}, 603 | } 604 | if not ok then 605 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 606 | return 607 | end 608 | '; 609 | } 610 | --- config 611 | location = /t { 612 | access_log off; 613 | content_by_lua ' 614 | ngx.sleep(0.52) 615 | 616 | local hc = require "resty.upstream.healthcheck" 617 | ngx.print(hc.status_page()) 618 | 619 | for i = 1, 2 do 620 | local res = ngx.location.capture("/proxy") 621 | ngx.say("upstream addr: ", res.header["X-Foo"]) 622 | end 623 | '; 624 | } 625 | 626 | location = /proxy { 627 | proxy_pass http://foo.com/; 628 | header_filter_by_lua ' 629 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 630 | '; 631 | } 632 | 633 | --- request 634 | GET /t 635 | 636 | --- response_body 637 | Upstream foo.com 638 | Primary Peers 639 | 127.0.0.1:12354 UP 640 | 127.0.0.1:12355 UP 641 | Backup Peers 642 | 127.0.0.1:12356 UP 643 | upstream addr: 127.0.0.1:12354 644 | upstream addr: 127.0.0.1:12355 645 | 646 | --- no_error_log 647 | [alert] 648 | failed to run healthcheck cycle 649 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|warn\(\): .*(?=,)|healthcheck: bad status code from .*(?=,)|publishing peers version \d+|upgrading peers version to \d+/ 650 | --- grep_error_log_out eval 651 | qr/^healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 652 | healthcheck: bad status code from 127\.0\.0\.1:12355: 403 653 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be not ok 654 | warn\(\): healthcheck: peer 127\.0\.0\.1:12355 is turned down after 1 failure\(s\) 655 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 656 | publishing peers version 1 657 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 658 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be not ok 659 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 660 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 661 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 662 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 663 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 664 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 665 | warn\(\): healthcheck: peer 127\.0\.0\.1:12355 is turned up after 2 success\(es\) 666 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 667 | publishing peers version 2 668 | (?:healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 669 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 670 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 671 | ){1,3}$/ 672 | --- timeout: 6 673 | 674 | 675 | 676 | === TEST 7: peers version upgrade (make up peers down) 677 | --- http_config eval 678 | "$::HttpConfig" 679 | . q{ 680 | upstream foo.com { 681 | server 127.0.0.1:12354; 682 | server 127.0.0.1:12355; 683 | server 127.0.0.1:12356 backup; 684 | } 685 | 686 | server { 687 | listen 12354; 688 | location = /status { 689 | return 200; 690 | } 691 | } 692 | 693 | server { 694 | listen 12355; 695 | location = /status { 696 | return 404; 697 | } 698 | } 699 | 700 | server { 701 | listen 12356; 702 | location = /status { 703 | return 503; 704 | } 705 | } 706 | 707 | lua_shared_dict healthcheck 1m; 708 | init_worker_by_lua ' 709 | ngx.config.debug = 1 710 | local dict = ngx.shared.healthcheck 711 | dict:flush_all() 712 | assert(dict:set("v:foo.com", 1)) 713 | assert(dict:set("d:foo.com:b0", true)) 714 | assert(dict:set("d:foo.com:p1", true)) 715 | local hc = require "resty.upstream.healthcheck" 716 | local ok, err = hc.spawn_checker{ 717 | shm = "healthcheck", 718 | upstream = "foo.com", 719 | type = "http", 720 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 721 | interval = 100, -- 100ms 722 | fall = 2, 723 | } 724 | if not ok then 725 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 726 | return 727 | end 728 | '; 729 | } 730 | --- config 731 | location = /t { 732 | access_log off; 733 | content_by_lua ' 734 | ngx.sleep(0.52) 735 | 736 | local hc = require "resty.upstream.healthcheck" 737 | ngx.print(hc.status_page()) 738 | 739 | for i = 1, 2 do 740 | local res = ngx.location.capture("/proxy") 741 | ngx.say("upstream addr: ", res.header["X-Foo"]) 742 | end 743 | '; 744 | } 745 | 746 | location = /proxy { 747 | proxy_pass http://foo.com/; 748 | header_filter_by_lua ' 749 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 750 | '; 751 | } 752 | --- request 753 | GET /t 754 | 755 | --- response_body 756 | Upstream foo.com 757 | Primary Peers 758 | 127.0.0.1:12354 UP 759 | 127.0.0.1:12355 UP 760 | Backup Peers 761 | 127.0.0.1:12356 UP 762 | upstream addr: 127.0.0.1:12354 763 | upstream addr: 127.0.0.1:12355 764 | 765 | --- no_error_log 766 | [error] 767 | [alert] 768 | was checked to be not ok 769 | failed to run healthcheck cycle 770 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|publishing peers version \d+|warn\(\): .*(?=,)|upgrading peers version to \d+/ 771 | --- grep_error_log_out eval 772 | qr/^upgrading peers version to 1 773 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 774 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 775 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 776 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 777 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 778 | warn\(\): healthcheck: peer 127\.0\.0\.1:12355 is turned up after 2 success\(es\) 779 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 780 | warn\(\): healthcheck: peer 127\.0\.0\.1:12356 is turned up after 2 success\(es\) 781 | publishing peers version 2 782 | (?:healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 783 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 784 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 785 | ){2,4}$/ 786 | 787 | 788 | 789 | === TEST 8: peers version upgrade (make down peers up) 790 | --- http_config eval 791 | "$::HttpConfig" 792 | . q{ 793 | upstream foo.com { 794 | server 127.0.0.1:12354 down; 795 | server 127.0.0.1:12355; 796 | server 127.0.0.1:12356 backup; 797 | } 798 | 799 | server { 800 | listen 12355; 801 | location = /status { 802 | return 404; 803 | } 804 | } 805 | 806 | server { 807 | listen 12356; 808 | location = /status { 809 | return 503; 810 | } 811 | } 812 | 813 | lua_shared_dict healthcheck 1m; 814 | init_worker_by_lua ' 815 | ngx.config.debug = 1 816 | local dict = ngx.shared.healthcheck 817 | dict:flush_all() 818 | assert(dict:set("v:foo.com", 1)) 819 | -- assert(dict:set("d:foo.com:b0", true)) 820 | -- assert(dict:set("d:foo.com:p1", true)) 821 | local hc = require "resty.upstream.healthcheck" 822 | local ok, err = hc.spawn_checker{ 823 | shm = "healthcheck", 824 | upstream = "foo.com", 825 | type = "http", 826 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 827 | interval = 100, -- 100ms 828 | fall = 2, 829 | } 830 | if not ok then 831 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 832 | return 833 | end 834 | '; 835 | } 836 | --- config 837 | location = /t { 838 | access_log off; 839 | content_by_lua ' 840 | ngx.sleep(0.52) 841 | 842 | local hc = require "resty.upstream.healthcheck" 843 | ngx.print(hc.status_page()) 844 | 845 | for i = 1, 2 do 846 | local res = ngx.location.capture("/proxy") 847 | ngx.say("upstream addr: ", res.header["X-Foo"]) 848 | end 849 | '; 850 | } 851 | 852 | location = /proxy { 853 | proxy_pass http://foo.com/; 854 | header_filter_by_lua ' 855 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 856 | '; 857 | } 858 | --- request 859 | GET /t 860 | 861 | --- response_body 862 | Upstream foo.com 863 | Primary Peers 864 | 127.0.0.1:12354 DOWN 865 | 127.0.0.1:12355 UP 866 | Backup Peers 867 | 127.0.0.1:12356 UP 868 | upstream addr: 127.0.0.1:12355 869 | upstream addr: 127.0.0.1:12355 870 | 871 | --- error_log 872 | failed to connect to 127.0.0.1:12354: connection refused 873 | --- no_error_log 874 | [alert] 875 | failed to run healthcheck cycle 876 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|publishing peers version \d+|warn\(\): .*(?=,)|upgrading peers version to \d+/ 877 | --- grep_error_log_out eval 878 | qr/^upgrading peers version to 1 879 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be not ok 880 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 881 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 882 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be not ok 883 | warn\(\): healthcheck: peer 127\.0\.0\.1:12354 is turned down after 2 failure\(s\) 884 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 885 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 886 | publishing peers version 2 887 | (?:healthcheck: peer 127\.0\.0\.1:12354 was checked to be not ok 888 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 889 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be ok 890 | ){3,5}$/ 891 | --- timeout: 6 892 | 893 | 894 | 895 | === TEST 9: concurrency == 2 (odd number of peers) 896 | --- http_config eval 897 | "$::HttpConfig" 898 | . q{ 899 | upstream foo.com { 900 | server 127.0.0.1:12354; 901 | server 127.0.0.1:12355; 902 | server 127.0.0.1:12356; 903 | server 127.0.0.1:12357; 904 | server 127.0.0.1:12358; 905 | server 127.0.0.1:12359 backup; 906 | } 907 | 908 | lua_shared_dict healthcheck 1m; 909 | init_worker_by_lua ' 910 | ngx.config.debug = 1 911 | ngx.shared.healthcheck:flush_all() 912 | local hc = require "resty.upstream.healthcheck" 913 | local ok, err = hc.spawn_checker{ 914 | shm = "healthcheck", 915 | upstream = "foo.com", 916 | type = "http", 917 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 918 | interval = 100, -- 100ms 919 | fall = 2, 920 | concurrency = 2, 921 | } 922 | if not ok then 923 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 924 | return 925 | end 926 | '; 927 | } 928 | --- config 929 | location = /t { 930 | access_log off; 931 | content_by_lua ' 932 | ngx.sleep(0.52) 933 | ngx.say("ok") 934 | '; 935 | } 936 | --- request 937 | GET /t 938 | 939 | --- response_body 940 | ok 941 | --- no_error_log 942 | [alert] 943 | failed to run healthcheck cycle 944 | --- error_log 945 | healthcheck: peer 127.0.0.1:12354 is turned down after 2 failure(s) 946 | healthcheck: peer 127.0.0.1:12355 is turned down after 2 failure(s) 947 | healthcheck: peer 127.0.0.1:12356 is turned down after 2 failure(s) 948 | healthcheck: peer 127.0.0.1:12357 is turned down after 2 failure(s) 949 | healthcheck: peer 127.0.0.1:12358 is turned down after 2 failure(s) 950 | healthcheck: peer 127.0.0.1:12359 is turned down after 2 failure(s) 951 | --- grep_error_log eval: qr/spawn a thread checking .* peer.*|check .*? peer.*/ 952 | --- grep_error_log_out eval 953 | qr/^(?:spawn a thread checking primary peers 0 to 2 954 | check primary peers 3 to 4 955 | check backup peer 0 956 | ){4,6}$/ 957 | 958 | 959 | 960 | === TEST 10: concurrency == 3 (odd number of peers) 961 | --- http_config eval 962 | "$::HttpConfig" 963 | . q{ 964 | upstream foo.com { 965 | server 127.0.0.1:12354; 966 | server 127.0.0.1:12355; 967 | server 127.0.0.1:12356; 968 | server 127.0.0.1:12359 backup; 969 | } 970 | 971 | lua_shared_dict healthcheck 1m; 972 | init_worker_by_lua ' 973 | ngx.config.debug = 1 974 | ngx.shared.healthcheck:flush_all() 975 | local hc = require "resty.upstream.healthcheck" 976 | local ok, err = hc.spawn_checker{ 977 | shm = "healthcheck", 978 | upstream = "foo.com", 979 | type = "http", 980 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 981 | interval = 100, -- 100ms 982 | fall = 2, 983 | concurrency = 3, 984 | } 985 | if not ok then 986 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 987 | return 988 | end 989 | '; 990 | } 991 | --- config 992 | location = /t { 993 | access_log off; 994 | content_by_lua ' 995 | ngx.sleep(0.52) 996 | ngx.say("ok") 997 | '; 998 | } 999 | --- request 1000 | GET /t 1001 | 1002 | --- response_body 1003 | ok 1004 | --- no_error_log 1005 | [alert] 1006 | failed to run healthcheck cycle 1007 | --- error_log 1008 | healthcheck: peer 127.0.0.1:12354 is turned down after 2 failure(s) 1009 | healthcheck: peer 127.0.0.1:12355 is turned down after 2 failure(s) 1010 | healthcheck: peer 127.0.0.1:12356 is turned down after 2 failure(s) 1011 | healthcheck: peer 127.0.0.1:12359 is turned down after 2 failure(s) 1012 | --- grep_error_log eval: qr/spawn a thread checking .* peer.*|check .*? peer.*/ 1013 | --- grep_error_log_out eval 1014 | qr/^(?:spawn a thread checking primary peer 0 1015 | spawn a thread checking primary peer 1 1016 | check primary peer 2 1017 | check backup peer 0 1018 | ){4,6}$/ 1019 | 1020 | 1021 | 1022 | === TEST 11: health check (good case), status ignored by default 1023 | --- http_config eval 1024 | "$::HttpConfig" 1025 | . q{ 1026 | upstream foo.com { 1027 | server 127.0.0.1:7983; 1028 | } 1029 | 1030 | server { 1031 | listen 12356; 1032 | location = /status { 1033 | return 503; 1034 | } 1035 | } 1036 | 1037 | lua_shared_dict healthcheck 1m; 1038 | init_worker_by_lua ' 1039 | ngx.config.debug = 1 1040 | ngx.shared.healthcheck:flush_all() 1041 | local hc = require "resty.upstream.healthcheck" 1042 | local ok, err = hc.spawn_checker{ 1043 | shm = "healthcheck", 1044 | upstream = "foo.com", 1045 | type = "http", 1046 | http_req = "GET /status HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 1047 | interval = 100, -- 100ms 1048 | fall = 1, 1049 | valid_statuses = {200}, 1050 | } 1051 | if not ok then 1052 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 1053 | return 1054 | end 1055 | '; 1056 | } 1057 | --- config 1058 | location = /t { 1059 | access_log off; 1060 | content_by_lua ' 1061 | ngx.sleep(0.12) 1062 | 1063 | local hc = require "resty.upstream.healthcheck" 1064 | ngx.print(hc.status_page()) 1065 | '; 1066 | } 1067 | 1068 | location = /proxy { 1069 | proxy_pass http://foo.com/; 1070 | header_filter_by_lua ' 1071 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 1072 | '; 1073 | } 1074 | --- request 1075 | GET /t 1076 | 1077 | --- tcp_listen: 7983 1078 | --- tcp_query eval: "GET /status HTTP/1.0\r\nHost: localhost\r\n\r\n" 1079 | --- tcp_reply 1080 | 1081 | --- response_body 1082 | Upstream foo.com 1083 | Primary Peers 1084 | 127.0.0.1:7983 DOWN 1085 | Backup Peers 1086 | 1087 | --- no_error_log 1088 | [alert] 1089 | bad argument #2 to 'sub' (number expected, got nil) 1090 | 1091 | 1092 | 1093 | === TEST 12: health check (bad case), bad status, multiple upstreams 1094 | --- http_config eval 1095 | "$::HttpConfig" 1096 | . q{ 1097 | upstream foo.com { 1098 | server 127.0.0.1:12354; 1099 | server 127.0.0.1:12355; 1100 | server 127.0.0.1:12356 backup; 1101 | } 1102 | 1103 | upstream bar.com { 1104 | server 127.0.0.1:12354; 1105 | server 127.0.0.1:12355; 1106 | server 127.0.0.1:12357; 1107 | server 127.0.0.1:12356 backup; 1108 | } 1109 | 1110 | server { 1111 | listen 12354; 1112 | location = /status { 1113 | return 200; 1114 | } 1115 | } 1116 | 1117 | server { 1118 | listen 12355; 1119 | listen 12357; 1120 | location = /status { 1121 | return 404; 1122 | } 1123 | } 1124 | 1125 | server { 1126 | listen 12356; 1127 | location = /status { 1128 | return 503; 1129 | } 1130 | } 1131 | 1132 | lua_shared_dict healthcheck 2m; 1133 | init_worker_by_lua_block { 1134 | ngx.config.debug = 1 1135 | ngx.shared.healthcheck:flush_all() 1136 | local hc = require "resty.upstream.healthcheck" 1137 | for i, upstream in ipairs{'foo.com', 'bar.com'} do 1138 | local ok, err = hc.spawn_checker{ 1139 | shm = "healthcheck", 1140 | upstream = upstream, 1141 | type = "http", 1142 | http_req = "GET /status HTTP/1.0\r\nHost: localhost\r\n\r\n", 1143 | interval = 50, -- ms 1144 | fall = 1, 1145 | valid_statuses = {200, 503}, 1146 | } 1147 | if not ok then 1148 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 1149 | return 1150 | end 1151 | end 1152 | } 1153 | 1154 | } 1155 | --- config 1156 | location = /t { 1157 | access_log off; 1158 | content_by_lua ' 1159 | ngx.sleep(0.52) 1160 | 1161 | local hc = require "resty.upstream.healthcheck" 1162 | ngx.print(hc.status_page()) 1163 | 1164 | for i = 1, 2 do 1165 | local res = ngx.location.capture("/proxy") 1166 | ngx.say("upstream addr: ", res.header["X-Foo"]) 1167 | end 1168 | '; 1169 | } 1170 | 1171 | location = /proxy { 1172 | proxy_pass http://foo.com/; 1173 | header_filter_by_lua ' 1174 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 1175 | '; 1176 | } 1177 | --- request 1178 | GET /t 1179 | 1180 | --- response_body 1181 | Upstream foo.com 1182 | Primary Peers 1183 | 127.0.0.1:12354 UP 1184 | 127.0.0.1:12355 DOWN 1185 | Backup Peers 1186 | 127.0.0.1:12356 UP 1187 | 1188 | Upstream bar.com 1189 | Primary Peers 1190 | 127.0.0.1:12354 UP 1191 | 127.0.0.1:12355 DOWN 1192 | 127.0.0.1:12357 DOWN 1193 | Backup Peers 1194 | 127.0.0.1:12356 UP 1195 | upstream addr: 127.0.0.1:12354 1196 | upstream addr: 127.0.0.1:12354 1197 | --- no_error_log 1198 | [alert] 1199 | failed to run healthcheck cycle 1200 | --- error_log 1201 | healthcheck: bad status code from 127.0.0.1:12355 1202 | healthcheck: bad status code from 127.0.0.1:12357 1203 | --- timeout: 6 1204 | 1205 | 1206 | 1207 | === TEST 13: crashes in init_by_lua_worker* 1208 | --- http_config eval 1209 | "$::HttpConfig" 1210 | . q{ 1211 | upstream foo.com { 1212 | server 127.0.0.1:12354; 1213 | server 127.0.0.1:12355; 1214 | server 127.0.0.1:12356 backup; 1215 | } 1216 | 1217 | server { 1218 | listen 12354; 1219 | location = /status { 1220 | return 200; 1221 | } 1222 | } 1223 | 1224 | server { 1225 | listen 12355; 1226 | location = /status { 1227 | return 404; 1228 | } 1229 | } 1230 | 1231 | server { 1232 | listen 12356; 1233 | location = /status { 1234 | return 503; 1235 | } 1236 | } 1237 | 1238 | lua_shared_dict healthcheck 1m; 1239 | init_worker_by_lua_block { 1240 | error("bad thing!") 1241 | } 1242 | } 1243 | --- config 1244 | location = /t { 1245 | access_log off; 1246 | content_by_lua ' 1247 | ngx.sleep(0.52) 1248 | 1249 | local hc = require "resty.upstream.healthcheck" 1250 | ngx.print(hc.status_page()) 1251 | 1252 | for i = 1, 2 do 1253 | local res = ngx.location.capture("/proxy") 1254 | ngx.say("upstream addr: ", res.header["X-Foo"]) 1255 | end 1256 | '; 1257 | } 1258 | 1259 | location = /proxy { 1260 | proxy_pass http://foo.com/; 1261 | header_filter_by_lua ' 1262 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 1263 | '; 1264 | } 1265 | --- request 1266 | GET /t 1267 | 1268 | --- response_body 1269 | Upstream foo.com (NO checkers) 1270 | Primary Peers 1271 | 127.0.0.1:12354 UP 1272 | 127.0.0.1:12355 UP 1273 | Backup Peers 1274 | 127.0.0.1:12356 UP 1275 | upstream addr: 127.0.0.1:12354 1276 | upstream addr: 127.0.0.1:12355 1277 | --- no_error_log 1278 | [alert] 1279 | failed to run healthcheck cycle 1280 | --- error_log 1281 | bad thing! 1282 | --- timeout: 4 1283 | 1284 | 1285 | 1286 | === TEST 14: health check with ipv6 backend (good case), status ignored by default 1287 | --- http_config eval 1288 | "$::HttpConfig" 1289 | . q{ 1290 | upstream foo.com { 1291 | server 127.0.0.1:12354; 1292 | server [::1]:12355; 1293 | server [0:0::1]:12356 backup; 1294 | } 1295 | 1296 | server { 1297 | listen 12354; 1298 | location = /status { 1299 | return 200; 1300 | } 1301 | } 1302 | 1303 | server { 1304 | listen [::1]:12355; 1305 | location = /status { 1306 | return 404; 1307 | } 1308 | } 1309 | 1310 | server { 1311 | listen [0:0::1]:12356; 1312 | location = /status { 1313 | return 503; 1314 | } 1315 | } 1316 | 1317 | lua_shared_dict healthcheck 1m; 1318 | init_worker_by_lua_block { 1319 | ngx.config.debug = 1 1320 | ngx.shared.healthcheck:flush_all() 1321 | local hc = require "resty.upstream.healthcheck" 1322 | local ok, err = hc.spawn_checker{ 1323 | shm = "healthcheck", 1324 | upstream = "foo.com", 1325 | type = "http", 1326 | http_req = "GET /status HTTP/1.0\r\nHost: localhost\r\n\r\n", 1327 | interval = 100, -- 100ms 1328 | fall = 2, 1329 | } 1330 | if not ok then 1331 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 1332 | return 1333 | end 1334 | } 1335 | } 1336 | --- config 1337 | location = /t { 1338 | access_log off; 1339 | content_by_lua_block { 1340 | ngx.sleep(0.52) 1341 | 1342 | local hc = require "resty.upstream.healthcheck" 1343 | ngx.print(hc.status_page()) 1344 | 1345 | for i = 1, 2 do 1346 | local res = ngx.location.capture("/proxy") 1347 | ngx.say("upstream addr: ", res.header["X-Foo"]) 1348 | end 1349 | } 1350 | } 1351 | 1352 | location = /proxy { 1353 | proxy_pass http://foo.com/; 1354 | header_filter_by_lua_block { 1355 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 1356 | } 1357 | } 1358 | --- request 1359 | GET /t 1360 | 1361 | --- response_body 1362 | Upstream foo.com 1363 | Primary Peers 1364 | 127.0.0.1:12354 UP 1365 | [::1]:12355 UP 1366 | Backup Peers 1367 | [::1]:12356 UP 1368 | upstream addr: 127.0.0.1:12354 1369 | upstream addr: [::1]:12355 1370 | 1371 | --- no_error_log 1372 | [error] 1373 | [alert] 1374 | [warn] 1375 | was checked to be not ok 1376 | failed to run healthcheck cycle 1377 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|publishing peers version \d+|upgrading peers version to \d+/ 1378 | --- grep_error_log_out eval 1379 | qr/^healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 1380 | healthcheck: peer \[::1\]:12355 was checked to be ok 1381 | healthcheck: peer \[::1\]:12356 was checked to be ok 1382 | (?:healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 1383 | healthcheck: peer \[::1\]:12355 was checked to be ok 1384 | healthcheck: peer \[::1\]:12356 was checked to be ok 1385 | ){3,7}$/ 1386 | --- wait: 0.2 1387 | --- timeout: 6 1388 | --- skip_eval: 8: system("ping6 -c 1 ::1 >/dev/null 2>&1") ne 0 1389 | 1390 | 1391 | 1392 | === TEST 15: peers > concurrency 1393 | --- http_config eval 1394 | "$::HttpConfig" 1395 | . q{ 1396 | upstream foo.com { 1397 | server 127.0.0.1:12354; 1398 | server 127.0.0.1:12355; 1399 | server 127.0.0.1:12356; 1400 | server 127.0.0.1:12357; 1401 | server 127.0.0.1:12358; 1402 | server 127.0.0.1:12359 backup; 1403 | } 1404 | 1405 | server { 1406 | listen 12354; 1407 | 1408 | location /status { 1409 | content_by_lua_block { 1410 | ngx.sleep(0.2) 1411 | } 1412 | } 1413 | } 1414 | 1415 | lua_shared_dict healthcheck 1m; 1416 | } 1417 | --- config 1418 | location = /t { 1419 | content_by_lua_block { 1420 | ngx.config.debug = 1 1421 | local hc = require "resty.upstream.healthcheck" 1422 | 1423 | local ok, err = hc.spawn_checker{ 1424 | shm = "healthcheck", 1425 | upstream = "foo.com", 1426 | type = "http", 1427 | http_req = "GET /status HTTP/1.0\r\nHost: localhost\r\n\r\n", 1428 | timeout = 100, 1429 | fall = 1, 1430 | concurrency = 2, 1431 | no_timer = true, 1432 | } 1433 | if not ok then 1434 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 1435 | return 1436 | end 1437 | 1438 | ngx.print(hc.status_page()) 1439 | } 1440 | } 1441 | --- request 1442 | GET /t 1443 | --- response_body_like 1444 | Upstream foo.com 1445 | Primary Peers 1446 | 127.0.0.1:12354 DOWN 1447 | 127.0.0.1:12355 \S+ 1448 | 127.0.0.1:12356 \S+ 1449 | 127.0.0.1:12357 \S+ 1450 | 127.0.0.1:12358 \S+ 1451 | Backup Peers 1452 | 127.0.0.1:12359 \S+ 1453 | --- no_error_log 1454 | [alert] 1455 | [emerg] 1456 | failed to run healthcheck cycle 1457 | --- wait: 0.3 1458 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|healthcheck: failed to receive status line from .*?:\d+/ 1459 | --- grep_error_log_out eval 1460 | qr/^healthcheck: peer 127\.0\.0\.1:12357 was checked to be not ok 1461 | healthcheck: peer 127\.0\.0\.1:12358 was checked to be not ok 1462 | healthcheck: failed to receive status line from 127\.0\.0\.1:12354 1463 | healthcheck: peer 127\.0\.0\.1:12354 was checked to be not ok 1464 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be not ok 1465 | healthcheck: peer 127\.0\.0\.1:12356 was checked to be not ok 1466 | healthcheck: peer 127\.0\.0\.1:12359 was checked to be not ok 1467 | $/ 1468 | 1469 | === TEST 16: health check using different port 1470 | --- http_config eval 1471 | "$::HttpConfig" 1472 | . q{ 1473 | upstream foo.com { 1474 | server 127.0.0.1:12354; 1475 | server 127.0.0.1:12355; 1476 | } 1477 | server { 1478 | listen 12354; 1479 | location = /status { 1480 | return 200; 1481 | } 1482 | } 1483 | server { 1484 | listen 12355; 1485 | location = /status { 1486 | return 404; 1487 | } 1488 | } 1489 | server { 1490 | listen 12356; 1491 | location = /healthz { 1492 | return 200; 1493 | } 1494 | } 1495 | lua_shared_dict healthcheck 1m; 1496 | init_worker_by_lua ' 1497 | ngx.config.debug = 1 1498 | ngx.shared.healthcheck:flush_all() 1499 | local hc = require "resty.upstream.healthcheck" 1500 | local ok, err = hc.spawn_checker{ 1501 | shm = "healthcheck", 1502 | upstream = "foo.com", 1503 | type = "http", 1504 | http_req = "GET /healthz HTTP/1.0\\\\r\\\\nHost: localhost\\\\r\\\\n\\\\r\\\\n", 1505 | interval = 100, -- 100ms 1506 | fall = 2, 1507 | port = 12356, 1508 | } 1509 | if not ok then 1510 | ngx.log(ngx.ERR, "failed to spawn health checker: ", err) 1511 | return 1512 | end 1513 | '; 1514 | } 1515 | --- config 1516 | location = /t { 1517 | access_log off; 1518 | content_by_lua ' 1519 | ngx.sleep(0.52) 1520 | local hc = require "resty.upstream.healthcheck" 1521 | ngx.print(hc.status_page()) 1522 | for i = 1, 2 do 1523 | local res = ngx.location.capture("/proxy") 1524 | ngx.say("upstream addr: ", res.header["X-Foo"]) 1525 | end 1526 | '; 1527 | } 1528 | location = /proxy { 1529 | proxy_pass http://foo.com/; 1530 | header_filter_by_lua ' 1531 | ngx.header["X-Foo"] = ngx.var.upstream_addr; 1532 | '; 1533 | } 1534 | --- request 1535 | GET /t 1536 | --- response_body 1537 | Upstream foo.com 1538 | Primary Peers 1539 | 127.0.0.1:12354 UP 1540 | 127.0.0.1:12355 UP 1541 | Backup Peers 1542 | upstream addr: 127.0.0.1:12354 1543 | upstream addr: 127.0.0.1:12355 1544 | --- no_error_log 1545 | [error] 1546 | [alert] 1547 | [warn] 1548 | was checked to be not ok 1549 | failed to run healthcheck cycle 1550 | --- grep_error_log eval: qr/healthcheck: .*? was checked .*|publishing peers version \d+|upgrading peers version to \d+/ 1551 | --- grep_error_log_out eval 1552 | qr/^healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 1553 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 1554 | (?:healthcheck: peer 127\.0\.0\.1:12354 was checked to be ok 1555 | healthcheck: peer 127\.0\.0\.1:12355 was checked to be ok 1556 | ){3,5}$/ 1557 | --- timeout: 6 1558 | -------------------------------------------------------------------------------- /valgrind.suppress: -------------------------------------------------------------------------------- 1 | { 2 | 3 | Memcheck:Leak 4 | fun:malloc 5 | fun:ngx_alloc 6 | fun:ngx_event_process_init 7 | } 8 | { 9 | 10 | Memcheck:Param 11 | epoll_ctl(event) 12 | fun:epoll_ctl 13 | fun:ngx_epoll_add_event 14 | } 15 | { 16 | 17 | Memcheck:Param 18 | epoll_ctl(event) 19 | fun:epoll_ctl 20 | fun:ngx_epoll_add_connection 21 | } 22 | { 23 | 24 | Memcheck:Addr4 25 | fun:ngx_init_cycle 26 | fun:ngx_master_process_cycle 27 | fun:main 28 | } 29 | { 30 | 31 | Memcheck:Cond 32 | fun:ngx_init_cycle 33 | fun:ngx_master_process_cycle 34 | fun:main 35 | } 36 | { 37 | 38 | Memcheck:Cond 39 | fun:index 40 | fun:expand_dynamic_string_token 41 | fun:_dl_map_object 42 | fun:map_doit 43 | fun:_dl_catch_error 44 | fun:do_preload 45 | fun:dl_main 46 | fun:_dl_sysdep_start 47 | fun:_dl_start 48 | } 49 | { 50 | 51 | Memcheck:Param 52 | epoll_ctl(event) 53 | fun:epoll_ctl 54 | fun:ngx_epoll_init 55 | fun:ngx_event_process_init 56 | } 57 | { 58 | 59 | Memcheck:Param 60 | epoll_ctl(event) 61 | fun:epoll_ctl 62 | fun:ngx_epoll_notify_init 63 | fun:ngx_epoll_init 64 | fun:ngx_event_process_init 65 | } 66 | { 67 | 68 | Memcheck:Param 69 | epoll_ctl(event) 70 | fun:epoll_ctl 71 | fun:ngx_epoll_test_rdhup 72 | } 73 | { 74 | 75 | Memcheck:Leak 76 | match-leak-kinds: definite 77 | fun:malloc 78 | fun:ngx_alloc 79 | fun:ngx_set_environment 80 | fun:ngx_single_process_cycle 81 | } 82 | { 83 | 84 | Memcheck:Leak 85 | match-leak-kinds: definite 86 | fun:malloc 87 | fun:ngx_alloc 88 | fun:ngx_set_environment 89 | fun:ngx_worker_process_init 90 | fun:ngx_worker_process_cycle 91 | } 92 | --------------------------------------------------------------------------------