├── LICENSE ├── README ├── config ├── nginx-1.10.1.patch ├── nginx-1.12.1-stream.patch ├── nginx-1.12.1.patch ├── nginx-1.4.4.patch ├── nginx-1.8.1.patch ├── nginx.patch ├── ngx_http_limit_upstream_module.c └── ngx_stream_limit_upstream_module.c /LICENSE: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2012-2016 Chen Chuanwen 3 | * Copyright (C) 2012-2016 cfsego 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions 8 | * are met: 9 | * 1. Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 | * SUCH DAMAGE. 26 | */ 27 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Limit upstream module for nginx 2 | 3 | =OVERVIEW=============================================================== 4 | 5 | When you use nginx as a proxy, there is a problem you may have to face: 6 | a upstream server is able to achieve high QPS within a relatively low resource 7 | stress. For example, mysql works well at the speed of 8000-9000 QPS within 60 parallel 8 | connections, but only 2000 QPS when the number of connections increases to 500. 9 | 10 | I do not find any module that will solve the problem because they all face client, 11 | not the upstream. I tried the limit_req, but failed. when I write down "limit_req rate=8000/s", 12 | I find there are over 2000 active connections between nginx and mysql, 13 | and when I write down "limit_req rate=60/s", Jesus, it is really 60 QPS for my nginx. 14 | 15 | So, I have a try, and the limit_upstream module comes. This module will limit the 16 | number of connections to a upstream server identified by its ip address. With it, I 17 | could control the upstream connections. When it reaches the set threshold, new requests 18 | will suspend and wait until a active upstream request finishes and wake them up, 19 | otherwise, they will die when they get timeout. Of course, I can also specify the length 20 | of waiting queue, and the waiting timeout, as limit_req do. 21 | 22 | In this module, each worker suspend its requests when upstreams' counters reach the 23 | threshold, and resume them or cancel them all by itself. The only things they share 24 | are the global upstreams' counters. A special case for a request be allowed to go to 25 | upstream even when the threshold is reached is that, no request related to that upstream 26 | is being processed at that time for the worker. as a result, the maximum number of 27 | established connections to a upstream server is ('limit' + 'worker_processes'). 28 | 29 | =MANUAL================================================================= 30 | 31 | Example: 32 | 33 | stream { 34 | server { 35 | proxy_pass spool; 36 | } 37 | 38 | limit_upstream_zone stest 10m; 39 | 40 | upstream spool { 41 | server 10.232.36.98:3112; 42 | ... 43 | 44 | limit_upstream_conn limit=260 zone=stest backlog=10000 timeout=180s; 45 | limit_upstream_log_level error; 46 | } 47 | } 48 | 49 | http { 50 | server { 51 | location =/test { 52 | proxy_pass http://pool; 53 | } 54 | } 55 | 56 | limit_upstream_zone test 10m; 57 | 58 | upstream pool { 59 | server 10.232.36.98:3111; 60 | ... 61 | 62 | limit_upstream_conn limit=260 zone=test backlog=10000 timeout=180s; 63 | limit_upstream_log_level error; 64 | } 65 | } 66 | 67 | syntax: limit_upstream_zone zone_name size; 68 | default: - 69 | context: http, stream 70 | 71 | Define a shared memory pool for the module. 72 | 73 | syntax: limit_upstream_conn zone=zone_name limit=limit [backlog=length] [timeout=timeout | nodelay] [instant_hook]; 74 | default: - 75 | context: upstream 76 | 77 | Set the zone used, the maximum allowed for each server of the upstream, 78 | the backlog length, and the timeout for each suspended request. 79 | The default for 'backlog' is 1000. Moreover, the default for 'timeout' is 80 | the same as the reading timeout set by proxy. For example, when using fastcgi, 81 | it is the same as the value set by 'fastcgi_read_timeout' or the default 82 | value of 'fastcgi_read_timeout' when not set. 83 | 84 | When backlog for a server is full, it may use other server, Otherwise, 85 | it wait in the queue. Timeout will cause 408 return status code. 86 | 87 | Parameter 'instant_hook' shouild be used when you are with a dynamic upstream 88 | updating module, eg. ngx_dyn_ups_module. What about with Nginx Plus? I didn't test that. 89 | For more details, you can refer to https://github.com/cfsego/nginx-limit-upstream/issues/13 90 | 91 | syntax: limit_upstream_log_level [ error | warn | notice | info ]; 92 | default: limit_upstream_log_level notice; 93 | context: http, stream, upstream 94 | 95 | Define the log level of ordinary logs for the module. 96 | 97 | =INSTALL=============================================================== 98 | 99 | patch -p2 < nginx.patch (nginx 1.0.X, 1.2.X) 100 | or 101 | patch -p1 < nginx-1.4.4.patch (nginx 1.4.X, nginx 1.6.X) 102 | or 103 | patch -p0 < nginx-1.8.1.patch (nginx 1.8.X, nginx 1.9.X) 104 | or 105 | patch -p0 < nginx-1.10.1.patch (nginx 1.10.X) 106 | or 107 | cp /path/to/module/nginx.1.10.1.patch /path/to/nginx-1.10.0/debian/patches (nginx 1.10 on debian) 108 | or 109 | patch -p1 < nginx-1.12.1.patch (nginx 1.12.X) 110 | 111 | if you want to use the module for stream, use this patch in addition: 112 | patch -p1 < nginx-1.12.1-stream.patch 113 | 114 | ./configure --add-module=/path/to/module 115 | make 116 | make install 117 | 118 | =PATCH================================================================= 119 | 120 | The nginx.patch is based on nginx 1.0.11, and it is compatible with nginx 1.2.0. 121 | However, I did not test it on the other versions :) 122 | The nginx.1.4.4.patch is based on nginx 1.4.4, and tested only under nginx 1.4.4. 123 | And there is patches for nginx.1.8.1 and 1.10.1. 124 | Thanks to @ivan-antonyuck, who made patch for nginx 1.10.X and 1.10 on debian. 125 | 126 | =BUGS================================================================== 127 | 128 | * bugfixed 2018.2.6 129 | 1. Some logs do not follow the directive 'limit_upstream_log_level', are always WARN. 130 | 2. There are some warnings in compilation. 131 | 132 | * bugfixed 2016.6.13 133 | The requests are choked when proxying https to backend. The cause is it transfers 134 | invalid data to set_session(). 135 | 136 | * bugfixed 2016.6.1 137 | The rbtree node are always deleted after reload, because snode->last is the timestamp 138 | used for removing useless node, but has never been set a value. This will cause the 139 | module to allow any request to pass after reload. 140 | 141 | * bugfixed 2014.6.5 142 | Local variable 'pc' is only used for printing debug infomation, so there is a 143 | warning when compiling nginx without the configuration '--with-debug'. 144 | 145 | * bugfixed 2013.11.11 146 | The worker crushes after it processes a request. The problem occurs under 147 | nginx 1.4.3, because nginx sets u->sockaddr to NULL before my module uses it. 148 | 149 | * bugfixed 2012.7.9 150 | There is a critical bug that may make upstreams stuck. The description is 151 | if a request uses a server, and the server fails, then the request tries another 152 | server with the help of ngx_http_upstream_next, however, the counter of the 153 | former server is not decreased, then the whole system fails to work after a number 154 | of such fails. 155 | 156 | =CHANGES=============================================================== 157 | 158 | 1.3.0 [2018.4.17] now we can use this module with stream. 159 | 160 | 1.2.3 [2018.2.6] change the default of 'timeout' to the same as the reading timeout 161 | set by proxy. 162 | 163 | 1.2.2 [2016.7.13] now can work with ngx_dyn_ups_module. 164 | 165 | 1.2.1 [2013.6.5] fixed a bug variable 'pc' is unused. 166 | 167 | 1.2.0 [2013.11.11] fixed a bug occured under nginx 1.4.4, new patch for nginx 1.4.4 168 | 169 | 1.1.0 [2012.7.17] bugfixed, and optimize for efficiency, 10% faster. 170 | 171 | 1.0.0 [2012.5.31] initial version, including some bug fixes. 172 | 173 | =USED WITH STREAM====================================================== 174 | 175 | On timeout, nginx will close the connection immediately, so the client may receive 176 | TCP datagram with RST flag. When the module drops a session, it returns NGX_DECLINED, so 177 | nginx will try another peer and mark current peer as failed, this may take effects on 178 | load balancer. If this cause any problem, let me know. 179 | 180 | =USED WITH UPSTREAM KEEPALIVE========================================== 181 | 182 | Once nginx upstream keepalive module returns NGX_DONE, it means there is already 183 | one setup connection, so my module let the request pass, no matter it overpasses the 184 | threshold or not. There is a problem with this case (when the number reaches, nginx 185 | trends to distribute upstream connections for newcomers among workers at the proportion 186 | of connections already setup, and it is unfair). I think set keepalive as formula below, 187 | keepalive = limit_conn / worker_processes 188 | may resolve the problem. 189 | 190 | =MISUSE================================================================ 191 | 192 | 1. someone uses it like this: 193 | 194 | limit_upstream_zone test 10m; 195 | 196 | upstream bb { 197 | server 127.0.0.1:3111; 198 | server 127.0.0.1:3112; 199 | 200 | limit_upstream_conn limit=8 zone=test backlog=2000 timeout=1500ms; 201 | } 202 | 203 | upstream cc { 204 | server 127.0.0.1:3111; 205 | server 127.0.0.1:3112; 206 | 207 | limit_upstream_conn limit=16 zone=test backlog=1000 timeout=500ms; 208 | } 209 | 210 | It will absolutely fail to work. The correct is: 211 | 212 | limit_upstream_zone test_high 10m; 213 | limit_upstream_zone test_low 10m; 214 | 215 | upstream bb { 216 | server 127.0.0.1:3111; 217 | server 127.0.0.1:3112; 218 | 219 | limit_upstream_conn limit=8 zone=test_low backlog=2000 timeout=1500ms; 220 | } 221 | 222 | upstream cc { 223 | server 127.0.0.1:3111; 224 | server 127.0.0.1:3112; 225 | 226 | limit_upstream_conn limit=16 zone=test_high backlog=1000 timeout=500ms; 227 | } 228 | -------------------------------------------------------------------------------- /config: -------------------------------------------------------------------------------- 1 | ngx_addon_name=ngx_limit_upstream 2 | 3 | ngx_module_name=ngx_http_limit_upstream_module 4 | ngx_module_deps= 5 | ngx_module_srcs=$ngx_addon_dir/ngx_http_limit_upstream_module.c 6 | ngx_module_libs= 7 | 8 | . auto/module 9 | 10 | if [ $STREAM != NO ]; then 11 | ngx_module_name=ngx_stream_limit_upstream_module 12 | ngx_module_deps= 13 | ngx_module_incs=src/stream 14 | ngx_module_srcs=$ngx_addon_dir/ngx_stream_limit_upstream_module.c 15 | ngx_module_libs= 16 | ngx_module_type=STREAM 17 | 18 | . auto/module 19 | fi 20 | -------------------------------------------------------------------------------- /nginx-1.10.1.patch: -------------------------------------------------------------------------------- 1 | --- src/http/ngx_http_upstream.h.1.10.1 2016-06-01 11:55:08.722172761 +0800 2 | +++ src/http/ngx_http_upstream.h 2016-06-01 11:55:37.870172018 +0800 3 | @@ -373,6 +373,7 @@ 4 | unsigned request_sent:1; 5 | unsigned request_body_sent:1; 6 | unsigned header_sent:1; 7 | + unsigned blocked:1; 8 | }; 9 | 10 | 11 | @@ -405,6 +406,8 @@ 12 | ngx_int_t ngx_http_upstream_hide_headers_hash(ngx_conf_t *cf, 13 | ngx_http_upstream_conf_t *conf, ngx_http_upstream_conf_t *prev, 14 | ngx_str_t *default_hide_headers, ngx_hash_init_t *hash); 15 | +void ngx_http_upstream_connect(ngx_http_request_t *r, 16 | + ngx_http_upstream_t *u); 17 | 18 | 19 | #define ngx_http_conf_upstream_srv_conf(uscf, module) \ 20 | --- src/http/ngx_http_upstream.c.1.10.1 2016-06-01 11:55:08.726172761 +0800 21 | +++ src/http/ngx_http_upstream.c 2016-06-01 11:56:49.238170197 +0800 22 | @@ -31,8 +31,6 @@ 23 | static void ngx_http_upstream_wr_check_broken_connection(ngx_http_request_t *r); 24 | static void ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, 25 | ngx_event_t *ev); 26 | -static void ngx_http_upstream_connect(ngx_http_request_t *r, 27 | - ngx_http_upstream_t *u); 28 | static ngx_int_t ngx_http_upstream_reinit(ngx_http_request_t *r, 29 | ngx_http_upstream_t *u); 30 | static void ngx_http_upstream_send_request(ngx_http_request_t *r, 31 | @@ -1325,35 +1323,39 @@ 32 | } 33 | 34 | 35 | -static void 36 | +void 37 | ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u) 38 | { 39 | ngx_int_t rc; 40 | ngx_connection_t *c; 41 | + if (!u->blocked) { 42 | + r->connection->log->action = "connecting to upstream"; 43 | 44 | - r->connection->log->action = "connecting to upstream"; 45 | - 46 | - if (u->state && u->state->response_time) { 47 | - u->state->response_time = ngx_current_msec - u->state->response_time; 48 | - } 49 | - 50 | - u->state = ngx_array_push(r->upstream_states); 51 | - if (u->state == NULL) { 52 | - ngx_http_upstream_finalize_request(r, u, 53 | - NGX_HTTP_INTERNAL_SERVER_ERROR); 54 | - return; 55 | - } 56 | + if (u->state && u->state->response_time) { 57 | + u->state->response_time = ngx_current_msec 58 | + - u->state->response_time; 59 | + } 60 | 61 | - ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t)); 62 | + u->state = ngx_array_push(r->upstream_states); 63 | + if (u->state == NULL) { 64 | + ngx_http_upstream_finalize_request(r, u, 65 | + NGX_HTTP_INTERNAL_SERVER_ERROR); 66 | + return; 67 | + } 68 | 69 | - u->state->response_time = ngx_current_msec; 70 | - u->state->connect_time = (ngx_msec_t) -1; 71 | - u->state->header_time = (ngx_msec_t) -1; 72 | + ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t)); 73 | 74 | + u->state->response_time = ngx_current_msec; 75 | + u->state->connect_time = (ngx_msec_t) -1; 76 | + u->state->header_time = (ngx_msec_t) -1; 77 | + } 78 | rc = ngx_event_connect_peer(&u->peer); 79 | 80 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 81 | "http upstream connect: %i", rc); 82 | + if (rc == NGX_BLOCK) { 83 | + return; 84 | + } 85 | 86 | if (rc == NGX_ERROR) { 87 | ngx_http_upstream_finalize_request(r, u, 88 | --- src/core/ngx_core.h.1.10.1 2016-06-01 11:55:08.726172761 +0800 89 | +++ src/core/ngx_core.h 2016-06-01 11:55:13.514172639 +0800 90 | @@ -40,7 +40,7 @@ 91 | #define NGX_DONE -4 92 | #define NGX_DECLINED -5 93 | #define NGX_ABORT -6 94 | - 95 | +#define NGX_BLOCK -7 96 | 97 | #include 98 | #include 99 | -------------------------------------------------------------------------------- /nginx-1.12.1-stream.patch: -------------------------------------------------------------------------------- 1 | --- a/src/stream/ngx_stream_upstream.h 2018-04-13 19:05:12.456091028 +0800 2 | +++ b/src/stream/ngx_stream_upstream.h 2018-04-13 20:07:45.604186242 +0800 3 | @@ -137,6 +137,7 @@ 4 | ngx_stream_upstream_state_t *state; 5 | unsigned connected:1; 6 | unsigned proxy_protocol:1; 7 | + unsigned blocked:1; 8 | } ngx_stream_upstream_t; 9 | 10 | 11 | --- a/src/stream/ngx_stream_proxy_module.c 2018-04-12 10:19:21.721098716 +0800 12 | +++ b/src/stream/ngx_stream_proxy_module.c 2018-04-17 14:58:23.554108490 +0800 13 | @@ -61,7 +61,7 @@ 14 | ngx_stream_proxy_srv_conf_t *pscf); 15 | static ngx_int_t ngx_stream_proxy_set_local(ngx_stream_session_t *s, 16 | ngx_stream_upstream_t *u, ngx_stream_upstream_local_t *local); 17 | -static void ngx_stream_proxy_connect(ngx_stream_session_t *s); 18 | +void ngx_stream_proxy_connect(ngx_stream_session_t *s); 19 | static void ngx_stream_proxy_init_upstream(ngx_stream_session_t *s); 20 | static void ngx_stream_proxy_resolve_handler(ngx_resolver_ctx_t *ctx); 21 | static void ngx_stream_proxy_upstream_handler(ngx_event_t *ev); 22 | @@ -73,7 +73,7 @@ 23 | static void ngx_stream_proxy_process(ngx_stream_session_t *s, 24 | ngx_uint_t from_upstream, ngx_uint_t do_write); 25 | static void ngx_stream_proxy_next_upstream(ngx_stream_session_t *s); 26 | -static void ngx_stream_proxy_finalize(ngx_stream_session_t *s, ngx_uint_t rc); 27 | +void ngx_stream_proxy_finalize(ngx_stream_session_t *s, ngx_uint_t rc); 28 | static u_char *ngx_stream_proxy_log_error(ngx_log_t *log, u_char *buf, 29 | size_t len); 30 | 31 | @@ -646,7 +646,7 @@ 32 | } 33 | 34 | 35 | -static void 36 | +void 37 | ngx_stream_proxy_connect(ngx_stream_session_t *s) 38 | { 39 | ngx_int_t rc; 40 | @@ -662,29 +662,35 @@ 41 | 42 | u = s->upstream; 43 | 44 | - u->connected = 0; 45 | - u->proxy_protocol = pscf->proxy_protocol; 46 | + if (!u->blocked) { 47 | + u->connected = 0; 48 | + u->proxy_protocol = pscf->proxy_protocol; 49 | 50 | - if (u->state) { 51 | - u->state->response_time = ngx_current_msec - u->state->response_time; 52 | - } 53 | + if (u->state) { 54 | + u->state->response_time = ngx_current_msec - u->state->response_time; 55 | + } 56 | 57 | - u->state = ngx_array_push(s->upstream_states); 58 | - if (u->state == NULL) { 59 | - ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR); 60 | - return; 61 | - } 62 | + u->state = ngx_array_push(s->upstream_states); 63 | + if (u->state == NULL) { 64 | + ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR); 65 | + return; 66 | + } 67 | 68 | - ngx_memzero(u->state, sizeof(ngx_stream_upstream_state_t)); 69 | + ngx_memzero(u->state, sizeof(ngx_stream_upstream_state_t)); 70 | 71 | - u->state->connect_time = (ngx_msec_t) -1; 72 | - u->state->first_byte_time = (ngx_msec_t) -1; 73 | - u->state->response_time = ngx_current_msec; 74 | + u->state->connect_time = (ngx_msec_t) -1; 75 | + u->state->first_byte_time = (ngx_msec_t) -1; 76 | + u->state->response_time = ngx_current_msec; 77 | + } 78 | 79 | rc = ngx_event_connect_peer(&u->peer); 80 | 81 | ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0, "proxy connect: %i", rc); 82 | 83 | + if (rc == NGX_BLOCK) { 84 | + return; 85 | + } 86 | + 87 | if (rc == NGX_ERROR) { 88 | ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR); 89 | return; 90 | @@ -1663,7 +1669,7 @@ 91 | } 92 | 93 | 94 | -static void 95 | +void 96 | ngx_stream_proxy_next_upstream(ngx_stream_session_t *s) 97 | { 98 | ngx_msec_t timeout; 99 | @@ -1729,7 +1735,7 @@ 100 | } 101 | 102 | 103 | -static void 104 | +void 105 | ngx_stream_proxy_finalize(ngx_stream_session_t *s, ngx_uint_t rc) 106 | { 107 | ngx_connection_t *pc; 108 | -------------------------------------------------------------------------------- /nginx-1.12.1.patch: -------------------------------------------------------------------------------- 1 | diff --git a/src/core/ngx_core.h b/src/core/ngx_core.h 2 | index 2069373b..e24e5df5 100644 3 | --- a/src/core/ngx_core.h 4 | +++ b/src/core/ngx_core.h 5 | @@ -39,6 +39,7 @@ typedef void (*ngx_connection_handler_pt)(ngx_connection_t *c); 6 | #define NGX_DONE -4 7 | #define NGX_DECLINED -5 8 | #define NGX_ABORT -6 9 | +#define NGX_BLOCK -7 10 | 11 | 12 | #include 13 | diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c 14 | index 36952860..a2fa6b34 100644 15 | --- a/src/http/ngx_http_upstream.c 16 | +++ b/src/http/ngx_http_upstream.c 17 | @@ -35,8 +35,6 @@ static void ngx_http_upstream_rd_check_broken_connection(ngx_http_request_t *r); 18 | static void ngx_http_upstream_wr_check_broken_connection(ngx_http_request_t *r); 19 | static void ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, 20 | ngx_event_t *ev); 21 | -static void ngx_http_upstream_connect(ngx_http_request_t *r, 22 | - ngx_http_upstream_t *u); 23 | static ngx_int_t ngx_http_upstream_reinit(ngx_http_request_t *r, 24 | ngx_http_upstream_t *u); 25 | static void ngx_http_upstream_send_request(ngx_http_request_t *r, 26 | @@ -1457,36 +1455,42 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, 27 | } 28 | 29 | 30 | -static void 31 | +void 32 | ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u) 33 | { 34 | ngx_int_t rc; 35 | ngx_connection_t *c; 36 | 37 | - r->connection->log->action = "connecting to upstream"; 38 | + if (!u->blocked) { 39 | + r->connection->log->action = "connecting to upstream"; 40 | 41 | - if (u->state && u->state->response_time) { 42 | - u->state->response_time = ngx_current_msec - u->state->response_time; 43 | - } 44 | + if (u->state && u->state->response_time) { 45 | + u->state->response_time = ngx_current_msec - u->state->response_time; 46 | + } 47 | 48 | - u->state = ngx_array_push(r->upstream_states); 49 | - if (u->state == NULL) { 50 | - ngx_http_upstream_finalize_request(r, u, 51 | - NGX_HTTP_INTERNAL_SERVER_ERROR); 52 | - return; 53 | - } 54 | + u->state = ngx_array_push(r->upstream_states); 55 | + if (u->state == NULL) { 56 | + ngx_http_upstream_finalize_request(r, u, 57 | + NGX_HTTP_INTERNAL_SERVER_ERROR); 58 | + return; 59 | + } 60 | 61 | - ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t)); 62 | + ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t)); 63 | 64 | - u->state->response_time = ngx_current_msec; 65 | - u->state->connect_time = (ngx_msec_t) -1; 66 | - u->state->header_time = (ngx_msec_t) -1; 67 | + u->state->response_time = ngx_current_msec; 68 | + u->state->connect_time = (ngx_msec_t) -1; 69 | + u->state->header_time = (ngx_msec_t) -1; 70 | + } 71 | 72 | rc = ngx_event_connect_peer(&u->peer); 73 | 74 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 75 | "http upstream connect: %i", rc); 76 | 77 | + if (rc == NGX_BLOCK) { 78 | + return; 79 | + } 80 | + 81 | if (rc == NGX_ERROR) { 82 | ngx_http_upstream_finalize_request(r, u, 83 | NGX_HTTP_INTERNAL_SERVER_ERROR); 84 | diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h 85 | index c552ac0c..92bb6229 100644 86 | --- a/src/http/ngx_http_upstream.h 87 | +++ b/src/http/ngx_http_upstream.h 88 | @@ -389,6 +389,7 @@ struct ngx_http_upstream_s { 89 | unsigned request_sent:1; 90 | unsigned request_body_sent:1; 91 | unsigned header_sent:1; 92 | + unsigned blocked:1; 93 | }; 94 | 95 | 96 | @@ -416,6 +417,8 @@ char *ngx_http_upstream_param_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, 97 | ngx_int_t ngx_http_upstream_hide_headers_hash(ngx_conf_t *cf, 98 | ngx_http_upstream_conf_t *conf, ngx_http_upstream_conf_t *prev, 99 | ngx_str_t *default_hide_headers, ngx_hash_init_t *hash); 100 | +void ngx_http_upstream_connect(ngx_http_request_t *r, 101 | + ngx_http_upstream_t *u); 102 | 103 | 104 | #define ngx_http_conf_upstream_srv_conf(uscf, module) \ 105 | -------------------------------------------------------------------------------- /nginx-1.4.4.patch: -------------------------------------------------------------------------------- 1 | diff --git a/src/core/ngx_core.h b/src/core/ngx_core.h 2 | index c78c73b..bc913ae 100644 3 | --- a/src/core/ngx_core.h 4 | +++ b/src/core/ngx_core.h 5 | @@ -33,6 +33,7 @@ typedef void (*ngx_connection_handler_pt)(ngx_connection_t *c); 6 | #define NGX_DONE -4 7 | #define NGX_DECLINED -5 8 | #define NGX_ABORT -6 9 | +#define NGX_BLOCK -7 10 | 11 | 12 | #include 13 | diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c 14 | index 58738a3..deda8fa 100644 15 | --- a/src/http/ngx_http_upstream.c 16 | +++ b/src/http/ngx_http_upstream.c 17 | @@ -25,8 +25,6 @@ static void ngx_http_upstream_rd_check_broken_connection(ngx_http_request_t *r); 18 | static void ngx_http_upstream_wr_check_broken_connection(ngx_http_request_t *r); 19 | static void ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, 20 | ngx_event_t *ev); 21 | -static void ngx_http_upstream_connect(ngx_http_request_t *r, 22 | - ngx_http_upstream_t *u); 23 | static ngx_int_t ngx_http_upstream_reinit(ngx_http_request_t *r, 24 | ngx_http_upstream_t *u); 25 | static void ngx_http_upstream_send_request(ngx_http_request_t *r, 26 | @@ -1149,39 +1147,45 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, 27 | } 28 | 29 | 30 | -static void 31 | +void 32 | ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u) 33 | { 34 | ngx_int_t rc; 35 | ngx_time_t *tp; 36 | ngx_connection_t *c; 37 | 38 | - r->connection->log->action = "connecting to upstream"; 39 | + if (!u->blocked) { 40 | + r->connection->log->action = "connecting to upstream"; 41 | 42 | - if (u->state && u->state->response_sec) { 43 | - tp = ngx_timeofday(); 44 | - u->state->response_sec = tp->sec - u->state->response_sec; 45 | - u->state->response_msec = tp->msec - u->state->response_msec; 46 | - } 47 | + if (u->state && u->state->response_sec) { 48 | + tp = ngx_timeofday(); 49 | + u->state->response_sec = tp->sec - u->state->response_sec; 50 | + u->state->response_msec = tp->msec - u->state->response_msec; 51 | + } 52 | + 53 | + u->state = ngx_array_push(r->upstream_states); 54 | + if (u->state == NULL) { 55 | + ngx_http_upstream_finalize_request(r, u, 56 | + NGX_HTTP_INTERNAL_SERVER_ERROR); 57 | + return; 58 | + } 59 | 60 | - u->state = ngx_array_push(r->upstream_states); 61 | - if (u->state == NULL) { 62 | - ngx_http_upstream_finalize_request(r, u, 63 | - NGX_HTTP_INTERNAL_SERVER_ERROR); 64 | - return; 65 | + ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t)); 66 | + 67 | + tp = ngx_timeofday(); 68 | + u->state->response_sec = tp->sec; 69 | + u->state->response_msec = tp->msec; 70 | } 71 | 72 | - ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t)); 73 | - 74 | - tp = ngx_timeofday(); 75 | - u->state->response_sec = tp->sec; 76 | - u->state->response_msec = tp->msec; 77 | - 78 | rc = ngx_event_connect_peer(&u->peer); 79 | 80 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 81 | "http upstream connect: %i", rc); 82 | 83 | + if (rc == NGX_BLOCK) { 84 | + return; 85 | + } 86 | + 87 | if (rc == NGX_ERROR) { 88 | ngx_http_upstream_finalize_request(r, u, 89 | NGX_HTTP_INTERNAL_SERVER_ERROR); 90 | diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h 91 | index e29772e..c598c1e 100644 92 | --- a/src/http/ngx_http_upstream.h 93 | +++ b/src/http/ngx_http_upstream.h 94 | @@ -352,6 +352,7 @@ struct ngx_http_upstream_s { 95 | 96 | unsigned request_sent:1; 97 | unsigned header_sent:1; 98 | + unsigned blocked:1; 99 | }; 100 | 101 | 102 | @@ -382,6 +383,8 @@ char *ngx_http_upstream_param_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, 103 | ngx_int_t ngx_http_upstream_hide_headers_hash(ngx_conf_t *cf, 104 | ngx_http_upstream_conf_t *conf, ngx_http_upstream_conf_t *prev, 105 | ngx_str_t *default_hide_headers, ngx_hash_init_t *hash); 106 | +void ngx_http_upstream_connect(ngx_http_request_t *r, 107 | + ngx_http_upstream_t *u); 108 | 109 | 110 | #define ngx_http_conf_upstream_srv_conf(uscf, module) \ 111 | -------------------------------------------------------------------------------- /nginx-1.8.1.patch: -------------------------------------------------------------------------------- 1 | --- src/http/ngx_http_upstream.h.1.8.1 2016-06-01 12:13:21.314144886 +0800 2 | +++ src/http/ngx_http_upstream.h 2016-06-01 12:13:32.678144596 +0800 3 | @@ -367,6 +367,7 @@ 4 | 5 | unsigned request_sent:1; 6 | unsigned header_sent:1; 7 | + unsigned blocked:1; 8 | }; 9 | 10 | 11 | @@ -399,6 +400,8 @@ 12 | ngx_int_t ngx_http_upstream_hide_headers_hash(ngx_conf_t *cf, 13 | ngx_http_upstream_conf_t *conf, ngx_http_upstream_conf_t *prev, 14 | ngx_str_t *default_hide_headers, ngx_hash_init_t *hash); 15 | +void ngx_http_upstream_connect(ngx_http_request_t *r, 16 | + ngx_http_upstream_t *u); 17 | 18 | 19 | #define ngx_http_conf_upstream_srv_conf(uscf, module) \ 20 | --- src/http/ngx_http_upstream.c.1.8.1 2016-06-01 12:13:21.318144886 +0800 21 | +++ src/http/ngx_http_upstream.c 2016-06-01 12:18:01.790137730 +0800 22 | @@ -31,8 +31,6 @@ 23 | static void ngx_http_upstream_wr_check_broken_connection(ngx_http_request_t *r); 24 | static void ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, 25 | ngx_event_t *ev); 26 | -static void ngx_http_upstream_connect(ngx_http_request_t *r, 27 | - ngx_http_upstream_t *u); 28 | static ngx_int_t ngx_http_upstream_reinit(ngx_http_request_t *r, 29 | ngx_http_upstream_t *u); 30 | static void ngx_http_upstream_send_request(ngx_http_request_t *r, 31 | @@ -1299,40 +1297,46 @@ 32 | } 33 | 34 | 35 | -static void 36 | +void 37 | ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u) 38 | { 39 | ngx_int_t rc; 40 | ngx_time_t *tp; 41 | ngx_connection_t *c; 42 | 43 | - r->connection->log->action = "connecting to upstream"; 44 | + if (!u->blocked) { 45 | + r->connection->log->action = "connecting to upstream"; 46 | 47 | - if (u->state && u->state->response_sec) { 48 | - tp = ngx_timeofday(); 49 | - u->state->response_sec = tp->sec - u->state->response_sec; 50 | - u->state->response_msec = tp->msec - u->state->response_msec; 51 | - } 52 | + if (u->state && u->state->response_sec) { 53 | + tp = ngx_timeofday(); 54 | + u->state->response_sec = tp->sec - u->state->response_sec; 55 | + u->state->response_msec = tp->msec - u->state->response_msec; 56 | + } 57 | 58 | - u->state = ngx_array_push(r->upstream_states); 59 | - if (u->state == NULL) { 60 | - ngx_http_upstream_finalize_request(r, u, 61 | - NGX_HTTP_INTERNAL_SERVER_ERROR); 62 | - return; 63 | - } 64 | + u->state = ngx_array_push(r->upstream_states); 65 | + if (u->state == NULL) { 66 | + ngx_http_upstream_finalize_request(r, u, 67 | + NGX_HTTP_INTERNAL_SERVER_ERROR); 68 | + return; 69 | + } 70 | 71 | - ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t)); 72 | + ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t)); 73 | 74 | - tp = ngx_timeofday(); 75 | - u->state->response_sec = tp->sec; 76 | - u->state->response_msec = tp->msec; 77 | - u->state->header_sec = (time_t) NGX_ERROR; 78 | + tp = ngx_timeofday(); 79 | + u->state->response_sec = tp->sec; 80 | + u->state->response_msec = tp->msec; 81 | + u->state->header_sec = (time_t) NGX_ERROR; 82 | + } 83 | 84 | rc = ngx_event_connect_peer(&u->peer); 85 | 86 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 87 | "http upstream connect: %i", rc); 88 | 89 | + if (rc == NGX_BLOCK) { 90 | + return; 91 | + } 92 | + 93 | if (rc == NGX_ERROR) { 94 | ngx_http_upstream_finalize_request(r, u, 95 | NGX_HTTP_INTERNAL_SERVER_ERROR); 96 | --- src/core/ngx_core.h.1.8.1 2016-06-01 12:13:21.318144886 +0800 97 | +++ src/core/ngx_core.h 2016-06-01 12:13:32.674144596 +0800 98 | @@ -40,6 +40,7 @@ 99 | #define NGX_DONE -4 100 | #define NGX_DECLINED -5 101 | #define NGX_ABORT -6 102 | +#define NGX_BLOCK -7 103 | 104 | 105 | #include 106 | -------------------------------------------------------------------------------- /nginx.patch: -------------------------------------------------------------------------------- 1 | Index: /branches/nginx/src/core/ngx_core.h 2 | =================================================================== 3 | --- /branches/nginx/src/core/ngx_core.h (revision 3473) 4 | +++ /branches/nginx/src/core/ngx_core.h (working copy) 5 | @@ -33,6 +33,7 @@ 6 | #define NGX_DONE -4 7 | #define NGX_DECLINED -5 8 | #define NGX_ABORT -6 9 | +#define NGX_BLOCK -7 10 | 11 | 12 | #include 13 | Index: /branches/nginx/src/http/ngx_http_upstream.h 14 | =================================================================== 15 | --- /branches/nginx/src/http/ngx_http_upstream.h (revision 3473) 16 | +++ /branches/nginx/src/http/ngx_http_upstream.h (working copy) 17 | @@ -320,6 +320,7 @@ 18 | 19 | unsigned request_sent:1; 20 | unsigned header_sent:1; 21 | + unsigned blocked:1; 22 | }; 23 | 24 | 25 | @@ -341,6 +342,8 @@ 26 | ngx_int_t ngx_http_upstream_hide_headers_hash(ngx_conf_t *cf, 27 | ngx_http_upstream_conf_t *conf, ngx_http_upstream_conf_t *prev, 28 | ngx_str_t *default_hide_headers, ngx_hash_init_t *hash); 29 | +void ngx_http_upstream_connect(ngx_http_request_t *r, 30 | + ngx_http_upstream_t *u); 31 | 32 | 33 | #define ngx_http_conf_upstream_srv_conf(uscf, module) \ 34 | Index: /branches/nginx/src/http/ngx_http_upstream.c 35 | =================================================================== 36 | --- /branches/nginx/src/http/ngx_http_upstream.c (revision 3473) 37 | +++ /branches/nginx/src/http/ngx_http_upstream.c (working copy) 38 | @@ -24,8 +24,6 @@ 39 | static void ngx_http_upstream_wr_check_broken_connection(ngx_http_request_t *r); 40 | static void ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, 41 | ngx_event_t *ev); 42 | -static void ngx_http_upstream_connect(ngx_http_request_t *r, 43 | - ngx_http_upstream_t *u); 44 | static ngx_int_t ngx_http_upstream_reinit(ngx_http_request_t *r, 45 | ngx_http_upstream_t *u); 46 | static void ngx_http_upstream_send_request(ngx_http_request_t *r, 47 | @@ -1070,41 +1068,48 @@ 48 | } 49 | 50 | 51 | -static void 52 | +void 53 | ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u) 54 | { 55 | ngx_int_t rc; 56 | ngx_time_t *tp; 57 | ngx_connection_t *c; 58 | 59 | - r->connection->log->action = "connecting to upstream"; 60 | + if (!u->blocked) { 61 | 62 | - r->connection->single_connection = 0; 63 | + r->connection->log->action = "connecting to upstream"; 64 | 65 | - if (u->state && u->state->response_sec) { 66 | - tp = ngx_timeofday(); 67 | - u->state->response_sec = tp->sec - u->state->response_sec; 68 | - u->state->response_msec = tp->msec - u->state->response_msec; 69 | - } 70 | + r->connection->single_connection = 0; 71 | 72 | - u->state = ngx_array_push(r->upstream_states); 73 | - if (u->state == NULL) { 74 | - ngx_http_upstream_finalize_request(r, u, 75 | - NGX_HTTP_INTERNAL_SERVER_ERROR); 76 | - return; 77 | - } 78 | + if (u->state && u->state->response_sec) { 79 | + tp = ngx_timeofday(); 80 | + u->state->response_sec = tp->sec - u->state->response_sec; 81 | + u->state->response_msec = tp->msec - u->state->response_msec; 82 | + } 83 | 84 | - ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t)); 85 | + u->state = ngx_array_push(r->upstream_states); 86 | + if (u->state == NULL) { 87 | + ngx_http_upstream_finalize_request(r, u, 88 | + NGX_HTTP_INTERNAL_SERVER_ERROR); 89 | + return; 90 | + } 91 | 92 | - tp = ngx_timeofday(); 93 | - u->state->response_sec = tp->sec; 94 | - u->state->response_msec = tp->msec; 95 | + ngx_memzero(u->state, sizeof(ngx_http_upstream_state_t)); 96 | + 97 | + tp = ngx_timeofday(); 98 | + u->state->response_sec = tp->sec; 99 | + u->state->response_msec = tp->msec; 100 | + } 101 | 102 | rc = ngx_event_connect_peer(&u->peer); 103 | 104 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 105 | "http upstream connect: %i", rc); 106 | 107 | + if (rc == NGX_BLOCK) { 108 | + return; 109 | + } 110 | + 111 | if (rc == NGX_ERROR) { 112 | ngx_http_upstream_finalize_request(r, u, 113 | NGX_HTTP_INTERNAL_SERVER_ERROR); 114 | -------------------------------------------------------------------------------- /ngx_http_limit_upstream_module.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Author: cfsego 3 | */ 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | static ngx_rbtree_t ngx_http_limit_upstream_loc_rbtree; 10 | static ngx_rbtree_node_t ngx_http_limit_upstream_sentinel; 11 | 12 | 13 | typedef struct { 14 | ngx_uint_t limit_conn; 15 | ngx_uint_t backlog; 16 | ngx_msec_t timeout; 17 | ngx_shm_zone_t *shm_zone; 18 | 19 | ngx_http_upstream_init_peer_pt init; 20 | 21 | ngx_uint_t log_level; 22 | 23 | unsigned hooked:1; 24 | } ngx_http_limit_upstream_conf_t; 25 | 26 | 27 | typedef struct { 28 | ngx_rbtree_t *rbtree; 29 | ngx_queue_t *queue; 30 | } ngx_http_limit_upstream_zone_t; 31 | 32 | 33 | typedef struct { 34 | u_char color; 35 | unsigned short port; 36 | ngx_uint_t counter; 37 | } ngx_http_limit_upstream_node_t; 38 | 39 | 40 | typedef struct { 41 | volatile ngx_uint_t counter; 42 | ngx_msec_t last; 43 | ngx_queue_t queue; 44 | } ngx_http_limit_upstream_shm_t; 45 | 46 | 47 | typedef struct { 48 | ngx_uint_t counter; 49 | ngx_uint_t qlen; 50 | ngx_uint_t work; 51 | ngx_queue_t wait; 52 | } ngx_http_limit_upstream_loc_t; 53 | 54 | 55 | typedef struct { 56 | ngx_http_request_t *r; 57 | ngx_http_limit_upstream_conf_t *lucf; 58 | ngx_http_limit_upstream_loc_t *lnode; 59 | void *data; 60 | void *wait; 61 | 62 | unsigned in_proc:1; 63 | unsigned cln:1; 64 | 65 | ngx_event_get_peer_pt get; 66 | ngx_event_free_peer_pt free; 67 | 68 | #if (NGX_HTTP_SSL) 69 | ngx_event_set_peer_session_pt set_session; 70 | ngx_event_save_peer_session_pt save_session; 71 | #endif 72 | 73 | struct sockaddr *sockaddr; 74 | } ngx_http_limit_upstream_ctx_t; 75 | 76 | 77 | typedef struct { 78 | ngx_queue_t queue; 79 | ngx_http_limit_upstream_ctx_t *ctx; 80 | ngx_http_event_handler_pt read_event_handler; 81 | ngx_http_event_handler_pt write_event_handler; 82 | unsigned r_timer_set:1; 83 | unsigned w_timer_set:1; 84 | } ngx_http_limit_upstream_wait_t; 85 | 86 | 87 | static ngx_conf_enum_t ngx_http_limit_upstream_log_levels[] = { 88 | { ngx_string("info"), NGX_LOG_INFO }, 89 | { ngx_string("notice"), NGX_LOG_NOTICE }, 90 | { ngx_string("warn"), NGX_LOG_WARN }, 91 | { ngx_string("error"), NGX_LOG_ERR }, 92 | { ngx_null_string, 0 } 93 | }; 94 | 95 | 96 | static void ngx_http_limit_upstream_timeout(ngx_http_request_t *r); 97 | static void ngx_http_limit_upstream_cleanup(void *data); 98 | static void ngx_http_limit_upstream_free_peer(ngx_peer_connection_t *pc, 99 | void *data, ngx_uint_t state); 100 | static ngx_int_t ngx_http_limit_upstream_get_peer(ngx_peer_connection_t *pc, 101 | void *data); 102 | #if (NGX_HTTP_SSL) 103 | static ngx_int_t ngx_http_limit_upstream_set_peer_session( 104 | ngx_peer_connection_t *pc, void *data); 105 | static void ngx_http_limit_upstream_save_peer_session(ngx_peer_connection_t *pc, 106 | void *data); 107 | #endif 108 | 109 | static ngx_int_t ngx_http_limit_upstream_init_peer(ngx_http_request_t *r, 110 | ngx_http_upstream_srv_conf_t *us); 111 | 112 | static void *ngx_http_limit_upstream_create_srv_conf(ngx_conf_t *cf); 113 | static char *ngx_http_limit_upstream_merge_srv_conf(ngx_conf_t *cf, void *parent, 114 | void *child); 115 | static char *ngx_http_limit_upstream_zone(ngx_conf_t *cf, ngx_command_t *cmd, 116 | void *conf); 117 | static char *ngx_http_limit_upstream_conn(ngx_conf_t *cf, ngx_command_t *cmd, 118 | void *conf); 119 | static ngx_int_t ngx_http_limit_upstream_init(ngx_conf_t *cf); 120 | 121 | static ngx_int_t ngx_http_limit_upstream_init_zone(ngx_shm_zone_t *shm_zone, 122 | void *data); 123 | static void ngx_http_limit_upstream_zone_expire(ngx_shm_zone_t *shm_zone); 124 | 125 | static void 126 | ngx_http_limit_upstream_rbtree_insert_value(ngx_rbtree_node_t *temp, 127 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); 128 | static ngx_rbtree_node_t * 129 | ngx_http_limit_upstream_rbtree_lookup(ngx_rbtree_t *rbtree, 130 | unsigned long addr, unsigned short port); 131 | 132 | 133 | static ngx_command_t ngx_http_limit_upstream_commands[] = { 134 | 135 | { ngx_string("limit_upstream_zone"), 136 | NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE2, 137 | ngx_http_limit_upstream_zone, 138 | 0, 139 | 0, 140 | NULL }, 141 | 142 | { ngx_string("limit_upstream_conn"), 143 | NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1234|NGX_CONF_TAKE5, 144 | ngx_http_limit_upstream_conn, 145 | NGX_HTTP_SRV_CONF_OFFSET, 146 | 0, 147 | NULL }, 148 | 149 | { ngx_string("limit_upstream_log_level"), 150 | NGX_HTTP_MAIN_CONF|NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1, 151 | ngx_conf_set_enum_slot, 152 | NGX_HTTP_SRV_CONF_OFFSET, 153 | offsetof(ngx_http_limit_upstream_conf_t, log_level), 154 | &ngx_http_limit_upstream_log_levels }, 155 | 156 | ngx_null_command 157 | }; 158 | 159 | 160 | static ngx_http_module_t ngx_http_limit_upstream_module_ctx = { 161 | NULL, 162 | ngx_http_limit_upstream_init, 163 | 164 | NULL, 165 | NULL, 166 | 167 | ngx_http_limit_upstream_create_srv_conf, 168 | ngx_http_limit_upstream_merge_srv_conf, 169 | 170 | NULL, 171 | NULL 172 | }; 173 | 174 | 175 | ngx_module_t ngx_http_limit_upstream_module = { 176 | NGX_MODULE_V1, 177 | &ngx_http_limit_upstream_module_ctx, 178 | ngx_http_limit_upstream_commands, 179 | NGX_HTTP_MODULE, 180 | NULL, 181 | NULL, 182 | NULL, 183 | NULL, 184 | NULL, 185 | NULL, 186 | NULL, 187 | NGX_MODULE_V1_PADDING 188 | }; 189 | 190 | 191 | static void 192 | ngx_http_limit_upstream_timeout(ngx_http_request_t *r) 193 | { 194 | ngx_event_t *wev; 195 | ngx_http_upstream_t *u; 196 | ngx_http_limit_upstream_wait_t *w; 197 | ngx_http_limit_upstream_loc_t *l; 198 | ngx_http_limit_upstream_ctx_t *ctx; 199 | 200 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 201 | "limit upstream: into timeout"); 202 | 203 | wev = r->connection->write; 204 | 205 | if (!wev->timedout) { 206 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 207 | "limit upstream: wev ignored"); 208 | return; 209 | } 210 | 211 | u = r->upstream; 212 | 213 | ctx = (ngx_http_limit_upstream_ctx_t *) u->peer.data; 214 | 215 | ngx_log_error(ctx->lucf->log_level, r->connection->log, 0, 216 | "limit upstream: request[%p] is timeout", r); 217 | 218 | wev->timedout = 0; 219 | 220 | if (u->cleanup) { 221 | *u->cleanup = NULL; 222 | u->cleanup = NULL; 223 | } 224 | 225 | w = ctx->wait; 226 | ngx_queue_remove(&w->queue); 227 | 228 | l = ctx->lnode; 229 | l->qlen--; 230 | 231 | ngx_http_finalize_request(r, NGX_HTTP_REQUEST_TIME_OUT); 232 | } 233 | 234 | 235 | static void 236 | ngx_http_limit_upstream_cleanup(void *data) 237 | { 238 | ngx_queue_t *q; 239 | ngx_rbtree_node_t *node_s, *node_l; 240 | struct sockaddr_in *sin; 241 | #if NGX_DEBUG 242 | ngx_peer_connection_t *pc; 243 | #endif 244 | ngx_http_limit_upstream_ctx_t *ctx; 245 | ngx_http_limit_upstream_loc_t *lnode; 246 | ngx_http_limit_upstream_shm_t *snode; 247 | ngx_http_limit_upstream_node_t *cnode; 248 | ngx_http_limit_upstream_wait_t *wnode; 249 | ngx_http_limit_upstream_zone_t *shmctx; 250 | 251 | ctx = (ngx_http_limit_upstream_ctx_t *) data; 252 | shmctx = ctx->lucf->shm_zone->data; 253 | 254 | if (!ctx->in_proc) { 255 | return; 256 | } 257 | 258 | #if NGX_DEBUG 259 | pc = &ctx->r->upstream->peer; 260 | #endif 261 | sin = (struct sockaddr_in *) ctx->sockaddr; 262 | 263 | node_s = ngx_http_limit_upstream_rbtree_lookup(shmctx->rbtree, 264 | sin->sin_addr.s_addr, 265 | sin->sin_port); 266 | 267 | node_l = ngx_http_limit_upstream_rbtree_lookup( 268 | &ngx_http_limit_upstream_loc_rbtree, 269 | sin->sin_addr.s_addr, 270 | sin->sin_port); 271 | 272 | if (node_l == NULL && node_s == NULL) { 273 | return; 274 | } 275 | 276 | #if 1 277 | 278 | if ((node_l == NULL && node_s) || (node_s == NULL && node_l)) { 279 | ngx_log_error(NGX_LOG_EMERG, ctx->r->connection->log, 0, 280 | "limit upstream: only local or shm node exists"); 281 | return; 282 | } 283 | 284 | #endif 285 | 286 | cnode = (ngx_http_limit_upstream_node_t *) &node_s->color; 287 | snode = (ngx_http_limit_upstream_shm_t *) &cnode->counter; 288 | 289 | cnode = (ngx_http_limit_upstream_node_t *) &node_l->color; 290 | lnode = (ngx_http_limit_upstream_loc_t *) &cnode->counter; 291 | 292 | lnode->work--; 293 | 294 | if ((snode->counter > ctx->lucf->limit_conn && lnode->work) 295 | || lnode->qlen == 0) 296 | { 297 | ngx_log_debug4(NGX_LOG_DEBUG_HTTP, ctx->r->connection->log, 0, 298 | "limit upstream: will not resume request for %V " 299 | "(counter: %d, active: %d, wait queue: %d)", 300 | pc->name, snode->counter, lnode->work, lnode->qlen); 301 | 302 | snode->counter--; 303 | lnode->counter--; 304 | return; 305 | } 306 | 307 | do { 308 | q = ngx_queue_last(&lnode->wait); 309 | wnode = ngx_queue_data(q, ngx_http_limit_upstream_wait_t, queue); 310 | 311 | ngx_log_debug5(NGX_LOG_DEBUG_HTTP, ctx->r->connection->log, 0, 312 | "limit upstream: remove queue node: %p for %V " 313 | "(counter: %d, active: %d, wait queue: %d)", 314 | q, pc->name, snode->counter, 315 | lnode->work, lnode->qlen); 316 | 317 | ngx_queue_remove(q); 318 | lnode->qlen--; 319 | 320 | /* resume a request in wait queue */ 321 | 322 | ctx = wnode->ctx; 323 | 324 | ngx_log_error(ctx->lucf->log_level, ctx->r->connection->log, 0, 325 | "limit upstream: request[%p] is resumed", ctx->r); 326 | 327 | ctx->r->read_event_handler = wnode->read_event_handler; 328 | ctx->r->write_event_handler = wnode->write_event_handler; 329 | 330 | ngx_del_timer(ctx->r->connection->read); 331 | 332 | if (wnode->r_timer_set) { 333 | if (ctx->r->connection->read->timedout) { 334 | ctx->r->read_event_handler(ctx->r); 335 | return; 336 | } 337 | 338 | if (ngx_handle_read_event(ctx->r->connection->read, 0) 339 | != NGX_OK) 340 | { 341 | ngx_http_finalize_request(ctx->r, 342 | NGX_HTTP_INTERNAL_SERVER_ERROR); 343 | return; 344 | } 345 | 346 | ngx_add_timer(ctx->r->connection->read, 347 | ctx->r->connection->read->timer.key); 348 | } 349 | 350 | ngx_del_timer(ctx->r->connection->write); 351 | 352 | if (wnode->w_timer_set) { 353 | if (ctx->r->connection->write->timedout) { 354 | ctx->r->write_event_handler(ctx->r); 355 | return; 356 | } 357 | 358 | if (ngx_handle_write_event(ctx->r->connection->write, 0) 359 | != NGX_OK) 360 | { 361 | ngx_http_finalize_request(ctx->r, 362 | NGX_HTTP_INTERNAL_SERVER_ERROR); 363 | return; 364 | } 365 | 366 | ngx_add_timer(ctx->r->connection->write, 367 | ctx->r->connection->write->timer.key); 368 | } 369 | 370 | lnode->work++; 371 | 372 | ngx_http_upstream_connect(ctx->r, ctx->r->upstream); 373 | } while ((ngx_uint_t) ngx_atomic_fetch_add(&snode->counter, 1) 374 | < ctx->lucf->limit_conn && lnode->qlen); 375 | 376 | snode->counter--; 377 | } 378 | 379 | 380 | static void 381 | ngx_http_limit_upstream_free_peer(ngx_peer_connection_t *pc, void *data, 382 | ngx_uint_t state) 383 | { 384 | ngx_http_limit_upstream_ctx_t *ctx; 385 | 386 | ctx = (ngx_http_limit_upstream_ctx_t *) data; 387 | 388 | ctx->free(pc, ctx->data, state); 389 | } 390 | 391 | 392 | static ngx_int_t 393 | ngx_http_limit_upstream_get_peer(ngx_peer_connection_t *pc, void *data) 394 | { 395 | size_t n; 396 | ngx_int_t rc; 397 | #if (NGX_DEBUG) 398 | ngx_uint_t active; 399 | #endif 400 | ngx_slab_pool_t *shpool; 401 | ngx_rbtree_node_t *node_s, *node_l; 402 | struct sockaddr_in *sin; 403 | ngx_http_cleanup_t *cln; 404 | ngx_http_upstream_t *u; 405 | ngx_http_limit_upstream_ctx_t *ctx; 406 | ngx_http_limit_upstream_loc_t *lnode; 407 | ngx_http_limit_upstream_shm_t *snode; 408 | ngx_http_limit_upstream_node_t *cnode; 409 | ngx_http_limit_upstream_wait_t *wnode; 410 | ngx_http_limit_upstream_zone_t *shmctx; 411 | ngx_rbtree_key_int_t t; 412 | 413 | ctx = (ngx_http_limit_upstream_ctx_t *) data; 414 | shpool = (ngx_slab_pool_t *) ctx->lucf->shm_zone->shm.addr; 415 | shmctx = ctx->lucf->shm_zone->data; 416 | 417 | if (ctx->r->upstream->blocked) { 418 | rc = NGX_OK; 419 | ctx->r->upstream->blocked = 0; 420 | 421 | goto set_and_ret; 422 | 423 | } else { 424 | 425 | if (ctx->in_proc) { 426 | 427 | sin = (struct sockaddr_in *) ctx->sockaddr; 428 | node_s = ngx_http_limit_upstream_rbtree_lookup(shmctx->rbtree, 429 | sin->sin_addr.s_addr, 430 | sin->sin_port); 431 | 432 | node_l = ngx_http_limit_upstream_rbtree_lookup( 433 | &ngx_http_limit_upstream_loc_rbtree, 434 | sin->sin_addr.s_addr, 435 | sin->sin_port); 436 | 437 | if (node_l && node_s) { 438 | cnode = (ngx_http_limit_upstream_node_t *) &node_s->color; 439 | snode = (ngx_http_limit_upstream_shm_t *) &cnode->counter; 440 | 441 | cnode = (ngx_http_limit_upstream_node_t *) &node_l->color; 442 | lnode = (ngx_http_limit_upstream_loc_t *) &cnode->counter; 443 | 444 | lnode->work--; 445 | snode->counter--; 446 | lnode->counter--; 447 | } 448 | 449 | ctx->in_proc = 0; 450 | } 451 | 452 | rc = ctx->get(pc, ctx->data); 453 | if (rc != NGX_OK && rc != NGX_DONE) { 454 | return rc; 455 | } 456 | } 457 | 458 | ctx->sockaddr = ngx_palloc(ctx->r->pool, pc->socklen); 459 | if (ctx->sockaddr == NULL) { 460 | return NGX_ERROR; 461 | } 462 | 463 | ngx_memcpy(ctx->sockaddr, pc->sockaddr, pc->socklen); 464 | 465 | sin = (struct sockaddr_in *) ctx->sockaddr; 466 | 467 | node_s = ngx_http_limit_upstream_rbtree_lookup(shmctx->rbtree, 468 | sin->sin_addr.s_addr, 469 | sin->sin_port); 470 | 471 | node_l = ngx_http_limit_upstream_rbtree_lookup( 472 | &ngx_http_limit_upstream_loc_rbtree, 473 | sin->sin_addr.s_addr, 474 | sin->sin_port); 475 | 476 | if (node_l == NULL) { 477 | n = offsetof(ngx_rbtree_node_t, color) 478 | + offsetof(ngx_http_limit_upstream_node_t, counter) 479 | + sizeof(ngx_http_limit_upstream_loc_t); 480 | 481 | node_l = ngx_pcalloc(ngx_cycle->pool, n); 482 | if (node_l == NULL) { 483 | return NGX_ERROR; 484 | } 485 | 486 | cnode = (ngx_http_limit_upstream_node_t *) &node_l->color; 487 | lnode = (ngx_http_limit_upstream_loc_t *) &cnode->counter; 488 | 489 | node_l->key = sin->sin_addr.s_addr; 490 | cnode->port = sin->sin_port; 491 | 492 | ngx_queue_init(&lnode->wait); 493 | 494 | ngx_rbtree_insert(&ngx_http_limit_upstream_loc_rbtree, node_l); 495 | 496 | } else { 497 | cnode = (ngx_http_limit_upstream_node_t *) &node_l->color; 498 | lnode = (ngx_http_limit_upstream_loc_t *) &cnode->counter; 499 | } 500 | 501 | if (node_s == NULL) { 502 | 503 | ngx_shmtx_lock(&shpool->mutex); 504 | 505 | n = offsetof(ngx_rbtree_node_t, color) 506 | + offsetof(ngx_http_limit_upstream_node_t, counter) 507 | + sizeof(ngx_http_limit_upstream_shm_t); 508 | 509 | node_s = ngx_slab_alloc_locked(shpool, n); 510 | if (node_s == NULL) { 511 | return NGX_ERROR; 512 | } 513 | 514 | cnode = (ngx_http_limit_upstream_node_t *) &node_s->color; 515 | snode = (ngx_http_limit_upstream_shm_t *) &cnode->counter; 516 | 517 | node_s->key = sin->sin_addr.s_addr; 518 | cnode->port = sin->sin_port; 519 | 520 | ngx_queue_insert_head(shmctx->queue, &snode->queue); 521 | 522 | ngx_rbtree_insert(shmctx->rbtree, node_s); 523 | 524 | ngx_shmtx_unlock(&shpool->mutex); 525 | 526 | } else { 527 | cnode = (ngx_http_limit_upstream_node_t *) &node_s->color; 528 | snode = (ngx_http_limit_upstream_shm_t *) &cnode->counter; 529 | } 530 | 531 | snode->last = ngx_current_msec; 532 | 533 | if (rc == NGX_DONE) { 534 | snode->counter++; 535 | lnode->counter++; 536 | lnode->work++; 537 | goto set_and_ret; 538 | } 539 | 540 | ngx_log_debug4(NGX_LOG_DEBUG_HTTP, ctx->r->connection->log, 0, 541 | "limit upstream: status for %V " 542 | "(counter: %d, active: %d, wait queue: %d)", 543 | pc->name, snode->counter, lnode->work, lnode->qlen); 544 | 545 | #if (NGX_DEBUG) 546 | active = ngx_atomic_fetch_add(&snode->counter, 1); 547 | 548 | if (active >= ctx->lucf->limit_conn && lnode->work) { 549 | #else 550 | if ((ngx_uint_t) ngx_atomic_fetch_add(&snode->counter, 1) 551 | >= ctx->lucf->limit_conn && lnode->work) 552 | { 553 | #endif 554 | 555 | snode->counter--; 556 | 557 | if (lnode->qlen >= ctx->lucf->backlog) { 558 | ngx_log_error(ctx->lucf->log_level, ctx->r->connection->log, 0, 559 | "limit upstream: request[%p] is dropped", ctx->r); 560 | return NGX_DECLINED; 561 | } 562 | 563 | wnode = ngx_pcalloc(ctx->r->pool, 564 | sizeof(ngx_http_limit_upstream_wait_t)); 565 | if (wnode == NULL) { 566 | return NGX_ERROR; 567 | } 568 | 569 | ngx_queue_insert_head(&lnode->wait, &wnode->queue); 570 | lnode->qlen++; 571 | 572 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->r->connection->log, 0, 573 | "limit upstream: add queue node: %p", &wnode->queue); 574 | 575 | ngx_log_error(ctx->lucf->log_level, ctx->r->connection->log, 0, 576 | "limit upstream: request[%p] is blocked", ctx->r); 577 | 578 | wnode->ctx = ctx; 579 | ctx->wait = wnode; 580 | ctx->lnode = lnode; 581 | 582 | wnode->read_event_handler = ctx->r->read_event_handler; 583 | wnode->r_timer_set = ctx->r->connection->read->timer_set; 584 | ctx->r->read_event_handler = ngx_http_block_reading; 585 | if (wnode->r_timer_set) { 586 | ngx_del_timer(ctx->r->connection->read); 587 | 588 | t = ctx->r->connection->read->timer.key; 589 | t -= (ngx_rbtree_key_int_t) ngx_current_msec; 590 | 591 | if (t > 0) { 592 | ctx->r->connection->read->timer.key = t; 593 | 594 | } else { 595 | ctx->r->connection->read->timedout = 1; 596 | } 597 | } 598 | 599 | wnode->write_event_handler = ctx->r->write_event_handler; 600 | wnode->w_timer_set = ctx->r->connection->write->timer_set; 601 | ctx->r->write_event_handler = ngx_http_limit_upstream_timeout; 602 | if (wnode->w_timer_set) { 603 | ngx_del_timer(ctx->r->connection->write); 604 | 605 | t = ctx->r->connection->write->timer.key; 606 | t -= (ngx_rbtree_key_int_t) ngx_current_msec; 607 | 608 | if (t > 0) { 609 | ctx->r->connection->write->timer.key = t; 610 | } else { 611 | ctx->r->connection->write->timedout = 1; 612 | } 613 | } 614 | 615 | if (ctx->lucf->timeout == 0) { 616 | u = ctx->r->upstream; 617 | ngx_add_timer(ctx->r->connection->write, u->conf->read_timeout); 618 | 619 | } else { 620 | ngx_add_timer(ctx->r->connection->write, ctx->lucf->timeout); 621 | } 622 | 623 | ctx->r->upstream->blocked = 1; 624 | 625 | return NGX_BLOCK; 626 | } 627 | 628 | #if (NGX_DEBUG) 629 | 630 | if (active >= ctx->lucf->limit_conn) { 631 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->r->connection->log, 0, 632 | "limit upstream: force continue request"); 633 | } 634 | 635 | #endif 636 | 637 | lnode->counter++; 638 | lnode->work++; 639 | 640 | set_and_ret: 641 | 642 | if (!ctx->cln) { 643 | cln = ngx_http_cleanup_add(ctx->r, 0); 644 | if (cln == NULL) { 645 | return NGX_ERROR; 646 | } 647 | 648 | cln->handler = ngx_http_limit_upstream_cleanup; 649 | cln->data = ctx; 650 | 651 | ctx->cln = 1; 652 | } 653 | 654 | ctx->in_proc = 1; 655 | 656 | return rc; 657 | } 658 | 659 | 660 | static ngx_int_t 661 | ngx_http_limit_upstream_init_peer(ngx_http_request_t *r, 662 | ngx_http_upstream_srv_conf_t *us) 663 | { 664 | ngx_int_t rc; 665 | ngx_http_limit_upstream_ctx_t *ctx; 666 | ngx_http_limit_upstream_conf_t *lucf; 667 | 668 | ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_limit_upstream_ctx_t)); 669 | if (ctx == NULL) { 670 | return NGX_ERROR; 671 | } 672 | 673 | lucf = us->srv_conf[ngx_http_limit_upstream_module.ctx_index]; 674 | 675 | rc = lucf->init(r, us); 676 | 677 | if (rc != NGX_OK) { 678 | return rc; 679 | } 680 | 681 | ctx->data = r->upstream->peer.data; 682 | r->upstream->peer.data = ctx; 683 | 684 | ctx->get = r->upstream->peer.get; 685 | r->upstream->peer.get = ngx_http_limit_upstream_get_peer; 686 | 687 | ctx->free = r->upstream->peer.free; 688 | r->upstream->peer.free = ngx_http_limit_upstream_free_peer; 689 | 690 | #if (NGX_HTTP_SSL) 691 | ctx->set_session = r->upstream->peer.set_session; 692 | r->upstream->peer.set_session = ngx_http_limit_upstream_set_peer_session; 693 | 694 | ctx->save_session = r->upstream->peer.save_session; 695 | r->upstream->peer.save_session = ngx_http_limit_upstream_save_peer_session; 696 | #endif 697 | 698 | ctx->r = r; 699 | ctx->lucf = lucf; 700 | r->upstream->blocked = 0; 701 | 702 | return NGX_OK; 703 | } 704 | 705 | 706 | static ngx_int_t 707 | ngx_http_limit_upstream_init(ngx_conf_t *cf) 708 | { 709 | ngx_uint_t i; 710 | ngx_http_limit_upstream_conf_t *lucf, *mlucf; 711 | ngx_http_upstream_srv_conf_t **uscfp; 712 | ngx_http_upstream_main_conf_t *umcf; 713 | 714 | umcf = ngx_http_conf_get_module_main_conf(cf, 715 | ngx_http_upstream_module); 716 | mlucf = ngx_http_conf_get_module_srv_conf(cf, 717 | ngx_http_limit_upstream_module); 718 | uscfp = umcf->upstreams.elts; 719 | 720 | ngx_rbtree_init(&ngx_http_limit_upstream_loc_rbtree, 721 | &ngx_http_limit_upstream_sentinel, 722 | ngx_http_limit_upstream_rbtree_insert_value); 723 | 724 | for (i = 0; i < umcf->upstreams.nelts; i++) { 725 | 726 | if (uscfp[i]->srv_conf) { 727 | lucf = uscfp[i]-> 728 | srv_conf[ngx_http_limit_upstream_module.ctx_index]; 729 | 730 | if (lucf->limit_conn != NGX_CONF_UNSET_UINT && !lucf->hooked) { 731 | lucf->init = uscfp[i]->peer.init; 732 | uscfp[i]->peer.init = ngx_http_limit_upstream_init_peer; 733 | 734 | if (lucf->backlog == NGX_CONF_UNSET_UINT) { 735 | lucf->backlog = 1000; 736 | } 737 | 738 | if (lucf->timeout == NGX_CONF_UNSET_MSEC) { 739 | lucf->timeout = 0; 740 | } 741 | 742 | if (ngx_http_limit_upstream_merge_srv_conf(cf, mlucf, lucf) 743 | != NGX_CONF_OK) 744 | { 745 | return NGX_ERROR; 746 | } 747 | } 748 | } 749 | } 750 | 751 | return NGX_OK; 752 | } 753 | 754 | 755 | static void * 756 | ngx_http_limit_upstream_create_srv_conf(ngx_conf_t *cf) 757 | { 758 | ngx_http_limit_upstream_conf_t *conf; 759 | 760 | conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_limit_upstream_conf_t)); 761 | if (conf == NULL) { 762 | return NULL; 763 | } 764 | 765 | /* 766 | * set by ngx_pcalloc(); 767 | * 768 | * conf->init = NULL; 769 | * conf->shm_zone = NULL; 770 | * conf->hooked = 0; 771 | */ 772 | 773 | conf->limit_conn = NGX_CONF_UNSET_UINT; 774 | conf->backlog = NGX_CONF_UNSET_UINT; 775 | conf->timeout = NGX_CONF_UNSET_MSEC; 776 | conf->log_level = NGX_CONF_UNSET_UINT; 777 | 778 | return conf; 779 | } 780 | 781 | 782 | static char * 783 | ngx_http_limit_upstream_merge_srv_conf(ngx_conf_t *cf, void *parent, 784 | void *child) 785 | { 786 | ngx_http_limit_upstream_conf_t *conf = child; 787 | ngx_http_limit_upstream_conf_t *prev = parent; 788 | 789 | ngx_conf_merge_uint_value(conf->log_level, 790 | prev->log_level, NGX_LOG_NOTICE); 791 | 792 | return NGX_CONF_OK; 793 | } 794 | 795 | 796 | static char * 797 | ngx_http_limit_upstream_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 798 | { 799 | ssize_t n; 800 | ngx_str_t *value; 801 | ngx_shm_zone_t *shm_zone; 802 | ngx_http_limit_upstream_zone_t *ctx; 803 | 804 | value = cf->args->elts; 805 | 806 | n = ngx_parse_size(&value[2]); 807 | if (n == NGX_ERROR) { 808 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 809 | "invalid size of limit_upstream_zone \"%V\"", 810 | &value[2]); 811 | return NGX_CONF_ERROR; 812 | } 813 | 814 | if (n < (ngx_int_t) (8 * ngx_pagesize)) { 815 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 816 | "limit_upstream_zone \"%V\" is too small", 817 | &value[1]); 818 | return NGX_CONF_ERROR; 819 | } 820 | 821 | shm_zone = ngx_shared_memory_add(cf, &value[1], n, 822 | &ngx_http_limit_upstream_module); 823 | if (shm_zone == NULL) { 824 | return NGX_CONF_ERROR; 825 | } 826 | 827 | if (shm_zone->data) { 828 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 829 | "limit_upstream_zone \"%V\" is duplicate", 830 | &value[1]); 831 | return NGX_CONF_ERROR; 832 | } 833 | 834 | ctx = ngx_pcalloc(cf->pool, sizeof(ngx_http_limit_upstream_zone_t)); 835 | if (ctx == NULL) { 836 | return NGX_CONF_ERROR; 837 | } 838 | 839 | shm_zone->data = ctx; 840 | shm_zone->init = ngx_http_limit_upstream_init_zone; 841 | 842 | return NGX_CONF_OK; 843 | } 844 | 845 | 846 | static char * 847 | ngx_http_limit_upstream_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 848 | { 849 | ngx_str_t *value, s; 850 | ngx_uint_t i; 851 | ngx_http_upstream_srv_conf_t *uscf; 852 | 853 | ngx_http_limit_upstream_conf_t *lucf = conf; 854 | 855 | if (lucf->shm_zone) { 856 | return "is duplicate"; 857 | } 858 | 859 | uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); 860 | 861 | value = cf->args->elts; 862 | 863 | for (i = 1; i < cf->args->nelts; i++) { 864 | 865 | if (ngx_strncmp("zone=", value[i].data, 5) == 0) { 866 | 867 | s.len = value[i].len - 5; 868 | s.data = value[i].data + 5; 869 | 870 | lucf->shm_zone = ngx_shared_memory_add(cf, &s, 0, 871 | &ngx_http_limit_upstream_module); 872 | if (lucf->shm_zone == NULL) { 873 | return NGX_CONF_ERROR; 874 | } 875 | 876 | continue; 877 | } 878 | 879 | if (ngx_strncmp("limit=", value[i].data, 6) == 0) { 880 | lucf->limit_conn = ngx_atoi(value[i].data + 6, value[i].len - 6); 881 | if (lucf->limit_conn <= 0) { 882 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 883 | "invalid limit \"%V\"", &value[i]); 884 | return NGX_CONF_ERROR; 885 | } 886 | 887 | continue; 888 | } 889 | 890 | if (ngx_strncmp("backlog=", value[i].data, 8) == 0) { 891 | 892 | lucf->backlog = ngx_atoi(value[i].data + 8, value[i].len - 8); 893 | if (lucf->backlog <= 0) { 894 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 895 | "invalid backlog \"%V\"", &value[i]); 896 | return NGX_CONF_ERROR; 897 | } 898 | 899 | continue; 900 | } 901 | 902 | if (ngx_strncmp("nodelay", value[i].data, 7) == 0) { 903 | lucf->backlog = 0; 904 | continue; 905 | } 906 | 907 | if (ngx_strncmp("timeout=", value[i].data, 8) == 0) { 908 | 909 | s.len = value[i].len - 8; 910 | s.data = value[i].data + 8; 911 | 912 | lucf->timeout = ngx_parse_time(&s, 0); 913 | if (lucf->timeout == (ngx_msec_t) NGX_ERROR) { 914 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 915 | "invalid timeout \"%V\"", &value[i]); 916 | return NGX_CONF_ERROR; 917 | } 918 | 919 | continue; 920 | } 921 | 922 | if (ngx_strncmp("instant_hook", value[i].data, 12) == 0) { 923 | lucf->init = uscf->peer.init; 924 | uscf->peer.init = ngx_http_limit_upstream_init_peer; 925 | lucf->hooked = 1; 926 | continue; 927 | } 928 | 929 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 930 | "invalid parameter \"%V\"", &value[i]); 931 | return NGX_CONF_ERROR; 932 | } 933 | 934 | if (lucf->limit_conn == NGX_CONF_UNSET_UINT) { 935 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 936 | "\"%V\" must have \"limit\" parameter", 937 | &cmd->name); 938 | return NGX_CONF_ERROR; 939 | } 940 | 941 | if (lucf->shm_zone == NULL) { 942 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 943 | "\"%V\" must have \"zone\" parameter", 944 | &cmd->name); 945 | return NGX_CONF_ERROR; 946 | } 947 | 948 | if (lucf->hooked) { 949 | if (lucf->backlog == NGX_CONF_UNSET_UINT) { 950 | lucf->backlog = 1000; 951 | } 952 | 953 | if (lucf->timeout == NGX_CONF_UNSET_MSEC) { 954 | lucf->timeout = 0; 955 | } 956 | 957 | if (lucf->log_level == NGX_CONF_UNSET_UINT) { 958 | lucf->log_level = NGX_LOG_NOTICE; 959 | } 960 | } 961 | 962 | return NGX_CONF_OK; 963 | } 964 | 965 | 966 | static void ngx_http_limit_upstream_zone_expire(ngx_shm_zone_t *shm_zone) 967 | { 968 | ngx_queue_t *q, *t; 969 | ngx_msec_int_t ms; 970 | ngx_slab_pool_t *shpool; 971 | ngx_rbtree_node_t *node; 972 | ngx_http_limit_upstream_shm_t *data; 973 | ngx_http_limit_upstream_zone_t *ctx; 974 | 975 | shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; 976 | ctx = shm_zone->data; 977 | 978 | ngx_shmtx_lock(&shpool->mutex); 979 | 980 | for (q = ngx_queue_head(ctx->queue), t = NULL; 981 | q != ctx->queue; 982 | q = t) 983 | { 984 | data = ngx_queue_data(q, ngx_http_limit_upstream_shm_t, queue); 985 | 986 | ms = (ngx_msec_int_t) ngx_current_msec - data->last; 987 | ms = ngx_abs(ms); 988 | 989 | t = ngx_queue_next(q); 990 | 991 | /* remove nodes not used for at least one hour */ 992 | 993 | if (ms >= 3600000) { 994 | 995 | ngx_queue_remove(q); 996 | 997 | node = (ngx_rbtree_node_t *) ((u_char *) data 998 | - offsetof(ngx_http_limit_upstream_node_t, counter) 999 | - offsetof(ngx_rbtree_node_t, color)); 1000 | 1001 | ngx_rbtree_delete(ctx->rbtree, node); 1002 | 1003 | ngx_slab_free_locked(shpool, node); 1004 | } 1005 | } 1006 | 1007 | ngx_shmtx_unlock(&shpool->mutex); 1008 | } 1009 | 1010 | 1011 | static ngx_int_t 1012 | ngx_http_limit_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data) 1013 | { 1014 | ngx_http_limit_upstream_zone_t *octx = data; 1015 | 1016 | ngx_slab_pool_t *shpool; 1017 | ngx_http_limit_upstream_zone_t *ctx; 1018 | 1019 | ctx = shm_zone->data; 1020 | 1021 | if (octx) { 1022 | ctx->rbtree = octx->rbtree; 1023 | ctx->queue = octx->queue; 1024 | 1025 | ngx_http_limit_upstream_zone_expire(shm_zone); 1026 | 1027 | return NGX_OK; 1028 | } 1029 | 1030 | shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; 1031 | 1032 | ctx->rbtree = ngx_slab_alloc(shpool, sizeof(ngx_rbtree_t)); 1033 | if (ctx->rbtree == NULL) { 1034 | return NGX_ERROR; 1035 | } 1036 | 1037 | ctx->queue = ngx_slab_alloc(shpool, sizeof(ngx_queue_t)); 1038 | if (ctx->queue == NULL) { 1039 | return NGX_ERROR; 1040 | } 1041 | 1042 | ngx_rbtree_init(ctx->rbtree, &ngx_http_limit_upstream_sentinel, 1043 | ngx_http_limit_upstream_rbtree_insert_value); 1044 | 1045 | ngx_queue_init(ctx->queue); 1046 | 1047 | return NGX_OK; 1048 | } 1049 | 1050 | 1051 | static ngx_rbtree_node_t * 1052 | ngx_http_limit_upstream_rbtree_lookup(ngx_rbtree_t *rbtree, unsigned long addr, 1053 | unsigned short port) 1054 | { 1055 | ngx_rbtree_node_t *node, *sentinel; 1056 | ngx_http_limit_upstream_node_t *cnode; 1057 | 1058 | node = rbtree->root; 1059 | sentinel = rbtree->sentinel; 1060 | 1061 | while (node != sentinel) { 1062 | if (addr < node->key) { 1063 | node = node->left; 1064 | continue; 1065 | } 1066 | 1067 | if (addr > node->key) { 1068 | node = node->right; 1069 | continue; 1070 | } 1071 | 1072 | do { 1073 | cnode = (ngx_http_limit_upstream_node_t *) &node->color; 1074 | 1075 | if (port == cnode->port) { 1076 | return node; 1077 | } 1078 | 1079 | node = port < cnode->port ? node->left : node->right; 1080 | } while (node != sentinel && addr == node->key); 1081 | } 1082 | 1083 | return NULL; 1084 | } 1085 | 1086 | 1087 | static void 1088 | ngx_http_limit_upstream_rbtree_insert_value(ngx_rbtree_node_t *temp, 1089 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) 1090 | { 1091 | ngx_rbtree_node_t **p; 1092 | ngx_http_limit_upstream_node_t *cn, *cnt; 1093 | 1094 | for ( ;; ) { 1095 | 1096 | if (node->key < temp->key) { 1097 | p = &temp->left; 1098 | } else if (node->key > temp->key) { 1099 | p = &temp->right; 1100 | } else { 1101 | cn = (ngx_http_limit_upstream_node_t *) &node->color; 1102 | cnt = (ngx_http_limit_upstream_node_t *) &temp->color; 1103 | 1104 | p = cn->port < cnt->port ? &temp->left : &temp->right; 1105 | } 1106 | 1107 | if (*p == sentinel) { 1108 | break; 1109 | } 1110 | 1111 | temp = *p; 1112 | } 1113 | 1114 | *p = node; 1115 | node->parent = temp; 1116 | node->left = sentinel; 1117 | node->right = sentinel; 1118 | ngx_rbt_red(node); 1119 | } 1120 | 1121 | 1122 | #if (NGX_HTTP_SSL) 1123 | 1124 | static ngx_int_t 1125 | ngx_http_limit_upstream_set_peer_session(ngx_peer_connection_t *pc, void *data) 1126 | { 1127 | ngx_http_limit_upstream_ctx_t *ctx; 1128 | 1129 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, 1130 | "limit upstream: set ssl session"); 1131 | 1132 | ctx = (ngx_http_limit_upstream_ctx_t *) data; 1133 | 1134 | if (ctx->set_session) { 1135 | return ctx->set_session(pc, ctx->data); 1136 | } 1137 | 1138 | return NGX_OK; 1139 | } 1140 | 1141 | 1142 | static void 1143 | ngx_http_limit_upstream_save_peer_session(ngx_peer_connection_t *pc, void *data) 1144 | { 1145 | ngx_http_limit_upstream_ctx_t *ctx; 1146 | 1147 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, 1148 | "limit upstream: save session"); 1149 | 1150 | ctx = (ngx_http_limit_upstream_ctx_t *) data; 1151 | 1152 | if (ctx->save_session) { 1153 | ctx->save_session(pc, ctx->data); 1154 | } 1155 | } 1156 | 1157 | #endif 1158 | -------------------------------------------------------------------------------- /ngx_stream_limit_upstream_module.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Author: cfsego 3 | */ 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | static ngx_rbtree_t ngx_stream_limit_ups_loc_rbtree; 10 | static ngx_rbtree_node_t ngx_stream_limit_ups_sentinel; 11 | 12 | 13 | typedef struct { 14 | ngx_uint_t limit_conn; 15 | ngx_uint_t backlog; 16 | ngx_msec_t timeout; 17 | ngx_shm_zone_t *shm_zone; 18 | 19 | ngx_stream_upstream_init_peer_pt init; 20 | 21 | ngx_uint_t log_level; 22 | 23 | unsigned hooked:1; 24 | } ngx_stream_limit_ups_conf_t; 25 | 26 | 27 | typedef struct { 28 | ngx_rbtree_t *rbtree; 29 | ngx_queue_t *queue; 30 | } ngx_stream_limit_ups_zone_t; 31 | 32 | 33 | typedef struct { 34 | u_char color; 35 | unsigned short port; 36 | ngx_uint_t counter; 37 | } ngx_stream_limit_ups_node_t; 38 | 39 | 40 | typedef struct { 41 | volatile ngx_uint_t counter; 42 | ngx_msec_t last; 43 | ngx_queue_t queue; 44 | } ngx_stream_limit_ups_shm_t; 45 | 46 | 47 | typedef struct { 48 | ngx_uint_t counter; 49 | ngx_uint_t qlen; 50 | ngx_uint_t work; 51 | ngx_queue_t wait; 52 | } ngx_stream_limit_ups_loc_t; 53 | 54 | 55 | typedef struct { 56 | ngx_stream_session_t *s; 57 | ngx_stream_limit_ups_conf_t *lucf; 58 | ngx_stream_limit_ups_loc_t *lnode; 59 | void *data; 60 | void *wait; 61 | 62 | unsigned in_proc:1; 63 | unsigned cln:1; 64 | 65 | ngx_event_get_peer_pt get; 66 | ngx_event_free_peer_pt free; 67 | ngx_event_notify_peer_pt notify; 68 | 69 | #if (NGX_HTTP_SSL) 70 | ngx_event_set_peer_session_pt set_session; 71 | ngx_event_save_peer_session_pt save_session; 72 | #endif 73 | 74 | struct sockaddr *sockaddr; 75 | } ngx_stream_limit_ups_ctx_t; 76 | 77 | 78 | typedef struct { 79 | ngx_queue_t queue; 80 | ngx_stream_limit_ups_ctx_t *ctx; 81 | ngx_event_handler_pt read_event_handler; 82 | ngx_event_handler_pt write_event_handler; 83 | unsigned r_timer_set:1; 84 | unsigned w_timer_set:1; 85 | } ngx_stream_limit_ups_wait_t; 86 | 87 | 88 | typedef struct { 89 | ngx_msec_t connect_timeout; 90 | ngx_msec_t timeout; 91 | } ngx_stream_proxy_srv_conf_t; 92 | 93 | 94 | static ngx_conf_enum_t ngx_stream_limit_ups_log_levels[] = { 95 | { ngx_string("info"), NGX_LOG_INFO }, 96 | { ngx_string("notice"), NGX_LOG_NOTICE }, 97 | { ngx_string("warn"), NGX_LOG_WARN }, 98 | { ngx_string("error"), NGX_LOG_ERR }, 99 | { ngx_null_string, 0 } 100 | }; 101 | 102 | 103 | static void ngx_stream_limit_ups_block_reading(ngx_event_t *ev); 104 | static void ngx_stream_limit_ups_timeout(ngx_event_t *ev); 105 | static void ngx_stream_limit_ups_cleanup(void *data); 106 | static void ngx_stream_limit_ups_notify_peer(ngx_peer_connection_t *pc, 107 | void *data, ngx_uint_t state); 108 | static void ngx_stream_limit_ups_free_peer(ngx_peer_connection_t *pc, 109 | void *data, ngx_uint_t state); 110 | static ngx_int_t ngx_stream_limit_ups_get_peer(ngx_peer_connection_t *pc, 111 | void *data); 112 | #if (NGX_HTTP_SSL) 113 | static ngx_int_t ngx_stream_limit_ups_set_peer_session( 114 | ngx_peer_connection_t *pc, void *data); 115 | static void ngx_stream_limit_ups_save_peer_session(ngx_peer_connection_t *pc, 116 | void *data); 117 | #endif 118 | 119 | static ngx_int_t ngx_stream_limit_ups_init_peer(ngx_stream_session_t *s, 120 | ngx_stream_upstream_srv_conf_t *us); 121 | 122 | static void *ngx_stream_limit_ups_create_srv_conf(ngx_conf_t *cf); 123 | static char *ngx_stream_limit_ups_merge_srv_conf(ngx_conf_t *cf, void *parent, 124 | void *child); 125 | static char *ngx_stream_limit_ups_zone(ngx_conf_t *cf, ngx_command_t *cmd, 126 | void *conf); 127 | static char *ngx_stream_limit_ups_conn(ngx_conf_t *cf, ngx_command_t *cmd, 128 | void *conf); 129 | static ngx_int_t ngx_stream_limit_ups_init(ngx_conf_t *cf); 130 | 131 | static ngx_int_t ngx_stream_limit_ups_init_zone(ngx_shm_zone_t *shm_zone, 132 | void *data); 133 | static void ngx_stream_limit_ups_zone_expire(ngx_shm_zone_t *shm_zone); 134 | 135 | static void 136 | ngx_stream_limit_ups_rbtree_insert_value(ngx_rbtree_node_t *temp, 137 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); 138 | static ngx_rbtree_node_t * 139 | ngx_stream_limit_ups_rbtree_lookup(ngx_rbtree_t *rbtree, 140 | unsigned long addr, unsigned short port); 141 | 142 | 143 | static ngx_command_t ngx_stream_limit_ups_commands[] = { 144 | 145 | { ngx_string("limit_upstream_zone"), 146 | NGX_STREAM_MAIN_CONF|NGX_CONF_TAKE2, 147 | ngx_stream_limit_ups_zone, 148 | 0, 149 | 0, 150 | NULL }, 151 | 152 | { ngx_string("limit_upstream_conn"), 153 | NGX_STREAM_UPS_CONF|NGX_CONF_TAKE1234|NGX_CONF_TAKE5, 154 | ngx_stream_limit_ups_conn, 155 | NGX_STREAM_SRV_CONF_OFFSET, 156 | 0, 157 | NULL }, 158 | 159 | { ngx_string("limit_upstream_log_level"), 160 | NGX_STREAM_MAIN_CONF|NGX_STREAM_UPS_CONF|NGX_CONF_TAKE1, 161 | ngx_conf_set_enum_slot, 162 | NGX_STREAM_SRV_CONF_OFFSET, 163 | offsetof(ngx_stream_limit_ups_conf_t, log_level), 164 | &ngx_stream_limit_ups_log_levels }, 165 | 166 | ngx_null_command 167 | }; 168 | 169 | 170 | static ngx_stream_module_t ngx_stream_limit_ups_module_ctx = { 171 | NULL, 172 | ngx_stream_limit_ups_init, 173 | 174 | NULL, 175 | NULL, 176 | 177 | ngx_stream_limit_ups_create_srv_conf, 178 | ngx_stream_limit_ups_merge_srv_conf, 179 | 180 | NULL, 181 | NULL 182 | }; 183 | 184 | 185 | ngx_module_t ngx_stream_limit_upstream_module = { 186 | NGX_MODULE_V1, 187 | &ngx_stream_limit_ups_module_ctx, 188 | ngx_stream_limit_ups_commands, 189 | NGX_STREAM_MODULE, 190 | NULL, 191 | NULL, 192 | NULL, 193 | NULL, 194 | NULL, 195 | NULL, 196 | NULL, 197 | NGX_MODULE_V1_PADDING 198 | }; 199 | 200 | 201 | extern void ngx_stream_proxy_finalize(ngx_stream_session_t *s, ngx_uint_t rc); 202 | extern void ngx_stream_proxy_connect(ngx_stream_session_t *s); 203 | extern ngx_module_t ngx_stream_proxy_module; 204 | 205 | 206 | static void 207 | ngx_stream_limit_ups_block_reading(ngx_event_t *ev) 208 | { 209 | ngx_connection_t *c; 210 | ngx_stream_session_t *s; 211 | 212 | c = ev->data; 213 | s = c->data; 214 | 215 | ngx_log_debug0(NGX_LOG_DEBUG_STREAM, c->log, 0, 216 | "limit upstream: stream reading blocked"); 217 | 218 | /* aio does not call this handler */ 219 | 220 | if ((ngx_event_flags & NGX_USE_LEVEL_EVENT) && c->read->active) 221 | { 222 | if (ngx_del_event(c->read, NGX_READ_EVENT, 0) != NGX_OK) { 223 | ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR); 224 | } 225 | } 226 | } 227 | 228 | 229 | static void 230 | ngx_stream_limit_ups_timeout(ngx_event_t *ev) 231 | { 232 | ngx_connection_t *c; 233 | ngx_stream_session_t *s; 234 | ngx_stream_upstream_t *u; 235 | ngx_stream_limit_ups_wait_t *w; 236 | ngx_stream_limit_ups_loc_t *l; 237 | ngx_stream_limit_ups_ctx_t *ctx; 238 | 239 | c = ev->data; 240 | s = c->data; 241 | 242 | ngx_log_debug0(NGX_LOG_DEBUG_STREAM, c->log, 0, 243 | "limit upstream: into timeout"); 244 | 245 | if (!ev->timedout) { 246 | ngx_log_debug0(NGX_LOG_DEBUG_STREAM, c->log, 0, 247 | "limit upstream: wev ignored"); 248 | return; 249 | } 250 | 251 | u = s->upstream; 252 | 253 | ctx = (ngx_stream_limit_ups_ctx_t *) u->peer.data; 254 | 255 | ngx_log_error(ctx->lucf->log_level, c->log, 0, 256 | "limit upstream: session[%p] is timeout", s); 257 | 258 | ev->timedout = 0; 259 | 260 | w = ctx->wait; 261 | ngx_queue_remove(&w->queue); 262 | 263 | l = ctx->lnode; 264 | l->qlen--; 265 | 266 | ngx_stream_proxy_finalize(s, NGX_STREAM_SERVICE_UNAVAILABLE); 267 | } 268 | 269 | 270 | static void 271 | ngx_stream_limit_ups_cleanup(void *data) 272 | { 273 | ngx_queue_t *q; 274 | ngx_rbtree_node_t *node_s, *node_l; 275 | struct sockaddr_in *sin; 276 | #if NGX_DEBUG 277 | ngx_peer_connection_t *pc; 278 | #endif 279 | ngx_stream_limit_ups_ctx_t *ctx; 280 | ngx_stream_limit_ups_loc_t *lnode; 281 | ngx_stream_limit_ups_shm_t *snode; 282 | ngx_stream_limit_ups_node_t *cnode; 283 | ngx_stream_limit_ups_wait_t *wnode; 284 | ngx_stream_limit_ups_zone_t *shmctx; 285 | 286 | ctx = (ngx_stream_limit_ups_ctx_t *) data; 287 | shmctx = ctx->lucf->shm_zone->data; 288 | 289 | if (!ctx->in_proc) { 290 | return; 291 | } 292 | 293 | #if NGX_DEBUG 294 | pc = &ctx->s->upstream->peer; 295 | #endif 296 | sin = (struct sockaddr_in *) ctx->sockaddr; 297 | 298 | node_s = ngx_stream_limit_ups_rbtree_lookup(shmctx->rbtree, 299 | sin->sin_addr.s_addr, 300 | sin->sin_port); 301 | 302 | node_l = ngx_stream_limit_ups_rbtree_lookup( 303 | &ngx_stream_limit_ups_loc_rbtree, 304 | sin->sin_addr.s_addr, 305 | sin->sin_port); 306 | 307 | if (node_l == NULL && node_s == NULL) { 308 | return; 309 | } 310 | 311 | #if 1 312 | 313 | if ((node_l == NULL && node_s) || (node_s == NULL && node_l)) { 314 | ngx_log_error(NGX_LOG_EMERG, ctx->s->connection->log, 0, 315 | "limit upstream: only local or shm node exists"); 316 | return; 317 | } 318 | 319 | #endif 320 | 321 | cnode = (ngx_stream_limit_ups_node_t *) &node_s->color; 322 | snode = (ngx_stream_limit_ups_shm_t *) &cnode->counter; 323 | 324 | cnode = (ngx_stream_limit_ups_node_t *) &node_l->color; 325 | lnode = (ngx_stream_limit_ups_loc_t *) &cnode->counter; 326 | 327 | lnode->work--; 328 | 329 | if ((snode->counter > ctx->lucf->limit_conn && lnode->work) 330 | || lnode->qlen == 0) 331 | { 332 | ngx_log_debug4(NGX_LOG_DEBUG_STREAM, ctx->s->connection->log, 0, 333 | "limit upstream: will not resume session for %V " 334 | "(counter: %d, active: %d, wait queue: %d)", 335 | pc->name, snode->counter, lnode->work, lnode->qlen); 336 | 337 | snode->counter--; 338 | lnode->counter--; 339 | return; 340 | } 341 | 342 | do { 343 | q = ngx_queue_last(&lnode->wait); 344 | wnode = ngx_queue_data(q, ngx_stream_limit_ups_wait_t, queue); 345 | 346 | ngx_log_debug5(NGX_LOG_DEBUG_STREAM, ctx->s->connection->log, 0, 347 | "limit upstream: remove queue node: %p for %V " 348 | "(counter: %d, active: %d, wait queue: %d)", 349 | q, pc->name, snode->counter, 350 | lnode->work, lnode->qlen); 351 | 352 | ngx_queue_remove(q); 353 | lnode->qlen--; 354 | 355 | /* resume a request in wait queue */ 356 | 357 | ctx = wnode->ctx; 358 | 359 | ngx_log_error(ctx->lucf->log_level, ctx->s->connection->log, 0, 360 | "limit upstream: session[%p] is resumed", ctx->s); 361 | 362 | ctx->s->connection->read->handler = wnode->read_event_handler; 363 | ctx->s->connection->write->handler = wnode->write_event_handler; 364 | 365 | if (wnode->r_timer_set) { 366 | if (ctx->s->connection->read->timedout) { 367 | ctx->s->connection->read->handler(ctx->s->connection->read); 368 | return; 369 | } 370 | 371 | if (ngx_handle_read_event(ctx->s->connection->read, 0) 372 | != NGX_OK) 373 | { 374 | ngx_stream_proxy_finalize(ctx->s, 375 | NGX_STREAM_INTERNAL_SERVER_ERROR); 376 | return; 377 | } 378 | 379 | ngx_add_timer(ctx->s->connection->read, 380 | ctx->s->connection->read->timer.key); 381 | } 382 | 383 | ngx_del_timer(ctx->s->connection->write); 384 | 385 | if (wnode->w_timer_set) { 386 | if (ctx->s->connection->write->timedout) { 387 | ctx->s->connection->write->handler(ctx->s->connection->write); 388 | return; 389 | } 390 | 391 | 392 | if (ngx_handle_write_event(ctx->s->connection->write, 0) 393 | != NGX_OK) 394 | { 395 | ngx_stream_proxy_finalize(ctx->s, 396 | NGX_STREAM_INTERNAL_SERVER_ERROR); 397 | return; 398 | } 399 | 400 | ngx_add_timer(ctx->s->connection->write, 401 | ctx->s->connection->write->timer.key); 402 | } 403 | 404 | lnode->work++; 405 | 406 | ngx_stream_proxy_connect(ctx->s); 407 | } while ((ngx_uint_t) ngx_atomic_fetch_add(&snode->counter, 1) 408 | < ctx->lucf->limit_conn && lnode->qlen); 409 | 410 | snode->counter--; 411 | } 412 | 413 | 414 | static void 415 | ngx_stream_limit_ups_notify_peer(ngx_peer_connection_t *pc, void *data, 416 | ngx_uint_t state) 417 | { 418 | ngx_stream_limit_ups_ctx_t *ctx; 419 | 420 | ctx = (ngx_stream_limit_ups_ctx_t *) data; 421 | 422 | if (ctx->notify) { 423 | ctx->notify(pc, ctx->data, state); 424 | } 425 | } 426 | 427 | 428 | static void 429 | ngx_stream_limit_ups_free_peer(ngx_peer_connection_t *pc, void *data, 430 | ngx_uint_t state) 431 | { 432 | ngx_stream_limit_ups_ctx_t *ctx; 433 | 434 | ctx = (ngx_stream_limit_ups_ctx_t *) data; 435 | 436 | ctx->free(pc, ctx->data, state); 437 | } 438 | 439 | 440 | static ngx_int_t 441 | ngx_stream_limit_ups_get_peer(ngx_peer_connection_t *pc, void *data) 442 | { 443 | size_t n; 444 | ngx_int_t rc; 445 | #if (NGX_DEBUG) 446 | ngx_uint_t active; 447 | #endif 448 | ngx_slab_pool_t *shpool; 449 | ngx_rbtree_node_t *node_s, *node_l; 450 | struct sockaddr_in *sin; 451 | ngx_pool_cleanup_t *cln; 452 | ngx_rbtree_key_int_t t; 453 | ngx_stream_limit_ups_ctx_t *ctx; 454 | ngx_stream_limit_ups_loc_t *lnode; 455 | ngx_stream_limit_ups_shm_t *snode; 456 | ngx_stream_limit_ups_node_t *cnode; 457 | ngx_stream_limit_ups_wait_t *wnode; 458 | ngx_stream_limit_ups_zone_t *shmctx; 459 | ngx_stream_proxy_srv_conf_t *pscf; 460 | 461 | ctx = (ngx_stream_limit_ups_ctx_t *) data; 462 | shpool = (ngx_slab_pool_t *) ctx->lucf->shm_zone->shm.addr; 463 | shmctx = ctx->lucf->shm_zone->data; 464 | 465 | if (ctx->s->upstream->blocked) { 466 | rc = NGX_OK; 467 | ctx->s->upstream->blocked = 0; 468 | 469 | goto set_and_ret; 470 | 471 | } else { 472 | 473 | if (ctx->in_proc) { 474 | 475 | sin = (struct sockaddr_in *) ctx->sockaddr; 476 | node_s = ngx_stream_limit_ups_rbtree_lookup(shmctx->rbtree, 477 | sin->sin_addr.s_addr, 478 | sin->sin_port); 479 | 480 | node_l = ngx_stream_limit_ups_rbtree_lookup( 481 | &ngx_stream_limit_ups_loc_rbtree, 482 | sin->sin_addr.s_addr, 483 | sin->sin_port); 484 | 485 | if (node_l && node_s) { 486 | cnode = (ngx_stream_limit_ups_node_t *) &node_s->color; 487 | snode = (ngx_stream_limit_ups_shm_t *) &cnode->counter; 488 | 489 | cnode = (ngx_stream_limit_ups_node_t *) &node_l->color; 490 | lnode = (ngx_stream_limit_ups_loc_t *) &cnode->counter; 491 | 492 | lnode->work--; 493 | snode->counter--; 494 | lnode->counter--; 495 | } 496 | 497 | ctx->in_proc = 0; 498 | } 499 | 500 | rc = ctx->get(pc, ctx->data); 501 | if (rc != NGX_OK && rc != NGX_DONE) { 502 | return rc; 503 | } 504 | } 505 | 506 | ctx->sockaddr = ngx_palloc(ctx->s->connection->pool, pc->socklen); 507 | if (ctx->sockaddr == NULL) { 508 | return NGX_ERROR; 509 | } 510 | 511 | ngx_memcpy(ctx->sockaddr, pc->sockaddr, pc->socklen); 512 | 513 | sin = (struct sockaddr_in *) ctx->sockaddr; 514 | 515 | node_s = ngx_stream_limit_ups_rbtree_lookup(shmctx->rbtree, 516 | sin->sin_addr.s_addr, 517 | sin->sin_port); 518 | 519 | node_l = ngx_stream_limit_ups_rbtree_lookup( 520 | &ngx_stream_limit_ups_loc_rbtree, 521 | sin->sin_addr.s_addr, 522 | sin->sin_port); 523 | 524 | if (node_l == NULL) { 525 | n = offsetof(ngx_rbtree_node_t, color) 526 | + offsetof(ngx_stream_limit_ups_node_t, counter) 527 | + sizeof(ngx_stream_limit_ups_loc_t); 528 | 529 | node_l = ngx_pcalloc(ngx_cycle->pool, n); 530 | if (node_l == NULL) { 531 | return NGX_ERROR; 532 | } 533 | 534 | cnode = (ngx_stream_limit_ups_node_t *) &node_l->color; 535 | lnode = (ngx_stream_limit_ups_loc_t *) &cnode->counter; 536 | 537 | node_l->key = sin->sin_addr.s_addr; 538 | cnode->port = sin->sin_port; 539 | 540 | ngx_queue_init(&lnode->wait); 541 | 542 | ngx_rbtree_insert(&ngx_stream_limit_ups_loc_rbtree, node_l); 543 | 544 | } else { 545 | cnode = (ngx_stream_limit_ups_node_t *) &node_l->color; 546 | lnode = (ngx_stream_limit_ups_loc_t *) &cnode->counter; 547 | } 548 | 549 | if (node_s == NULL) { 550 | 551 | ngx_shmtx_lock(&shpool->mutex); 552 | 553 | n = offsetof(ngx_rbtree_node_t, color) 554 | + offsetof(ngx_stream_limit_ups_node_t, counter) 555 | + sizeof(ngx_stream_limit_ups_shm_t); 556 | 557 | node_s = ngx_slab_alloc_locked(shpool, n); 558 | if (node_s == NULL) { 559 | return NGX_ERROR; 560 | } 561 | 562 | cnode = (ngx_stream_limit_ups_node_t *) &node_s->color; 563 | snode = (ngx_stream_limit_ups_shm_t *) &cnode->counter; 564 | 565 | node_s->key = sin->sin_addr.s_addr; 566 | cnode->port = sin->sin_port; 567 | 568 | ngx_queue_insert_head(shmctx->queue, &snode->queue); 569 | 570 | ngx_rbtree_insert(shmctx->rbtree, node_s); 571 | 572 | ngx_shmtx_unlock(&shpool->mutex); 573 | 574 | } else { 575 | cnode = (ngx_stream_limit_ups_node_t *) &node_s->color; 576 | snode = (ngx_stream_limit_ups_shm_t *) &cnode->counter; 577 | } 578 | 579 | snode->last = ngx_current_msec; 580 | 581 | if (rc == NGX_DONE) { 582 | snode->counter++; 583 | lnode->counter++; 584 | lnode->work++; 585 | goto set_and_ret; 586 | } 587 | 588 | ngx_log_debug4(NGX_LOG_DEBUG_STREAM, ctx->s->connection->log, 0, 589 | "limit upstream: status for %V " 590 | "(counter: %d, active: %d, wait queue: %d)", 591 | pc->name, snode->counter, lnode->work, lnode->qlen); 592 | 593 | #if (NGX_DEBUG) 594 | active = ngx_atomic_fetch_add(&snode->counter, 1); 595 | 596 | if (active >= ctx->lucf->limit_conn && lnode->work) { 597 | #else 598 | if ((ngx_uint_t) ngx_atomic_fetch_add(&snode->counter, 1) 599 | >= ctx->lucf->limit_conn && lnode->work) 600 | { 601 | #endif 602 | 603 | snode->counter--; 604 | 605 | if (lnode->qlen >= ctx->lucf->backlog) { 606 | ngx_log_error(ctx->lucf->log_level, ctx->s->connection->log, 0, 607 | "limit upstream: session[%p] is dropped", ctx->s); 608 | return NGX_DECLINED; 609 | } 610 | 611 | wnode = ngx_pcalloc(ctx->s->connection->pool, 612 | sizeof(ngx_stream_limit_ups_wait_t)); 613 | if (wnode == NULL) { 614 | return NGX_ERROR; 615 | } 616 | 617 | ngx_queue_insert_head(&lnode->wait, &wnode->queue); 618 | lnode->qlen++; 619 | 620 | ngx_log_debug1(NGX_LOG_DEBUG_STREAM, ctx->s->connection->log, 0, 621 | "limit upstream: add queue node: %p", &wnode->queue); 622 | 623 | ngx_log_error(ctx->lucf->log_level, ctx->s->connection->log, 0, 624 | "limit upstream: session[%p] is blocked", ctx->s); 625 | 626 | wnode->ctx = ctx; 627 | ctx->wait = wnode; 628 | ctx->lnode = lnode; 629 | 630 | wnode->read_event_handler = ctx->s->connection->read->handler; 631 | wnode->r_timer_set = ctx->s->connection->read->timer_set; 632 | ctx->s->connection->read->handler = ngx_stream_limit_ups_block_reading; 633 | if (wnode->r_timer_set) { 634 | ngx_del_timer(ctx->s->connection->read); 635 | 636 | t = ctx->s->connection->read->timer.key; 637 | t -= (ngx_rbtree_key_int_t) ngx_current_msec; 638 | 639 | if (t > 0) { 640 | ctx->s->connection->read->timer.key = t; 641 | 642 | } else { 643 | ctx->s->connection->read->timedout = 1; 644 | } 645 | } 646 | 647 | wnode->write_event_handler = ctx->s->connection->write->handler; 648 | wnode->w_timer_set = ctx->s->connection->write->timer_set; 649 | ctx->s->connection->write->handler = ngx_stream_limit_ups_timeout; 650 | if (wnode->w_timer_set) { 651 | ngx_del_timer(ctx->s->connection->write); 652 | 653 | t = ctx->s->connection->write->timer.key; 654 | t -= (ngx_rbtree_key_int_t) ngx_current_msec; 655 | 656 | if (t > 0) { 657 | ctx->s->connection->write->timer.key = t; 658 | } else { 659 | ctx->s->connection->write->timedout = 1; 660 | } 661 | } 662 | 663 | if (ctx->lucf->timeout == 0) { 664 | pscf = ngx_stream_get_module_srv_conf(ctx->s, 665 | ngx_stream_proxy_module); 666 | ngx_add_timer(ctx->s->connection->write, pscf->timeout); 667 | 668 | } else { 669 | ngx_add_timer(ctx->s->connection->write, ctx->lucf->timeout); 670 | } 671 | 672 | ctx->s->upstream->blocked = 1; 673 | 674 | return NGX_BLOCK; 675 | } 676 | 677 | #if (NGX_DEBUG) 678 | 679 | if (active >= ctx->lucf->limit_conn) { 680 | ngx_log_debug0(NGX_LOG_DEBUG_STREAM, ctx->s->connection->log, 0, 681 | "limit upstream: force continue request"); 682 | } 683 | 684 | #endif 685 | 686 | lnode->counter++; 687 | lnode->work++; 688 | 689 | set_and_ret: 690 | 691 | if (!ctx->cln) { 692 | cln = ngx_pool_cleanup_add(ctx->s->connection->pool, 0); 693 | if (cln == NULL) { 694 | return NGX_ERROR; 695 | } 696 | 697 | cln->handler = ngx_stream_limit_ups_cleanup; 698 | cln->data = ctx; 699 | 700 | ctx->cln = 1; 701 | } 702 | 703 | ctx->in_proc = 1; 704 | 705 | return rc; 706 | } 707 | 708 | 709 | static ngx_int_t 710 | ngx_stream_limit_ups_init_peer(ngx_stream_session_t *s, 711 | ngx_stream_upstream_srv_conf_t *us) 712 | { 713 | ngx_int_t rc; 714 | ngx_stream_limit_ups_ctx_t *ctx; 715 | ngx_stream_limit_ups_conf_t *lucf; 716 | 717 | ctx = ngx_pcalloc(s->connection->pool, 718 | sizeof(ngx_stream_limit_ups_ctx_t)); 719 | if (ctx == NULL) { 720 | return NGX_ERROR; 721 | } 722 | 723 | lucf = us->srv_conf[ngx_stream_limit_upstream_module.ctx_index]; 724 | 725 | rc = lucf->init(s, us); 726 | 727 | if (rc != NGX_OK) { 728 | return rc; 729 | } 730 | 731 | ctx->data = s->upstream->peer.data; 732 | s->upstream->peer.data = ctx; 733 | 734 | ctx->get = s->upstream->peer.get; 735 | s->upstream->peer.get = ngx_stream_limit_ups_get_peer; 736 | 737 | ctx->free = s->upstream->peer.free; 738 | s->upstream->peer.free = ngx_stream_limit_ups_free_peer; 739 | 740 | ctx->notify = s->upstream->peer.notify; 741 | s->upstream->peer.notify = ngx_stream_limit_ups_notify_peer; 742 | 743 | #if (NGX_HTTP_SSL) 744 | ctx->set_session = s->upstream->peer.set_session; 745 | s->upstream->peer.set_session = ngx_stream_limit_ups_set_peer_session; 746 | 747 | ctx->save_session = s->upstream->peer.save_session; 748 | s->upstream->peer.save_session = ngx_stream_limit_ups_save_peer_session; 749 | #endif 750 | 751 | ctx->s = s; 752 | ctx->lucf = lucf; 753 | s->upstream->blocked = 0; 754 | 755 | return NGX_OK; 756 | } 757 | 758 | 759 | static ngx_int_t 760 | ngx_stream_limit_ups_init(ngx_conf_t *cf) 761 | { 762 | ngx_uint_t i; 763 | ngx_stream_limit_ups_conf_t *lucf, *mlucf; 764 | ngx_stream_upstream_srv_conf_t **uscfp; 765 | ngx_stream_upstream_main_conf_t *umcf; 766 | 767 | umcf = ngx_stream_conf_get_module_main_conf(cf, 768 | ngx_stream_upstream_module); 769 | mlucf = ngx_stream_conf_get_module_srv_conf(cf, 770 | ngx_stream_limit_upstream_module); 771 | uscfp = umcf->upstreams.elts; 772 | 773 | ngx_rbtree_init(&ngx_stream_limit_ups_loc_rbtree, 774 | &ngx_stream_limit_ups_sentinel, 775 | ngx_stream_limit_ups_rbtree_insert_value); 776 | 777 | for (i = 0; i < umcf->upstreams.nelts; i++) { 778 | 779 | if (uscfp[i]->srv_conf) { 780 | lucf = uscfp[i]-> 781 | srv_conf[ngx_stream_limit_upstream_module.ctx_index]; 782 | 783 | if (lucf->limit_conn != NGX_CONF_UNSET_UINT && !lucf->hooked) { 784 | lucf->init = uscfp[i]->peer.init; 785 | uscfp[i]->peer.init = ngx_stream_limit_ups_init_peer; 786 | 787 | if (lucf->backlog == NGX_CONF_UNSET_UINT) { 788 | lucf->backlog = 1000; 789 | } 790 | 791 | if (lucf->timeout == NGX_CONF_UNSET_MSEC) { 792 | lucf->timeout = 0; 793 | } 794 | 795 | if (ngx_stream_limit_ups_merge_srv_conf(cf, mlucf, lucf) 796 | != NGX_CONF_OK) 797 | { 798 | return NGX_ERROR; 799 | } 800 | } 801 | } 802 | } 803 | 804 | return NGX_OK; 805 | } 806 | 807 | 808 | static void * 809 | ngx_stream_limit_ups_create_srv_conf(ngx_conf_t *cf) 810 | { 811 | ngx_stream_limit_ups_conf_t *conf; 812 | 813 | conf = ngx_pcalloc(cf->pool, sizeof(ngx_stream_limit_ups_conf_t)); 814 | if (conf == NULL) { 815 | return NULL; 816 | } 817 | 818 | /* 819 | * set by ngx_pcalloc(); 820 | * 821 | * conf->init = NULL; 822 | * conf->shm_zone = NULL; 823 | * conf->hooked = 0; 824 | */ 825 | 826 | conf->limit_conn = NGX_CONF_UNSET_UINT; 827 | conf->backlog = NGX_CONF_UNSET_UINT; 828 | conf->timeout = NGX_CONF_UNSET_MSEC; 829 | conf->log_level = NGX_CONF_UNSET_UINT; 830 | 831 | return conf; 832 | } 833 | 834 | 835 | static char * 836 | ngx_stream_limit_ups_merge_srv_conf(ngx_conf_t *cf, void *parent, 837 | void *child) 838 | { 839 | ngx_stream_limit_ups_conf_t *conf = child; 840 | ngx_stream_limit_ups_conf_t *prev = parent; 841 | 842 | ngx_conf_merge_uint_value(conf->log_level, 843 | prev->log_level, NGX_LOG_NOTICE); 844 | 845 | return NGX_CONF_OK; 846 | } 847 | 848 | 849 | static char * 850 | ngx_stream_limit_ups_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 851 | { 852 | ssize_t n; 853 | ngx_str_t *value; 854 | ngx_shm_zone_t *shm_zone; 855 | ngx_stream_limit_ups_zone_t *ctx; 856 | 857 | value = cf->args->elts; 858 | 859 | n = ngx_parse_size(&value[2]); 860 | if (n == NGX_ERROR) { 861 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 862 | "invalid size of limit_upstream_zone \"%V\"", 863 | &value[2]); 864 | return NGX_CONF_ERROR; 865 | } 866 | 867 | if (n < (ngx_int_t) (8 * ngx_pagesize)) { 868 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 869 | "limit_upstream_zone \"%V\" is too small", 870 | &value[1]); 871 | return NGX_CONF_ERROR; 872 | } 873 | 874 | shm_zone = ngx_shared_memory_add(cf, &value[1], n, 875 | &ngx_stream_limit_upstream_module); 876 | if (shm_zone == NULL) { 877 | return NGX_CONF_ERROR; 878 | } 879 | 880 | if (shm_zone->data) { 881 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 882 | "limit_upstream_zone \"%V\" is duplicate", 883 | &value[1]); 884 | return NGX_CONF_ERROR; 885 | } 886 | 887 | ctx = ngx_pcalloc(cf->pool, sizeof(ngx_stream_limit_ups_zone_t)); 888 | if (ctx == NULL) { 889 | return NGX_CONF_ERROR; 890 | } 891 | 892 | shm_zone->data = ctx; 893 | shm_zone->init = ngx_stream_limit_ups_init_zone; 894 | 895 | return NGX_CONF_OK; 896 | } 897 | 898 | 899 | static char * 900 | ngx_stream_limit_ups_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 901 | { 902 | ngx_str_t *value, s; 903 | ngx_uint_t i; 904 | ngx_stream_upstream_srv_conf_t *uscf; 905 | 906 | ngx_stream_limit_ups_conf_t *lucf = conf; 907 | 908 | if (lucf->shm_zone) { 909 | return "is duplicate"; 910 | } 911 | 912 | uscf = ngx_stream_conf_get_module_srv_conf(cf, ngx_stream_upstream_module); 913 | 914 | value = cf->args->elts; 915 | 916 | for (i = 1; i < cf->args->nelts; i++) { 917 | 918 | if (ngx_strncmp("zone=", value[i].data, 5) == 0) { 919 | 920 | s.len = value[i].len - 5; 921 | s.data = value[i].data + 5; 922 | 923 | lucf->shm_zone = ngx_shared_memory_add(cf, &s, 0, 924 | &ngx_stream_limit_upstream_module); 925 | if (lucf->shm_zone == NULL) { 926 | return NGX_CONF_ERROR; 927 | } 928 | 929 | continue; 930 | } 931 | 932 | if (ngx_strncmp("limit=", value[i].data, 6) == 0) { 933 | lucf->limit_conn = ngx_atoi(value[i].data + 6, value[i].len - 6); 934 | if (lucf->limit_conn <= 0) { 935 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 936 | "invalid limit \"%V\"", &value[i]); 937 | return NGX_CONF_ERROR; 938 | } 939 | 940 | continue; 941 | } 942 | 943 | if (ngx_strncmp("backlog=", value[i].data, 8) == 0) { 944 | 945 | lucf->backlog = ngx_atoi(value[i].data + 8, value[i].len - 8); 946 | if (lucf->backlog <= 0) { 947 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 948 | "invalid backlog \"%V\"", &value[i]); 949 | return NGX_CONF_ERROR; 950 | } 951 | 952 | continue; 953 | } 954 | 955 | if (ngx_strncmp("nodelay", value[i].data, 7) == 0) { 956 | lucf->backlog = 0; 957 | continue; 958 | } 959 | 960 | if (ngx_strncmp("timeout=", value[i].data, 8) == 0) { 961 | 962 | s.len = value[i].len - 8; 963 | s.data = value[i].data + 8; 964 | 965 | lucf->timeout = ngx_parse_time(&s, 0); 966 | if (lucf->timeout == (ngx_msec_t) NGX_ERROR) { 967 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 968 | "invalid timeout \"%V\"", &value[i]); 969 | return NGX_CONF_ERROR; 970 | } 971 | 972 | continue; 973 | } 974 | 975 | if (ngx_strncmp("instant_hook", value[i].data, 12) == 0) { 976 | lucf->init = uscf->peer.init; 977 | uscf->peer.init = ngx_stream_limit_ups_init_peer; 978 | lucf->hooked = 1; 979 | continue; 980 | } 981 | 982 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 983 | "invalid parameter \"%V\"", &value[i]); 984 | return NGX_CONF_ERROR; 985 | } 986 | 987 | if (lucf->limit_conn == NGX_CONF_UNSET_UINT) { 988 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 989 | "\"%V\" must have \"limit\" parameter", 990 | &cmd->name); 991 | return NGX_CONF_ERROR; 992 | } 993 | 994 | if (lucf->shm_zone == NULL) { 995 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 996 | "\"%V\" must have \"zone\" parameter", 997 | &cmd->name); 998 | return NGX_CONF_ERROR; 999 | } 1000 | 1001 | if (lucf->hooked) { 1002 | if (lucf->backlog == NGX_CONF_UNSET_UINT) { 1003 | lucf->backlog = 1000; 1004 | } 1005 | 1006 | if (lucf->timeout == NGX_CONF_UNSET_MSEC) { 1007 | lucf->timeout = 0; 1008 | } 1009 | 1010 | if (lucf->log_level == NGX_CONF_UNSET_UINT) { 1011 | lucf->log_level = NGX_LOG_NOTICE; 1012 | } 1013 | } 1014 | 1015 | return NGX_CONF_OK; 1016 | } 1017 | 1018 | 1019 | static void ngx_stream_limit_ups_zone_expire(ngx_shm_zone_t *shm_zone) 1020 | { 1021 | ngx_queue_t *q, *t; 1022 | ngx_msec_int_t ms; 1023 | ngx_slab_pool_t *shpool; 1024 | ngx_rbtree_node_t *node; 1025 | ngx_stream_limit_ups_shm_t *data; 1026 | ngx_stream_limit_ups_zone_t *ctx; 1027 | 1028 | shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; 1029 | ctx = shm_zone->data; 1030 | 1031 | ngx_shmtx_lock(&shpool->mutex); 1032 | 1033 | for (q = ngx_queue_head(ctx->queue), t = NULL; 1034 | q != ctx->queue; 1035 | q = t) 1036 | { 1037 | data = ngx_queue_data(q, ngx_stream_limit_ups_shm_t, queue); 1038 | 1039 | ms = (ngx_msec_int_t) ngx_current_msec - data->last; 1040 | ms = ngx_abs(ms); 1041 | 1042 | t = ngx_queue_next(q); 1043 | 1044 | /* remove nodes not used for at least one hour */ 1045 | 1046 | if (ms >= 3600000) { 1047 | 1048 | ngx_queue_remove(q); 1049 | 1050 | node = (ngx_rbtree_node_t *) ((u_char *) data 1051 | - offsetof(ngx_stream_limit_ups_node_t, counter) 1052 | - offsetof(ngx_rbtree_node_t, color)); 1053 | 1054 | ngx_rbtree_delete(ctx->rbtree, node); 1055 | 1056 | ngx_slab_free_locked(shpool, node); 1057 | } 1058 | } 1059 | 1060 | ngx_shmtx_unlock(&shpool->mutex); 1061 | } 1062 | 1063 | 1064 | static ngx_int_t 1065 | ngx_stream_limit_ups_init_zone(ngx_shm_zone_t *shm_zone, void *data) 1066 | { 1067 | ngx_stream_limit_ups_zone_t *octx = data; 1068 | 1069 | ngx_slab_pool_t *shpool; 1070 | ngx_stream_limit_ups_zone_t *ctx; 1071 | 1072 | ctx = shm_zone->data; 1073 | 1074 | if (octx) { 1075 | ctx->rbtree = octx->rbtree; 1076 | ctx->queue = octx->queue; 1077 | 1078 | ngx_stream_limit_ups_zone_expire(shm_zone); 1079 | 1080 | return NGX_OK; 1081 | } 1082 | 1083 | shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; 1084 | 1085 | ctx->rbtree = ngx_slab_alloc(shpool, sizeof(ngx_rbtree_t)); 1086 | if (ctx->rbtree == NULL) { 1087 | return NGX_ERROR; 1088 | } 1089 | 1090 | ctx->queue = ngx_slab_alloc(shpool, sizeof(ngx_queue_t)); 1091 | if (ctx->queue == NULL) { 1092 | return NGX_ERROR; 1093 | } 1094 | 1095 | ngx_rbtree_init(ctx->rbtree, &ngx_stream_limit_ups_sentinel, 1096 | ngx_stream_limit_ups_rbtree_insert_value); 1097 | 1098 | ngx_queue_init(ctx->queue); 1099 | 1100 | return NGX_OK; 1101 | } 1102 | 1103 | 1104 | static ngx_rbtree_node_t * 1105 | ngx_stream_limit_ups_rbtree_lookup(ngx_rbtree_t *rbtree, 1106 | unsigned long addr, unsigned short port) 1107 | { 1108 | ngx_rbtree_node_t *node, *sentinel; 1109 | ngx_stream_limit_ups_node_t *cnode; 1110 | 1111 | node = rbtree->root; 1112 | sentinel = rbtree->sentinel; 1113 | 1114 | while (node != sentinel) { 1115 | if (addr < node->key) { 1116 | node = node->left; 1117 | continue; 1118 | } 1119 | 1120 | if (addr > node->key) { 1121 | node = node->right; 1122 | continue; 1123 | } 1124 | 1125 | do { 1126 | cnode = (ngx_stream_limit_ups_node_t *) &node->color; 1127 | 1128 | if (port == cnode->port) { 1129 | return node; 1130 | } 1131 | 1132 | node = port < cnode->port ? node->left : node->right; 1133 | } while (node != sentinel && addr == node->key); 1134 | } 1135 | 1136 | return NULL; 1137 | } 1138 | 1139 | 1140 | static void 1141 | ngx_stream_limit_ups_rbtree_insert_value(ngx_rbtree_node_t *temp, 1142 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) 1143 | { 1144 | ngx_rbtree_node_t **p; 1145 | ngx_stream_limit_ups_node_t *cn, *cnt; 1146 | 1147 | for ( ;; ) { 1148 | 1149 | if (node->key < temp->key) { 1150 | p = &temp->left; 1151 | } else if (node->key > temp->key) { 1152 | p = &temp->right; 1153 | } else { 1154 | cn = (ngx_stream_limit_ups_node_t *) &node->color; 1155 | cnt = (ngx_stream_limit_ups_node_t *) &temp->color; 1156 | 1157 | p = cn->port < cnt->port ? &temp->left : &temp->right; 1158 | } 1159 | 1160 | if (*p == sentinel) { 1161 | break; 1162 | } 1163 | 1164 | temp = *p; 1165 | } 1166 | 1167 | *p = node; 1168 | node->parent = temp; 1169 | node->left = sentinel; 1170 | node->right = sentinel; 1171 | ngx_rbt_red(node); 1172 | } 1173 | 1174 | 1175 | #if (NGX_HTTP_SSL) 1176 | 1177 | static ngx_int_t 1178 | ngx_stream_limit_ups_set_peer_session(ngx_peer_connection_t *pc, 1179 | void *data) 1180 | { 1181 | ngx_stream_limit_ups_ctx_t *ctx; 1182 | 1183 | ngx_log_debug0(NGX_LOG_DEBUG_STREAM, pc->log, 0, 1184 | "limit upstream: set ssl session"); 1185 | 1186 | ctx = (ngx_stream_limit_ups_ctx_t *) data; 1187 | 1188 | if (ctx->set_session) { 1189 | return ctx->set_session(pc, ctx->data); 1190 | } 1191 | 1192 | return NGX_OK; 1193 | } 1194 | 1195 | 1196 | static void 1197 | ngx_stream_limit_ups_save_peer_session(ngx_peer_connection_t *pc, 1198 | void *data) 1199 | { 1200 | ngx_stream_limit_ups_ctx_t *ctx; 1201 | 1202 | ngx_log_debug0(NGX_LOG_DEBUG_STREAM, pc->log, 0, 1203 | "limit upstream: save session"); 1204 | 1205 | ctx = (ngx_stream_limit_ups_ctx_t *) data; 1206 | 1207 | if (ctx->save_session) { 1208 | ctx->save_session(pc, ctx->data); 1209 | } 1210 | } 1211 | 1212 | #endif 1213 | --------------------------------------------------------------------------------