├── config ├── .gdbinit ├── README └── ngx_http_upstream_fair_module.c /config: -------------------------------------------------------------------------------- 1 | ngx_addon_name=ngx_http_upstream_fair_module 2 | HTTP_MODULES="$HTTP_MODULES ngx_http_upstream_fair_module" 3 | NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/ngx_http_upstream_fair_module.c" 4 | -------------------------------------------------------------------------------- /.gdbinit: -------------------------------------------------------------------------------- 1 | define show_fair_peer 2 | set $n = (ngx_http_upstream_fair_shm_block_t *)$arg0 3 | set $peers = $n->peers 4 | printf "upstream id: 0x%08x (%s), current peer: %d/%d\n", $n->node.key, $peers->name.data, $peers->current, $peers->number 5 | set $i = 0 6 | while $i < $peers->number 7 | set $peer = &$peers->peer[$i] 8 | printf "peer %d: %s weight: %d/%d fails: %d/%d acc: %d down: %d nreq: %u last_req_id: %u\n", $i, $peer->name.data,\ 9 | $peer->shared->current_weight, $peer->weight,\ 10 | $peer->shared->fails, $peer->max_fails,\ 11 | $peer->accessed, $peer->down,\ 12 | $peer->shared->nreq, $peer->shared->last_req_id 13 | set $i = $i + 1 14 | end 15 | printf "-----------------\n" 16 | if ($n->node.left != $arg1) 17 | show_fair_peer $n->node.left $arg1 18 | end 19 | if ($n->node.right != $arg1) 20 | show_fair_peer $n->node.right $arg1 21 | end 22 | end 23 | 24 | define show_fair_peers 25 | set $tree = ngx_http_upstream_fair_rbtree 26 | if (!$tree) 27 | printf "Cannot find the upstream_fair peer information tree\n" 28 | else 29 | set $root = (ngx_http_upstream_fair_shm_block_t *)($tree->root) 30 | if ($root != $tree->sentinel) 31 | show_fair_peer $root $tree->sentinel 32 | else 33 | printf "No upstream_fair peer information\n" 34 | end 35 | end 36 | end 37 | document show_fair_peers 38 | Dump upstream_fair peer infromation 39 | end 40 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Nginx Upstream Fair Proxy Load Balancer 2 | -- 3 | 4 | Description: 5 | -- 6 | 7 | The Nginx fair proxy balancer enhances the standard round-robin load balancer provided 8 | with Nginx so that it will track busy back end servers (e.g. Thin, Ebb, Mongrel) 9 | and balance the load to non-busy server processes. 10 | 11 | Further information can be found on http://nginx.localdomain.pl/ 12 | 13 | Ezra Zygmuntowicz has a good writeup of the fair proxy load balancer and how to use it here: 14 | http://brainspl.at/articles/2007/11/09/a-fair-proxy-balancer-for-nginx-and-mongrel 15 | 16 | 17 | Installation: 18 | -- 19 | 20 | You'll need to re-compile Nginx from source to include this module. 21 | Modify your compile of Nginx by adding the following directive 22 | (modified to suit your path of course): 23 | 24 | ./configure --with-http_ssl_module --add-module=/absolute/path/to/nginx-upstream-fair 25 | make 26 | make install 27 | 28 | 29 | Usage: 30 | -- 31 | 32 | Change your Nginx config file's upstream block to include the 'fair' directive: 33 | 34 | upstream mongrel { 35 | fair; 36 | server 127.0.0.1:5000; 37 | server 127.0.0.1:5001; 38 | server 127.0.0.1:5002; 39 | } 40 | 41 | 42 | If you encounter any issues, please report them using the bugtracker at 43 | http://nginx.localdomain.pl/ 44 | 45 | Contributing: 46 | -- 47 | 48 | Git source repositories: 49 | http://github.com/gnosek/nginx-upstream-fair/tree/master 50 | http://git.localdomain.pl/?p=nginx-upstream-fair.git;a=summary 51 | 52 | Please feel free to fork the project at GitHub and submit pull requests or patches. 53 | 54 | -------------------------------------------------------------------------------- /ngx_http_upstream_fair_module.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2007 Grzegorz Nosek 3 | * Work sponsored by Ezra Zygmuntowicz & EngineYard.com 4 | * 5 | * Based on nginx source (C) Igor Sysoev 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | typedef struct { 13 | ngx_uint_t nreq; 14 | ngx_uint_t total_req; 15 | ngx_uint_t last_req_id; 16 | ngx_uint_t fails; 17 | ngx_uint_t current_weight; 18 | } ngx_http_upstream_fair_shared_t; 19 | 20 | typedef struct ngx_http_upstream_fair_peers_s ngx_http_upstream_fair_peers_t; 21 | 22 | typedef struct { 23 | ngx_rbtree_node_t node; 24 | ngx_uint_t generation; 25 | uintptr_t peers; /* forms a unique cookie together with generation */ 26 | ngx_uint_t total_nreq; 27 | ngx_uint_t total_requests; 28 | ngx_atomic_t lock; 29 | ngx_http_upstream_fair_shared_t stats[1]; 30 | } ngx_http_upstream_fair_shm_block_t; 31 | 32 | /* ngx_spinlock is defined without a matching unlock primitive */ 33 | #define ngx_spinlock_unlock(lock) (void) ngx_atomic_cmp_set(lock, ngx_pid, 0) 34 | 35 | typedef struct { 36 | ngx_http_upstream_fair_shared_t *shared; 37 | struct sockaddr *sockaddr; 38 | socklen_t socklen; 39 | ngx_str_t name; 40 | 41 | ngx_uint_t weight; 42 | ngx_uint_t max_fails; 43 | time_t fail_timeout; 44 | 45 | time_t accessed; 46 | ngx_uint_t down:1; 47 | 48 | #if (NGX_HTTP_SSL) 49 | ngx_ssl_session_t *ssl_session; /* local to a process */ 50 | #endif 51 | 52 | } ngx_http_upstream_fair_peer_t; 53 | 54 | #define NGX_HTTP_UPSTREAM_FAIR_NO_RR (1<<26) 55 | #define NGX_HTTP_UPSTREAM_FAIR_WEIGHT_MODE_IDLE (1<<27) 56 | #define NGX_HTTP_UPSTREAM_FAIR_WEIGHT_MODE_PEAK (1<<28) 57 | #define NGX_HTTP_UPSTREAM_FAIR_WEIGHT_MODE_MASK ((1<<27) | (1<<28)) 58 | 59 | enum { WM_DEFAULT = 0, WM_IDLE, WM_PEAK }; 60 | 61 | struct ngx_http_upstream_fair_peers_s { 62 | ngx_http_upstream_fair_shm_block_t *shared; 63 | ngx_uint_t current; 64 | ngx_uint_t size_err:1; 65 | ngx_uint_t no_rr:1; 66 | ngx_uint_t weight_mode:2; 67 | ngx_uint_t number; 68 | ngx_str_t *name; 69 | ngx_http_upstream_fair_peers_t *next; /* for backup peers support, not really used yet */ 70 | ngx_http_upstream_fair_peer_t peer[1]; 71 | }; 72 | 73 | 74 | #define NGX_PEER_INVALID (~0UL) 75 | 76 | typedef struct { 77 | ngx_http_upstream_fair_peers_t *peers; 78 | ngx_uint_t current; 79 | uintptr_t *tried; 80 | uintptr_t *done; 81 | uintptr_t data; 82 | uintptr_t data2; 83 | } ngx_http_upstream_fair_peer_data_t; 84 | 85 | 86 | static ngx_int_t ngx_http_upstream_init_fair(ngx_conf_t *cf, 87 | ngx_http_upstream_srv_conf_t *us); 88 | static ngx_int_t ngx_http_upstream_get_fair_peer(ngx_peer_connection_t *pc, 89 | void *data); 90 | static void ngx_http_upstream_free_fair_peer(ngx_peer_connection_t *pc, 91 | void *data, ngx_uint_t state); 92 | static ngx_int_t ngx_http_upstream_init_fair_peer(ngx_http_request_t *r, 93 | ngx_http_upstream_srv_conf_t *us); 94 | static char *ngx_http_upstream_fair(ngx_conf_t *cf, ngx_command_t *cmd, 95 | void *conf); 96 | static char *ngx_http_upstream_fair_set_shm_size(ngx_conf_t *cf, 97 | ngx_command_t *cmd, void *conf); 98 | static ngx_int_t ngx_http_upstream_fair_init_module(ngx_cycle_t *cycle); 99 | 100 | #if (NGX_HTTP_EXTENDED_STATUS) 101 | static ngx_chain_t *ngx_http_upstream_fair_report_status(ngx_http_request_t *r, 102 | ngx_int_t *length); 103 | #endif 104 | 105 | #if (NGX_HTTP_SSL) 106 | static ngx_int_t ngx_http_upstream_fair_set_session(ngx_peer_connection_t *pc, 107 | void *data); 108 | static void ngx_http_upstream_fair_save_session(ngx_peer_connection_t *pc, 109 | void *data); 110 | #endif 111 | 112 | static ngx_command_t ngx_http_upstream_fair_commands[] = { 113 | 114 | { ngx_string("fair"), 115 | NGX_HTTP_UPS_CONF|NGX_CONF_ANY, 116 | ngx_http_upstream_fair, 117 | 0, 118 | 0, 119 | NULL }, 120 | 121 | { ngx_string("upstream_fair_shm_size"), 122 | NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, 123 | ngx_http_upstream_fair_set_shm_size, 124 | 0, 125 | 0, 126 | NULL }, 127 | 128 | ngx_null_command 129 | }; 130 | 131 | 132 | static ngx_http_module_t ngx_http_upstream_fair_module_ctx = { 133 | NULL, /* preconfiguration */ 134 | NULL, /* postconfiguration */ 135 | 136 | NULL, /* create main configuration */ 137 | NULL, /* init main configuration */ 138 | 139 | NULL, /* create server configuration */ 140 | NULL, /* merge server configuration */ 141 | 142 | NULL, /* create location configuration */ 143 | NULL, /* merge location configuration */ 144 | 145 | #if (NGX_HTTP_EXTENDED_STATUS) 146 | ngx_http_upstream_fair_report_status, 147 | #endif 148 | }; 149 | 150 | 151 | ngx_module_t ngx_http_upstream_fair_module = { 152 | NGX_MODULE_V1, 153 | &ngx_http_upstream_fair_module_ctx, /* module context */ 154 | ngx_http_upstream_fair_commands, /* module directives */ 155 | NGX_HTTP_MODULE, /* module type */ 156 | NULL, /* init master */ 157 | ngx_http_upstream_fair_init_module, /* init module */ 158 | NULL, /* init process */ 159 | NULL, /* init thread */ 160 | NULL, /* exit thread */ 161 | NULL, /* exit process */ 162 | NULL, /* exit master */ 163 | NGX_MODULE_V1_PADDING 164 | }; 165 | 166 | 167 | static ngx_uint_t ngx_http_upstream_fair_shm_size; 168 | static ngx_shm_zone_t * ngx_http_upstream_fair_shm_zone; 169 | static ngx_rbtree_t * ngx_http_upstream_fair_rbtree; 170 | static ngx_uint_t ngx_http_upstream_fair_generation; 171 | 172 | static int 173 | ngx_http_upstream_fair_compare_rbtree_node(const ngx_rbtree_node_t *v_left, 174 | const ngx_rbtree_node_t *v_right) 175 | { 176 | ngx_http_upstream_fair_shm_block_t *left, *right; 177 | 178 | left = (ngx_http_upstream_fair_shm_block_t *) v_left; 179 | right = (ngx_http_upstream_fair_shm_block_t *) v_right; 180 | 181 | if (left->generation < right->generation) { 182 | return -1; 183 | } else if (left->generation > right->generation) { 184 | return 1; 185 | } else { /* left->generation == right->generation */ 186 | if (left->peers < right->peers) { 187 | return -1; 188 | } else if (left->peers > right->peers) { 189 | return 1; 190 | } else { 191 | return 0; 192 | } 193 | } 194 | } 195 | 196 | /* 197 | * generic functions start here 198 | */ 199 | static void 200 | ngx_rbtree_generic_insert(ngx_rbtree_node_t *temp, 201 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel, 202 | int (*compare)(const ngx_rbtree_node_t *left, const ngx_rbtree_node_t *right)) 203 | { 204 | for ( ;; ) { 205 | if (node->key < temp->key) { 206 | 207 | if (temp->left == sentinel) { 208 | temp->left = node; 209 | break; 210 | } 211 | 212 | temp = temp->left; 213 | 214 | } else if (node->key > temp->key) { 215 | 216 | if (temp->right == sentinel) { 217 | temp->right = node; 218 | break; 219 | } 220 | 221 | temp = temp->right; 222 | 223 | } else { /* node->key == temp->key */ 224 | if (compare(node, temp) < 0) { 225 | 226 | if (temp->left == sentinel) { 227 | temp->left = node; 228 | break; 229 | } 230 | 231 | temp = temp->left; 232 | 233 | } else { 234 | 235 | if (temp->right == sentinel) { 236 | temp->right = node; 237 | break; 238 | } 239 | 240 | temp = temp->right; 241 | } 242 | } 243 | } 244 | 245 | node->parent = temp; 246 | node->left = sentinel; 247 | node->right = sentinel; 248 | ngx_rbt_red(node); 249 | } 250 | 251 | #define NGX_BITVECTOR_ELT_SIZE (sizeof(uintptr_t) * 8) 252 | 253 | static uintptr_t * 254 | ngx_bitvector_alloc(ngx_pool_t *pool, ngx_uint_t size, uintptr_t *small) 255 | { 256 | ngx_uint_t nelts = (size + NGX_BITVECTOR_ELT_SIZE - 1) / NGX_BITVECTOR_ELT_SIZE; 257 | 258 | if (small && nelts == 1) { 259 | *small = 0; 260 | return small; 261 | } 262 | 263 | return ngx_pcalloc(pool, nelts * NGX_BITVECTOR_ELT_SIZE); 264 | } 265 | 266 | static ngx_int_t 267 | ngx_bitvector_test(uintptr_t *bv, ngx_uint_t bit) 268 | { 269 | ngx_uint_t n, m; 270 | 271 | n = bit / NGX_BITVECTOR_ELT_SIZE; 272 | m = 1 << (bit % NGX_BITVECTOR_ELT_SIZE); 273 | 274 | return bv[n] & m; 275 | } 276 | 277 | static void 278 | ngx_bitvector_set(uintptr_t *bv, ngx_uint_t bit) 279 | { 280 | ngx_uint_t n, m; 281 | 282 | n = bit / NGX_BITVECTOR_ELT_SIZE; 283 | m = 1 << (bit % NGX_BITVECTOR_ELT_SIZE); 284 | 285 | bv[n] |= m; 286 | } 287 | 288 | /* 289 | * generic functions end here 290 | */ 291 | 292 | static ngx_int_t 293 | ngx_http_upstream_fair_init_module(ngx_cycle_t *cycle) 294 | { 295 | ngx_http_upstream_fair_generation++; 296 | return NGX_OK; 297 | } 298 | 299 | static void 300 | ngx_http_upstream_fair_rbtree_insert(ngx_rbtree_node_t *temp, 301 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) { 302 | 303 | ngx_rbtree_generic_insert(temp, node, sentinel, 304 | ngx_http_upstream_fair_compare_rbtree_node); 305 | } 306 | 307 | 308 | static ngx_int_t 309 | ngx_http_upstream_fair_init_shm_zone(ngx_shm_zone_t *shm_zone, void *data) 310 | { 311 | ngx_slab_pool_t *shpool; 312 | ngx_rbtree_t *tree; 313 | ngx_rbtree_node_t *sentinel; 314 | 315 | if (data) { 316 | shm_zone->data = data; 317 | return NGX_OK; 318 | } 319 | 320 | shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; 321 | tree = ngx_slab_alloc(shpool, sizeof *tree); 322 | if (tree == NULL) { 323 | return NGX_ERROR; 324 | } 325 | 326 | sentinel = ngx_slab_alloc(shpool, sizeof *sentinel); 327 | if (sentinel == NULL) { 328 | return NGX_ERROR; 329 | } 330 | 331 | ngx_rbtree_sentinel_init(sentinel); 332 | tree->root = sentinel; 333 | tree->sentinel = sentinel; 334 | tree->insert = ngx_http_upstream_fair_rbtree_insert; 335 | shm_zone->data = tree; 336 | ngx_http_upstream_fair_rbtree = tree; 337 | 338 | return NGX_OK; 339 | } 340 | 341 | 342 | static char * 343 | ngx_http_upstream_fair_set_shm_size(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 344 | { 345 | ssize_t new_shm_size; 346 | ngx_str_t *value; 347 | 348 | value = cf->args->elts; 349 | 350 | new_shm_size = ngx_parse_size(&value[1]); 351 | if (new_shm_size == NGX_ERROR) { 352 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "Invalid memory area size `%V'", &value[1]); 353 | return NGX_CONF_ERROR; 354 | } 355 | 356 | new_shm_size = ngx_align(new_shm_size, ngx_pagesize); 357 | 358 | if (new_shm_size < 8 * (ssize_t) ngx_pagesize) { 359 | ngx_conf_log_error(NGX_LOG_WARN, cf, 0, "The upstream_fair_shm_size value must be at least %udKiB", (8 * ngx_pagesize) >> 10); 360 | new_shm_size = 8 * ngx_pagesize; 361 | } 362 | 363 | if (ngx_http_upstream_fair_shm_size && 364 | ngx_http_upstream_fair_shm_size != (ngx_uint_t) new_shm_size) { 365 | ngx_conf_log_error(NGX_LOG_WARN, cf, 0, "Cannot change memory area size without restart, ignoring change"); 366 | } else { 367 | ngx_http_upstream_fair_shm_size = new_shm_size; 368 | } 369 | ngx_conf_log_error(NGX_LOG_DEBUG, cf, 0, "Using %udKiB of shared memory for upstream_fair", new_shm_size >> 10); 370 | 371 | return NGX_CONF_OK; 372 | } 373 | 374 | 375 | static char * 376 | ngx_http_upstream_fair(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 377 | { 378 | ngx_http_upstream_srv_conf_t *uscf; 379 | ngx_uint_t i; 380 | ngx_uint_t extra_peer_flags = 0; 381 | 382 | for (i = 1; i < cf->args->nelts; i++) { 383 | ngx_str_t *value = cf->args->elts; 384 | if (ngx_strcmp(value[i].data, "no_rr") == 0) { 385 | extra_peer_flags |= NGX_HTTP_UPSTREAM_FAIR_NO_RR; 386 | } else if (ngx_strcmp(value[i].data, "weight_mode=peak") == 0) { 387 | if (extra_peer_flags & NGX_HTTP_UPSTREAM_FAIR_WEIGHT_MODE_MASK) { 388 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "weight_mode= options are mutually exclusive"); 389 | return NGX_CONF_ERROR; 390 | } 391 | extra_peer_flags |= NGX_HTTP_UPSTREAM_FAIR_WEIGHT_MODE_PEAK; 392 | } else if (ngx_strcmp(value[i].data, "weight_mode=idle") == 0) { 393 | if (extra_peer_flags & NGX_HTTP_UPSTREAM_FAIR_WEIGHT_MODE_MASK) { 394 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "weight_mode= options are mutually exclusive"); 395 | return NGX_CONF_ERROR; 396 | } 397 | extra_peer_flags |= NGX_HTTP_UPSTREAM_FAIR_WEIGHT_MODE_IDLE; 398 | } else { 399 | ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "Invalid `fair' parameter `%V'", &value[i]); 400 | return NGX_CONF_ERROR; 401 | } 402 | } 403 | 404 | uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); 405 | 406 | uscf->peer.init_upstream = ngx_http_upstream_init_fair; 407 | 408 | uscf->flags = NGX_HTTP_UPSTREAM_CREATE 409 | |NGX_HTTP_UPSTREAM_WEIGHT 410 | |NGX_HTTP_UPSTREAM_MAX_FAILS 411 | |NGX_HTTP_UPSTREAM_FAIL_TIMEOUT 412 | |NGX_HTTP_UPSTREAM_DOWN 413 | |extra_peer_flags; 414 | 415 | return NGX_CONF_OK; 416 | } 417 | 418 | 419 | static ngx_int_t 420 | ngx_http_upstream_cmp_servers(const void *one, const void *two) 421 | { 422 | const ngx_http_upstream_fair_peer_t *first, *second; 423 | 424 | first = one; 425 | second = two; 426 | 427 | return (first->weight < second->weight); 428 | } 429 | 430 | 431 | /* TODO: Actually support backup servers */ 432 | static ngx_int_t 433 | ngx_http_upstream_init_fair_rr(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) 434 | { 435 | ngx_url_t u; 436 | ngx_uint_t i, j, n; 437 | ngx_http_upstream_server_t *server; 438 | ngx_http_upstream_fair_peers_t *peers, *backup; 439 | 440 | if (us->servers) { 441 | server = us->servers->elts; 442 | 443 | n = 0; 444 | 445 | for (i = 0; i < us->servers->nelts; i++) { 446 | if (server[i].backup) { 447 | continue; 448 | } 449 | 450 | n += server[i].naddrs; 451 | } 452 | 453 | peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_fair_peers_t) 454 | + sizeof(ngx_http_upstream_fair_peer_t) * (n - 1)); 455 | if (peers == NULL) { 456 | return NGX_ERROR; 457 | } 458 | 459 | peers->number = n; 460 | peers->name = &us->host; 461 | 462 | n = 0; 463 | 464 | for (i = 0; i < us->servers->nelts; i++) { 465 | for (j = 0; j < server[i].naddrs; j++) { 466 | if (server[i].backup) { 467 | continue; 468 | } 469 | 470 | peers->peer[n].sockaddr = server[i].addrs[j].sockaddr; 471 | peers->peer[n].socklen = server[i].addrs[j].socklen; 472 | peers->peer[n].name = server[i].addrs[j].name; 473 | peers->peer[n].max_fails = server[i].max_fails; 474 | peers->peer[n].fail_timeout = server[i].fail_timeout; 475 | peers->peer[n].down = server[i].down; 476 | peers->peer[n].weight = server[i].down ? 0 : server[i].weight; 477 | n++; 478 | } 479 | } 480 | 481 | us->peer.data = peers; 482 | 483 | ngx_sort(&peers->peer[0], (size_t) n, 484 | sizeof(ngx_http_upstream_fair_peer_t), 485 | ngx_http_upstream_cmp_servers); 486 | 487 | /* backup servers */ 488 | 489 | n = 0; 490 | 491 | for (i = 0; i < us->servers->nelts; i++) { 492 | if (!server[i].backup) { 493 | continue; 494 | } 495 | 496 | n += server[i].naddrs; 497 | } 498 | 499 | if (n == 0) { 500 | return NGX_OK; 501 | } 502 | 503 | backup = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_fair_peers_t) 504 | + sizeof(ngx_http_upstream_fair_peer_t) * (n - 1)); 505 | if (backup == NULL) { 506 | return NGX_ERROR; 507 | } 508 | 509 | backup->number = n; 510 | backup->name = &us->host; 511 | 512 | n = 0; 513 | 514 | for (i = 0; i < us->servers->nelts; i++) { 515 | for (j = 0; j < server[i].naddrs; j++) { 516 | if (!server[i].backup) { 517 | continue; 518 | } 519 | 520 | backup->peer[n].sockaddr = server[i].addrs[j].sockaddr; 521 | backup->peer[n].socklen = server[i].addrs[j].socklen; 522 | backup->peer[n].name = server[i].addrs[j].name; 523 | backup->peer[n].weight = server[i].weight; 524 | backup->peer[n].max_fails = server[i].max_fails; 525 | backup->peer[n].fail_timeout = server[i].fail_timeout; 526 | backup->peer[n].down = server[i].down; 527 | n++; 528 | } 529 | } 530 | 531 | peers->next = backup; 532 | 533 | ngx_sort(&backup->peer[0], (size_t) n, 534 | sizeof(ngx_http_upstream_fair_peer_t), 535 | ngx_http_upstream_cmp_servers); 536 | 537 | return NGX_OK; 538 | } 539 | 540 | 541 | /* an upstream implicitly defined by proxy_pass, etc. */ 542 | 543 | if (us->port == 0 && us->default_port == 0) { 544 | ngx_log_error(NGX_LOG_EMERG, cf->log, 0, 545 | "no port in upstream \"%V\" in %s:%ui", 546 | &us->host, us->file_name, us->line); 547 | return NGX_ERROR; 548 | } 549 | 550 | ngx_memzero(&u, sizeof(ngx_url_t)); 551 | 552 | u.host = us->host; 553 | u.port = (in_port_t) (us->port ? us->port : us->default_port); 554 | 555 | if (ngx_inet_resolve_host(cf->pool, &u) != NGX_OK) { 556 | if (u.err) { 557 | ngx_log_error(NGX_LOG_EMERG, cf->log, 0, 558 | "%s in upstream \"%V\" in %s:%ui", 559 | u.err, &us->host, us->file_name, us->line); 560 | } 561 | 562 | return NGX_ERROR; 563 | } 564 | 565 | n = u.naddrs; 566 | 567 | peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_fair_peers_t) 568 | + sizeof(ngx_http_upstream_fair_peer_t) * (n - 1)); 569 | if (peers == NULL) { 570 | return NGX_ERROR; 571 | } 572 | 573 | peers->number = n; 574 | peers->name = &us->host; 575 | 576 | for (i = 0; i < u.naddrs; i++) { 577 | peers->peer[i].sockaddr = u.addrs[i].sockaddr; 578 | peers->peer[i].socklen = u.addrs[i].socklen; 579 | peers->peer[i].name = u.addrs[i].name; 580 | peers->peer[i].weight = 1; 581 | peers->peer[i].max_fails = 1; 582 | peers->peer[i].fail_timeout = 10; 583 | } 584 | 585 | us->peer.data = peers; 586 | 587 | /* implicitly defined upstream has no backup servers */ 588 | 589 | return NGX_OK; 590 | } 591 | 592 | static ngx_int_t 593 | ngx_http_upstream_init_fair(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) 594 | { 595 | ngx_http_upstream_fair_peers_t *peers; 596 | ngx_uint_t n; 597 | ngx_str_t *shm_name; 598 | 599 | /* do the dirty work using rr module */ 600 | if (ngx_http_upstream_init_fair_rr(cf, us) != NGX_OK) { 601 | return NGX_ERROR; 602 | } 603 | 604 | /* setup our wrapper around rr */ 605 | peers = ngx_palloc(cf->pool, sizeof *peers); 606 | if (peers == NULL) { 607 | return NGX_ERROR; 608 | } 609 | peers = us->peer.data; 610 | n = peers->number; 611 | 612 | shm_name = ngx_palloc(cf->pool, sizeof *shm_name); 613 | shm_name->len = sizeof("upstream_fair") - 1; 614 | shm_name->data = (unsigned char *) "upstream_fair"; 615 | 616 | if (ngx_http_upstream_fair_shm_size == 0) { 617 | ngx_http_upstream_fair_shm_size = 8 * ngx_pagesize; 618 | } 619 | 620 | ngx_http_upstream_fair_shm_zone = ngx_shared_memory_add( 621 | cf, shm_name, ngx_http_upstream_fair_shm_size, &ngx_http_upstream_fair_module); 622 | if (ngx_http_upstream_fair_shm_zone == NULL) { 623 | return NGX_ERROR; 624 | } 625 | ngx_http_upstream_fair_shm_zone->init = ngx_http_upstream_fair_init_shm_zone; 626 | 627 | peers->shared = NULL; 628 | peers->current = n - 1; 629 | if (us->flags & NGX_HTTP_UPSTREAM_FAIR_NO_RR) { 630 | peers->no_rr = 1; 631 | } 632 | if (us->flags & NGX_HTTP_UPSTREAM_FAIR_WEIGHT_MODE_IDLE) { 633 | peers->weight_mode = WM_IDLE; 634 | } else if (us->flags & NGX_HTTP_UPSTREAM_FAIR_WEIGHT_MODE_PEAK) { 635 | peers->weight_mode = WM_PEAK; 636 | } 637 | peers->size_err = 0; 638 | 639 | us->peer.init = ngx_http_upstream_init_fair_peer; 640 | 641 | return NGX_OK; 642 | } 643 | 644 | 645 | static void 646 | ngx_http_upstream_fair_update_nreq(ngx_http_upstream_fair_peer_data_t *fp, int delta, ngx_log_t *log) 647 | { 648 | #if (NGX_DEBUG) 649 | ngx_uint_t nreq; 650 | ngx_uint_t total_nreq; 651 | 652 | nreq = (fp->peers->peer[fp->current].shared->nreq += delta); 653 | total_nreq = (fp->peers->shared->total_nreq += delta); 654 | 655 | ngx_log_debug6(NGX_LOG_DEBUG_HTTP, log, 0, 656 | "[upstream_fair] nreq for peer %ui @ %p/%p now %d, total %d, delta %d", 657 | fp->current, fp->peers, fp->peers->peer[fp->current].shared, nreq, 658 | total_nreq, delta); 659 | #endif 660 | } 661 | 662 | /* 663 | * SCHED_COUNTER_BITS is the portion of an ngx_uint_t which represents 664 | * the req_delta part (number of requests serviced on _other_ 665 | * backends). The rest (top bits) represents the number of currently 666 | * processed requests. 667 | * 668 | * The value is not too critical because overflow is handled via 669 | * saturation. With the default value of 20, scheduling is exact for 670 | * fewer than 4k concurrent requests per backend (on 32-bit 671 | * architectures) and fewer than 1M concurrent requests to all backends 672 | * together. Beyond these limits, the algorithm essentially falls back 673 | * to pure weighted round-robin. 674 | * 675 | * A higher score means less suitable. 676 | * 677 | * The `delta' parameter is bit-negated so that high values yield low 678 | * scores and get chosen more often. 679 | */ 680 | 681 | #define SCHED_COUNTER_BITS 20 682 | #define SCHED_NREQ_MAX ((~0UL) >> SCHED_COUNTER_BITS) 683 | #define SCHED_COUNTER_MAX ((1 << SCHED_COUNTER_BITS) - 1) 684 | #define SCHED_SCORE(nreq,delta) (((nreq) << SCHED_COUNTER_BITS) | (~(delta) & SCHED_COUNTER_MAX)) 685 | #define ngx_upstream_fair_min(a,b) (((a) < (b)) ? (a) : (b)) 686 | 687 | static ngx_uint_t 688 | ngx_http_upstream_fair_sched_score(ngx_peer_connection_t *pc, 689 | ngx_http_upstream_fair_peer_data_t *fp, 690 | ngx_uint_t n) 691 | { 692 | ngx_http_upstream_fair_peer_t *peer = &fp->peers->peer[n]; 693 | ngx_http_upstream_fair_shared_t *fs = peer->shared; 694 | ngx_uint_t req_delta = fp->peers->shared->total_requests - fs->last_req_id; 695 | 696 | /* sanity check */ 697 | if ((ngx_int_t)fs->nreq < 0) { 698 | ngx_log_error(NGX_LOG_WARN, pc->log, 0, "[upstream_fair] upstream %ui has negative nreq (%i)", n, fs->nreq); 699 | return SCHED_SCORE(0, req_delta); 700 | } 701 | 702 | ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] peer %ui: nreq = %i, req_delta = %ui", n, fs->nreq, req_delta); 703 | 704 | return SCHED_SCORE( 705 | ngx_upstream_fair_min(fs->nreq, SCHED_NREQ_MAX), 706 | ngx_upstream_fair_min(req_delta, SCHED_COUNTER_MAX)); 707 | } 708 | 709 | /* 710 | * the core of load balancing logic 711 | */ 712 | 713 | static ngx_int_t 714 | ngx_http_upstream_fair_try_peer(ngx_peer_connection_t *pc, 715 | ngx_http_upstream_fair_peer_data_t *fp, 716 | ngx_uint_t peer_id) 717 | { 718 | ngx_http_upstream_fair_peer_t *peer; 719 | 720 | if (ngx_bitvector_test(fp->tried, peer_id)) 721 | return NGX_BUSY; 722 | 723 | peer = &fp->peers->peer[peer_id]; 724 | 725 | if (!peer->down) { 726 | if (peer->max_fails == 0 || peer->shared->fails < peer->max_fails) { 727 | return NGX_OK; 728 | } 729 | 730 | if (ngx_time() - peer->accessed > peer->fail_timeout) { 731 | ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] resetting fail count for peer %d, time delta %d > %d", 732 | peer_id, ngx_time() - peer->accessed, peer->fail_timeout); 733 | peer->shared->fails = 0; 734 | return NGX_OK; 735 | } 736 | } 737 | 738 | return NGX_BUSY; 739 | } 740 | 741 | static ngx_uint_t 742 | ngx_http_upstream_choose_fair_peer_idle(ngx_peer_connection_t *pc, 743 | ngx_http_upstream_fair_peer_data_t *fp) 744 | { 745 | ngx_uint_t i, n; 746 | ngx_uint_t npeers = fp->peers->number; 747 | ngx_uint_t weight_mode = fp->peers->weight_mode; 748 | ngx_uint_t best_idx = NGX_PEER_INVALID; 749 | ngx_uint_t best_nreq = ~0U; 750 | 751 | for (i = 0, n = fp->current; i < npeers; i++, n = (n + 1) % npeers) { 752 | ngx_uint_t nreq = fp->peers->peer[n].shared->nreq; 753 | ngx_uint_t weight = fp->peers->peer[n].weight; 754 | 755 | if (fp->peers->peer[n].shared->fails > 0) 756 | continue; 757 | 758 | if (nreq >= weight || (nreq > 0 && weight_mode != WM_IDLE)) { 759 | continue; 760 | } 761 | 762 | if (ngx_http_upstream_fair_try_peer(pc, fp, n) != NGX_OK) { 763 | continue; 764 | } 765 | 766 | /* not in WM_IDLE+no_rr mode: the first completely idle backend gets chosen */ 767 | if (weight_mode != WM_IDLE || !fp->peers->no_rr) { 768 | best_idx = n; 769 | break; 770 | } 771 | 772 | /* in WM_IDLE+no_rr mode we actually prefer slightly loaded backends 773 | * to totally idle ones, under the assumption that they're spawned 774 | * on demand and can handle up to 'weight' concurrent requests 775 | */ 776 | if (best_idx == NGX_PEER_INVALID || nreq) { 777 | if (best_nreq <= nreq) { 778 | continue; 779 | } 780 | best_idx = n; 781 | best_nreq = nreq; 782 | } 783 | } 784 | 785 | return best_idx; 786 | } 787 | 788 | static ngx_int_t 789 | ngx_http_upstream_choose_fair_peer_busy(ngx_peer_connection_t *pc, 790 | ngx_http_upstream_fair_peer_data_t *fp) 791 | { 792 | ngx_uint_t i, n; 793 | ngx_uint_t npeers = fp->peers->number; 794 | ngx_uint_t weight_mode = fp->peers->weight_mode; 795 | ngx_uint_t best_idx = NGX_PEER_INVALID; 796 | ngx_uint_t sched_score; 797 | ngx_uint_t best_sched_score = ~0UL; 798 | 799 | /* 800 | * calculate sched scores for all the peers, choosing the lowest one 801 | */ 802 | for (i = 0, n = fp->current; i < npeers; i++, n = (n + 1) % npeers) { 803 | ngx_http_upstream_fair_peer_t *peer; 804 | ngx_uint_t nreq; 805 | ngx_uint_t weight; 806 | 807 | peer = &fp->peers->peer[n]; 808 | nreq = fp->peers->peer[n].shared->nreq; 809 | 810 | if (weight_mode == WM_PEAK && nreq >= peer->weight) { 811 | ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] backend %d has nreq %ui >= weight %ui in WM_PEAK mode", n, nreq, peer->weight); 812 | continue; 813 | } 814 | 815 | if (ngx_http_upstream_fair_try_peer(pc, fp, n) != NGX_OK) { 816 | if (!pc->tries) { 817 | ngx_log_debug(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] all backends exhausted"); 818 | return NGX_PEER_INVALID; 819 | } 820 | 821 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] backend %d already tried", n); 822 | continue; 823 | } 824 | 825 | sched_score = ngx_http_upstream_fair_sched_score(pc, fp, n); 826 | 827 | if (weight_mode == WM_DEFAULT) { 828 | /* 829 | * take peer weight into account 830 | */ 831 | weight = peer->shared->current_weight; 832 | if (peer->max_fails) { 833 | ngx_uint_t mf = peer->max_fails; 834 | weight = peer->shared->current_weight * (mf - peer->shared->fails) / mf; 835 | } 836 | if (weight > 0) { 837 | sched_score /= weight; 838 | } 839 | ngx_log_debug8(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] bss = %ui, ss = %ui (n = %d, w = %d/%d, f = %d/%d, weight = %d)", 840 | best_sched_score, sched_score, n, peer->shared->current_weight, peer->weight, peer->shared->fails, peer->max_fails, weight); 841 | } 842 | 843 | if (sched_score <= best_sched_score) { 844 | best_idx = n; 845 | best_sched_score = sched_score; 846 | } 847 | } 848 | 849 | return best_idx; 850 | } 851 | 852 | static ngx_int_t 853 | ngx_http_upstream_choose_fair_peer(ngx_peer_connection_t *pc, 854 | ngx_http_upstream_fair_peer_data_t *fp, ngx_uint_t *peer_id) 855 | { 856 | ngx_uint_t npeers; 857 | ngx_uint_t best_idx = NGX_PEER_INVALID; 858 | ngx_uint_t weight_mode; 859 | 860 | npeers = fp->peers->number; 861 | weight_mode = fp->peers->weight_mode; 862 | 863 | /* just a single backend */ 864 | if (npeers == 1) { 865 | *peer_id = 0; 866 | return NGX_OK; 867 | } 868 | 869 | /* any idle backends? */ 870 | best_idx = ngx_http_upstream_choose_fair_peer_idle(pc, fp); 871 | if (best_idx != NGX_PEER_INVALID) { 872 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] peer %i is idle", best_idx); 873 | goto chosen; 874 | } 875 | 876 | /* no idle backends, choose the least loaded one */ 877 | best_idx = ngx_http_upstream_choose_fair_peer_busy(pc, fp); 878 | if (best_idx != NGX_PEER_INVALID) { 879 | goto chosen; 880 | } 881 | 882 | return NGX_BUSY; 883 | 884 | chosen: 885 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] chose peer %i", best_idx); 886 | *peer_id = best_idx; 887 | ngx_bitvector_set(fp->tried, best_idx); 888 | 889 | if (weight_mode == WM_DEFAULT) { 890 | ngx_http_upstream_fair_peer_t *peer = &fp->peers->peer[best_idx]; 891 | 892 | if (peer->shared->current_weight-- == 0) { 893 | peer->shared->current_weight = peer->weight; 894 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] peer %d expired weight, reset to %d", best_idx, peer->weight); 895 | } 896 | } 897 | return NGX_OK; 898 | } 899 | 900 | ngx_int_t 901 | ngx_http_upstream_get_fair_peer(ngx_peer_connection_t *pc, void *data) 902 | { 903 | ngx_int_t ret; 904 | ngx_uint_t peer_id, i; 905 | ngx_http_upstream_fair_peer_data_t *fp = data; 906 | ngx_http_upstream_fair_peer_t *peer; 907 | ngx_atomic_t *lock; 908 | 909 | peer_id = fp->current; 910 | fp->current = (fp->current + 1) % fp->peers->number; 911 | 912 | lock = &fp->peers->shared->lock; 913 | ngx_spinlock(lock, ngx_pid, 1024); 914 | ret = ngx_http_upstream_choose_fair_peer(pc, fp, &peer_id); 915 | ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] fp->current = %d, peer_id = %d, ret = %d", 916 | fp->current, peer_id, ret); 917 | 918 | if (pc) 919 | pc->tries--; 920 | 921 | if (ret == NGX_BUSY) { 922 | for (i = 0; i < fp->peers->number; i++) { 923 | fp->peers->peer[i].shared->fails = 0; 924 | } 925 | 926 | pc->name = fp->peers->name; 927 | fp->current = NGX_PEER_INVALID; 928 | ngx_spinlock_unlock(lock); 929 | return NGX_BUSY; 930 | } 931 | 932 | /* assert(ret == NGX_OK); */ 933 | peer = &fp->peers->peer[peer_id]; 934 | fp->current = peer_id; 935 | if (!fp->peers->no_rr) { 936 | fp->peers->current = peer_id; 937 | } 938 | pc->sockaddr = peer->sockaddr; 939 | pc->socklen = peer->socklen; 940 | pc->name = &peer->name; 941 | 942 | peer->shared->last_req_id = fp->peers->shared->total_requests; 943 | ngx_http_upstream_fair_update_nreq(fp, 1, pc->log); 944 | peer->shared->total_req++; 945 | ngx_spinlock_unlock(lock); 946 | return ret; 947 | } 948 | 949 | 950 | void 951 | ngx_http_upstream_free_fair_peer(ngx_peer_connection_t *pc, void *data, 952 | ngx_uint_t state) 953 | { 954 | ngx_http_upstream_fair_peer_data_t *fp = data; 955 | ngx_http_upstream_fair_peer_t *peer; 956 | ngx_atomic_t *lock; 957 | 958 | ngx_log_debug4(NGX_LOG_DEBUG_HTTP, pc->log, 0, "[upstream_fair] fp->current = %d, state = %ui, pc->tries = %d, pc->data = %p", 959 | fp->current, state, pc->tries, pc->data); 960 | 961 | if (fp->current == NGX_PEER_INVALID) { 962 | return; 963 | } 964 | 965 | lock = &fp->peers->shared->lock; 966 | ngx_spinlock(lock, ngx_pid, 1024); 967 | if (!ngx_bitvector_test(fp->done, fp->current)) { 968 | ngx_bitvector_set(fp->done, fp->current); 969 | ngx_http_upstream_fair_update_nreq(fp, -1, pc->log); 970 | } 971 | 972 | if (fp->peers->number == 1) { 973 | pc->tries = 0; 974 | } 975 | 976 | if (state & NGX_PEER_FAILED) { 977 | peer = &fp->peers->peer[fp->current]; 978 | 979 | peer->shared->fails++; 980 | peer->accessed = ngx_time(); 981 | } 982 | ngx_spinlock_unlock(lock); 983 | } 984 | 985 | /* 986 | * walk through the rbtree, removing old entries and looking for 987 | * a matching one -- compared by (cycle, peers) pair 988 | * 989 | * no attempt at optimisation is made, for two reasons: 990 | * - the tree will be quite small, anyway 991 | * - being called once per worker startup per upstream block, 992 | * this code isn't really the hot path 993 | */ 994 | static ngx_http_upstream_fair_shm_block_t * 995 | ngx_http_upstream_fair_walk_shm( 996 | ngx_slab_pool_t *shpool, 997 | ngx_rbtree_node_t *node, 998 | ngx_rbtree_node_t *sentinel, 999 | ngx_http_upstream_fair_peers_t *peers) 1000 | { 1001 | ngx_http_upstream_fair_shm_block_t *uf_node; 1002 | ngx_http_upstream_fair_shm_block_t *found_node = NULL; 1003 | ngx_http_upstream_fair_shm_block_t *tmp_node; 1004 | 1005 | if (node == sentinel) { 1006 | return NULL; 1007 | } 1008 | 1009 | /* visit left node */ 1010 | if (node->left != sentinel) { 1011 | tmp_node = ngx_http_upstream_fair_walk_shm(shpool, node->left, 1012 | sentinel, peers); 1013 | if (tmp_node) { 1014 | found_node = tmp_node; 1015 | } 1016 | } 1017 | 1018 | /* visit right node */ 1019 | if (node->right != sentinel) { 1020 | tmp_node = ngx_http_upstream_fair_walk_shm(shpool, node->right, 1021 | sentinel, peers); 1022 | if (tmp_node) { 1023 | found_node = tmp_node; 1024 | } 1025 | } 1026 | 1027 | /* visit current node */ 1028 | uf_node = (ngx_http_upstream_fair_shm_block_t *) node; 1029 | if (uf_node->generation != ngx_http_upstream_fair_generation) { 1030 | ngx_spinlock(&uf_node->lock, ngx_pid, 1024); 1031 | if (uf_node->total_nreq == 0) { 1032 | /* don't bother unlocking */ 1033 | ngx_rbtree_delete(ngx_http_upstream_fair_rbtree, node); 1034 | ngx_slab_free_locked(shpool, node); 1035 | } 1036 | ngx_spinlock_unlock(&uf_node->lock); 1037 | } else if (uf_node->peers == (uintptr_t) peers) { 1038 | found_node = uf_node; 1039 | } 1040 | 1041 | return found_node; 1042 | } 1043 | 1044 | static ngx_int_t 1045 | ngx_http_upstream_fair_shm_alloc(ngx_http_upstream_fair_peers_t *usfp, ngx_log_t *log) 1046 | { 1047 | ngx_slab_pool_t *shpool; 1048 | ngx_uint_t i; 1049 | 1050 | if (usfp->shared) { 1051 | return NGX_OK; 1052 | } 1053 | 1054 | shpool = (ngx_slab_pool_t *)ngx_http_upstream_fair_shm_zone->shm.addr; 1055 | 1056 | ngx_shmtx_lock(&shpool->mutex); 1057 | 1058 | usfp->shared = ngx_http_upstream_fair_walk_shm(shpool, 1059 | ngx_http_upstream_fair_rbtree->root, 1060 | ngx_http_upstream_fair_rbtree->sentinel, 1061 | usfp); 1062 | 1063 | if (usfp->shared) { 1064 | ngx_shmtx_unlock(&shpool->mutex); 1065 | return NGX_OK; 1066 | } 1067 | 1068 | usfp->shared = ngx_slab_alloc_locked(shpool, 1069 | sizeof(ngx_http_upstream_fair_shm_block_t) + 1070 | (usfp->number - 1) * sizeof(ngx_http_upstream_fair_shared_t)); 1071 | 1072 | if (!usfp->shared) { 1073 | ngx_shmtx_unlock(&shpool->mutex); 1074 | if (!usfp->size_err) { 1075 | ngx_log_error(NGX_LOG_EMERG, log, 0, 1076 | "upstream_fair_shm_size too small (current value is %udKiB)", 1077 | ngx_http_upstream_fair_shm_size >> 10); 1078 | usfp->size_err = 1; 1079 | } 1080 | return NGX_ERROR; 1081 | } 1082 | 1083 | usfp->shared->node.key = ngx_crc32_short((u_char *) &ngx_cycle, sizeof ngx_cycle) ^ 1084 | ngx_crc32_short((u_char *) &usfp, sizeof(usfp)); 1085 | 1086 | usfp->shared->generation = ngx_http_upstream_fair_generation; 1087 | usfp->shared->peers = (uintptr_t) usfp; 1088 | usfp->shared->total_nreq = 0; 1089 | usfp->shared->total_requests = 0; 1090 | 1091 | for (i = 0; i < usfp->number; i++) { 1092 | usfp->shared->stats[i].nreq = 0; 1093 | usfp->shared->stats[i].last_req_id = 0; 1094 | usfp->shared->stats[i].total_req = 0; 1095 | } 1096 | 1097 | ngx_rbtree_insert(ngx_http_upstream_fair_rbtree, &usfp->shared->node); 1098 | 1099 | ngx_shmtx_unlock(&shpool->mutex); 1100 | return NGX_OK; 1101 | } 1102 | 1103 | ngx_int_t 1104 | ngx_http_upstream_init_fair_peer(ngx_http_request_t *r, 1105 | ngx_http_upstream_srv_conf_t *us) 1106 | { 1107 | ngx_http_upstream_fair_peer_data_t *fp; 1108 | ngx_http_upstream_fair_peers_t *usfp; 1109 | ngx_uint_t n; 1110 | 1111 | fp = r->upstream->peer.data; 1112 | 1113 | if (fp == NULL) { 1114 | fp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_fair_peer_data_t)); 1115 | if (fp == NULL) { 1116 | return NGX_ERROR; 1117 | } 1118 | 1119 | r->upstream->peer.data = fp; 1120 | } 1121 | 1122 | usfp = us->peer.data; 1123 | 1124 | fp->tried = ngx_bitvector_alloc(r->pool, usfp->number, &fp->data); 1125 | fp->done = ngx_bitvector_alloc(r->pool, usfp->number, &fp->data2); 1126 | 1127 | if (fp->tried == NULL || fp->done == NULL) { 1128 | return NGX_ERROR; 1129 | } 1130 | 1131 | /* set up shared memory area */ 1132 | ngx_http_upstream_fair_shm_alloc(usfp, r->connection->log); 1133 | 1134 | fp->current = usfp->current; 1135 | fp->peers = usfp; 1136 | usfp->shared->total_requests++; 1137 | 1138 | for (n = 0; n < usfp->number; n++) { 1139 | usfp->peer[n].shared = &usfp->shared->stats[n]; 1140 | } 1141 | 1142 | r->upstream->peer.get = ngx_http_upstream_get_fair_peer; 1143 | r->upstream->peer.free = ngx_http_upstream_free_fair_peer; 1144 | r->upstream->peer.tries = usfp->number; 1145 | #if (NGX_HTTP_SSL) 1146 | r->upstream->peer.set_session = 1147 | ngx_http_upstream_fair_set_session; 1148 | r->upstream->peer.save_session = 1149 | ngx_http_upstream_fair_save_session; 1150 | #endif 1151 | 1152 | return NGX_OK; 1153 | } 1154 | 1155 | #if (NGX_HTTP_SSL) 1156 | static ngx_int_t 1157 | ngx_http_upstream_fair_set_session(ngx_peer_connection_t *pc, void *data) 1158 | { 1159 | ngx_http_upstream_fair_peer_data_t *fp = data; 1160 | 1161 | ngx_int_t rc; 1162 | ngx_ssl_session_t *ssl_session; 1163 | ngx_http_upstream_fair_peer_t *peer; 1164 | 1165 | if (fp->current == NGX_PEER_INVALID) 1166 | return NGX_OK; 1167 | 1168 | peer = &fp->peers->peer[fp->current]; 1169 | 1170 | /* TODO: threads only mutex */ 1171 | /* ngx_lock_mutex(fp->peers->mutex); */ 1172 | 1173 | ssl_session = peer->ssl_session; 1174 | 1175 | rc = ngx_ssl_set_session(pc->connection, ssl_session); 1176 | 1177 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, 1178 | "set session: %p:%d", 1179 | ssl_session, ssl_session ? ssl_session->references : 0); 1180 | 1181 | /* ngx_unlock_mutex(fp->peers->mutex); */ 1182 | 1183 | return rc; 1184 | } 1185 | 1186 | static void 1187 | ngx_http_upstream_fair_save_session(ngx_peer_connection_t *pc, void *data) 1188 | { 1189 | ngx_http_upstream_fair_peer_data_t *fp = data; 1190 | 1191 | ngx_ssl_session_t *old_ssl_session, *ssl_session; 1192 | ngx_http_upstream_fair_peer_t *peer; 1193 | 1194 | if (fp->current == NGX_PEER_INVALID) 1195 | return; 1196 | 1197 | ssl_session = ngx_ssl_get_session(pc->connection); 1198 | 1199 | if (ssl_session == NULL) { 1200 | return; 1201 | } 1202 | 1203 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, 1204 | "save session: %p:%d", ssl_session, ssl_session->references); 1205 | 1206 | peer = &fp->peers->peer[fp->current]; 1207 | 1208 | /* TODO: threads only mutex */ 1209 | /* ngx_lock_mutex(fp->peers->mutex); */ 1210 | 1211 | old_ssl_session = peer->ssl_session; 1212 | peer->ssl_session = ssl_session; 1213 | 1214 | /* ngx_unlock_mutex(fp->peers->mutex); */ 1215 | 1216 | if (old_ssl_session) { 1217 | 1218 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, 1219 | "old session: %p:%d", 1220 | old_ssl_session, old_ssl_session->references); 1221 | 1222 | /* TODO: may block */ 1223 | 1224 | ngx_ssl_free_session(old_ssl_session); 1225 | } 1226 | } 1227 | 1228 | #endif 1229 | 1230 | #if (NGX_HTTP_EXTENDED_STATUS) 1231 | static void 1232 | ngx_http_upstream_fair_walk_status(ngx_pool_t *pool, ngx_chain_t *cl, ngx_int_t *length, 1233 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) 1234 | { 1235 | ngx_http_upstream_fair_shm_block_t *s_node = (ngx_http_upstream_fair_shm_block_t *) node; 1236 | ngx_http_upstream_fair_peers_t *peers; 1237 | ngx_chain_t *new_cl; 1238 | ngx_buf_t *b; 1239 | ngx_uint_t size, i; 1240 | 1241 | if (node == sentinel) { 1242 | return; 1243 | } 1244 | 1245 | if (node->left != sentinel) { 1246 | ngx_http_upstream_fair_walk_status(pool, cl, length, node->left, sentinel); 1247 | } 1248 | 1249 | if (s_node->generation != ngx_http_upstream_fair_generation) { 1250 | size = 100; 1251 | peers = NULL; 1252 | } else { 1253 | /* this is rather ugly (casting an uintptr_t back into a pointer 1254 | * but as long as the generation is still the same (verified above), 1255 | * it should be still safe 1256 | */ 1257 | peers = (ngx_http_upstream_fair_peers_t *) s_node->peers; 1258 | if (!peers->shared) { 1259 | goto next; 1260 | } 1261 | 1262 | size = 200 + peers->number * 120; /* LOTS of slack */ 1263 | } 1264 | 1265 | b = ngx_create_temp_buf(pool, size); 1266 | if (!b) { 1267 | goto next; 1268 | } 1269 | 1270 | new_cl = ngx_alloc_chain_link(pool); 1271 | if (!new_cl) { 1272 | goto next; 1273 | } 1274 | 1275 | new_cl->buf = b; 1276 | new_cl->next = NULL; 1277 | 1278 | while (cl->next) { 1279 | cl = cl->next; 1280 | } 1281 | cl->next = new_cl; 1282 | 1283 | if (peers) { 1284 | b->last = ngx_sprintf(b->last, "upstream %V (%p): current peer %d/%d, total requests: %ui\n", peers->name, (void*) node, peers->current, peers->number, s_node->total_requests); 1285 | for (i = 0; i < peers->number; i++) { 1286 | ngx_http_upstream_fair_peer_t *peer = &peers->peer[i]; 1287 | ngx_http_upstream_fair_shared_t *sh = peer->shared; 1288 | b->last = ngx_sprintf(b->last, " peer %d: %V weight: %d/%d, fails: %d/%d, acc: %d, down: %d, nreq: %d, total_req: %ui, last_req: %ui\n", 1289 | i, &peer->name, sh->current_weight, peer->weight, sh->fails, peer->max_fails, peer->accessed, peer->down, 1290 | sh->nreq, sh->total_req, sh->last_req_id); 1291 | } 1292 | } else { 1293 | b->last = ngx_sprintf(b->last, "upstream %p: gen %ui != %ui, total_nreq = %ui", (void*) node, s_node->generation, ngx_http_upstream_fair_generation, s_node->total_nreq); 1294 | } 1295 | b->last = ngx_sprintf(b->last, "\n"); 1296 | b->last_buf = 1; 1297 | 1298 | *length += b->last - b->pos; 1299 | 1300 | if (cl->buf) { 1301 | cl->buf->last_buf = 0; 1302 | } 1303 | 1304 | cl = cl->next; 1305 | next: 1306 | 1307 | if (node->right != sentinel) { 1308 | ngx_http_upstream_fair_walk_status(pool, cl, length, node->right, sentinel); 1309 | } 1310 | } 1311 | 1312 | static ngx_chain_t* 1313 | ngx_http_upstream_fair_report_status(ngx_http_request_t *r, ngx_int_t *length) 1314 | { 1315 | ngx_buf_t *b; 1316 | ngx_chain_t *cl; 1317 | ngx_slab_pool_t *shpool; 1318 | 1319 | b = ngx_create_temp_buf(r->pool, sizeof("\nupstream_fair status report:\n")); 1320 | if (!b) { 1321 | return NULL; 1322 | } 1323 | 1324 | cl = ngx_alloc_chain_link(r->pool); 1325 | if (!cl) { 1326 | return NULL; 1327 | } 1328 | cl->next = NULL; 1329 | cl->buf = b; 1330 | 1331 | b->last = ngx_cpymem(b->last, "\nupstream_fair status report:\n", 1332 | sizeof("\nupstream_fair status report:\n") - 1); 1333 | 1334 | *length = b->last - b->pos; 1335 | 1336 | shpool = (ngx_slab_pool_t *)ngx_http_upstream_fair_shm_zone->shm.addr; 1337 | 1338 | ngx_shmtx_lock(&shpool->mutex); 1339 | 1340 | ngx_http_upstream_fair_walk_status(r->pool, cl, 1341 | length, 1342 | ngx_http_upstream_fair_rbtree->root, 1343 | ngx_http_upstream_fair_rbtree->sentinel); 1344 | 1345 | ngx_shmtx_unlock(&shpool->mutex); 1346 | 1347 | if (!cl->next || !cl->next->buf) { 1348 | /* no upstream_fair status to report */ 1349 | return NULL; 1350 | } 1351 | 1352 | return cl; 1353 | } 1354 | #endif 1355 | 1356 | /* vim: set et ts=4 sw=4: */ 1357 | --------------------------------------------------------------------------------