├── README.md ├── nginx-dyntls-1.11.5.diff ├── nginx__dynamic_tls_records.patch ├── nginx__dynamic_tls_records_1.13.0+.patch ├── nginx__dynamic_tls_records_1.15.3+.patch ├── nginx__dynamic_tls_records_1.15.5+.patch ├── nginx__dynamic_tls_records_1.17.7+.patch ├── nginx__dynamic_tls_records_1.25.1+.patch ├── nginx__dynamic_tls_records_1.27.2+.patch ├── nginx__dynamic_tls_records_1.27.4+.patch └── nginx__dynamic_tls_records_1.27.5+.patch /README.md: -------------------------------------------------------------------------------- 1 | # Optimizing TLS over TCP to reduce latency for NGINX 2 | 3 | * [`nginx__dynamic_tls_records`](https://github.com/cloudflare/sslconfig/blob/3e45b99/patches/) 4 | * [`Optimizing HTTP/2 prioritization with BBR and tcp_notsent_lowat`](https://blog.cloudflare.com/http-2-prioritization-with-nginx/) 5 | 6 | ### What we do now 7 | 8 | We use a static record size of 4K. 9 | This gives a good balance of latency and throughput. 10 | 11 | #### Configuration 12 | 13 | *Example* 14 | 15 | ```nginx 16 | http { 17 | ssl_dyn_rec_enable on; 18 | } 19 | ``` 20 | 21 | #### Optimize latency 22 | 23 | By initialy sending small (1 TCP segment) sized records, 24 | we are able to avoid HoL blocking of the first byte. 25 | This means TTFB is sometime lower by a whole RTT. 26 | 27 | #### Optimizing throughput 28 | 29 | By sending increasingly larger records later in the connection, 30 | when HoL is not a problem, we reduce the overhead of TLS record 31 | (29 bytes per record with GCM/CHACHA-POLY). 32 | 33 | #### Logic 34 | 35 | Start each connection with small records 36 | (1369 byte default, change with `ssl_dyn_rec_size_lo`). 37 | 38 | After a given number of records (40, change with `ssl_dyn_rec_threshold`) 39 | start sending larger records (4229, `ssl_dyn_rec_size_hi`). 40 | 41 | Eventually after the same number of records, 42 | start sending the largest records (`ssl_buffer_size`). 43 | 44 | In case the connection idles for a given amount of time 45 | (1s, `ssl_dyn_rec_timeout`), the process repeats itself 46 | (i.e. begin sending small records again). 47 | 48 | ### Configuration directives 49 | 50 | #### ssl_dyn_rec_enable 51 | * **syntax**: `ssl_dyn_rec_enable bool` 52 | * **default**: `off` 53 | * **context**: `http`, `server` 54 | 55 | #### ssl_dyn_rec_timeout 56 | * **syntax**: `ssl_dyn_rec_timeout number` 57 | * **default**: `1000` 58 | * **context**: `http`, `server` 59 | 60 | We want the initial records to fit into one TCP segment 61 | so we don't get TCP HoL blocking due to TCP Slow Start. 62 | 63 | A connection always starts with small records, but after 64 | a given amount of records sent, we make the records larger 65 | to reduce header overhead. 66 | 67 | After a connection has idled for a given timeout, begin 68 | the process from the start. The actual parameters are 69 | configurable. If `ssl_dyn_rec_timeout` is `0`, we assume `ssl_dyn_rec` is `off`. 70 | 71 | #### ssl_dyn_rec_size_lo 72 | * **syntax**: `ssl_dyn_rec_size_lo number` 73 | * **default**: `1369` 74 | * **context**: `http`, `server` 75 | 76 | Default sizes for the dynamic record sizes are defined to fit maximal 77 | TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 78 | 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) 79 | 80 | #### ssl_dyn_rec_size_hi 81 | * **syntax**: `ssl_dyn_rec_size_hi number` 82 | * **default**: `4229` 83 | * **context**: `http`, `server` 84 | 85 | 4229 = (1500 - 40 - 20 - 10) * 3 - 61 86 | 87 | #### ssl_dyn_rec_threshold 88 | * **syntax**: `ssl_dyn_rec_threshold number` 89 | * **default**: `40` 90 | * **context**: `http`, `server` 91 | 92 | ### License 93 | 94 | * [Cloudflare](https://github.com/cloudflare), [Vlad Krasnov](https://github.com/vkrasnov) 95 | -------------------------------------------------------------------------------- /nginx-dyntls-1.11.5.diff: -------------------------------------------------------------------------------- 1 | diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c 2 | index 68d02bf..2b24542 100644 3 | --- a/src/event/ngx_event_openssl.c 4 | +++ b/src/event/ngx_event_openssl.c 5 | @@ -1131,6 +1131,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) 6 | 7 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 8 | sc->buffer_size = ssl->buffer_size; 9 | + sc->dyn_rec = ssl->dyn_rec; 10 | 11 | sc->session_ctx = ssl->ctx; 12 | 13 | @@ -1669,6 +1670,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) 14 | 15 | for ( ;; ) { 16 | 17 | + /* Dynamic record resizing: 18 | + We want the initial records to fit into one TCP segment 19 | + so we don't get TCP HoL blocking due to TCP Slow Start. 20 | + A connection always starts with small records, but after 21 | + a given amount of records sent, we make the records larger 22 | + to reduce header overhead. 23 | + After a connection has idled for a given timeout, begin 24 | + the process from the start. The actual parameters are 25 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 26 | + 27 | + if (c->ssl->dyn_rec.timeout > 0 ) { 28 | + 29 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 30 | + c->ssl->dyn_rec.timeout) 31 | + { 32 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 33 | + c->ssl->dyn_rec_records_sent = 0; 34 | + 35 | + } else { 36 | + if (c->ssl->dyn_rec_records_sent > 37 | + c->ssl->dyn_rec.threshold * 2) 38 | + { 39 | + buf->end = buf->start + c->ssl->buffer_size; 40 | + 41 | + } else if (c->ssl->dyn_rec_records_sent > 42 | + c->ssl->dyn_rec.threshold) 43 | + { 44 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 45 | + 46 | + } else { 47 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 48 | + } 49 | + } 50 | + } 51 | + 52 | while (in && buf->last < buf->end && send < limit) { 53 | if (in->buf->last_buf || in->buf->flush) { 54 | flush = 1; 55 | @@ -1770,6 +1806,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) 56 | 57 | if (n > 0) { 58 | 59 | + c->ssl->dyn_rec_records_sent++; 60 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 61 | + 62 | if (c->ssl->saved_read_handler) { 63 | 64 | c->read->handler = c->ssl->saved_read_handler; 65 | diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h 66 | index 24b812f..504e943 100644 67 | --- a/src/event/ngx_event_openssl.h 68 | +++ b/src/event/ngx_event_openssl.h 69 | @@ -53,11 +53,18 @@ 70 | #define ngx_ssl_session_t SSL_SESSION 71 | #define ngx_ssl_conn_t SSL 72 | 73 | +typedef struct { 74 | + ngx_msec_t timeout; 75 | + ngx_uint_t threshold; 76 | + size_t size_lo; 77 | + size_t size_hi; 78 | +} ngx_ssl_dyn_rec_t; 79 | 80 | struct ngx_ssl_s { 81 | SSL_CTX *ctx; 82 | ngx_log_t *log; 83 | size_t buffer_size; 84 | + ngx_ssl_dyn_rec_t dyn_rec; 85 | }; 86 | 87 | 88 | @@ -80,6 +87,10 @@ struct ngx_ssl_connection_s { 89 | unsigned no_wait_shutdown:1; 90 | unsigned no_send_shutdown:1; 91 | unsigned handshake_buffer_set:1; 92 | + 93 | + ngx_ssl_dyn_rec_t dyn_rec; 94 | + ngx_msec_t dyn_rec_last_write; 95 | + ngx_uint_t dyn_rec_records_sent; 96 | }; 97 | 98 | 99 | @@ -89,7 +100,7 @@ struct ngx_ssl_connection_s { 100 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 101 | 102 | 103 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 104 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 105 | 106 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 107 | 108 | diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c 109 | index d685ae9..2362632 100644 110 | --- a/src/http/modules/ngx_http_ssl_module.c 111 | +++ b/src/http/modules/ngx_http_ssl_module.c 112 | @@ -233,6 +233,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { 113 | offsetof(ngx_http_ssl_srv_conf_t, stapling_verify), 114 | NULL }, 115 | 116 | + { ngx_string("ssl_dyn_rec_enable"), 117 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 118 | + ngx_conf_set_flag_slot, 119 | + NGX_HTTP_SRV_CONF_OFFSET, 120 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 121 | + NULL }, 122 | + 123 | + { ngx_string("ssl_dyn_rec_timeout"), 124 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 125 | + ngx_conf_set_msec_slot, 126 | + NGX_HTTP_SRV_CONF_OFFSET, 127 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 128 | + NULL }, 129 | + 130 | + { ngx_string("ssl_dyn_rec_size_lo"), 131 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 132 | + ngx_conf_set_size_slot, 133 | + NGX_HTTP_SRV_CONF_OFFSET, 134 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 135 | + NULL }, 136 | + 137 | + { ngx_string("ssl_dyn_rec_size_hi"), 138 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 139 | + ngx_conf_set_size_slot, 140 | + NGX_HTTP_SRV_CONF_OFFSET, 141 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 142 | + NULL }, 143 | + 144 | + { ngx_string("ssl_dyn_rec_threshold"), 145 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 146 | + ngx_conf_set_num_slot, 147 | + NGX_HTTP_SRV_CONF_OFFSET, 148 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 149 | + NULL }, 150 | + 151 | ngx_null_command 152 | }; 153 | 154 | @@ -533,6 +568,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) 155 | sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; 156 | sscf->stapling = NGX_CONF_UNSET; 157 | sscf->stapling_verify = NGX_CONF_UNSET; 158 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 159 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 160 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 161 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 162 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 163 | 164 | return sscf; 165 | } 166 | @@ -598,6 +638,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 167 | ngx_conf_merge_str_value(conf->stapling_responder, 168 | prev->stapling_responder, ""); 169 | 170 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 171 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 172 | + 1000); 173 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 174 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 175 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 176 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 177 | + 1369); 178 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 179 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 180 | + 4229); 181 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 182 | + 40); 183 | + 184 | conf->ssl.log = cf->log; 185 | 186 | if (conf->enable) { 187 | @@ -778,6 +832,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 188 | 189 | } 190 | 191 | + if (conf->dyn_rec_enable) { 192 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 193 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 194 | + 195 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 196 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 197 | + 198 | + } else { 199 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 200 | + } 201 | + 202 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 203 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 204 | + 205 | + } else { 206 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 207 | + } 208 | + 209 | + } else { 210 | + conf->ssl.dyn_rec.timeout = 0; 211 | + } 212 | + 213 | return NGX_CONF_OK; 214 | } 215 | 216 | diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h 217 | index 57f5941..309ac00 100644 218 | --- a/src/http/modules/ngx_http_ssl_module.h 219 | +++ b/src/http/modules/ngx_http_ssl_module.h 220 | @@ -57,6 +57,12 @@ typedef struct { 221 | 222 | u_char *file; 223 | ngx_uint_t line; 224 | + 225 | + ngx_flag_t dyn_rec_enable; 226 | + ngx_msec_t dyn_rec_timeout; 227 | + size_t dyn_rec_size_lo; 228 | + size_t dyn_rec_size_hi; 229 | + ngx_uint_t dyn_rec_threshold; 230 | } ngx_http_ssl_srv_conf_t; 231 | 232 | 233 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records.patch: -------------------------------------------------------------------------------- 1 | From a424fefb0a638eb6d32756b5a0c471efc63e5384 Mon Sep 17 00:00:00 2001 2 | From: Vlad Krasnov 3 | Date: Sat, 9 Jan 2016 06:53:14 -0800 4 | Subject: [PATCH] - Add TLS Dynamic Record Resizing 5 | 6 | What we do now: 7 | We use a static record size of 4K. This gives a good balance of latency and 8 | throughput. 9 | 10 | Optimize latency: 11 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 12 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 13 | RTT. 14 | 15 | Optimizing throughput: 16 | By sending increasingly larger records later in the connection, when HoL is not 17 | a problem, we reduce the overhead of TLS record (29 bytes per record with 18 | GCM/CHACHA-POLY). 19 | 20 | Logic: 21 | Start each connection with small records (1369 byte default, change with 22 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 23 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 24 | Eventually after the same number of records, start sending the largest records 25 | (ssl_buffer_size). 26 | In case the connection idles for a given amount of time (1s, 27 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 28 | records again). 29 | 30 | --- 31 | src/event/ngx_event_openssl.c | 39 +++++++++++++++++ 32 | src/event/ngx_event_openssl.h | 15 ++++++- 33 | src/http/modules/ngx_http_ssl_module.c | 76 ++++++++++++++++++++++++++++++++++ 34 | src/http/modules/ngx_http_ssl_module.h | 6 +++ 35 | 4 files changed, 135 insertions(+), 1 deletion(-) 36 | 37 | diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c 38 | index 57dfc6c..4a0d41a 100644 39 | --- a/src/event/ngx_event_openssl.c 40 | +++ b/src/event/ngx_event_openssl.c 41 | @@ -1037,6 +1037,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) 42 | 43 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 44 | sc->buffer_size = ssl->buffer_size; 45 | + sc->dyn_rec = ssl->dyn_rec; 46 | 47 | sc->session_ctx = ssl->ctx; 48 | 49 | @@ -1575,6 +1576,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) 50 | 51 | for ( ;; ) { 52 | 53 | + /* Dynamic record resizing: 54 | + We want the initial records to fit into one TCP segment 55 | + so we don't get TCP HoL blocking due to TCP Slow Start. 56 | + A connection always starts with small records, but after 57 | + a given amount of records sent, we make the records larger 58 | + to reduce header overhead. 59 | + After a connection has idled for a given timeout, begin 60 | + the process from the start. The actual parameters are 61 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 62 | + 63 | + if (c->ssl->dyn_rec.timeout > 0 ) { 64 | + 65 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 66 | + c->ssl->dyn_rec.timeout) 67 | + { 68 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 69 | + c->ssl->dyn_rec_records_sent = 0; 70 | + 71 | + } else { 72 | + if (c->ssl->dyn_rec_records_sent > 73 | + c->ssl->dyn_rec.threshold * 2) 74 | + { 75 | + buf->end = buf->start + c->ssl->buffer_size; 76 | + 77 | + } else if (c->ssl->dyn_rec_records_sent > 78 | + c->ssl->dyn_rec.threshold) 79 | + { 80 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 81 | + 82 | + } else { 83 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 84 | + } 85 | + } 86 | + } 87 | + 88 | while (in && buf->last < buf->end && send < limit) { 89 | if (in->buf->last_buf || in->buf->flush) { 90 | flush = 1; 91 | @@ -1676,6 +1712,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) 92 | 93 | if (n > 0) { 94 | 95 | + c->ssl->dyn_rec_records_sent++; 96 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 97 | + 98 | if (c->ssl->saved_read_handler) { 99 | 100 | c->read->handler = c->ssl->saved_read_handler; 101 | diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h 102 | index c86be2a..4a45934 100644 103 | --- a/src/event/ngx_event_openssl.h 104 | +++ b/src/event/ngx_event_openssl.h 105 | @@ -38,9 +38,18 @@ 106 | 107 | 108 | typedef struct { 109 | + ngx_msec_t timeout; 110 | + ngx_uint_t threshold; 111 | + size_t size_lo; 112 | + size_t size_hi; 113 | +} ngx_ssl_dyn_rec_t; 114 | + 115 | + 116 | +typedef struct { 117 | SSL_CTX *ctx; 118 | ngx_log_t *log; 119 | size_t buffer_size; 120 | + ngx_ssl_dyn_rec_t dyn_rec; 121 | } ngx_ssl_t; 122 | 123 | 124 | @@ -63,6 +72,10 @@ typedef struct { 125 | unsigned no_wait_shutdown:1; 126 | unsigned no_send_shutdown:1; 127 | unsigned handshake_buffer_set:1; 128 | + 129 | + ngx_ssl_dyn_rec_t dyn_rec; 130 | + ngx_msec_t dyn_rec_last_write; 131 | + ngx_uint_t dyn_rec_records_sent; 132 | } ngx_ssl_connection_t; 133 | 134 | 135 | @@ -72,7 +85,7 @@ typedef struct { 136 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 137 | 138 | 139 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 140 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 141 | 142 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 143 | 144 | diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c 145 | index 7b051ea..e2941af 100644 146 | --- a/src/http/modules/ngx_http_ssl_module.c 147 | +++ b/src/http/modules/ngx_http_ssl_module.c 148 | @@ -233,6 +233,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { 149 | offsetof(ngx_http_ssl_srv_conf_t, stapling_verify), 150 | NULL }, 151 | 152 | + { ngx_string("ssl_dyn_rec_enable"), 153 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 154 | + ngx_conf_set_flag_slot, 155 | + NGX_HTTP_SRV_CONF_OFFSET, 156 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 157 | + NULL }, 158 | + 159 | + { ngx_string("ssl_dyn_rec_timeout"), 160 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 161 | + ngx_conf_set_msec_slot, 162 | + NGX_HTTP_SRV_CONF_OFFSET, 163 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 164 | + NULL }, 165 | + 166 | + { ngx_string("ssl_dyn_rec_size_lo"), 167 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 168 | + ngx_conf_set_size_slot, 169 | + NGX_HTTP_SRV_CONF_OFFSET, 170 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 171 | + NULL }, 172 | + 173 | + { ngx_string("ssl_dyn_rec_size_hi"), 174 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 175 | + ngx_conf_set_size_slot, 176 | + NGX_HTTP_SRV_CONF_OFFSET, 177 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 178 | + NULL }, 179 | + 180 | + { ngx_string("ssl_dyn_rec_threshold"), 181 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 182 | + ngx_conf_set_num_slot, 183 | + NGX_HTTP_SRV_CONF_OFFSET, 184 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 185 | + NULL }, 186 | + 187 | ngx_null_command 188 | }; 189 | 190 | @@ -532,6 +567,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) 191 | sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; 192 | sscf->stapling = NGX_CONF_UNSET; 193 | sscf->stapling_verify = NGX_CONF_UNSET; 194 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 195 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 196 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 197 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 198 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 199 | 200 | return sscf; 201 | } 202 | @@ -596,6 +636,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 203 | ngx_conf_merge_str_value(conf->stapling_responder, 204 | prev->stapling_responder, ""); 205 | 206 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 207 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 208 | + 1000); 209 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 210 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 211 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 212 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 213 | + 1369); 214 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 215 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 216 | + 4229); 217 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 218 | + 40); 219 | + 220 | conf->ssl.log = cf->log; 221 | 222 | if (conf->enable) { 223 | @@ -773,6 +827,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 224 | 225 | } 226 | 227 | + if (conf->dyn_rec_enable) { 228 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 229 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 230 | + 231 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 232 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 233 | + 234 | + } else { 235 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 236 | + } 237 | + 238 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 239 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 240 | + 241 | + } else { 242 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 243 | + } 244 | + 245 | + } else { 246 | + conf->ssl.dyn_rec.timeout = 0; 247 | + } 248 | + 249 | return NGX_CONF_OK; 250 | } 251 | 252 | diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h 253 | index 8e69e9e..05967d4 100644 254 | --- a/src/http/modules/ngx_http_ssl_module.h 255 | +++ b/src/http/modules/ngx_http_ssl_module.h 256 | @@ -56,6 +56,12 @@ typedef struct { 257 | 258 | u_char *file; 259 | ngx_uint_t line; 260 | + 261 | + ngx_flag_t dyn_rec_enable; 262 | + ngx_msec_t dyn_rec_timeout; 263 | + size_t dyn_rec_size_lo; 264 | + size_t dyn_rec_size_hi; 265 | + ngx_uint_t dyn_rec_threshold; 266 | } ngx_http_ssl_srv_conf_t; 267 | 268 | 269 | -- 270 | 2.7.4 (Apple Git-66) 271 | 272 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records_1.13.0+.patch: -------------------------------------------------------------------------------- 1 | What we do now: 2 | We use a static record size of 4K. This gives a good balance of latency and 3 | throughput. 4 | 5 | Optimize latency: 6 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 7 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 8 | RTT. 9 | 10 | Optimizing throughput: 11 | By sending increasingly larger records later in the connection, when HoL is not 12 | a problem, we reduce the overhead of TLS record (29 bytes per record with 13 | GCM/CHACHA-POLY). 14 | 15 | Logic: 16 | Start each connection with small records (1369 byte default, change with 17 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 18 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 19 | Eventually after the same number of records, start sending the largest records 20 | (ssl_buffer_size). 21 | In case the connection idles for a given amount of time (1s, 22 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 23 | records again). 24 | 25 | Upstream source: 26 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 27 | 28 | --- a/src/event/ngx_event_openssl.c 29 | +++ b/src/event/ngx_event_openssl.c 30 | @@ -1131,6 +1131,7 @@ 31 | 32 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 33 | sc->buffer_size = ssl->buffer_size; 34 | + sc->dyn_rec = ssl->dyn_rec; 35 | 36 | sc->session_ctx = ssl->ctx; 37 | 38 | @@ -1669,6 +1670,41 @@ 39 | 40 | for ( ;; ) { 41 | 42 | + /* Dynamic record resizing: 43 | + We want the initial records to fit into one TCP segment 44 | + so we don't get TCP HoL blocking due to TCP Slow Start. 45 | + A connection always starts with small records, but after 46 | + a given amount of records sent, we make the records larger 47 | + to reduce header overhead. 48 | + After a connection has idled for a given timeout, begin 49 | + the process from the start. The actual parameters are 50 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 51 | + 52 | + if (c->ssl->dyn_rec.timeout > 0 ) { 53 | + 54 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 55 | + c->ssl->dyn_rec.timeout) 56 | + { 57 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 58 | + c->ssl->dyn_rec_records_sent = 0; 59 | + 60 | + } else { 61 | + if (c->ssl->dyn_rec_records_sent > 62 | + c->ssl->dyn_rec.threshold * 2) 63 | + { 64 | + buf->end = buf->start + c->ssl->buffer_size; 65 | + 66 | + } else if (c->ssl->dyn_rec_records_sent > 67 | + c->ssl->dyn_rec.threshold) 68 | + { 69 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 70 | + 71 | + } else { 72 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 73 | + } 74 | + } 75 | + } 76 | + 77 | while (in && buf->last < buf->end && send < limit) { 78 | if (in->buf->last_buf || in->buf->flush) { 79 | flush = 1; 80 | @@ -1770,6 +1806,9 @@ 81 | 82 | if (n > 0) { 83 | 84 | + c->ssl->dyn_rec_records_sent++; 85 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 86 | + 87 | if (c->ssl->saved_read_handler) { 88 | 89 | c->read->handler = c->ssl->saved_read_handler; 90 | --- a/src/event/ngx_event_openssl.h 91 | +++ b/src/event/ngx_event_openssl.h 92 | @@ -54,10 +54,19 @@ 93 | #endif 94 | 95 | 96 | +typedef struct { 97 | + ngx_msec_t timeout; 98 | + ngx_uint_t threshold; 99 | + size_t size_lo; 100 | + size_t size_hi; 101 | +} ngx_ssl_dyn_rec_t; 102 | + 103 | + 104 | struct ngx_ssl_s { 105 | SSL_CTX *ctx; 106 | ngx_log_t *log; 107 | size_t buffer_size; 108 | + ngx_ssl_dyn_rec_t dyn_rec; 109 | }; 110 | 111 | 112 | @@ -80,6 +89,10 @@ 113 | unsigned no_wait_shutdown:1; 114 | unsigned no_send_shutdown:1; 115 | unsigned handshake_buffer_set:1; 116 | + 117 | + ngx_ssl_dyn_rec_t dyn_rec; 118 | + ngx_msec_t dyn_rec_last_write; 119 | + ngx_uint_t dyn_rec_records_sent; 120 | }; 121 | 122 | 123 | @@ -89,7 +102,7 @@ 124 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 125 | 126 | 127 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 128 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 129 | 130 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 131 | 132 | --- a/src/http/modules/ngx_http_ssl_module.c 133 | +++ b/src/http/modules/ngx_http_ssl_module.c 134 | @@ -233,6 +233,41 @@ 135 | offsetof(ngx_http_ssl_srv_conf_t, stapling_verify), 136 | NULL }, 137 | 138 | + { ngx_string("ssl_dyn_rec_enable"), 139 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 140 | + ngx_conf_set_flag_slot, 141 | + NGX_HTTP_SRV_CONF_OFFSET, 142 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 143 | + NULL }, 144 | + 145 | + { ngx_string("ssl_dyn_rec_timeout"), 146 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 147 | + ngx_conf_set_msec_slot, 148 | + NGX_HTTP_SRV_CONF_OFFSET, 149 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 150 | + NULL }, 151 | + 152 | + { ngx_string("ssl_dyn_rec_size_lo"), 153 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 154 | + ngx_conf_set_size_slot, 155 | + NGX_HTTP_SRV_CONF_OFFSET, 156 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 157 | + NULL }, 158 | + 159 | + { ngx_string("ssl_dyn_rec_size_hi"), 160 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 161 | + ngx_conf_set_size_slot, 162 | + NGX_HTTP_SRV_CONF_OFFSET, 163 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 164 | + NULL }, 165 | + 166 | + { ngx_string("ssl_dyn_rec_threshold"), 167 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 168 | + ngx_conf_set_num_slot, 169 | + NGX_HTTP_SRV_CONF_OFFSET, 170 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 171 | + NULL }, 172 | + 173 | ngx_null_command 174 | }; 175 | 176 | @@ -533,6 +568,11 @@ 177 | sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; 178 | sscf->stapling = NGX_CONF_UNSET; 179 | sscf->stapling_verify = NGX_CONF_UNSET; 180 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 181 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 182 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 183 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 184 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 185 | 186 | return sscf; 187 | } 188 | @@ -598,6 +638,20 @@ 189 | ngx_conf_merge_str_value(conf->stapling_responder, 190 | prev->stapling_responder, ""); 191 | 192 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 193 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 194 | + 1000); 195 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 196 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 197 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 198 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 199 | + 1369); 200 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 201 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 202 | + 4229); 203 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 204 | + 40); 205 | + 206 | conf->ssl.log = cf->log; 207 | 208 | if (conf->enable) { 209 | @@ -778,6 +832,28 @@ 210 | 211 | } 212 | 213 | + if (conf->dyn_rec_enable) { 214 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 215 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 216 | + 217 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 218 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 219 | + 220 | + } else { 221 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 222 | + } 223 | + 224 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 225 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 226 | + 227 | + } else { 228 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 229 | + } 230 | + 231 | + } else { 232 | + conf->ssl.dyn_rec.timeout = 0; 233 | + } 234 | + 235 | return NGX_CONF_OK; 236 | } 237 | 238 | --- a/src/http/modules/ngx_http_ssl_module.h 239 | +++ b/src/http/modules/ngx_http_ssl_module.h 240 | @@ -57,6 +57,12 @@ 241 | 242 | u_char *file; 243 | ngx_uint_t line; 244 | + 245 | + ngx_flag_t dyn_rec_enable; 246 | + ngx_msec_t dyn_rec_timeout; 247 | + size_t dyn_rec_size_lo; 248 | + size_t dyn_rec_size_hi; 249 | + ngx_uint_t dyn_rec_threshold; 250 | } ngx_http_ssl_srv_conf_t; 251 | 252 | 253 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records_1.15.3+.patch: -------------------------------------------------------------------------------- 1 | What we do now: 2 | We use a static record size of 4K. This gives a good balance of latency and 3 | throughput. 4 | 5 | Optimize latency: 6 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 7 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 8 | RTT. 9 | 10 | Optimizing throughput: 11 | By sending increasingly larger records later in the connection, when HoL is not 12 | a problem, we reduce the overhead of TLS record (29 bytes per record with 13 | GCM/CHACHA-POLY). 14 | 15 | Logic: 16 | Start each connection with small records (1369 byte default, change with 17 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 18 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 19 | Eventually after the same number of records, start sending the largest records 20 | (ssl_buffer_size). 21 | In case the connection idles for a given amount of time (1s, 22 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 23 | records again). 24 | 25 | Upstream source: 26 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 27 | 28 | --- a/src/event/ngx_event_openssl.c 29 | +++ b/src/event/ngx_event_openssl.c 30 | @@ -1239,6 +1239,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) 31 | 32 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 33 | sc->buffer_size = ssl->buffer_size; 34 | + sc->dyn_rec = ssl->dyn_rec; 35 | 36 | sc->session_ctx = ssl->ctx; 37 | 38 | @@ -1806,6 +1807,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) 39 | 40 | for ( ;; ) { 41 | 42 | + /* Dynamic record resizing: 43 | + We want the initial records to fit into one TCP segment 44 | + so we don't get TCP HoL blocking due to TCP Slow Start. 45 | + A connection always starts with small records, but after 46 | + a given amount of records sent, we make the records larger 47 | + to reduce header overhead. 48 | + After a connection has idled for a given timeout, begin 49 | + the process from the start. The actual parameters are 50 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 51 | + 52 | + if (c->ssl->dyn_rec.timeout > 0 ) { 53 | + 54 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 55 | + c->ssl->dyn_rec.timeout) 56 | + { 57 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 58 | + c->ssl->dyn_rec_records_sent = 0; 59 | + 60 | + } else { 61 | + if (c->ssl->dyn_rec_records_sent > 62 | + c->ssl->dyn_rec.threshold * 2) 63 | + { 64 | + buf->end = buf->start + c->ssl->buffer_size; 65 | + 66 | + } else if (c->ssl->dyn_rec_records_sent > 67 | + c->ssl->dyn_rec.threshold) 68 | + { 69 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 70 | + 71 | + } else { 72 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 73 | + } 74 | + } 75 | + } 76 | + 77 | while (in && buf->last < buf->end && send < limit) { 78 | if (in->buf->last_buf || in->buf->flush) { 79 | flush = 1; 80 | @@ -1907,6 +1943,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) 81 | 82 | if (n > 0) { 83 | 84 | + c->ssl->dyn_rec_records_sent++; 85 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 86 | + 87 | if (c->ssl->saved_read_handler) { 88 | 89 | c->read->handler = c->ssl->saved_read_handler; 90 | --- a/src/event/ngx_event_openssl.h 91 | +++ b/src/event/ngx_event_openssl.h 92 | @@ -64,10 +64,19 @@ 93 | #endif 94 | 95 | 96 | +typedef struct { 97 | + ngx_msec_t timeout; 98 | + ngx_uint_t threshold; 99 | + size_t size_lo; 100 | + size_t size_hi; 101 | +} ngx_ssl_dyn_rec_t; 102 | + 103 | + 104 | struct ngx_ssl_s { 105 | SSL_CTX *ctx; 106 | ngx_log_t *log; 107 | size_t buffer_size; 108 | + ngx_ssl_dyn_rec_t dyn_rec; 109 | }; 110 | 111 | 112 | @@ -93,6 +102,10 @@ struct ngx_ssl_connection_s { 113 | unsigned no_wait_shutdown:1; 114 | unsigned no_send_shutdown:1; 115 | unsigned handshake_buffer_set:1; 116 | + 117 | + ngx_ssl_dyn_rec_t dyn_rec; 118 | + ngx_msec_t dyn_rec_last_write; 119 | + ngx_uint_t dyn_rec_records_sent; 120 | }; 121 | 122 | 123 | @@ -102,7 +115,7 @@ struct ngx_ssl_connection_s { 124 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 125 | 126 | 127 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 128 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 129 | 130 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 131 | 132 | --- a/src/http/modules/ngx_http_ssl_module.c 133 | +++ b/src/http/modules/ngx_http_ssl_module.c 134 | @@ -246,6 +246,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { 135 | offsetof(ngx_http_ssl_srv_conf_t, early_data), 136 | NULL }, 137 | 138 | + { ngx_string("ssl_dyn_rec_enable"), 139 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 140 | + ngx_conf_set_flag_slot, 141 | + NGX_HTTP_SRV_CONF_OFFSET, 142 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 143 | + NULL }, 144 | + 145 | + { ngx_string("ssl_dyn_rec_timeout"), 146 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 147 | + ngx_conf_set_msec_slot, 148 | + NGX_HTTP_SRV_CONF_OFFSET, 149 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 150 | + NULL }, 151 | + 152 | + { ngx_string("ssl_dyn_rec_size_lo"), 153 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 154 | + ngx_conf_set_size_slot, 155 | + NGX_HTTP_SRV_CONF_OFFSET, 156 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 157 | + NULL }, 158 | + 159 | + { ngx_string("ssl_dyn_rec_size_hi"), 160 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 161 | + ngx_conf_set_size_slot, 162 | + NGX_HTTP_SRV_CONF_OFFSET, 163 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 164 | + NULL }, 165 | + 166 | + { ngx_string("ssl_dyn_rec_threshold"), 167 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 168 | + ngx_conf_set_num_slot, 169 | + NGX_HTTP_SRV_CONF_OFFSET, 170 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 171 | + NULL }, 172 | + 173 | ngx_null_command 174 | }; 175 | 176 | @@ -576,6 +611,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) 177 | sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; 178 | sscf->stapling = NGX_CONF_UNSET; 179 | sscf->stapling_verify = NGX_CONF_UNSET; 180 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 181 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 182 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 183 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 184 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 185 | 186 | return sscf; 187 | } 188 | @@ -643,6 +683,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 189 | ngx_conf_merge_str_value(conf->stapling_responder, 190 | prev->stapling_responder, ""); 191 | 192 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 193 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 194 | + 1000); 195 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 196 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 197 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 198 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 199 | + 1369); 200 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 201 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 202 | + 4229); 203 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 204 | + 40); 205 | + 206 | conf->ssl.log = cf->log; 207 | 208 | if (conf->enable) { 209 | @@ -827,6 +881,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 210 | return NGX_CONF_ERROR; 211 | } 212 | 213 | + if (conf->dyn_rec_enable) { 214 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 215 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 216 | + 217 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 218 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 219 | + 220 | + } else { 221 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 222 | + } 223 | + 224 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 225 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 226 | + 227 | + } else { 228 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 229 | + } 230 | + 231 | + } else { 232 | + conf->ssl.dyn_rec.timeout = 0; 233 | + } 234 | + 235 | return NGX_CONF_OK; 236 | } 237 | 238 | --- a/src/http/modules/ngx_http_ssl_module.h 239 | +++ b/src/http/modules/ngx_http_ssl_module.h 240 | @@ -58,6 +58,12 @@ typedef struct { 241 | 242 | u_char *file; 243 | ngx_uint_t line; 244 | + 245 | + ngx_flag_t dyn_rec_enable; 246 | + ngx_msec_t dyn_rec_timeout; 247 | + size_t dyn_rec_size_lo; 248 | + size_t dyn_rec_size_hi; 249 | + ngx_uint_t dyn_rec_threshold; 250 | } ngx_http_ssl_srv_conf_t; 251 | 252 | 253 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records_1.15.5+.patch: -------------------------------------------------------------------------------- 1 | What we do now: 2 | We use a static record size of 4K. This gives a good balance of latency and 3 | throughput. 4 | 5 | Optimize latency: 6 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 7 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 8 | RTT. 9 | 10 | Optimizing throughput: 11 | By sending increasingly larger records later in the connection, when HoL is not 12 | a problem, we reduce the overhead of TLS record (29 bytes per record with 13 | GCM/CHACHA-POLY). 14 | 15 | Logic: 16 | Start each connection with small records (1369 byte default, change with 17 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 18 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 19 | Eventually after the same number of records, start sending the largest records 20 | (ssl_buffer_size). 21 | In case the connection idles for a given amount of time (1s, 22 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 23 | records again). 24 | 25 | Upstream source: 26 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 27 | 28 | --- a/src/event/ngx_event_openssl.c 29 | +++ b/src/event/ngx_event_openssl.c 30 | @@ -1267,6 +1267,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) 31 | 32 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 33 | sc->buffer_size = ssl->buffer_size; 34 | + sc->dyn_rec = ssl->dyn_rec; 35 | 36 | sc->session_ctx = ssl->ctx; 37 | 38 | @@ -2115,6 +2116,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) 39 | 40 | for ( ;; ) { 41 | 42 | + /* Dynamic record resizing: 43 | + We want the initial records to fit into one TCP segment 44 | + so we don't get TCP HoL blocking due to TCP Slow Start. 45 | + A connection always starts with small records, but after 46 | + a given amount of records sent, we make the records larger 47 | + to reduce header overhead. 48 | + After a connection has idled for a given timeout, begin 49 | + the process from the start. The actual parameters are 50 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 51 | + 52 | + if (c->ssl->dyn_rec.timeout > 0 ) { 53 | + 54 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 55 | + c->ssl->dyn_rec.timeout) 56 | + { 57 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 58 | + c->ssl->dyn_rec_records_sent = 0; 59 | + 60 | + } else { 61 | + if (c->ssl->dyn_rec_records_sent > 62 | + c->ssl->dyn_rec.threshold * 2) 63 | + { 64 | + buf->end = buf->start + c->ssl->buffer_size; 65 | + 66 | + } else if (c->ssl->dyn_rec_records_sent > 67 | + c->ssl->dyn_rec.threshold) 68 | + { 69 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 70 | + 71 | + } else { 72 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 73 | + } 74 | + } 75 | + } 76 | + 77 | while (in && buf->last < buf->end && send < limit) { 78 | if (in->buf->last_buf || in->buf->flush) { 79 | flush = 1; 80 | @@ -2222,6 +2258,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) 81 | 82 | if (n > 0) { 83 | 84 | + c->ssl->dyn_rec_records_sent++; 85 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 86 | + 87 | if (c->ssl->saved_read_handler) { 88 | 89 | c->read->handler = c->ssl->saved_read_handler; 90 | --- a/src/event/ngx_event_openssl.h 91 | +++ b/src/event/ngx_event_openssl.h 92 | @@ -64,10 +64,19 @@ 93 | #endif 94 | 95 | 96 | +typedef struct { 97 | + ngx_msec_t timeout; 98 | + ngx_uint_t threshold; 99 | + size_t size_lo; 100 | + size_t size_hi; 101 | +} ngx_ssl_dyn_rec_t; 102 | + 103 | + 104 | struct ngx_ssl_s { 105 | SSL_CTX *ctx; 106 | ngx_log_t *log; 107 | size_t buffer_size; 108 | + ngx_ssl_dyn_rec_t dyn_rec; 109 | }; 110 | 111 | 112 | @@ -95,6 +104,11 @@ struct ngx_ssl_connection_s { 113 | unsigned no_wait_shutdown:1; 114 | unsigned no_send_shutdown:1; 115 | unsigned handshake_buffer_set:1; 116 | + 117 | + ngx_ssl_dyn_rec_t dyn_rec; 118 | + ngx_msec_t dyn_rec_last_write; 119 | + ngx_uint_t dyn_rec_records_sent; 120 | + 121 | unsigned try_early_data:1; 122 | unsigned in_early:1; 123 | unsigned early_preread:1; 124 | @@ -107,7 +121,7 @@ struct ngx_ssl_connection_s { 125 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 126 | 127 | 128 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 129 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 130 | 131 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 132 | 133 | --- a/src/http/modules/ngx_http_ssl_module.c 134 | +++ b/src/http/modules/ngx_http_ssl_module.c 135 | @@ -246,6 +246,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { 136 | offsetof(ngx_http_ssl_srv_conf_t, early_data), 137 | NULL }, 138 | 139 | + { ngx_string("ssl_dyn_rec_enable"), 140 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 141 | + ngx_conf_set_flag_slot, 142 | + NGX_HTTP_SRV_CONF_OFFSET, 143 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 144 | + NULL }, 145 | + 146 | + { ngx_string("ssl_dyn_rec_timeout"), 147 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 148 | + ngx_conf_set_msec_slot, 149 | + NGX_HTTP_SRV_CONF_OFFSET, 150 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 151 | + NULL }, 152 | + 153 | + { ngx_string("ssl_dyn_rec_size_lo"), 154 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 155 | + ngx_conf_set_size_slot, 156 | + NGX_HTTP_SRV_CONF_OFFSET, 157 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 158 | + NULL }, 159 | + 160 | + { ngx_string("ssl_dyn_rec_size_hi"), 161 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 162 | + ngx_conf_set_size_slot, 163 | + NGX_HTTP_SRV_CONF_OFFSET, 164 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 165 | + NULL }, 166 | + 167 | + { ngx_string("ssl_dyn_rec_threshold"), 168 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 169 | + ngx_conf_set_num_slot, 170 | + NGX_HTTP_SRV_CONF_OFFSET, 171 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 172 | + NULL }, 173 | + 174 | ngx_null_command 175 | }; 176 | 177 | @@ -576,6 +611,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) 178 | sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; 179 | sscf->stapling = NGX_CONF_UNSET; 180 | sscf->stapling_verify = NGX_CONF_UNSET; 181 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 182 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 183 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 184 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 185 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 186 | 187 | return sscf; 188 | } 189 | @@ -643,6 +683,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 190 | ngx_conf_merge_str_value(conf->stapling_responder, 191 | prev->stapling_responder, ""); 192 | 193 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 194 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 195 | + 1000); 196 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 197 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 198 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 199 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 200 | + 1369); 201 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 202 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 203 | + 4229); 204 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 205 | + 40); 206 | + 207 | conf->ssl.log = cf->log; 208 | 209 | if (conf->enable) { 210 | @@ -827,6 +881,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 211 | return NGX_CONF_ERROR; 212 | } 213 | 214 | + if (conf->dyn_rec_enable) { 215 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 216 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 217 | + 218 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 219 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 220 | + 221 | + } else { 222 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 223 | + } 224 | + 225 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 226 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 227 | + 228 | + } else { 229 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 230 | + } 231 | + 232 | + } else { 233 | + conf->ssl.dyn_rec.timeout = 0; 234 | + } 235 | + 236 | return NGX_CONF_OK; 237 | } 238 | 239 | --- a/src/http/modules/ngx_http_ssl_module.h 240 | +++ b/src/http/modules/ngx_http_ssl_module.h 241 | @@ -58,6 +58,12 @@ typedef struct { 242 | 243 | u_char *file; 244 | ngx_uint_t line; 245 | + 246 | + ngx_flag_t dyn_rec_enable; 247 | + ngx_msec_t dyn_rec_timeout; 248 | + size_t dyn_rec_size_lo; 249 | + size_t dyn_rec_size_hi; 250 | + ngx_uint_t dyn_rec_threshold; 251 | } ngx_http_ssl_srv_conf_t; 252 | 253 | 254 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records_1.17.7+.patch: -------------------------------------------------------------------------------- 1 | What we do now: 2 | We use a static record size of 4K. This gives a good balance of latency and 3 | throughput. 4 | 5 | Optimize latency: 6 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 7 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 8 | RTT. 9 | 10 | Optimizing throughput: 11 | By sending increasingly larger records later in the connection, when HoL is not 12 | a problem, we reduce the overhead of TLS record (29 bytes per record with 13 | GCM/CHACHA-POLY). 14 | 15 | Logic: 16 | Start each connection with small records (1369 byte default, change with 17 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 18 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 19 | Eventually after the same number of records, start sending the largest records 20 | (ssl_buffer_size). 21 | In case the connection idles for a given amount of time (1s, 22 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 23 | records again). 24 | 25 | Upstream source: 26 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 27 | 28 | diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c 29 | index 6a0e8c0..384dd4a 100644 30 | --- a/src/event/ngx_event_openssl.c 31 | +++ b/src/event/ngx_event_openssl.c 32 | @@ -1507,6 +1507,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) 33 | 34 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 35 | sc->buffer_size = ssl->buffer_size; 36 | + sc->dyn_rec = ssl->dyn_rec; 37 | 38 | sc->session_ctx = ssl->ctx; 39 | 40 | @@ -2400,6 +2401,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) 41 | 42 | for ( ;; ) { 43 | 44 | + /* Dynamic record resizing: 45 | + We want the initial records to fit into one TCP segment 46 | + so we don't get TCP HoL blocking due to TCP Slow Start. 47 | + A connection always starts with small records, but after 48 | + a given amount of records sent, we make the records larger 49 | + to reduce header overhead. 50 | + After a connection has idled for a given timeout, begin 51 | + the process from the start. The actual parameters are 52 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 53 | + 54 | + if (c->ssl->dyn_rec.timeout > 0 ) { 55 | + 56 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 57 | + c->ssl->dyn_rec.timeout) 58 | + { 59 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 60 | + c->ssl->dyn_rec_records_sent = 0; 61 | + 62 | + } else { 63 | + if (c->ssl->dyn_rec_records_sent > 64 | + c->ssl->dyn_rec.threshold * 2) 65 | + { 66 | + buf->end = buf->start + c->ssl->buffer_size; 67 | + 68 | + } else if (c->ssl->dyn_rec_records_sent > 69 | + c->ssl->dyn_rec.threshold) 70 | + { 71 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 72 | + 73 | + } else { 74 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 75 | + } 76 | + } 77 | + } 78 | + 79 | while (in && buf->last < buf->end && send < limit) { 80 | if (in->buf->last_buf || in->buf->flush) { 81 | flush = 1; 82 | @@ -2507,6 +2543,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) 83 | 84 | if (n > 0) { 85 | 86 | + c->ssl->dyn_rec_records_sent++; 87 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 88 | + 89 | if (c->ssl->saved_read_handler) { 90 | 91 | c->read->handler = c->ssl->saved_read_handler; 92 | diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h 93 | index 61da0c5..cd2759e 100644 94 | --- a/src/event/ngx_event_openssl.h 95 | +++ b/src/event/ngx_event_openssl.h 96 | @@ -64,10 +64,19 @@ 97 | #endif 98 | 99 | 100 | +typedef struct { 101 | + ngx_msec_t timeout; 102 | + ngx_uint_t threshold; 103 | + size_t size_lo; 104 | + size_t size_hi; 105 | +} ngx_ssl_dyn_rec_t; 106 | + 107 | + 108 | struct ngx_ssl_s { 109 | SSL_CTX *ctx; 110 | ngx_log_t *log; 111 | size_t buffer_size; 112 | + ngx_ssl_dyn_rec_t dyn_rec; 113 | }; 114 | 115 | 116 | @@ -99,6 +108,10 @@ struct ngx_ssl_connection_s { 117 | unsigned in_early:1; 118 | unsigned early_preread:1; 119 | unsigned write_blocked:1; 120 | + 121 | + ngx_ssl_dyn_rec_t dyn_rec; 122 | + ngx_msec_t dyn_rec_last_write; 123 | + ngx_uint_t dyn_rec_records_sent; 124 | }; 125 | 126 | 127 | @@ -108,7 +121,7 @@ struct ngx_ssl_connection_s { 128 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 129 | 130 | 131 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 132 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 133 | 134 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 135 | 136 | diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c 137 | index 495e628..9480b73 100644 138 | --- a/src/http/modules/ngx_http_ssl_module.c 139 | +++ b/src/http/modules/ngx_http_ssl_module.c 140 | @@ -249,6 +249,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { 141 | offsetof(ngx_http_ssl_srv_conf_t, early_data), 142 | NULL }, 143 | 144 | + { ngx_string("ssl_dyn_rec_enable"), 145 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 146 | + ngx_conf_set_flag_slot, 147 | + NGX_HTTP_SRV_CONF_OFFSET, 148 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 149 | + NULL }, 150 | + 151 | + { ngx_string("ssl_dyn_rec_timeout"), 152 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 153 | + ngx_conf_set_msec_slot, 154 | + NGX_HTTP_SRV_CONF_OFFSET, 155 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 156 | + NULL }, 157 | + 158 | + { ngx_string("ssl_dyn_rec_size_lo"), 159 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 160 | + ngx_conf_set_size_slot, 161 | + NGX_HTTP_SRV_CONF_OFFSET, 162 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 163 | + NULL }, 164 | + 165 | + { ngx_string("ssl_dyn_rec_size_hi"), 166 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 167 | + ngx_conf_set_size_slot, 168 | + NGX_HTTP_SRV_CONF_OFFSET, 169 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 170 | + NULL }, 171 | + 172 | + { ngx_string("ssl_dyn_rec_threshold"), 173 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 174 | + ngx_conf_set_num_slot, 175 | + NGX_HTTP_SRV_CONF_OFFSET, 176 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 177 | + NULL }, 178 | + 179 | ngx_null_command 180 | }; 181 | 182 | @@ -580,6 +615,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) 183 | sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; 184 | sscf->stapling = NGX_CONF_UNSET; 185 | sscf->stapling_verify = NGX_CONF_UNSET; 186 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 187 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 188 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 189 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 190 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 191 | 192 | return sscf; 193 | } 194 | @@ -647,6 +687,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 195 | ngx_conf_merge_str_value(conf->stapling_responder, 196 | prev->stapling_responder, ""); 197 | 198 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 199 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 200 | + 1000); 201 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 202 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 203 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 204 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 205 | + 1369); 206 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 207 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 208 | + 4229); 209 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 210 | + 40); 211 | + 212 | conf->ssl.log = cf->log; 213 | 214 | if (conf->enable) { 215 | @@ -857,6 +911,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 216 | return NGX_CONF_ERROR; 217 | } 218 | 219 | + if (conf->dyn_rec_enable) { 220 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 221 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 222 | + 223 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 224 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 225 | + 226 | + } else { 227 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 228 | + } 229 | + 230 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 231 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 232 | + 233 | + } else { 234 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 235 | + } 236 | + 237 | + } else { 238 | + conf->ssl.dyn_rec.timeout = 0; 239 | + } 240 | + 241 | return NGX_CONF_OK; 242 | } 243 | 244 | diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h 245 | index 26fdccf..b14b52a 100644 246 | --- a/src/http/modules/ngx_http_ssl_module.h 247 | +++ b/src/http/modules/ngx_http_ssl_module.h 248 | @@ -61,6 +61,12 @@ typedef struct { 249 | 250 | u_char *file; 251 | ngx_uint_t line; 252 | + 253 | + ngx_flag_t dyn_rec_enable; 254 | + ngx_msec_t dyn_rec_timeout; 255 | + size_t dyn_rec_size_lo; 256 | + size_t dyn_rec_size_hi; 257 | + ngx_uint_t dyn_rec_threshold; 258 | } ngx_http_ssl_srv_conf_t; 259 | 260 | 261 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records_1.25.1+.patch: -------------------------------------------------------------------------------- 1 | What we do now: 2 | We use a static record size of 4K. This gives a good balance of latency and 3 | throughput. 4 | 5 | Optimize latency: 6 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 7 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 8 | RTT. 9 | 10 | Optimizing throughput: 11 | By sending increasingly larger records later in the connection, when HoL is not 12 | a problem, we reduce the overhead of TLS record (29 bytes per record with 13 | GCM/CHACHA-POLY). 14 | 15 | Logic: 16 | Start each connection with small records (1369 byte default, change with 17 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 18 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 19 | Eventually after the same number of records, start sending the largest records 20 | (ssl_buffer_size). 21 | In case the connection idles for a given amount of time (1s, 22 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 23 | records again). 24 | 25 | Upstream source: 26 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 27 | 28 | diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c 29 | index c38aa27..4c36728 100644 30 | --- a/src/event/ngx_event_openssl.c 31 | +++ b/src/event/ngx_event_openssl.c 32 | @@ -1674,6 +1674,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) 33 | 34 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 35 | sc->buffer_size = ssl->buffer_size; 36 | + sc->dyn_rec = ssl->dyn_rec; 37 | 38 | sc->session_ctx = ssl->ctx; 39 | 40 | @@ -2645,6 +2646,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) 41 | 42 | for ( ;; ) { 43 | 44 | + /* Dynamic record resizing: 45 | + We want the initial records to fit into one TCP segment 46 | + so we don't get TCP HoL blocking due to TCP Slow Start. 47 | + A connection always starts with small records, but after 48 | + a given amount of records sent, we make the records larger 49 | + to reduce header overhead. 50 | + After a connection has idled for a given timeout, begin 51 | + the process from the start. The actual parameters are 52 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 53 | + 54 | + if (c->ssl->dyn_rec.timeout > 0 ) { 55 | + 56 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 57 | + c->ssl->dyn_rec.timeout) 58 | + { 59 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 60 | + c->ssl->dyn_rec_records_sent = 0; 61 | + 62 | + } else { 63 | + if (c->ssl->dyn_rec_records_sent > 64 | + c->ssl->dyn_rec.threshold * 2) 65 | + { 66 | + buf->end = buf->start + c->ssl->buffer_size; 67 | + 68 | + } else if (c->ssl->dyn_rec_records_sent > 69 | + c->ssl->dyn_rec.threshold) 70 | + { 71 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 72 | + 73 | + } else { 74 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 75 | + } 76 | + } 77 | + } 78 | + 79 | while (in && buf->last < buf->end && send < limit) { 80 | if (in->buf->last_buf || in->buf->flush) { 81 | flush = 1; 82 | @@ -2784,6 +2820,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) 83 | 84 | if (n > 0) { 85 | 86 | + c->ssl->dyn_rec_records_sent++; 87 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 88 | + 89 | if (c->ssl->saved_read_handler) { 90 | 91 | c->read->handler = c->ssl->saved_read_handler; 92 | diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h 93 | index c062f91..468911b 100644 94 | --- a/src/event/ngx_event_openssl.h 95 | +++ b/src/event/ngx_event_openssl.h 96 | @@ -86,10 +86,19 @@ 97 | typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t; 98 | 99 | 100 | +typedef struct { 101 | + ngx_msec_t timeout; 102 | + ngx_uint_t threshold; 103 | + size_t size_lo; 104 | + size_t size_hi; 105 | +} ngx_ssl_dyn_rec_t; 106 | + 107 | + 108 | struct ngx_ssl_s { 109 | SSL_CTX *ctx; 110 | ngx_log_t *log; 111 | size_t buffer_size; 112 | + ngx_ssl_dyn_rec_t dyn_rec; 113 | }; 114 | 115 | 116 | @@ -128,6 +137,10 @@ struct ngx_ssl_connection_s { 117 | unsigned in_ocsp:1; 118 | unsigned early_preread:1; 119 | unsigned write_blocked:1; 120 | + 121 | + ngx_ssl_dyn_rec_t dyn_rec; 122 | + ngx_msec_t dyn_rec_last_write; 123 | + ngx_uint_t dyn_rec_records_sent; 124 | }; 125 | 126 | 127 | @@ -137,7 +150,7 @@ struct ngx_ssl_connection_s { 128 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 129 | 130 | 131 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 132 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 133 | 134 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 135 | 136 | diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c 137 | index 1c92d9f..f32ae33 100644 138 | --- a/src/http/modules/ngx_http_ssl_module.c 139 | +++ b/src/http/modules/ngx_http_ssl_module.c 140 | @@ -290,6 +290,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { 141 | offsetof(ngx_http_ssl_srv_conf_t, reject_handshake), 142 | NULL }, 143 | 144 | + { ngx_string("ssl_dyn_rec_enable"), 145 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 146 | + ngx_conf_set_flag_slot, 147 | + NGX_HTTP_SRV_CONF_OFFSET, 148 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 149 | + NULL }, 150 | + 151 | + { ngx_string("ssl_dyn_rec_timeout"), 152 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 153 | + ngx_conf_set_msec_slot, 154 | + NGX_HTTP_SRV_CONF_OFFSET, 155 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 156 | + NULL }, 157 | + 158 | + { ngx_string("ssl_dyn_rec_size_lo"), 159 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 160 | + ngx_conf_set_size_slot, 161 | + NGX_HTTP_SRV_CONF_OFFSET, 162 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 163 | + NULL }, 164 | + 165 | + { ngx_string("ssl_dyn_rec_size_hi"), 166 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 167 | + ngx_conf_set_size_slot, 168 | + NGX_HTTP_SRV_CONF_OFFSET, 169 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 170 | + NULL }, 171 | + 172 | + { ngx_string("ssl_dyn_rec_threshold"), 173 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 174 | + ngx_conf_set_num_slot, 175 | + NGX_HTTP_SRV_CONF_OFFSET, 176 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 177 | + NULL }, 178 | + 179 | ngx_null_command 180 | }; 181 | 182 | @@ -629,6 +664,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) 183 | sscf->ocsp_cache_zone = NGX_CONF_UNSET_PTR; 184 | sscf->stapling = NGX_CONF_UNSET; 185 | sscf->stapling_verify = NGX_CONF_UNSET; 186 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 187 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 188 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 189 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 190 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 191 | 192 | return sscf; 193 | } 194 | @@ -694,6 +734,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 195 | ngx_conf_merge_str_value(conf->stapling_responder, 196 | prev->stapling_responder, ""); 197 | 198 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 199 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 200 | + 1000); 201 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 202 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 203 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 204 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 205 | + 1369); 206 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 207 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 208 | + 4229); 209 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 210 | + 40); 211 | + 212 | conf->ssl.log = cf->log; 213 | 214 | if (conf->certificates) { 215 | @@ -890,6 +944,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 216 | return NGX_CONF_ERROR; 217 | } 218 | 219 | + if (conf->dyn_rec_enable) { 220 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 221 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 222 | + 223 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 224 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 225 | + 226 | + } else { 227 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 228 | + } 229 | + 230 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 231 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 232 | + 233 | + } else { 234 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 235 | + } 236 | + 237 | + } else { 238 | + conf->ssl.dyn_rec.timeout = 0; 239 | + } 240 | + 241 | return NGX_CONF_OK; 242 | } 243 | 244 | diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h 245 | index c69c8ff..1d7e63e 100644 246 | --- a/src/http/modules/ngx_http_ssl_module.h 247 | +++ b/src/http/modules/ngx_http_ssl_module.h 248 | @@ -62,6 +62,12 @@ typedef struct { 249 | ngx_flag_t stapling_verify; 250 | ngx_str_t stapling_file; 251 | ngx_str_t stapling_responder; 252 | + 253 | + ngx_flag_t dyn_rec_enable; 254 | + ngx_msec_t dyn_rec_timeout; 255 | + size_t dyn_rec_size_lo; 256 | + size_t dyn_rec_size_hi; 257 | + ngx_uint_t dyn_rec_threshold; 258 | } ngx_http_ssl_srv_conf_t; 259 | 260 | 261 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records_1.27.2+.patch: -------------------------------------------------------------------------------- 1 | What we do now: 2 | We use a static record size of 4K. This gives a good balance of latency and 3 | throughput. 4 | 5 | Optimize latency: 6 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 7 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 8 | RTT. 9 | 10 | Optimizing throughput: 11 | By sending increasingly larger records later in the connection, when HoL is not 12 | a problem, we reduce the overhead of TLS record (29 bytes per record with 13 | GCM/CHACHA-POLY). 14 | 15 | Logic: 16 | Start each connection with small records (1369 byte default, change with 17 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 18 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 19 | Eventually after the same number of records, start sending the largest records 20 | (ssl_buffer_size). 21 | In case the connection idles for a given amount of time (1s, 22 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 23 | records again). 24 | 25 | Upstream source: 26 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 27 | 28 | diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c 29 | index 2b1d107df..88ae40ecc 100644 30 | --- a/src/event/ngx_event_openssl.c 31 | +++ b/src/event/ngx_event_openssl.c 32 | @@ -1594,6 +1594,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) 33 | 34 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 35 | sc->buffer_size = ssl->buffer_size; 36 | + sc->dyn_rec = ssl->dyn_rec; 37 | 38 | sc->session_ctx = ssl->ctx; 39 | 40 | @@ -2565,6 +2566,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) 41 | 42 | for ( ;; ) { 43 | 44 | + /* Dynamic record resizing: 45 | + We want the initial records to fit into one TCP segment 46 | + so we don't get TCP HoL blocking due to TCP Slow Start. 47 | + A connection always starts with small records, but after 48 | + a given amount of records sent, we make the records larger 49 | + to reduce header overhead. 50 | + After a connection has idled for a given timeout, begin 51 | + the process from the start. The actual parameters are 52 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 53 | + 54 | + if (c->ssl->dyn_rec.timeout > 0 ) { 55 | + 56 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 57 | + c->ssl->dyn_rec.timeout) 58 | + { 59 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 60 | + c->ssl->dyn_rec_records_sent = 0; 61 | + 62 | + } else { 63 | + if (c->ssl->dyn_rec_records_sent > 64 | + c->ssl->dyn_rec.threshold * 2) 65 | + { 66 | + buf->end = buf->start + c->ssl->buffer_size; 67 | + 68 | + } else if (c->ssl->dyn_rec_records_sent > 69 | + c->ssl->dyn_rec.threshold) 70 | + { 71 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 72 | + 73 | + } else { 74 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 75 | + } 76 | + } 77 | + } 78 | + 79 | while (in && buf->last < buf->end && send < limit) { 80 | if (in->buf->last_buf || in->buf->flush) { 81 | flush = 1; 82 | @@ -2704,6 +2740,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) 83 | 84 | if (n > 0) { 85 | 86 | + c->ssl->dyn_rec_records_sent++; 87 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 88 | + 89 | if (c->ssl->saved_read_handler) { 90 | 91 | c->read->handler = c->ssl->saved_read_handler; 92 | diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h 93 | index 6d171229c..246413fbb 100644 94 | --- a/src/event/ngx_event_openssl.h 95 | +++ b/src/event/ngx_event_openssl.h 96 | @@ -86,10 +86,19 @@ 97 | typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t; 98 | 99 | 100 | +typedef struct { 101 | + ngx_msec_t timeout; 102 | + ngx_uint_t threshold; 103 | + size_t size_lo; 104 | + size_t size_hi; 105 | +} ngx_ssl_dyn_rec_t; 106 | + 107 | + 108 | struct ngx_ssl_s { 109 | SSL_CTX *ctx; 110 | ngx_log_t *log; 111 | size_t buffer_size; 112 | + ngx_ssl_dyn_rec_t dyn_rec; 113 | 114 | ngx_array_t certs; 115 | 116 | @@ -133,6 +142,10 @@ struct ngx_ssl_connection_s { 117 | unsigned in_ocsp:1; 118 | unsigned early_preread:1; 119 | unsigned write_blocked:1; 120 | + 121 | + ngx_ssl_dyn_rec_t dyn_rec; 122 | + ngx_msec_t dyn_rec_last_write; 123 | + ngx_uint_t dyn_rec_records_sent; 124 | }; 125 | 126 | 127 | @@ -142,7 +155,7 @@ struct ngx_ssl_connection_s { 128 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 129 | 130 | 131 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 132 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 133 | 134 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 135 | 136 | diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c 137 | index abc8d49ab..3a614ab9c 100644 138 | --- a/src/http/modules/ngx_http_ssl_module.c 139 | +++ b/src/http/modules/ngx_http_ssl_module.c 140 | @@ -290,6 +290,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { 141 | offsetof(ngx_http_ssl_srv_conf_t, reject_handshake), 142 | NULL }, 143 | 144 | + { ngx_string("ssl_dyn_rec_enable"), 145 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 146 | + ngx_conf_set_flag_slot, 147 | + NGX_HTTP_SRV_CONF_OFFSET, 148 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 149 | + NULL }, 150 | + 151 | + { ngx_string("ssl_dyn_rec_timeout"), 152 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 153 | + ngx_conf_set_msec_slot, 154 | + NGX_HTTP_SRV_CONF_OFFSET, 155 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 156 | + NULL }, 157 | + 158 | + { ngx_string("ssl_dyn_rec_size_lo"), 159 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 160 | + ngx_conf_set_size_slot, 161 | + NGX_HTTP_SRV_CONF_OFFSET, 162 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 163 | + NULL }, 164 | + 165 | + { ngx_string("ssl_dyn_rec_size_hi"), 166 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 167 | + ngx_conf_set_size_slot, 168 | + NGX_HTTP_SRV_CONF_OFFSET, 169 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 170 | + NULL }, 171 | + 172 | + { ngx_string("ssl_dyn_rec_threshold"), 173 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 174 | + ngx_conf_set_num_slot, 175 | + NGX_HTTP_SRV_CONF_OFFSET, 176 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 177 | + NULL }, 178 | + 179 | ngx_null_command 180 | }; 181 | 182 | @@ -629,6 +664,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) 183 | sscf->ocsp_cache_zone = NGX_CONF_UNSET_PTR; 184 | sscf->stapling = NGX_CONF_UNSET; 185 | sscf->stapling_verify = NGX_CONF_UNSET; 186 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 187 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 188 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 189 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 190 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 191 | 192 | return sscf; 193 | } 194 | @@ -694,6 +734,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 195 | ngx_conf_merge_str_value(conf->stapling_responder, 196 | prev->stapling_responder, ""); 197 | 198 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 199 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 200 | + 1000); 201 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 202 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 203 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 204 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 205 | + 1369); 206 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 207 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 208 | + 4229); 209 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 210 | + 40); 211 | + 212 | conf->ssl.log = cf->log; 213 | 214 | if (conf->certificates) { 215 | @@ -709,6 +763,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 216 | return NGX_CONF_ERROR; 217 | } 218 | 219 | + if (conf->dyn_rec_enable) { 220 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 221 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 222 | + 223 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 224 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 225 | + 226 | + } else { 227 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 228 | + } 229 | + 230 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 231 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 232 | + 233 | + } else { 234 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 235 | + } 236 | + 237 | + } else { 238 | + conf->ssl.dyn_rec.timeout = 0; 239 | + } 240 | + 241 | } else if (!conf->reject_handshake) { 242 | return NGX_CONF_OK; 243 | } 244 | diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h 245 | index c69c8ffd2..1d7e63e2b 100644 246 | --- a/src/http/modules/ngx_http_ssl_module.h 247 | +++ b/src/http/modules/ngx_http_ssl_module.h 248 | @@ -62,6 +62,12 @@ typedef struct { 249 | ngx_flag_t stapling_verify; 250 | ngx_str_t stapling_file; 251 | ngx_str_t stapling_responder; 252 | + 253 | + ngx_flag_t dyn_rec_enable; 254 | + ngx_msec_t dyn_rec_timeout; 255 | + size_t dyn_rec_size_lo; 256 | + size_t dyn_rec_size_hi; 257 | + ngx_uint_t dyn_rec_threshold; 258 | } ngx_http_ssl_srv_conf_t; 259 | 260 | 261 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records_1.27.4+.patch: -------------------------------------------------------------------------------- 1 | What we do now: 2 | We use a static record size of 4K. This gives a good balance of latency and 3 | throughput. 4 | 5 | Optimize latency: 6 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 7 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 8 | RTT. 9 | 10 | Optimizing throughput: 11 | By sending increasingly larger records later in the connection, when HoL is not 12 | a problem, we reduce the overhead of TLS record (29 bytes per record with 13 | GCM/CHACHA-POLY). 14 | 15 | Logic: 16 | Start each connection with small records (1369 byte default, change with 17 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 18 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 19 | Eventually after the same number of records, start sending the largest records 20 | (ssl_buffer_size). 21 | In case the connection idles for a given amount of time (1s, 22 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 23 | records again). 24 | 25 | Upstream source: 26 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 27 | 28 | diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c 29 | index 2b1d107df..88ae40ecc 100644 30 | --- a/src/event/ngx_event_openssl.c 31 | +++ b/src/event/ngx_event_openssl.c 32 | @@ -1594,6 +1594,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) 33 | 34 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 35 | sc->buffer_size = ssl->buffer_size; 36 | + sc->dyn_rec = ssl->dyn_rec; 37 | 38 | sc->session_ctx = ssl->ctx; 39 | 40 | @@ -2565,6 +2566,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) 41 | 42 | for ( ;; ) { 43 | 44 | + /* Dynamic record resizing: 45 | + We want the initial records to fit into one TCP segment 46 | + so we don't get TCP HoL blocking due to TCP Slow Start. 47 | + A connection always starts with small records, but after 48 | + a given amount of records sent, we make the records larger 49 | + to reduce header overhead. 50 | + After a connection has idled for a given timeout, begin 51 | + the process from the start. The actual parameters are 52 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 53 | + 54 | + if (c->ssl->dyn_rec.timeout > 0 ) { 55 | + 56 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 57 | + c->ssl->dyn_rec.timeout) 58 | + { 59 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 60 | + c->ssl->dyn_rec_records_sent = 0; 61 | + 62 | + } else { 63 | + if (c->ssl->dyn_rec_records_sent > 64 | + c->ssl->dyn_rec.threshold * 2) 65 | + { 66 | + buf->end = buf->start + c->ssl->buffer_size; 67 | + 68 | + } else if (c->ssl->dyn_rec_records_sent > 69 | + c->ssl->dyn_rec.threshold) 70 | + { 71 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 72 | + 73 | + } else { 74 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 75 | + } 76 | + } 77 | + } 78 | + 79 | while (in && buf->last < buf->end && send < limit) { 80 | if (in->buf->last_buf || in->buf->flush) { 81 | flush = 1; 82 | @@ -2704,6 +2740,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) 83 | 84 | if (n > 0) { 85 | 86 | + c->ssl->dyn_rec_records_sent++; 87 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 88 | + 89 | if (c->ssl->saved_read_handler) { 90 | 91 | c->read->handler = c->ssl->saved_read_handler; 92 | diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h 93 | index 6d171229c..246413fbb 100644 94 | --- a/src/event/ngx_event_openssl.h 95 | +++ b/src/event/ngx_event_openssl.h 96 | @@ -86,10 +86,19 @@ 97 | typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t; 98 | 99 | 100 | +typedef struct { 101 | + ngx_msec_t timeout; 102 | + ngx_uint_t threshold; 103 | + size_t size_lo; 104 | + size_t size_hi; 105 | +} ngx_ssl_dyn_rec_t; 106 | + 107 | + 108 | struct ngx_ssl_s { 109 | SSL_CTX *ctx; 110 | ngx_log_t *log; 111 | size_t buffer_size; 112 | + ngx_ssl_dyn_rec_t dyn_rec; 113 | 114 | ngx_array_t certs; 115 | 116 | @@ -133,6 +142,10 @@ struct ngx_ssl_connection_s { 117 | unsigned in_ocsp:1; 118 | unsigned early_preread:1; 119 | unsigned write_blocked:1; 120 | + 121 | + ngx_ssl_dyn_rec_t dyn_rec; 122 | + ngx_msec_t dyn_rec_last_write; 123 | + ngx_uint_t dyn_rec_records_sent; 124 | }; 125 | 126 | 127 | @@ -142,7 +155,7 @@ struct ngx_ssl_connection_s { 128 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 129 | 130 | 131 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 132 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 133 | 134 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 135 | 136 | diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c 137 | index abc8d49ab..3a614ab9c 100644 138 | --- a/src/http/modules/ngx_http_ssl_module.c 139 | +++ b/src/http/modules/ngx_http_ssl_module.c 140 | @@ -290,6 +290,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { 141 | offsetof(ngx_http_ssl_srv_conf_t, reject_handshake), 142 | NULL }, 143 | 144 | + { ngx_string("ssl_dyn_rec_enable"), 145 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 146 | + ngx_conf_set_flag_slot, 147 | + NGX_HTTP_SRV_CONF_OFFSET, 148 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 149 | + NULL }, 150 | + 151 | + { ngx_string("ssl_dyn_rec_timeout"), 152 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 153 | + ngx_conf_set_msec_slot, 154 | + NGX_HTTP_SRV_CONF_OFFSET, 155 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 156 | + NULL }, 157 | + 158 | + { ngx_string("ssl_dyn_rec_size_lo"), 159 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 160 | + ngx_conf_set_size_slot, 161 | + NGX_HTTP_SRV_CONF_OFFSET, 162 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 163 | + NULL }, 164 | + 165 | + { ngx_string("ssl_dyn_rec_size_hi"), 166 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 167 | + ngx_conf_set_size_slot, 168 | + NGX_HTTP_SRV_CONF_OFFSET, 169 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 170 | + NULL }, 171 | + 172 | + { ngx_string("ssl_dyn_rec_threshold"), 173 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 174 | + ngx_conf_set_num_slot, 175 | + NGX_HTTP_SRV_CONF_OFFSET, 176 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 177 | + NULL }, 178 | + 179 | ngx_null_command 180 | }; 181 | 182 | @@ -629,6 +664,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) 183 | sscf->ocsp_cache_zone = NGX_CONF_UNSET_PTR; 184 | sscf->stapling = NGX_CONF_UNSET; 185 | sscf->stapling_verify = NGX_CONF_UNSET; 186 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 187 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 188 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 189 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 190 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 191 | 192 | return sscf; 193 | } 194 | @@ -694,6 +734,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 195 | ngx_conf_merge_str_value(conf->stapling_responder, 196 | prev->stapling_responder, ""); 197 | 198 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 199 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 200 | + 1000); 201 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 202 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 203 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 204 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 205 | + 1369); 206 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 207 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 208 | + 4229); 209 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 210 | + 40); 211 | + 212 | conf->ssl.log = cf->log; 213 | 214 | if (conf->certificates) { 215 | @@ -709,6 +763,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 216 | return NGX_CONF_ERROR; 217 | } 218 | 219 | + if (conf->dyn_rec_enable) { 220 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 221 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 222 | + 223 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 224 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 225 | + 226 | + } else { 227 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 228 | + } 229 | + 230 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 231 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 232 | + 233 | + } else { 234 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 235 | + } 236 | + 237 | + } else { 238 | + conf->ssl.dyn_rec.timeout = 0; 239 | + } 240 | + 241 | } else if (!conf->reject_handshake) { 242 | return NGX_CONF_OK; 243 | } 244 | diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h 245 | index c69c8ffd2..1d7e63e2b 100644 246 | --- a/src/http/modules/ngx_http_ssl_module.h 247 | +++ b/src/http/modules/ngx_http_ssl_module.h 248 | @@ -62,6 +62,12 @@ typedef struct { 249 | ngx_flag_t stapling_verify; 250 | ngx_str_t stapling_file; 251 | ngx_str_t stapling_responder; 252 | + 253 | + ngx_flag_t dyn_rec_enable; 254 | + ngx_msec_t dyn_rec_timeout; 255 | + size_t dyn_rec_size_lo; 256 | + size_t dyn_rec_size_hi; 257 | + ngx_uint_t dyn_rec_threshold; 258 | } ngx_http_ssl_srv_conf_t; 259 | 260 | 261 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records_1.27.5+.patch: -------------------------------------------------------------------------------- 1 | What we do now: 2 | We use a static record size of 4K. This gives a good balance of latency and 3 | throughput. 4 | 5 | Optimize latency: 6 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 7 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 8 | RTT. 9 | 10 | Optimizing throughput: 11 | By sending increasingly larger records later in the connection, when HoL is not 12 | a problem, we reduce the overhead of TLS record (29 bytes per record with 13 | GCM/CHACHA-POLY). 14 | 15 | Logic: 16 | Start each connection with small records (1369 byte default, change with 17 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 18 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 19 | Eventually after the same number of records, start sending the largest records 20 | (ssl_buffer_size). 21 | In case the connection idles for a given amount of time (1s, 22 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 23 | records again). 24 | 25 | Upstream source: 26 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 27 | 28 | diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c 29 | index 2b1d107df..88ae40ecc 100644 30 | --- a/src/event/ngx_event_openssl.c 31 | +++ b/src/event/ngx_event_openssl.c 32 | @@ -1594,6 +1594,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) 33 | 34 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 35 | sc->buffer_size = ssl->buffer_size; 36 | + sc->dyn_rec = ssl->dyn_rec; 37 | 38 | sc->session_ctx = ssl->ctx; 39 | 40 | @@ -2565,6 +2566,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) 41 | 42 | for ( ;; ) { 43 | 44 | + /* Dynamic record resizing: 45 | + We want the initial records to fit into one TCP segment 46 | + so we don't get TCP HoL blocking due to TCP Slow Start. 47 | + A connection always starts with small records, but after 48 | + a given amount of records sent, we make the records larger 49 | + to reduce header overhead. 50 | + After a connection has idled for a given timeout, begin 51 | + the process from the start. The actual parameters are 52 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 53 | + 54 | + if (c->ssl->dyn_rec.timeout > 0 ) { 55 | + 56 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 57 | + c->ssl->dyn_rec.timeout) 58 | + { 59 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 60 | + c->ssl->dyn_rec_records_sent = 0; 61 | + 62 | + } else { 63 | + if (c->ssl->dyn_rec_records_sent > 64 | + c->ssl->dyn_rec.threshold * 2) 65 | + { 66 | + buf->end = buf->start + c->ssl->buffer_size; 67 | + 68 | + } else if (c->ssl->dyn_rec_records_sent > 69 | + c->ssl->dyn_rec.threshold) 70 | + { 71 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 72 | + 73 | + } else { 74 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 75 | + } 76 | + } 77 | + } 78 | + 79 | while (in && buf->last < buf->end && send < limit) { 80 | if (in->buf->last_buf || in->buf->flush) { 81 | flush = 1; 82 | @@ -2704,6 +2740,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) 83 | 84 | if (n > 0) { 85 | 86 | + c->ssl->dyn_rec_records_sent++; 87 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 88 | + 89 | if (c->ssl->saved_read_handler) { 90 | 91 | c->read->handler = c->ssl->saved_read_handler; 92 | diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h 93 | index 6d171229c..246413fbb 100644 94 | --- a/src/event/ngx_event_openssl.h 95 | +++ b/src/event/ngx_event_openssl.h 96 | @@ -86,10 +86,19 @@ 97 | typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t; 98 | 99 | 100 | +typedef struct { 101 | + ngx_msec_t timeout; 102 | + ngx_uint_t threshold; 103 | + size_t size_lo; 104 | + size_t size_hi; 105 | +} ngx_ssl_dyn_rec_t; 106 | + 107 | + 108 | struct ngx_ssl_s { 109 | SSL_CTX *ctx; 110 | ngx_log_t *log; 111 | size_t buffer_size; 112 | + ngx_ssl_dyn_rec_t dyn_rec; 113 | 114 | ngx_array_t certs; 115 | 116 | @@ -133,6 +142,10 @@ struct ngx_ssl_connection_s { 117 | unsigned in_ocsp:1; 118 | unsigned early_preread:1; 119 | unsigned write_blocked:1; 120 | + 121 | + ngx_ssl_dyn_rec_t dyn_rec; 122 | + ngx_msec_t dyn_rec_last_write; 123 | + ngx_uint_t dyn_rec_records_sent; 124 | }; 125 | 126 | 127 | @@ -142,7 +155,7 @@ struct ngx_ssl_connection_s { 128 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 129 | 130 | 131 | -#define NGX_SSL_MAX_SESSION_SIZE 8192 132 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 133 | 134 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 135 | 136 | diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c 137 | index abc8d49ab..3a614ab9c 100644 138 | --- a/src/http/modules/ngx_http_ssl_module.c 139 | +++ b/src/http/modules/ngx_http_ssl_module.c 140 | @@ -290,6 +290,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { 141 | offsetof(ngx_http_ssl_srv_conf_t, reject_handshake), 142 | NULL }, 143 | 144 | + { ngx_string("ssl_dyn_rec_enable"), 145 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 146 | + ngx_conf_set_flag_slot, 147 | + NGX_HTTP_SRV_CONF_OFFSET, 148 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 149 | + NULL }, 150 | + 151 | + { ngx_string("ssl_dyn_rec_timeout"), 152 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 153 | + ngx_conf_set_msec_slot, 154 | + NGX_HTTP_SRV_CONF_OFFSET, 155 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 156 | + NULL }, 157 | + 158 | + { ngx_string("ssl_dyn_rec_size_lo"), 159 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 160 | + ngx_conf_set_size_slot, 161 | + NGX_HTTP_SRV_CONF_OFFSET, 162 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 163 | + NULL }, 164 | + 165 | + { ngx_string("ssl_dyn_rec_size_hi"), 166 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 167 | + ngx_conf_set_size_slot, 168 | + NGX_HTTP_SRV_CONF_OFFSET, 169 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 170 | + NULL }, 171 | + 172 | + { ngx_string("ssl_dyn_rec_threshold"), 173 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 174 | + ngx_conf_set_num_slot, 175 | + NGX_HTTP_SRV_CONF_OFFSET, 176 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 177 | + NULL }, 178 | + 179 | ngx_null_command 180 | }; 181 | 182 | @@ -629,6 +664,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) 183 | sscf->ocsp_cache_zone = NGX_CONF_UNSET_PTR; 184 | sscf->stapling = NGX_CONF_UNSET; 185 | sscf->stapling_verify = NGX_CONF_UNSET; 186 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 187 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 188 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 189 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 190 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 191 | 192 | return sscf; 193 | } 194 | @@ -694,6 +734,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 195 | ngx_conf_merge_str_value(conf->stapling_responder, 196 | prev->stapling_responder, ""); 197 | 198 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 199 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 200 | + 1000); 201 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 202 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 203 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 204 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 205 | + 1369); 206 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 207 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 208 | + 4229); 209 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 210 | + 40); 211 | + 212 | conf->ssl.log = cf->log; 213 | 214 | if (conf->certificates) { 215 | @@ -709,6 +763,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 216 | return NGX_CONF_ERROR; 217 | } 218 | 219 | + if (conf->dyn_rec_enable) { 220 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 221 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 222 | + 223 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 224 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 225 | + 226 | + } else { 227 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 228 | + } 229 | + 230 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 231 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 232 | + 233 | + } else { 234 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 235 | + } 236 | + 237 | + } else { 238 | + conf->ssl.dyn_rec.timeout = 0; 239 | + } 240 | + 241 | } else if (!conf->reject_handshake) { 242 | return NGX_CONF_OK; 243 | } 244 | diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h 245 | index c69c8ffd2..1d7e63e2b 100644 246 | --- a/src/http/modules/ngx_http_ssl_module.h 247 | +++ b/src/http/modules/ngx_http_ssl_module.h 248 | @@ -62,6 +62,12 @@ typedef struct { 249 | ngx_flag_t stapling_verify; 250 | ngx_str_t stapling_file; 251 | ngx_str_t stapling_responder; 252 | + 253 | + ngx_flag_t dyn_rec_enable; 254 | + ngx_msec_t dyn_rec_timeout; 255 | + size_t dyn_rec_size_lo; 256 | + size_t dyn_rec_size_hi; 257 | + ngx_uint_t dyn_rec_threshold; 258 | } ngx_http_ssl_srv_conf_t; 259 | 260 | 261 | --------------------------------------------------------------------------------