├── README.md ├── nginx__dynamic_tls_records.patch ├── nginx__dynamic_tls_records_1.11.5+.patch └── nginx__dynamic_tls_records_1.13.0+.patch /README.md: -------------------------------------------------------------------------------- 1 | # nginx-dynamic-tls-records-patch 2 | Add TLS Dynamic Record Resizing to Nginx 3 | 4 | From a424fefb0a638eb6d32756b5a0c471efc63e5384 Mon Sep 17 00:00:00 2001 5 | From: Vlad Krasnov 6 | 7 | Date: Sat, 9 Jan 2016 06:53:14 -0800 8 | 9 | Subject: [PATCH] - Add TLS Dynamic Record Resizing 10 | 11 | 12 | 13 | What we do now: 14 | 15 | We use a static record size of 4K. This gives a good balance of latency and 16 | throughput. 17 | 18 | 19 | 20 | Optimize latency: 21 | 22 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 23 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 24 | RTT. 25 | 26 | 27 | 28 | Optimizing throughput: 29 | 30 | By sending increasingly larger records later in the connection, when HoL is not 31 | a problem, we reduce the overhead of TLS record (29 bytes per record with 32 | GCM/CHACHA-POLY). 33 | 34 | 35 | 36 | Logic: 37 | 38 | Start each connection with small records (1369 byte default, change with 39 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 40 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 41 | Eventually after the same number of records, start sending the largest records 42 | (ssl_buffer_size). 43 | 44 | In case the connection idles for a given amount of time (1s, 45 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 46 | records again). 47 | 48 | 49 | 50 | Upstream source: 51 | 52 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 53 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records.patch: -------------------------------------------------------------------------------- 1 | From a424fefb0a638eb6d32756b5a0c471efc63e5384 Mon Sep 17 00:00:00 2001 2 | From: Vlad Krasnov 3 | Date: Sat, 9 Jan 2016 06:53:14 -0800 4 | Subject: [PATCH] - Add TLS Dynamic Record Resizing 5 | 6 | What we do now: 7 | We use a static record size of 4K. This gives a good balance of latency and 8 | throughput. 9 | 10 | Optimize latency: 11 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 12 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 13 | RTT. 14 | 15 | Optimizing throughput: 16 | By sending increasingly larger records later in the connection, when HoL is not 17 | a problem, we reduce the overhead of TLS record (29 bytes per record with 18 | GCM/CHACHA-POLY). 19 | 20 | Logic: 21 | Start each connection with small records (1369 byte default, change with 22 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 23 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 24 | Eventually after the same number of records, start sending the largest records 25 | (ssl_buffer_size). 26 | In case the connection idles for a given amount of time (1s, 27 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 28 | records again). 29 | 30 | --- 31 | src/event/ngx_event_openssl.c | 39 +++++++++++++++++ 32 | src/event/ngx_event_openssl.h | 15 ++++++- 33 | src/http/modules/ngx_http_ssl_module.c | 76 ++++++++++++++++++++++++++++++++++ 34 | src/http/modules/ngx_http_ssl_module.h | 6 +++ 35 | 4 files changed, 135 insertions(+), 1 deletion(-) 36 | 37 | diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c 38 | index 57dfc6c..4a0d41a 100644 39 | --- a/src/event/ngx_event_openssl.c 40 | +++ b/src/event/ngx_event_openssl.c 41 | @@ -1037,6 +1037,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) 42 | 43 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 44 | sc->buffer_size = ssl->buffer_size; 45 | + sc->dyn_rec = ssl->dyn_rec; 46 | 47 | sc->session_ctx = ssl->ctx; 48 | 49 | @@ -1575,6 +1576,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) 50 | 51 | for ( ;; ) { 52 | 53 | + /* Dynamic record resizing: 54 | + We want the initial records to fit into one TCP segment 55 | + so we don't get TCP HoL blocking due to TCP Slow Start. 56 | + A connection always starts with small records, but after 57 | + a given amount of records sent, we make the records larger 58 | + to reduce header overhead. 59 | + After a connection has idled for a given timeout, begin 60 | + the process from the start. The actual parameters are 61 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 62 | + 63 | + if (c->ssl->dyn_rec.timeout > 0 ) { 64 | + 65 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 66 | + c->ssl->dyn_rec.timeout) 67 | + { 68 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 69 | + c->ssl->dyn_rec_records_sent = 0; 70 | + 71 | + } else { 72 | + if (c->ssl->dyn_rec_records_sent > 73 | + c->ssl->dyn_rec.threshold * 2) 74 | + { 75 | + buf->end = buf->start + c->ssl->buffer_size; 76 | + 77 | + } else if (c->ssl->dyn_rec_records_sent > 78 | + c->ssl->dyn_rec.threshold) 79 | + { 80 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 81 | + 82 | + } else { 83 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 84 | + } 85 | + } 86 | + } 87 | + 88 | while (in && buf->last < buf->end && send < limit) { 89 | if (in->buf->last_buf || in->buf->flush) { 90 | flush = 1; 91 | @@ -1676,6 +1712,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) 92 | 93 | if (n > 0) { 94 | 95 | + c->ssl->dyn_rec_records_sent++; 96 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 97 | + 98 | if (c->ssl->saved_read_handler) { 99 | 100 | c->read->handler = c->ssl->saved_read_handler; 101 | diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h 102 | index c86be2a..4a45934 100644 103 | --- a/src/event/ngx_event_openssl.h 104 | +++ b/src/event/ngx_event_openssl.h 105 | @@ -38,9 +38,18 @@ 106 | 107 | 108 | typedef struct { 109 | + ngx_msec_t timeout; 110 | + ngx_uint_t threshold; 111 | + size_t size_lo; 112 | + size_t size_hi; 113 | +} ngx_ssl_dyn_rec_t; 114 | + 115 | + 116 | +typedef struct { 117 | SSL_CTX *ctx; 118 | ngx_log_t *log; 119 | size_t buffer_size; 120 | + ngx_ssl_dyn_rec_t dyn_rec; 121 | } ngx_ssl_t; 122 | 123 | 124 | @@ -63,6 +72,10 @@ typedef struct { 125 | unsigned no_wait_shutdown:1; 126 | unsigned no_send_shutdown:1; 127 | unsigned handshake_buffer_set:1; 128 | + 129 | + ngx_ssl_dyn_rec_t dyn_rec; 130 | + ngx_msec_t dyn_rec_last_write; 131 | + ngx_uint_t dyn_rec_records_sent; 132 | } ngx_ssl_connection_t; 133 | 134 | 135 | @@ -72,7 +85,7 @@ typedef struct { 136 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 137 | 138 | 139 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 140 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 141 | 142 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 143 | 144 | diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c 145 | index 7b051ea..e2941af 100644 146 | --- a/src/http/modules/ngx_http_ssl_module.c 147 | +++ b/src/http/modules/ngx_http_ssl_module.c 148 | @@ -233,6 +233,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { 149 | offsetof(ngx_http_ssl_srv_conf_t, stapling_verify), 150 | NULL }, 151 | 152 | + { ngx_string("ssl_dyn_rec_enable"), 153 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 154 | + ngx_conf_set_flag_slot, 155 | + NGX_HTTP_SRV_CONF_OFFSET, 156 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 157 | + NULL }, 158 | + 159 | + { ngx_string("ssl_dyn_rec_timeout"), 160 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 161 | + ngx_conf_set_msec_slot, 162 | + NGX_HTTP_SRV_CONF_OFFSET, 163 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 164 | + NULL }, 165 | + 166 | + { ngx_string("ssl_dyn_rec_size_lo"), 167 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 168 | + ngx_conf_set_size_slot, 169 | + NGX_HTTP_SRV_CONF_OFFSET, 170 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 171 | + NULL }, 172 | + 173 | + { ngx_string("ssl_dyn_rec_size_hi"), 174 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 175 | + ngx_conf_set_size_slot, 176 | + NGX_HTTP_SRV_CONF_OFFSET, 177 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 178 | + NULL }, 179 | + 180 | + { ngx_string("ssl_dyn_rec_threshold"), 181 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 182 | + ngx_conf_set_num_slot, 183 | + NGX_HTTP_SRV_CONF_OFFSET, 184 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 185 | + NULL }, 186 | + 187 | ngx_null_command 188 | }; 189 | 190 | @@ -532,6 +567,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) 191 | sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; 192 | sscf->stapling = NGX_CONF_UNSET; 193 | sscf->stapling_verify = NGX_CONF_UNSET; 194 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 195 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 196 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 197 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 198 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 199 | 200 | return sscf; 201 | } 202 | @@ -596,6 +636,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 203 | ngx_conf_merge_str_value(conf->stapling_responder, 204 | prev->stapling_responder, ""); 205 | 206 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 207 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 208 | + 1000); 209 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 210 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 211 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 212 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 213 | + 1369); 214 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 215 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 216 | + 4229); 217 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 218 | + 40); 219 | + 220 | conf->ssl.log = cf->log; 221 | 222 | if (conf->enable) { 223 | @@ -773,6 +827,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) 224 | 225 | } 226 | 227 | + if (conf->dyn_rec_enable) { 228 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 229 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 230 | + 231 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 232 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 233 | + 234 | + } else { 235 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 236 | + } 237 | + 238 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 239 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 240 | + 241 | + } else { 242 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 243 | + } 244 | + 245 | + } else { 246 | + conf->ssl.dyn_rec.timeout = 0; 247 | + } 248 | + 249 | return NGX_CONF_OK; 250 | } 251 | 252 | diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h 253 | index 8e69e9e..05967d4 100644 254 | --- a/src/http/modules/ngx_http_ssl_module.h 255 | +++ b/src/http/modules/ngx_http_ssl_module.h 256 | @@ -56,6 +56,12 @@ typedef struct { 257 | 258 | u_char *file; 259 | ngx_uint_t line; 260 | + 261 | + ngx_flag_t dyn_rec_enable; 262 | + ngx_msec_t dyn_rec_timeout; 263 | + size_t dyn_rec_size_lo; 264 | + size_t dyn_rec_size_hi; 265 | + ngx_uint_t dyn_rec_threshold; 266 | } ngx_http_ssl_srv_conf_t; 267 | 268 | 269 | -- 270 | 2.7.4 (Apple Git-66) 271 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records_1.11.5+.patch: -------------------------------------------------------------------------------- 1 | What we do now: 2 | We use a static record size of 4K. This gives a good balance of latency and 3 | throughput. 4 | 5 | Optimize latency: 6 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 7 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 8 | RTT. 9 | 10 | Optimizing throughput: 11 | By sending increasingly larger records later in the connection, when HoL is not 12 | a problem, we reduce the overhead of TLS record (29 bytes per record with 13 | GCM/CHACHA-POLY). 14 | 15 | Logic: 16 | Start each connection with small records (1369 byte default, change with 17 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 18 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 19 | Eventually after the same number of records, start sending the largest records 20 | (ssl_buffer_size). 21 | In case the connection idles for a given amount of time (1s, 22 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 23 | records again). 24 | 25 | Upstream source: 26 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 27 | 28 | --- a/src/event/ngx_event_openssl.c 29 | +++ b/src/event/ngx_event_openssl.c 30 | @@ -1131,6 +1131,7 @@ 31 | 32 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 33 | sc->buffer_size = ssl->buffer_size; 34 | + sc->dyn_rec = ssl->dyn_rec; 35 | 36 | sc->session_ctx = ssl->ctx; 37 | 38 | @@ -1669,6 +1670,41 @@ 39 | 40 | for ( ;; ) { 41 | 42 | + /* Dynamic record resizing: 43 | + We want the initial records to fit into one TCP segment 44 | + so we don't get TCP HoL blocking due to TCP Slow Start. 45 | + A connection always starts with small records, but after 46 | + a given amount of records sent, we make the records larger 47 | + to reduce header overhead. 48 | + After a connection has idled for a given timeout, begin 49 | + the process from the start. The actual parameters are 50 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 51 | + 52 | + if (c->ssl->dyn_rec.timeout > 0 ) { 53 | + 54 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 55 | + c->ssl->dyn_rec.timeout) 56 | + { 57 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 58 | + c->ssl->dyn_rec_records_sent = 0; 59 | + 60 | + } else { 61 | + if (c->ssl->dyn_rec_records_sent > 62 | + c->ssl->dyn_rec.threshold * 2) 63 | + { 64 | + buf->end = buf->start + c->ssl->buffer_size; 65 | + 66 | + } else if (c->ssl->dyn_rec_records_sent > 67 | + c->ssl->dyn_rec.threshold) 68 | + { 69 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 70 | + 71 | + } else { 72 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 73 | + } 74 | + } 75 | + } 76 | + 77 | while (in && buf->last < buf->end && send < limit) { 78 | if (in->buf->last_buf || in->buf->flush) { 79 | flush = 1; 80 | @@ -1770,6 +1806,9 @@ 81 | 82 | if (n > 0) { 83 | 84 | + c->ssl->dyn_rec_records_sent++; 85 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 86 | + 87 | if (c->ssl->saved_read_handler) { 88 | 89 | c->read->handler = c->ssl->saved_read_handler; 90 | --- a/src/event/ngx_event_openssl.h 91 | +++ b/src/event/ngx_event_openssl.h 92 | @@ -54,10 +54,19 @@ 93 | #define ngx_ssl_conn_t SSL 94 | 95 | 96 | +typedef struct { 97 | + ngx_msec_t timeout; 98 | + ngx_uint_t threshold; 99 | + size_t size_lo; 100 | + size_t size_hi; 101 | +} ngx_ssl_dyn_rec_t; 102 | + 103 | + 104 | struct ngx_ssl_s { 105 | SSL_CTX *ctx; 106 | ngx_log_t *log; 107 | size_t buffer_size; 108 | + ngx_ssl_dyn_rec_t dyn_rec; 109 | }; 110 | 111 | 112 | @@ -80,6 +89,10 @@ 113 | unsigned no_wait_shutdown:1; 114 | unsigned no_send_shutdown:1; 115 | unsigned handshake_buffer_set:1; 116 | + 117 | + ngx_ssl_dyn_rec_t dyn_rec; 118 | + ngx_msec_t dyn_rec_last_write; 119 | + ngx_uint_t dyn_rec_records_sent; 120 | }; 121 | 122 | 123 | @@ -89,7 +102,7 @@ 124 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 125 | 126 | 127 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 128 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 129 | 130 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 131 | 132 | --- a/src/http/modules/ngx_http_ssl_module.c 133 | +++ b/src/http/modules/ngx_http_ssl_module.c 134 | @@ -233,6 +233,41 @@ 135 | offsetof(ngx_http_ssl_srv_conf_t, stapling_verify), 136 | NULL }, 137 | 138 | + { ngx_string("ssl_dyn_rec_enable"), 139 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 140 | + ngx_conf_set_flag_slot, 141 | + NGX_HTTP_SRV_CONF_OFFSET, 142 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 143 | + NULL }, 144 | + 145 | + { ngx_string("ssl_dyn_rec_timeout"), 146 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 147 | + ngx_conf_set_msec_slot, 148 | + NGX_HTTP_SRV_CONF_OFFSET, 149 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 150 | + NULL }, 151 | + 152 | + { ngx_string("ssl_dyn_rec_size_lo"), 153 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 154 | + ngx_conf_set_size_slot, 155 | + NGX_HTTP_SRV_CONF_OFFSET, 156 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 157 | + NULL }, 158 | + 159 | + { ngx_string("ssl_dyn_rec_size_hi"), 160 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 161 | + ngx_conf_set_size_slot, 162 | + NGX_HTTP_SRV_CONF_OFFSET, 163 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 164 | + NULL }, 165 | + 166 | + { ngx_string("ssl_dyn_rec_threshold"), 167 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 168 | + ngx_conf_set_num_slot, 169 | + NGX_HTTP_SRV_CONF_OFFSET, 170 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 171 | + NULL }, 172 | + 173 | ngx_null_command 174 | }; 175 | 176 | @@ -533,6 +568,11 @@ 177 | sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; 178 | sscf->stapling = NGX_CONF_UNSET; 179 | sscf->stapling_verify = NGX_CONF_UNSET; 180 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 181 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 182 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 183 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 184 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 185 | 186 | return sscf; 187 | } 188 | @@ -598,6 +638,20 @@ 189 | ngx_conf_merge_str_value(conf->stapling_responder, 190 | prev->stapling_responder, ""); 191 | 192 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 193 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 194 | + 1000); 195 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 196 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 197 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 198 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 199 | + 1369); 200 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 201 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 202 | + 4229); 203 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 204 | + 40); 205 | + 206 | conf->ssl.log = cf->log; 207 | 208 | if (conf->enable) { 209 | @@ -778,6 +832,28 @@ 210 | 211 | } 212 | 213 | + if (conf->dyn_rec_enable) { 214 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 215 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 216 | + 217 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 218 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 219 | + 220 | + } else { 221 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 222 | + } 223 | + 224 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 225 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 226 | + 227 | + } else { 228 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 229 | + } 230 | + 231 | + } else { 232 | + conf->ssl.dyn_rec.timeout = 0; 233 | + } 234 | + 235 | return NGX_CONF_OK; 236 | } 237 | 238 | --- a/src/http/modules/ngx_http_ssl_module.h 239 | +++ b/src/http/modules/ngx_http_ssl_module.h 240 | @@ -57,6 +57,12 @@ 241 | 242 | u_char *file; 243 | ngx_uint_t line; 244 | + 245 | + ngx_flag_t dyn_rec_enable; 246 | + ngx_msec_t dyn_rec_timeout; 247 | + size_t dyn_rec_size_lo; 248 | + size_t dyn_rec_size_hi; 249 | + ngx_uint_t dyn_rec_threshold; 250 | } ngx_http_ssl_srv_conf_t; 251 | 252 | 253 | -------------------------------------------------------------------------------- /nginx__dynamic_tls_records_1.13.0+.patch: -------------------------------------------------------------------------------- 1 | What we do now: 2 | We use a static record size of 4K. This gives a good balance of latency and 3 | throughput. 4 | 5 | Optimize latency: 6 | By initialy sending small (1 TCP segment) sized records, we are able to avoid 7 | HoL blocking of the first byte. This means TTFB is sometime lower by a whole 8 | RTT. 9 | 10 | Optimizing throughput: 11 | By sending increasingly larger records later in the connection, when HoL is not 12 | a problem, we reduce the overhead of TLS record (29 bytes per record with 13 | GCM/CHACHA-POLY). 14 | 15 | Logic: 16 | Start each connection with small records (1369 byte default, change with 17 | ssl_dyn_rec_size_lo). After a given number of records (40, change with 18 | ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). 19 | Eventually after the same number of records, start sending the largest records 20 | (ssl_buffer_size). 21 | In case the connection idles for a given amount of time (1s, 22 | ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small 23 | records again). 24 | 25 | Upstream source: 26 | https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch 27 | 28 | --- a/src/event/ngx_event_openssl.c 29 | +++ b/src/event/ngx_event_openssl.c 30 | @@ -1131,6 +1131,7 @@ 31 | 32 | sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); 33 | sc->buffer_size = ssl->buffer_size; 34 | + sc->dyn_rec = ssl->dyn_rec; 35 | 36 | sc->session_ctx = ssl->ctx; 37 | 38 | @@ -1669,6 +1670,41 @@ 39 | 40 | for ( ;; ) { 41 | 42 | + /* Dynamic record resizing: 43 | + We want the initial records to fit into one TCP segment 44 | + so we don't get TCP HoL blocking due to TCP Slow Start. 45 | + A connection always starts with small records, but after 46 | + a given amount of records sent, we make the records larger 47 | + to reduce header overhead. 48 | + After a connection has idled for a given timeout, begin 49 | + the process from the start. The actual parameters are 50 | + configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ 51 | + 52 | + if (c->ssl->dyn_rec.timeout > 0 ) { 53 | + 54 | + if (ngx_current_msec - c->ssl->dyn_rec_last_write > 55 | + c->ssl->dyn_rec.timeout) 56 | + { 57 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 58 | + c->ssl->dyn_rec_records_sent = 0; 59 | + 60 | + } else { 61 | + if (c->ssl->dyn_rec_records_sent > 62 | + c->ssl->dyn_rec.threshold * 2) 63 | + { 64 | + buf->end = buf->start + c->ssl->buffer_size; 65 | + 66 | + } else if (c->ssl->dyn_rec_records_sent > 67 | + c->ssl->dyn_rec.threshold) 68 | + { 69 | + buf->end = buf->start + c->ssl->dyn_rec.size_hi; 70 | + 71 | + } else { 72 | + buf->end = buf->start + c->ssl->dyn_rec.size_lo; 73 | + } 74 | + } 75 | + } 76 | + 77 | while (in && buf->last < buf->end && send < limit) { 78 | if (in->buf->last_buf || in->buf->flush) { 79 | flush = 1; 80 | @@ -1770,6 +1806,9 @@ 81 | 82 | if (n > 0) { 83 | 84 | + c->ssl->dyn_rec_records_sent++; 85 | + c->ssl->dyn_rec_last_write = ngx_current_msec; 86 | + 87 | if (c->ssl->saved_read_handler) { 88 | 89 | c->read->handler = c->ssl->saved_read_handler; 90 | --- a/src/event/ngx_event_openssl.h 91 | +++ b/src/event/ngx_event_openssl.h 92 | @@ -54,10 +54,19 @@ 93 | #endif 94 | 95 | 96 | +typedef struct { 97 | + ngx_msec_t timeout; 98 | + ngx_uint_t threshold; 99 | + size_t size_lo; 100 | + size_t size_hi; 101 | +} ngx_ssl_dyn_rec_t; 102 | + 103 | + 104 | struct ngx_ssl_s { 105 | SSL_CTX *ctx; 106 | ngx_log_t *log; 107 | size_t buffer_size; 108 | + ngx_ssl_dyn_rec_t dyn_rec; 109 | }; 110 | 111 | 112 | @@ -80,6 +89,10 @@ 113 | unsigned no_wait_shutdown:1; 114 | unsigned no_send_shutdown:1; 115 | unsigned handshake_buffer_set:1; 116 | + 117 | + ngx_ssl_dyn_rec_t dyn_rec; 118 | + ngx_msec_t dyn_rec_last_write; 119 | + ngx_uint_t dyn_rec_records_sent; 120 | }; 121 | 122 | 123 | @@ -89,7 +102,7 @@ 124 | #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 125 | 126 | 127 | -#define NGX_SSL_MAX_SESSION_SIZE 4096 128 | +#define NGX_SSL_MAX_SESSION_SIZE 16384 129 | 130 | typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; 131 | 132 | --- a/src/http/modules/ngx_http_ssl_module.c 133 | +++ b/src/http/modules/ngx_http_ssl_module.c 134 | @@ -233,6 +233,41 @@ 135 | offsetof(ngx_http_ssl_srv_conf_t, stapling_verify), 136 | NULL }, 137 | 138 | + { ngx_string("ssl_dyn_rec_enable"), 139 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 140 | + ngx_conf_set_flag_slot, 141 | + NGX_HTTP_SRV_CONF_OFFSET, 142 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), 143 | + NULL }, 144 | + 145 | + { ngx_string("ssl_dyn_rec_timeout"), 146 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 147 | + ngx_conf_set_msec_slot, 148 | + NGX_HTTP_SRV_CONF_OFFSET, 149 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), 150 | + NULL }, 151 | + 152 | + { ngx_string("ssl_dyn_rec_size_lo"), 153 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 154 | + ngx_conf_set_size_slot, 155 | + NGX_HTTP_SRV_CONF_OFFSET, 156 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), 157 | + NULL }, 158 | + 159 | + { ngx_string("ssl_dyn_rec_size_hi"), 160 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 161 | + ngx_conf_set_size_slot, 162 | + NGX_HTTP_SRV_CONF_OFFSET, 163 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), 164 | + NULL }, 165 | + 166 | + { ngx_string("ssl_dyn_rec_threshold"), 167 | + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, 168 | + ngx_conf_set_num_slot, 169 | + NGX_HTTP_SRV_CONF_OFFSET, 170 | + offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), 171 | + NULL }, 172 | + 173 | ngx_null_command 174 | }; 175 | 176 | @@ -533,6 +568,11 @@ 177 | sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; 178 | sscf->stapling = NGX_CONF_UNSET; 179 | sscf->stapling_verify = NGX_CONF_UNSET; 180 | + sscf->dyn_rec_enable = NGX_CONF_UNSET; 181 | + sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; 182 | + sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; 183 | + sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; 184 | + sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; 185 | 186 | return sscf; 187 | } 188 | @@ -598,6 +638,20 @@ 189 | ngx_conf_merge_str_value(conf->stapling_responder, 190 | prev->stapling_responder, ""); 191 | 192 | + ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); 193 | + ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, 194 | + 1000); 195 | + /* Default sizes for the dynamic record sizes are defined to fit maximal 196 | + TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: 197 | + 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ 198 | + ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, 199 | + 1369); 200 | + /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ 201 | + ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, 202 | + 4229); 203 | + ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, 204 | + 40); 205 | + 206 | conf->ssl.log = cf->log; 207 | 208 | if (conf->enable) { 209 | @@ -778,6 +832,28 @@ 210 | 211 | } 212 | 213 | + if (conf->dyn_rec_enable) { 214 | + conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; 215 | + conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; 216 | + 217 | + if (conf->buffer_size > conf->dyn_rec_size_lo) { 218 | + conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; 219 | + 220 | + } else { 221 | + conf->ssl.dyn_rec.size_lo = conf->buffer_size; 222 | + } 223 | + 224 | + if (conf->buffer_size > conf->dyn_rec_size_hi) { 225 | + conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; 226 | + 227 | + } else { 228 | + conf->ssl.dyn_rec.size_hi = conf->buffer_size; 229 | + } 230 | + 231 | + } else { 232 | + conf->ssl.dyn_rec.timeout = 0; 233 | + } 234 | + 235 | return NGX_CONF_OK; 236 | } 237 | 238 | --- a/src/http/modules/ngx_http_ssl_module.h 239 | +++ b/src/http/modules/ngx_http_ssl_module.h 240 | @@ -57,6 +57,12 @@ 241 | 242 | u_char *file; 243 | ngx_uint_t line; 244 | + 245 | + ngx_flag_t dyn_rec_enable; 246 | + ngx_msec_t dyn_rec_timeout; 247 | + size_t dyn_rec_size_lo; 248 | + size_t dyn_rec_size_hi; 249 | + ngx_uint_t dyn_rec_threshold; 250 | } ngx_http_ssl_srv_conf_t; 251 | 252 | 253 | --------------------------------------------------------------------------------