├── README.md └── default.vcl /README.md: -------------------------------------------------------------------------------- 1 | ## Varnish Configuration Templates (boilerplate) 2 | 3 | ### 🚀 Need help implementing Varnish? 4 | 5 | I'm available [for consultancy](https://ma.ttias.be/consultancy/) if you're struggling with implementing Varnish and speeding up your site. Don't be afraid to [reach out!](https://ma.ttias.be/consultancy/) 6 | 7 | ### Installation 8 | 9 | You can use the configuration templates found in this repository to quickly get started with a complete Varnish configuration that offers support for most functionality. Start of by looking into "default.vcl" and taking the bits you need, copy it to your own default.vcl. 10 | 11 | ### What is it? 12 | 13 | A set of configuration samples used for Varnish 5.0 with various features: 14 | 15 | * Server-side URL rewriting 16 | * Clean error pages for debugging 17 | * Virtual Host implementations 18 | * Various header normalizations 19 | * Cookie manipulations 20 | * 301/302 redirects from within Varnish 21 | 22 | ### Common troubleshooting 23 | 24 | Common list of errors and their fixes: 25 | 26 | * [FetchError http first read error: -1 11 (Resource temporarily unavailable)](https://ma.ttias.be/varnish-fetcherror-http-first-read-error-1-11-resource-temporarily-unavailable/) 27 | * [FetchError: straight insufficient bytes](https://ma.ttias.be/varnish-fetcherror-straight-insufficient-bytes/) 28 | * [FetchError: Gunzip+ESI Failed at the very end](https://ma.ttias.be/varnish-fetcherror-testgunzip-gunzip-esi-failed-very-end/) 29 | 30 | Basic troubleshooting: 31 | 32 | * [Test if your Varnish VCL compiles and Varnish starts](https://ma.ttias.be/varnish-running-in-foreground-but-fails-to-run-as-servicedaemon/) 33 | * [See which cookies are being stripped in your VCL](https://ma.ttias.be/varnish-tip-see-cookies-stripped-vcl/) 34 | * [Reload Varnish VCL without losing cache data](https://ma.ttias.be/reload-varnish-vcl-without-losing-cache-data/) 35 | * [Combine Apache'S HTTP authentication with Varnish IP whitelisting](https://ma.ttias.be/apache-http-authentication-with-x-forwarded-for-ip-whitelisting-in-varnish/) 36 | 37 | [Click here for a Varnish 3 VCL config template](https://github.com/mattiasgeniar/varnish-3.0-configuration-templates) 38 | 39 | [Click here for a Varnish 4 VCL config template](https://github.com/mattiasgeniar/varnish-4.0-configuration-templates) 40 | 41 | [Click here for a Varnish 5 VCL config template](https://github.com/mattiasgeniar/varnish-5.0-configuration-templates) 42 | -------------------------------------------------------------------------------- /default.vcl: -------------------------------------------------------------------------------- 1 | vcl 4.1; 2 | # Based on: https://github.com/mattiasgeniar/varnish-6.0-configuration-templates/blob/master/default.vcl 3 | 4 | import std; 5 | import directors; 6 | 7 | backend server1 { # Define one backend 8 | .host = "127.0.0.1"; # IP or Hostname of backend 9 | .port = "80"; # Port Apache or whatever is listening 10 | .max_connections = 300; # That's it 11 | 12 | .probe = { 13 | #.url = "/"; # short easy way (GET /) 14 | # We prefer to only do a HEAD / 15 | .request = 16 | "HEAD / HTTP/1.1" 17 | "Host: localhost" 18 | "Connection: close" 19 | "User-Agent: Varnish Health Probe"; 20 | 21 | .interval = 5s; # check the health of each backend every 5 seconds 22 | .timeout = 1s; # timing out after 1 second. 23 | .window = 5; # If 3 out of the last 5 polls succeeded the backend is considered healthy, otherwise it will be marked as sick 24 | .threshold = 3; 25 | } 26 | 27 | .first_byte_timeout = 300s; # How long to wait before we receive a first byte from our backend? 28 | .connect_timeout = 5s; # How long to wait for a backend connection? 29 | .between_bytes_timeout = 2s; # How long to wait between bytes received from our backend? 30 | } 31 | 32 | acl purge { 33 | # ACL we'll use later to allow purges 34 | "localhost"; 35 | "127.0.0.1"; 36 | "::1"; 37 | } 38 | 39 | sub vcl_init { 40 | # Called when VCL is loaded, before any requests pass through it. 41 | # Typically used to initialize VMODs. 42 | 43 | new vdir = directors.round_robin(); 44 | vdir.add_backend(server1); 45 | # vdir.add_backend(server...); 46 | # vdir.add_backend(servern); 47 | } 48 | 49 | sub vcl_recv { 50 | # Called at the beginning of a request, after the complete request has been received and parsed. 51 | # Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable, 52 | # which backend to use. 53 | # also used to modify the request 54 | 55 | set req.backend_hint = vdir.backend(); # send all traffic to the vdir director 56 | 57 | # Normalize the header if it exists, remove the port (in case you're testing this on various TCP ports) 58 | if (req.http.Host) { 59 | set req.http.Host = regsub(req.http.Host, ":[0-9]+", ""); 60 | } 61 | 62 | # Remove the proxy header (see https://httpoxy.org/#mitigate-varnish) 63 | unset req.http.proxy; 64 | 65 | # Normalize the query arguments 66 | set req.url = std.querysort(req.url); 67 | 68 | # Allow purging 69 | if (req.method == "PURGE") { 70 | if (!client.ip ~ purge) { # purge is the ACL defined at the begining 71 | # Not from an allowed IP? Then die with an error. 72 | return (synth(405, "This IP is not allowed to send PURGE requests.")); 73 | } 74 | # If you got this stage (and didn't error out above), purge the cached result 75 | return (purge); 76 | } 77 | 78 | # Only deal with "normal" types 79 | if (req.method != "GET" && 80 | req.method != "HEAD" && 81 | req.method != "PUT" && 82 | req.method != "POST" && 83 | req.method != "TRACE" && 84 | req.method != "OPTIONS" && 85 | req.method != "PATCH" && 86 | req.method != "DELETE") { 87 | /* Non-RFC2616 or CONNECT which is weird. */ 88 | return (pipe); 89 | } 90 | 91 | # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html) 92 | if (req.http.Upgrade ~ "(?i)websocket") { 93 | return (pipe); 94 | } 95 | 96 | # Only cache GET or HEAD requests. This makes sure the POST requests are always passed. 97 | if (req.method != "GET" && req.method != "HEAD") { 98 | return (pass); 99 | } 100 | 101 | # Some generic URL manipulation, useful for all templates that follow 102 | # First remove the Google Analytics added parameters, useless for our backend 103 | if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=") { 104 | set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", ""); 105 | set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?"); 106 | set req.url = regsub(req.url, "\?&", "?"); 107 | set req.url = regsub(req.url, "\?$", ""); 108 | } 109 | 110 | # Strip hash, server doesn't need it. 111 | if (req.url ~ "\#") { 112 | set req.url = regsub(req.url, "\#.*$", ""); 113 | } 114 | 115 | # Strip a trailing ? if it exists 116 | if (req.url ~ "\?$") { 117 | set req.url = regsub(req.url, "\?$", ""); 118 | } 119 | 120 | # Some generic cookie manipulation, useful for all templates that follow 121 | # Don't manipulate empty cookies 122 | if (req.http.Cookie !~ "^\s*$") { 123 | # Remove the "has_js" cookie 124 | set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", ""); 125 | 126 | # Remove any Google Analytics based cookies 127 | set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", ""); 128 | set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", ""); 129 | set req.http.Cookie = regsuball(req.http.Cookie, "_gat=[^;]+(; )?", ""); 130 | set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", ""); 131 | set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", ""); 132 | set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", ""); 133 | 134 | # Remove DoubleClick offensive cookies 135 | set req.http.Cookie = regsuball(req.http.Cookie, "__gads=[^;]+(; )?", ""); 136 | 137 | # Remove the Quant Capital cookies (added by some plugin, all __qca) 138 | set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", ""); 139 | 140 | # Remove the AddThis cookies 141 | set req.http.Cookie = regsuball(req.http.Cookie, "__atuv.=[^;]+(; )?", ""); 142 | 143 | # Remove a ";" prefix in the cookie if present 144 | set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", ""); 145 | } 146 | 147 | # Are there cookies left with only spaces or that are empty? 148 | if (req.http.cookie ~ "^\s*$") { 149 | unset req.http.cookie; 150 | } 151 | 152 | #if (req.http.Cache-Control ~ "(?i)no-cache") { 153 | #if (client.ip ~ purge) { 154 | # Ignore requests via proxy caches and badly behaved crawlers 155 | # like msnbot that send no-cache with every request. 156 | #if (! (req.http.Via || req.http.User-Agent ~ "(?i)bot" || req.http.X-Purge)) { 157 | #set req.hash_always_miss = true; # Doesn't seems to refresh the object in the cache 158 | #return(purge); # Couple this with restart in vcl_purge and X-Purge header to avoid loops 159 | #} 160 | #} 161 | #} 162 | 163 | # Large static files are delivered directly to the end-user without 164 | # waiting for Varnish to fully read the file first. 165 | # Varnish 4 fully supports Streaming, so set do_stream in vcl_backend_response() 166 | if (req.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") { 167 | unset req.http.Cookie; 168 | return (hash); 169 | } 170 | 171 | # Remove all cookies for static files 172 | # A valid discussion could be held on this line: do you really need to cache static files that don't cause load? Only if you have memory left. 173 | # Sure, there's disk I/O, but chances are your OS will already have these files in their buffers (thus memory). 174 | # Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/ 175 | if (req.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") { 176 | unset req.http.Cookie; 177 | return (hash); 178 | } 179 | 180 | # Send Surrogate-Capability headers to announce ESI support to backend 181 | set req.http.Surrogate-Capability = "key=ESI/1.0"; 182 | 183 | if (req.http.Authorization) { 184 | # Not cacheable by default 185 | return (pass); 186 | } 187 | 188 | return (hash); 189 | } 190 | 191 | sub vcl_pipe { 192 | # Called upon entering pipe mode. 193 | # In this mode, the request is passed on to the backend, and any further data from both the client 194 | # and backend is passed on unaltered until either end closes the connection. Basically, Varnish will 195 | # degrade into a simple TCP proxy, shuffling bytes back and forth. For a connection in pipe mode, 196 | # no other VCL subroutine will ever get called after vcl_pipe. 197 | 198 | # Note that only the first request to the backend will have 199 | # X-Forwarded-For set. If you use X-Forwarded-For and want to 200 | # have it set for all requests, make sure to have: 201 | # set bereq.http.connection = "close"; 202 | # here. It is not set by default as it might break some broken web 203 | # applications, like IIS with NTLM authentication. 204 | 205 | # set bereq.http.Connection = "Close"; 206 | 207 | # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html) 208 | if (req.http.upgrade) { 209 | set bereq.http.upgrade = req.http.upgrade; 210 | } 211 | 212 | return (pipe); 213 | } 214 | 215 | sub vcl_pass { 216 | # Called upon entering pass mode. In this mode, the request is passed on to the backend, and the 217 | # backend's response is passed on to the client, but is not entered into the cache. Subsequent 218 | # requests submitted over the same client connection are handled normally. 219 | 220 | # return (pass); 221 | } 222 | 223 | # The data on which the hashing will take place 224 | sub vcl_hash { 225 | # Called after vcl_recv to create a hash value for the request. This is used as a key 226 | # to look up the object in Varnish. 227 | 228 | hash_data(req.url); 229 | 230 | if (req.http.host) { 231 | hash_data(req.http.host); 232 | } else { 233 | hash_data(server.ip); 234 | } 235 | 236 | # hash cookies for requests that have them 237 | if (req.http.Cookie) { 238 | hash_data(req.http.Cookie); 239 | } 240 | 241 | # Cache the HTTP vs HTTPs separately 242 | if (req.http.X-Forwarded-Proto) { 243 | hash_data(req.http.X-Forwarded-Proto); 244 | } 245 | } 246 | 247 | sub vcl_hit { 248 | # Called when a cache lookup is successful. 249 | 250 | if (obj.ttl >= 0s) { 251 | # A pure unadultered hit, deliver it 252 | return (deliver); 253 | } 254 | 255 | # https://www.varnish-cache.org/docs/trunk/users-guide/vcl-grace.html 256 | # When several clients are requesting the same page Varnish will send one request to the backend and place the others 257 | # on hold while fetching one copy from the backend. In some products this is called request coalescing and Varnish does 258 | # this automatically. 259 | # If you are serving thousands of hits per second the queue of waiting requests can get huge. There are two potential 260 | # problems - one is a thundering herd problem - suddenly releasing a thousand threads to serve content might send the 261 | # load sky high. Secondly - nobody likes to wait. To deal with this we can instruct Varnish to keep the objects in cache 262 | # beyond their TTL and to serve the waiting requests somewhat stale content. 263 | 264 | # if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) { 265 | # return (deliver); 266 | # } else { 267 | # return (miss); 268 | # } 269 | 270 | # We have no fresh fish. Lets look at the stale ones. 271 | if (std.healthy(req.backend_hint)) { 272 | # Backend is healthy. Limit age to 10s. 273 | if (obj.ttl + 10s > 0s) { 274 | #set req.http.grace = "normal(limited)"; 275 | return (deliver); 276 | } 277 | } else { 278 | # backend is sick - use full grace 279 | if (obj.ttl + obj.grace > 0s) { 280 | #set req.http.grace = "full"; 281 | return (deliver); 282 | } 283 | } 284 | } 285 | 286 | sub vcl_miss { 287 | # Called after a cache lookup if the requested document was not found in the cache. Its purpose 288 | # is to decide whether or not to attempt to retrieve the document from the backend, and which 289 | # backend to use. 290 | 291 | return (fetch); 292 | } 293 | 294 | # Handle the HTTP request coming from our backend 295 | sub vcl_backend_response { 296 | # Called after the response headers has been successfully retrieved from the backend. 297 | 298 | # Pause ESI request and remove Surrogate-Control header 299 | if (beresp.http.Surrogate-Control ~ "ESI/1.0") { 300 | unset beresp.http.Surrogate-Control; 301 | set beresp.do_esi = true; 302 | } 303 | 304 | # Enable cache for all static files 305 | # The same argument as the static caches from above: monitor your cache size, if you get data nuked out of it, consider giving up the static file cache. 306 | # Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/ 307 | if (bereq.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") { 308 | unset beresp.http.set-cookie; 309 | } 310 | 311 | # Large static files are delivered directly to the end-user without 312 | # waiting for Varnish to fully read the file first. 313 | # Varnish 4 fully supports Streaming, so use streaming here to avoid locking. 314 | if (bereq.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") { 315 | unset beresp.http.set-cookie; 316 | set beresp.do_stream = true; # Check memory usage it'll grow in fetch_chunksize blocks (128k by default) if the backend doesn't send a Content-Length header, so only enable it for big objects 317 | } 318 | 319 | # Sometimes, a 301 or 302 redirect formed via Apache's mod_rewrite can mess with the HTTP port that is being passed along. 320 | # This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box. 321 | # A redirect can then often redirect the end-user to a URL on :8080, where it should be :80. 322 | # This may need finetuning on your setup. 323 | # 324 | # To prevent accidental replace, we only filter the 301/302 redirects for now. 325 | if (beresp.status == 301 || beresp.status == 302) { 326 | set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", ""); 327 | } 328 | 329 | # Set 2min cache if unset for static files 330 | if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") { 331 | set beresp.ttl = 120s; # Important, you shouldn't rely on this, SET YOUR HEADERS in the backend 332 | set beresp.uncacheable = true; 333 | return (deliver); 334 | } 335 | 336 | # Don't cache 50x responses 337 | if (beresp.status == 500 || beresp.status == 502 || beresp.status == 503 || beresp.status == 504) { 338 | return (abandon); 339 | } 340 | 341 | # Allow stale content, in case the backend goes down. 342 | # make Varnish keep all objects for 6 hours beyond their TTL 343 | set beresp.grace = 6h; 344 | 345 | return (deliver); 346 | } 347 | 348 | # The routine when we deliver the HTTP request to the user 349 | # Last chance to modify headers that are sent to the client 350 | sub vcl_deliver { 351 | # Called before a cached object is delivered to the client. 352 | 353 | if (obj.hits > 0) { # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed 354 | set resp.http.X-Cache = "HIT"; 355 | } else { 356 | set resp.http.X-Cache = "MISS"; 357 | } 358 | 359 | # Please note that obj.hits behaviour changed in 4.0, now it counts per objecthead, not per object 360 | # and obj.hits may not be reset in some cases where bans are in use. See bug 1492 for details. 361 | # So take hits with a grain of salt 362 | set resp.http.X-Cache-Hits = obj.hits; 363 | 364 | # Remove some headers: PHP version 365 | unset resp.http.X-Powered-By; 366 | 367 | # Remove some headers: Apache version & OS 368 | unset resp.http.Server; 369 | unset resp.http.X-Drupal-Cache; 370 | unset resp.http.X-Varnish; 371 | unset resp.http.Via; 372 | unset resp.http.Link; 373 | unset resp.http.X-Generator; 374 | 375 | return (deliver); 376 | } 377 | 378 | sub vcl_purge { 379 | # Only handle actual PURGE HTTP methods, everything else is discarded 380 | if (req.method == "PURGE") { 381 | # restart request 382 | set req.http.X-Purge = "Yes"; 383 | return(restart); 384 | } 385 | } 386 | 387 | sub vcl_synth { 388 | if (resp.status == 720) { 389 | # We use this special error status 720 to force redirects with 301 (permanent) redirects 390 | # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html")); 391 | set resp.http.Location = resp.reason; 392 | set resp.status = 301; 393 | return (deliver); 394 | } elseif (resp.status == 721) { 395 | # And we use error status 721 to force redirects with a 302 (temporary) redirect 396 | # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html")); 397 | set resp.http.Location = resp.reason; 398 | set resp.status = 302; 399 | return (deliver); 400 | } 401 | 402 | return (deliver); 403 | } 404 | 405 | 406 | sub vcl_fini { 407 | # Called when VCL is discarded only after all requests have exited the VCL. 408 | # Typically used to clean up VMODs. 409 | 410 | return (ok); 411 | } 412 | --------------------------------------------------------------------------------