├── Dockerfile ├── README.md └── watch.py /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.5 2 | MAINTAINER Hubert Chathi 3 | EXPOSE 80 4 | ENV K8SBASE="http://127.0.0.1:8080" 5 | RUN apk --no-cache add --update varnish python py-jinja2 py-requests py-gevent ca-certificates wget \ 6 | && wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.1.1/dumb-init_1.1.1_amd64 \ 7 | && chmod +x /usr/local/bin/dumb-init 8 | WORKDIR /opt/varnish 9 | COPY watch.py /opt/varnish/ 10 | CMD ["/usr/local/bin/dumb-init", "-c", "./watch.py"] 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dynamic Varnish for Kubernetes 2 | 3 | by MuchLearning 4 | 5 | Updates Varnish config based on Kubernetes configuration changes. Intended for 6 | use with https://github.com/muchlearning/kubernetes-haproxy. 7 | 8 | ## Introduction 9 | 10 | This pod watches Kubernetes (or more specifically, the etcd2 used by 11 | Kubernetes) for configuration changes (creates, deletes). When a change is 12 | detected, the configuration is updated, and Varnish is gracefully reloaded if 13 | needed. It uses etcd2's watch feature rather than polling, so updates should 14 | be near-instantaneous. 15 | 16 | ### Important Note 17 | 18 | Previous versions pulled the configuration from etcd, but this failed when 19 | Kubernetes used etcd3. This version now pulls the configuration from the 20 | Kubernetes server instead, but will require changes to the configuration. In 21 | particular, the `K8SBASE` environment variable needs to be set, pointing to the 22 | URL of the Kubernetes API server. 23 | 24 | ## Configuration 25 | 26 | ### Environment variables 27 | 28 | - `K8SBASE`: (required) the base URL for the Kubernetes API server (with no 29 | trailing slash). The URL must be an HTTP URL; HTTPS is not (yet) supported. 30 | Defaults to `http://127.0.0.1:8080` (which will probably not work). 31 | 32 | ### ConfigMaps 33 | 34 | The HAProxy configuration is driven by some Kubernetes configmaps and secrets 35 | in the `lb` namespace. The pod watches these and updates the configuration 36 | when they change. 37 | 38 | - `services` configmap: each key defines a service to be exposed. The value is 39 | a JSON object as defined in 40 | https://github.com/muchlearning/kubernetes-haproxy, but with the following 41 | additional keys used by the example template 42 | - `varnish`: an object with the following keys: 43 | - `recv`, `backend_fetch`, `backend_response`, `deliver`: (optional) VCL to 44 | execute in the `vcl_*` subroutines when a request is made for the service 45 | - `config` configmap: the `varnishtemplate` key in this configmap defines a 46 | Jinja2 template to use to generate the Varnish VCL file. The template is 47 | passed these replacements: 48 | - `services`: a list of services, each of which is a dict corresponding to 49 | the values given in the `services` configmap above. The list is sorted in 50 | the order of the service name (the keys in the `services` configmap). In 51 | addition to the keys given in the service's JSON object, each service has 52 | the following keys: 53 | - `name`: the name of the service 54 | - `env`: a dict containing the process' environment variables 55 | 56 | #### Examples 57 | 58 | See `examples/varnish.yaml` in 59 | https://github.com/muchlearning/kubernetes-haproxy. 60 | -------------------------------------------------------------------------------- /watch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import base64 4 | import gevent 5 | import gevent.event 6 | import hashlib 7 | import json 8 | import jinja2 9 | from operator import itemgetter 10 | import os 11 | import os.path 12 | import re 13 | import requests 14 | import subprocess 15 | import sys 16 | import time 17 | 18 | from gevent import monkey 19 | monkey.patch_all() 20 | 21 | K8SBASE = os.getenv("K8SBASE") or "http://127.0.0.1:8000" 22 | 23 | change_event = gevent.event.Event() 24 | 25 | generation = None 26 | 27 | def re_escape(text): 28 | # NOTE: for now, only escape the special characters that can show up in 29 | # domain names (i.e., just '.') 30 | # FIXME: escape other characters 31 | return re.sub('([.])', r'\\\1', text) 32 | 33 | def load_services(services_nodes): 34 | services = {} 35 | cache = {} 36 | for key, service in services_nodes.iteritems(): 37 | service_config = json.loads(service) 38 | set_service(services, key, service_config) 39 | return services 40 | 41 | def set_service(services, key, service_config): 42 | services[key] = service_config 43 | services[key]["name"] = key 44 | 45 | class K8sWatcher(gevent.Greenlet): 46 | def _run(self): 47 | while True: 48 | req = requests.get(K8SBASE + "/api/v1/watch/" + self._path, stream=True) 49 | lines = req.iter_lines() 50 | for line in lines: 51 | self._process_line(line) 52 | 53 | def _process_line(self, line): 54 | data = json.loads(line) 55 | return self._process_json(data) 56 | 57 | class ConfigWatcher(K8sWatcher): 58 | def __init__(self, namespace, configmap = None, configname = None): 59 | K8sWatcher.__init__(self) 60 | self._path = "namespaces/" + namespace + "/configmaps" 61 | self.configmap = configmap 62 | if configmap: 63 | self._path = self.path + "/" + configmap 64 | self.configname = configname 65 | if configname: 66 | self.config = None 67 | else: 68 | self.config = {} 69 | 70 | def _process_json(self, json): 71 | if (json["object"] and json["object"]["kind"] == "ConfigMap"): 72 | obj = json["object"] 73 | if self.configname: 74 | if "data" in obj and self.configname in obj["data"]: 75 | self.config = obj["data"][self.configname] 76 | elif self.configmap: 77 | if "data" in obj: 78 | self.config = obj["data"] 79 | else: 80 | if obj["metadata"]["name"] not in self.config: 81 | self.config[obj["metadata"]["name"]] = {} 82 | if "data" in obj: 83 | self.config[obj["metadata"]["name"]] = obj["data"] 84 | change_event.set() 85 | 86 | def refresh(): 87 | global generation 88 | try: 89 | response = etcd_open("/v2/keys/registry/configmaps/lb/services") 90 | response_json = response.read() 91 | generation = int(response.info().getheader('X-Etcd-Index')) 92 | finally: 93 | try: 94 | response.close() 95 | except: 96 | pass 97 | 98 | services = load_services(json.loads(json.loads(response_json)["node"]["value"])) 99 | 100 | try: 101 | response = etcd_open("/v2/keys/registry/configmaps/lb/config") 102 | response_json = response.read() 103 | finally: 104 | try: 105 | response.close() 106 | except: 107 | pass 108 | 109 | node = json.loads(response_json)["node"] 110 | config = json.loads(node["value"])["data"] 111 | template = config["varnishtemplate"] 112 | return { 113 | "services": services, 114 | "template": template 115 | } 116 | 117 | config_watcher = ConfigWatcher("lb") 118 | config_watcher.start() 119 | 120 | if __name__ == "__main__": 121 | lasthash = None 122 | count = 0 123 | started = False 124 | templ_env = jinja2.Environment() 125 | templ_env.filters['re_escape'] = re_escape 126 | while True: 127 | change_event.wait() 128 | change_event.clear() 129 | if "config" in config_watcher.config and "varnishtemplate" in config_watcher.config["config"] \ 130 | and "services" in config_watcher.config: 131 | cfg = config_watcher.config 132 | serviceslist = load_services(cfg["services"]).values() 133 | serviceslist.sort(key=itemgetter("name")) 134 | config = templ_env.from_string(cfg["config"]["varnishtemplate"]).render( 135 | services=serviceslist, 136 | env=os.environ) 137 | changed = False 138 | currhash = hashlib.sha512(config).digest() 139 | if currhash != lasthash: 140 | changed = True 141 | sys.stderr.write("Debug: writing new config\n") 142 | with open("varnish.vcl", "w") as f: 143 | f.write(config) 144 | else: 145 | sys.stderr.write("Debug: config file did not change\n") 146 | lasthash = currhash 147 | 148 | if changed: 149 | count += 1 150 | vclname = "%s-%d" % (time.strftime("%Y-%m-%dT%H:%M:%S"), count) 151 | if started: 152 | cmd = ["/usr/bin/varnishadm", "vcl.load", vclname, os.path.abspath("varnish.vcl")] 153 | sys.stderr.write("Debug: compiling new VCL (%s)\n" % vclname) 154 | if subprocess.call(cmd): 155 | sys.stderr.write("ERROR: could not compile VCL") 156 | else: 157 | cmd = ["/usr/bin/varnishadm", "vcl.use", vclname] 158 | sys.stderr.write("Debug: using new VCL\n") 159 | subprocess.call(cmd) 160 | else: 161 | cmd = ["/usr/sbin/varnishd", "-f", os.path.abspath("varnish.vcl"), "-a", ":80"] 162 | storage = (os.getenv("STORAGE") or "malloc").split(";") 163 | for s in storage: 164 | cmd += ["-s", s] 165 | 166 | sys.stderr.write("Debug: starting varnish\n") 167 | subprocess.call(cmd) 168 | started = True 169 | --------------------------------------------------------------------------------