├── README.md └── main.go /README.md: -------------------------------------------------------------------------------- 1 | # Swarm Proxy 2 | 3 | ## About 4 | 5 | Swarm Proxy is a program for managing Docker Swarm services behind a reverse 6 | proxy. It uses the Docker events stream to monitor for services being created 7 | and removed, and then uses Swarm Configs to update a reverse proxy service to 8 | correctly route traffic to those services. It is stateless. 9 | 10 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | "text/template" 10 | 11 | "github.com/pkg/errors" 12 | 13 | "github.com/docker/docker/api/types" 14 | "github.com/docker/docker/api/types/events" 15 | "github.com/docker/docker/api/types/filters" 16 | "github.com/docker/docker/api/types/swarm" 17 | "github.com/docker/docker/client" 18 | 19 | log "github.com/sirupsen/logrus" 20 | ) 21 | 22 | // TODO LIST: 23 | // * support generic templates (allows using any reverse proxy) 24 | // * reconcile the proxy state when the manager comes up by checking that 25 | // all the configs are in the right place 26 | // * support updates 27 | // * support configuring/changing default values 28 | // * optimize by caching 29 | // * auto-generate proxy service and network, instead of relying on them 30 | // already existing. 31 | // * allow non-standard ports; right now we default to port 80 32 | // * batch updates to the proxy, so that creating a lot of services in a short 33 | // time period is done more efficiently 34 | // * handle more errors and edge cases 35 | // * write Dockerfile 36 | 37 | // I'm cheating and baking this templating into the file because then it gets 38 | // compiled into the binary. In a more complete and less proof-of-concept 39 | // implementation, we would read this template from a file (which the user 40 | // could customize in a child image!) 41 | const nginxConfigTemplate = `server { 42 | # This config file was generated by ProxyManager 43 | server_name {{.ServerName}}; 44 | {{/* TODO(dperny): what to do w/ access_log directive? */}} 45 | location / { 46 | proxy_pass http://{{.ServiceName}}; 47 | } 48 | }` 49 | 50 | const ( 51 | configServerNameLabel = "proxymanager_config_server_name" 52 | 53 | defaultConfigFilePrefix = "proxymanager_config_" 54 | defaultProxyService = "proxy" 55 | defaultProxyNetwork = "proxynet" 56 | nginxConfigLocation = "/etc/nginx/conf.d/" 57 | configServiceIDLabel = "proxyServiceId" 58 | ) 59 | 60 | type ServerConfig struct { 61 | ServerName string // the domain name to respond as 62 | ServiceName string // the name of the service responding 63 | } 64 | 65 | // ProxyManager is a struct that holds data for the proxy manager 66 | type ProxyManager struct { 67 | Cli *client.Client 68 | ProxyService string // the name of the service acting as the proxy 69 | ProxyNetwork string // the network that proxied services are connected to 70 | ConfigTemplate *template.Template 71 | proxyNwId string // the ID of the proxy network 72 | } 73 | 74 | func NewProxyManager() (*ProxyManager, error) { 75 | t := template.Must(template.New("config").Parse(nginxConfigTemplate)) 76 | cli, err := client.NewEnvClient() 77 | if err != nil { 78 | return nil, err 79 | } 80 | return &ProxyManager{ 81 | Cli: cli, 82 | ProxyService: defaultProxyService, 83 | ProxyNetwork: defaultProxyNetwork, 84 | ConfigTemplate: t, 85 | }, nil 86 | } 87 | 88 | func (p *ProxyManager) IsServiceManaged(service swarm.Service) bool { 89 | log.Debugf("Checking if service has nw %v", p.ProxyNetwork) 90 | managed := false 91 | for _, network := range service.Spec.TaskTemplate.Networks { 92 | if network.Target == p.ProxyNetwork { 93 | log.Debugf("Service has nw %v", network.Target) 94 | managed = true 95 | break 96 | } 97 | } 98 | return managed 99 | } 100 | 101 | func (p *ProxyManager) CreateConfig(ctx context.Context, service swarm.Service) error { 102 | log.Debug("Creating config for service") 103 | // Generate the service config 104 | // set the server name to be the service name 105 | serverConfig := ServerConfig{ 106 | ServerName: service.Spec.Name, 107 | ServiceName: service.Spec.Name, 108 | } 109 | 110 | // if the override is set, use it as the name instead of the server name 111 | if name, ok := service.Spec.Labels[configServerNameLabel]; ok { 112 | log.Debugf("Overriding service server name with %v", name) 113 | serverConfig.ServerName = name 114 | } 115 | 116 | // create a buffer to hold the generated config 117 | buf := &bytes.Buffer{} 118 | err := p.ConfigTemplate.Execute(buf, serverConfig) 119 | log.Debug(buf.String()) 120 | // then create a config spec with the newly created config 121 | configSpec := swarm.ConfigSpec{ 122 | Annotations: swarm.Annotations{ 123 | Name: defaultConfigFilePrefix + serverConfig.ServiceName, 124 | Labels: map[string]string{ 125 | // add the ID of this managed service to the config's labels 126 | // helps with removals later 127 | configServiceIDLabel: service.ID, 128 | }, 129 | }, 130 | Data: buf.Bytes(), 131 | } 132 | // create the config 133 | log.Debugf("Creating new config %v", configSpec.Name) 134 | resp, err := p.Cli.ConfigCreate(ctx, configSpec) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | // create the config reference 140 | ref := &swarm.ConfigReference{ 141 | File: &swarm.ConfigReferenceFileTarget{ 142 | Name: nginxConfigLocation + configSpec.Name + ".conf", 143 | // we have to set UID and GID. they're unset, this will fail. 144 | // unsure why sensible defaults aren't assumed. 145 | UID: "0", 146 | GID: "0", 147 | Mode: 444, 148 | }, 149 | ConfigID: resp.ID, 150 | ConfigName: configSpec.Name, 151 | } 152 | 153 | log.Debug("Updating proxy service") 154 | // now update the proxy service to include this config 155 | proxy, _, err := p.Cli.ServiceInspectWithRaw(ctx, p.ProxyService, types.ServiceInspectOptions{}) 156 | if err != nil { 157 | return err 158 | } 159 | // add the config to the service's configs 160 | proxy.Spec.TaskTemplate.ContainerSpec.Configs = append(proxy.Spec.TaskTemplate.ContainerSpec.Configs, ref) 161 | _, err = p.Cli.ServiceUpdate(ctx, proxy.ID, proxy.Version, proxy.Spec, types.ServiceUpdateOptions{}) 162 | if err != nil { 163 | return err 164 | } 165 | log.Infof("Added config for %v", service.ID) 166 | return nil 167 | } 168 | 169 | // RemoveConfig handles events where a proxied service is removed. 170 | func (p *ProxyManager) RemoveConfig(ctx context.Context, actor events.Actor) error { 171 | log.Debug("Checking for remove") 172 | // this function is kinda tricky because we only have the ID of the removed 173 | // service, because it's removed. 174 | 175 | // get the id of the service. this is just for programmer clarity, in case 176 | // the implementation of this function changes later 177 | serviceID := actor.ID 178 | 179 | // First, list all configs 180 | configs, err := p.Cli.ConfigList(ctx, types.ConfigListOptions{}) 181 | if err != nil { 182 | return err 183 | } 184 | 185 | for _, config := range configs { 186 | if id := config.Spec.Labels[configServiceIDLabel]; id == serviceID { 187 | log.Debugf("Found config for %v", serviceID) 188 | // we found a config for this service 189 | // get the proxy service and remove this config from it 190 | service, _, err := p.Cli.ServiceInspectWithRaw(ctx, p.ProxyService, types.ServiceInspectOptions{}) 191 | if err != nil { 192 | return err 193 | } 194 | // `i` is the location of the config in the proxy service's 195 | // config references. if it's still -1 after this loop, that means 196 | // the proxy service doesn't have this config 197 | i := -1 198 | // take the configs array as a variable so we don't have to type the whole chain 199 | refs := service.Spec.TaskTemplate.ContainerSpec.Configs 200 | for j, configRef := range refs { 201 | if configRef.ConfigID == config.ID { 202 | i = j 203 | } 204 | } 205 | if i >= 0 { 206 | log.Debug("Removing config from proxy") 207 | // from github.com/golang/go/wiki/SliceTricks 208 | // delete without preserving order (memory safe) 209 | // replace the target element with the last element 210 | refs[i] = refs[len(refs)-1] 211 | // nil the last element 212 | refs[len(refs)-1] = nil 213 | // cut of the last element 214 | refs = refs[:len(refs)-1] 215 | // now update the service 216 | service.Spec.TaskTemplate.ContainerSpec.Configs = refs 217 | _, err = p.Cli.ServiceUpdate(ctx, service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) 218 | if err != nil { 219 | // TODO(dperny) handle case where the service has been updated 220 | // in the meantime and the version is old 221 | return err 222 | } 223 | } else { 224 | log.Infof("A config for service %v exists, but is not in the proxy's configs") 225 | } 226 | // now delete the old config 227 | // TODO(dperny) what happens if we try to remove the config while 228 | // the service update removing the config is in progress? 229 | log.Infof("Removing config %v", config.Spec.Name) 230 | err = p.Cli.ConfigRemove(ctx, config.ID) 231 | if err != nil { 232 | return err 233 | } 234 | return nil 235 | } 236 | } 237 | 238 | log.Infof("Service %v had no config to remove", serviceID) 239 | return nil 240 | } 241 | 242 | func (p *ProxyManager) HandleEvent(ctx context.Context, event events.Message) { 243 | // check that the service belongs to our network and is managed by us 244 | var err error 245 | switch event.Action { 246 | case "create": 247 | var service swarm.Service 248 | service, _, err = p.Cli.ServiceInspectWithRaw(ctx, event.Actor.ID, types.ServiceInspectOptions{}) 249 | if err != nil { 250 | break 251 | } 252 | if !p.IsServiceManaged(service) { 253 | // TODO(dperny) log that we passed 254 | log.Infof("Service %v is not managed", service.Spec.Name) 255 | break 256 | } 257 | err = p.CreateConfig(ctx, service) 258 | case "update": 259 | err = errors.New("service updates not supported") 260 | case "remove": 261 | err = p.RemoveConfig(ctx, event.Actor) 262 | } 263 | if err != nil { 264 | log.Warnf("Event handling error: %v", err) 265 | } 266 | } 267 | 268 | // Reconcile checks the cluster state to see if all services are wired up 269 | // correctly. It's called once at the start of the program. 270 | func (p *ProxyManager) Reconcile(ctx context.Context) error { 271 | log.Info("Reconciling proxy configs with cluster state") 272 | log.Error("Reconciling not implemented!") 273 | // TODO(dperny) implement 274 | return nil 275 | } 276 | 277 | func (p *ProxyManager) Run(ctx context.Context) error { 278 | // replace the network name with id 279 | nw, err := p.Cli.NetworkInspect(ctx, p.ProxyNetwork, false) 280 | if err != nil { 281 | return err 282 | } 283 | log.Debugf("Setting ProxyNetwork to %v", nw.ID) 284 | p.ProxyNetwork = nw.ID 285 | // Reconcile our current state 286 | if err := p.Reconcile(ctx); err != nil { 287 | return err 288 | } 289 | 290 | // subscribe to service events 291 | filter := filters.NewArgs() 292 | filter.Add("type", events.ServiceEventType) 293 | opts := types.EventsOptions{Filters: filter} 294 | log.Debug("Subscribing to Docker events") 295 | events, errors := p.Cli.Events(ctx, opts) 296 | 297 | for { 298 | select { 299 | case <-ctx.Done(): 300 | // TODO(dperny) check if I need to perform any cleanup 301 | log.Debug("Context done, returning") 302 | return nil 303 | // handle context done 304 | case event := <-events: 305 | // handle events 306 | log.Debug("Recieved Docker event, calling handler") 307 | // TODO(dperny) Can I safely do HandleEvent async? 308 | p.HandleEvent(ctx, event) 309 | case err := <-errors: 310 | log.Error("Recieved event stream error") 311 | // TODO(dperny) Restart event stream 312 | return err 313 | // handle event stream errors 314 | } 315 | } 316 | } 317 | 318 | func main() { 319 | log.SetLevel(log.DebugLevel) 320 | log.Info("Running main function") 321 | ctx, cancel := context.WithCancel(context.Background()) 322 | log.Info("Registering Signal Handlers") 323 | ch := make(chan os.Signal, 1) 324 | signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, os.Interrupt) 325 | defer func() { 326 | signal.Stop(ch) 327 | cancel() 328 | }() 329 | 330 | go func() { 331 | select { 332 | case <-ch: 333 | log.Info("Recieved signal, canceling context") 334 | cancel() 335 | case <-ctx.Done(): 336 | } 337 | }() 338 | 339 | p, err := NewProxyManager() 340 | if err != nil { 341 | log.Fatal(err.Error()) 342 | } 343 | 344 | if err := p.Run(ctx); err != nil { 345 | log.Fatal(err.Error()) 346 | } 347 | log.Info("Exited cleanly") 348 | } 349 | --------------------------------------------------------------------------------