├── .gitignore ├── .dockerignore ├── Dockerfile ├── glide.yaml ├── LICENSE ├── ingress.go ├── route53.go ├── README.md ├── main.go ├── route53_test.go ├── ingress_test.go ├── glide.lock ├── registrator.go └── registrator_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | ingress53 3 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | ingress53 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.6 2 | 3 | ENV IMPORT_PATH="github.com/utilitywarehouse/ingress53" 4 | 5 | ADD . /go/src/${IMPORT_PATH} 6 | 7 | RUN apk add --no-cache \ 8 | ca-certificates \ 9 | && apk add --no-cache --virtual=.builddeps \ 10 | -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ 11 | git \ 12 | go \ 13 | musl-dev \ 14 | && export GOPATH=/go \ 15 | && cd $GOPATH/src/${IMPORT_PATH} \ 16 | && go get ./... \ 17 | && CGO_ENABLED=0 go test -v "${IMPORT_PATH}" \ 18 | && CGO_ENABLED=0 go build -v -ldflags '-s -X "main.appGitHash=$(git rev-parse HEAD)" -extldflags "-static"' -o "/$(basename ${IMPORT_PATH})" . \ 19 | && apk del --no-cache .builddeps \ 20 | && rm -rf $GOPATH 21 | 22 | ENTRYPOINT ["/ingress53"] 23 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/utilitywarehouse/ingress53 2 | import: 3 | - package: github.com/aws/aws-sdk-go 4 | version: ~1.12.25 5 | subpackages: 6 | - aws 7 | - aws/session 8 | - service/route53 9 | - service/route53/route53iface 10 | - package: github.com/hashicorp/logutils 11 | - package: github.com/miekg/dns 12 | - package: github.com/prometheus/client_golang 13 | version: ~0.9.0-pre1 14 | subpackages: 15 | - prometheus 16 | - prometheus/promhttp 17 | - package: github.com/utilitywarehouse/go-operational 18 | subpackages: 19 | - op 20 | - package: k8s.io/api 21 | subpackages: 22 | - extensions/v1beta1 23 | - package: k8s.io/apimachinery 24 | subpackages: 25 | - pkg/apis/meta/v1 26 | - pkg/labels 27 | - pkg/runtime 28 | - pkg/watch 29 | - package: k8s.io/client-go 30 | version: ~5.0.1 31 | subpackages: 32 | - kubernetes 33 | - rest 34 | - tools/cache 35 | - tools/clientcmd 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Utility Warehouse 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /ingress.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "k8s.io/api/extensions/v1beta1" 8 | "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/apimachinery/pkg/runtime" 10 | "k8s.io/apimachinery/pkg/watch" 11 | "k8s.io/client-go/kubernetes" 12 | "k8s.io/client-go/tools/cache" 13 | ) 14 | 15 | type eventHandlerFunc func(eventType watch.EventType, oldIngress *v1beta1.Ingress, newIngress *v1beta1.Ingress) 16 | 17 | type ingressWatcher struct { 18 | client kubernetes.Interface 19 | eventHandler eventHandlerFunc 20 | resyncPeriod time.Duration 21 | labelSelector string 22 | stopChannel chan struct{} 23 | store cache.Store 24 | } 25 | 26 | func newIngressWatcher(client kubernetes.Interface, eventHandler eventHandlerFunc, labelSelector string, resyncPeriod time.Duration) *ingressWatcher { 27 | return &ingressWatcher{ 28 | client: client, 29 | eventHandler: eventHandler, 30 | resyncPeriod: resyncPeriod, 31 | labelSelector: labelSelector, 32 | stopChannel: make(chan struct{}), 33 | } 34 | } 35 | 36 | func (iw *ingressWatcher) Start() { 37 | lw := &cache.ListWatch{ 38 | ListFunc: func(options v1.ListOptions) (runtime.Object, error) { 39 | options.LabelSelector = iw.labelSelector 40 | return iw.client.Extensions().Ingresses(v1.NamespaceAll).List(options) 41 | }, 42 | WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { 43 | options.LabelSelector = iw.labelSelector 44 | return iw.client.Extensions().Ingresses(v1.NamespaceAll).Watch(options) 45 | }, 46 | } 47 | eh := cache.ResourceEventHandlerFuncs{ 48 | AddFunc: func(obj interface{}) { 49 | iw.eventHandler(watch.Added, nil, obj.(*v1beta1.Ingress)) 50 | }, 51 | UpdateFunc: func(oldObj, newObj interface{}) { 52 | iw.eventHandler(watch.Modified, oldObj.(*v1beta1.Ingress), newObj.(*v1beta1.Ingress)) 53 | }, 54 | DeleteFunc: func(obj interface{}) { 55 | iw.eventHandler(watch.Deleted, obj.(*v1beta1.Ingress), nil) 56 | }, 57 | } 58 | store, controller := cache.NewInformer(lw, &v1beta1.Ingress{}, iw.resyncPeriod, eh) 59 | iw.store = store 60 | log.Println("[INFO] starting ingress watcher") 61 | controller.Run(iw.stopChannel) 62 | log.Println("[INFO] ingress watcher stopped") 63 | } 64 | 65 | func (iw *ingressWatcher) Stop() { 66 | log.Println("[INFO] stopping ingress watcher ...") 67 | close(iw.stopChannel) 68 | } 69 | 70 | func (iw *ingressWatcher) HostnameOwners(hostname string) []string { 71 | owners := []string{} 72 | for _, i := range iw.store.List() { 73 | for _, h := range getHostnamesFromIngress(i.(*v1beta1.Ingress)) { 74 | if hostname == h { 75 | owners = append(owners, i.(*v1beta1.Ingress).Name) 76 | } 77 | } 78 | } 79 | return owners 80 | } 81 | 82 | func getHostnamesFromIngress(ingress *v1beta1.Ingress) []string { 83 | hostnames := []string{} 84 | for _, rule := range ingress.Spec.Rules { 85 | found := false 86 | for _, h := range hostnames { 87 | if h == rule.Host { 88 | found = true 89 | break 90 | } 91 | } 92 | if !found { 93 | hostnames = append(hostnames, rule.Host) 94 | } 95 | } 96 | return hostnames 97 | } 98 | -------------------------------------------------------------------------------- /route53.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/service/route53" 11 | "github.com/aws/aws-sdk-go/service/route53/route53iface" 12 | ) 13 | 14 | var ( 15 | errRoute53WaitWatchTimedOut = errors.New("timed out waiting for changes to be applied") 16 | 17 | defaultRoute53RecordTTL int64 = 60 18 | defaultRoute53ZoneWaitWatchInterval = 10 * time.Second 19 | defaultRoute53ZoneWaitWatchTimeout = 2 * time.Minute 20 | ) 21 | 22 | type route53Zone struct { 23 | api route53iface.Route53API 24 | Name string 25 | ID string 26 | Nameservers []string 27 | } 28 | 29 | func newRoute53Zone(zoneID string, route53session route53iface.Route53API) (*route53Zone, error) { 30 | ret := &route53Zone{api: route53session} 31 | if err := ret.setZone(zoneID); err != nil { 32 | return nil, err 33 | } 34 | return ret, nil 35 | } 36 | 37 | func (z *route53Zone) UpsertCnames(records []cnameRecord) error { 38 | return z.changeCnames(route53.ChangeActionUpsert, records) 39 | } 40 | 41 | func (z *route53Zone) DeleteCnames(records []cnameRecord) error { 42 | return z.changeCnames(route53.ChangeActionDelete, records) 43 | } 44 | 45 | func (z *route53Zone) changeCnames(action string, records []cnameRecord) error { 46 | changes := make([]*route53.Change, len(records)) 47 | for i, r := range records { 48 | changes[i] = &route53.Change{ 49 | Action: aws.String(action), 50 | ResourceRecordSet: &route53.ResourceRecordSet{ 51 | Name: aws.String(r.Hostname), 52 | TTL: aws.Int64(defaultRoute53RecordTTL), 53 | Type: aws.String(route53.RRTypeCname), 54 | ResourceRecords: []*route53.ResourceRecord{{Value: aws.String(r.Target)}}, 55 | }, 56 | } 57 | } 58 | resp, err := z.api.ChangeResourceRecordSets(&route53.ChangeResourceRecordSetsInput{ 59 | ChangeBatch: &route53.ChangeBatch{ 60 | Changes: changes, 61 | Comment: aws.String("updated by ingress53"), 62 | }, 63 | HostedZoneId: aws.String(z.ID), 64 | }) 65 | if err != nil { 66 | return err 67 | } 68 | log.Println("[DEBUG] route53 changes have been submitted, waiting for nameservers to sync") 69 | return z.waitForSync(*resp.ChangeInfo.Id) 70 | } 71 | 72 | func (z *route53Zone) waitForSync(changeID string) error { 73 | timeout := time.NewTimer(defaultRoute53ZoneWaitWatchTimeout) 74 | tick := time.NewTicker(defaultRoute53ZoneWaitWatchInterval) 75 | defer func() { 76 | timeout.Stop() 77 | tick.Stop() 78 | }() 79 | var err error 80 | var change *route53.GetChangeOutput 81 | for { 82 | select { 83 | case <-tick.C: 84 | change, err = z.api.GetChange(&route53.GetChangeInput{Id: aws.String(changeID)}) 85 | if err != nil { 86 | return err 87 | } 88 | if *change.ChangeInfo.Status == route53.ChangeStatusInsync { 89 | return nil 90 | } 91 | log.Printf("[DEBUG] route53 changes are still being applied, waiting for %s", defaultRoute53ZoneWaitWatchInterval.String()) 92 | case <-timeout.C: 93 | return errRoute53WaitWatchTimedOut 94 | } 95 | } 96 | } 97 | 98 | func (z *route53Zone) setZone(id string) error { 99 | zone, err := z.api.GetHostedZone(&route53.GetHostedZoneInput{Id: aws.String(id)}) 100 | if err != nil { 101 | return err 102 | } 103 | z.Name = *zone.HostedZone.Name 104 | z.ID = *zone.HostedZone.Id 105 | z.Nameservers = make([]string, len(zone.DelegationSet.NameServers)) 106 | for i, ns := range zone.DelegationSet.NameServers { 107 | z.Nameservers[i] = *ns 108 | } 109 | return nil 110 | } 111 | 112 | func (z *route53Zone) Domain() string { 113 | return z.Name 114 | } 115 | 116 | func (z *route53Zone) ListNameservers() []string { 117 | ret := make([]string, len(z.Nameservers)) 118 | for i, ns := range z.Nameservers { 119 | ret[i] = fmt.Sprintf("%s:53", ns) 120 | } 121 | return ret 122 | } 123 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ingress53 - deprecated 2 | 3 | DEPRECATED in favour of https://github.com/kubernetes-incubator/external-dns 4 | 5 | ingress53 is a service designed to run in kubernetes and maintain DNS records for the cluster's ingress resources in AWS Route53. 6 | 7 | It will watch the kubernetes API (using the service token) for any Ingress resource changes and try to apply those records to route53 in Amazon, mapping the record to the "target", which is the dns name of the ingress endpoint for your cluster. 8 | 9 | # Requirements 10 | 11 | You need to export the following env variables to be able to use AWS APIs: 12 | 13 | ```sh 14 | export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE 15 | export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY 16 | ``` 17 | 18 | The minimum AWS policy you can use: 19 | ```json 20 | { 21 | "Version": "2012-10-17", 22 | "Statement": [ 23 | { 24 | "Effect": "Allow", 25 | "Action": "route53:ListHostedZonesByName", 26 | "Resource": "*" 27 | }, 28 | { 29 | "Effect": "Allow", 30 | "Action": [ 31 | "route53:GetHostedZone", 32 | "route53:ChangeResourceRecordSets" 33 | ], 34 | "Resource": "arn:aws:route53:::hostedzone/XXXXXXXXXXXXXX" 35 | }, 36 | { 37 | "Effect": "Allow", 38 | "Action": "route53:GetChange", 39 | "Resource": "arn:aws:route53:::change/*" 40 | } 41 | ] 42 | } 43 | ``` 44 | 45 | # Usage 46 | 47 | A kubernetes selector is used to specify the target (entry point of the cluster). 48 | 49 | You will need to create a dns record that points to your ingress endpoint[s]. We will use this to CNAME all ingress resource entries to that "target". 50 | 51 | Your setup might look like this: 52 | 53 | - A ingress controller (nginx/traefik) kubernetes service running on a nodePort (:8080) 54 | - ELB that serves all worker nodes on :8080 55 | - A CNAME for the elb `private.cluster-entrypoint.com` > `my-loadbalancer-1234567890.us-west-2.elb.amazonaws.com` 56 | - ingress53 service running inside the cluster 57 | 58 | Now, if you were to create an ingress kubernetes resource: 59 | 60 | ```yaml 61 | apiVersion: extensions/v1beta1 62 | kind: Ingress 63 | metadata: 64 | name: my-app 65 | labels: 66 | ingress53.target: private.cluster-entrypoint.com 67 | spec: 68 | rules: 69 | - host: my-app.example.com 70 | http: 71 | paths: 72 | - path: / 73 | backend: 74 | serviceName: my-app 75 | servicePort: 80 76 | ``` 77 | 78 | ingress53 will create a CNAME record in route53: `my-app.example.com` > `private.cluster-entrypoint.com` 79 | 80 | You can test it locally (please refer to the command line help for more options): 81 | 82 | ```sh 83 | ./ingress53 \ 84 | -route53-zone-id=XXXXXXXXXXXXXX \ 85 | -target=private.cluster-entrypoint.com \ 86 | -target=public.cluster-entrypoint.com \ 87 | -kubernetes-config=$HOME/.kube/config \ 88 | -dry-run 89 | ``` 90 | 91 | You can use the generated docker image ([quay.io/utilitywarehouse/ingress53](https://quay.io/repository/utilitywarehouse/ingress53?tab=tags)) to deploy it on your kubernetes cluster. 92 | 93 | ## Example kubernetes manifests 94 | 95 | ```yaml 96 | --- 97 | apiVersion: v1 98 | kind: Service 99 | metadata: 100 | name: ingress53 101 | labels: 102 | app: ingress53 103 | namespace: kube-system 104 | annotations: 105 | prometheus.io/scrape: 'true' 106 | prometheus.io/path: /metrics 107 | prometheus.io/port: '5000' 108 | spec: 109 | ports: 110 | - name: web 111 | protocol: TCP 112 | port: 80 113 | targetPort: 5000 114 | selector: 115 | app: ingress53 116 | --- 117 | apiVersion: extensions/v1beta1 118 | kind: Deployment 119 | metadata: 120 | labels: 121 | app: ingress53 122 | name: ingress53 123 | namespace: kube-system 124 | spec: 125 | replicas: 1 126 | template: 127 | metadata: 128 | labels: 129 | app: ingress53 130 | name: ingress53 131 | spec: 132 | containers: 133 | - name: ingress53 134 | image: quay.io/repository/utilitywarehouse/ingress53:2.0.0 135 | args: 136 | - -route53-zone-id=XXXXXXXXXXXXXX 137 | - -target=private.cluster-entrypoint.com 138 | - -target=public.cluster-entrypoint.com 139 | resources: 140 | requests: 141 | cpu: 10m 142 | memory: 64Mi 143 | ports: 144 | - containerPort: 5000 145 | name: web 146 | protocol: TCP 147 | env: 148 | - name: AWS_ACCESS_KEY_ID 149 | valueFrom: 150 | secretKeyRef: 151 | name: ingress53 152 | key: aws_access_key_id 153 | - name: AWS_SECRET_ACCESS_KEY 154 | valueFrom: 155 | secretKeyRef: 156 | name: ingress53 157 | key: aws_secret_access_key 158 | ``` 159 | 160 | ## Building 161 | 162 | If you need to build manually: 163 | 164 | ``` 165 | $ git clone git@github.com:utilitywarehouse/ingress53.git 166 | $ cd ingress53 167 | $ go build . 168 | ``` 169 | 170 | The project uses [glide](https://glide.sh/) to manage dependencies for development purposes but you don't need to use it, `go get` will work just as well. 171 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "os" 9 | "os/signal" 10 | "strings" 11 | 12 | "github.com/hashicorp/logutils" 13 | "github.com/prometheus/client_golang/prometheus" 14 | "github.com/prometheus/client_golang/prometheus/promhttp" 15 | "github.com/utilitywarehouse/go-operational/op" 16 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 17 | "k8s.io/client-go/tools/clientcmd" 18 | ) 19 | 20 | // Define a type named "strslice" as a slice of strings 21 | type strslice []string 22 | 23 | // Now, for our new type, implement the two methods of 24 | // the flag.Value interface... 25 | // The first method is String() string 26 | func (s *strslice) String() string { 27 | return fmt.Sprint(*s) 28 | } 29 | 30 | // Set is the method to set the flag value, part of the flag.Value interface. 31 | // Set's argument is a string to be parsed to set the flag. 32 | // It's a comma-separated list, so we split it. 33 | func (s *strslice) Set(value string) error { 34 | for _, target := range strings.Split(value, ",") { 35 | *s = append(*s, target) 36 | } 37 | return nil 38 | } 39 | 40 | var ( 41 | appGitHash = "master" 42 | 43 | // Define a flag to accumulate durations. Because it has a special type, 44 | // we need to use the Var function and therefore create the flag during 45 | // init. 46 | targets strslice 47 | 48 | kubeConfig = flag.String("kubernetes-config", "", "path to the kubeconfig file, if unspecified then in-cluster config will be used") 49 | targetLabelName = flag.String("target-label", "ingress53.target", "Kubernetes key of the label that specifies the target type") 50 | r53ZoneID = flag.String("route53-zone-id", "", "route53 hosted DNS zone id") 51 | debugLogs = flag.Bool("debug", false, "enables debug logs") 52 | dryRun = flag.Bool("dry-run", false, "if set, ingress53 will not make any Route53 changes") 53 | 54 | metricUpdatesApplied = prometheus.NewCounterVec( 55 | prometheus.CounterOpts{ 56 | Namespace: "ingress53", 57 | 58 | Subsystem: "route53", 59 | Name: "updates_applied", 60 | Help: "number of route53 updates", 61 | }, 62 | []string{"hostname", "action"}, 63 | ) 64 | 65 | metricUpdatesReceived = prometheus.NewCounterVec( 66 | prometheus.CounterOpts{ 67 | Namespace: "ingress53", 68 | Subsystem: "kubernetes", 69 | Name: "updates_received", 70 | Help: "number of route53 updates", 71 | }, 72 | []string{"ingress", "action"}, 73 | ) 74 | 75 | metricUpdatesRejected = prometheus.NewCounter( 76 | prometheus.CounterOpts{ 77 | Namespace: "ingress53", 78 | Subsystem: "kubernetes", 79 | Name: "updates_rejected", 80 | Help: "number of route53 updates rejected", 81 | }, 82 | ) 83 | 84 | // ListWatch runs every 30 seconds (approx). That means that we can allow up to 9 85 | // errors on a per 5m rate of the following metric otherwise every call to kube 86 | // api is failing (so rate > 0.03 => evry call to api fails) 87 | metricKubernetesIOError = prometheus.NewCounter( 88 | prometheus.CounterOpts{ 89 | Namespace: "ingress53", 90 | Subsystem: "kubernetes", 91 | Name: "io_error", 92 | Help: "number of errors occured while talking to kube api", 93 | }, 94 | ) 95 | ) 96 | 97 | // UpdateKubernetesIOErrorCount: to keep count of errors while talking to kube api 98 | func UpdateKubernetesIOErrorCount(err error) { 99 | metricKubernetesIOError.Inc() 100 | } 101 | 102 | func main() { 103 | prometheus.MustRegister(metricUpdatesApplied) 104 | prometheus.MustRegister(metricUpdatesReceived) 105 | prometheus.MustRegister(metricUpdatesRejected) 106 | prometheus.MustRegister(metricKubernetesIOError) 107 | 108 | // Register KubernetesIO Error Handling 109 | utilruntime.ErrorHandlers = append(utilruntime.ErrorHandlers, UpdateKubernetesIOErrorCount) 110 | 111 | flag.Var(&targets, "target", "List of endpoints (ELB) targets to map ingress records to") 112 | flag.Parse() 113 | 114 | luf := &logutils.LevelFilter{ 115 | Levels: []logutils.LogLevel{"DEBUG", "INFO", "ERROR"}, 116 | MinLevel: logutils.LogLevel("INFO"), 117 | Writer: os.Stdout, 118 | } 119 | if *debugLogs { 120 | luf.MinLevel = logutils.LogLevel("DEBUG") 121 | } 122 | log.SetOutput(luf) 123 | 124 | ro := registratorOptions{ 125 | Targets: targets, 126 | TargetLabelName: *targetLabelName, 127 | Route53ZoneID: *r53ZoneID, 128 | } 129 | if *kubeConfig != "" { 130 | config, err := clientcmd.BuildConfigFromFlags("", *kubeConfig) 131 | if err != nil { 132 | log.Printf("[ERROR] could not create kubernetes client: %+v", err) 133 | os.Exit(1) 134 | } 135 | ro.KubernetesConfig = config 136 | } 137 | 138 | r, err := newRegistratorWithOptions(ro) 139 | if err != nil { 140 | log.Printf("[ERROR] could not create registrator: %+v", err) 141 | os.Exit(1) 142 | } 143 | 144 | sigChannel := make(chan os.Signal, 1) 145 | signal.Notify(sigChannel, os.Interrupt) 146 | go func() { 147 | <-sigChannel 148 | log.Println("[INFO] interrupt singal: shutting down ...") 149 | r.Stop() 150 | }() 151 | 152 | go func() { 153 | log.Printf("[INFO] starting HTTP endpoints ...") 154 | 155 | mux := http.NewServeMux() 156 | mux.Handle("/__/", op.NewHandler( 157 | op.NewStatus("ingress53", "ingress53 updates Route53 DNS records based on the ingresses available on the kubernetes cluster it runs on"). 158 | AddOwner("infrastructure", "#infra"). 159 | AddLink("github", "https://github.com/utilitywarehouse/ingress53"). 160 | SetRevision(appGitHash). 161 | AddChecker("running", func(cr *op.CheckResponse) { cr.Healthy("service is running") }). 162 | ReadyAlways(), 163 | )) 164 | mux.Handle("/metrics", promhttp.Handler()) 165 | 166 | if err := http.ListenAndServe(":5000", mux); err != nil { 167 | log.Printf("[ERROR] could not start HTTP router: %+v", err) 168 | os.Exit(1) 169 | } 170 | }() 171 | 172 | if err := r.Start(); err != nil { 173 | log.Printf("[ERROR] registrator returned an error: %+v", err) 174 | os.Exit(1) 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /route53_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/service/route53" 11 | "github.com/aws/aws-sdk-go/service/route53/route53iface" 12 | ) 13 | 14 | type mockRoute53API struct { 15 | route53iface.Route53API 16 | getZoneResp *route53.GetHostedZoneOutput 17 | getZoneErr error 18 | getChangeResp *route53.GetChangeOutput 19 | getChangeErr error 20 | changeRRResp *route53.ChangeResourceRecordSetsOutput 21 | changeRRErr error 22 | } 23 | 24 | func (m mockRoute53API) GetHostedZone(in *route53.GetHostedZoneInput) (*route53.GetHostedZoneOutput, error) { 25 | return m.getZoneResp, m.getZoneErr 26 | } 27 | 28 | func (m mockRoute53API) ChangeResourceRecordSets(in *route53.ChangeResourceRecordSetsInput) (*route53.ChangeResourceRecordSetsOutput, error) { 29 | return m.changeRRResp, m.changeRRErr 30 | } 31 | 32 | func (m mockRoute53API) GetChange(in *route53.GetChangeInput) (*route53.GetChangeOutput, error) { 33 | return m.getChangeResp, m.getChangeErr 34 | } 35 | 36 | func mockRoute53Timers() func() { 37 | dwi := defaultRoute53ZoneWaitWatchInterval 38 | dwt := defaultRoute53ZoneWaitWatchTimeout 39 | defaultRoute53ZoneWaitWatchInterval = 1 * time.Second 40 | defaultRoute53ZoneWaitWatchTimeout = 2 * time.Second 41 | return func() { 42 | defaultRoute53ZoneWaitWatchInterval = dwi 43 | defaultRoute53ZoneWaitWatchTimeout = dwt 44 | } 45 | } 46 | 47 | var ( 48 | errTestRoute53ZoneMock = errors.New("test error") 49 | 50 | testRoute53ZoneGetZoneOK = &route53.GetHostedZoneOutput{ 51 | HostedZone: &route53.HostedZone{ 52 | ResourceRecordSetCount: aws.Int64(1), 53 | CallerReference: aws.String(""), 54 | Config: &route53.HostedZoneConfig{ 55 | Comment: aws.String(""), 56 | PrivateZone: aws.Bool(false), 57 | }, 58 | Id: aws.String("/hostedzone/XXXXXXXXXXXXXX"), 59 | Name: aws.String("example.com."), 60 | }, 61 | DelegationSet: &route53.DelegationSet{ 62 | NameServers: []*string{ 63 | aws.String("0.ns.example.com"), 64 | aws.String("1.ns.example.com"), 65 | aws.String("2.ns.example.com"), 66 | aws.String("3.ns.example.com"), 67 | }, 68 | CallerReference: aws.String(""), 69 | Id: aws.String("/delegationset/XXXXXXXXXXXXXX"), 70 | }, 71 | } 72 | 73 | testRoute53ZoneChangeRROK = &route53.ChangeResourceRecordSetsOutput{ 74 | ChangeInfo: &route53.ChangeInfo{ 75 | Comment: aws.String(""), 76 | Id: aws.String("123456789"), 77 | Status: aws.String(route53.ChangeStatusPending), 78 | SubmittedAt: aws.Time(time.Now()), 79 | }, 80 | } 81 | 82 | testRoute53ZoneGetChangeOK = &route53.GetChangeOutput{ 83 | ChangeInfo: &route53.ChangeInfo{ 84 | Comment: aws.String(""), 85 | Id: aws.String("123456789"), 86 | Status: aws.String(route53.ChangeStatusInsync), 87 | SubmittedAt: aws.Time(time.Now()), 88 | }, 89 | } 90 | 91 | testRoute53ZoneGetChangePending = &route53.GetChangeOutput{ 92 | ChangeInfo: &route53.ChangeInfo{ 93 | Comment: aws.String(""), 94 | Id: aws.String("123456789"), 95 | Status: aws.String(route53.ChangeStatusPending), 96 | SubmittedAt: aws.Time(time.Now()), 97 | }, 98 | } 99 | 100 | expectedNameservers = []string{ 101 | "0.ns.example.com", 102 | "1.ns.example.com", 103 | "2.ns.example.com", 104 | "3.ns.example.com", 105 | } 106 | ) 107 | 108 | func TestRoute53Zone_UpsertCname(t *testing.T) { 109 | testCases := []struct { 110 | getZoneErr error 111 | getZoneResponse *route53.GetHostedZoneOutput 112 | 113 | changeRRErr error 114 | changeRRResponse *route53.ChangeResourceRecordSetsOutput 115 | getChangeErr error 116 | getChangeResponse *route53.GetChangeOutput 117 | 118 | zoneID string 119 | record cnameRecord 120 | 121 | expectedNewErr error 122 | expectedUpsertErr error 123 | }{ 124 | { // error in get zone 125 | errTestRoute53ZoneMock, 126 | nil, 127 | nil, 128 | nil, 129 | nil, 130 | nil, 131 | "example.com.", 132 | cnameRecord{"test.example.com", "cname.example.com"}, 133 | errTestRoute53ZoneMock, 134 | nil, 135 | }, 136 | { // error in change request 137 | nil, 138 | testRoute53ZoneGetZoneOK, 139 | errTestRoute53ZoneMock, 140 | nil, 141 | nil, 142 | nil, 143 | "example.com.", 144 | cnameRecord{"test.example.com", "cname.example.com"}, 145 | nil, 146 | errTestRoute53ZoneMock, 147 | }, 148 | { // error in get change request 149 | nil, 150 | testRoute53ZoneGetZoneOK, 151 | nil, 152 | testRoute53ZoneChangeRROK, 153 | errTestRoute53ZoneMock, 154 | nil, 155 | "example.com.", 156 | cnameRecord{"test.example.com", "cname.example.com"}, 157 | nil, 158 | errTestRoute53ZoneMock, 159 | }, 160 | { // timeout in get change 161 | nil, 162 | testRoute53ZoneGetZoneOK, 163 | nil, 164 | testRoute53ZoneChangeRROK, 165 | nil, 166 | testRoute53ZoneGetChangePending, 167 | "example.com.", 168 | cnameRecord{"test.example.com", "cname.example.com"}, 169 | nil, 170 | errRoute53WaitWatchTimedOut, 171 | }, 172 | { // works end to end 173 | nil, 174 | testRoute53ZoneGetZoneOK, 175 | nil, 176 | testRoute53ZoneChangeRROK, 177 | nil, 178 | testRoute53ZoneGetChangeOK, 179 | "example.com.", 180 | cnameRecord{"test.example.com", "cname.example.com"}, 181 | nil, 182 | nil, 183 | }, 184 | } 185 | 186 | defer mockRoute53Timers()() 187 | 188 | for i, tc := range testCases { 189 | p, err := newRoute53Zone(tc.zoneID, &mockRoute53API{ 190 | getZoneResp: tc.getZoneResponse, 191 | getZoneErr: tc.getZoneErr, 192 | getChangeResp: tc.getChangeResponse, 193 | getChangeErr: tc.getChangeErr, 194 | changeRRResp: tc.changeRRResponse, 195 | changeRRErr: tc.changeRRErr, 196 | }) 197 | if err != tc.expectedNewErr { 198 | t.Fatalf("newRoute53Zone returned unexpected error: %+v", err) 199 | } 200 | 201 | if tc.expectedNewErr != nil { 202 | continue 203 | } 204 | 205 | if p.Name != "example.com." { 206 | t.Errorf("Route53Zone has unexpected Name: %+v", p.Name) 207 | } 208 | 209 | if p.ID != "/hostedzone/XXXXXXXXXXXXXX" { 210 | t.Errorf("Route53Zone has unexpected ID: %+v", p.ID) 211 | } 212 | 213 | if !reflect.DeepEqual(p.Nameservers, expectedNameservers) { 214 | t.Errorf("Route53Zone has unexpected Nameservers: %+v", p.Nameservers) 215 | } 216 | 217 | if err := p.UpsertCnames([]cnameRecord{tc.record}); err != tc.expectedUpsertErr { 218 | t.Errorf("Route53Zone.UpsertCname returned unexpected error for case #%02d: %+v", i, err) 219 | } 220 | } 221 | } 222 | 223 | func TestRoute53Zone_DeleteCname(t *testing.T) { 224 | defer mockRoute53Timers()() 225 | 226 | p, err := newRoute53Zone("example.com.", &mockRoute53API{ 227 | getZoneResp: testRoute53ZoneGetZoneOK, 228 | getChangeResp: testRoute53ZoneGetChangeOK, 229 | changeRRResp: testRoute53ZoneChangeRROK, 230 | }) 231 | if err != nil { 232 | t.Fatalf("newRoute53Zone returned unexpected error: %+v", err) 233 | } 234 | 235 | if err := p.DeleteCnames([]cnameRecord{{Hostname: "test.example.com", Target: "foo.example.com"}}); err != nil { 236 | t.Errorf("Route53Zone.DeleteCname returned unexpected error: %+v", err) 237 | } 238 | } 239 | 240 | func TestRoute53Zone_Domain(t *testing.T) { 241 | z := route53Zone{Name: "test"} 242 | if z.Domain() != "test" { 243 | t.Errorf("Route53Zone.Domain return unexpected value") 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /ingress_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | "k8s.io/api/extensions/v1beta1" 11 | "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/apimachinery/pkg/watch" 13 | "k8s.io/client-go/kubernetes/fake" 14 | testcore "k8s.io/client-go/testing" 15 | ) 16 | 17 | var ( 18 | privateIngressHostsAB = &v1beta1.Ingress{ 19 | ObjectMeta: v1.ObjectMeta{ 20 | Name: "privateIngressHostsAB", 21 | Namespace: v1.NamespaceDefault, 22 | Labels: map[string]string{ 23 | testTargetLabelName: testPrivateTarget, 24 | }, 25 | }, 26 | Spec: v1beta1.IngressSpec{ 27 | Rules: []v1beta1.IngressRule{ 28 | {Host: "a.example.com"}, 29 | {Host: "b.example.com"}, 30 | }, 31 | }, 32 | } 33 | 34 | publicIngressHostC = &v1beta1.Ingress{ 35 | ObjectMeta: v1.ObjectMeta{ 36 | Name: "publicIngressHostCD", 37 | Namespace: v1.NamespaceDefault, 38 | Labels: map[string]string{ 39 | testTargetLabelName: testPublicTarget, 40 | }, 41 | }, 42 | Spec: v1beta1.IngressSpec{ 43 | Rules: []v1beta1.IngressRule{ 44 | {Host: "c.example.com"}, 45 | }, 46 | }, 47 | } 48 | 49 | publicIngressHostD = &v1beta1.Ingress{ 50 | ObjectMeta: v1.ObjectMeta{ 51 | Name: "publicIngressHostCD", 52 | Namespace: v1.NamespaceDefault, 53 | Labels: map[string]string{ 54 | testTargetLabelName: testPublicTarget, 55 | }, 56 | }, 57 | Spec: v1beta1.IngressSpec{ 58 | Rules: []v1beta1.IngressRule{ 59 | {Host: "d.example.com"}, 60 | }, 61 | }, 62 | } 63 | 64 | privateIngressHostE = &v1beta1.Ingress{ 65 | ObjectMeta: v1.ObjectMeta{ 66 | Name: "ingressHostE", 67 | Namespace: v1.NamespaceDefault, 68 | Labels: map[string]string{ 69 | testTargetLabelName: testPrivateTarget, 70 | }, 71 | }, 72 | Spec: v1beta1.IngressSpec{ 73 | Rules: []v1beta1.IngressRule{ 74 | {Host: "e.example.com"}, 75 | }, 76 | }, 77 | } 78 | 79 | privateIngressHostEDup = &v1beta1.Ingress{ 80 | ObjectMeta: v1.ObjectMeta{ 81 | Name: "ingressHostE", 82 | Namespace: v1.NamespaceDefault, 83 | Labels: map[string]string{ 84 | testTargetLabelName: testPrivateTarget, 85 | }, 86 | }, 87 | Spec: v1beta1.IngressSpec{ 88 | Rules: []v1beta1.IngressRule{ 89 | {Host: "e.example.com"}, 90 | }, 91 | }, 92 | } 93 | 94 | publicIngressHostEDup = &v1beta1.Ingress{ 95 | ObjectMeta: v1.ObjectMeta{ 96 | Name: "ingressHostE", 97 | Namespace: v1.NamespaceDefault, 98 | Labels: map[string]string{ 99 | testTargetLabelName: testPublicTarget, 100 | }, 101 | }, 102 | Spec: v1beta1.IngressSpec{ 103 | Rules: []v1beta1.IngressRule{ 104 | {Host: "e.example.com"}, 105 | }, 106 | }, 107 | } 108 | 109 | privateIngressHostE2 = &v1beta1.Ingress{ 110 | ObjectMeta: v1.ObjectMeta{ 111 | Name: "ingressHostE2", 112 | Namespace: v1.NamespaceDefault, 113 | Labels: map[string]string{ 114 | testTargetLabelName: testPrivateTarget, 115 | }, 116 | }, 117 | Spec: v1beta1.IngressSpec{ 118 | Rules: []v1beta1.IngressRule{ 119 | {Host: "e.example.com"}, 120 | }, 121 | }, 122 | } 123 | 124 | privateIngressHostE2Fixed = &v1beta1.Ingress{ 125 | ObjectMeta: v1.ObjectMeta{ 126 | Name: "ingressHostE2", 127 | Namespace: v1.NamespaceDefault, 128 | Labels: map[string]string{ 129 | testTargetLabelName: testPrivateTarget, 130 | }, 131 | }, 132 | Spec: v1beta1.IngressSpec{ 133 | Rules: []v1beta1.IngressRule{ 134 | {Host: "e2.example.com"}, 135 | }, 136 | }, 137 | } 138 | 139 | ingressNoLabels = &v1beta1.Ingress{ 140 | ObjectMeta: v1.ObjectMeta{ 141 | Name: "ingressNoLabels", 142 | Namespace: v1.NamespaceDefault, 143 | Labels: map[string]string{}, 144 | }, 145 | Spec: v1beta1.IngressSpec{ 146 | Rules: []v1beta1.IngressRule{ 147 | {Host: "no-labels.example.com"}, 148 | }, 149 | }, 150 | } 151 | 152 | nonRegisteredIngress = &v1beta1.Ingress{ 153 | ObjectMeta: v1.ObjectMeta{ 154 | Name: "nonRegisteredIngress", 155 | Namespace: v1.NamespaceDefault, 156 | Labels: map[string]string{ 157 | testTargetLabelName: "non-registered-target.aws.com", 158 | }, 159 | }, 160 | Spec: v1beta1.IngressSpec{ 161 | Rules: []v1beta1.IngressRule{ 162 | {Host: "non-registered-target.example.com"}, 163 | }, 164 | }, 165 | } 166 | ) 167 | 168 | func Test_getHostnamesFromIngress(t *testing.T) { 169 | testCases := []struct { 170 | Spec v1beta1.IngressSpec 171 | Expected []string 172 | }{ 173 | // single value 174 | { 175 | Spec: v1beta1.IngressSpec{ 176 | Rules: []v1beta1.IngressRule{ 177 | {Host: "foo.example.com"}, 178 | }, 179 | }, 180 | Expected: []string{"foo.example.com"}, 181 | }, 182 | // two values 183 | { 184 | Spec: v1beta1.IngressSpec{ 185 | Rules: []v1beta1.IngressRule{ 186 | {Host: "foo.example.com"}, 187 | {Host: "bar.example.com"}, 188 | }, 189 | }, 190 | Expected: []string{"foo.example.com", "bar.example.com"}, 191 | }, 192 | // duplicate 193 | { 194 | Spec: v1beta1.IngressSpec{ 195 | Rules: []v1beta1.IngressRule{ 196 | {Host: "foo.example.com"}, 197 | {Host: "foo.example.com"}, 198 | }, 199 | }, 200 | Expected: []string{"foo.example.com"}, 201 | }, 202 | } 203 | 204 | for i, tc := range testCases { 205 | ingress := &v1beta1.Ingress{Spec: tc.Spec} 206 | hostnames := getHostnamesFromIngress(ingress) 207 | 208 | if !reflect.DeepEqual(hostnames, tc.Expected) { 209 | t.Errorf("getHostnamesFromIngress returned unexpected results for test case #%02d: %+v", i, hostnames) 210 | } 211 | } 212 | } 213 | 214 | type testIngressEvent struct { 215 | et watch.EventType 216 | old *v1beta1.Ingress 217 | new *v1beta1.Ingress 218 | } 219 | 220 | func newTestIngressWatcherClient(initial ...v1beta1.Ingress) (*fake.Clientset, *watch.FakeWatcher) { 221 | client := fake.NewSimpleClientset(&v1beta1.IngressList{Items: initial}) 222 | watcher := watch.NewFake() 223 | client.PrependWatchReactor("ingresses", testcore.DefaultWatchReactor(watcher, nil)) 224 | return client, watcher 225 | } 226 | 227 | func waitForTrue(test func() bool, timeout time.Duration) error { 228 | timer := time.NewTimer(timeout) 229 | ticker := time.NewTicker(timeout / 100) 230 | defer func() { 231 | timer.Stop() 232 | ticker.Stop() 233 | }() 234 | for { 235 | select { 236 | case <-ticker.C: 237 | if test() { 238 | return nil 239 | } 240 | case <-timer.C: 241 | return fmt.Errorf("timed out") 242 | } 243 | } 244 | } 245 | 246 | func TestIngressWatcher(t *testing.T) { 247 | expected := []testIngressEvent{ 248 | {watch.Added, nil, privateIngressHostsAB}, 249 | {watch.Added, nil, publicIngressHostC}, 250 | {watch.Deleted, privateIngressHostsAB, nil}, 251 | {watch.Modified, publicIngressHostC, publicIngressHostD}, 252 | } 253 | 254 | client, watcher := newTestIngressWatcherClient(*privateIngressHostsAB, *publicIngressHostC) 255 | 256 | pM := &sync.Mutex{} 257 | processed := []testIngressEvent{} 258 | iw := newIngressWatcher(client, func(t watch.EventType, o, n *v1beta1.Ingress) { 259 | pM.Lock() 260 | processed = append(processed, testIngressEvent{t, o, n}) 261 | pM.Unlock() 262 | }, "", 0) 263 | 264 | wg := sync.WaitGroup{} 265 | wg.Add(1) 266 | go func() { 267 | defer wg.Done() 268 | iw.Start() 269 | }() 270 | 271 | // because the events are processed asynchronously, using a wait function 272 | // here guarantees that they will arrive in the correct order and enables 273 | // testing for equality 274 | pLenIs := func(n int) func() bool { 275 | return func() bool { 276 | pM.Lock() 277 | defer pM.Unlock() 278 | if len(processed) == n { 279 | return true 280 | } 281 | return false 282 | } 283 | } 284 | if err := waitForTrue(pLenIs(2), 10*time.Second); err != nil { 285 | t.Fatalf("timed out waiting for ingressWatcher to process events") 286 | } 287 | watcher.Delete(privateIngressHostsAB) 288 | if err := waitForTrue(pLenIs(3), 10*time.Second); err != nil { 289 | t.Fatalf("timed out waiting for ingressWatcher to process events") 290 | } 291 | watcher.Modify(publicIngressHostD) 292 | if err := waitForTrue(pLenIs(4), 10*time.Second); err != nil { 293 | t.Fatalf("timed out waiting for ingressWatcher to process events") 294 | } 295 | 296 | iw.Stop() 297 | wg.Wait() 298 | 299 | pM.Lock() 300 | if !reflect.DeepEqual(processed, expected) { 301 | t.Errorf("ingressWatcher did not produce expected results: %+v != %+v", processed, expected) 302 | } 303 | pM.Unlock() 304 | } 305 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: 9745379bcd31320fded5612b0d135d3f48fab47706462e4315f5880d794bd0bc 2 | updated: 2017-11-09T10:56:13.556110246Z 3 | imports: 4 | - name: github.com/aws/aws-sdk-go 5 | version: a201bf33b18ad4ab54344e4bc26b87eb6ad37b8e 6 | subpackages: 7 | - aws 8 | - aws/awserr 9 | - aws/awsutil 10 | - aws/client 11 | - aws/client/metadata 12 | - aws/corehandlers 13 | - aws/credentials 14 | - aws/credentials/ec2rolecreds 15 | - aws/credentials/endpointcreds 16 | - aws/credentials/stscreds 17 | - aws/defaults 18 | - aws/ec2metadata 19 | - aws/endpoints 20 | - aws/request 21 | - aws/session 22 | - aws/signer/v4 23 | - internal/shareddefaults 24 | - private/protocol 25 | - private/protocol/query 26 | - private/protocol/query/queryutil 27 | - private/protocol/rest 28 | - private/protocol/restxml 29 | - private/protocol/xml/xmlutil 30 | - service/route53 31 | - service/route53/route53iface 32 | - service/sts 33 | - name: github.com/beorn7/perks 34 | version: 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 35 | subpackages: 36 | - quantile 37 | - name: github.com/davecgh/go-spew 38 | version: 782f4967f2dc4564575ca782fe2d04090b5faca8 39 | subpackages: 40 | - spew 41 | - name: github.com/emicklei/go-restful 42 | version: ff4f55a206334ef123e4f79bbf348980da81ca46 43 | subpackages: 44 | - log 45 | - name: github.com/emicklei/go-restful-swagger12 46 | version: dcef7f55730566d41eae5db10e7d6981829720f6 47 | - name: github.com/ghodss/yaml 48 | version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee 49 | - name: github.com/go-ini/ini 50 | version: f280b3ba517bf5fc98922624f21fb0e7a92adaec 51 | - name: github.com/go-openapi/jsonpointer 52 | version: 46af16f9f7b149af66e5d1bd010e3574dc06de98 53 | - name: github.com/go-openapi/jsonreference 54 | version: 13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272 55 | - name: github.com/go-openapi/spec 56 | version: 6aced65f8501fe1217321abf0749d354824ba2ff 57 | - name: github.com/go-openapi/swag 58 | version: 1d0bd113de87027671077d3c71eb3ac5d7dbba72 59 | - name: github.com/gogo/protobuf 60 | version: c0656edd0d9eab7c66d1eb0c568f9039345796f7 61 | subpackages: 62 | - proto 63 | - sortkeys 64 | - name: github.com/golang/glog 65 | version: 44145f04b68cf362d9c4df2182967c2275eaefed 66 | - name: github.com/golang/protobuf 67 | version: 4bd1920723d7b7c925de087aa32e2187708897f7 68 | subpackages: 69 | - proto 70 | - ptypes 71 | - ptypes/any 72 | - ptypes/duration 73 | - ptypes/timestamp 74 | - name: github.com/google/btree 75 | version: 7d79101e329e5a3adf994758c578dab82b90c017 76 | - name: github.com/google/gofuzz 77 | version: 44d81051d367757e1c7c6a5a86423ece9afcf63c 78 | - name: github.com/googleapis/gnostic 79 | version: 0c5108395e2debce0d731cf0287ddf7242066aba 80 | subpackages: 81 | - OpenAPIv2 82 | - compiler 83 | - extensions 84 | - name: github.com/gregjones/httpcache 85 | version: 787624de3eb7bd915c329cba748687a3b22666a6 86 | subpackages: 87 | - diskcache 88 | - name: github.com/hashicorp/golang-lru 89 | version: a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 90 | subpackages: 91 | - simplelru 92 | - name: github.com/hashicorp/logutils 93 | version: 0dc08b1671f34c4250ce212759ebd880f743d883 94 | - name: github.com/howeyc/gopass 95 | version: bf9dde6d0d2c004a008c27aaee91170c786f6db8 96 | - name: github.com/imdario/mergo 97 | version: 6633656539c1639d9d78127b7d47c622b5d7b6dc 98 | - name: github.com/jmespath/go-jmespath 99 | version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d 100 | - name: github.com/json-iterator/go 101 | version: 36b14963da70d11297d313183d7e6388c8510e1e 102 | - name: github.com/juju/ratelimit 103 | version: 5b9ff866471762aa2ab2dced63c9fb6f53921342 104 | - name: github.com/mailru/easyjson 105 | version: d5b7844b561a7bc640052f1b935f7b800330d7e0 106 | subpackages: 107 | - buffer 108 | - jlexer 109 | - jwriter 110 | - name: github.com/matttproud/golang_protobuf_extensions 111 | version: c12348ce28de40eed0136aa2b644d0ee0650e56c 112 | subpackages: 113 | - pbutil 114 | - name: github.com/miekg/dns 115 | version: 8223ae840e47f0c9a1ea1d5c22bdd03b30b6b3cf 116 | subpackages: 117 | - internal/socket 118 | - name: github.com/peterbourgon/diskv 119 | version: 5f041e8faa004a95c88a202771f4cc3e991971e6 120 | - name: github.com/prometheus/client_golang 121 | version: 967789050ba94deca04a5e84cce8ad472ce313c1 122 | subpackages: 123 | - prometheus 124 | - prometheus/promhttp 125 | - name: github.com/prometheus/client_model 126 | version: 6f3806018612930941127f2a7c6c453ba2c527d2 127 | subpackages: 128 | - go 129 | - name: github.com/prometheus/common 130 | version: e3fb1a1acd7605367a2b378bc2e2f893c05174b7 131 | subpackages: 132 | - expfmt 133 | - internal/bitbucket.org/ww/goautoneg 134 | - model 135 | - name: github.com/prometheus/procfs 136 | version: a6e9df898b1336106c743392c48ee0b71f5c4efa 137 | subpackages: 138 | - xfs 139 | - name: github.com/PuerkitoBio/purell 140 | version: 8a290539e2e8629dbc4e6bad948158f790ec31f4 141 | - name: github.com/PuerkitoBio/urlesc 142 | version: 5bd2802263f21d8788851d5305584c82a5c75d7e 143 | - name: github.com/spf13/pflag 144 | version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 145 | - name: github.com/utilitywarehouse/go-operational 146 | version: 1893d0e062cc2c6e11c95f074c17378f85836f7b 147 | subpackages: 148 | - op 149 | - name: golang.org/x/crypto 150 | version: 81e90905daefcd6fd217b62423c0908922eadb30 151 | subpackages: 152 | - ssh/terminal 153 | - name: golang.org/x/net 154 | version: 1c05540f6879653db88113bc4a2b70aec4bd491f 155 | subpackages: 156 | - context 157 | - http2 158 | - http2/hpack 159 | - idna 160 | - lex/httplex 161 | - name: golang.org/x/sys 162 | version: 7ddbeae9ae08c6a06a59597f0c9edbc5ff2444ce 163 | subpackages: 164 | - unix 165 | - windows 166 | - name: golang.org/x/text 167 | version: b19bf474d317b857955b12035d2c5acb57ce8b01 168 | subpackages: 169 | - cases 170 | - internal 171 | - internal/tag 172 | - language 173 | - runes 174 | - secure/bidirule 175 | - secure/precis 176 | - transform 177 | - unicode/bidi 178 | - unicode/norm 179 | - width 180 | - name: gopkg.in/inf.v0 181 | version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 182 | - name: gopkg.in/yaml.v2 183 | version: 53feefa2559fb8dfa8d81baad31be332c97d6c77 184 | - name: k8s.io/api 185 | version: 6c6dac0277229b9e9578c5ca3f74a4345d35cdc2 186 | subpackages: 187 | - admissionregistration/v1alpha1 188 | - apps/v1beta1 189 | - apps/v1beta2 190 | - authentication/v1 191 | - authentication/v1beta1 192 | - authorization/v1 193 | - authorization/v1beta1 194 | - autoscaling/v1 195 | - autoscaling/v2beta1 196 | - batch/v1 197 | - batch/v1beta1 198 | - batch/v2alpha1 199 | - certificates/v1beta1 200 | - core/v1 201 | - extensions/v1beta1 202 | - networking/v1 203 | - policy/v1beta1 204 | - rbac/v1 205 | - rbac/v1alpha1 206 | - rbac/v1beta1 207 | - scheduling/v1alpha1 208 | - settings/v1alpha1 209 | - storage/v1 210 | - storage/v1beta1 211 | - name: k8s.io/apimachinery 212 | version: 019ae5ada31de202164b118aee88ee2d14075c31 213 | subpackages: 214 | - pkg/api/equality 215 | - pkg/api/errors 216 | - pkg/api/meta 217 | - pkg/api/resource 218 | - pkg/apis/meta/internalversion 219 | - pkg/apis/meta/v1 220 | - pkg/apis/meta/v1/unstructured 221 | - pkg/apis/meta/v1alpha1 222 | - pkg/conversion 223 | - pkg/conversion/queryparams 224 | - pkg/conversion/unstructured 225 | - pkg/fields 226 | - pkg/labels 227 | - pkg/runtime 228 | - pkg/runtime/schema 229 | - pkg/runtime/serializer 230 | - pkg/runtime/serializer/json 231 | - pkg/runtime/serializer/protobuf 232 | - pkg/runtime/serializer/recognizer 233 | - pkg/runtime/serializer/streaming 234 | - pkg/runtime/serializer/versioning 235 | - pkg/selection 236 | - pkg/types 237 | - pkg/util/cache 238 | - pkg/util/clock 239 | - pkg/util/diff 240 | - pkg/util/errors 241 | - pkg/util/framer 242 | - pkg/util/intstr 243 | - pkg/util/json 244 | - pkg/util/net 245 | - pkg/util/runtime 246 | - pkg/util/sets 247 | - pkg/util/validation 248 | - pkg/util/validation/field 249 | - pkg/util/wait 250 | - pkg/util/yaml 251 | - pkg/version 252 | - pkg/watch 253 | - third_party/forked/golang/reflect 254 | - name: k8s.io/client-go 255 | version: 2ae454230481a7cb5544325e12ad7658ecccd19b 256 | subpackages: 257 | - discovery 258 | - discovery/fake 259 | - kubernetes 260 | - kubernetes/fake 261 | - kubernetes/scheme 262 | - kubernetes/typed/admissionregistration/v1alpha1 263 | - kubernetes/typed/admissionregistration/v1alpha1/fake 264 | - kubernetes/typed/apps/v1beta1 265 | - kubernetes/typed/apps/v1beta1/fake 266 | - kubernetes/typed/apps/v1beta2 267 | - kubernetes/typed/apps/v1beta2/fake 268 | - kubernetes/typed/authentication/v1 269 | - kubernetes/typed/authentication/v1/fake 270 | - kubernetes/typed/authentication/v1beta1 271 | - kubernetes/typed/authentication/v1beta1/fake 272 | - kubernetes/typed/authorization/v1 273 | - kubernetes/typed/authorization/v1/fake 274 | - kubernetes/typed/authorization/v1beta1 275 | - kubernetes/typed/authorization/v1beta1/fake 276 | - kubernetes/typed/autoscaling/v1 277 | - kubernetes/typed/autoscaling/v1/fake 278 | - kubernetes/typed/autoscaling/v2beta1 279 | - kubernetes/typed/autoscaling/v2beta1/fake 280 | - kubernetes/typed/batch/v1 281 | - kubernetes/typed/batch/v1/fake 282 | - kubernetes/typed/batch/v1beta1 283 | - kubernetes/typed/batch/v1beta1/fake 284 | - kubernetes/typed/batch/v2alpha1 285 | - kubernetes/typed/batch/v2alpha1/fake 286 | - kubernetes/typed/certificates/v1beta1 287 | - kubernetes/typed/certificates/v1beta1/fake 288 | - kubernetes/typed/core/v1 289 | - kubernetes/typed/core/v1/fake 290 | - kubernetes/typed/extensions/v1beta1 291 | - kubernetes/typed/extensions/v1beta1/fake 292 | - kubernetes/typed/networking/v1 293 | - kubernetes/typed/networking/v1/fake 294 | - kubernetes/typed/policy/v1beta1 295 | - kubernetes/typed/policy/v1beta1/fake 296 | - kubernetes/typed/rbac/v1 297 | - kubernetes/typed/rbac/v1/fake 298 | - kubernetes/typed/rbac/v1alpha1 299 | - kubernetes/typed/rbac/v1alpha1/fake 300 | - kubernetes/typed/rbac/v1beta1 301 | - kubernetes/typed/rbac/v1beta1/fake 302 | - kubernetes/typed/scheduling/v1alpha1 303 | - kubernetes/typed/scheduling/v1alpha1/fake 304 | - kubernetes/typed/settings/v1alpha1 305 | - kubernetes/typed/settings/v1alpha1/fake 306 | - kubernetes/typed/storage/v1 307 | - kubernetes/typed/storage/v1/fake 308 | - kubernetes/typed/storage/v1beta1 309 | - kubernetes/typed/storage/v1beta1/fake 310 | - pkg/version 311 | - rest 312 | - rest/watch 313 | - testing 314 | - tools/auth 315 | - tools/cache 316 | - tools/clientcmd 317 | - tools/clientcmd/api 318 | - tools/clientcmd/api/latest 319 | - tools/clientcmd/api/v1 320 | - tools/metrics 321 | - tools/pager 322 | - tools/reference 323 | - transport 324 | - util/cert 325 | - util/flowcontrol 326 | - util/homedir 327 | - util/integer 328 | - name: k8s.io/kube-openapi 329 | version: 868f2f29720b192240e18284659231b440f9cda5 330 | subpackages: 331 | - pkg/common 332 | testImports: [] 333 | -------------------------------------------------------------------------------- /registrator.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "regexp" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | "github.com/aws/aws-sdk-go/aws/session" 13 | "github.com/aws/aws-sdk-go/service/route53" 14 | "github.com/miekg/dns" 15 | "k8s.io/api/extensions/v1beta1" 16 | "k8s.io/apimachinery/pkg/labels" 17 | "k8s.io/apimachinery/pkg/watch" 18 | "k8s.io/client-go/kubernetes" 19 | "k8s.io/client-go/rest" 20 | ) 21 | 22 | var ( 23 | errRegistratorMissingOption = errors.New("missing required registrator option") 24 | errDNSEmptyAnswer = errors.New("DNS nameserver returned an empty answer") 25 | defaultResyncPeriod = 15 * time.Minute 26 | defaultBatchProcessCycle = 5 * time.Second 27 | dnsClient = &dns.Client{} 28 | ) 29 | 30 | type dnsZone interface { 31 | UpsertCnames(records []cnameRecord) error 32 | DeleteCnames(records []cnameRecord) error 33 | Domain() string 34 | ListNameservers() []string 35 | } 36 | 37 | type cnameChange struct { 38 | Action string 39 | Record cnameRecord 40 | } 41 | 42 | type cnameRecord struct { 43 | Hostname string 44 | Target string 45 | } 46 | 47 | type registrator struct { 48 | dnsZone 49 | *ingressWatcher 50 | options registratorOptions 51 | sats []selectorAndTarget 52 | updateQueue chan cnameChange 53 | } 54 | 55 | type registratorOptions struct { 56 | AWSSessionOptions *session.Options 57 | KubernetesConfig *rest.Config 58 | Targets []string // required 59 | TargetLabelName string // required 60 | Route53ZoneID string // required 61 | ResyncPeriod time.Duration 62 | } 63 | 64 | type selectorAndTarget struct { 65 | Selector labels.Selector 66 | Target string 67 | } 68 | 69 | func newRegistrator(zoneID string, targets []string, targetLabelName string) (*registrator, error) { 70 | return newRegistratorWithOptions( 71 | registratorOptions{ 72 | Route53ZoneID: zoneID, 73 | Targets: targets, 74 | TargetLabelName: targetLabelName, 75 | }) 76 | } 77 | 78 | func newRegistratorWithOptions(options registratorOptions) (*registrator, error) { 79 | // check required options are set 80 | if len(options.Targets) == 0 || options.Route53ZoneID == "" || options.TargetLabelName == "" { 81 | return nil, errRegistratorMissingOption 82 | } 83 | var sats []selectorAndTarget 84 | for _, target := range options.Targets { 85 | s, err := labels.Parse(options.TargetLabelName + "=" + target) 86 | if err != nil { 87 | return nil, err 88 | } 89 | sats = append(sats, selectorAndTarget{Selector: s, Target: target}) 90 | } 91 | if options.AWSSessionOptions == nil { 92 | options.AWSSessionOptions = &session.Options{} 93 | } 94 | if options.KubernetesConfig == nil { 95 | c, err := rest.InClusterConfig() 96 | if err != nil { 97 | return nil, err 98 | } 99 | options.KubernetesConfig = c 100 | } 101 | if options.ResyncPeriod == 0 { 102 | options.ResyncPeriod = defaultResyncPeriod 103 | } 104 | return ®istrator{ 105 | options: options, 106 | sats: sats, 107 | updateQueue: make(chan cnameChange, 64), 108 | }, nil 109 | } 110 | 111 | func (r *registrator) Start() error { 112 | sess, err := session.NewSessionWithOptions(*r.options.AWSSessionOptions) 113 | if err != nil { 114 | return err 115 | } 116 | dns, err := newRoute53Zone(r.options.Route53ZoneID, route53.New(sess)) 117 | if err != nil { 118 | return err 119 | } 120 | r.dnsZone = dns 121 | log.Println("[INFO] setup route53 session") 122 | kubeClient, err := kubernetes.NewForConfig(r.options.KubernetesConfig) 123 | if err != nil { 124 | return err 125 | } 126 | r.ingressWatcher = newIngressWatcher(kubeClient, r.handler, r.options.TargetLabelName, r.options.ResyncPeriod) 127 | log.Println("[INFO] setup kubernetes ingress watcher") 128 | wg := sync.WaitGroup{} 129 | wg.Add(1) 130 | go func() { 131 | defer wg.Done() 132 | r.processUpdateQueue() 133 | }() 134 | r.ingressWatcher.Start() 135 | wg.Wait() 136 | return nil 137 | } 138 | 139 | func (r *registrator) handler(eventType watch.EventType, oldIngress *v1beta1.Ingress, newIngress *v1beta1.Ingress) { 140 | switch eventType { 141 | case watch.Added: 142 | log.Printf("[DEBUG] received %s event for %s", eventType, newIngress.Name) 143 | metricUpdatesReceived.WithLabelValues(newIngress.Name, "add").Inc() 144 | hostnames := getHostnamesFromIngress(newIngress) 145 | target := r.getTargetForIngress(newIngress) 146 | if target == "" { 147 | log.Printf("[INFO] invalid ingress target for new ingress %s: %s", newIngress.Name, newIngress.Labels[r.options.TargetLabelName]) 148 | } else if len(hostnames) == 0 { 149 | log.Printf("[INFO] could not extract hostnames from new ingress %s", newIngress.Name) 150 | } else { 151 | log.Printf("[DEBUG] queued update of %d record(s) for new ingress %s, pointing to %s", len(hostnames), newIngress.Name, target) 152 | r.queueUpdates(route53.ChangeActionUpsert, hostnames, target) 153 | } 154 | case watch.Modified: 155 | log.Printf("[DEBUG] received %s event for %s", eventType, newIngress.Name) 156 | metricUpdatesReceived.WithLabelValues(newIngress.Name, "modify").Inc() 157 | newHostnames := getHostnamesFromIngress(newIngress) 158 | newTarget := r.getTargetForIngress(newIngress) 159 | oldHostnames := getHostnamesFromIngress(oldIngress) 160 | oldTarget := r.getTargetForIngress(oldIngress) 161 | diffHostnames := diffStringSlices(oldHostnames, newHostnames) 162 | if len(diffHostnames) == 0 && newIngress.Labels[r.options.TargetLabelName] == oldIngress.Labels[r.options.TargetLabelName] { 163 | log.Printf("[DEBUG] no changes for ingress %s, looks like a no-op resync", newIngress.Name) 164 | break 165 | } 166 | if newTarget == "" { 167 | log.Printf("[INFO] invalid ingress target for modified ingress %s: %s", newIngress.Name, newIngress.Labels[r.options.TargetLabelName]) 168 | } else if len(newHostnames) == 0 { 169 | log.Printf("[INFO] could not extract hostnames from modified ingress %s", newIngress.Name) 170 | } else { 171 | log.Printf("[DEBUG] queued update of %d record(s) for modified ingress %s, pointing to %s", len(newHostnames), newIngress.Name, newTarget) 172 | r.queueUpdates(route53.ChangeActionUpsert, newHostnames, newTarget) 173 | } 174 | if oldTarget == "" { 175 | log.Printf("[INFO] invalid ingress target for previous ingress %s: %s", oldIngress.Name, oldIngress.Labels[r.options.TargetLabelName]) 176 | } else if len(diffHostnames) == 0 { 177 | log.Printf("[DEBUG] no difference in hostnames from previous ingress %s", oldIngress.Name) 178 | } else { 179 | log.Printf("[DEBUG] queued deletion of %d record(s) for previous ingress %s", len(diffHostnames), oldIngress.Name) 180 | r.queueUpdates(route53.ChangeActionDelete, diffHostnames, oldTarget) 181 | } 182 | case watch.Deleted: 183 | log.Printf("[DEBUG] received %s event for %s", eventType, oldIngress.Name) 184 | metricUpdatesReceived.WithLabelValues(oldIngress.Name, "delete").Inc() 185 | hostnames := getHostnamesFromIngress(oldIngress) 186 | target := r.getTargetForIngress(oldIngress) 187 | if target == "" { 188 | log.Printf("[INFO] invalid ingress target for old ingress %s: %s", oldIngress.Name, oldIngress.Labels[r.options.TargetLabelName]) 189 | } else if len(hostnames) == 0 { 190 | log.Printf("[INFO] could not extract hostnames from old ingress %s", oldIngress.Name) 191 | } else { 192 | log.Printf("[DEBUG] queued deletion of %d record(s) for old ingress %s", len(hostnames), oldIngress.Name) 193 | r.queueUpdates(route53.ChangeActionDelete, hostnames, target) 194 | } 195 | default: 196 | log.Printf("[DEBUG] received %s event: cannot handle", eventType) 197 | } 198 | } 199 | 200 | func (r *registrator) queueUpdates(action string, hostnames []string, target string) { 201 | for _, h := range hostnames { 202 | r.updateQueue <- cnameChange{action, cnameRecord{h, target}} 203 | } 204 | } 205 | 206 | func (r *registrator) processUpdateQueue() { 207 | ret := []cnameChange{} 208 | for { 209 | select { 210 | case t := <-r.updateQueue: 211 | if len(ret) > 0 && ((ret[0].Action == route53.ChangeActionDelete && t.Action != route53.ChangeActionDelete) || (ret[0].Action != route53.ChangeActionDelete && t.Action == route53.ChangeActionDelete)) { 212 | r.applyBatch(ret) 213 | ret = []cnameChange{} 214 | } 215 | ret = append(ret, t) 216 | case <-r.stopChannel: 217 | if len(ret) > 0 { 218 | r.applyBatch(ret) 219 | ret = []cnameChange{} 220 | } 221 | return 222 | default: 223 | if len(ret) > 0 { 224 | r.applyBatch(ret) 225 | ret = []cnameChange{} 226 | } 227 | time.Sleep(100 * time.Millisecond) 228 | } 229 | } 230 | } 231 | 232 | func (r *registrator) applyBatch(changes []cnameChange) { 233 | action := changes[0].Action 234 | records := make([]cnameRecord, len(changes)) 235 | for i, c := range changes { 236 | records[i] = c.Record 237 | } 238 | pruned := r.pruneBatch(action, records) 239 | if len(pruned) == 0 { 240 | return 241 | } 242 | hostnames := make([]string, len(pruned)) 243 | for i, p := range pruned { 244 | hostnames[i] = p.Hostname 245 | } 246 | if action == route53.ChangeActionDelete { 247 | log.Printf("[INFO] deleting %d record(s): %+v", len(pruned), hostnames) 248 | if !*dryRun { 249 | if err := r.DeleteCnames(pruned); err != nil { 250 | log.Printf("[ERROR] error deleting records: %+v", err) 251 | } else { 252 | log.Printf("[INFO] records were deleted") 253 | for _, p := range pruned { 254 | metricUpdatesApplied.WithLabelValues(p.Hostname, "delete").Inc() 255 | } 256 | } 257 | } 258 | } else { 259 | log.Printf("[INFO] modifying %d record(s): %+v", len(pruned), hostnames) 260 | if !*dryRun { 261 | if err := r.UpsertCnames(pruned); err != nil { 262 | log.Printf("[ERROR] error modifying records: %+v", err) 263 | } else { 264 | log.Printf("[INFO] records were modified") 265 | for _, p := range pruned { 266 | metricUpdatesApplied.WithLabelValues(p.Hostname, "upsert").Inc() 267 | } 268 | } 269 | } 270 | } 271 | } 272 | 273 | func (r *registrator) getTargetForIngress(ingress *v1beta1.Ingress) string { 274 | for _, sat := range r.sats { 275 | if sat.Selector.Matches(labels.Set(ingress.Labels)) { 276 | return sat.Target 277 | } 278 | } 279 | return "" 280 | } 281 | 282 | func (r *registrator) pruneBatch(action string, records []cnameRecord) []cnameRecord { 283 | pruned := []cnameRecord{} 284 | for _, u := range records { 285 | if !r.canHandleRecord(u.Hostname) { 286 | metricUpdatesRejected.Inc() 287 | log.Printf("[INFO] cannot handle dns record %s, will ignore it", u.Hostname) 288 | continue 289 | } 290 | t, err := resolveCname(fmt.Sprintf("%s.", strings.Trim(u.Hostname, ".")), r.ListNameservers()) 291 | switch action { 292 | case route53.ChangeActionDelete: 293 | o := r.ingressWatcher.HostnameOwners(u.Hostname) 294 | if len(o) > 0 { 295 | log.Printf("[DEBUG] will not delete record %s because it's still claimed by: %s", u.Hostname, strings.Join(o, ",")) 296 | } else if err == nil { 297 | pruned = append(pruned, u) 298 | } else if err != errDNSEmptyAnswer { 299 | log.Printf("[DEBUG] error resolving %s: %+v, will try to delete the record", u.Hostname, err) 300 | pruned = append(pruned, u) 301 | } else { 302 | log.Printf("[DEBUG] %s does not resolve, no-op", u.Hostname) 303 | } 304 | case route53.ChangeActionUpsert: 305 | if err != nil { 306 | log.Printf("[DEBUG] error resolving %s: %+v, will try to update the record", u.Hostname, err) 307 | pruned = append(pruned, u) 308 | } else if strings.Trim(t, ".") != u.Target { 309 | pruned = append(pruned, u) 310 | } else { 311 | log.Printf("[DEBUG] %s resolves correctly, no-op", u.Hostname) 312 | } 313 | } 314 | } 315 | pruned = uniqueRecords(pruned) 316 | return pruned 317 | } 318 | 319 | func (r *registrator) canHandleRecord(record string) bool { 320 | zone := strings.Trim(r.Domain(), ".") 321 | record = strings.Trim(record, ".") 322 | matches, err := regexp.MatchString(fmt.Sprintf("^[^.]+\\.%s$", strings.Replace(zone, ".", "\\.", -1)), record) 323 | if err != nil { 324 | log.Printf("[DEBUG] regexp match error, will not handle record %s: %+v", record, err) 325 | return false 326 | } 327 | return matches 328 | } 329 | 330 | func resolveCname(name string, nameservers []string) (string, error) { 331 | m := dns.Msg{} 332 | m.SetQuestion(name, dns.TypeCNAME) 333 | var retError error 334 | var retTarget string 335 | for _, nameserver := range nameservers { 336 | r, _, err := dnsClient.Exchange(&m, nameserver) 337 | if err != nil { 338 | retError = err 339 | continue 340 | } 341 | if len(r.Answer) == 0 { 342 | retError = errDNSEmptyAnswer 343 | continue 344 | } 345 | retTarget = r.Answer[0].(*dns.CNAME).Target 346 | retError = nil 347 | break 348 | } 349 | return retTarget, retError 350 | } 351 | 352 | func diffStringSlices(a []string, b []string) []string { 353 | ret := []string{} 354 | for _, va := range a { 355 | exists := false 356 | for _, vb := range b { 357 | if va == vb { 358 | exists = true 359 | break 360 | } 361 | } 362 | if !exists { 363 | ret = append(ret, va) 364 | } 365 | } 366 | return ret 367 | } 368 | 369 | func uniqueRecords(records []cnameRecord) []cnameRecord { 370 | uniqueRecords := []cnameRecord{} 371 | rejectedRecords := []string{} 372 | for i, r1 := range records { 373 | if stringInSlice(r1.Hostname, rejectedRecords) || recordHostnameInSlice(r1.Hostname, uniqueRecords) { 374 | continue 375 | } 376 | duplicates := []cnameRecord{} 377 | for j, r2 := range records { 378 | if i != j && r1.Hostname == r2.Hostname { 379 | duplicates = append(duplicates, r2) 380 | } 381 | } 382 | if recordTargetsAllMatch(r1.Target, duplicates) { 383 | uniqueRecords = append(uniqueRecords, r1) 384 | } else { 385 | rejectedRecords = append(rejectedRecords, r1.Hostname) 386 | } 387 | } 388 | if len(rejectedRecords) > 0 { 389 | metricUpdatesRejected.Add(float64(len(rejectedRecords))) 390 | log.Printf("[INFO] refusing to modify the following records: [%s]: they are claimed by multiple ingresses but are pointing to different targets", strings.Join(rejectedRecords, ", ")) 391 | } 392 | return uniqueRecords 393 | } 394 | 395 | func stringInSlice(s string, slice []string) bool { 396 | for _, x := range slice { 397 | if s == x { 398 | return true 399 | } 400 | } 401 | return false 402 | } 403 | 404 | func recordHostnameInSlice(h string, records []cnameRecord) bool { 405 | for _, x := range records { 406 | if h == x.Hostname { 407 | return true 408 | } 409 | } 410 | return false 411 | } 412 | 413 | func recordTargetsAllMatch(target string, records []cnameRecord) bool { 414 | for _, r := range records { 415 | if target != r.Target { 416 | return false 417 | } 418 | } 419 | return true 420 | } 421 | -------------------------------------------------------------------------------- /registrator_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "reflect" 7 | "strings" 8 | "sync" 9 | "testing" 10 | "time" 11 | 12 | "github.com/miekg/dns" 13 | 14 | "k8s.io/api/extensions/v1beta1" 15 | "k8s.io/apimachinery/pkg/labels" 16 | "k8s.io/apimachinery/pkg/watch" 17 | "k8s.io/client-go/rest" 18 | ) 19 | 20 | const ( 21 | testPrivateTarget string = "private.cluster-entrypoint.com" 22 | testPublicTarget string = "public.cluster-entrypoint.com" 23 | testTargetLabelName string = "ingress53.target" 24 | ) 25 | 26 | func TestNewRegistrator_defaults(t *testing.T) { 27 | _, err := newRegistrator("z", []string{testPrivateTarget, testPublicTarget}, testTargetLabelName) 28 | if err == nil || err.Error() != "unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined" { 29 | t.Errorf("newRegistrator did not return expected error") 30 | } 31 | 32 | // missing options 33 | _, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}}) 34 | if err != errRegistratorMissingOption { 35 | t.Errorf("newRegistrator did not return expected error") 36 | } 37 | 38 | // invalid label name 39 | _, err = newRegistrator("z", []string{testPrivateTarget, testPublicTarget}, "!^7") 40 | if err == nil { 41 | t.Errorf("newRegistrator did not return expected error") 42 | } 43 | 44 | // invalid label name 45 | _, err = newRegistrator("z", []string{testPrivateTarget, testPublicTarget}, testTargetLabelName) 46 | if err == nil { 47 | t.Errorf("newRegistrator did not return expected error") 48 | } 49 | 50 | // working 51 | _, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{testPrivateTarget, testPublicTarget}, TargetLabelName: testTargetLabelName, Route53ZoneID: "c"}) 52 | if err != nil { 53 | t.Errorf("newRegistrator returned an unexpected error: %+v", err) 54 | } 55 | } 56 | 57 | func TestRegistrator_GetTargetForIngress(t *testing.T) { 58 | // ingress ab 59 | r, err := newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{testPrivateTarget, testPublicTarget}, TargetLabelName: testTargetLabelName, Route53ZoneID: "c"}) 60 | if err != nil { 61 | t.Errorf("newRegistrator returned an unexpected error: %+v", err) 62 | } 63 | 64 | target := r.getTargetForIngress(privateIngressHostsAB) 65 | if target != testPrivateTarget { 66 | t.Errorf("getTargetForIngress returned unexpected value") 67 | } 68 | 69 | // ingress c 70 | r, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{testPrivateTarget, testPublicTarget}, TargetLabelName: testTargetLabelName, Route53ZoneID: "c"}) 71 | if err != nil { 72 | t.Errorf("newRegistrator returned an unexpected error: %+v", err) 73 | } 74 | target = r.getTargetForIngress(publicIngressHostC) 75 | if target != testPublicTarget { 76 | t.Errorf("getTargetForIngress returned unexpected value") 77 | } 78 | 79 | // ingress target not registered with ingress53 80 | r, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{testPrivateTarget, testPublicTarget}, TargetLabelName: testTargetLabelName, Route53ZoneID: "c"}) 81 | if err != nil { 82 | t.Errorf("newRegistrator returned an unexpected error: %+v", err) 83 | } 84 | target = r.getTargetForIngress(nonRegisteredIngress) 85 | if target != "" { 86 | t.Errorf("getTargetForIngress returned unexpected value") 87 | } 88 | } 89 | 90 | type mockDNSZone struct { 91 | zoneData map[string]string 92 | domain string 93 | nameservers []string 94 | } 95 | 96 | func (m *mockDNSZone) UpsertCnames(records []cnameRecord) error { 97 | for _, r := range records { 98 | m.zoneData[r.Hostname] = r.Target 99 | } 100 | return nil 101 | } 102 | 103 | func (m *mockDNSZone) DeleteCnames(records []cnameRecord) error { 104 | for _, r := range records { 105 | delete(m.zoneData, r.Hostname) 106 | } 107 | return nil 108 | } 109 | 110 | func (m *mockDNSZone) Domain() string { return m.domain } 111 | 112 | func (m *mockDNSZone) ListNameservers() []string { return m.nameservers } 113 | 114 | func (m *mockDNSZone) startMockDNSServer() (*dns.Server, error) { 115 | pc, err := net.ListenPacket("udp", "127.0.0.1:0") 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | mux := dns.NewServeMux() 121 | mux.HandleFunc(".", func(w dns.ResponseWriter, req *dns.Msg) { 122 | msg := new(dns.Msg) 123 | msg.SetReply(req) 124 | msg.Authoritative = true 125 | if target, ok := m.zoneData[strings.Trim(req.Question[0].Name, ".")]; ok { 126 | msg.Answer = append(msg.Answer, &dns.CNAME{ 127 | Hdr: dns.RR_Header{ 128 | Name: req.Question[0].Name, 129 | Rrtype: dns.TypeCNAME, 130 | Class: dns.ClassINET, 131 | Ttl: 0, 132 | }, 133 | Target: fmt.Sprintf("%s.", target), 134 | }) 135 | } 136 | w.WriteMsg(msg) 137 | }) 138 | 139 | server := &dns.Server{ 140 | PacketConn: pc, 141 | ReadTimeout: time.Hour, 142 | WriteTimeout: time.Hour, 143 | Handler: mux, 144 | } 145 | 146 | waitLock := sync.Mutex{} 147 | waitLock.Lock() 148 | server.NotifyStartedFunc = waitLock.Unlock 149 | 150 | go func() { 151 | server.ActivateAndServe() 152 | pc.Close() 153 | }() 154 | 155 | waitLock.Lock() 156 | 157 | m.nameservers = []string{pc.LocalAddr().String()} 158 | 159 | return server, nil 160 | } 161 | 162 | type mockEvent struct { 163 | et watch.EventType 164 | old *v1beta1.Ingress 165 | new *v1beta1.Ingress 166 | } 167 | 168 | type mockStore struct { 169 | items []interface{} 170 | } 171 | 172 | func (m *mockStore) Add(obj interface{}) error { return nil } 173 | func (m *mockStore) Update(obj interface{}) error { return nil } 174 | func (m *mockStore) Delete(obj interface{}) error { return nil } 175 | func (m *mockStore) List() []interface{} { return m.items } 176 | func (m *mockStore) ListKeys() []string { return nil } 177 | func (m *mockStore) Get(obj interface{}) (item interface{}, exists bool, err error) { 178 | return nil, false, nil 179 | } 180 | func (m *mockStore) GetByKey(key string) (item interface{}, exists bool, err error) { 181 | return nil, false, nil 182 | } 183 | func (m *mockStore) Replace([]interface{}, string) error { return nil } 184 | func (m *mockStore) Resync() error { return nil } 185 | 186 | func TestRegistratorHandler(t *testing.T) { 187 | privateSelector, _ := labels.Parse(fmt.Sprintf("%s=%s", testTargetLabelName, testPrivateTarget)) 188 | publicSelector, _ := labels.Parse(fmt.Sprintf("%s=%s", testTargetLabelName, testPublicTarget)) 189 | sats := []selectorAndTarget{selectorAndTarget{Selector: privateSelector, Target: testPrivateTarget}, selectorAndTarget{Selector: publicSelector, Target: testPublicTarget}} 190 | 191 | mdz := &mockDNSZone{} 192 | server, err := mdz.startMockDNSServer() 193 | defer server.Shutdown() 194 | if err != nil { 195 | t.Fatalf("dnstest: unable to run test server: %v", err) 196 | } 197 | 198 | r := ®istrator{ 199 | dnsZone: mdz, 200 | sats: sats, 201 | updateQueue: make(chan cnameChange, 16), 202 | ingressWatcher: &ingressWatcher{ 203 | stopChannel: make(chan struct{}), 204 | store: &mockStore{}, 205 | }, 206 | options: registratorOptions{ 207 | Targets: []string{testPrivateTarget, testPublicTarget}, 208 | TargetLabelName: testTargetLabelName, 209 | Route53ZoneID: "c", 210 | }, 211 | } 212 | 213 | testCases := []struct { 214 | domain string 215 | events []mockEvent 216 | data map[string]string 217 | storeIngresses []interface{} 218 | }{ 219 | { 220 | "", 221 | []mockEvent{}, 222 | map[string]string{}, 223 | nil, 224 | }, 225 | { 226 | "example.com.", 227 | []mockEvent{ 228 | {watch.Added, nil, privateIngressHostsAB}, 229 | }, 230 | map[string]string{ 231 | "a.example.com": testPrivateTarget, 232 | "b.example.com": testPrivateTarget, 233 | }, 234 | nil, 235 | }, 236 | { 237 | "example.com.", 238 | []mockEvent{ 239 | {watch.Added, nil, privateIngressHostsAB}, 240 | {watch.Deleted, privateIngressHostsAB, nil}, 241 | }, 242 | map[string]string{}, 243 | nil, 244 | }, 245 | { 246 | "example.com.", 247 | []mockEvent{ 248 | {watch.Added, nil, privateIngressHostsAB}, 249 | {watch.Modified, privateIngressHostsAB, publicIngressHostC}, 250 | }, 251 | map[string]string{ 252 | "c.example.com": testPublicTarget, 253 | }, 254 | nil, 255 | }, 256 | { 257 | "example.com.", 258 | []mockEvent{ 259 | {watch.Added, nil, privateIngressHostsAB}, 260 | {watch.Deleted, privateIngressHostsAB, nil}, 261 | {watch.Added, nil, publicIngressHostC}, 262 | }, 263 | map[string]string{ 264 | "c.example.com": testPublicTarget, 265 | }, 266 | nil, 267 | }, 268 | { 269 | "an.example.com.", 270 | []mockEvent{ 271 | {watch.Added, nil, privateIngressHostsAB}, 272 | }, 273 | map[string]string{}, 274 | nil, 275 | }, 276 | { 277 | "example.com.", 278 | []mockEvent{ 279 | {watch.Added, nil, privateIngressHostE}, 280 | }, 281 | map[string]string{ 282 | "e.example.com": testPrivateTarget, 283 | }, 284 | nil, 285 | }, 286 | { 287 | "example.com.", 288 | []mockEvent{ 289 | {watch.Added, nil, privateIngressHostE}, 290 | {watch.Added, nil, privateIngressHostEDup}, 291 | }, 292 | map[string]string{ 293 | "e.example.com": testPrivateTarget, 294 | }, 295 | nil, 296 | }, 297 | { 298 | "example.com.", 299 | []mockEvent{ 300 | {watch.Added, nil, privateIngressHostE}, 301 | {watch.Added, nil, publicIngressHostEDup}, 302 | }, 303 | map[string]string{}, 304 | nil, 305 | }, 306 | { 307 | "example.com.", 308 | []mockEvent{ 309 | {watch.Added, nil, privateIngressHostE}, 310 | {watch.Deleted, privateIngressHostsAB, nil}, 311 | {watch.Modified, privateIngressHostE, publicIngressHostEDup}, 312 | }, 313 | map[string]string{ 314 | "e.example.com": testPublicTarget, 315 | }, 316 | nil, 317 | }, 318 | { 319 | "example.com.", 320 | []mockEvent{ 321 | {watch.Added, nil, privateIngressHostE2}, 322 | {watch.Modified, privateIngressHostE2, privateIngressHostE2Fixed}, 323 | }, 324 | map[string]string{ 325 | "e.example.com": testPrivateTarget, 326 | "e2.example.com": testPrivateTarget, 327 | }, 328 | []interface{}{privateIngressHostE}, 329 | }, 330 | } 331 | 332 | for i, test := range testCases { 333 | r.ingressWatcher.stopChannel = make(chan struct{}) 334 | r.ingressWatcher.store = &mockStore{items: test.storeIngresses} 335 | mdz.domain = test.domain 336 | mdz.zoneData = map[string]string{} 337 | r.updateQueue = make(chan cnameChange, 16) 338 | for _, e := range test.events { 339 | r.handler(e.et, e.old, e.new) 340 | } 341 | wg := sync.WaitGroup{} 342 | wg.Add(1) 343 | go func() { 344 | defer wg.Done() 345 | r.processUpdateQueue() 346 | }() 347 | time.Sleep(1000 * time.Millisecond) // XXX 348 | close(r.stopChannel) 349 | wg.Wait() 350 | if !reflect.DeepEqual(mdz.zoneData, test.data) { 351 | t.Errorf("handler produced unexcepted zone data for test case #%02d: %+v, expected: %+v", i, mdz.zoneData, test.data) 352 | } 353 | } 354 | } 355 | 356 | func TestRegistrator_canHandleRecord(t *testing.T) { 357 | testCases := []struct { 358 | record string 359 | expected bool 360 | }{ 361 | {"example.com", false}, // apex 362 | {"test.example.org", false}, // different zone 363 | {"wrong.test.example.com.", false}, // too deep 364 | {"test.example.com", true}, 365 | {"test.example.com.", true}, 366 | } 367 | defer mockRoute53Timers()() 368 | r := registrator{dnsZone: &mockDNSZone{domain: "example.com"}} 369 | 370 | for i, tc := range testCases { 371 | v := r.canHandleRecord(tc.record) 372 | if v != tc.expected { 373 | t.Errorf("newRoute53Zone returned unexpected value for test case #%02d: %v", i, v) 374 | } 375 | } 376 | } 377 | 378 | func setupMockDNSRecord(mux *dns.ServeMux, name string, target string) { 379 | mux.HandleFunc(name, func(w dns.ResponseWriter, req *dns.Msg) { 380 | m := new(dns.Msg) 381 | m.SetReply(req) 382 | m.Authoritative = true 383 | m.Answer = append(m.Answer, &dns.CNAME{ 384 | Hdr: dns.RR_Header{ 385 | Name: req.Question[0].Name, 386 | Rrtype: dns.TypeCNAME, 387 | Class: dns.ClassINET, 388 | Ttl: 0, 389 | }, 390 | Target: target, 391 | }) 392 | w.WriteMsg(m) 393 | }) 394 | } 395 | 396 | func startMockDNSServer(laddr string, records map[string]string) (*dns.Server, string, error) { 397 | pc, err := net.ListenPacket("udp", laddr) 398 | if err != nil { 399 | return nil, "", err 400 | } 401 | 402 | mux := dns.NewServeMux() 403 | for n, r := range records { 404 | setupMockDNSRecord(mux, n, r) 405 | } 406 | 407 | server := &dns.Server{ 408 | PacketConn: pc, 409 | ReadTimeout: time.Hour, 410 | WriteTimeout: time.Hour, 411 | Handler: mux, 412 | } 413 | 414 | waitLock := sync.Mutex{} 415 | waitLock.Lock() 416 | server.NotifyStartedFunc = waitLock.Unlock 417 | 418 | go func() { 419 | server.ActivateAndServe() 420 | pc.Close() 421 | }() 422 | 423 | waitLock.Lock() 424 | return server, pc.LocalAddr().String(), nil 425 | } 426 | 427 | func startMockDNSServerFleet(records map[string]string) ([]*dns.Server, []string, error) { 428 | servers := []*dns.Server{} 429 | serverAddresses := []string{} 430 | 431 | s, addr, err := startMockDNSServer("127.0.0.1:0", records) 432 | if err != nil { 433 | return nil, nil, err 434 | } 435 | servers = append(servers, s) 436 | serverAddresses = append(serverAddresses, addr) 437 | 438 | s, addr, err = startMockDNSServer("127.0.0.1:0", records) 439 | if err != nil { 440 | return nil, nil, err 441 | } 442 | servers = append(servers, s) 443 | serverAddresses = append(serverAddresses, addr) 444 | 445 | s, addr, err = startMockDNSServer("127.0.0.1:0", records) 446 | if err != nil { 447 | return nil, nil, err 448 | } 449 | servers = append(servers, s) 450 | serverAddresses = append(serverAddresses, addr) 451 | 452 | s, addr, err = startMockDNSServer("127.0.0.1:0", records) 453 | if err != nil { 454 | return nil, nil, err 455 | } 456 | servers = append(servers, s) 457 | serverAddresses = append(serverAddresses, addr) 458 | 459 | return servers, serverAddresses, nil 460 | } 461 | 462 | func startMockSemiBrokenDNSServerFleet(records map[string]string) ([]*dns.Server, []string, error) { 463 | servers := []*dns.Server{&dns.Server{}} 464 | serverAddresses := []string{"127.0.0.1:10000"} 465 | 466 | s, addr, err := startMockDNSServer("127.0.0.1:0", nil) 467 | if err != nil { 468 | return nil, nil, err 469 | } 470 | servers = append(servers, s) 471 | serverAddresses = append(serverAddresses, addr) 472 | 473 | s, addr, err = startMockDNSServer("127.0.0.1:0", records) 474 | if err != nil { 475 | return nil, nil, err 476 | } 477 | servers = append(servers, s) 478 | serverAddresses = append(serverAddresses, addr) 479 | 480 | s, addr, err = startMockDNSServer("127.0.0.1:0", records) 481 | if err != nil { 482 | return nil, nil, err 483 | } 484 | servers = append(servers, s) 485 | serverAddresses = append(serverAddresses, addr) 486 | 487 | return servers, serverAddresses, nil 488 | } 489 | 490 | func stopMockDNSServerFleet(servers []*dns.Server) { 491 | for _, s := range servers { 492 | s.Shutdown() 493 | } 494 | } 495 | 496 | func TestDNSClient_ResolveCname_noServer(t *testing.T) { 497 | _, err := resolveCname("example.com.", []string{"127.0.0.1:65111"}) 498 | if err == nil { 499 | t.Fatalf("Client.ResolveA should have returned an error") 500 | } 501 | } 502 | 503 | func TestDNSClient_ResolveCname_empty(t *testing.T) { 504 | servers, serverAddresses, err := startMockDNSServerFleet(map[string]string{}) 505 | defer stopMockDNSServerFleet(servers) 506 | if err != nil { 507 | t.Fatalf("dnstest: unable to run test server: %v", err) 508 | } 509 | 510 | _, err = resolveCname("example.com.", serverAddresses) 511 | if err != errDNSEmptyAnswer { 512 | t.Fatalf("Client.ResolveA should have returned an empty answer error") 513 | } 514 | } 515 | 516 | func TestDNSClient_ResolveCname_broken(t *testing.T) { 517 | servers, serverAddresses, err := startMockSemiBrokenDNSServerFleet(map[string]string{"example.com.": "target.example.com."}) 518 | defer stopMockDNSServerFleet(servers) 519 | if err != nil { 520 | t.Fatalf("dnstest: unable to run test server: %v", err) 521 | } 522 | 523 | resp, err := resolveCname("example.com.", serverAddresses) 524 | if err != nil { 525 | t.Fatalf("Client.ResolveA returned unexpected error: %+v", err) 526 | } 527 | 528 | if resp != "target.example.com." { 529 | t.Fatalf("Client.ResolveA returned unexpected response") 530 | } 531 | } 532 | 533 | func TestDiffStringSlices(t *testing.T) { 534 | testCases := []struct { 535 | A []string 536 | B []string 537 | R []string 538 | }{ 539 | { 540 | A: []string{"A"}, 541 | B: []string{"A"}, 542 | R: []string{}, 543 | }, 544 | { 545 | A: []string{"A"}, 546 | B: []string{"B"}, 547 | R: []string{"A"}, 548 | }, 549 | { 550 | A: []string{}, 551 | B: []string{"B"}, 552 | R: []string{}, 553 | }, 554 | { 555 | A: []string{"A"}, 556 | B: []string{}, 557 | R: []string{"A"}, 558 | }, 559 | { 560 | A: []string{"A", "B", "C"}, 561 | B: []string{"B", "D", "A"}, 562 | R: []string{"C"}, 563 | }, 564 | } 565 | 566 | for i, test := range testCases { 567 | r := diffStringSlices(test.A, test.B) 568 | if !reflect.DeepEqual(r, test.R) { 569 | t.Errorf("diffStringSlices returned unexpected result for test case #%02d: %+v", i, r) 570 | } 571 | } 572 | 573 | } 574 | --------------------------------------------------------------------------------