├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── RELEASE_NOTES.md ├── demo ├── Dockerfile ├── deployment.yaml ├── index.js └── package.json ├── examples ├── aws-cluster-deployment.yaml ├── ingress-daemonset.yaml └── multipurpose-router-daemonset.yaml ├── glide.lock ├── glide.yaml ├── kubernetes ├── client.go └── client_test.go ├── main.go ├── nginx ├── config.go ├── config_test.go └── server.go ├── router ├── config.go ├── config_test.go ├── pods.go ├── pods_test.go ├── secrets.go ├── secrets_test.go └── types.go └── utils ├── validation.go └── validation_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.test 3 | 4 | .classpath 5 | .project 6 | .projectile 7 | .tern-project 8 | coverage.out 9 | k8s-router 10 | 11 | .idea/ 12 | .settings/ 13 | .vscode/ 14 | demo/node_modules/ 15 | typings/ 16 | vendor/ 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx:stable-alpine 2 | 3 | MAINTAINER Jeremy Whitlock 4 | 5 | ARG GIT_COMMIT 6 | 7 | LABEL Description="A general purpose router for Kubernetes." 8 | LABEL GitCommit=${GIT_COMMIT} 9 | 10 | # Prepare the environment 11 | RUN apk update \ 12 | # Install the CA Certificates for SSL usage 13 | && apk add --no-cache ca-certificates \ 14 | # Update the CA Certificates 15 | && update-ca-certificates \ 16 | # Remove the nginx log symlinks to give more control over how logging is controlled 17 | && unlink /var/log/nginx/access.log \ 18 | && unlink /var/log/nginx/error.log 19 | 20 | COPY k8s-router / 21 | 22 | CMD ["/k8s-router"] 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2016 Apigee Corporation 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | APPENDIX: Attributions for bundled softwares. 204 | 205 | This product bundles portions of Kubernetes (Apache 2.0 License) 206 | Copyright 2016 The Kubernetes Authors. 207 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GIT_COMMIT=$(shell git rev-parse HEAD) 2 | 3 | all: build 4 | 5 | check: test lint 6 | 7 | clean: 8 | rm -f coverage.out k8s-router router/router.test kubernetes/kubernetes.test nginx/nginx.test utils/utils.test 9 | 10 | lint: 11 | golint router 12 | golint kubernetes 13 | golint nginx 14 | 15 | test: 16 | go test -cover $$(glide novendor) 17 | 18 | build: main.go 19 | go build 20 | 21 | build-for-container: main.go 22 | CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -o k8s-router . 23 | 24 | build-image: build-for-container 25 | docker build -t k8s-router --build-arg GIT_COMMIT=$(GIT_COMMIT) . 26 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | k8s-router 2 | Copyright 2016 Apigee Corporation 3 | 4 | This product includes software developed at the 5 | Apigee Corporation (http://apigee.com/). 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | The purpose of this project is to provide a name and path based router for Kubernetes. It started out as an ingress 4 | controller but has since been repurposed to allow for both ingress and other types of routing via to its 5 | configurability. From an ingress perspective, this router does things a little different than your typical 6 | [Kubernetes Ingress controller](http://kubernetes.io/docs/user-guide/ingress/): 7 | 8 | * This version does pod-level routing instead of service-level routing 9 | * This version does not use the 10 | [Kubernetes Ingress Resource](http://kubernetes.io/docs/user-guide/ingress/#the-ingress-resource) definitions 11 | and instead uses pod-level annotations to wire things up _(This is partially because the built-in ingress resource 12 | is intended for service-based ingress instead of pod-based ingress.)_ 13 | 14 | But in the end, you have a routing controller capable of doing routing based on the combination of hostname/IP and path. 15 | 16 | # Design 17 | 18 | This router is written in Go and is intended to be deployed within a container on Kubernetes. Upon startup, this router 19 | will find the [Pods](http://kubernetes.io/docs/user-guide/pods/) marked for routing _(using a configurable label 20 | selector)_ and the [Secrets](http://kubernetes.io/docs/user-guide/secrets/) *(using a configurable location)_ used to 21 | secure routing for those pods. _(For more details on the role secrets play in this router, please see the 22 | [Security section](#security) of this document.)_ The Pods marked for routing are then analyzed to identify the wiring 23 | information used for routing stored in the Pod's [annotations](http://kubernetes.io/docs/user-guide/annotations/): 24 | 25 | * `routingHosts`: This is a space delimited array of hostnames and/or IP addresses that are expected to route to the 26 | Pod _(Example: `test.github.com 192.168.0.1`)_ 27 | * `routingPaths`: This is the space delimited array of request path or path prefixes that are expected to route to the 28 | Pod and its appropriate container port. _(The value's format is `{PORT}:{PATH}` where `{PORT}` corresponds to the 29 | container port serving the traffic for the `{PATH}`. Example: `3000:/nodejs 8080:/java`.)_ 30 | 31 | Once we've found all Pods and Secrets that are involved in routing, we generate an nginx configuration file and start 32 | nginx. At this point, we cache Pods and Secrets to avoid having to requery the full list each time and instead listen 33 | for Pod and Secret events. Any time a Pod or Secret event occurs that would have an impact on routing, we regenerate 34 | the nginx configuration and reload it. _(The idea here was to allow for an initial hit to pull all pods but to then 35 | to use the events for as quick a turnaround as possible.)_ Events are processed in 2 second chunks. 36 | 37 | Each Pod can expose one or more services by using one or more entries in the `routingPaths` annotation. All of the 38 | paths exposed via `routingPaths` are exposed for each of the hosts listed in the `routingHosts` annotation. _(So if 39 | you have a trafficHosts of `host1 host2` and a `routingPaths` of `80:/ 3000:/nodejs`, you would have 4 separate nginx 40 | location blocks: `host1/ -> {PodIP}:80`, `host2/ -> {PodIP}:80`, `host1/nodejs -> {PodIP}:3000` and 41 | `host2/nodejs -> {PodIP}:3000` Right now there is no way to associate specific paths to specific hosts but it may be 42 | something we support in the future.)_ 43 | 44 | # Configuration 45 | 46 | All of the touch points for this router are configurable via environment variables: 47 | 48 | * `API_KEY_HEADER`: This is the header name used by nginx to identify the API Key used _(Default: `X-ROUTING-API-KEY`)_ 49 | * `API_KEY_SECRET_LOCATION`: This is the location of the optional API Key to use to secure communication to your Pods. 50 | _(The format for this key is `{SECRET_NAME}:{SECRET_DATA_FIELD_NAME}`. Default: `routing:api-key`)_ 51 | * `CLIENT_MAX_BODY_SIZE`: Configures the max client request body size of nginx. _(Default: `0`, Disables checking of client request body size.)_ 52 | * `HOSTS_ANNOTATION`: This is the annotation name used to store the space delimited array of hosts used for routing to 53 | your Pods _(Default: `routingHosts`)_ 54 | * `PATHS_ANNOTATION`: This is the annotation name used to store the space delimited array of routing path configurations for your Pods _(Default: `routingPaths`)_ 55 | * `PORT`: This is the port that nginx will listen on _(Default: `80`)_ 56 | * `ROUTABLE_LABEL_SELECTOR`: This is the [label selector](http://kubernetes.io/docs/user-guide/labels/#label-selectors) 57 | used to identify Pods that are marked for routing _(Default: `routable=true`)_ 58 | 59 | # Security 60 | 61 | While most routers will perform routing only, we have added a very simple mechanism to do API Key based authorization 62 | at the router level. Why might you want this? Imagine you've got multi-tenancy in Kubernetes where each namespace is 63 | specific to a single tentant. To avoid a Pod in namespace `X` configuring itself to receive traffic from namespace `Y`, 64 | this router allows you to create a specially named secret _(`routing` in this case)_ with a specially named data field 65 | _(`api-key`)_ and the value stored in this secret will be used to secure traffic to all Pods in your namespace wired up 66 | for routing. To do this, nginx is configured to ensure that all requests routed to your Pod have the 67 | `X-ROUTING-API-KEY` header provided with its value being the base64-encoded value of your secret. 68 | 69 | Here is an example of how you might create this secret so that all Pods wired up for routing in the `my-namespace` 70 | namespace are secured via API Key: 71 | 72 | ``` 73 | kubectl create secret generic routing --from-literal=api-key=supersecret --namespace=my-namespace 74 | ``` 75 | 76 | Based on the example, any routes that points to Pods in the `my-namespace` namespace will be required to have 77 | `X-ROUTING-API-KEY: c3VwZXJzZWNyZXQ=` set in their request for the router to allow routing to the Pods. Otherwise, a 78 | `403` is returned. Of course, if your namespace does not have the specially named secret, you do not have to adhere to 79 | provide this header. 80 | 81 | **Note:** This feature is written assuming that each combination of `routingHosts` and `routingPaths` will only be 82 | configured such that the Pods servicing the traffice are from a single namespace. Once you start allowing pods from 83 | multiple namespaces to consume traffic for the same host and path combination, this falls apart. While the routing will 84 | work fine in this situation, the router's API Key is namespace specific and the first seen API Key is the one that is 85 | used. 86 | 87 | # Streaming Support 88 | 89 | By default, nginx will buffer responses for proxied servers. Unfortunately, this can be a problem if you deploy a 90 | streaming APIs. Thankfully, nginx makes it easy for proxied applications to disable proxy buffering by setting the 91 | `X-Accel-Buffering` header to `no`. Doing this will make your streaming API work as expected. For more details, view 92 | the nginx documentation: http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering 93 | 94 | # WebSocket Support 95 | 96 | Why are we bringing up WebSocket support? Well, nginx itself operates in a way that makes routing to Pods that are 97 | WebSocket servers a little difficult. For more details, read the nginx documentation located here: 98 | https://www.nginx.com/blog/websocket-nginx/ The way that the k8s-router addresses things is at each `location` 99 | block, we throw in some WebSocket configuration. It's very simple stuff but since there is some reasoning behind the 100 | location where this is applied and the approach taken, it makes sense to explain it here. 101 | 102 | The WebSocket configuration is at the `location` level, and it is there because nginx does not allow us to use the 103 | [set directive](http://nginx.org/en/docs/http/ngx_http_rewrite_module.html#set) at the `server` or `http` level. We 104 | have to use the `set` directive to properly handle the `Connection` header. See, nginx uses `close` as the default 105 | value for the `Connection` header when there is no `Connection` header provided. So if we just passed through the 106 | `Connection` header and there was no `Connection` header provided, instead of using the default value of `close` the 107 | value would be `''` which would basically delete the `Connection` header which is not how nginx operates. So we have to 108 | conditionally set a variable based on the `Connection` header value and `set` is the only way. 109 | 110 | The other part of this implementation that is worth documenting is that in the previously linked documentation for 111 | enabling WebSockets in nginx, you see they use `proxy_http_version 1.1;` to force HTTP 1.1. Well, for a generic server 112 | where not all `location` blocks are for WebSockets, we needed a way to conditionally enable HTTP 1.1. Well...there is 113 | no way to do this. `proxy_http_version` cannot be used in an `if` directive and `proxy_http_version` cannot be set to 114 | a string value, which is the only value you can use for nginx variables. So since we do not want to force HTTP 1.1 on 115 | everyone, we just leave it up to the client to make an HTTP 1.1 request. 116 | 117 | So when you look at the generated nginx configuration and see some duplicate configuration related to WebSockets, or 118 | you see that we are not forcing HTTP 1.1, now you know. 119 | 120 | # Examples 121 | 122 | ## An Ingress Controller 123 | 124 | Let's assume you've already deployed the router controller. _(If you haven't, feel free to look at the 125 | [Building and Running](#building-and-running) section of the documentation.)_ When the router starts up, nginx is 126 | configured and started on your behalf. The generated `/etc/nginx/nginx.conf` that the router starts with looks like 127 | this _(assuming you do not have any deployed Pods marked for routing)_: 128 | 129 | ``` nginx 130 | # A very simple nginx configuration file that forces nginx to start as a daemon. 131 | events {} 132 | http { 133 | # Default server that will just close the connection as if there was no server available 134 | server { 135 | listen 80 default_server; 136 | return 444; 137 | } 138 | } 139 | daemon on; 140 | ``` 141 | 142 | This configuration will tell nginx to listen on port `80` and all requests for unknown hosts will be closed, as if there 143 | was no server listening for traffic. _(This approach is better than reporting a `404` because a `404` says someone is 144 | there but the request was for a missing resource while closing the connection says that the request was for a server 145 | that didn't exist, or in our case a request was made to a host that our router is unaware of.)_ 146 | 147 | Now that we know how the router spins up nginx initially, let's deploy a _microservice_ to Kubernetes. To do that, we 148 | will be packaging up a simple Node.js application that prints out the environment details of its running container, 149 | including the IP address(es) of its host. To do this, we will build a Docker image, publish the Docker image and then 150 | create a Kubernetes ReplicationController that will deploy one pod representing our microservice. 151 | 152 | **Note:** All commands you see in this demo assume you are already within the `demo` directory. 153 | These commands are written assuming you are running docker at `192.168.64.1:5000` so please adjust your Docker commands 154 | accordingly. 155 | 156 | First things first, let's build our Docker image using `docker build -t nodejs-k8s-env .`, tag the Docker image using 157 | `docker tag -f nodejs-k8s-env 192.168.64.1:5000/nodejs-k8s-env` and finally push the Docker image to your Docker 158 | registry using `docker push 192.168.64.1:5000/nodejs-k8s-env`. At this point, we have built and published a Docker 159 | image for our microservice. 160 | 161 | The next step is to deploy our microservice to Kubernetes but before we do this, let's look at the ReplicationController 162 | configuration file to see what is going on. Here is the `rc.yaml` we'll be using to deploy our microservice 163 | _(Same as `demo/rc.yaml`)_: 164 | 165 | ``` yaml 166 | apiVersion: v1 167 | kind: ReplicationController 168 | metadata: 169 | name: nodejs-k8s-env 170 | labels: 171 | name: nodejs-k8s-env 172 | spec: 173 | replicas: 1 174 | selector: 175 | name: nodejs-k8s-env 176 | template: 177 | metadata: 178 | labels: 179 | name: nodejs-k8s-env 180 | # This marks the pod as routable 181 | routable: "true" 182 | annotations: 183 | # This says that only traffic for the "test.k8s.local" host will be routed to this pod 184 | routingHosts: "test.k8s.local" 185 | # This says that only traffic for the "/nodejs" path and its sub paths will be routed to this pod, on port 3000 186 | routingPaths: "3000:/nodejs" 187 | spec: 188 | containers: 189 | - name: nodejs-k8s-env 190 | image: 192.168.64.1:5000/nodejs-k8s-env 191 | env: 192 | - name: PORT 193 | value: "3000" 194 | ports: 195 | - containerPort: 3000 196 | ``` 197 | 198 | When we deploy our microservice using `kubectl create -f rc.yaml`, the router will notice that we now have one Pod 199 | running that is marked for routing. If you were tailing the logs, or you were to review the content of 200 | `/etc/nginx/nginx.conf` in the container, you should see that it now reflects that we have a new microservice 201 | deployed: 202 | 203 | ``` nginx 204 | events { 205 | worker_connections 1024; 206 | } 207 | http { 208 | # http://nginx.org/en/docs/http/ngx_http_core_module.html 209 | types_hash_max_size 2048; 210 | server_names_hash_max_size 512; 211 | server_names_hash_bucket_size 64; 212 | 213 | # Force HTTP 1.1 for upstream requests 214 | proxy_http_version 1.1; 215 | 216 | # When nginx proxies to an upstream, the default value used for 'Connection' is 'close'. We use this variable to do 217 | # the same thing so that whenever a 'Connection' header is in the request, the variable reflects the provided value 218 | # otherwise, it defaults to 'close'. This is opposed to just using "proxy_set_header Connection $http_connection" 219 | # which would remove the 'Connection' header from the upstream request whenever the request does not contain a 220 | # 'Connection' header, which is a deviation from the nginx norm. 221 | map $http_connection $p_connection { 222 | default $http_connection; 223 | '' close; 224 | } 225 | 226 | # Pass through the appropriate headers 227 | proxy_set_header Connection $p_connection; 228 | proxy_set_header Host $host; 229 | proxy_set_header Upgrade $http_upgrade; 230 | 231 | server { 232 | listen 80; 233 | server_name test.k8s.local; 234 | 235 | location /nodejs { 236 | # Pod nodejs-k8s-env-eq7mh 237 | proxy_pass http://10.244.69.6:3000; 238 | } 239 | } 240 | 241 | # Default server that will just close the connection as if there was no server available 242 | server { 243 | listen 80 default_server; 244 | return 444; 245 | } 246 | } 247 | ``` 248 | 249 | This means that if someone requests `http://test.k8s.local/nodejs`, assuming you've got `test.k8s.local` pointed to the 250 | edge of your Kubernetes cluster, it should get routed to the proper Pod _(`nodejs-k8s-env-eq7mh` in our example)_. If 251 | everything worked out properly, you should see output like this: 252 | 253 | ``` json 254 | { 255 | "env": { 256 | "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 257 | "HOSTNAME": "nodejs-k8s-env-eq7mh", 258 | "PORT": "3000", 259 | "KUBERNETES_PORT": "tcp://10.100.0.1:443", 260 | "KUBERNETES_PORT_443_TCP": "tcp://10.100.0.1:443", 261 | "KUBERNETES_PORT_443_TCP_PROTO": "tcp", 262 | "KUBERNETES_PORT_443_TCP_PORT": "443", 263 | "KUBERNETES_PORT_443_TCP_ADDR": "10.100.0.1", 264 | "KUBERNETES_SERVICE_HOST": "10.100.0.1", 265 | "KUBERNETES_SERVICE_PORT": "443", 266 | "VERSION": "v5.8.0", 267 | "NPM_VERSION": "3", 268 | "HOME": "/root" 269 | }, 270 | "ips": { 271 | "lo": "127.0.0.1", 272 | "eth0": "10.244.69.6" 273 | } 274 | } 275 | ``` 276 | 277 | If you've noticed in the example nginx configuration files above, nginx is configured appropriately to reverse proxy to 278 | WebSocket servers. _(The only reason we bring this up is that it's not something you get out of the box.)_ To test 279 | this using the the previously-deployed application, use the following Node.js application: 280 | 281 | ```js 282 | var socket = require('socket.io-client')('http://test.k8s.local', {path: '/nodejs/socket.io'}); 283 | 284 | socket.on('env', function (env) { 285 | console.log(JSON.stringify(env, null, 2)); 286 | }); 287 | 288 | // Emit the 'env' event to the server, which emits an 'env' event to the client with the server environment details. 289 | socket.emit('env'); 290 | ``` 291 | 292 | Now that's cool and all but what happens when we scale our application? Let's scale our microservice to `3` instances 293 | using `kubectl scale --replicas=3 replicationcontrollers nodejs-k8s-env`. Your `/etc/nginx/nginx.conf` should look 294 | something like this: 295 | 296 | ``` nginx 297 | events { 298 | worker_connections 1024; 299 | } 300 | http { 301 | # http://nginx.org/en/docs/http/ngx_http_core_module.html 302 | types_hash_max_size 2048; 303 | server_names_hash_max_size 512; 304 | server_names_hash_bucket_size 64; 305 | 306 | # Force HTTP 1.1 for upstream requests 307 | proxy_http_version 1.1; 308 | 309 | # When nginx proxies to an upstream, the default value used for 'Connection' is 'close'. We use this variable to do 310 | # the same thing so that whenever a 'Connection' header is in the request, the variable reflects the provided value 311 | # otherwise, it defaults to 'close'. This is opposed to just using "proxy_set_header Connection $http_connection" 312 | # which would remove the 'Connection' header from the upstream request whenever the request does not contain a 313 | # 'Connection' header, which is a deviation from the nginx norm. 314 | map $http_connection $p_connection { 315 | default $http_connection; 316 | '' close; 317 | } 318 | 319 | # Pass through the appropriate headers 320 | proxy_set_header Connection $p_connection; 321 | proxy_set_header Host $host; 322 | proxy_set_header Upgrade $http_upgrade; 323 | 324 | # Upstream for /nodejs traffic on test.k8s.local 325 | upstream upstream1866206336 { 326 | # Pod nodejs-k8s-env-eq7mh 327 | server 10.244.69.6:3000; 328 | # Pod nodejs-k8s-env-yr1my 329 | server 10.244.69.8:3000; 330 | # Pod nodejs-k8s-env-oq9xn 331 | server 10.244.69.9:3000; 332 | } 333 | 334 | server { 335 | listen 80; 336 | server_name test.k8s.local; 337 | 338 | location /nodejs { 339 | # Upstream upstream1866206336 340 | proxy_pass http://upstream1866206336; 341 | } 342 | } 343 | 344 | # Default server that will just close the connection as if there was no server available 345 | server { 346 | listen 80 default_server; 347 | return 444; 348 | } 349 | } 350 | ``` 351 | 352 | The big change between the one Pod microservice and the N Pod microservice is that now the nginx configuration uses 353 | the nginx [upstream](http://nginx.org/en/docs/http/ngx_http_upstream_module.html) to do load balancing across the N 354 | different Pods. And due to the default load balancer in nginx being round-robin based, requests for 355 | `http://test.k8s.local/nodejs` should return a different payload for each request showing that you are indeed 356 | hitting each individual Pod. 357 | 358 | I hope this example gave you a better idea of how this all works. If not, let us know how to make it better. 359 | 360 | ## Multipurpose Deployments 361 | 362 | As mentioned above, this project started out as an ingress with the sole purpose of routing traffic from the internet 363 | to Pods within the Kubernetes cluster. One of the use cases we have at work is we need an general ingress but we also 364 | want to use this router for a simplistic service router. So essentially, we have a public ingress and a 365 | private...router. Here is an example deployment file where you use the configurability of this router to serve both 366 | purposes: 367 | 368 | ``` yaml 369 | apiVersion: extensions/v1beta1 370 | kind: DaemonSet 371 | metadata: 372 | name: k8s-pods-router 373 | labels: 374 | app: k8s-pods-router 375 | spec: 376 | template: 377 | metadata: 378 | labels: 379 | app: k8s-pods-router 380 | spec: 381 | containers: 382 | - image: thirtyx/k8s-router:latest 383 | imagePullPolicy: Always 384 | name: k8s-pods-router-public 385 | ports: 386 | - containerPort: 80 387 | hostPort: 80 388 | env: 389 | - name: POD_NAME 390 | valueFrom: 391 | fieldRef: 392 | fieldPath: metadata.name 393 | - name: POD_NAMESPACE 394 | valueFrom: 395 | fieldRef: 396 | fieldPath: metadata.namespace 397 | # Use the configuration to use the public/private paradigm (public version) 398 | - name: API_KEY_SECRET_LOCATION 399 | value: routing:public-api-key 400 | - name: HOSTS_ANNOTATION 401 | value: publicHosts 402 | - name: PATHS_ANNOTATION 403 | value: publicPaths 404 | - image: thirtyx/k8s-router:latest 405 | imagePullPolicy: Always 406 | name: k8s-pods-router-private 407 | ports: 408 | - containerPort: 81 409 | # We should probably avoid using host port and if needed, at least lock it down from external access 410 | hostPort: 81 411 | env: 412 | - name: POD_NAME 413 | valueFrom: 414 | fieldRef: 415 | fieldPath: metadata.name 416 | - name: POD_NAMESPACE 417 | valueFrom: 418 | fieldRef: 419 | fieldPath: metadata.namespace 420 | # Use the configuration to use the public/private paradigm (private version) 421 | - name: API_KEY_SECRET_LOCATION 422 | value: routing:private-api-key 423 | - name: HOSTS_ANNOTATION 424 | value: privateHosts 425 | - name: PATHS_ANNOTATION 426 | value: privatePaths 427 | # Since we cannot have two containers listening on the same port, use a different port for the private router 428 | - name: PORT 429 | value: "81" 430 | ``` 431 | 432 | Based on this deployment, we have an ingress that serves `publicHosts` and `publicPaths` combinations and an internal 433 | router that serves `privateHosts` and `privatePaths` combinations. With this being the case, let's take our example 434 | Node.js application deployed above and lets deploy a variant over it that has both public and private paths: 435 | 436 | ``` yaml 437 | apiVersion: v1 438 | kind: ReplicationController 439 | metadata: 440 | name: nodejs-k8s-env 441 | labels: 442 | name: nodejs-k8s-env 443 | spec: 444 | replicas: 1 445 | selector: 446 | name: nodejs-k8s-env 447 | template: 448 | metadata: 449 | labels: 450 | name: nodejs-k8s-env 451 | routable: "true" 452 | annotations: 453 | # Private routing information 454 | privateHosts: "test.k8s.local" 455 | privatePaths: "3000:/internal" 456 | # Public routing information 457 | publicHosts: "test.k8s.com" 458 | publicPaths: "3000:/public" 459 | spec: 460 | containers: 461 | - name: nodejs-k8s-env 462 | image: thirtyx/nodejs-k8s-env 463 | env: 464 | - name: PORT 465 | value: "3000" 466 | ports: 467 | - containerPort: 3000 468 | ``` 469 | 470 | Now if we were to `curl http://test.k8s.com/nodejs` from outside of Kubernetes, assuming DNS was setup properly, the 471 | ingress router would route properly but if we were to `curl http://test.k8s.local/nodejs`, it wouldn't go anywhere. Not 472 | only that, if I were to `curl http://test.k8s.com/internal`, it also would not go anywhere. The only way to access the 473 | `/internal` path would be to be within Kubernetes, with DNS properly setup, and to 474 | `curl http://test.k8s.local/internal`. 475 | 476 | Now I realize this is a somewhat convoluted example but the purpose was to show how we could use the same code base to 477 | serve different roles using configuration alone. Thet network isolation and security required to do this properly is 478 | outside the scope of this example. 479 | 480 | # Building and Running 481 | 482 | If you want to run `k8s-router` in _mock mode_, you can use `go build` followed by `./k8s-router`. Running in mock mode 483 | means that `nginx` is not actually started and managed, nor is any actual routing performed. `k8s-router` will use your 484 | `kubectl` configuration to identify the cluster connection details by using the current context `kubectl` is configured 485 | for. This is very useful in the event you want to connect to an external Kubernetes cluster and see the generated 486 | routing configuration. 487 | 488 | If you're building this to run on Kubernetes, you'll need to do the following: 489 | 490 | * `CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -o k8s-router .` 491 | * `docker build ...` 492 | * `docker tag ...` 493 | * `docker push ...` 494 | 495 | _(The `...` are there because your Docker comands will likely be different than mine or someone else's)_ We have an 496 | example DaemonSet for deploying the k8s-router as an ingress controller to Kubernetes located at 497 | `examples/ingress-daemonset.yaml`. Here is how I test locally: 498 | 499 | * `CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -o k8s-router .` 500 | * `docker build -t k8s-router .` 501 | * `docker tag -f k8s-router 192.168.64.1:5000/k8s-router` 502 | * `docker push 192.168.64.1:5000/k8s-router` 503 | * `kubectl create -f examples/ingress-daemonset.yaml` 504 | 505 | **Note:** This router is written to be ran within Kubernetes but for testing purposes, it can be ran outside of 506 | Kubernetes. When ran outside of Kubernetes, you will have have a kube config file. When ran outside the container, nginx itself will not be 507 | started and its configuration file will not be written to disk, only printed to stdout. This might change in the future 508 | but for now, this support is only as a convenience. 509 | 510 | # Credit 511 | 512 | This project was largely based after the `nginx-alpha` example in the 513 | [kubernetes/contrib](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx-alpha) repository. 514 | -------------------------------------------------------------------------------- /RELEASE_NOTES.md: -------------------------------------------------------------------------------- 1 | # k8s-router Releases 2 | 3 | ## 1.0.8 (2016-10-28) 4 | 5 | * Updated to allow for configuring the maximum client request size _(Issue #51)_ 6 | 7 | ## 1.0.7 (2016-10-26) 8 | 9 | * Enhanced `Pod` routability validation to check the routing path container against the list of container ports _(Issue #12)_ 10 | * Fix logging of unroutable pods _(Issue #46)_ 11 | * Improved internal caching, only cache needed data for pods _(Issue #44)_ 12 | * Remove environment variable resolution of Kubernetes. Now uses kubectl config _(Issue #45)_ 13 | 14 | ## 1.0.5 (2016-09-02) 15 | 16 | * Fixed bug where error check didn't happen in the right spot 17 | 18 | ## 1.0.4 (2016-08-29) 19 | 20 | * Fixed bug where we assumed that a running `Pod` meant the `Pod` had an IP _(Issue #38)_ 21 | 22 | ## 1.0.3 (2016-08-29) 23 | 24 | * Updated the initial startup to always succeed even if the `Pod` state would generate an invalid `/etc/nginx/nginx.conf` _(Issue #37)_ 25 | 26 | -------------------------------------------------------------------------------- /demo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:4-onbuild 2 | 3 | EXPOSE 3000 4 | -------------------------------------------------------------------------------- /demo/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: nodejs-k8s-env 5 | spec: 6 | replicas: 1 7 | template: 8 | metadata: 9 | labels: 10 | app: nodejs-k8s-env 11 | routable: "true" 12 | annotations: 13 | routingHosts: "test.k8s.local" 14 | routingPaths: "3000:/nodejs" 15 | spec: 16 | containers: 17 | - name: nodejs-k8s-env 18 | image: thirtyx/nodejs-k8s-env:latest 19 | env: 20 | - name: PORT 21 | value: "3000" 22 | ports: 23 | - containerPort: 3000 24 | env: 25 | - name: POD_NAME 26 | valueFrom: 27 | fieldRef: 28 | fieldPath: metadata.name 29 | - name: POD_NAMESPACE 30 | valueFrom: 31 | fieldRef: 32 | fieldPath: metadata.namespace 33 | - name: BASE_PATH 34 | value: /nodejs 35 | -------------------------------------------------------------------------------- /demo/index.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 'use strict' 18 | 19 | const http = require('http') 20 | const os = require('os') 21 | const socketIO = require('socket.io') 22 | const allIfcs = os.networkInterfaces() 23 | 24 | const basePath = process.env.BASE_PATH || '/nodejs' 25 | const getEnv = (req) => { 26 | var env = { 27 | env: process.env, 28 | ips: ifcs 29 | } 30 | 31 | if (typeof req !== 'undefined') { 32 | env.req = { 33 | headers: req.headers, 34 | method: req.method, 35 | url: req.url 36 | } 37 | } 38 | 39 | return env 40 | } 41 | const ifcs = {} 42 | const port = process.env.PORT || 3000 43 | const server = http.createServer((req, res) => { 44 | console.log(req) 45 | res.writeHead(200, { 46 | 'Content-Type': 'application/json' 47 | }) 48 | res.end(JSON.stringify(getEnv(req), null, 2)) 49 | }) 50 | const io = socketIO(server, {path: basePath + '/socket.io'}) 51 | 52 | // Generate the list of IPs 53 | Object.keys(allIfcs).forEach((name) => { 54 | allIfcs[name].forEach((ifc) => { 55 | if (ifc.family === 'IPv4') 56 | ifcs[name] = ifc.address 57 | }) 58 | }) 59 | 60 | server.listen(port, () => { 61 | console.log('Current Environment') 62 | console.log('-------------------') 63 | 64 | Object.keys(process.env).forEach((key) => { 65 | console.log(' %s: %s', key, process.env[key]) 66 | }) 67 | 68 | console.log(); 69 | console.log('Current IPs') 70 | console.log('-----------'); 71 | 72 | Object.keys(ifcs).forEach((name) => { 73 | console.log(' %s: %s', name, ifcs[name]); 74 | }) 75 | 76 | console.log('Server listening on port', port) 77 | }) 78 | 79 | io.on('connection', function (socket) { 80 | // Emit the environment back upon request 81 | socket.on('env', function () { 82 | socket.emit('env', getEnv()); 83 | }) 84 | }); 85 | -------------------------------------------------------------------------------- /demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nodejs-k8s-env", 3 | "version": "0.0.0", 4 | "description": "Simple application for getting the Kubernetes environment information.", 5 | "main": "index.js", 6 | "author": "Jeremy Whitlock ", 7 | "license": "Apache-2", 8 | "private": true, 9 | "scripts": { 10 | "start": "node ." 11 | }, 12 | "dependencies": { 13 | "socket.io": "^1.4.6" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/aws-cluster-deployment.yaml: -------------------------------------------------------------------------------- 1 | #### 2 | # Deployes the ingress controller as a daemon set, then creates the external ELB to serve it traffic 3 | ### 4 | apiVersion: extensions/v1beta1 5 | kind: DaemonSet 6 | metadata: 7 | name: k8s-routers 8 | labels: 9 | app: k8s-routers 10 | spec: 11 | template: 12 | metadata: 13 | labels: 14 | app: k8s-routers 15 | annotations: 16 | projectcalico.org/policy: "allow tcp from cidr 192.168.0.0/16; allow tcp from cidr 10.129.0.0/16" 17 | spec: 18 | containers: 19 | - image: registry-1.docker.io/thirtyx/k8s-router:latest 20 | imagePullPolicy: Always 21 | name: k8s-public-router 22 | ports: 23 | - containerPort: 80 24 | hostPort: 80 25 | env: 26 | - name: POD_NAME 27 | valueFrom: 28 | fieldRef: 29 | fieldPath: metadata.name 30 | - name: POD_NAMESPACE 31 | valueFrom: 32 | fieldRef: 33 | fieldPath: metadata.namespace 34 | # Use the configuration to use the public/private paradigm (public version) 35 | - name: API_KEY_SECRET_LOCATION 36 | value: routing:public-api-key 37 | - name: HOSTS_ANNOTATION 38 | value: publicHosts 39 | - name: PATHS_ANNOTATION 40 | value: publicPaths 41 | - image: registry-1.docker.io/thirtyx/k8s-router:latest 42 | imagePullPolicy: Always 43 | name: k8s-private-router 44 | ports: 45 | - containerPort: 80 46 | # If we were using services, we could avoid exposing this port and could use service mechanisms for 47 | # identifying the port. For now, we expose the host port and should lock it down to disallow external 48 | # access. 49 | hostPort: 81 50 | env: 51 | - name: POD_NAME 52 | valueFrom: 53 | fieldRef: 54 | fieldPath: metadata.name 55 | - name: POD_NAMESPACE 56 | valueFrom: 57 | fieldRef: 58 | fieldPath: metadata.namespace 59 | # Use the configuration to use the public/private paradigm (private version) 60 | - name: API_KEY_SECRET_LOCATION 61 | value: routing:private-api-key 62 | - name: HOSTS_ANNOTATION 63 | value: privateHosts 64 | - name: PATHS_ANNOTATION 65 | value: privatePaths 66 | # Since we cannot have two containers listening on the same port, use a different port for the private router 67 | - name: PORT 68 | value: "81" 69 | --- 70 | apiVersion: v1 71 | kind: Service 72 | metadata: 73 | name: k8s-public-router-lb 74 | spec: 75 | type: LoadBalancer 76 | selector: 77 | app: k8s-routers 78 | ports: 79 | - name: http 80 | protocol: TCP 81 | port: 80 82 | targetPort: 80 83 | -------------------------------------------------------------------------------- /examples/ingress-daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: k8s-router 5 | labels: 6 | app: k8s-router 7 | spec: 8 | template: 9 | metadata: 10 | labels: 11 | app: k8s-router 12 | spec: 13 | containers: 14 | - image: thirtyx/k8s-router:latest 15 | imagePullPolicy: Always 16 | name: k8s-router 17 | ports: 18 | - containerPort: 80 19 | hostPort: 80 20 | env: 21 | - name: POD_NAME 22 | valueFrom: 23 | fieldRef: 24 | fieldPath: metadata.name 25 | - name: POD_NAMESPACE 26 | valueFrom: 27 | fieldRef: 28 | fieldPath: metadata.namespace 29 | -------------------------------------------------------------------------------- /examples/multipurpose-router-daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: k8s-routers 5 | labels: 6 | app: k8s-routers 7 | spec: 8 | template: 9 | metadata: 10 | labels: 11 | app: k8s-routers 12 | spec: 13 | containers: 14 | - image: thirtyx/k8s-router:latest 15 | imagePullPolicy: Always 16 | name: k8s-router-public 17 | ports: 18 | - containerPort: 80 19 | hostPort: 80 20 | env: 21 | - name: POD_NAME 22 | valueFrom: 23 | fieldRef: 24 | fieldPath: metadata.name 25 | - name: POD_NAMESPACE 26 | valueFrom: 27 | fieldRef: 28 | fieldPath: metadata.namespace 29 | # Use the configuration to use the public/private paradigm (public version) 30 | - name: API_KEY_SECRET_LOCATION 31 | value: routing:public-api-key 32 | - name: HOSTS_ANNOTATION 33 | value: publicHosts 34 | - name: PATHS_ANNOTATION 35 | value: publicPaths 36 | - image: thirtyx/k8s-router:latest 37 | imagePullPolicy: Always 38 | name: k8s-router-private 39 | ports: 40 | - containerPort: 81 41 | # We should probably avoid using host port and if needed, at least lock it down from external access 42 | hostPort: 81 43 | env: 44 | - name: POD_NAME 45 | valueFrom: 46 | fieldRef: 47 | fieldPath: metadata.name 48 | - name: POD_NAMESPACE 49 | valueFrom: 50 | fieldRef: 51 | fieldPath: metadata.namespace 52 | # Use the configuration to use the public/private paradigm (private version) 53 | - name: API_KEY_SECRET_LOCATION 54 | value: routing:private-api-key 55 | - name: HOSTS_ANNOTATION 56 | value: privateHosts 57 | - name: PATHS_ANNOTATION 58 | value: privatePaths 59 | # Since we cannot have two containers listening on the same port, use a different port for the private router 60 | - name: PORT 61 | value: "81" 62 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: 927780719238b01a74704d85ffc3bd53bd1e21b104c7701ae80b3e63567f4a0d 2 | updated: 2016-09-28T14:09:57.746768022-06:00 3 | imports: 4 | - name: github.com/beorn7/perks 5 | version: 3ac7bf7a47d159a033b107610db8a1b6575507a4 6 | subpackages: 7 | - quantile 8 | - name: github.com/blang/semver 9 | version: 31b736133b98f26d5e078ec9eb591666edfd091f 10 | - name: github.com/coreos/go-oidc 11 | version: 5cf2aa52da8c574d3aa4458f471ad6ae2240fe6b 12 | subpackages: 13 | - http 14 | - jose 15 | - key 16 | - oauth2 17 | - oidc 18 | - name: github.com/coreos/go-systemd 19 | version: 4484981625c1a6a2ecb40a390fcb6a9bcfee76e3 20 | subpackages: 21 | - activation 22 | - daemon 23 | - dbus 24 | - journal 25 | - unit 26 | - util 27 | - name: github.com/coreos/pkg 28 | version: 7f080b6c11ac2d2347c3cd7521e810207ea1a041 29 | subpackages: 30 | - capnslog 31 | - dlopen 32 | - health 33 | - httputil 34 | - timeutil 35 | - name: github.com/davecgh/go-spew 36 | version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d 37 | subpackages: 38 | - spew 39 | - name: github.com/docker/distribution 40 | version: cd27f179f2c10c5d300e6d09025b538c475b0d51 41 | subpackages: 42 | - digest 43 | - reference 44 | - name: github.com/docker/docker 45 | version: 0f5c9d301b9b1cca66b3ea0f9dec3b5317d3686d 46 | subpackages: 47 | - pkg/jsonmessage 48 | - pkg/mount 49 | - pkg/stdcopy 50 | - pkg/symlink 51 | - pkg/term 52 | - pkg/term/winconsole 53 | - pkg/timeutils 54 | - pkg/units 55 | - name: github.com/docker/go-units 56 | version: 0bbddae09c5a5419a8c6dcdd7ff90da3d450393b 57 | - name: github.com/emicklei/go-restful 58 | version: 7c47e2558a0bbbaba9ecab06bc6681e73028a28a 59 | subpackages: 60 | - log 61 | - swagger 62 | - name: github.com/ghodss/yaml 63 | version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee 64 | - name: github.com/gogo/protobuf 65 | version: 82d16f734d6d871204a3feb1a73cb220cc92574c 66 | subpackages: 67 | - gogoproto 68 | - plugin/defaultcheck 69 | - plugin/description 70 | - plugin/embedcheck 71 | - plugin/enumstringer 72 | - plugin/equal 73 | - plugin/face 74 | - plugin/gostring 75 | - plugin/grpc 76 | - plugin/marshalto 77 | - plugin/oneofcheck 78 | - plugin/populate 79 | - plugin/size 80 | - plugin/stringer 81 | - plugin/testgen 82 | - plugin/union 83 | - plugin/unmarshal 84 | - proto 85 | - protoc-gen-gogo/descriptor 86 | - protoc-gen-gogo/generator 87 | - protoc-gen-gogo/plugin 88 | - sortkeys 89 | - vanity 90 | - vanity/command 91 | - name: github.com/golang/glog 92 | version: 44145f04b68cf362d9c4df2182967c2275eaefed 93 | - name: github.com/golang/protobuf 94 | version: b982704f8bb716bb608144408cff30e15fbde841 95 | subpackages: 96 | - proto 97 | - name: github.com/google/cadvisor 98 | version: 4dbefc9b671b81257973a33211fb12370c1a526e 99 | subpackages: 100 | - api 101 | - cache/memory 102 | - collector 103 | - container 104 | - container/common 105 | - container/docker 106 | - container/libcontainer 107 | - container/raw 108 | - container/rkt 109 | - container/systemd 110 | - devicemapper 111 | - events 112 | - fs 113 | - healthz 114 | - http 115 | - http/mux 116 | - info/v1 117 | - info/v1/test 118 | - info/v2 119 | - machine 120 | - manager 121 | - manager/watcher 122 | - manager/watcher/raw 123 | - manager/watcher/rkt 124 | - metrics 125 | - pages 126 | - pages/static 127 | - storage 128 | - summary 129 | - utils 130 | - utils/cloudinfo 131 | - utils/cpuload 132 | - utils/cpuload/netlink 133 | - utils/docker 134 | - utils/oomparser 135 | - utils/sysfs 136 | - utils/sysinfo 137 | - utils/tail 138 | - validate 139 | - version 140 | - name: github.com/google/gofuzz 141 | version: bbcb9da2d746f8bdbd6a936686a0a6067ada0ec5 142 | - name: github.com/jonboulle/clockwork 143 | version: 3f831b65b61282ba6bece21b91beea2edc4c887a 144 | - name: github.com/juju/ratelimit 145 | version: 77ed1c8a01217656d2080ad51981f6e99adaa177 146 | - name: github.com/matttproud/golang_protobuf_extensions 147 | version: fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a 148 | subpackages: 149 | - pbutil 150 | - name: github.com/opencontainers/runc 151 | version: 7ca2aa4873aea7cb4265b1726acb24b90d8726c6 152 | subpackages: 153 | - libcontainer 154 | - libcontainer/apparmor 155 | - libcontainer/cgroups 156 | - libcontainer/cgroups/fs 157 | - libcontainer/cgroups/systemd 158 | - libcontainer/configs 159 | - libcontainer/configs/validate 160 | - libcontainer/criurpc 161 | - libcontainer/label 162 | - libcontainer/seccomp 163 | - libcontainer/selinux 164 | - libcontainer/stacktrace 165 | - libcontainer/system 166 | - libcontainer/user 167 | - libcontainer/utils 168 | - name: github.com/pborman/uuid 169 | version: ca53cad383cad2479bbba7f7a1a05797ec1386e4 170 | - name: github.com/prometheus/client_golang 171 | version: 3b78d7a77f51ccbc364d4bc170920153022cfd08 172 | subpackages: 173 | - prometheus 174 | - name: github.com/prometheus/client_model 175 | version: fa8ad6fec33561be4280a8f0514318c79d7f6cb6 176 | subpackages: 177 | - go 178 | - name: github.com/prometheus/common 179 | version: a6ab08426bb262e2d190097751f5cfd1cfdfd17d 180 | subpackages: 181 | - expfmt 182 | - internal/bitbucket.org/ww/goautoneg 183 | - model 184 | - name: github.com/prometheus/procfs 185 | version: 490cc6eb5fa45bf8a8b7b73c8bc82a8160e8531d 186 | - name: github.com/spf13/pflag 187 | version: 08b1a584251b5b62f458943640fc8ebd4d50aaa5 188 | - name: github.com/ugorji/go 189 | version: f4485b318aadd133842532f841dc205a8e339d74 190 | subpackages: 191 | - codec 192 | - codec/codecgen 193 | - name: golang.org/x/net 194 | version: 62685c2d7ca23c807425dca88b11a3e2323dab41 195 | subpackages: 196 | - context 197 | - context/ctxhttp 198 | - html 199 | - html/atom 200 | - http2 201 | - http2/hpack 202 | - internal/timeseries 203 | - proxy 204 | - trace 205 | - websocket 206 | - name: golang.org/x/oauth2 207 | version: b5adcc2dcdf009d0391547edc6ecbaff889f5bb9 208 | subpackages: 209 | - google 210 | - internal 211 | - jws 212 | - jwt 213 | - name: google.golang.org/appengine 214 | version: 12d5545dc1cfa6047a286d5e853841b6471f4c19 215 | subpackages: 216 | - internal 217 | - internal/app_identity 218 | - internal/base 219 | - internal/datastore 220 | - internal/log 221 | - internal/modules 222 | - internal/remote_api 223 | - internal/urlfetch 224 | - urlfetch 225 | - name: google.golang.org/cloud 226 | version: eb47ba841d53d93506cfbfbc03927daf9cc48f88 227 | subpackages: 228 | - compute/metadata 229 | - internal 230 | - name: gopkg.in/inf.v0 231 | version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 232 | - name: gopkg.in/yaml.v2 233 | version: a83829b6f1293c91addabc89d0571c246397bbf4 234 | - name: k8s.io/kubernetes 235 | version: 283137936a498aed572ee22af6774b6fb6e9fd94 236 | subpackages: 237 | - pkg/api 238 | - pkg/api/endpoints 239 | - pkg/api/errors 240 | - pkg/api/install 241 | - pkg/api/meta 242 | - pkg/api/meta/metatypes 243 | - pkg/api/pod 244 | - pkg/api/resource 245 | - pkg/api/service 246 | - pkg/api/unversioned 247 | - pkg/api/unversioned/validation 248 | - pkg/api/util 249 | - pkg/api/v1 250 | - pkg/api/validation 251 | - pkg/apimachinery 252 | - pkg/apimachinery/registered 253 | - pkg/apis/apps 254 | - pkg/apis/apps/install 255 | - pkg/apis/apps/v1alpha1 256 | - pkg/apis/authentication.k8s.io 257 | - pkg/apis/authentication.k8s.io/install 258 | - pkg/apis/authentication.k8s.io/v1beta1 259 | - pkg/apis/authorization 260 | - pkg/apis/authorization/install 261 | - pkg/apis/authorization/v1beta1 262 | - pkg/apis/autoscaling 263 | - pkg/apis/autoscaling/install 264 | - pkg/apis/autoscaling/v1 265 | - pkg/apis/batch 266 | - pkg/apis/batch/install 267 | - pkg/apis/batch/v1 268 | - pkg/apis/batch/v2alpha1 269 | - pkg/apis/componentconfig 270 | - pkg/apis/componentconfig/install 271 | - pkg/apis/componentconfig/v1alpha1 272 | - pkg/apis/extensions 273 | - pkg/apis/extensions/install 274 | - pkg/apis/extensions/v1beta1 275 | - pkg/apis/policy 276 | - pkg/apis/policy/install 277 | - pkg/apis/policy/v1alpha1 278 | - pkg/apis/rbac 279 | - pkg/apis/rbac/install 280 | - pkg/apis/rbac/v1alpha1 281 | - pkg/auth/user 282 | - pkg/capabilities 283 | - pkg/client/metrics 284 | - pkg/client/restclient 285 | - pkg/client/transport 286 | - pkg/client/typed/discovery 287 | - pkg/client/unversioned 288 | - pkg/client/unversioned/clientcmd/api 289 | - pkg/conversion 290 | - pkg/conversion/queryparams 291 | - pkg/fields 292 | - pkg/kubelet/qos 293 | - pkg/kubelet/qos/util 294 | - pkg/labels 295 | - pkg/master/ports 296 | - pkg/runtime 297 | - pkg/runtime/serializer 298 | - pkg/runtime/serializer/json 299 | - pkg/runtime/serializer/protobuf 300 | - pkg/runtime/serializer/recognizer 301 | - pkg/runtime/serializer/streaming 302 | - pkg/runtime/serializer/versioning 303 | - pkg/types 304 | - pkg/util 305 | - pkg/util/crypto 306 | - pkg/util/errors 307 | - pkg/util/flowcontrol 308 | - pkg/util/framer 309 | - pkg/util/hash 310 | - pkg/util/integer 311 | - pkg/util/intstr 312 | - pkg/util/json 313 | - pkg/util/net 314 | - pkg/util/net/sets 315 | - pkg/util/parsers 316 | - pkg/util/rand 317 | - pkg/util/runtime 318 | - pkg/util/sets 319 | - pkg/util/validation 320 | - pkg/util/validation/field 321 | - pkg/util/wait 322 | - pkg/util/yaml 323 | - pkg/version 324 | - pkg/watch 325 | - pkg/watch/versioned 326 | - plugin/pkg/client/auth 327 | - plugin/pkg/client/auth/gcp 328 | - plugin/pkg/client/auth/oidc 329 | - third_party/forked/reflect 330 | testImports: [] 331 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/30x/k8s-router 2 | import: 3 | - package: k8s.io/kubernetes 4 | version: 1.3.0 5 | -------------------------------------------------------------------------------- /kubernetes/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubernetes 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | "k8s.io/kubernetes/pkg/client/restclient" 24 | "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" 25 | client "k8s.io/kubernetes/pkg/client/unversioned" 26 | ) 27 | 28 | // Check if running in cluster 29 | func RunningInCluster() (bool) { 30 | if _, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount/token"); err == nil { 31 | return true; 32 | } else { 33 | return false; 34 | } 35 | } 36 | 37 | /* 38 | GetClient returns a Kubernetes client. 39 | */ 40 | func GetClient() (*client.Client, error) { 41 | var kubeConfig restclient.Config 42 | 43 | // Set the Kubernetes configuration based on the environment 44 | if RunningInCluster() { 45 | config, err := restclient.InClusterConfig() 46 | 47 | if err != nil { 48 | return nil, fmt.Errorf("Failed to create in-cluster config: %v.", err) 49 | } 50 | 51 | kubeConfig = *config 52 | } else { 53 | loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() 54 | configOverrides := &clientcmd.ConfigOverrides{} 55 | config := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) 56 | tmpKubeConfig, err := config.ClientConfig() 57 | if err != nil { 58 | return nil, fmt.Errorf("Failed to load local kube config: %v", err) 59 | } 60 | kubeConfig = *tmpKubeConfig; 61 | } 62 | 63 | 64 | // Create the Kubernetes client based on the configuration 65 | return client.New(&kubeConfig) 66 | } 67 | -------------------------------------------------------------------------------- /kubernetes/client_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubernetes 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | const ( 24 | ErrUnexpected = "Unexpected error: %v." 25 | ) 26 | 27 | /* 28 | Test for github.com/30x/k8s-router/kubernetes/client#GetClient 29 | */ 30 | func TestGetClient(t *testing.T) { 31 | // Test will need proper kube config, dosen't need to be reachable 32 | client, err := GetClient() 33 | 34 | if err != nil { 35 | t.Fatalf(ErrUnexpected, err) 36 | } else if client == nil { 37 | t.Fatal("Client should not be nil") 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "log" 21 | "time" 22 | 23 | "github.com/30x/k8s-router/kubernetes" 24 | "github.com/30x/k8s-router/nginx" 25 | "github.com/30x/k8s-router/router" 26 | 27 | "k8s.io/kubernetes/pkg/api" 28 | client "k8s.io/kubernetes/pkg/client/unversioned" 29 | "k8s.io/kubernetes/pkg/watch" 30 | ) 31 | 32 | func initController(config *router.Config, kubeClient *client.Client) (*router.Cache, watch.Interface, watch.Interface) { 33 | log.Println("Searching for routable pods") 34 | 35 | // Query the initial list of Pods 36 | pods, err := router.GetRoutablePodList(config, kubeClient) 37 | 38 | if err != nil { 39 | log.Fatalf("Failed to query the initial list of pods: %v.", err) 40 | } 41 | 42 | log.Printf(" Pods found: %d", len(pods.Items)) 43 | 44 | // Create a cache to keep track of the router "API Keys" and Pods (with routes) 45 | cache := &router.Cache{ 46 | Pods: make(map[string]*router.PodWithRoutes), 47 | Secrets: make(map[string][]byte), 48 | } 49 | 50 | // Turn the pods into a map based on the pod's name 51 | for i, pod := range pods.Items { 52 | cache.Pods[pod.Name] = router.ConvertPodToModel(config, &(pods.Items[i])) 53 | } 54 | 55 | // Query the initial list of Secrets 56 | secrets, err := router.GetRouterSecretList(config, kubeClient) 57 | 58 | if err != nil { 59 | log.Fatalf("Failed to query the initial list of secrets: %v", err) 60 | } 61 | 62 | // Turn the secrets into a map based on the secret's namespace 63 | for i, secret := range secrets.Items { 64 | cache.Secrets[secret.Namespace] = router.ConvertSecretToModel(config, &(secrets.Items[i])) 65 | } 66 | 67 | log.Printf(" Secrets found: %d", len(secrets.Items)) 68 | 69 | // Generate the nginx configuration and restart nginx 70 | nginx.RestartServer(nginx.GetConf(config, cache), false) 71 | 72 | // Get the list options so we can create the watch 73 | podWatchOptions := api.ListOptions{ 74 | LabelSelector: config.RoutableLabelSelector, 75 | ResourceVersion: pods.ListMeta.ResourceVersion, 76 | } 77 | 78 | // Create a watcher to be notified of Pod events 79 | podWatcher, err := kubeClient.Pods(api.NamespaceAll).Watch(podWatchOptions) 80 | 81 | if err != nil { 82 | log.Fatalf("Failed to create pod watcher: %v.", err) 83 | } 84 | 85 | // Get the list options so we can create the watch 86 | secretWatchOptions := api.ListOptions{ 87 | ResourceVersion: pods.ListMeta.ResourceVersion, 88 | } 89 | 90 | // Create a watcher to be notified of Pod events 91 | secretWatcher, err := kubeClient.Secrets(api.NamespaceAll).Watch(secretWatchOptions) 92 | 93 | if err != nil { 94 | log.Fatalf("Failed to create secret watcher: %v.", err) 95 | } 96 | 97 | return cache, podWatcher, secretWatcher 98 | } 99 | 100 | /* 101 | Simple Go application that provides routing for host+path combinations to Kubernetes pods. For more details on how to 102 | configure this, please review the design document located here: 103 | 104 | https://github.com/30x/k8s-router#design 105 | 106 | This application is written to run inside the Kubernetes cluster but can be run outside the Kubernetes cluster if a 107 | proper kube config is detected. (This can be useful for inspecting the routing table of an external Kubernetes 108 | cluster.) 109 | */ 110 | func main() { 111 | log.Println("Starting the Kubernetes Router") 112 | 113 | // Get the configuration 114 | config, err := router.ConfigFromEnv() 115 | 116 | if err != nil { 117 | log.Fatalf("Invalid configuration: %v.", err) 118 | } 119 | 120 | // Print the configuration 121 | log.Println(" Using configuration:") 122 | log.Printf(" API Key Header Name: %s\n", config.APIKeyHeader) 123 | log.Printf(" API Key Secret Name: %s\n", config.APIKeySecret) 124 | log.Printf(" API Key Secret Data Field: %s\n", config.APIKeySecretDataField) 125 | log.Printf(" Hosts Annotation: %s\n", config.HostsAnnotation) 126 | log.Printf(" Max client request size (0 indicates there is no maximum): %s\n", config.ClientMaxBodySize) 127 | log.Printf(" Paths Annotation: %s\n", config.PathsAnnotation) 128 | log.Printf(" Port (nginx): %d\n", config.Port) 129 | log.Printf(" Routable Label Selector: %s\n", config.RoutableLabelSelector) 130 | log.Println("") 131 | 132 | // Create the Kubernetes Client 133 | kubeClient, err := kubernetes.GetClient() 134 | 135 | if err != nil { 136 | log.Fatalf("Failed to create client: %v.", err) 137 | } 138 | 139 | // Don't write nginx conf when not in cluster 140 | nginx.RunInMockMode = !(kubernetes.RunningInCluster()) 141 | 142 | // Start nginx with the default configuration to start nginx as a daemon 143 | nginx.StartServer(nginx.GetDefaultConf(config)) 144 | 145 | // Create the initial cache and watcher 146 | cache, podWatcher, secretWatcher := initController(config, kubeClient) 147 | 148 | // Loop forever 149 | for { 150 | var podEvents []watch.Event 151 | var secretEvents []watch.Event 152 | 153 | // Get a 2 seconds window worth of events 154 | for { 155 | doRestart := false 156 | doStop := false 157 | 158 | select { 159 | case event, ok := <-podWatcher.ResultChan(): 160 | if !ok { 161 | log.Println("Kubernetes closed the pod watcher, restarting") 162 | 163 | doRestart = true 164 | } else { 165 | podEvents = append(podEvents, event) 166 | } 167 | 168 | case event, ok := <-secretWatcher.ResultChan(): 169 | if !ok { 170 | log.Println("Kubernetes closed the secret watcher, restarting") 171 | 172 | doRestart = true 173 | } else { 174 | secret := event.Object.(*api.Secret) 175 | 176 | // Only record secret events for secrets with the name we are interested in 177 | if secret.Name == config.APIKeySecret { 178 | secretEvents = append(secretEvents, event) 179 | } 180 | } 181 | 182 | // TODO: Rewrite to start the two seconds after the first post-restart event is seen 183 | case <-time.After(2 * time.Second): 184 | doStop = true 185 | } 186 | 187 | if doStop { 188 | break 189 | } else if doRestart { 190 | podWatcher.Stop() 191 | secretWatcher.Stop() 192 | 193 | cache, podWatcher, secretWatcher = initController(config, kubeClient) 194 | } 195 | } 196 | 197 | needsRestart := false 198 | 199 | if len(podEvents) > 0 { 200 | log.Printf("%d pod events found", len(podEvents)) 201 | 202 | // Update the cache based on the events and check if the server needs to be restarted 203 | needsRestart = router.UpdatePodCacheForEvents(config, cache.Pods, podEvents) 204 | } 205 | 206 | if !needsRestart && len(secretEvents) > 0 { 207 | log.Printf("%d secret events found", len(secretEvents)) 208 | 209 | // Update the cache based on the events and check if the server needs to be restarted 210 | needsRestart = router.UpdateSecretCacheForEvents(config, cache.Secrets, secretEvents) 211 | } 212 | 213 | // Wrapped in an if/else to limit logging 214 | if len(podEvents) > 0 || len(secretEvents) > 0 { 215 | if needsRestart { 216 | log.Println(" Requires nginx restart: yes") 217 | 218 | // Restart nginx 219 | nginx.RestartServer(nginx.GetConf(config, cache), false) 220 | } else { 221 | log.Println(" Requires nginx restart: no") 222 | } 223 | } 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /nginx/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package nginx 18 | 19 | import ( 20 | "bytes" 21 | "encoding/base64" 22 | "fmt" 23 | "hash/fnv" 24 | "log" 25 | "regexp" 26 | "sort" 27 | "strings" 28 | "text/template" 29 | 30 | "github.com/30x/k8s-router/router" 31 | ) 32 | 33 | const ( 34 | defaultNginxConfTmpl = ` 35 | # A very simple nginx configuration file that forces nginx to start as a daemon. 36 | events {} 37 | http {` + defaultNginxServerConfTmpl + `} 38 | daemon on; 39 | ` 40 | defaultNginxServerConfTmpl = ` 41 | # Default server that will just close the connection as if there was no server available 42 | server { 43 | listen {{.Port}} default_server; 44 | return 444; 45 | } 46 | ` 47 | defaultNginxLocationTmpl = ` 48 | # Here to avoid returning the nginx welcome page for servers that do not have a "/" location. (Issue #35) 49 | location / { 50 | return 404; 51 | } 52 | ` 53 | httpConfPreambleTmpl = ` 54 | # http://nginx.org/en/docs/http/ngx_http_core_module.html 55 | types_hash_max_size 2048; 56 | server_names_hash_max_size 512; 57 | server_names_hash_bucket_size 64; 58 | 59 | # Maximum body size in request 60 | client_max_body_size {{.Config.ClientMaxBodySize}}; 61 | 62 | # Force HTTP 1.1 for upstream requests 63 | proxy_http_version 1.1; 64 | 65 | # When nginx proxies to an upstream, the default value used for 'Connection' is 'close'. We use this variable to do 66 | # the same thing so that whenever a 'Connection' header is in the request, the variable reflects the provided value 67 | # otherwise, it defaults to 'close'. This is opposed to just using "proxy_set_header Connection $http_connection" 68 | # which would remove the 'Connection' header from the upstream request whenever the request does not contain a 69 | # 'Connection' header, which is a deviation from the nginx norm. 70 | map $http_connection $p_connection { 71 | default $http_connection; 72 | '' close; 73 | } 74 | 75 | # Pass through the appropriate headers 76 | proxy_set_header Connection $p_connection; 77 | proxy_set_header Host $http_host; 78 | proxy_set_header Upgrade $http_upgrade; 79 | ` 80 | nginxConfTmpl = ` 81 | events { 82 | worker_connections 1024; 83 | } 84 | http {` + httpConfPreambleTmpl + `{{range $key, $upstream := .Upstreams}} 85 | # Upstream for {{$upstream.Path}} traffic on {{$upstream.Host}} 86 | upstream {{$upstream.Name}} { 87 | {{range $server := $upstream.Servers}} # Pod {{$server.Pod.Name}} (namespace: {{$server.Pod.Namespace}}) 88 | server {{$server.Target}}; 89 | {{end}} } 90 | {{end}}{{range $host, $server := .Hosts}} 91 | server { 92 | listen {{$.Port}}; 93 | server_name {{$host}}; 94 | {{if $server.NeedsDefaultLocation}}` + defaultNginxLocationTmpl + `{{end}}{{range $path, $location := $server.Locations}} 95 | location {{$path}} { 96 | {{if ne $location.Secret ""}}# Check the Routing API Key (namespace: {{$location.Namespace}}) 97 | if ($http_{{$.APIKeyHeader}} != "{{$location.Secret}}") { 98 | return 403; 99 | } 100 | 101 | {{end}}{{if $location.Server.IsUpstream}}# Upstream {{$location.Server.Target}}{{else}}# Pod {{$location.Server.Pod.Name}} (namespace: {{$location.Server.Pod.Namespace}}){{end}} 102 | proxy_pass http://{{$location.Server.Target}}; 103 | } 104 | {{end}} } 105 | {{end}}` + defaultNginxServerConfTmpl + `} 106 | ` 107 | // NginxConfPath is The nginx configuration file path 108 | NginxConfPath = "/etc/nginx/nginx.conf" 109 | ) 110 | 111 | // Cannot declare as a constant 112 | var defaultNginxConf string 113 | var defaultNginxConfTemplate *template.Template 114 | var nginxAPIKeyHeader string 115 | var nginxConfTemplate *template.Template 116 | 117 | type hostT struct { 118 | Locations map[string]*locationT 119 | NeedsDefaultLocation bool 120 | } 121 | 122 | type locationT struct { 123 | Namespace string 124 | Path string 125 | Secret string 126 | Server *serverT 127 | } 128 | 129 | type serverT struct { 130 | IsUpstream bool 131 | Pod *router.PodWithRoutes 132 | Target string 133 | } 134 | 135 | type serversT []*serverT 136 | 137 | type templateDataT struct { 138 | APIKeyHeader string 139 | Hosts map[string]*hostT 140 | Port int 141 | Upstreams map[string]*upstreamT 142 | Config *router.Config 143 | } 144 | 145 | type upstreamT struct { 146 | Host string 147 | Name string 148 | Path string 149 | Servers serversT 150 | } 151 | 152 | func (slice serversT) Len() int { 153 | return len(slice) 154 | } 155 | 156 | func (slice serversT) Less(i, j int) bool { 157 | return slice[i].Pod.Name < slice[j].Pod.Name 158 | } 159 | 160 | func (slice serversT) Swap(i, j int) { 161 | slice[i], slice[j] = slice[j], slice[i] 162 | } 163 | 164 | func hash(s string) uint32 { 165 | h := fnv.New32a() 166 | h.Write([]byte(s)) 167 | return h.Sum32() 168 | } 169 | 170 | func convertAPIKeyHeaderForNginx(config *router.Config) { 171 | if nginxAPIKeyHeader == "" { 172 | // Convert the API Key header to nginx 173 | nginxAPIKeyHeader = strings.ToLower(regexp.MustCompile("[^A-Za-z0-9]").ReplaceAllString(config.APIKeyHeader, "_")) 174 | } 175 | } 176 | 177 | func init() { 178 | // Parse the default nginx.conf template 179 | t, err := template.New("nginx-default").Parse(defaultNginxConfTmpl) 180 | 181 | if err != nil { 182 | log.Fatalf("Failed to render default nginx.conf template: %v.", err) 183 | } 184 | 185 | defaultNginxConfTemplate = t 186 | 187 | // Parse the nginx.conf template 188 | t2, err := template.New("nginx").Parse(nginxConfTmpl) 189 | 190 | if err != nil { 191 | log.Fatalf("Failed to render nginx.conf template: %v.", err) 192 | } 193 | 194 | nginxConfTemplate = t2 195 | } 196 | 197 | /* 198 | GetConf takes the router cache and returns a generated nginx configuration 199 | */ 200 | func GetConf(config *router.Config, cache *router.Cache) string { 201 | // Quick out if there are no pods in the cache 202 | if len(cache.Pods) == 0 { 203 | return GetDefaultConf(config) 204 | } 205 | 206 | // Make sure we've converted the API Key to nginx format 207 | convertAPIKeyHeaderForNginx(config) 208 | 209 | tmplData := templateDataT{ 210 | APIKeyHeader: nginxAPIKeyHeader, 211 | Hosts: make(map[string]*hostT), 212 | Port: config.Port, 213 | Upstreams: make(map[string]*upstreamT), 214 | Config: config, 215 | } 216 | 217 | // Process the pods to populate the nginx configuration data structure 218 | for _, cacheEntry := range cache.Pods { 219 | // Process each pod route 220 | for _, route := range cacheEntry.Routes { 221 | host, ok := tmplData.Hosts[route.Incoming.Host] 222 | 223 | if !ok { 224 | tmplData.Hosts[route.Incoming.Host] = &hostT{ 225 | Locations: make(map[string]*locationT), 226 | NeedsDefaultLocation: true, 227 | } 228 | host = tmplData.Hosts[route.Incoming.Host] 229 | } 230 | 231 | var locationSecret string 232 | namespace := cacheEntry.Namespace 233 | secret, ok := cache.Secrets[namespace] 234 | 235 | if ok { 236 | // There is guaranteed to be an API Key so no need to double check 237 | locationSecret = base64.StdEncoding.EncodeToString(secret) 238 | } 239 | 240 | location, ok := host.Locations[route.Incoming.Path] 241 | upstreamKey := route.Incoming.Host + route.Incoming.Path 242 | upstreamHash := fmt.Sprint(hash(upstreamKey)) 243 | upstreamName := "upstream" + upstreamHash 244 | target := route.Outgoing.IP 245 | 246 | if route.Outgoing.Port != "80" && route.Outgoing.Port != "443" { 247 | target += ":" + route.Outgoing.Port 248 | } 249 | 250 | // Unset the need for a default location if necessary 251 | if host.NeedsDefaultLocation && route.Incoming.Path == "/" { 252 | host.NeedsDefaultLocation = false 253 | } 254 | 255 | if ok { 256 | // If the current target is different than the new one, create/update the upstream accordingly 257 | if location.Server.Target != target { 258 | if upstream, ok := tmplData.Upstreams[upstreamKey]; ok { 259 | ok = true 260 | 261 | // Check to see if there is a server with the corresponding target 262 | for _, server := range upstream.Servers { 263 | if server.Target == target { 264 | ok = false 265 | break 266 | } 267 | } 268 | 269 | // If there is no server for this target, create one 270 | if ok { 271 | upstream.Servers = append(upstream.Servers, &serverT{ 272 | Pod: cacheEntry, 273 | Target: target, 274 | }) 275 | 276 | // Sort to make finding your pods in an upstream easier 277 | sort.Sort(upstream.Servers) 278 | } 279 | } else { 280 | // Create the new upstream 281 | tmplData.Upstreams[upstreamKey] = &upstreamT{ 282 | Name: upstreamName, 283 | Host: route.Incoming.Host, 284 | Path: route.Incoming.Path, 285 | Servers: []*serverT{ 286 | location.Server, 287 | &serverT{ 288 | Pod: cacheEntry, 289 | Target: target, 290 | }, 291 | }, 292 | } 293 | } 294 | 295 | // Update the location server 296 | location.Server = &serverT{ 297 | IsUpstream: true, 298 | Target: upstreamName, 299 | } 300 | } 301 | } else { 302 | host.Locations[route.Incoming.Path] = &locationT{ 303 | Namespace: namespace, 304 | Path: route.Incoming.Path, 305 | Secret: locationSecret, 306 | Server: &serverT{ 307 | Pod: cacheEntry, 308 | Target: target, 309 | }, 310 | } 311 | } 312 | } 313 | } 314 | 315 | var doc bytes.Buffer 316 | 317 | // Useful for debugging 318 | if err := nginxConfTemplate.Execute(&doc, tmplData); err != nil { 319 | log.Fatalf("Failed to write template %v", err) 320 | } 321 | 322 | return doc.String() 323 | } 324 | 325 | /* 326 | GetDefaultConf returns the default nginx.conf 327 | */ 328 | func GetDefaultConf(config *router.Config) string { 329 | // Make sure we've converted the API Key to nginx format 330 | convertAPIKeyHeaderForNginx(config) 331 | 332 | if defaultNginxConf == "" { 333 | var doc bytes.Buffer 334 | 335 | if err := defaultNginxConfTemplate.Execute(&doc, config); err != nil { 336 | log.Fatalf("Failed to write template %v", err) 337 | } else { 338 | defaultNginxConf = doc.String() 339 | } 340 | } 341 | 342 | return defaultNginxConf 343 | } 344 | -------------------------------------------------------------------------------- /nginx/config_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package nginx 18 | 19 | import ( 20 | "bytes" 21 | "encoding/base64" 22 | "log" 23 | "strings" 24 | "testing" 25 | "text/template" 26 | 27 | "github.com/30x/k8s-router/router" 28 | 29 | "k8s.io/kubernetes/pkg/api" 30 | ) 31 | 32 | var config *router.Config 33 | 34 | func init() { 35 | envConfig, err := router.ConfigFromEnv() 36 | 37 | if err != nil { 38 | log.Fatalf("Unable to get configuration from environment: %v", err) 39 | } 40 | 41 | config = envConfig 42 | } 43 | 44 | func getDefaultServerConf(config *router.Config) string { 45 | var doc bytes.Buffer 46 | 47 | // Parse the default nginx server block template 48 | t, err := template.New("nginx-default-server").Parse(defaultNginxServerConfTmpl) 49 | 50 | if err != nil { 51 | log.Fatalf("Failed to render nginx.conf default server block template: %v.", err) 52 | } 53 | 54 | if err := t.Execute(&doc, config); err != nil { 55 | log.Fatalf("Failed to write template %v", err) 56 | 57 | return "" 58 | } 59 | 60 | return doc.String() 61 | } 62 | 63 | func getConfPreamble(config *router.Config) string { 64 | var doc bytes.Buffer 65 | 66 | // Parse the default nginx server block template 67 | t, err := template.New("nginx-http-preamble").Parse(httpConfPreambleTmpl) 68 | 69 | if err != nil { 70 | log.Fatalf("Failed to render nginx.conf http preamble template: %v.", err) 71 | } 72 | 73 | data := struct { 74 | Config *router.Config 75 | }{ 76 | config, 77 | } 78 | 79 | if err := t.Execute(&doc, data); err != nil { 80 | log.Fatalf("Failed to write template %v", err) 81 | 82 | return "" 83 | } 84 | 85 | return doc.String() 86 | } 87 | 88 | func resetConf() { 89 | // Reset the cached default server (At runtime, we cache the results because they will never change) 90 | defaultNginxConf = "" 91 | // Change the config port 92 | config.Port = 80 93 | // Reset the cached API Key header (At runtime, we cache the results because they will never change) 94 | nginxAPIKeyHeader = "" 95 | } 96 | 97 | func validateConf(t *testing.T, desc, expected string, pods []*api.Pod, secrets []*api.Secret) { 98 | cache := &router.Cache{ 99 | Pods: make(map[string]*router.PodWithRoutes), 100 | Secrets: make(map[string][]byte), 101 | } 102 | 103 | for _, pod := range pods { 104 | cache.Pods[pod.Name] = router.ConvertPodToModel(config, pod) 105 | } 106 | 107 | for _, secret := range secrets { 108 | cache.Secrets[secret.Namespace] = router.ConvertSecretToModel(config, secret) 109 | } 110 | 111 | actual := GetConf(config, cache) 112 | 113 | if expected != actual { 114 | t.Fatalf("Unexpected nginx.conf was generated (%s)\nExpected: %s\n\nActual: %s\n", desc, expected, actual) 115 | } 116 | } 117 | 118 | /* 119 | Test for github.com/30x/k8s-router/nginx/config#GetConf with an empty cache 120 | */ 121 | func TestGetConfNoRoutablePods(t *testing.T) { 122 | conf := GetConf(config, &router.Cache{}) 123 | 124 | if conf != ` 125 | # A very simple nginx configuration file that forces nginx to start as a daemon. 126 | events {} 127 | http { 128 | # Default server that will just close the connection as if there was no server available 129 | server { 130 | listen 80 default_server; 131 | return 444; 132 | } 133 | } 134 | daemon on; 135 | ` { 136 | t.Fatal("The default nginx.conf should be returned for an empty cache") 137 | } 138 | } 139 | 140 | /* 141 | Test for github.com/30x/k8s-router/nginx/config#GetConf with an empty cache and a custom port 142 | */ 143 | func TestGetConfNoRoutablePodsCustomPort(t *testing.T) { 144 | resetConf() 145 | 146 | // Change the config port 147 | config.Port = 90 148 | 149 | conf := GetConf(config, &router.Cache{}) 150 | 151 | if conf != ` 152 | # A very simple nginx configuration file that forces nginx to start as a daemon. 153 | events {} 154 | http { 155 | # Default server that will just close the connection as if there was no server available 156 | server { 157 | listen 90 default_server; 158 | return 444; 159 | } 160 | } 161 | daemon on; 162 | ` { 163 | t.Fatal("The default nginx.conf should be returned for an empty cache and a custom port") 164 | } 165 | 166 | resetConf() 167 | } 168 | 169 | /* 170 | Test for github.com/30x/k8s-router/nginx/config#GetConf with single pod and multiple paths 171 | */ 172 | func TestGetConfMultiplePaths(t *testing.T) { 173 | expectedConf := ` 174 | events { 175 | worker_connections 1024; 176 | } 177 | http {` + getConfPreamble(config) + ` 178 | server { 179 | listen 80; 180 | server_name test.github.com; 181 | ` + defaultNginxLocationTmpl + ` 182 | location /prod { 183 | # Pod testing (namespace: testing) 184 | proxy_pass http://10.244.1.16; 185 | } 186 | 187 | location /test { 188 | # Pod testing (namespace: testing) 189 | proxy_pass http://10.244.1.16:3000; 190 | } 191 | } 192 | ` + getDefaultServerConf(config) + `} 193 | ` 194 | 195 | pod := api.Pod{ 196 | ObjectMeta: api.ObjectMeta{ 197 | Annotations: map[string]string{ 198 | "routingHosts": "test.github.com", 199 | "routingPaths": "80:/prod 3000:/test", 200 | }, 201 | Name: "testing", 202 | Namespace: "testing", 203 | }, 204 | Spec: api.PodSpec{ 205 | Containers: []api.Container{ 206 | api.Container{ 207 | Ports: []api.ContainerPort{ 208 | api.ContainerPort{ 209 | ContainerPort: int32(80), 210 | }, 211 | api.ContainerPort{ 212 | ContainerPort: int32(3000), 213 | }, 214 | }, 215 | }, 216 | }, 217 | }, 218 | Status: api.PodStatus{ 219 | Phase: api.PodRunning, 220 | PodIP: "10.244.1.16", 221 | }, 222 | } 223 | 224 | validateConf(t, "single pod multiple paths", expectedConf, []*api.Pod{&pod}, []*api.Secret{}) 225 | } 226 | 227 | /* 228 | Test for github.com/30x/k8s-router/nginx/config#GetConf with single pod, multiple paths and a custom port 229 | */ 230 | func TestGetConfMultiplePathsCustomPort(t *testing.T) { 231 | resetConf() 232 | 233 | // Change the config port 234 | config.Port = 90 235 | 236 | expectedConf := ` 237 | events { 238 | worker_connections 1024; 239 | } 240 | http {` + getConfPreamble(config) + ` 241 | server { 242 | listen 90; 243 | server_name test.github.com; 244 | ` + defaultNginxLocationTmpl + ` 245 | location /prod { 246 | # Pod testing (namespace: testing) 247 | proxy_pass http://10.244.1.16; 248 | } 249 | 250 | location /test { 251 | # Pod testing (namespace: testing) 252 | proxy_pass http://10.244.1.16:3000; 253 | } 254 | } 255 | ` + getDefaultServerConf(config) + `} 256 | ` 257 | 258 | pod := api.Pod{ 259 | ObjectMeta: api.ObjectMeta{ 260 | Annotations: map[string]string{ 261 | "routingHosts": "test.github.com", 262 | "routingPaths": "80:/prod 3000:/test", 263 | }, 264 | Name: "testing", 265 | Namespace: "testing", 266 | }, 267 | Spec: api.PodSpec{ 268 | Containers: []api.Container{ 269 | api.Container{ 270 | Ports: []api.ContainerPort{ 271 | api.ContainerPort{ 272 | ContainerPort: int32(80), 273 | }, 274 | api.ContainerPort{ 275 | ContainerPort: int32(3000), 276 | }, 277 | }, 278 | }, 279 | }, 280 | }, 281 | Status: api.PodStatus{ 282 | Phase: api.PodRunning, 283 | PodIP: "10.244.1.16", 284 | }, 285 | } 286 | 287 | validateConf(t, "single pod multiple paths", expectedConf, []*api.Pod{&pod}, []*api.Secret{}) 288 | 289 | resetConf() 290 | } 291 | 292 | /* 293 | Test for github.com/30x/k8s-router/nginx/config#GetConf with multiple, single pod services 294 | */ 295 | func TestGetConfMultipleRoutableServices(t *testing.T) { 296 | expectedConf := ` 297 | events { 298 | worker_connections 1024; 299 | } 300 | http {` + getConfPreamble(config) + ` 301 | server { 302 | listen 80; 303 | server_name prod.github.com; 304 | 305 | location / { 306 | # Pod testing2 (namespace: testing) 307 | proxy_pass http://10.244.1.17; 308 | } 309 | } 310 | 311 | server { 312 | listen 80; 313 | server_name test.github.com; 314 | ` + defaultNginxLocationTmpl + ` 315 | location /nodejs { 316 | # Pod testing (namespace: testing) 317 | proxy_pass http://10.244.1.16:3000; 318 | } 319 | } 320 | ` + getDefaultServerConf(config) + `} 321 | ` 322 | 323 | pods := []*api.Pod{ 324 | &api.Pod{ 325 | ObjectMeta: api.ObjectMeta{ 326 | Annotations: map[string]string{ 327 | "routingHosts": "test.github.com", 328 | "routingPaths": "3000:/nodejs", 329 | }, 330 | Name: "testing", 331 | Namespace: "testing", 332 | }, 333 | Spec: api.PodSpec{ 334 | Containers: []api.Container{ 335 | api.Container{ 336 | Ports: []api.ContainerPort{ 337 | api.ContainerPort{ 338 | ContainerPort: int32(3000), 339 | }, 340 | }, 341 | }, 342 | }, 343 | }, 344 | Status: api.PodStatus{ 345 | Phase: api.PodRunning, 346 | PodIP: "10.244.1.16", 347 | }, 348 | }, 349 | &api.Pod{ 350 | ObjectMeta: api.ObjectMeta{ 351 | Annotations: map[string]string{ 352 | "routingHosts": "prod.github.com", 353 | "routingPaths": "80:/", 354 | }, 355 | Name: "testing2", 356 | Namespace: "testing", 357 | }, 358 | Spec: api.PodSpec{ 359 | Containers: []api.Container{ 360 | api.Container{ 361 | Ports: []api.ContainerPort{ 362 | api.ContainerPort{ 363 | ContainerPort: int32(80), 364 | }, 365 | }, 366 | }, 367 | }, 368 | }, 369 | Status: api.PodStatus{ 370 | Phase: api.PodRunning, 371 | PodIP: "10.244.1.17", 372 | }, 373 | }, 374 | } 375 | 376 | validateConf(t, "multiple pods, different services", expectedConf, pods, []*api.Secret{}) 377 | } 378 | 379 | /* 380 | Test for github.com/30x/k8s-router/nginx/config#GetConf with single, multiple pod services 381 | */ 382 | func TestGetConfMultiplePodRoutableServices(t *testing.T) { 383 | expectedConf := ` 384 | events { 385 | worker_connections 1024; 386 | } 387 | http {` + getConfPreamble(config) + ` 388 | # Upstream for / traffic on test.github.com 389 | upstream upstream619897598 { 390 | # Pod testing (namespace: testing) 391 | server 10.244.1.16; 392 | # Pod testing2 (namespace: testing) 393 | server 10.244.1.17; 394 | # Pod testing3 (namespace: testing) 395 | server 10.244.1.18:3000; 396 | } 397 | 398 | server { 399 | listen 80; 400 | server_name test.github.com; 401 | 402 | location / { 403 | # Upstream upstream619897598 404 | proxy_pass http://upstream619897598; 405 | } 406 | } 407 | ` + getDefaultServerConf(config) + `} 408 | ` 409 | 410 | pods := []*api.Pod{ 411 | &api.Pod{ 412 | ObjectMeta: api.ObjectMeta{ 413 | Annotations: map[string]string{ 414 | "routingHosts": "test.github.com", 415 | "routingPaths": "80:/", 416 | }, 417 | Name: "testing", 418 | Namespace: "testing", 419 | }, 420 | Spec: api.PodSpec{ 421 | Containers: []api.Container{ 422 | api.Container{ 423 | Ports: []api.ContainerPort{ 424 | api.ContainerPort{ 425 | ContainerPort: int32(80), 426 | }, 427 | }, 428 | }, 429 | }, 430 | }, 431 | Status: api.PodStatus{ 432 | Phase: api.PodRunning, 433 | PodIP: "10.244.1.16", 434 | }, 435 | }, 436 | &api.Pod{ 437 | ObjectMeta: api.ObjectMeta{ 438 | Annotations: map[string]string{ 439 | "routingHosts": "test.github.com", 440 | "routingPaths": "80:/", 441 | }, 442 | Name: "testing2", 443 | Namespace: "testing", 444 | }, 445 | Spec: api.PodSpec{ 446 | Containers: []api.Container{ 447 | api.Container{ 448 | Ports: []api.ContainerPort{ 449 | api.ContainerPort{ 450 | ContainerPort: int32(80), 451 | }, 452 | }, 453 | }, 454 | }, 455 | }, 456 | Status: api.PodStatus{ 457 | Phase: api.PodRunning, 458 | PodIP: "10.244.1.17", 459 | }, 460 | }, 461 | &api.Pod{ 462 | ObjectMeta: api.ObjectMeta{ 463 | Annotations: map[string]string{ 464 | "routingHosts": "test.github.com", 465 | "routingPaths": "3000:/", 466 | }, 467 | Name: "testing3", 468 | Namespace: "testing", 469 | }, 470 | Spec: api.PodSpec{ 471 | Containers: []api.Container{ 472 | api.Container{ 473 | Ports: []api.ContainerPort{ 474 | api.ContainerPort{ 475 | ContainerPort: int32(3000), 476 | }, 477 | }, 478 | }, 479 | }, 480 | }, 481 | Status: api.PodStatus{ 482 | Phase: api.PodRunning, 483 | PodIP: "10.244.1.18", 484 | }, 485 | }, 486 | } 487 | 488 | validateConf(t, "multiple pods, same service", expectedConf, pods, []*api.Secret{}) 489 | } 490 | 491 | /* 492 | Test for github.com/30x/k8s-router/nginx/config#GetConf with API Key 493 | */ 494 | func TestGetConfWithAPIKey(t *testing.T) { 495 | apiKey := []byte("Updated-API-Key") 496 | expectedConf := ` 497 | events { 498 | worker_connections 1024; 499 | } 500 | http {` + getConfPreamble(config) + ` 501 | server { 502 | listen 80; 503 | server_name test.github.com; 504 | 505 | location / { 506 | # Check the Routing API Key (namespace: testing) 507 | if ($http_x_routing_api_key != "` + base64.StdEncoding.EncodeToString(apiKey) + `") { 508 | return 403; 509 | } 510 | 511 | # Pod testing (namespace: testing) 512 | proxy_pass http://10.244.1.16; 513 | } 514 | } 515 | ` + getDefaultServerConf(config) + `} 516 | ` 517 | 518 | pod := api.Pod{ 519 | ObjectMeta: api.ObjectMeta{ 520 | Annotations: map[string]string{ 521 | "routingHosts": "test.github.com", 522 | "routingPaths": "80:/", 523 | }, 524 | Name: "testing", 525 | Namespace: "testing", 526 | }, 527 | Spec: api.PodSpec{ 528 | Containers: []api.Container{ 529 | api.Container{ 530 | Ports: []api.ContainerPort{ 531 | api.ContainerPort{ 532 | ContainerPort: int32(80), 533 | }, 534 | }, 535 | }, 536 | }, 537 | }, 538 | Status: api.PodStatus{ 539 | Phase: api.PodRunning, 540 | PodIP: "10.244.1.16", 541 | }, 542 | } 543 | secret := api.Secret{ 544 | ObjectMeta: api.ObjectMeta{ 545 | Name: config.APIKeySecret, 546 | Namespace: "testing", 547 | }, 548 | Data: map[string][]byte{ 549 | "api-key": apiKey, 550 | }, 551 | } 552 | 553 | validateConf(t, "pod with API Key", expectedConf, []*api.Pod{&pod}, []*api.Secret{&secret}) 554 | } 555 | 556 | /* 557 | Test for github.com/30x/k8s-router/nginx/config#GetConf with custom API Key header 558 | */ 559 | func TestGetConfWithCustomAPIKeyHeader(t *testing.T) { 560 | resetConf() 561 | 562 | // Change the API Key Header 563 | config.APIKeyHeader = "X-SOMETHING-CUSTOM_API*KEY" 564 | 565 | apiKey := []byte("Updated-API-Key") 566 | expectedConf := ` 567 | events { 568 | worker_connections 1024; 569 | } 570 | http {` + getConfPreamble(config) + ` 571 | server { 572 | listen 80; 573 | server_name test.github.com; 574 | 575 | location / { 576 | # Check the Routing API Key (namespace: testing) 577 | if ($http_x_something_custom_api_key != "` + base64.StdEncoding.EncodeToString(apiKey) + `") { 578 | return 403; 579 | } 580 | 581 | # Pod testing (namespace: testing) 582 | proxy_pass http://10.244.1.16; 583 | } 584 | } 585 | ` + getDefaultServerConf(config) + `} 586 | ` 587 | 588 | pod := api.Pod{ 589 | ObjectMeta: api.ObjectMeta{ 590 | Annotations: map[string]string{ 591 | "routingHosts": "test.github.com", 592 | "routingPaths": "80:/", 593 | }, 594 | Name: "testing", 595 | Namespace: "testing", 596 | }, 597 | Spec: api.PodSpec{ 598 | Containers: []api.Container{ 599 | api.Container{ 600 | Ports: []api.ContainerPort{ 601 | api.ContainerPort{ 602 | ContainerPort: int32(80), 603 | }, 604 | }, 605 | }, 606 | }, 607 | }, 608 | Status: api.PodStatus{ 609 | Phase: api.PodRunning, 610 | PodIP: "10.244.1.16", 611 | }, 612 | } 613 | secret := api.Secret{ 614 | ObjectMeta: api.ObjectMeta{ 615 | Name: config.APIKeySecret, 616 | Namespace: "testing", 617 | }, 618 | Data: map[string][]byte{ 619 | "api-key": apiKey, 620 | }, 621 | } 622 | 623 | validateConf(t, "pod with API Key", expectedConf, []*api.Pod{&pod}, []*api.Secret{&secret}) 624 | 625 | resetConf() 626 | } 627 | 628 | /* 629 | Test for ClientMaxBodySize config variable in Nginx Template 630 | */ 631 | func TestClientMaxBodySize(t *testing.T) { 632 | config.ClientMaxBodySize = "1234m" 633 | doc := getConfPreamble(config) 634 | idx := strings.Index(doc, "client_max_body_size 1234m;") 635 | if (idx < 0) { 636 | log.Fatalf("Failed to include client_max_body_size from config.") 637 | } 638 | } 639 | -------------------------------------------------------------------------------- /nginx/server.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package nginx 18 | 19 | import ( 20 | "fmt" 21 | "io" 22 | "log" 23 | "os" 24 | "os/exec" 25 | ) 26 | 27 | // If running locally enabled mock mode to not call sh commands or write config 28 | var RunInMockMode bool 29 | 30 | func shellOut(cmd string, exitOnFailure bool) { 31 | if RunInMockMode { 32 | return 33 | } 34 | 35 | out, err := exec.Command("sh", "-c", cmd).CombinedOutput() 36 | 37 | if err != nil { 38 | msg := fmt.Sprintf("Failed to execute (%v): %v, err: %v", cmd, string(out), err) 39 | 40 | if exitOnFailure { 41 | log.Fatal(msg) 42 | } else { 43 | log.Println(msg) 44 | } 45 | } 46 | } 47 | 48 | func writeNginxConf(conf string) { 49 | log.Println(conf) 50 | 51 | if RunInMockMode { 52 | return; 53 | } 54 | 55 | // Create the nginx.conf file based on the template 56 | if w, err := os.Create(NginxConfPath); err != nil { 57 | log.Fatalf("Failed to open %s: %v", NginxConfPath, err) 58 | } else if _, err := io.WriteString(w, conf); err != nil { 59 | log.Fatalf("Failed to write template %v", err) 60 | } 61 | 62 | log.Printf("Wrote nginx configuration to %s\n", NginxConfPath) 63 | } 64 | 65 | /* 66 | RestartServer restarts nginx using the provided configuration. 67 | */ 68 | func RestartServer(conf string, exitOnFailure bool) { 69 | log.Println("Reloading nginx with the following configuration:") 70 | 71 | writeNginxConf(conf) 72 | 73 | log.Println("Restarting nginx") 74 | 75 | shellOut("nginx -s reload", exitOnFailure) 76 | } 77 | 78 | /* 79 | StartServer starts nginx using the provided configuration. 80 | */ 81 | func StartServer(conf string) { 82 | log.Println("Starting nginx with the following configuration:") 83 | 84 | writeNginxConf(conf) 85 | 86 | log.Println("Starting nginx") 87 | 88 | shellOut("nginx", true) 89 | } 90 | -------------------------------------------------------------------------------- /router/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package router 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "strconv" 23 | "strings" 24 | 25 | "github.com/30x/k8s-router/utils" 26 | 27 | "k8s.io/kubernetes/pkg/labels" 28 | "k8s.io/kubernetes/pkg/util/validation" 29 | ) 30 | 31 | const ( 32 | // DefaultAPIKeyHeader is the default value for the header used to identify the API Key (X-ROUTING-API-KEY) 33 | DefaultAPIKeyHeader = "X-ROUTING-API-KEY" 34 | // DefaultAPIKeySecret is the default value for the first portion of the DefaultAPIKeySecretLocation (routing) 35 | DefaultAPIKeySecret = "routing" 36 | // DefaultAPIKeySecretDataField is the default value for the second portion of the DefaultAPIKeySecretDataField (api-key) 37 | DefaultAPIKeySecretDataField = "api-key" 38 | // DefaultAPIKeySecretLocation is the default value for the EnvVarAPIKeySecretLocation (routing:api-key) 39 | DefaultAPIKeySecretLocation = DefaultAPIKeySecret + ":" + DefaultAPIKeySecretDataField 40 | // DefaultClientMaxBodySize for nginx max client request size. Default 100mb 41 | DefaultClientMaxBodySize = "0" 42 | // DefaultHostsAnnotation is the default value for EnvVarHostsAnnotation (routingHosts) 43 | DefaultHostsAnnotation = "routingHosts" 44 | // DefaultPathsAnnotation is the default value for the EnvVarHostsAnnotation (routingPaths) 45 | DefaultPathsAnnotation = "routingPaths" 46 | // DefaultPort is the default value for the EnvVarPort (80) 47 | DefaultPort = 80 48 | // DefaultRoutableLabelSelector is the default value for EnvVarRoutableLabelSelector (routable=true) 49 | DefaultRoutableLabelSelector = "routable=true" 50 | // EnvVarAPIKeyHeader Environment variable name for providing the header name used to identify the API Key header 51 | EnvVarAPIKeyHeader = "API_KEY_HEADER" 52 | // EnvVarAPIKeySecretLocation Environment variable name for providing the location of the secret (name:field) to identify API Key secrets 53 | EnvVarAPIKeySecretLocation = "API_KEY_SECRET_LOCATION" 54 | // EnvVarHostsAnnotation Environment variable name for providing the name of the hosts annotation 55 | EnvVarHostsAnnotation = "HOSTS_ANNOTATION" 56 | // EnvVarPathsAnnotation Environment variable name for providing the the name of the paths annotation 57 | EnvVarPathsAnnotation = "PATHS_ANNOTATION" 58 | // EnvVarPort Environment variable for providing the port nginx should listen on 59 | EnvVarPort = "PORT" 60 | // EnvClientMaxBodySize Environment variable for max client request body size 61 | EnvClientMaxBodySize = "CLIENT_MAX_BODY_SIZE" 62 | // EnvVarRoutableLabelSelector Environment variable name for providing the label selector for identifying routable objects 63 | EnvVarRoutableLabelSelector = "ROUTABLE_LABEL_SELECTOR" 64 | // ErrMsgTmplInvalidAnnotationName is the error message template for an invalid annotation name 65 | ErrMsgTmplInvalidAnnotationName = "%s has an invalid annotation name: %s" 66 | // ErrMsgTmplInvalidAPIKeySecretLocation is the error message template for invalid API Key Secret location environment variable values 67 | ErrMsgTmplInvalidAPIKeySecretLocation = "%s is not in the format of {API_KEY_SECRET_NAME}:{API_KEY_SECRET_DATA_FIELD_NAME}" 68 | // ErrMsgTmplInvalidLabelSelector is the error message template for an invalid label selector 69 | ErrMsgTmplInvalidLabelSelector = "%s has an invalid label selector: %s\n" 70 | // ErrMsgTmplInvalidPort is the error message template for an invalid port 71 | ErrMsgTmplInvalidPort = "%s is an invalid port: %s\n" 72 | ) 73 | 74 | /* 75 | ConfigFromEnv returns the configuration based on the environment variables and validates the values 76 | */ 77 | func ConfigFromEnv() (*Config, error) { 78 | config := &Config{ 79 | APIKeyHeader: os.Getenv(EnvVarAPIKeyHeader), 80 | HostsAnnotation: os.Getenv(EnvVarHostsAnnotation), 81 | PathsAnnotation: os.Getenv(EnvVarPathsAnnotation), 82 | ClientMaxBodySize: os.Getenv(EnvClientMaxBodySize), 83 | } 84 | 85 | // Apply defaults 86 | if config.APIKeyHeader == "" { 87 | config.APIKeyHeader = DefaultAPIKeyHeader 88 | } 89 | 90 | if config.HostsAnnotation == "" { 91 | config.HostsAnnotation = DefaultHostsAnnotation 92 | } 93 | 94 | if config.PathsAnnotation == "" { 95 | config.PathsAnnotation = DefaultPathsAnnotation 96 | } 97 | 98 | if config.ClientMaxBodySize == "" { 99 | config.ClientMaxBodySize = DefaultClientMaxBodySize 100 | } 101 | 102 | // Validate configuration 103 | apiKeySecretLocation := os.Getenv(EnvVarAPIKeySecretLocation) 104 | var apiKeySecretLocationParts []string 105 | 106 | if apiKeySecretLocation == "" { 107 | // No need to validate, just use the default 108 | config.APIKeySecret = DefaultAPIKeySecret 109 | config.APIKeySecretDataField = DefaultAPIKeySecretDataField 110 | } else { 111 | apiKeySecretLocationParts = strings.Split(apiKeySecretLocation, ":") 112 | 113 | if len(apiKeySecretLocationParts) == 2 { 114 | config.APIKeySecret = apiKeySecretLocationParts[0] 115 | config.APIKeySecretDataField = apiKeySecretLocationParts[1] 116 | } else { 117 | return nil, fmt.Errorf(ErrMsgTmplInvalidAPIKeySecretLocation, EnvVarAPIKeySecretLocation) 118 | } 119 | } 120 | 121 | hostErrs := validation.IsQualifiedName(strings.ToLower(config.HostsAnnotation)) 122 | pathErrs := validation.IsQualifiedName(strings.ToLower(config.PathsAnnotation)) 123 | 124 | if len(hostErrs) > 0 { 125 | return nil, fmt.Errorf(ErrMsgTmplInvalidAnnotationName, EnvVarHostsAnnotation, config.HostsAnnotation) 126 | } else if len(pathErrs) > 0 { 127 | return nil, fmt.Errorf(ErrMsgTmplInvalidAnnotationName, EnvVarPathsAnnotation, config.PathsAnnotation) 128 | } 129 | 130 | portStr := os.Getenv(EnvVarPort) 131 | 132 | if portStr == "" { 133 | config.Port = DefaultPort 134 | } else { 135 | port, err := strconv.Atoi(portStr) 136 | 137 | if err != nil || !utils.IsValidPort(port) { 138 | return nil, fmt.Errorf(ErrMsgTmplInvalidPort, EnvVarPort, portStr) 139 | } 140 | 141 | config.Port = port 142 | } 143 | 144 | routableLabelSelector := os.Getenv(EnvVarRoutableLabelSelector) 145 | 146 | if routableLabelSelector == "" { 147 | routableLabelSelector = DefaultRoutableLabelSelector 148 | } 149 | 150 | selector, err := labels.Parse(routableLabelSelector) 151 | 152 | if err == nil { 153 | config.RoutableLabelSelector = selector 154 | } else { 155 | return nil, fmt.Errorf(ErrMsgTmplInvalidLabelSelector, EnvVarRoutableLabelSelector, routableLabelSelector) 156 | } 157 | 158 | return config, nil 159 | } 160 | -------------------------------------------------------------------------------- /router/config_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package router 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "strconv" 23 | "testing" 24 | 25 | "k8s.io/kubernetes/pkg/labels" 26 | ) 27 | 28 | func getConfig(t *testing.T) *Config { 29 | config, err := ConfigFromEnv() 30 | 31 | if err != nil { 32 | t.Fatalf("Problem retrieving configuration") 33 | } 34 | 35 | return config 36 | } 37 | 38 | func getLabelSelector(t *testing.T, labelSelector string) labels.Selector { 39 | selector, err := labels.Parse(labelSelector) 40 | 41 | if err != nil { 42 | t.Fatalf("Unable to parse the label selector (%s): %v\n", labelSelector, err) 43 | } 44 | 45 | return selector 46 | } 47 | 48 | func resetEnv(t *testing.T) { 49 | unsetEnv := func(name string) { 50 | err := os.Unsetenv(name) 51 | 52 | if err != nil { 53 | t.Fatalf("Unable to unset environment variable (%s): %v\n", name, err) 54 | } 55 | } 56 | 57 | unsetEnv(EnvVarAPIKeySecretLocation) 58 | unsetEnv(EnvVarHostsAnnotation) 59 | unsetEnv(EnvVarPathsAnnotation) 60 | unsetEnv(EnvVarPort) 61 | unsetEnv(EnvVarRoutableLabelSelector) 62 | } 63 | 64 | func setEnv(t *testing.T, key, value string) { 65 | err := os.Setenv(key, value) 66 | 67 | if err != nil { 68 | t.Fatalf("Unable to set environment variable (%s = %s): %v\n", key, value, err) 69 | } 70 | } 71 | 72 | func validateConfig(t *testing.T, desc string, expected *Config, actual *Config) { 73 | makeError := func(field, eValue, aValue string) string { 74 | return fmt.Sprintf("Expected %s (%s) does not match actual %s (%s): %s\n", field, eValue, field, aValue, desc) 75 | } 76 | 77 | if expected.APIKeySecret != actual.APIKeySecret { 78 | t.Fatalf(makeError("APIKeySecret", expected.APIKeySecret, actual.APIKeySecret)) 79 | } else if expected.APIKeySecretDataField != actual.APIKeySecretDataField { 80 | t.Fatalf(makeError("APIKeySecretDataField", expected.APIKeySecretDataField, actual.APIKeySecretDataField)) 81 | } else if expected.HostsAnnotation != actual.HostsAnnotation { 82 | t.Fatalf(makeError("HostsAnnotation", expected.HostsAnnotation, actual.HostsAnnotation)) 83 | } else if expected.PathsAnnotation != actual.PathsAnnotation { 84 | t.Fatalf(makeError("PathsAnnotation", expected.PathsAnnotation, actual.PathsAnnotation)) 85 | } else if expected.Port != actual.Port { 86 | t.Fatalf(makeError("Port", strconv.Itoa(expected.Port), strconv.Itoa(actual.Port))) 87 | } else if expected.RoutableLabelSelector.String() != actual.RoutableLabelSelector.String() { 88 | t.Fatalf(makeError("RoutableLabelSelector", expected.RoutableLabelSelector.String(), actual.RoutableLabelSelector.String())) 89 | } 90 | } 91 | 92 | /* 93 | Test for github.com/30x/k8s-router/router/config#ConfigFromEnv using the default environment 94 | */ 95 | func TestConfigFromEnvDefaultConfig(t *testing.T) { 96 | validateConfig(t, "default configuration", getConfig(t), &Config{ 97 | APIKeySecret: DefaultAPIKeySecret, 98 | APIKeySecretDataField: DefaultAPIKeySecretDataField, 99 | HostsAnnotation: DefaultHostsAnnotation, 100 | PathsAnnotation: DefaultPathsAnnotation, 101 | Port: DefaultPort, 102 | RoutableLabelSelector: getLabelSelector(t, DefaultRoutableLabelSelector), 103 | }) 104 | } 105 | 106 | /* 107 | Test for github.com/30x/k8s-router/router/config#ConfigFromEnv using invalid configurations 108 | */ 109 | func TestConfigFromEnvInvalidEnv(t *testing.T) { 110 | validateInvalidConfig := func(errMsg string) { 111 | config, err := ConfigFromEnv() 112 | 113 | if config != nil { 114 | t.Fatal("Config should be nil") 115 | } else if errMsg != err.Error() { 116 | t.Fatalf("Expected error message (%s) but found: %s\n", errMsg, err.Error()) 117 | } 118 | 119 | resetEnv(t) 120 | } 121 | 122 | // Reset the environment variables just in case 123 | resetEnv(t) 124 | 125 | // Invalid API Key Secret location 126 | setEnv(t, EnvVarAPIKeySecretLocation, "routing") 127 | 128 | validateInvalidConfig(fmt.Sprintf(ErrMsgTmplInvalidAPIKeySecretLocation, EnvVarAPIKeySecretLocation)) 129 | 130 | // Invalid hosts annotation 131 | invalidName := "*&^^%&%$$^&%&" 132 | 133 | setEnv(t, EnvVarHostsAnnotation, invalidName) 134 | 135 | validateInvalidConfig(fmt.Sprintf(ErrMsgTmplInvalidAnnotationName, EnvVarHostsAnnotation, invalidName)) 136 | 137 | // Invalid paths annotation 138 | setEnv(t, EnvVarPathsAnnotation, invalidName) 139 | 140 | validateInvalidConfig(fmt.Sprintf(ErrMsgTmplInvalidAnnotationName, EnvVarPathsAnnotation, invalidName)) 141 | 142 | // Invalid port (not a number) 143 | setEnv(t, EnvVarPort, invalidName) 144 | 145 | validateInvalidConfig(fmt.Sprintf(ErrMsgTmplInvalidPort, EnvVarPort, invalidName)) 146 | 147 | // Invalid port (not a valid port) 148 | invalidPort := "-1" 149 | 150 | setEnv(t, EnvVarPort, invalidPort) 151 | 152 | validateInvalidConfig(fmt.Sprintf(ErrMsgTmplInvalidPort, EnvVarPort, invalidPort)) 153 | 154 | // Invalid routable label selector 155 | setEnv(t, EnvVarRoutableLabelSelector, invalidName) 156 | 157 | validateInvalidConfig(fmt.Sprintf(ErrMsgTmplInvalidLabelSelector, EnvVarRoutableLabelSelector, invalidName)) 158 | } 159 | 160 | /* 161 | Test for github.com/30x/k8s-router/router/config#ConfigFromEnv using a valid environment 162 | */ 163 | func TestConfigFromEnvValidConfig(t *testing.T) { 164 | resetEnv(t) 165 | 166 | hostsAnnotation := "trafficHosts" 167 | pathsAnnotation := "publicPaths" 168 | port := "81" 169 | routableLabelSelector := "route-me=true" 170 | secretName := "custom" 171 | secretDataField := "another-custom" 172 | 173 | setEnv(t, EnvVarAPIKeySecretLocation, secretName+":"+secretDataField) 174 | setEnv(t, EnvVarHostsAnnotation, hostsAnnotation) 175 | setEnv(t, EnvVarPathsAnnotation, pathsAnnotation) 176 | setEnv(t, EnvVarPort, port) 177 | setEnv(t, EnvVarRoutableLabelSelector, routableLabelSelector) 178 | 179 | validateConfig(t, "default configuration", getConfig(t), &Config{ 180 | APIKeySecret: secretName, 181 | APIKeySecretDataField: secretDataField, 182 | HostsAnnotation: hostsAnnotation, 183 | PathsAnnotation: pathsAnnotation, 184 | Port: 81, 185 | RoutableLabelSelector: getLabelSelector(t, routableLabelSelector), 186 | }) 187 | } 188 | -------------------------------------------------------------------------------- /router/pods.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package router 18 | 19 | import ( 20 | "log" 21 | "strconv" 22 | "hash/fnv" 23 | "regexp" 24 | "strings" 25 | 26 | "github.com/30x/k8s-router/utils" 27 | 28 | "k8s.io/kubernetes/pkg/api" 29 | client "k8s.io/kubernetes/pkg/client/unversioned" 30 | "k8s.io/kubernetes/pkg/fields" 31 | "k8s.io/kubernetes/pkg/labels" 32 | "k8s.io/kubernetes/pkg/watch" 33 | ) 34 | 35 | const ( 36 | hostnameRegexStr = "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$" 37 | ipRegexStr = "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" 38 | pathSegmentRegexStr = "^[A-Za-z0-9\\-._~!$&'()*+,;=:@]|%[0-9A-Fa-f]{2}$" 39 | ) 40 | 41 | type pathPair struct { 42 | Path string 43 | Port string 44 | } 45 | 46 | /* 47 | String implements the Stringer interface 48 | */ 49 | func (r *Route) String() string { 50 | return r.Incoming.Host + r.Incoming.Path + " -> " + r.Outgoing.IP + ":" + r.Outgoing.Port 51 | } 52 | 53 | var hostnameRegex *regexp.Regexp 54 | var ipRegex *regexp.Regexp 55 | var pathSegmentRegex *regexp.Regexp 56 | 57 | func compileRegex(regexStr string) *regexp.Regexp { 58 | compiled, err := regexp.Compile(regexStr) 59 | 60 | if err != nil { 61 | log.Fatalf("Failed to compile regular expression (%s): %v\n", regexStr, err) 62 | } 63 | 64 | return compiled 65 | } 66 | 67 | func init() { 68 | // Compile all regular expressions 69 | hostnameRegex = compileRegex(hostnameRegexStr) 70 | ipRegex = compileRegex(ipRegexStr) 71 | pathSegmentRegex = compileRegex(pathSegmentRegexStr) 72 | } 73 | 74 | func isContainerPort(ports []int32, port int32) bool { 75 | for _, vPort := range ports { 76 | if vPort == port { 77 | return true 78 | } 79 | } 80 | return false 81 | } 82 | 83 | /* 84 | GetRoutablePodList returns the routable pods list. 85 | */ 86 | func GetRoutablePodList(config *Config, kubeClient *client.Client) (*api.PodList, error) { 87 | // Query the initial list of Pods 88 | podList, err := kubeClient.Pods(api.NamespaceAll).List(api.ListOptions{ 89 | FieldSelector: fields.Everything(), 90 | LabelSelector: config.RoutableLabelSelector, 91 | }) 92 | 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | return podList, nil 98 | } 99 | 100 | 101 | /* 102 | Calculate hash for hosts and paths annotations to compare when pod is modified 103 | */ 104 | func calculateAnnotationHash(config *Config, pod *api.Pod) (uint64) { 105 | h := fnv.New64() 106 | h.Write([]byte(pod.Annotations[config.HostsAnnotation])) 107 | h.Write([]byte(pod.Annotations[config.PathsAnnotation])) 108 | return h.Sum64() 109 | } 110 | 111 | /* 112 | Converts a Kubernetes pod model to our model 113 | */ 114 | func ConvertPodToModel(config *Config, pod *api.Pod) (*PodWithRoutes) { 115 | return &PodWithRoutes{ 116 | Name: pod.Name, 117 | Namespace: pod.Namespace, 118 | Status: pod.Status.Phase, 119 | AnnotationHash: calculateAnnotationHash(config, pod), 120 | Routes: GetRoutes(config, pod), 121 | } 122 | } 123 | 124 | /* 125 | GetRoutes returns an array of routes defined within the provided pod 126 | */ 127 | func GetRoutes(config *Config, pod *api.Pod) []*Route { 128 | var routes []*Route 129 | 130 | // Do not process pods that are not running 131 | if pod.Status.Phase == api.PodRunning { 132 | // Do not process pods without an IP 133 | if pod.Status.PodIP != "" { 134 | var hosts []string 135 | var pathPairs []*pathPair 136 | var ports []int32 137 | 138 | annotation, ok := pod.Annotations[config.HostsAnnotation] 139 | 140 | // This pod does not have the hosts annotation set 141 | if ok { 142 | // Process the routing hosts 143 | for _, host := range strings.Split(annotation, " ") { 144 | valid := hostnameRegex.MatchString(host) 145 | 146 | if !valid { 147 | valid = ipRegex.MatchString(host) 148 | 149 | if !valid { 150 | log.Printf(" Pod (%s) routing issue: %s (%s) is not a valid hostname/ip\n", pod.Name, config.HostsAnnotation, host) 151 | 152 | continue 153 | } 154 | } 155 | 156 | // Record the host 157 | hosts = append(hosts, host) 158 | } 159 | 160 | // Do not process the routing paths if there are no valid hosts 161 | if len(hosts) > 0 { 162 | annotation, ok = pod.Annotations[config.PathsAnnotation] 163 | 164 | // Create a list of valid routing ports 165 | for _, container := range pod.Spec.Containers { 166 | for _, port := range container.Ports { 167 | ports = append(ports, port.ContainerPort) 168 | } 169 | } 170 | 171 | if ok { 172 | for _, publicPath := range strings.Split(annotation, " ") { 173 | pathParts := strings.Split(publicPath, ":") 174 | 175 | if len(pathParts) == 2 { 176 | cPathPair := &pathPair{} 177 | 178 | // Validate the port 179 | port, err := strconv.Atoi(pathParts[0]) 180 | 181 | if err != nil || !utils.IsValidPort(port) { 182 | log.Printf(" Pod (%s) routing issue: %s port (%s) is not valid\n", pod.Name, config.PathsAnnotation, pathParts[0]) 183 | } else if !isContainerPort(ports, int32(port)) { 184 | log.Printf(" Pod (%s) routing issue: %s port (%s) is not an exposed container port\n", pod.Name, config.PathsAnnotation, pathParts[0]) 185 | } else { 186 | cPathPair.Port = pathParts[0] 187 | } 188 | 189 | // Validate the path (when necessary) 190 | if port > 0 { 191 | pathSegments := strings.Split(pathParts[1], "/") 192 | valid := true 193 | 194 | for i, pathSegment := range pathSegments { 195 | // Skip the first and last entry 196 | if (i == 0 || i == len(pathSegments)-1) && pathSegment == "" { 197 | continue 198 | } else if !pathSegmentRegex.MatchString(pathSegment) { 199 | log.Printf(" Pod (%s) routing issue: publicPath path (%s) is not valid\n", pod.Name, pathParts[1]) 200 | 201 | valid = false 202 | 203 | break 204 | } 205 | } 206 | 207 | if valid { 208 | cPathPair.Path = pathParts[1] 209 | } 210 | } 211 | 212 | if cPathPair.Path != "" && cPathPair.Port != "" { 213 | pathPairs = append(pathPairs, cPathPair) 214 | } 215 | } else { 216 | log.Printf(" Pod (%s) routing issue: publicPath (%s) is not a valid PORT:PATH combination\n", pod.Name, annotation) 217 | } 218 | } 219 | } else { 220 | log.Printf(" Pod (%s) is not routable: Missing '%s' annotation\n", pod.Name, config.PathsAnnotation) 221 | } 222 | } 223 | 224 | // Turn the hosts and path pairs into routes 225 | if hosts != nil && pathPairs != nil { 226 | for _, host := range hosts { 227 | for _, cPathPair := range pathPairs { 228 | routes = append(routes, &Route{ 229 | Incoming: &Incoming{ 230 | Host: host, 231 | Path: cPathPair.Path, 232 | }, 233 | Outgoing: &Outgoing{ 234 | IP: pod.Status.PodIP, 235 | Port: cPathPair.Port, 236 | }, 237 | }) 238 | } 239 | } 240 | } 241 | } else { 242 | log.Printf(" Pod (%s) is not routable: Missing '%s' annotation\n", pod.Name, config.HostsAnnotation) 243 | } 244 | } else { 245 | log.Printf(" Pod (%s) is not routable: Pod does not have an IP\n", pod.Name) 246 | } 247 | } else { 248 | log.Printf(" Pod (%s) is not routable: Not running (%s)\n", pod.Name, pod.Status.Phase) 249 | } 250 | 251 | return routes 252 | } 253 | 254 | /* 255 | UpdatePodCacheForEvents updates the cache based on the pod events and returns if the changes warrant an nginx restart. 256 | */ 257 | func UpdatePodCacheForEvents(config *Config, cache map[string]*PodWithRoutes, events []watch.Event) bool { 258 | needsRestart := false 259 | 260 | for _, event := range events { 261 | pod := event.Object.(*api.Pod) 262 | 263 | log.Printf(" Pod (%s) event: %s\n", pod.Name, event.Type) 264 | 265 | // Process the event 266 | switch event.Type { 267 | case watch.Added: 268 | // This event is likely never going to be handled in the real world because most pod add events happen prior to 269 | // pod being routable but it's here just in case. 270 | cache[pod.Name] = ConvertPodToModel(config, pod) 271 | 272 | needsRestart = len(cache[pod.Name].Routes) > 0 273 | 274 | case watch.Deleted: 275 | needsRestart = true 276 | delete(cache, pod.Name) 277 | 278 | case watch.Modified: 279 | podLabels := labels.Set(pod.Labels) 280 | 281 | // Check if the pod still has the routable label 282 | if config.RoutableLabelSelector.Matches(podLabels) { 283 | cached, ok := cache[pod.Name] 284 | 285 | // If anything routing related changes, trigger a server restart 286 | if !ok || calculateAnnotationHash(config, pod) != cached.AnnotationHash || pod.Status.Phase != cached.Status { 287 | needsRestart = true 288 | } 289 | 290 | // Add/Update the cache entry 291 | cache[pod.Name] = ConvertPodToModel(config, pod) 292 | } else { 293 | log.Println(" Pod is no longer routable") 294 | 295 | // Pod no longer matches the routable label selector so we need to remove it from the cache 296 | needsRestart = true 297 | delete(cache, pod.Name) 298 | } 299 | } 300 | 301 | cacheEntry, ok := cache[pod.Name] 302 | 303 | if ok { 304 | if len(cacheEntry.Routes) > 0 { 305 | log.Println(" Pod is routable") 306 | } else { 307 | log.Println(" Pod is not routable") 308 | } 309 | } 310 | } 311 | 312 | return needsRestart 313 | } 314 | -------------------------------------------------------------------------------- /router/pods_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package router 18 | 19 | import ( 20 | "io/ioutil" 21 | "log" 22 | "testing" 23 | 24 | "github.com/30x/k8s-router/kubernetes" 25 | 26 | "k8s.io/kubernetes/pkg/api" 27 | "k8s.io/kubernetes/pkg/labels" 28 | "k8s.io/kubernetes/pkg/watch" 29 | ) 30 | 31 | var config *Config 32 | 33 | func init() { 34 | envConfig, err := ConfigFromEnv() 35 | 36 | if err != nil { 37 | log.Fatalf("Unable to get configuration from environment: %v", err) 38 | } 39 | 40 | config = envConfig 41 | 42 | log.SetOutput(ioutil.Discard) 43 | } 44 | 45 | func validateRoutes(t *testing.T, desc string, expected, actual []*Route) { 46 | aCount := 0 47 | eCount := 0 48 | 49 | if actual != nil { 50 | aCount = len(actual) 51 | } 52 | 53 | if expected != nil { 54 | eCount = len(expected) 55 | } 56 | 57 | // First check that we have the proper number of routes 58 | if aCount != eCount { 59 | t.Fatalf("Expected %d routes but found %d routes: %s\n", eCount, aCount, desc) 60 | } 61 | 62 | // Validate each route positionally 63 | find := func(items []*Route, item *Route) *Route { 64 | var route *Route 65 | 66 | for _, cRoute := range items { 67 | if item.Incoming.Host == cRoute.Incoming.Host && 68 | item.Incoming.Path == cRoute.Incoming.Path && 69 | item.Outgoing.IP == cRoute.Outgoing.IP && 70 | item.Outgoing.Port == cRoute.Outgoing.Port { 71 | route = cRoute 72 | 73 | break 74 | } 75 | } 76 | 77 | return route 78 | } 79 | 80 | for _, route := range expected { 81 | if find(actual, route) == nil { 82 | t.Fatalf("Unable to find route (%s): %s\n", route, desc) 83 | } 84 | } 85 | } 86 | 87 | /* 88 | Test for github.com/30x/k8s-router/router/pods#GetRoutablePodList 89 | */ 90 | func TestGetRoutablePodList(t *testing.T) { 91 | kubeClient, err := kubernetes.GetClient() 92 | 93 | if err != nil { 94 | t.Fatalf("Failed to create k8s client: %v.", err) 95 | } 96 | 97 | podsList, err := GetRoutablePodList(config, kubeClient) 98 | 99 | if err != nil { 100 | t.Fatalf("Failed to get the routable pods: %v.", err) 101 | } 102 | 103 | for _, pod := range podsList.Items { 104 | podLabels := labels.Set(pod.Labels) 105 | 106 | // Check if the pod still has the routable label 107 | if !config.RoutableLabelSelector.Matches(podLabels) { 108 | t.Fatalf("Every pod should match the (%s) label selector", config.RoutableLabelSelector) 109 | } 110 | } 111 | } 112 | 113 | /* 114 | Test for github.com/30x/k8s-router/router/pods#GetRoutes where the pod is not running 115 | */ 116 | func TestGetRoutesNotRunning(t *testing.T) { 117 | validateRoutes(t, "pod not running", []*Route{}, GetRoutes(config, &api.Pod{ 118 | Status: api.PodStatus{ 119 | Phase: api.PodPending, 120 | }, 121 | })) 122 | } 123 | 124 | /* 125 | Test for github.com/30x/k8s-router/router/pods#GetRoutes where the pod is running but does not have an IP 126 | */ 127 | func TestGetRoutesRunningWithoutIP(t *testing.T) { 128 | validateRoutes(t, "pod does not have an IP", []*Route{}, GetRoutes(config, &api.Pod{ 129 | Status: api.PodStatus{ 130 | Phase: api.PodRunning, 131 | }, 132 | })) 133 | } 134 | 135 | /* 136 | Test for github.com/30x/k8s-router/router/pods#GetRoutes where the pod has no routingHosts annotation 137 | */ 138 | func TestGetRoutesNoTrafficHosts(t *testing.T) { 139 | validateRoutes(t, "pod has no routingHosts annotation", []*Route{}, GetRoutes(config, &api.Pod{ 140 | Status: api.PodStatus{ 141 | Phase: api.PodRunning, 142 | }, 143 | })) 144 | } 145 | 146 | /* 147 | Test for github.com/30x/k8s-router/router/pods#GetRoutes where the pod has an invalid routingHosts annotation 148 | */ 149 | func TestGetRoutesInvalidTrafficHosts(t *testing.T) { 150 | validateRoutes(t, "pod has an invalid routingHosts host", []*Route{}, GetRoutes(config, &api.Pod{ 151 | ObjectMeta: api.ObjectMeta{ 152 | Annotations: map[string]string{ 153 | "routingHosts": "test.github.com test.", 154 | }, 155 | }, 156 | Status: api.PodStatus{ 157 | Phase: api.PodRunning, 158 | }, 159 | })) 160 | } 161 | 162 | /* 163 | Test for github.com/30x/k8s-router/router/pods#GetRoutes where the pod has an invalid port value in the routingPaths annotation 164 | */ 165 | func TestGetRoutesInvalidPublicPathsPort(t *testing.T) { 166 | // Not a valid integer 167 | validateRoutes(t, "pod has an invalid routingPaths port (invalid integer)", []*Route{}, GetRoutes(config, &api.Pod{ 168 | ObjectMeta: api.ObjectMeta{ 169 | Annotations: map[string]string{ 170 | "routingHosts": "test.github.com", 171 | "routingPaths": "abcdef:/", 172 | }, 173 | }, 174 | Status: api.PodStatus{ 175 | Phase: api.PodRunning, 176 | }, 177 | })) 178 | 179 | // Port is less than 0 180 | validateRoutes(t, "pod has an invalid routingPaths port (port < 0)", []*Route{}, GetRoutes(config, &api.Pod{ 181 | ObjectMeta: api.ObjectMeta{ 182 | Annotations: map[string]string{ 183 | "routingHosts": "test.github.com", 184 | "routingPaths": "-1:/", 185 | }, 186 | }, 187 | Status: api.PodStatus{ 188 | Phase: api.PodRunning, 189 | }, 190 | })) 191 | 192 | // Port is greater than 65535 193 | validateRoutes(t, "pod has an invalid routingPaths port (port > 65536)", []*Route{}, GetRoutes(config, &api.Pod{ 194 | ObjectMeta: api.ObjectMeta{ 195 | Annotations: map[string]string{ 196 | "routingHosts": "test.github.com", 197 | "routingPaths": "77777:/", 198 | }, 199 | }, 200 | Status: api.PodStatus{ 201 | Phase: api.PodRunning, 202 | }, 203 | })) 204 | 205 | // Port is not an exposed container port 206 | validateRoutes(t, "pod has an invalid routingPaths port (port > 65536)", []*Route{}, GetRoutes(config, &api.Pod{ 207 | ObjectMeta: api.ObjectMeta{ 208 | Annotations: map[string]string{ 209 | "routingHosts": "test.github.com", 210 | "routingPaths": "81:/", 211 | }, 212 | }, 213 | Spec: api.PodSpec{ 214 | Containers: []api.Container{ 215 | api.Container{ 216 | Ports: []api.ContainerPort{ 217 | api.ContainerPort{ 218 | ContainerPort: int32(80), 219 | }, 220 | }, 221 | }, 222 | }, 223 | }, 224 | Status: api.PodStatus{ 225 | Phase: api.PodRunning, 226 | }, 227 | })) 228 | } 229 | 230 | /* 231 | Test for github.com/30x/k8s-router/router/pods#GetRoutes where the pod has an invalid path value in the routingPaths annotation 232 | */ 233 | func TestGetRoutesInvalidPublicPathsPath(t *testing.T) { 234 | // "%ZZ" is not a valid path segment 235 | validateRoutes(t, "pod has an invalid routingPaths path", []*Route{}, GetRoutes(config, &api.Pod{ 236 | ObjectMeta: api.ObjectMeta{ 237 | Annotations: map[string]string{ 238 | "routingHosts": "test.github.com", 239 | "routingPaths": "3000:/people/%ZZ", 240 | }, 241 | }, 242 | Spec: api.PodSpec{ 243 | Containers: []api.Container{ 244 | api.Container{ 245 | Ports: []api.ContainerPort{ 246 | api.ContainerPort{ 247 | ContainerPort: int32(3000), 248 | }, 249 | }, 250 | }, 251 | }, 252 | }, 253 | Status: api.PodStatus{ 254 | Phase: api.PodRunning, 255 | }, 256 | })) 257 | } 258 | 259 | /* 260 | Test for github.com/30x/k8s-router/router/pods#GetRoutes where the pod has no routingPaths annotation 261 | */ 262 | func TestGetRoutesValidPods(t *testing.T) { 263 | host1 := "test.github.com" 264 | host2 := "www.github.com" 265 | ip := "10.244.1.17" 266 | path1 := "/" 267 | path2 := "/admin/" 268 | port1 := "3000" 269 | port2 := "3001" 270 | 271 | // A single host and path 272 | validateRoutes(t, "single host and path", []*Route{ 273 | &Route{ 274 | Incoming: &Incoming{ 275 | Host: host1, 276 | Path: path1, 277 | }, 278 | Outgoing: &Outgoing{ 279 | IP: ip, 280 | Port: port1, 281 | }, 282 | }, 283 | }, GetRoutes(config, &api.Pod{ 284 | ObjectMeta: api.ObjectMeta{ 285 | Annotations: map[string]string{ 286 | "routingHosts": host1, 287 | "routingPaths": port1 + ":" + path1, 288 | }, 289 | }, 290 | Spec: api.PodSpec{ 291 | Containers: []api.Container{ 292 | api.Container{ 293 | Ports: []api.ContainerPort{ 294 | api.ContainerPort{ 295 | ContainerPort: int32(3000), 296 | }, 297 | }, 298 | }, 299 | }, 300 | }, 301 | Status: api.PodStatus{ 302 | Phase: api.PodRunning, 303 | PodIP: ip, 304 | }, 305 | })) 306 | 307 | // A single host and multiple paths 308 | validateRoutes(t, "single host and multiple paths", []*Route{ 309 | &Route{ 310 | Incoming: &Incoming{ 311 | Host: host1, 312 | Path: path1, 313 | }, 314 | Outgoing: &Outgoing{ 315 | IP: ip, 316 | Port: port1, 317 | }, 318 | }, 319 | &Route{ 320 | Incoming: &Incoming{ 321 | Host: host1, 322 | Path: path2, 323 | }, 324 | Outgoing: &Outgoing{ 325 | IP: ip, 326 | Port: port2, 327 | }, 328 | }, 329 | }, GetRoutes(config, &api.Pod{ 330 | ObjectMeta: api.ObjectMeta{ 331 | Annotations: map[string]string{ 332 | "routingHosts": host1, 333 | "routingPaths": port1 + ":" + path1 + " " + port2 + ":" + path2, 334 | }, 335 | }, 336 | Spec: api.PodSpec{ 337 | Containers: []api.Container{ 338 | api.Container{ 339 | Ports: []api.ContainerPort{ 340 | api.ContainerPort{ 341 | ContainerPort: int32(3000), 342 | }, 343 | api.ContainerPort{ 344 | ContainerPort: int32(3001), 345 | }, 346 | }, 347 | }, 348 | }, 349 | }, 350 | Status: api.PodStatus{ 351 | Phase: api.PodRunning, 352 | PodIP: ip, 353 | }, 354 | })) 355 | 356 | // Multiple hosts and single path 357 | validateRoutes(t, "multiple hosts and single path", []*Route{ 358 | &Route{ 359 | Incoming: &Incoming{ 360 | Host: host1, 361 | Path: path1, 362 | }, 363 | Outgoing: &Outgoing{ 364 | IP: ip, 365 | Port: port1, 366 | }, 367 | }, 368 | &Route{ 369 | Incoming: &Incoming{ 370 | Host: host2, 371 | Path: path1, 372 | }, 373 | Outgoing: &Outgoing{ 374 | IP: ip, 375 | Port: port1, 376 | }, 377 | }, 378 | }, GetRoutes(config, &api.Pod{ 379 | ObjectMeta: api.ObjectMeta{ 380 | Annotations: map[string]string{ 381 | "routingHosts": host1 + " " + host2, 382 | "routingPaths": port1 + ":" + path1, 383 | }, 384 | }, 385 | Spec: api.PodSpec{ 386 | Containers: []api.Container{ 387 | api.Container{ 388 | Ports: []api.ContainerPort{ 389 | api.ContainerPort{ 390 | ContainerPort: int32(3000), 391 | }, 392 | }, 393 | }, 394 | }, 395 | }, 396 | Status: api.PodStatus{ 397 | Phase: api.PodRunning, 398 | PodIP: ip, 399 | }, 400 | })) 401 | 402 | // Multiple hosts and multiple paths 403 | validateRoutes(t, "multiple hosts and multiple paths", []*Route{ 404 | &Route{ 405 | Incoming: &Incoming{ 406 | Host: host1, 407 | Path: path1, 408 | }, 409 | Outgoing: &Outgoing{ 410 | IP: ip, 411 | Port: port1, 412 | }, 413 | }, 414 | &Route{ 415 | Incoming: &Incoming{ 416 | Host: host1, 417 | Path: path2, 418 | }, 419 | Outgoing: &Outgoing{ 420 | IP: ip, 421 | Port: port2, 422 | }, 423 | }, 424 | &Route{ 425 | Incoming: &Incoming{ 426 | Host: host2, 427 | Path: path1, 428 | }, 429 | Outgoing: &Outgoing{ 430 | IP: ip, 431 | Port: port1, 432 | }, 433 | }, 434 | &Route{ 435 | Incoming: &Incoming{ 436 | Host: host2, 437 | Path: path2, 438 | }, 439 | Outgoing: &Outgoing{ 440 | IP: ip, 441 | Port: port2, 442 | }, 443 | }, 444 | }, GetRoutes(config, &api.Pod{ 445 | ObjectMeta: api.ObjectMeta{ 446 | Annotations: map[string]string{ 447 | "routingHosts": host1 + " " + host2, 448 | "routingPaths": port1 + ":" + path1 + " " + port2 + ":" + path2, 449 | }, 450 | }, 451 | Spec: api.PodSpec{ 452 | Containers: []api.Container{ 453 | api.Container{ 454 | Ports: []api.ContainerPort{ 455 | api.ContainerPort{ 456 | ContainerPort: int32(3000), 457 | }, 458 | api.ContainerPort{ 459 | ContainerPort: int32(3001), 460 | }, 461 | }, 462 | }, 463 | }, 464 | }, 465 | Status: api.PodStatus{ 466 | Phase: api.PodRunning, 467 | PodIP: ip, 468 | }, 469 | })) 470 | } 471 | 472 | /* 473 | Test for github.com/30x/k8s-router/router/pods#UpdatePodCacheForEvents 474 | */ 475 | func TestUpdatePodCacheForEvents(t *testing.T) { 476 | annotations := map[string]string{ 477 | "routingHosts": "test.github.com", 478 | "routingPaths": "80:/", 479 | } 480 | cache := map[string]*PodWithRoutes{} 481 | labels := map[string]string{ 482 | "routable": "true", 483 | } 484 | podName := "test-pod" 485 | 486 | modifiedPodRoutableFalse := &api.Pod{ 487 | ObjectMeta: api.ObjectMeta{ 488 | Labels: map[string]string{ 489 | "routable": "false", 490 | }, 491 | Name: podName, 492 | }, 493 | Status: api.PodStatus{ 494 | Phase: api.PodRunning, 495 | PodIP: "10.244.1.17", 496 | }, 497 | } 498 | modifiedPodWithRoutes := &api.Pod{ 499 | ObjectMeta: api.ObjectMeta{ 500 | Annotations: annotations, 501 | Labels: labels, 502 | Name: podName, 503 | }, 504 | Status: api.PodStatus{ 505 | Phase: api.PodRunning, 506 | PodIP: "10.244.1.17", 507 | }, 508 | } 509 | unroutablePod := &api.Pod{ 510 | ObjectMeta: api.ObjectMeta{ 511 | Annotations: annotations, 512 | Labels: labels, 513 | Name: podName, 514 | }, 515 | Status: api.PodStatus{ 516 | Phase: api.PodPending, 517 | }, 518 | } 519 | 520 | // Test adding an unroutable pod 521 | needsRestart := UpdatePodCacheForEvents(config, cache, []watch.Event{ 522 | watch.Event{ 523 | Type: watch.Added, 524 | Object: unroutablePod, 525 | }, 526 | }) 527 | 528 | if needsRestart { 529 | t.Fatal("Server should not need a restart") 530 | } else if _, ok := cache[podName]; !ok { 531 | t.Fatal("Cache should reflect the added pod") 532 | } 533 | 534 | // Test modifying a pod to make it routable 535 | needsRestart = UpdatePodCacheForEvents(config, cache, []watch.Event{ 536 | watch.Event{ 537 | Type: watch.Modified, 538 | Object: modifiedPodWithRoutes, 539 | }, 540 | }) 541 | 542 | if !needsRestart { 543 | t.Fatal("Server should need a restart") 544 | } 545 | 546 | // Test modifying a pod that does not change routes 547 | needsRestart = UpdatePodCacheForEvents(config, cache, []watch.Event{ 548 | watch.Event{ 549 | Type: watch.Modified, 550 | Object: modifiedPodWithRoutes, 551 | }, 552 | }) 553 | 554 | if needsRestart { 555 | t.Fatal("Server should not need a restart") 556 | } 557 | 558 | // Test modifying a pod to set the routable label to false 559 | needsRestart = UpdatePodCacheForEvents(config, cache, []watch.Event{ 560 | watch.Event{ 561 | Type: watch.Modified, 562 | Object: modifiedPodRoutableFalse, 563 | }, 564 | }) 565 | 566 | if !needsRestart { 567 | t.Fatal("Server should need a restart") 568 | } else if len(cache) > 0 { 569 | t.Fatal("Cache should reflect the modified (but removed) pod") 570 | } 571 | 572 | // Test modifying a pod to remove its routable label 573 | _ = UpdatePodCacheForEvents(config, cache, []watch.Event{ 574 | watch.Event{ 575 | Type: watch.Added, 576 | Object: modifiedPodWithRoutes, 577 | }, 578 | }) 579 | 580 | if len(cache) != 1 { 581 | t.Fatal("There was an issue updating the cache") 582 | } 583 | 584 | needsRestart = UpdatePodCacheForEvents(config, cache, []watch.Event{ 585 | watch.Event{ 586 | Type: watch.Modified, 587 | Object: modifiedPodRoutableFalse, 588 | }, 589 | }) 590 | 591 | if !needsRestart { 592 | t.Fatal("Server should need a restart") 593 | } else if len(cache) > 0 { 594 | t.Fatal("Cache should reflect the modified (but removed) pod") 595 | } 596 | 597 | // Test deleting a pod 598 | _ = UpdatePodCacheForEvents(config, cache, []watch.Event{ 599 | watch.Event{ 600 | Type: watch.Added, 601 | Object: modifiedPodWithRoutes, 602 | }, 603 | }) 604 | 605 | if len(cache) != 1 { 606 | t.Fatal("There was an issue updating the cache") 607 | } 608 | 609 | needsRestart = UpdatePodCacheForEvents(config, cache, []watch.Event{ 610 | watch.Event{ 611 | Type: watch.Deleted, 612 | Object: modifiedPodWithRoutes, 613 | }, 614 | }) 615 | 616 | if !needsRestart { 617 | t.Fatal("Server should need a restart") 618 | } else if len(cache) > 0 { 619 | t.Fatal("Cache should reflect the deleted pod") 620 | } 621 | } 622 | -------------------------------------------------------------------------------- /router/secrets.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package router 18 | 19 | import ( 20 | "log" 21 | 22 | "k8s.io/kubernetes/pkg/api" 23 | client "k8s.io/kubernetes/pkg/client/unversioned" 24 | "k8s.io/kubernetes/pkg/watch" 25 | ) 26 | 27 | func ConvertSecretToModel(config *Config, secret *api.Secret) ([]byte) { 28 | apikey, _ := secret.Data[config.APIKeySecretDataField] 29 | return apikey 30 | } 31 | /* 32 | GetRouterSecretList returns the router secrets. 33 | */ 34 | func GetRouterSecretList(config *Config, kubeClient *client.Client) (*api.SecretList, error) { 35 | // Query all secrets 36 | secretList, err := kubeClient.Secrets(api.NamespaceAll).List(api.ListOptions{}) 37 | 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | // Filter out the secrets that are not router API Key secrets or that do not have the proper secret key 43 | var filtered []api.Secret 44 | 45 | for _, secret := range secretList.Items { 46 | if secret.Name == config.APIKeySecret { 47 | _, ok := secret.Data[config.APIKeySecretDataField] 48 | 49 | if ok { 50 | filtered = append(filtered, secret) 51 | } else { 52 | log.Printf(" Router secret for namespace (%s) is not usable: Missing '%s' key\n", secret.Namespace, config.APIKeySecretDataField) 53 | } 54 | } 55 | } 56 | 57 | secretList.Items = filtered 58 | 59 | return secretList, nil 60 | } 61 | 62 | /* 63 | UpdateSecretCacheForEvents updates the cache based on the secret events and returns if the changes warrant an nginx restart. 64 | */ 65 | func UpdateSecretCacheForEvents(config *Config, cache map[string][]byte, events []watch.Event) bool { 66 | needsRestart := false 67 | 68 | for _, event := range events { 69 | secret := event.Object.(*api.Secret) 70 | namespace := secret.Namespace 71 | 72 | log.Printf(" Secret (%s in %s namespace) event: %s\n", secret.Name, secret.Namespace, event.Type) 73 | 74 | // Process the event 75 | switch event.Type { 76 | case watch.Added: 77 | cache[namespace] = ConvertSecretToModel(config, secret) 78 | needsRestart = true 79 | 80 | case watch.Deleted: 81 | delete(cache, namespace) 82 | needsRestart = true 83 | 84 | case watch.Modified: 85 | cachedAPIKey, ok := cache[namespace] 86 | apiKey := ConvertSecretToModel(config, secret) 87 | 88 | if ok { 89 | 90 | if (apiKey == nil && cachedAPIKey != nil) || (apiKey != nil && cachedAPIKey == nil) { 91 | needsRestart = true 92 | } else if apiKey != nil && cachedAPIKey != nil && len(apiKey) != len(cachedAPIKey) { 93 | needsRestart = true 94 | } else { 95 | for i := range apiKey { 96 | if apiKey[i] != cachedAPIKey[i] { 97 | needsRestart = true 98 | 99 | break 100 | } 101 | } 102 | } 103 | } 104 | 105 | cache[namespace] = apiKey 106 | } 107 | 108 | if _, ok := cache[namespace]; ok { 109 | apiKey := ConvertSecretToModel(config, secret) 110 | 111 | if apiKey == nil { 112 | log.Printf(" Secret has an %s value: no\n", config.APIKeySecretDataField) 113 | } else { 114 | log.Printf(" Secret has an %s value: yes\n", config.APIKeySecretDataField) 115 | } 116 | } 117 | } 118 | 119 | return needsRestart 120 | } 121 | -------------------------------------------------------------------------------- /router/secrets_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package router 18 | 19 | import ( 20 | "io/ioutil" 21 | "log" 22 | "testing" 23 | 24 | "github.com/30x/k8s-router/kubernetes" 25 | 26 | "k8s.io/kubernetes/pkg/api" 27 | "k8s.io/kubernetes/pkg/watch" 28 | ) 29 | 30 | // config is set in pods_test.go 31 | 32 | func init() { 33 | log.SetOutput(ioutil.Discard) 34 | } 35 | 36 | /* 37 | Test for github.com/30x/k8s-router/router/secrets#GetRouterSecretList 38 | */ 39 | func TestGetRouterSecretList(t *testing.T) { 40 | kubeClient, err := kubernetes.GetClient() 41 | 42 | if err != nil { 43 | t.Fatalf("Failed to create k8s client: %v.", err) 44 | } 45 | 46 | secretList, err := GetRouterSecretList(config, kubeClient) 47 | 48 | if err != nil { 49 | t.Fatalf("Failed to get the router secrets: %v.", err) 50 | } 51 | 52 | for _, secret := range secretList.Items { 53 | if secret.Name != config.APIKeySecret { 54 | t.Fatalf("Every secret should have a %s name", config.APIKeySecret) 55 | } 56 | } 57 | } 58 | 59 | /* 60 | Test for github.com/30x/k8s-router/router/secrets#UpdateSecretCacheForEvents 61 | */ 62 | func TestUpdateSecretCacheForEvents(t *testing.T) { 63 | apiKeyStr := "API-Key" 64 | apiKey := []byte(apiKeyStr) 65 | cache := make(map[string][]byte) 66 | namespace := "my-namespace" 67 | 68 | addedSecret := &api.Secret{ 69 | ObjectMeta: api.ObjectMeta{ 70 | Name: config.APIKeySecret, 71 | Namespace: "my-namespace", 72 | }, 73 | Data: map[string][]byte{ 74 | "api-key": apiKey, 75 | }, 76 | } 77 | modifiedSecretNoRestart := &api.Secret{ 78 | ObjectMeta: api.ObjectMeta{ 79 | Name: config.APIKeySecret, 80 | Namespace: "my-namespace", 81 | }, 82 | Data: map[string][]byte{ 83 | "api-key": apiKey, 84 | "new-key": []byte("New-API-Key"), 85 | }, 86 | } 87 | modifiedSecretRestart := &api.Secret{ 88 | ObjectMeta: api.ObjectMeta{ 89 | Name: config.APIKeySecret, 90 | Namespace: "my-namespace", 91 | }, 92 | Data: map[string][]byte{ 93 | "api-key": []byte("Updated-API-Key"), 94 | }, 95 | } 96 | 97 | // Test add event 98 | needsRestart := UpdateSecretCacheForEvents(config, cache, []watch.Event{ 99 | watch.Event{ 100 | Type: watch.Added, 101 | Object: addedSecret, 102 | }, 103 | }) 104 | 105 | if !needsRestart { 106 | t.Fatal("Server should require a restart") 107 | } else if _, ok := cache[namespace]; !ok { 108 | t.Fatal("Cache should reflect the added secret") 109 | } 110 | 111 | // Test modify event with unchanged api-key 112 | needsRestart = UpdateSecretCacheForEvents(config, cache, []watch.Event{ 113 | watch.Event{ 114 | Type: watch.Modified, 115 | Object: modifiedSecretNoRestart, 116 | }, 117 | }) 118 | 119 | if needsRestart { 120 | t.Fatal("Server should not require a restart") 121 | } 122 | 123 | // Test modify event with changed api-key 124 | needsRestart = UpdateSecretCacheForEvents(config, cache, []watch.Event{ 125 | watch.Event{ 126 | Type: watch.Modified, 127 | Object: modifiedSecretRestart, 128 | }, 129 | }) 130 | 131 | if !needsRestart { 132 | t.Fatal("Server should require a restart") 133 | } 134 | 135 | if apiKeyStr == string(cache[namespace][:]) { 136 | t.Fatal("Cache should have the updated secret") 137 | } 138 | 139 | // Test delete event 140 | needsRestart = UpdateSecretCacheForEvents(config, cache, []watch.Event{ 141 | watch.Event{ 142 | Type: watch.Deleted, 143 | Object: addedSecret, 144 | }, 145 | }) 146 | 147 | if !needsRestart { 148 | t.Fatal("Server should require a restart") 149 | } else if _, ok := cache[namespace]; ok { 150 | t.Fatal("Cache should not have the deleted secret") 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /router/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package router 18 | 19 | import ( 20 | "k8s.io/kubernetes/pkg/api" 21 | "k8s.io/kubernetes/pkg/labels" 22 | ) 23 | 24 | /* 25 | Cache is the structure containing the router API Keys and the routable pods cache 26 | */ 27 | type Cache struct { 28 | Pods map[string]*PodWithRoutes 29 | Secrets map[string][]byte 30 | } 31 | 32 | /* 33 | Config is the structure containing the configuration 34 | */ 35 | type Config struct { 36 | // The header name used to identify the API Key 37 | APIKeyHeader string 38 | // The secret name used to store the API Key for the namespace 39 | APIKeySecret string 40 | // The secret data field name to store the API Key for the namespace 41 | APIKeySecretDataField string 42 | // The name of the annotation used to find hosts to route 43 | HostsAnnotation string 44 | // The name of the annotation used to find paths to route 45 | PathsAnnotation string 46 | // The port that nginx will listen on 47 | Port int 48 | // The label selector used to identify routable objects 49 | RoutableLabelSelector labels.Selector 50 | // Max client request body size. nginx config: client_max_body_size. eg 10m 51 | ClientMaxBodySize string 52 | } 53 | 54 | /* 55 | Incoming describes the information required to route an incoming request 56 | */ 57 | type Incoming struct { 58 | Host string 59 | Path string 60 | } 61 | 62 | /* 63 | Outgoing describes the information required to proxy to a backend 64 | */ 65 | type Outgoing struct { 66 | IP string 67 | Port string 68 | } 69 | 70 | /* 71 | PodWithRoutes contains a pod and its routes 72 | */ 73 | type PodWithRoutes struct { 74 | Name string 75 | Namespace string 76 | Status api.PodPhase 77 | AnnotationHash uint64 78 | Routes []*Route 79 | } 80 | 81 | /* 82 | Route describes the incoming route matching details and the outgoing proxy backend details 83 | */ 84 | type Route struct { 85 | Incoming *Incoming 86 | Outgoing *Outgoing 87 | } 88 | -------------------------------------------------------------------------------- /utils/validation.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package utils 18 | 19 | /* 20 | IsValidPort returns whether the provided integer is a valid port 21 | */ 22 | func IsValidPort(port int) bool { 23 | return port > 0 && port < 65536 24 | } 25 | -------------------------------------------------------------------------------- /utils/validation_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2016 Apigee Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package utils 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | /* 24 | Test for github.com/30x/k8s-router/utils/validation#IsValidPort with invalid values 25 | */ 26 | func TestIsValidPortNotNumberInvalidValues(t *testing.T) { 27 | makeError := func() { 28 | t.Fatal("Should had returned false") 29 | } 30 | 31 | if IsValidPort(0) { 32 | makeError() 33 | } else if IsValidPort(70000) { 34 | makeError() 35 | } 36 | } 37 | 38 | /* 39 | Test for github.com/30x/k8s-router/utils/validation#IsValidPort with valid values 40 | */ 41 | func TestIsValidPortNotNumberValidValues(t *testing.T) { 42 | makeError := func() { 43 | t.Fatal("Should had returned true") 44 | } 45 | 46 | if !IsValidPort(1) { 47 | makeError() 48 | } else if !IsValidPort(65000) { 49 | makeError() 50 | } 51 | } 52 | --------------------------------------------------------------------------------