├── .github └── workflows │ └── go.yml ├── .gitignore ├── .golangci.yml ├── Dockerfile ├── Makefile ├── README.md ├── api ├── api.go ├── api_test.go ├── handler.go └── handler_test.go ├── backend ├── backend.go ├── local-cluster.go ├── multi-cluster.go └── multi-cluster_test.go ├── cmd ├── daemon.go ├── flags.go ├── flags_test.go └── router │ └── main.go ├── deployments ├── local.yml └── rbac.yml ├── go.mod ├── go.sum ├── kubernetes ├── ingress.go ├── ingress_test.go ├── istiogateway.go ├── istiogateway_test.go ├── loadbalancer.go ├── loadbalancer_test.go ├── service.go └── service_test.go ├── observability ├── middlware.go ├── observability.go └── transport.go └── router ├── mock └── service.go ├── service.go └── service_test.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | test: 8 | name: Test 9 | runs-on: ubuntu-latest 10 | steps: 11 | 12 | - name: Set up Go 1.x 13 | uses: actions/setup-go@v2 14 | with: 15 | go-version: ^1.22 16 | id: go 17 | 18 | - name: Check out code into the Go module directory 19 | uses: actions/checkout@v2 20 | 21 | - name: Get dependencies 22 | run: | 23 | go get -v -t -d ./... 24 | if [ -f Gopkg.toml ]; then 25 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 26 | dep ensure 27 | fi 28 | 29 | - name: Build 30 | run: go build -v -o test-build ./cmd/router 31 | 32 | - name: Test 33 | run: go test -v ./... 34 | 35 | lint: 36 | name: Lint 37 | runs-on: ubuntu-latest 38 | steps: 39 | - uses: actions/setup-go@v3 40 | with: 41 | go-version: '1.22' 42 | - uses: actions/checkout@v2 43 | - uses: actions/cache@v2 44 | with: 45 | path: ~/go/pkg/mod 46 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 47 | restore-keys: | 48 | ${{ runner.os }}-go- 49 | - uses: golangci/golangci-lint-action@v6 50 | with: 51 | version: v1.59 52 | args: --timeout=10m 53 | 54 | docker-image: 55 | name: "Push to dockerhub" 56 | needs: 57 | - test 58 | - lint 59 | runs-on: ubuntu-latest 60 | if: github.event_name != 'pull_request' 61 | steps: 62 | - uses: actions/checkout@v2 63 | - name: Set up Docker Buildx 64 | id: buildx 65 | uses: docker/setup-buildx-action@v1 66 | 67 | - name: Available platforms 68 | run: echo ${{ steps.buildx.outputs.platforms }} 69 | - uses: actions/cache@v2 70 | with: 71 | path: /tmp/.buildx-cache 72 | key: ${{ runner.os }}-buildx-${{ github.sha }} 73 | restore-keys: | 74 | ${{ runner.os }}-buildx- 75 | - uses: Surgo/docker-smart-tag-action@v1 76 | id: smarttag 77 | with: 78 | docker_image: tsuru/kubernetes-router 79 | default_branch: main 80 | tag_with_sha: "true" 81 | - uses: docker/login-action@v1 82 | with: 83 | username: ${{ secrets.DOCKERHUB_USERNAME }} 84 | password: ${{ secrets.DOCKERHUB_PASSWORD }} 85 | - uses: docker/build-push-action@v2 86 | with: 87 | push: true 88 | tags: ${{ steps.smarttag.outputs.tag }} 89 | cache-from: type=local,src=/tmp/.buildx-cache 90 | cache-to: type=local,dest=/tmp/.buildx-cache 91 | platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 92 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | kubernetes-router 2 | .vscode/ 3 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | deadline: 30m 3 | tests: true 4 | 5 | linters-settings: 6 | govet: 7 | shadow: true 8 | gofmt: 9 | simplify: true 10 | maligned: 11 | suggest-new: true 12 | 13 | linters: 14 | enable: 15 | - goimports 16 | - gofmt 17 | - misspell 18 | disable: 19 | - errcheck 20 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG alpine_version=3.19 2 | ARG golang_version=1.22 3 | FROM --platform=$BUILDPLATFORM golang:${golang_version}-alpine${alpine_version} as builder 4 | ARG TARGETARCH 5 | ENV GOARCH=$TARGETARCH 6 | RUN apk update && apk add make 7 | 8 | COPY . /go/src/github.com/tsuru/kubernetes-router/ 9 | WORKDIR /go/src/github.com/tsuru/kubernetes-router/ 10 | RUN CGO_ENABLED=0 make build 11 | 12 | FROM alpine:${alpine_version} 13 | RUN apk --no-cache add ca-certificates 14 | WORKDIR /root/ 15 | COPY --from=builder /go/src/github.com/tsuru/kubernetes-router/kubernetes-router . 16 | 17 | EXPOSE 8077 18 | 19 | CMD ["./kubernetes-router"] 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BINARY=kubernetes-router 2 | TAG=latest 3 | IMAGE=tsuru/$(BINARY) 4 | LOCAL_REGISTRY=10.200.10.1:5000 5 | NAMESPACE=tsuru 6 | LINTER_ARGS = \ 7 | -j 4 --enable-gc -s vendor -e '.*/vendor/.*' --vendor --enable=misspell --enable=gofmt --enable=goimports \ 8 | --disable=gocyclo --disable=gosec --deadline=60m --tests 9 | RUN_FLAGS=-v 9 10 | 11 | .PHONY: run 12 | run: build 13 | ./$(BINARY) $(RUN_FLAGS) 14 | 15 | .PHONY: build 16 | build: 17 | go build -o $(BINARY) ./cmd/router 18 | 19 | .PHONY: build-docker 20 | build-docker: 21 | docker build -t localhost:5000/kubernetes-router:latest . 22 | 23 | .PHONY: push 24 | push: build-docker 25 | docker push localhost:5000/kubernetes-router:latest 26 | kubectl get po -l app=kubernetes-router --no-headers | awk '{ print $$1 }' | xargs -I{} kubectl delete po {} 27 | 28 | .PHONY: test 29 | test: 30 | go test ./... -race -cover 31 | 32 | .PHONY: lint 33 | lint: 34 | curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin 35 | go install ./... 36 | go test -i ./... 37 | $$(go env GOPATH)/bin/golangci-lint run -c ./.golangci.yml ./... 38 | 39 | .PHONY: minikube 40 | minikube: 41 | make IMAGE=$(LOCAL_REGISTRY)/$(BINARY) push 42 | kubectl delete -f deployments/local.yml || true 43 | cat deployments/rbac.yml | sed 's~NAMESPACE~$(NAMESPACE)~g' | kubectl apply -f - 44 | cat deployments/local.yml | sed 's~IMAGE~$(LOCAL_REGISTRY)/$(BINARY)~g' | sed 's~NAMESPACE~$(NAMESPACE)~g' | kubectl apply -f - 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Router 2 | 3 | Kubernetes router implements the tsuru router http API and manages the creation and removal of 4 | load balancer services or ingress resources on a kubernetes cluster. It expects to be run as a pod in the cluster itself. 5 | 6 | ## Flags 7 | 8 | - `-alsologtostderr`: log to standard error as well as files; 9 | - `-cert-file`: Path to certificate used to serve https requests; 10 | - `-controller-modes`: Defines enabled controller running modes: service, ingress, ingress-nginx or istio-gateway; 11 | - `-ingress-domain`: Default domain to be used on created vhosts, local is the default. (eg: serviceName.local) (default "local"); 12 | - `-istio-gateway.gateway-selector`: Gateway selector used in gateways created for apps; 13 | - `-k8s-annotations`: Annotations to be added to each resource created. Expects KEY=VALUE format; 14 | - `-k8s-labels`: Labels to be added to each resource created. Expects KEY=VALUE format; 15 | - `-k8s-namespace`: Kubernetes namespace to create resources (default "default"); 16 | - `-k8s-timeout`: Kubernetes per-request timeout (default 10s); 17 | - `-key-file`: Path to private key used to serve https requests; 18 | - `-listen-addr`: Listen address (default ":8077"); 19 | - `-log_backtrace_at`: when logging hits line file:N, emit a stack trace; 20 | - `-log_dir`: If non-empty, write log files in this directory; 21 | - `-logtostderr`: log to standard error instead of files; 22 | - `-opts-to-label`: Mapping between router options and service labels. Expects KEY=VALUE format; 23 | - `-opts-to-label-doc`: Mapping between router options and user friendly help. Expects KEY=VALUE format; 24 | - `-opts-to-ingress-annotations`: Mapping between router options and ingress annotations. Expects KEY=VALUE format; 25 | - `-opts-to-ingress-annotations-doc`: Mapping between router options and user friendly help. Expects KEY=VALUE format; 26 | - `-ingress-class`: Default class annotation for ingress objects; 27 | - `-ingress-annotations-prefix`: Default prefix for annotations in ingress objects; 28 | - `-pool-labels`: Default labels for a given pool. Expects POOL={"LABEL":"VALUE"} format; 29 | - `-stderrthreshold`: logs at or above this threshold go to stderr; 30 | - `-v`: log level for V logs; 31 | - `-vmodule`: comma-separated list of pattern=N settings for file-filtered logging. 32 | 33 | ## Envs 34 | 35 | - `ROUTER_API_USER`/`ROUTER_API_PASSWORD`: Basic auth user and password to be checked for every request to the router API. Optional. 36 | 37 | ## Running locally with Tsuru and Minikube 38 | 39 | 1. Setup tsuru + minikube (https://docs.tsuru.io/master/contributing/compose.html) 40 | 41 | 2. Run the ingress-router in your minikube cluster 42 | 43 | $ make minikube 44 | 45 | 3. Fetch the URL of the ingress-router service 46 | 47 | $ minikube service list 48 | 49 | 4. Add the ingress router to tsuru.conf 50 | 51 | $ vim ../tsuru/etc/tsuru-compose.conf 52 | 53 | Add: 54 | 55 | routers: 56 | ingress-router: 57 | type: api 58 | api-url: http://192.168.99.100:31647 59 | 60 | Replace http://192.168.99.100:31647 with the URL shown by `minikube service list`. 61 | 62 | 5. Reload tsuru 63 | 64 | $ $GOPATH/src/github.com/tsuru/tsuru/build-compose.sh 65 | 66 | 6. Create an app using the ingress-router 67 | 68 | $ tsuru app-create myapp static --router ingress-router 69 | 70 | -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package api 6 | 7 | import ( 8 | "context" 9 | "encoding/json" 10 | "fmt" 11 | "log" 12 | "net/http" 13 | "regexp" 14 | "strings" 15 | "sync" 16 | "time" 17 | 18 | "github.com/golang/glog" 19 | "github.com/gorilla/mux" 20 | "github.com/tsuru/kubernetes-router/backend" 21 | "github.com/tsuru/kubernetes-router/router" 22 | "golang.org/x/sync/errgroup" 23 | ) 24 | 25 | var httpSchemeRegex = regexp.MustCompile(`^https?://`) 26 | 27 | const checkPathTimeout = 2 * time.Second 28 | 29 | // RouterAPI implements Tsuru HTTP router API 30 | type RouterAPI struct { 31 | Backend backend.Backend 32 | } 33 | 34 | // Routes returns an mux for the API routes 35 | func (a *RouterAPI) Routes() *mux.Router { 36 | r := mux.NewRouter() 37 | a.registerRoutes(r.PathPrefix("/api").Subrouter()) 38 | a.registerRoutes(r.PathPrefix("/api/{mode}").Subrouter()) 39 | return r 40 | } 41 | 42 | func (a *RouterAPI) registerRoutes(r *mux.Router) { 43 | r.Handle("/backend/{name}", handler(a.getBackend)).Methods(http.MethodGet) 44 | r.Handle("/backend/{name}", handler(a.ensureBackend)).Methods(http.MethodPut) 45 | r.Handle("/backend/{name}", handler(a.removeBackend)).Methods(http.MethodDelete) 46 | r.Handle("/backend/{name}/status", handler(a.status)).Methods(http.MethodGet) 47 | r.Handle("/backend/{name}/routes", handler(a.getRoutes)).Methods(http.MethodGet) 48 | r.Handle("/info", handler(a.info)).Methods(http.MethodGet) 49 | 50 | // TLS 51 | r.Handle("/backend/{name}/certificate/{certname}", handler(a.addCertificate)).Methods(http.MethodPut) 52 | r.Handle("/backend/{name}/certificate/{certname}", handler(a.getCertificate)).Methods(http.MethodGet) 53 | r.Handle("/backend/{name}/certificate/{certname}", handler(a.removeCertificate)).Methods(http.MethodDelete) 54 | 55 | // Supports 56 | r.Handle("/support/tls", handler(a.supportTLS)).Methods(http.MethodGet) 57 | r.Handle("/support/info", handler(func(w http.ResponseWriter, r *http.Request) error { 58 | w.WriteHeader(http.StatusOK) 59 | return nil 60 | })).Methods(http.MethodGet) 61 | r.Handle("/support/status", handler(func(w http.ResponseWriter, r *http.Request) error { 62 | w.WriteHeader(http.StatusOK) 63 | return nil 64 | })).Methods(http.MethodGet) 65 | r.Handle("/support/prefix", handler(func(w http.ResponseWriter, r *http.Request) error { 66 | w.WriteHeader(http.StatusOK) 67 | return nil 68 | })).Methods(http.MethodGet) 69 | r.Handle("/support/v2", handler(func(w http.ResponseWriter, r *http.Request) error { 70 | w.WriteHeader(http.StatusOK) 71 | return nil 72 | })).Methods(http.MethodGet) 73 | } 74 | 75 | func (a *RouterAPI) router(ctx context.Context, mode string, header http.Header) (router.Router, error) { 76 | router, err := a.Backend.Router(ctx, mode, header) 77 | if err != nil { 78 | if err == backend.ErrBackendNotFound { 79 | return nil, httpError{Status: http.StatusNotFound} 80 | } 81 | 82 | return nil, err 83 | } 84 | 85 | return router, nil 86 | } 87 | 88 | func instanceID(r *http.Request) router.InstanceID { 89 | vars := mux.Vars(r) 90 | return router.InstanceID{ 91 | AppName: vars["name"], 92 | InstanceName: r.Header.Get("X-Router-Instance"), 93 | } 94 | } 95 | 96 | // getBackend returns the address for the load balancer registered in 97 | // the ingress by a ingress controller 98 | func (a *RouterAPI) getBackend(w http.ResponseWriter, r *http.Request) error { 99 | ctx := r.Context() 100 | vars := mux.Vars(r) 101 | svc, err := a.router(ctx, vars["mode"], r.Header) 102 | if err != nil { 103 | return err 104 | } 105 | addrs, err := svc.GetAddresses(ctx, instanceID(r)) 106 | if err != nil { 107 | return err 108 | } 109 | type getBackendResponse struct { 110 | Address string `json:"address"` 111 | Addresses []string `json:"addresses"` 112 | } 113 | rsp := getBackendResponse{ 114 | Addresses: addrs, 115 | } 116 | if len(addrs) > 0 { 117 | rsp.Address = addrs[0] 118 | } 119 | return json.NewEncoder(w).Encode(rsp) 120 | } 121 | 122 | type statusResp struct { 123 | Status router.BackendStatus `json:"status"` 124 | Detail string `json:"detail"` 125 | Checks []urlCheck `json:"checks,omitempty"` 126 | } 127 | 128 | type urlCheck struct { 129 | Address string `json:"address"` 130 | Status int `json:"status"` 131 | Error string `json:"error"` 132 | } 133 | 134 | // status returns backend events 135 | func (a *RouterAPI) status(w http.ResponseWriter, r *http.Request) error { 136 | ctx := r.Context() 137 | vars := mux.Vars(r) 138 | svc, err := a.router(ctx, vars["mode"], r.Header) 139 | if err != nil { 140 | return err 141 | } 142 | 143 | rsp := statusResp{ 144 | Status: router.BackendStatusReady, 145 | } 146 | 147 | grp, ctx := errgroup.WithContext(ctx) 148 | 149 | grp.Go(func() error { 150 | checks, checkErr := checkPath(ctx, r.URL.Query().Get("checkpath"), svc, instanceID(r)) 151 | if checkErr != nil { 152 | return checkErr 153 | } 154 | rsp.Checks = checks 155 | return nil 156 | }) 157 | 158 | grp.Go(func() error { 159 | statusRouter, ok := svc.(router.RouterStatus) 160 | if !ok { 161 | return nil 162 | } 163 | status, detail, statusErr := statusRouter.GetStatus(ctx, instanceID(r)) 164 | if statusErr != nil { 165 | return statusErr 166 | } 167 | rsp.Status = status 168 | rsp.Detail = detail 169 | return nil 170 | }) 171 | 172 | err = grp.Wait() 173 | if err != nil { 174 | return err 175 | } 176 | 177 | return json.NewEncoder(w).Encode(rsp) 178 | } 179 | 180 | // removeBackend removes the Ingress for a given app 181 | func (a *RouterAPI) removeBackend(w http.ResponseWriter, r *http.Request) error { 182 | ctx := r.Context() 183 | vars := mux.Vars(r) 184 | svc, err := a.router(ctx, vars["mode"], r.Header) 185 | if err != nil { 186 | return err 187 | } 188 | return svc.Remove(ctx, instanceID(r)) 189 | } 190 | 191 | // addRoutes updates the Ingress to point to the correct service 192 | func (a *RouterAPI) ensureBackend(w http.ResponseWriter, r *http.Request) error { 193 | vars := mux.Vars(r) 194 | ctx := r.Context() 195 | 196 | opts := &router.EnsureBackendOpts{ 197 | Opts: router.Opts{ 198 | HeaderOpts: r.Header.Values("X-Router-Opt"), 199 | }, 200 | } 201 | err := json.NewDecoder(r.Body).Decode(opts) 202 | if err != nil { 203 | return err 204 | } 205 | 206 | svc, err := a.router(ctx, vars["mode"], r.Header) 207 | if err != nil { 208 | return err 209 | } 210 | 211 | return svc.Ensure(ctx, instanceID(r), *opts) 212 | } 213 | 214 | // getRoutes always returns an empty address list to force tsuru to call 215 | // addRoutes on every routes rebuild call. 216 | func (a *RouterAPI) getRoutes(w http.ResponseWriter, r *http.Request) error { 217 | type resp struct { 218 | Addresses []string `json:"addresses"` 219 | } 220 | return json.NewEncoder(w).Encode(resp{}) 221 | } 222 | 223 | func (a *RouterAPI) info(w http.ResponseWriter, r *http.Request) error { 224 | ctx := r.Context() 225 | vars := mux.Vars(r) 226 | svc, err := a.router(ctx, vars["mode"], r.Header) 227 | if err != nil { 228 | return err 229 | } 230 | opts := svc.SupportedOptions(ctx) 231 | allOpts := router.DescribedOptions() 232 | info := make(map[string]string) 233 | for k, v := range opts { 234 | vv := v 235 | if vv == "" { 236 | vv = allOpts[k] 237 | } 238 | info[k] = vv 239 | } 240 | return json.NewEncoder(w).Encode(info) 241 | } 242 | 243 | // Healthcheck checks the health of the service 244 | func (a *RouterAPI) Healthcheck(w http.ResponseWriter, r *http.Request) { 245 | err := a.Backend.Healthcheck(r.Context()) 246 | if err != nil { 247 | glog.Errorf("failed to write healthcheck: %v", err) 248 | w.WriteHeader(http.StatusInternalServerError) 249 | fmt.Fprint(w, err.Error()) 250 | return 251 | } 252 | 253 | fmt.Fprint(w, "WORKING") 254 | } 255 | 256 | // addCertificate Add certificate to app 257 | func (a *RouterAPI) addCertificate(w http.ResponseWriter, r *http.Request) error { 258 | ctx := r.Context() 259 | vars := mux.Vars(r) 260 | name := vars["name"] 261 | certName := vars["certname"] 262 | log.Printf("Adding on %s certificate %s", name, certName) 263 | cert := router.CertData{} 264 | err := json.NewDecoder(r.Body).Decode(&cert) 265 | if err != nil { 266 | return err 267 | } 268 | svc, err := a.router(ctx, vars["mode"], r.Header) 269 | if err != nil { 270 | return err 271 | } 272 | return svc.(router.RouterTLS).AddCertificate(ctx, instanceID(r), certName, cert) 273 | } 274 | 275 | // getCertificate Return certificate for app 276 | func (a *RouterAPI) getCertificate(w http.ResponseWriter, r *http.Request) error { 277 | ctx := r.Context() 278 | vars := mux.Vars(r) 279 | name := vars["name"] 280 | certName := vars["certname"] 281 | log.Printf("Getting certificate %s from %s", certName, name) 282 | svc, err := a.router(ctx, vars["mode"], r.Header) 283 | if err != nil { 284 | return err 285 | } 286 | cert, err := svc.(router.RouterTLS).GetCertificate(ctx, instanceID(r), certName) 287 | 288 | if err == router.ErrCertificateNotFound { 289 | w.WriteHeader(http.StatusNotFound) 290 | return nil 291 | } 292 | 293 | if err != nil { 294 | return err 295 | } 296 | b, err := json.Marshal(&cert) 297 | if err != nil { 298 | return err 299 | } 300 | w.Header().Set("Content-Type", "application/json") 301 | _, err = w.Write(b) 302 | return err 303 | } 304 | 305 | // removeCertificate Delete certificate for app 306 | func (a *RouterAPI) removeCertificate(w http.ResponseWriter, r *http.Request) error { 307 | ctx := r.Context() 308 | vars := mux.Vars(r) 309 | name := vars["name"] 310 | certName := vars["certname"] 311 | log.Printf("Removing certificate %s from %s", certName, name) 312 | svc, err := a.router(ctx, vars["mode"], r.Header) 313 | if err != nil { 314 | return err 315 | } 316 | err = svc.(router.RouterTLS).RemoveCertificate(ctx, instanceID(r), certName) 317 | if err == router.ErrCertificateNotFound { 318 | w.WriteHeader(http.StatusNotFound) 319 | return nil 320 | } 321 | return err 322 | } 323 | 324 | // Check for TLS Support 325 | func (a *RouterAPI) supportTLS(w http.ResponseWriter, r *http.Request) error { 326 | var err error 327 | vars := mux.Vars(r) 328 | svc, err := a.router(r.Context(), vars["mode"], r.Header) 329 | if err != nil { 330 | return err 331 | } 332 | 333 | headerOpts := r.Header.Values("X-Router-Opt") 334 | for _, opt := range headerOpts { 335 | if opt == "http-only=true" { 336 | w.WriteHeader(http.StatusNotFound) 337 | fmt.Fprintf(w, "No TLS Capabilities, disabled via header") 338 | return nil 339 | } 340 | } 341 | 342 | _, ok := svc.(router.RouterTLS) 343 | if !ok { 344 | w.WriteHeader(http.StatusNotFound) 345 | fmt.Fprintf(w, "No TLS Capabilities") 346 | return nil 347 | } 348 | fmt.Fprintf(w, "OK") 349 | return nil 350 | } 351 | 352 | func checkPath(ctx context.Context, path string, svc router.Router, instance router.InstanceID) ([]urlCheck, error) { 353 | if path == "" { 354 | return nil, nil 355 | } 356 | 357 | addrs, err := svc.GetAddresses(ctx, instance) 358 | if err != nil { 359 | return nil, err 360 | } 361 | 362 | wg := sync.WaitGroup{} 363 | checks := make(chan urlCheck, len(addrs)) 364 | for _, addr := range addrs { 365 | wg.Add(1) 366 | go func(addr string) { 367 | defer wg.Done() 368 | check := urlCheck{ 369 | Address: addr, 370 | } 371 | 372 | url := fmt.Sprintf("%s/%s", strings.TrimSuffix(addr, "/"), strings.TrimPrefix(path, "/")) 373 | if !httpSchemeRegex.MatchString(url) { 374 | url = "http://" + url 375 | } 376 | 377 | ctxWithTimeout, cancel := context.WithTimeout(ctx, checkPathTimeout) 378 | defer cancel() 379 | req, err := http.NewRequestWithContext(ctxWithTimeout, http.MethodGet, url, nil) 380 | if err != nil { 381 | check.Error = err.Error() 382 | checks <- check 383 | return 384 | } 385 | rsp, err := http.DefaultClient.Do(req) 386 | if err != nil { 387 | check.Error = err.Error() 388 | checks <- check 389 | return 390 | } 391 | check.Status = rsp.StatusCode 392 | checks <- check 393 | }(addr) 394 | } 395 | 396 | wg.Wait() 397 | close(checks) 398 | 399 | var ret []urlCheck 400 | for check := range checks { 401 | ret = append(ret, check) 402 | } 403 | return ret, nil 404 | } 405 | -------------------------------------------------------------------------------- /api/api_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package api 6 | 7 | import ( 8 | "bytes" 9 | "encoding/json" 10 | "io" 11 | "net/http" 12 | "net/http/httptest" 13 | "sort" 14 | "testing" 15 | 16 | "github.com/stretchr/testify/suite" 17 | "github.com/tsuru/kubernetes-router/backend" 18 | "github.com/tsuru/kubernetes-router/router" 19 | "github.com/tsuru/kubernetes-router/router/mock" 20 | ) 21 | 22 | type RouterAPISuite struct { 23 | suite.Suite 24 | 25 | api *RouterAPI 26 | mockRouter *mock.RouterMock 27 | handler http.Handler 28 | } 29 | 30 | func TestRouterAPISuite(t *testing.T) { 31 | suite.Run(t, &RouterAPISuite{}) 32 | } 33 | 34 | func (s *RouterAPISuite) SetupTest() { 35 | s.mockRouter = &mock.RouterMock{} 36 | s.api = &RouterAPI{ 37 | Backend: &backend.LocalCluster{ 38 | DefaultMode: "mymode", 39 | Routers: map[string]router.Router{ 40 | "mymode": s.mockRouter, 41 | }, 42 | }, 43 | } 44 | s.handler = s.api.Routes() 45 | } 46 | 47 | func (s *RouterAPISuite) TestHealthcheckOK() { 48 | req := httptest.NewRequest("GET", "http://localhost", nil) 49 | w := httptest.NewRecorder() 50 | 51 | s.api.Healthcheck(w, req) 52 | 53 | resp := w.Result() 54 | body, _ := io.ReadAll(resp.Body) 55 | 56 | s.Equal(http.StatusOK, resp.StatusCode) 57 | s.Equal("WORKING", string(body)) 58 | } 59 | 60 | func (s *RouterAPISuite) TestGetBackend() { 61 | s.mockRouter.GetAddressesFn = func(id router.InstanceID) ([]string, error) { 62 | s.Assert().Equal("myapp", id.AppName) 63 | return []string{"myapp"}, nil 64 | } 65 | req := httptest.NewRequest(http.MethodGet, "http://localhost/api/backend/myapp", nil) 66 | w := httptest.NewRecorder() 67 | 68 | s.handler.ServeHTTP(w, req) 69 | resp := w.Result() 70 | s.Equal(http.StatusOK, resp.StatusCode) 71 | s.True(s.mockRouter.GetAddressesInvoked) 72 | var data map[string]interface{} 73 | err := json.Unmarshal(w.Body.Bytes(), &data) 74 | s.NoError(err) 75 | s.Equal(map[string]interface{}{ 76 | "address": "myapp", 77 | "addresses": []interface{}{"myapp"}, 78 | }, data) 79 | } 80 | 81 | func (s *RouterAPISuite) TestGetBackendExplicitMode() { 82 | mockRouter := &mock.RouterMock{} 83 | api := RouterAPI{ 84 | Backend: &backend.LocalCluster{ 85 | DefaultMode: "xyz", 86 | Routers: map[string]router.Router{"mymode": mockRouter}, 87 | }, 88 | } 89 | r := api.Routes() 90 | mockRouter.GetAddressesFn = func(id router.InstanceID) ([]string, error) { 91 | s.Equal("myapp", id.AppName) 92 | return []string{"myapp"}, nil 93 | } 94 | req := httptest.NewRequest(http.MethodGet, "http://localhost/api/mymode/backend/myapp", nil) 95 | w := httptest.NewRecorder() 96 | 97 | r.ServeHTTP(w, req) 98 | resp := w.Result() 99 | s.Equal(http.StatusOK, resp.StatusCode) 100 | s.True(mockRouter.GetAddressesInvoked) 101 | } 102 | 103 | func (s *RouterAPISuite) TestGetBackendInvalidMode() { 104 | req := httptest.NewRequest(http.MethodGet, "http://localhost/api/othermode/backend/myapp", nil) 105 | w := httptest.NewRecorder() 106 | 107 | s.handler.ServeHTTP(w, req) 108 | resp := w.Result() 109 | s.Equal(http.StatusNotFound, resp.StatusCode) 110 | } 111 | 112 | func (s *RouterAPISuite) TestEnsureBackend() { 113 | s.mockRouter.EnsureFn = func(id router.InstanceID, o router.EnsureBackendOpts) error { 114 | s.Equal("myapp", id.AppName) 115 | s.Equal([]router.BackendPrefix{ 116 | { 117 | Target: router.BackendTarget{Namespace: "tsuru", Service: "myapp-web"}, 118 | }, 119 | }, o.Prefixes) 120 | s.Equal("mypool", o.Opts.Pool) 121 | s.Equal("443", o.Opts.ExposedPort) 122 | s.Equal(map[string]string{"custom": "val"}, o.Opts.AdditionalOpts) 123 | return nil 124 | } 125 | 126 | reqData, _ := json.Marshal( 127 | map[string]interface{}{ 128 | "opts": map[string]interface{}{ 129 | "tsuru.io/app-pool": "mypool", 130 | "exposed-port": "443", 131 | "custom": "val", 132 | }, 133 | "prefixes": []map[string]interface{}{ 134 | { 135 | "prefix": "", 136 | "target": map[string]string{ 137 | "service": "myapp-web", 138 | "namespace": "tsuru", 139 | }, 140 | }, 141 | }, 142 | }) 143 | body := bytes.NewReader(reqData) 144 | req := httptest.NewRequest(http.MethodPut, "http://localhost/api/backend/myapp", body) 145 | w := httptest.NewRecorder() 146 | 147 | s.handler.ServeHTTP(w, req) 148 | resp := w.Result() 149 | s.Equal(http.StatusOK, resp.StatusCode) 150 | s.True(s.mockRouter.EnsureInvoked) 151 | } 152 | 153 | func (s *RouterAPISuite) TestEnsureBackendWithHeaderOpts() { 154 | s.mockRouter.EnsureFn = func(id router.InstanceID, o router.EnsureBackendOpts) error { 155 | s.Equal("myapp", id.AppName) 156 | s.Equal([]router.BackendPrefix{ 157 | {Prefix: "", Target: router.BackendTarget{Namespace: "tsuru", Service: "myapp-web"}}, 158 | }, o.Prefixes) 159 | s.Equal("mypool", o.Opts.Pool) 160 | s.Equal("443", o.Opts.ExposedPort) 161 | s.Equal("a.b", o.Opts.Domain) 162 | s.Equal("test.io", o.Opts.DomainSuffix) 163 | expectedAdditional := map[string]string{"custom": "val", "custom2": "val2"} 164 | s.Equal(expectedAdditional, o.Opts.AdditionalOpts) 165 | 166 | return nil 167 | } 168 | 169 | reqData, _ := json.Marshal( 170 | map[string]interface{}{ 171 | "opts": map[string]interface{}{ 172 | "tsuru.io/app-pool": "mypool", 173 | "exposed-port": "443", 174 | "custom": "val", 175 | }, 176 | "prefixes": []map[string]interface{}{ 177 | { 178 | "prefix": "", 179 | "target": map[string]string{ 180 | "service": "myapp-web", 181 | "namespace": "tsuru", 182 | }, 183 | }, 184 | }, 185 | }) 186 | body := bytes.NewReader(reqData) 187 | req := httptest.NewRequest(http.MethodPut, "http://localhost/api/backend/myapp", body) 188 | req.Header.Add("X-Router-Opt", "domain=a.b") 189 | req.Header.Add("X-Router-Opt", "domain-suffix=test.io") 190 | req.Header.Add("X-Router-Opt", "custom2=val2") 191 | w := httptest.NewRecorder() 192 | 193 | s.handler.ServeHTTP(w, req) 194 | resp := w.Result() 195 | s.Equal(http.StatusOK, resp.StatusCode) 196 | s.True(s.mockRouter.EnsureInvoked) 197 | } 198 | 199 | func (s *RouterAPISuite) TestRemoveBackend() { 200 | s.mockRouter.RemoveFn = func(id router.InstanceID) error { 201 | s.Equal("myapp", id.AppName) 202 | return nil 203 | } 204 | 205 | req := httptest.NewRequest(http.MethodDelete, "http://localhost/api/backend/myapp", nil) 206 | w := httptest.NewRecorder() 207 | 208 | s.handler.ServeHTTP(w, req) 209 | resp := w.Result() 210 | s.Equal(http.StatusOK, resp.StatusCode) 211 | 212 | if !s.mockRouter.RemoveInvoked { 213 | s.Fail("Service Remove function not invoked") 214 | } 215 | } 216 | 217 | func (s *RouterAPISuite) TestInfo() { 218 | s.mockRouter.SupportedOptionsFn = func() map[string]string { 219 | return map[string]string{router.ExposedPort: "", router.Domain: "Custom help."} 220 | } 221 | 222 | req := httptest.NewRequest(http.MethodGet, "http://localhost/api/info", nil) 223 | w := httptest.NewRecorder() 224 | 225 | s.handler.ServeHTTP(w, req) 226 | resp := w.Result() 227 | s.Equal(http.StatusOK, resp.StatusCode) 228 | 229 | var info map[string]string 230 | err := json.Unmarshal(w.Body.Bytes(), &info) 231 | s.Require().NoError(err) 232 | 233 | expected := map[string]string{ 234 | "exposed-port": "Port to be exposed by the Load Balancer. Defaults to 80.", 235 | "domain": "Custom help.", 236 | } 237 | s.Equal(expected, info) 238 | } 239 | 240 | func (s *RouterAPISuite) TestGetRoutes() { 241 | req := httptest.NewRequest(http.MethodGet, "http://localhost/api/backend/myapp/routes", nil) 242 | w := httptest.NewRecorder() 243 | 244 | s.handler.ServeHTTP(w, req) 245 | resp := w.Result() 246 | s.Equal(http.StatusOK, resp.StatusCode) 247 | var data map[string][]string 248 | err := json.Unmarshal(w.Body.Bytes(), &data) 249 | s.Require().NoError(err) 250 | expected := map[string][]string{"addresses": nil} 251 | s.Equal(expected, data) 252 | } 253 | 254 | func (s *RouterAPISuite) TestAddCertificate() { 255 | certExpected := router.CertData{Certificate: "Certz", Key: "keyz"} 256 | 257 | s.mockRouter.AddCertificateFn = func(id router.InstanceID, certName string, cert router.CertData) error { 258 | s.Require().Equal(certExpected, cert) 259 | return nil 260 | } 261 | 262 | reqData, _ := json.Marshal(certExpected) 263 | body := bytes.NewReader(reqData) 264 | 265 | req := httptest.NewRequest(http.MethodPut, "http://localhost/api/backend/myapp/certificate/certname", body) 266 | w := httptest.NewRecorder() 267 | 268 | s.handler.ServeHTTP(w, req) 269 | resp := w.Result() 270 | s.Equal(http.StatusOK, resp.StatusCode) 271 | if !s.mockRouter.AddCertificateInvoked { 272 | s.Fail("Service Addresses function not invoked") 273 | } 274 | } 275 | 276 | func (s *RouterAPISuite) TestGetCertificate() { 277 | s.mockRouter.GetCertificateFn = func(id router.InstanceID, certName string) (*router.CertData, error) { 278 | cert := router.CertData{Certificate: "Certz", Key: "keyz"} 279 | return &cert, nil 280 | } 281 | 282 | req := httptest.NewRequest(http.MethodGet, "http://localhost/api/backend/myapp/certificate/certname", nil) 283 | w := httptest.NewRecorder() 284 | 285 | s.handler.ServeHTTP(w, req) 286 | resp := w.Result() 287 | s.Equal(http.StatusOK, resp.StatusCode) 288 | 289 | if !s.mockRouter.GetCertificateInvoked { 290 | s.Fail("Service Addresses function not invoked") 291 | } 292 | var data router.CertData 293 | err := json.Unmarshal(w.Body.Bytes(), &data) 294 | s.Require().NoError(err) 295 | 296 | expected := router.CertData{Certificate: "Certz", Key: "keyz"} 297 | s.Equal(expected, data) 298 | } 299 | 300 | func (s *RouterAPISuite) TestRemoveCertificate() { 301 | s.mockRouter.RemoveCertificateFn = func(id router.InstanceID, certName string) error { 302 | return nil 303 | } 304 | 305 | req := httptest.NewRequest(http.MethodDelete, "http://localhost/api/backend/myapp/certificate/certname", nil) 306 | w := httptest.NewRecorder() 307 | 308 | s.handler.ServeHTTP(w, req) 309 | resp := w.Result() 310 | s.Equal(http.StatusOK, resp.StatusCode) 311 | 312 | if !s.mockRouter.RemoveCertificateInvoked { 313 | s.Fail("Service Addresses function not invoked") 314 | } 315 | } 316 | 317 | func (s *RouterAPISuite) TestGetBackendStatus() { 318 | s.mockRouter.GetStatusFn = func(id router.InstanceID) (router.BackendStatus, string, error) { 319 | s.Assert().Equal("myapp", id.AppName) 320 | return router.BackendStatusReady, "xyz", nil 321 | } 322 | req := httptest.NewRequest(http.MethodGet, "http://localhost/api/backend/myapp/status", nil) 323 | w := httptest.NewRecorder() 324 | 325 | s.handler.ServeHTTP(w, req) 326 | resp := w.Result() 327 | s.Equal(http.StatusOK, resp.StatusCode) 328 | s.False(s.mockRouter.GetAddressesInvoked) 329 | s.True(s.mockRouter.GetStatusInvoked) 330 | 331 | expectedStatus := statusResp{ 332 | Status: "ready", 333 | Detail: "xyz", 334 | } 335 | 336 | var data statusResp 337 | err := json.Unmarshal(w.Body.Bytes(), &data) 338 | s.NoError(err) 339 | s.Equal(expectedStatus, data) 340 | } 341 | 342 | func (s *RouterAPISuite) TestGetBackendStatusWithCheckPath() { 343 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 344 | w.WriteHeader(209) 345 | })) 346 | defer srv.Close() 347 | 348 | s.mockRouter.GetAddressesFn = func(id router.InstanceID) ([]string, error) { 349 | s.Assert().Equal("myapp", id.AppName) 350 | return []string{srv.URL, srv.URL + "/x"}, nil 351 | } 352 | s.mockRouter.GetStatusFn = func(id router.InstanceID) (router.BackendStatus, string, error) { 353 | s.Assert().Equal("myapp", id.AppName) 354 | return router.BackendStatusReady, "xyz", nil 355 | } 356 | req := httptest.NewRequest(http.MethodGet, "http://localhost/api/backend/myapp/status?checkpath=/", nil) 357 | w := httptest.NewRecorder() 358 | 359 | s.handler.ServeHTTP(w, req) 360 | resp := w.Result() 361 | s.Equal(http.StatusOK, resp.StatusCode) 362 | s.True(s.mockRouter.GetAddressesInvoked) 363 | s.True(s.mockRouter.GetStatusInvoked) 364 | 365 | expectedStatus := statusResp{ 366 | Status: "ready", 367 | Detail: "xyz", 368 | Checks: []urlCheck{ 369 | { 370 | Address: srv.URL, 371 | Status: 209, 372 | }, 373 | { 374 | Address: srv.URL + "/x", 375 | Status: 209, 376 | }, 377 | }, 378 | } 379 | 380 | var parsedRsp statusResp 381 | err := json.Unmarshal(w.Body.Bytes(), &parsedRsp) 382 | s.NoError(err) 383 | 384 | sort.Slice(parsedRsp.Checks, func(i, j int) bool { 385 | return parsedRsp.Checks[i].Address < parsedRsp.Checks[j].Address 386 | }) 387 | 388 | s.Equal(expectedStatus, parsedRsp) 389 | } 390 | -------------------------------------------------------------------------------- /api/handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package api 6 | 7 | import ( 8 | "log" 9 | "net/http" 10 | 11 | "github.com/tsuru/kubernetes-router/router" 12 | ) 13 | 14 | type httpError struct { 15 | Body string 16 | Status int 17 | } 18 | 19 | func (h httpError) Error() string { 20 | return h.Body 21 | } 22 | 23 | type handler func(http.ResponseWriter, *http.Request) error 24 | 25 | // ServeHTTP serves an HTTP request 26 | func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 27 | handleError(h(w, r), w, r) 28 | } 29 | 30 | func handleError(err error, w http.ResponseWriter, r *http.Request) { 31 | if err != nil { 32 | log.Printf("error during request %v %v: %v", r.Method, r.URL.Path, err) 33 | if httpErr, ok := err.(httpError); ok { 34 | http.Error(w, httpErr.Error(), httpErr.Status) 35 | return 36 | } 37 | if err == router.ErrIngressAlreadyExists { 38 | http.Error(w, err.Error(), http.StatusConflict) 39 | return 40 | } 41 | http.Error(w, err.Error(), http.StatusInternalServerError) 42 | } 43 | } 44 | 45 | // AuthMiddleware is an http.Handler with Basic Auth 46 | type AuthMiddleware struct { 47 | User string 48 | Pass string 49 | } 50 | 51 | // ServeHTTP serves an HTTP request with Basic Auth 52 | func (h AuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { 53 | if h.User == "" && h.Pass == "" { 54 | next(w, r) 55 | return 56 | } 57 | rUser, rPass, _ := r.BasicAuth() 58 | 59 | if rUser != h.User || rPass != h.Pass { 60 | w.Header().Set("WWW-Authenticate", "Basic realm=\"Authorization Required\"") 61 | http.Error(w, "Not Authorized", http.StatusUnauthorized) 62 | return 63 | } 64 | next(w, r) 65 | } 66 | -------------------------------------------------------------------------------- /api/handler_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package api 6 | 7 | import ( 8 | "errors" 9 | "net/http" 10 | "net/http/httptest" 11 | "testing" 12 | ) 13 | 14 | func TestHandler(t *testing.T) { 15 | tt := []struct { 16 | name string 17 | err error 18 | expectedBody string 19 | expectedStatus int 20 | }{ 21 | {"withoutError", nil, "", http.StatusOK}, 22 | {"withGenericError", errors.New("internal error"), "internal error\n", http.StatusInternalServerError}, 23 | } 24 | for _, tc := range tt { 25 | tc := tc 26 | t.Run(tc.name, func(t *testing.T) { 27 | myHandler := func(w http.ResponseWriter, r *http.Request) error { 28 | return tc.err 29 | } 30 | req := httptest.NewRequest(http.MethodGet, "http://localhost", nil) 31 | w := httptest.NewRecorder() 32 | wrapped := handler(myHandler) 33 | wrapped.ServeHTTP(w, req) 34 | 35 | response := w.Result() 36 | 37 | if w.Body.String() != tc.expectedBody { 38 | t.Errorf("Expected body to be %q. Got %q", tc.expectedBody, w.Body.String()) 39 | } 40 | if response.StatusCode != tc.expectedStatus { 41 | t.Errorf("Expected status %d. Got %d", tc.expectedStatus, response.StatusCode) 42 | } 43 | }) 44 | } 45 | } 46 | 47 | func TestAuthHandler(t *testing.T) { 48 | h := AuthMiddleware{"user", "god"} 49 | tt := []struct { 50 | name string 51 | user string 52 | password string 53 | expectedStatus int 54 | }{ 55 | {"rightCredentials", "user", "god", http.StatusOK}, 56 | {"wrongCredentials", "bla", "wrong", http.StatusUnauthorized}, 57 | } 58 | for _, tc := range tt { 59 | tc := tc 60 | t.Run(tc.name, func(t *testing.T) { 61 | req := httptest.NewRequest(http.MethodGet, "http://localhost", nil) 62 | req.SetBasicAuth(tc.user, tc.password) 63 | w := httptest.NewRecorder() 64 | h.ServeHTTP(w, req, func(http.ResponseWriter, *http.Request) { 65 | }) 66 | 67 | response := w.Result() 68 | 69 | if response.StatusCode != tc.expectedStatus { 70 | t.Errorf("Expected status %d. Got %d", tc.expectedStatus, response.StatusCode) 71 | } 72 | }) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /backend/backend.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package backend 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "net/http" 11 | 12 | "github.com/tsuru/kubernetes-router/router" 13 | ) 14 | 15 | var ( 16 | ErrBackendNotFound = errors.New("Backend not found") 17 | ) 18 | 19 | type Backend interface { 20 | Router(ctx context.Context, mode string, header http.Header) (router.Router, error) 21 | Healthcheck(ctx context.Context) error 22 | } 23 | -------------------------------------------------------------------------------- /backend/local-cluster.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | package backend 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "net/http" 10 | "strings" 11 | 12 | "github.com/tsuru/kubernetes-router/router" 13 | ) 14 | 15 | var _ Backend = &LocalCluster{} 16 | 17 | type LocalCluster struct { 18 | DefaultMode string 19 | Routers map[string]router.Router 20 | } 21 | 22 | func (m *LocalCluster) Router(ctx context.Context, mode string, _ http.Header) (router.Router, error) { 23 | if mode == "" { 24 | mode = m.DefaultMode 25 | } 26 | svc, ok := m.Routers[mode] 27 | if !ok { 28 | return nil, ErrBackendNotFound 29 | } 30 | return svc, nil 31 | } 32 | 33 | func (m *LocalCluster) Healthcheck(ctx context.Context) error { 34 | errAccumulator := &multiRoutersErrors{} 35 | 36 | for mode, svc := range m.Routers { 37 | if hc, ok := svc.(router.HealthcheckableRouter); ok { 38 | if err := hc.Healthcheck(); err != nil { 39 | errAccumulator.errors = append(errAccumulator.errors, fmt.Sprintf("failed to check IngressService %v: %v", mode, err)) 40 | } 41 | } 42 | } 43 | 44 | if len(errAccumulator.errors) > 0 { 45 | return errAccumulator 46 | } 47 | 48 | return nil 49 | } 50 | 51 | type multiRoutersErrors struct { 52 | errors []string 53 | } 54 | 55 | func (m *multiRoutersErrors) Error() string { 56 | return strings.Join(m.errors, " - ") 57 | } 58 | -------------------------------------------------------------------------------- /backend/multi-cluster.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | package backend 5 | 6 | import ( 7 | "context" 8 | "encoding/base64" 9 | "errors" 10 | "net/http" 11 | "time" 12 | 13 | "github.com/opentracing/opentracing-go" 14 | "github.com/tsuru/kubernetes-router/kubernetes" 15 | "github.com/tsuru/kubernetes-router/observability" 16 | "github.com/tsuru/kubernetes-router/router" 17 | kubernetesGO "k8s.io/client-go/kubernetes" 18 | "k8s.io/client-go/rest" 19 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 20 | "k8s.io/client-go/transport" 21 | 22 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 23 | ) 24 | 25 | var _ Backend = &MultiCluster{} 26 | 27 | type ClusterConfig struct { 28 | Name string `json:"name"` 29 | Default bool `json:"default"` 30 | Address string `json:"address"` 31 | Token string `json:"token"` 32 | CA string `json:"ca"` 33 | 34 | AuthProvider *clientcmdapi.AuthProviderConfig `json:"authProvider"` 35 | Exec *clientcmdapi.ExecConfig `json:"exec"` 36 | } 37 | 38 | type ClustersFile struct { 39 | Clusters []ClusterConfig `json:"clusters"` 40 | } 41 | 42 | type MultiCluster struct { 43 | Namespace string 44 | Fallback Backend 45 | K8sTimeout *time.Duration 46 | Modes []string 47 | Clusters []ClusterConfig 48 | } 49 | 50 | func (m *MultiCluster) Router(ctx context.Context, mode string, headers http.Header) (router.Router, error) { 51 | name := headers.Get("X-Tsuru-Cluster-Name") 52 | address := headers.Get("X-Tsuru-Cluster-Addresses") 53 | 54 | if address == "" { 55 | return m.Fallback.Router(ctx, mode, headers) 56 | } 57 | 58 | span := opentracing.SpanFromContext(ctx) 59 | if span != nil { 60 | span.SetTag("cluster.name", name) 61 | span.SetTag("cluster.address", address) 62 | } 63 | 64 | timeout := time.Second * 10 65 | if m.K8sTimeout != nil { 66 | timeout = *m.K8sTimeout 67 | } 68 | 69 | kubernetesRestConfig, err := m.getKubeConfig(name, address, timeout) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | k8sClient, err := kubernetesGO.NewForConfig(kubernetesRestConfig) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | baseService := &kubernetes.BaseService{ 80 | Namespace: m.Namespace, 81 | Timeout: timeout, 82 | Client: k8sClient, 83 | RestConfig: kubernetesRestConfig, 84 | } 85 | 86 | if mode == "service" || mode == "loadbalancer" || mode == "" { 87 | return &kubernetes.LBService{ 88 | BaseService: baseService, 89 | }, nil 90 | } 91 | 92 | if mode == "ingress" { 93 | return &kubernetes.IngressService{ 94 | BaseService: baseService, 95 | }, nil 96 | } 97 | 98 | if mode == "nginx-ingress" { 99 | return &kubernetes.IngressService{ 100 | BaseService: baseService, 101 | IngressClass: "nginx", 102 | AnnotationsPrefix: "nginx.ingress.kubernetes.io", 103 | // TODO(wpjunior): may router opts plugged in here ? 104 | }, nil 105 | } 106 | 107 | if mode == "istio-gateway" { 108 | return &kubernetes.IstioGateway{ 109 | BaseService: baseService, 110 | }, nil 111 | } 112 | 113 | return nil, errors.New("Mode not found") 114 | } 115 | 116 | func (m *MultiCluster) Healthcheck(ctx context.Context) error { 117 | return m.Fallback.Healthcheck(ctx) 118 | } 119 | 120 | func (m *MultiCluster) getKubeConfig(name, address string, timeout time.Duration) (*rest.Config, error) { 121 | selectedCluster := ClusterConfig{} 122 | 123 | for _, cluster := range m.Clusters { 124 | if cluster.Default { 125 | selectedCluster = cluster 126 | } 127 | if cluster.Name == name { 128 | selectedCluster = cluster 129 | break 130 | } 131 | } 132 | 133 | if selectedCluster.Name == "" { 134 | return nil, errors.New("cluster not found") 135 | } 136 | 137 | if selectedCluster.Address != "" { 138 | address = selectedCluster.Address 139 | } 140 | 141 | restConfig := &rest.Config{ 142 | Host: address, 143 | BearerToken: selectedCluster.Token, 144 | Timeout: timeout, 145 | WrapTransport: func(rt http.RoundTripper) http.RoundTripper { 146 | return transport.DebugWrappers(observability.WrapTransport(rt)) 147 | }, 148 | } 149 | 150 | if selectedCluster.Exec != nil && selectedCluster.AuthProvider != nil { 151 | return nil, errors.New("both exec and authProvider mutually exclusive are set in the cluster config") 152 | } 153 | 154 | if selectedCluster.AuthProvider != nil { 155 | restConfig.AuthProvider = selectedCluster.AuthProvider 156 | } 157 | 158 | if selectedCluster.Exec != nil { 159 | restConfig.ExecProvider = selectedCluster.Exec 160 | restConfig.ExecProvider.InteractiveMode = "Never" 161 | } 162 | 163 | if selectedCluster.CA != "" { 164 | caData, err := base64.StdEncoding.DecodeString(selectedCluster.CA) 165 | if err != nil { 166 | return nil, err 167 | } 168 | restConfig.TLSClientConfig.CAData = caData 169 | } 170 | 171 | return restConfig, nil 172 | 173 | } 174 | -------------------------------------------------------------------------------- /backend/multi-cluster_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package backend 6 | 7 | import ( 8 | "context" 9 | "encoding/base64" 10 | "errors" 11 | "net/http" 12 | "testing" 13 | "time" 14 | 15 | "github.com/opentracing/opentracing-go" 16 | "github.com/opentracing/opentracing-go/mocktracer" 17 | "github.com/stretchr/testify/assert" 18 | "github.com/stretchr/testify/require" 19 | "github.com/tsuru/kubernetes-router/kubernetes" 20 | "github.com/tsuru/kubernetes-router/router" 21 | restclient "k8s.io/client-go/rest" 22 | "k8s.io/client-go/tools/clientcmd/api" 23 | ) 24 | 25 | var _ Backend = &fakeBackend{} 26 | var ctx = context.TODO() 27 | 28 | type fakeBackend struct{} 29 | 30 | func (*fakeBackend) Router(ctx context.Context, mode string, headers http.Header) (router.Router, error) { 31 | return nil, errors.New("not implemented yet") 32 | } 33 | func (*fakeBackend) Healthcheck(context.Context) error { 34 | return errors.New("not implemented yet") 35 | } 36 | 37 | func TestMultiClusterFallback(t *testing.T) { 38 | backend := &MultiCluster{ 39 | Namespace: "tsuru-test", 40 | Fallback: &fakeBackend{}, 41 | } 42 | router, err := backend.Router(ctx, "service", http.Header{}) 43 | if assert.Error(t, err) { 44 | assert.Equal(t, err.Error(), "not implemented yet") 45 | } 46 | assert.Nil(t, router) 47 | } 48 | 49 | func TestMultiClusterHealthcheck(t *testing.T) { 50 | backend := &MultiCluster{ 51 | Namespace: "tsuru-test", 52 | Fallback: &fakeBackend{}, 53 | } 54 | err := backend.Healthcheck(context.TODO()) 55 | if assert.Error(t, err) { 56 | assert.Equal(t, err.Error(), "not implemented yet") 57 | } 58 | } 59 | 60 | func TestMultiClusterService(t *testing.T) { 61 | backend := &MultiCluster{ 62 | Namespace: "tsuru-test", 63 | Fallback: &fakeBackend{}, 64 | Clusters: []ClusterConfig{ 65 | { 66 | Name: "my-cluster", 67 | Token: "my-token", 68 | }, 69 | }, 70 | } 71 | mockTracer := mocktracer.New() 72 | span := mockTracer.StartSpan("test") 73 | spanCtx := opentracing.ContextWithSpan(ctx, span) 74 | router, err := backend.Router(spanCtx, "service", http.Header{ 75 | "X-Tsuru-Cluster-Name": { 76 | "my-cluster", 77 | }, 78 | "X-Tsuru-Cluster-Addresses": { 79 | "https://mycluster.com", 80 | }, 81 | }) 82 | assert.NoError(t, err) 83 | lbService, ok := router.(*kubernetes.LBService) 84 | require.True(t, ok) 85 | assert.Equal(t, "tsuru-test", lbService.BaseService.Namespace) 86 | assert.Equal(t, 10*time.Second, lbService.BaseService.Timeout) 87 | assert.Equal(t, "https://mycluster.com", lbService.BaseService.RestConfig.Host) 88 | assert.Equal(t, "my-token", lbService.BaseService.RestConfig.BearerToken) 89 | assert.Equal(t, span.(*mocktracer.MockSpan).Tags(), map[string]interface{}{ 90 | "cluster.address": "https://mycluster.com", 91 | "cluster.name": "my-cluster", 92 | }) 93 | } 94 | 95 | type dummyAuthProvider struct{} 96 | 97 | func (d *dummyAuthProvider) WrapTransport(rt http.RoundTripper) http.RoundTripper { 98 | return rt 99 | } 100 | 101 | func (d *dummyAuthProvider) Login() error { 102 | return nil 103 | } 104 | 105 | func TestMultiClusterAuthProvider(t *testing.T) { 106 | newDummyProvider := func(clusterAddress string, cfg map[string]string, persister restclient.AuthProviderConfigPersister) (restclient.AuthProvider, error) { 107 | return &dummyAuthProvider{}, nil 108 | } 109 | 110 | err := restclient.RegisterAuthProviderPlugin("dummy-test", newDummyProvider) 111 | require.NoError(t, err) 112 | 113 | backend := &MultiCluster{ 114 | Namespace: "tsuru-test", 115 | Fallback: &fakeBackend{}, 116 | Clusters: []ClusterConfig{ 117 | { 118 | Name: "my-cluster", 119 | Address: "https://example.org", 120 | AuthProvider: &api.AuthProviderConfig{Name: "dummy-test"}, 121 | }, 122 | }, 123 | } 124 | mockTracer := mocktracer.New() 125 | span := mockTracer.StartSpan("test") 126 | spanCtx := opentracing.ContextWithSpan(ctx, span) 127 | router, err := backend.Router(spanCtx, "service", http.Header{ 128 | "X-Tsuru-Cluster-Name": { 129 | "my-cluster", 130 | }, 131 | "X-Tsuru-Cluster-Addresses": { 132 | "https://mycluster.com", 133 | }, 134 | }) 135 | assert.NoError(t, err) 136 | lbService, ok := router.(*kubernetes.LBService) 137 | require.True(t, ok) 138 | assert.Equal(t, "tsuru-test", lbService.BaseService.Namespace) 139 | assert.Equal(t, 10*time.Second, lbService.BaseService.Timeout) 140 | assert.Equal(t, "https://example.org", lbService.BaseService.RestConfig.Host) 141 | assert.Equal(t, "dummy-test", lbService.BaseService.RestConfig.AuthProvider.Name) 142 | assert.Equal(t, span.(*mocktracer.MockSpan).Tags(), map[string]interface{}{ 143 | "cluster.address": "https://mycluster.com", 144 | "cluster.name": "my-cluster", 145 | }) 146 | } 147 | 148 | func TestMultiClusterExecProvider(t *testing.T) { 149 | backend := &MultiCluster{ 150 | Namespace: "tsuru-test", 151 | Fallback: &fakeBackend{}, 152 | Clusters: []ClusterConfig{ 153 | { 154 | Name: "my-cluster", 155 | Address: "https://example.org", 156 | Exec: &api.ExecConfig{ 157 | APIVersion: "client.authentication.k8s.io/v1beta1", 158 | Command: "echo", 159 | Args: []string{"arg1", "arg2"}, 160 | }, 161 | }, 162 | }, 163 | } 164 | mockTracer := mocktracer.New() 165 | span := mockTracer.StartSpan("test") 166 | spanCtx := opentracing.ContextWithSpan(ctx, span) 167 | router, err := backend.Router(spanCtx, "service", http.Header{ 168 | "X-Tsuru-Cluster-Name": { 169 | "my-cluster", 170 | }, 171 | "X-Tsuru-Cluster-Addresses": { 172 | "https://mycluster.com", 173 | }, 174 | }) 175 | assert.NoError(t, err) 176 | lbService, ok := router.(*kubernetes.LBService) 177 | require.True(t, ok) 178 | assert.Equal(t, "tsuru-test", lbService.BaseService.Namespace) 179 | assert.Equal(t, 10*time.Second, lbService.BaseService.Timeout) 180 | assert.Equal(t, "https://example.org", lbService.BaseService.RestConfig.Host) 181 | assert.Equal(t, "echo", lbService.BaseService.RestConfig.ExecProvider.Command) 182 | assert.Equal(t, []string{"arg1", "arg2"}, lbService.BaseService.RestConfig.ExecProvider.Args) 183 | assert.Equal(t, api.ExecInteractiveMode("Never"), lbService.BaseService.RestConfig.ExecProvider.InteractiveMode) 184 | assert.Equal(t, span.(*mocktracer.MockSpan).Tags(), map[string]interface{}{ 185 | "cluster.address": "https://mycluster.com", 186 | "cluster.name": "my-cluster", 187 | }) 188 | } 189 | 190 | func TestMultiClusterSetBothAuthMechanism(t *testing.T) { 191 | newDummyProvider := func(clusterAddress string, cfg map[string]string, persister restclient.AuthProviderConfigPersister) (restclient.AuthProvider, error) { 192 | return &dummyAuthProvider{}, nil 193 | } 194 | 195 | err := restclient.RegisterAuthProviderPlugin("dummy-test2", newDummyProvider) 196 | require.NoError(t, err) 197 | 198 | backend := &MultiCluster{ 199 | Namespace: "tsuru-test", 200 | Fallback: &fakeBackend{}, 201 | Clusters: []ClusterConfig{ 202 | { 203 | Name: "my-cluster", 204 | Address: "https://example.org", 205 | AuthProvider: &api.AuthProviderConfig{Name: "dummy-test2"}, 206 | Exec: &api.ExecConfig{ 207 | Command: "echo", 208 | }, 209 | }, 210 | }, 211 | } 212 | mockTracer := mocktracer.New() 213 | span := mockTracer.StartSpan("test") 214 | spanCtx := opentracing.ContextWithSpan(ctx, span) 215 | _, err = backend.Router(spanCtx, "service", http.Header{ 216 | "X-Tsuru-Cluster-Name": { 217 | "my-cluster", 218 | }, 219 | "X-Tsuru-Cluster-Addresses": { 220 | "https://mycluster.com", 221 | }, 222 | }) 223 | assert.Error(t, err, "both exec and authProvider mutually exclusive are set in the cluster config") 224 | } 225 | 226 | func TestMultiClusterCA(t *testing.T) { 227 | fakeCA := `-----BEGIN CERTIFICATE----- 228 | MIIGFDCCA/ygAwIBAgIIU+w77vuySF8wDQYJKoZIhvcNAQEFBQAwUTELMAkGA1UE 229 | BhMCRVMxQjBABgNVBAMMOUF1dG9yaWRhZCBkZSBDZXJ0aWZpY2FjaW9uIEZpcm1h 230 | cHJvZmVzaW9uYWwgQ0lGIEE2MjYzNDA2ODAeFw0wOTA1MjAwODM4MTVaFw0zMDEy 231 | MzEwODM4MTVaMFExCzAJBgNVBAYTAkVTMUIwQAYDVQQDDDlBdXRvcmlkYWQgZGUg 232 | Q2VydGlmaWNhY2lvbiBGaXJtYXByb2Zlc2lvbmFsIENJRiBBNjI2MzQwNjgwggIi 233 | MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKlmuO6vj78aI14H9M2uDDUtd9 234 | thDIAl6zQyrET2qyyhxdKJp4ERppWVevtSBC5IsP5t9bpgOSL/UR5GLXMnE42QQM 235 | cas9UX4PB99jBVzpv5RvwSmCwLTaUbDBPLutN0pcyvFLNg4kq7/DhHf9qFD0sefG 236 | L9ItWY16Ck6WaVICqjaY7Pz6FIMMNx/Jkjd/14Et5cS54D40/mf0PmbR0/RAz15i 237 | NA9wBj4gGFrO93IbJWyTdBSTo3OxDqqHECNZXyAFGUftaI6SEspd/NYrspI8IM/h 238 | X68gvqB2f3bl7BqGYTM+53u0P6APjqK5am+5hyZvQWyIplD9amML9ZMWGxmPsu2b 239 | m8mQ9QEM3xk9Dz44I8kvjwzRAv4bVdZO0I08r0+k8/6vKtMFnXkIoctXMbScyJCy 240 | Z/QYFpM6/EfY0XiWMR+6KwxfXZmtY4laJCB22N/9q06mIqqdXuYnin1oKaPnirja 241 | EbsXLZmdEyRG98Xi2J+Of8ePdG1asuhy9azuJBCtLxTa/y2aRnFHvkLfuwHb9H/T 242 | KI8xWVvTyQKmtFLKbpf7Q8UIJm+K9Lv9nyiqDdVF8xM6HdjAeI9BZzwelGSuewvF 243 | 6NkBiDkal4ZkQdU7hwxu+g/GvUgUvzlN1J5Bto+WHWOWk9mVBngxaJ43BjuAiUVh 244 | OSPHG0SjFeUc+JIwuwIDAQABo4HvMIHsMBIGA1UdEwEB/wQIMAYBAf8CAQEwDgYD 245 | VR0PAQH/BAQDAgEGMB0GA1UdDgQWBBRlzeurNR4APn7VdMActHNHDhpkLzCBpgYD 246 | VR0gBIGeMIGbMIGYBgRVHSAAMIGPMC8GCCsGAQUFBwIBFiNodHRwOi8vd3d3LmZp 247 | cm1hcHJvZmVzaW9uYWwuY29tL2NwczBcBggrBgEFBQcCAjBQHk4AUABhAHMAZQBv 248 | ACAAZABlACAAbABhACAAQgBvAG4AYQBuAG8AdgBhACAANAA3ACAAQgBhAHIAYwBl 249 | AGwAbwBuAGEAIAAwADgAMAAxADcwDQYJKoZIhvcNAQEFBQADggIBABd9oPm03cXF 250 | 661LJLWhAqvdpYhKsg9VSytXjDvlMd3+xDLx51tkljYyGOylMnfX40S2wBEqgLk9 251 | am58m9Ot/MPWo+ZkKXzR4Tgegiv/J2Wv+xYVxC5xhOW1//qkR71kMrv2JYSiJ0L1 252 | ILDCExARzRAVukKQKtJE4ZYm6zFIEv0q2skGz3QeqUvVhyj5eTSSPi5E6PaPT481 253 | PyWzOdxjKpBrIF/EUhJOlywqrJ2X3kjyo2bbwtKDlaZmp54lD+kLM5FlClrD2VQS 254 | 3a/DTg4fJl4N3LON7NWBcN7STyQF82xO9UxJZo3R/9ILJUFI/lGExkKvgATP0H5k 255 | SeTy36LssUzAKh3ntLFlosS88Zj0qnAHY7S42jtM+kAiMFsRpvAFDsYCA0irhpuF 256 | 3dvd6qJ2gHN99ZwExEWN57kci57q13XRcrHedUTnQn3iV2t93Jm8PYMo6oCTjcVM 257 | ZcFwgbg4/EMxsvYDNEeyrPsiBsse3RdHHF9mudMaotoRsaS8I8nkvof/uZS2+F0g 258 | StRf571oe2XyFR7SOqkt6dhrJKyXWERHrVkY8SFlcN7ONGCoQPHzPKTDKCOM/icz 259 | Q0CgFzzr6juwcqajuUpLXhZI9LK8yIySxZ2frHI2vDSANGupi5LAuBft7HZT9SQB 260 | jLMi6Et8Vcad+qMUu2WFbm5PEn4KPJ2V 261 | -----END CERTIFICATE-----` 262 | 263 | encodedFakeCA := base64.StdEncoding.EncodeToString([]byte(fakeCA)) 264 | 265 | backend := &MultiCluster{ 266 | Namespace: "tsuru-test", 267 | Fallback: &fakeBackend{}, 268 | Clusters: []ClusterConfig{ 269 | { 270 | Name: "my-cluster", 271 | Address: "https://example.org", 272 | CA: encodedFakeCA, 273 | }, 274 | }, 275 | } 276 | mockTracer := mocktracer.New() 277 | span := mockTracer.StartSpan("test") 278 | spanCtx := opentracing.ContextWithSpan(ctx, span) 279 | router, err := backend.Router(spanCtx, "service", http.Header{ 280 | "X-Tsuru-Cluster-Name": { 281 | "my-cluster", 282 | }, 283 | "X-Tsuru-Cluster-Addresses": { 284 | "https://mycluster.com", 285 | }, 286 | }) 287 | assert.NoError(t, err) 288 | lbService, ok := router.(*kubernetes.LBService) 289 | require.True(t, ok) 290 | assert.Equal(t, "tsuru-test", lbService.BaseService.Namespace) 291 | assert.Equal(t, 10*time.Second, lbService.BaseService.Timeout) 292 | assert.Equal(t, "https://example.org", lbService.BaseService.RestConfig.Host) 293 | assert.Equal(t, []byte(fakeCA), lbService.BaseService.RestConfig.TLSClientConfig.CAData) 294 | assert.Equal(t, span.(*mocktracer.MockSpan).Tags(), map[string]interface{}{ 295 | "cluster.address": "https://mycluster.com", 296 | "cluster.name": "my-cluster", 297 | }) 298 | } 299 | 300 | func TestMultiClusterIngress(t *testing.T) { 301 | backend := &MultiCluster{ 302 | Namespace: "tsuru-test", 303 | Fallback: &fakeBackend{}, 304 | Clusters: []ClusterConfig{ 305 | { 306 | Name: "default-token", 307 | Token: "my-token", 308 | Default: true, 309 | }, 310 | }, 311 | } 312 | router, err := backend.Router(ctx, "ingress", http.Header{ 313 | "X-Tsuru-Cluster-Name": { 314 | "my-cluster", 315 | }, 316 | "X-Tsuru-Cluster-Addresses": { 317 | "https://mycluster.com", 318 | }, 319 | }) 320 | assert.NoError(t, err) 321 | ingressService, ok := router.(*kubernetes.IngressService) 322 | require.True(t, ok) 323 | assert.Equal(t, "tsuru-test", ingressService.BaseService.Namespace) 324 | assert.Equal(t, "", ingressService.IngressClass) 325 | assert.Equal(t, 10*time.Second, ingressService.BaseService.Timeout) 326 | assert.Equal(t, "https://mycluster.com", ingressService.BaseService.RestConfig.Host) 327 | assert.Equal(t, "my-token", ingressService.BaseService.RestConfig.BearerToken) 328 | } 329 | 330 | func TestMultiClusterNginxIngress(t *testing.T) { 331 | backend := &MultiCluster{ 332 | Namespace: "tsuru-test", 333 | Fallback: &fakeBackend{}, 334 | Clusters: []ClusterConfig{ 335 | { 336 | Name: "default-token", 337 | Token: "my-token", 338 | Default: true, 339 | }, 340 | }, 341 | } 342 | router, err := backend.Router(ctx, "nginx-ingress", http.Header{ 343 | "X-Tsuru-Cluster-Name": { 344 | "my-cluster", 345 | }, 346 | "X-Tsuru-Cluster-Addresses": { 347 | "https://mycluster.com", 348 | }, 349 | }) 350 | assert.NoError(t, err) 351 | ingressService, ok := router.(*kubernetes.IngressService) 352 | require.True(t, ok) 353 | assert.Equal(t, "tsuru-test", ingressService.BaseService.Namespace) 354 | assert.Equal(t, "nginx", ingressService.IngressClass) 355 | assert.Equal(t, "nginx.ingress.kubernetes.io", ingressService.AnnotationsPrefix) 356 | assert.Equal(t, 10*time.Second, ingressService.BaseService.Timeout) 357 | assert.Equal(t, "https://mycluster.com", ingressService.BaseService.RestConfig.Host) 358 | assert.Equal(t, "my-token", ingressService.BaseService.RestConfig.BearerToken) 359 | } 360 | 361 | func TestMultiClusterIstioGateway(t *testing.T) { 362 | backend := &MultiCluster{ 363 | Namespace: "tsuru-test", 364 | Fallback: &fakeBackend{}, 365 | Clusters: []ClusterConfig{ 366 | { 367 | Name: "default-token", 368 | Token: "my-token", 369 | Default: true, 370 | }, 371 | }, 372 | } 373 | router, err := backend.Router(ctx, "istio-gateway", http.Header{ 374 | "X-Tsuru-Cluster-Name": { 375 | "my-cluster", 376 | }, 377 | "X-Tsuru-Cluster-Addresses": { 378 | "https://mycluster.com", 379 | }, 380 | }) 381 | assert.NoError(t, err) 382 | istioGateway, ok := router.(*kubernetes.IstioGateway) 383 | require.True(t, ok) 384 | assert.Equal(t, "tsuru-test", istioGateway.BaseService.Namespace) 385 | assert.Equal(t, 10*time.Second, istioGateway.BaseService.Timeout) 386 | assert.Equal(t, "https://mycluster.com", istioGateway.BaseService.RestConfig.Host) 387 | assert.Equal(t, "my-token", istioGateway.BaseService.RestConfig.BearerToken) 388 | } 389 | -------------------------------------------------------------------------------- /cmd/daemon.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net/http" 7 | "net/http/pprof" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/gorilla/mux" 14 | "github.com/prometheus/client_golang/prometheus/promhttp" 15 | "github.com/tsuru/kubernetes-router/api" 16 | "github.com/tsuru/kubernetes-router/backend" 17 | "github.com/tsuru/kubernetes-router/observability" 18 | "github.com/urfave/negroni" 19 | ) 20 | 21 | type DaemonOpts struct { 22 | Name string 23 | ListenAddr string 24 | Backend backend.Backend 25 | KeyFile string 26 | CertFile string 27 | } 28 | 29 | func StartDaemon(opts DaemonOpts) { 30 | routerAPI := api.RouterAPI{ 31 | Backend: opts.Backend, 32 | } 33 | 34 | r := mux.NewRouter().StrictSlash(true) 35 | 36 | r.PathPrefix("/api").Handler(negroni.New( 37 | api.AuthMiddleware{ 38 | User: os.Getenv("ROUTER_API_USER"), 39 | Pass: os.Getenv("ROUTER_API_PASSWORD"), 40 | }, 41 | negroni.Wrap(routerAPI.Routes()), 42 | )) 43 | r.HandleFunc("/healthcheck", routerAPI.Healthcheck) 44 | r.Handle("/metrics", promhttp.Handler()) 45 | 46 | r.HandleFunc("/debug/pprof/", pprof.Index) 47 | r.HandleFunc("/debug/pprof/heap", pprof.Index) 48 | r.HandleFunc("/debug/pprof/mutex", pprof.Index) 49 | r.HandleFunc("/debug/pprof/goroutine", pprof.Index) 50 | r.HandleFunc("/debug/pprof/threadcreate", pprof.Index) 51 | r.HandleFunc("/debug/pprof/block", pprof.Index) 52 | r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) 53 | r.HandleFunc("/debug/pprof/profile", pprof.Profile) 54 | r.HandleFunc("/debug/pprof/symbol", pprof.Symbol) 55 | r.HandleFunc("/debug/pprof/trace", pprof.Trace) 56 | 57 | n := negroni.New(observability.Middleware(), negroni.NewLogger(), negroni.NewRecovery()) 58 | n.UseHandler(r) 59 | 60 | server := http.Server{ 61 | Addr: opts.ListenAddr, 62 | Handler: n, 63 | ReadTimeout: 10 * time.Second, 64 | WriteTimeout: 30 * time.Second, 65 | } 66 | 67 | go handleSignals(&server) 68 | 69 | if opts.KeyFile != "" && opts.CertFile != "" { 70 | log.Printf("Started listening and serving TLS at %s", opts.ListenAddr) 71 | if err := server.ListenAndServeTLS(opts.CertFile, opts.KeyFile); err != nil && err != http.ErrServerClosed { 72 | log.Fatalf("fail serve: %v", err) 73 | } 74 | return 75 | } 76 | log.Printf("Started listening and serving %s at %s", opts.Name, opts.ListenAddr) 77 | if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { 78 | log.Fatalf("fail serve: %v", err) 79 | } 80 | } 81 | 82 | func handleSignals(server *http.Server) { 83 | signals := make(chan os.Signal, 1) 84 | signal.Notify(signals, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT) 85 | sig := <-signals 86 | log.Printf("Received %s. Terminating...", sig) 87 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) 88 | defer cancel() 89 | err := server.Shutdown(ctx) 90 | if err != nil { 91 | log.Fatalf("Error during server shutdown: %v", err) 92 | } 93 | log.Print("Server shutdown succeeded.") 94 | } 95 | -------------------------------------------------------------------------------- /cmd/flags.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package cmd 6 | 7 | import ( 8 | "encoding/json" 9 | "errors" 10 | "strings" 11 | ) 12 | 13 | // MapFlag wraps a map[string]string to be populated from 14 | // flags with KEY=VALUE format 15 | type MapFlag map[string]string 16 | 17 | // String prints the json representation 18 | func (f *MapFlag) String() string { 19 | repr := *f 20 | if repr == nil { 21 | repr = MapFlag{} 22 | } 23 | data, err := json.Marshal(repr) 24 | if err != nil { 25 | panic(err) 26 | } 27 | return string(data) 28 | } 29 | 30 | // Set sets a value on the underlying map 31 | func (f *MapFlag) Set(val string) error { 32 | parts := strings.SplitN(val, "=", 2) 33 | if *f == nil { 34 | *f = map[string]string{} 35 | } 36 | if len(parts) < 2 { 37 | return errors.New("must be on the form \"key=value\"") 38 | } 39 | (*f)[parts[0]] = parts[1] 40 | return nil 41 | } 42 | 43 | // MultiMapFlag wraps a map[string]map[string]string to be populated from 44 | // flags with KEY={K: V} format 45 | type MultiMapFlag map[string]map[string]string 46 | 47 | // String prints the json representation 48 | func (f *MultiMapFlag) String() string { 49 | repr := *f 50 | if repr == nil { 51 | repr = MultiMapFlag{} 52 | } 53 | data, err := json.Marshal(repr) 54 | if err != nil { 55 | panic(err) 56 | } 57 | return string(data) 58 | } 59 | 60 | // Set sets a value on the underlying map 61 | func (f *MultiMapFlag) Set(val string) error { 62 | parts := strings.SplitN(val, "=", 2) 63 | if *f == nil { 64 | *f = map[string]map[string]string{} 65 | } 66 | if len(parts) < 2 { 67 | return errors.New("must be on the form \"key={\"key\": \"value\"}\"") 68 | } 69 | var innerMap map[string]string 70 | err := json.Unmarshal([]byte(parts[1]), &innerMap) 71 | if err != nil { 72 | return err 73 | } 74 | (*f)[parts[0]] = innerMap 75 | return nil 76 | } 77 | 78 | // StringSliceFlag wraps a string slice populated by multiple flags. 79 | type StringSliceFlag []string 80 | 81 | // String prints a json representation 82 | func (f *StringSliceFlag) String() string { 83 | repr := *f 84 | if repr == nil { 85 | repr = StringSliceFlag{} 86 | } 87 | data, _ := json.Marshal(repr) 88 | return string(data) 89 | } 90 | 91 | // Set appends a new string to the slice 92 | func (f *StringSliceFlag) Set(val string) error { 93 | *f = append(*f, val) 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /cmd/flags_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package cmd 6 | 7 | import ( 8 | "reflect" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestMapFlag(t *testing.T) { 15 | var f MapFlag 16 | err := f.Set("a=1") 17 | if err != nil { 18 | t.Fatalf("Expected nil. Got %v.", err) 19 | } 20 | err = f.Set("b=2") 21 | if err != nil { 22 | t.Fatalf("Expected nil. Got %v.", err) 23 | } 24 | err = f.Set("c=3") 25 | if err != nil { 26 | t.Fatalf("Expected nil. Got %v.", err) 27 | } 28 | expected := MapFlag{"a": "1", "b": "2", "c": "3"} 29 | if !reflect.DeepEqual(f, expected) { 30 | t.Fatalf("Expected %v. Got %v.", expected, f) 31 | } 32 | } 33 | 34 | func TestMapFlagInvalid(t *testing.T) { 35 | var f MapFlag 36 | err := f.Set("a") 37 | if err == nil { 38 | t.Fatal("Expected err. Got nil") 39 | } 40 | } 41 | 42 | func TestMultiMapFlag(t *testing.T) { 43 | var f MultiMapFlag 44 | err := f.Set("a={\"v\": \"1\"}") 45 | if err != nil { 46 | t.Fatalf("Expected nil. Got %v.", err) 47 | } 48 | err = f.Set("b={\"v\": \"2\", \"x\":\"3\"}") 49 | if err != nil { 50 | t.Fatalf("Expected nil. Got %v.", err) 51 | } 52 | expected := MultiMapFlag{"a": {"v": "1"}, "b": {"v": "2", "x": "3"}} 53 | if !reflect.DeepEqual(f, expected) { 54 | t.Fatalf("Expected %v. Got %v.", expected, f) 55 | } 56 | } 57 | 58 | func TestStringSliceFlag(t *testing.T) { 59 | var f StringSliceFlag 60 | err := f.Set("a") 61 | require.NoError(t, err) 62 | err = f.Set("b") 63 | require.NoError(t, err) 64 | err = f.Set("c") 65 | require.NoError(t, err) 66 | expected := StringSliceFlag{ 67 | "a", "b", "c", 68 | } 69 | if !reflect.DeepEqual(f, expected) { 70 | t.Fatalf("Expected %v. Got %v.", expected, f) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /cmd/router/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "io" 10 | "log" 11 | "os" 12 | "time" 13 | 14 | "github.com/ghodss/yaml" 15 | "github.com/tsuru/kubernetes-router/backend" 16 | "github.com/tsuru/kubernetes-router/cmd" 17 | "github.com/tsuru/kubernetes-router/kubernetes" 18 | _ "github.com/tsuru/kubernetes-router/observability" 19 | "github.com/tsuru/kubernetes-router/router" 20 | ) 21 | 22 | func main() { 23 | listenAddr := flag.String("listen-addr", ":8077", "Listen address") 24 | ingressPort := flag.Int("ingress-http-port", 0, "The port that ingress services are exposed") 25 | k8sNamespace := flag.String("k8s-namespace", "tsuru", "Kubernetes namespace to create resources") 26 | k8sTimeout := flag.Duration("k8s-timeout", time.Second*10, "Kubernetes per-request timeout") 27 | k8sLabels := &cmd.MapFlag{} 28 | flag.Var(k8sLabels, "k8s-labels", "Labels to be added to each resource created. Expects KEY=VALUE format.") 29 | k8sAnnotations := &cmd.MapFlag{} 30 | flag.Var(k8sAnnotations, "k8s-annotations", "Annotations to be added to each resource created. Expects KEY=VALUE format.") 31 | runModes := cmd.StringSliceFlag{} 32 | flag.Var(&runModes, "controller-modes", "Defines enabled controller running modes: service, ingress, ingress-nginx or istio-gateway.") 33 | 34 | ingressDomain := flag.String("ingress-domain", "local", "Default domain to be used on created vhosts, local is the default. (eg: serviceName.local)") 35 | 36 | istioGatewaySelector := &cmd.MapFlag{} 37 | flag.Var(istioGatewaySelector, "istio-gateway.gateway-selector", "Gateway selector used in gateways created for apps.") 38 | 39 | certFile := flag.String("cert-file", "", "Path to certificate used to serve https requests") 40 | keyFile := flag.String("key-file", "", "Path to private key used to serve https requests") 41 | 42 | optsToLabels := &cmd.MapFlag{} 43 | flag.Var(optsToLabels, "opts-to-label", "Mapping between router options and service labels. Expects KEY=VALUE format.") 44 | 45 | optsToLabelsDocs := &cmd.MapFlag{} 46 | flag.Var(optsToLabelsDocs, "opts-to-label-doc", "Mapping between router options and user friendly help. Expects KEY=VALUE format.") 47 | 48 | optsToIngressAnnotations := &cmd.MapFlag{} 49 | flag.Var(optsToIngressAnnotations, "opts-to-ingress-annotations", "Mapping between router options and ingress annotations. Expects KEY=VALUE format.") 50 | 51 | optsToIngressAnnotationsDocs := &cmd.MapFlag{} 52 | flag.Var(optsToIngressAnnotationsDocs, "opts-to-ingress-annotations-doc", "Mapping between router options and user friendly help. Expects KEY=VALUE format.") 53 | 54 | ingressClass := flag.String("ingress-class", "", "Default class used for ingress objects") 55 | 56 | useIngressClassName := flag.Bool("use-ingress-class-name", false, "If true, the ingress.spec.ingressClassName will be used instead of the ingress.class annotation") 57 | 58 | ingressAnnotationsPrefix := flag.String("ingress-annotations-prefix", "", "Default prefix for annotations based on options") 59 | 60 | poolLabels := &cmd.MultiMapFlag{} 61 | flag.Var(poolLabels, "pool-labels", "Default labels for a given pool. Expects POOL={\"LABEL\":\"VALUE\"} format.") 62 | clustersFilePath := flag.String("clusters-file", "", "Path to file that describes clusters, when inform this file enable the multi-cluster support") 63 | 64 | flag.Parse() 65 | 66 | err := flag.Lookup("logtostderr").Value.Set("true") 67 | if err != nil { 68 | log.Printf("failed to set log to stderr: %v\n", err) 69 | } 70 | 71 | base := &kubernetes.BaseService{ 72 | Namespace: *k8sNamespace, 73 | Timeout: *k8sTimeout, 74 | Labels: *k8sLabels, 75 | Annotations: *k8sAnnotations, 76 | } 77 | 78 | if len(runModes) == 0 { 79 | runModes = append(runModes, "service") 80 | } 81 | 82 | localBackend := &backend.LocalCluster{ 83 | DefaultMode: runModes[0], 84 | Routers: map[string]router.Router{}, 85 | } 86 | 87 | for _, mode := range runModes { 88 | switch mode { 89 | case "istio-gateway": 90 | localBackend.Routers[mode] = &kubernetes.IstioGateway{ 91 | BaseService: base, 92 | DomainSuffix: *ingressDomain, 93 | GatewaySelector: *istioGatewaySelector, 94 | } 95 | case "ingress-nginx": 96 | *ingressClass = "nginx" 97 | *ingressAnnotationsPrefix = "nginx.ingress.kubernetes.io" 98 | fallthrough 99 | case "ingress": 100 | localBackend.Routers[mode] = &kubernetes.IngressService{ 101 | BaseService: base, 102 | DomainSuffix: *ingressDomain, 103 | OptsAsAnnotations: *optsToIngressAnnotations, 104 | OptsAsAnnotationsDocs: *optsToIngressAnnotationsDocs, 105 | IngressClass: *ingressClass, 106 | AnnotationsPrefix: *ingressAnnotationsPrefix, 107 | HTTPPort: *ingressPort, 108 | UseIngressClassName: *useIngressClassName, 109 | } 110 | case "service", "loadbalancer": 111 | localBackend.Routers[mode] = &kubernetes.LBService{ 112 | BaseService: base, 113 | OptsAsLabels: *optsToLabels, 114 | OptsAsLabelsDocs: *optsToLabelsDocs, 115 | PoolLabels: *poolLabels, 116 | } 117 | default: 118 | log.Fatalf("fail parameters: Use one of the following modes: service, ingress, ingress-nginx or istio-gateway.") 119 | } 120 | } 121 | 122 | var routerBackend backend.Backend = localBackend 123 | // enable multi-cluster support when file is provided 124 | if *clustersFilePath != "" { 125 | f, err := os.Open(*clustersFilePath) 126 | if err != nil { 127 | log.Printf("failed to load clusters file: %v\n", err) 128 | return 129 | } 130 | clustersFile := &backend.ClustersFile{} 131 | data, err := io.ReadAll(f) 132 | if err != nil { 133 | log.Printf("failed to load clusters file: %v\n", err) 134 | return 135 | } 136 | err = yaml.Unmarshal(data, clustersFile) 137 | if err != nil { 138 | log.Printf("failed to load clusters file: %v\n", err) 139 | return 140 | } 141 | 142 | routerBackend = &backend.MultiCluster{ 143 | Namespace: *k8sNamespace, 144 | Fallback: routerBackend, 145 | K8sTimeout: k8sTimeout, 146 | Modes: runModes, 147 | Clusters: clustersFile.Clusters, 148 | } 149 | } 150 | 151 | cmd.StartDaemon(cmd.DaemonOpts{ 152 | Name: "kubernetes-router", 153 | ListenAddr: *listenAddr, 154 | Backend: routerBackend, 155 | KeyFile: *keyFile, 156 | CertFile: *certFile, 157 | }) 158 | } 159 | -------------------------------------------------------------------------------- /deployments/local.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: kubernetes-router 5 | namespace: NAMESPACE 6 | spec: 7 | replicas: 1 8 | strategy: 9 | type: Recreate 10 | selector: 11 | matchLabels: 12 | app: kubernetes-router 13 | template: 14 | metadata: 15 | labels: 16 | app: kubernetes-router 17 | spec: 18 | serviceAccountName: kubernetes-router 19 | containers: 20 | - name: kubernetes-router 21 | image: IMAGE 22 | imagePullPolicy: Always 23 | livenessProbe: 24 | httpGet: 25 | path: /healthcheck 26 | port: 8077 27 | scheme: HTTP 28 | timeoutSeconds: 5 29 | command: ["./kubernetes-router"] 30 | args: ["-v", "3", "--k8s-namespace", "NAMESPACE"] 31 | ports: 32 | - containerPort: 8077 33 | --- 34 | apiVersion: v1 35 | kind: Service 36 | metadata: 37 | name: kubernetes-router 38 | namespace: NAMESPACE 39 | spec: 40 | type: LoadBalancer 41 | ports: 42 | - port: 80 43 | targetPort: 8077 44 | selector: 45 | app: kubernetes-router 46 | -------------------------------------------------------------------------------- /deployments/rbac.yml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: kubernetes-router 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: kubernetes-router 9 | subjects: 10 | - kind: ServiceAccount 11 | name: kubernetes-router 12 | namespace: NAMESPACE 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | kind: ClusterRole 16 | metadata: 17 | name: kubernetes-router 18 | rules: 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - "services" 23 | - "secrets" 24 | verbs: 25 | - "*" 26 | - apiGroups: 27 | - "" 28 | resources: 29 | - "nodes" 30 | verbs: 31 | - "list" 32 | - apiGroups: 33 | - "apiextensions.k8s.io" 34 | resources: 35 | - "customresourcedefinitions" 36 | verbs: 37 | - "get" 38 | - apiGroups: 39 | - "tsuru.io" 40 | resources: 41 | - "apps" 42 | verbs: 43 | - "get" 44 | - apiGroups: 45 | - "networking.k8s.io" 46 | resources: 47 | - "ingresses" 48 | verbs: 49 | - "*" 50 | - apiGroups: 51 | - "cert-manager.io" 52 | resources: 53 | - "issuers" 54 | - "clusterissuers" 55 | verbs: 56 | - "get" 57 | - "list" 58 | --- 59 | apiVersion: v1 60 | kind: ServiceAccount 61 | metadata: 62 | name: kubernetes-router 63 | namespace: NAMESPACE 64 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tsuru/kubernetes-router 2 | 3 | go 1.22.0 4 | 5 | toolchain go1.22.5 6 | 7 | require ( 8 | github.com/cert-manager/cert-manager v1.15.3 9 | github.com/ghodss/yaml v1.0.0 10 | github.com/golang/glog v1.2.1 11 | github.com/gorilla/mux v1.8.0 12 | github.com/opentracing-contrib/go-stdlib v1.0.0 13 | github.com/opentracing/opentracing-go v1.2.0 14 | github.com/pkg/errors v0.9.1 15 | github.com/prometheus/client_golang v1.19.1 16 | github.com/stretchr/testify v1.9.0 17 | github.com/tsuru/tsuru v0.0.0-20201016203419-9a2686f0f674 18 | github.com/uber/jaeger-client-go v2.25.0+incompatible 19 | github.com/urfave/negroni v0.2.0 20 | golang.org/x/sync v0.7.0 21 | istio.io/api v0.0.0-20200911191701-0dc35ad5c478 22 | istio.io/client-go v0.0.0-20200807182027-d287a5abb594 23 | k8s.io/api v0.31.0 24 | k8s.io/apiextensions-apiserver v0.31.0 25 | k8s.io/apimachinery v0.31.0 26 | k8s.io/client-go v0.31.0 27 | ) 28 | 29 | require ( 30 | github.com/emicklei/go-restful/v3 v3.12.0 // indirect 31 | github.com/evanphx/json-patch/v5 v5.9.0 // indirect 32 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 33 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 34 | github.com/go-openapi/jsonreference v0.21.0 // indirect 35 | github.com/go-openapi/swag v0.23.0 // indirect 36 | github.com/google/gnostic-models v0.6.8 // indirect 37 | github.com/google/uuid v1.6.0 // indirect 38 | github.com/josharian/intern v1.0.0 // indirect 39 | github.com/mailru/easyjson v0.7.7 // indirect 40 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 41 | github.com/x448/float16 v0.8.4 // indirect 42 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect 43 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 44 | sigs.k8s.io/gateway-api v1.1.0 // indirect 45 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 46 | ) 47 | 48 | require ( 49 | github.com/beorn7/perks v1.0.1 // indirect 50 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 51 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 52 | github.com/go-logr/logr v1.4.2 // indirect 53 | github.com/gogo/protobuf v1.3.2 // indirect 54 | github.com/golang/protobuf v1.5.4 // indirect 55 | github.com/google/go-cmp v0.6.0 // indirect 56 | github.com/google/gofuzz v1.2.0 // indirect 57 | github.com/json-iterator/go v1.1.12 // indirect 58 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 59 | github.com/modern-go/reflect2 v1.0.2 // indirect 60 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 61 | github.com/prometheus/client_model v0.6.1 // indirect 62 | github.com/prometheus/common v0.55.0 // indirect 63 | github.com/prometheus/procfs v0.15.1 // indirect 64 | github.com/uber/jaeger-lib v2.2.0+incompatible // indirect 65 | go.uber.org/atomic v1.7.0 // indirect 66 | golang.org/x/net v0.26.0 // indirect 67 | golang.org/x/oauth2 v0.21.0 // indirect 68 | golang.org/x/sys v0.21.0 // indirect 69 | golang.org/x/term v0.21.0 // indirect 70 | golang.org/x/text v0.16.0 // indirect 71 | golang.org/x/time v0.5.0 // indirect 72 | google.golang.org/grpc v1.65.0 // indirect 73 | google.golang.org/protobuf v1.34.2 // indirect 74 | gopkg.in/inf.v0 v0.9.1 // indirect 75 | gopkg.in/yaml.v2 v2.4.0 // indirect 76 | gopkg.in/yaml.v3 v3.0.1 // indirect 77 | istio.io/gogo-genproto v0.0.0-20201015184601-1e80d26d6249 // indirect 78 | k8s.io/klog/v2 v2.130.1 // indirect 79 | k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f // indirect 80 | k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 81 | sigs.k8s.io/controller-runtime v0.19.0 82 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 83 | sigs.k8s.io/yaml v1.4.0 // indirect 84 | ) 85 | -------------------------------------------------------------------------------- /kubernetes/ingress.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package kubernetes 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "log" 11 | "net" 12 | "reflect" 13 | "strconv" 14 | "strings" 15 | 16 | "github.com/opentracing/opentracing-go" 17 | "github.com/pkg/errors" 18 | "github.com/tsuru/kubernetes-router/router" 19 | v1 "k8s.io/api/core/v1" 20 | networkingV1 "k8s.io/api/networking/v1" 21 | 22 | k8sErrors "k8s.io/apimachinery/pkg/api/errors" 23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 25 | "k8s.io/apimachinery/pkg/runtime/schema" 26 | "k8s.io/apimachinery/pkg/types" 27 | "k8s.io/apimachinery/pkg/util/validation" 28 | typedV1 "k8s.io/client-go/kubernetes/typed/core/v1" 29 | networkingTypedV1 "k8s.io/client-go/kubernetes/typed/networking/v1" 30 | ) 31 | 32 | var ( 33 | // AnnotationsACMEKey defines the common annotation used to enable acme-tls 34 | AnnotationsACMEKey = "kubernetes.io/tls-acme" 35 | labelCNameIngress = "router.tsuru.io/is-cname-ingress" 36 | AnnotationsCNames = "router.tsuru.io/cnames" 37 | AnnotationFreeze = "router.tsuru.io/freeze" 38 | 39 | defaultClassOpt = "class" 40 | defaultOptsAsAnnotations = map[string]string{ 41 | defaultClassOpt: "kubernetes.io/ingress.class", 42 | } 43 | defaultOptsAsAnnotationsDocs = map[string]string{ 44 | defaultClassOpt: "Ingress class for the Ingress object", 45 | } 46 | 47 | certManagerIssuerKey = "cert-manager.io/issuer" 48 | certManagerClusterIssuerKey = "cert-manager.io/cluster-issuer" 49 | certManagerIssuerKindKey = "cert-manager.io/issuer-kind" 50 | certManagerIssuerGroupKey = "cert-manager.io/issuer-group" 51 | certManagerCommonName = "cert-manager.io/common-name" 52 | 53 | certManagerAnnotations = []string{ 54 | certManagerIssuerKey, 55 | certManagerClusterIssuerKey, 56 | certManagerIssuerKindKey, 57 | certManagerIssuerGroupKey, 58 | } 59 | ) 60 | 61 | var ( 62 | _ router.Router = &IngressService{} 63 | _ router.RouterTLS = &IngressService{} 64 | _ router.RouterStatus = &IngressService{} 65 | ) 66 | 67 | // Cert-manager types 68 | type CertManagerIssuerType int 69 | 70 | const ( 71 | certManagerIssuerTypeIssuer = iota 72 | certManagerIssuerTypeClusterIssuer 73 | certManagerIssuerTypeExternalIssuer 74 | ) 75 | 76 | type CertManagerIssuerData struct { 77 | name string 78 | kind string 79 | group string 80 | issuerType CertManagerIssuerType 81 | } 82 | 83 | const ( 84 | errIssuerNotFound = "issuer %s not found" 85 | errExternalIssuerNotFound = "external issuer %s not found, err: %s" 86 | errExternalIssuerInvalid = "invalid external issuer: %s (requires ..)" 87 | ) 88 | 89 | // IngressService manages ingresses in a Kubernetes cluster that uses ingress-nginx 90 | type IngressService struct { 91 | *BaseService 92 | DomainSuffix string 93 | 94 | // AnnotationsPrefix defines the common prefix used in the nginx ingress controller 95 | AnnotationsPrefix string 96 | // IngressClass defines the default ingress class used by the controller 97 | IngressClass string 98 | UseIngressClassName bool 99 | HTTPPort int 100 | OptsAsAnnotations map[string]string 101 | OptsAsAnnotationsDocs map[string]string 102 | } 103 | 104 | // Ensure creates or updates an Ingress resource to point it to either 105 | // the only service or the one responsible for the process web 106 | func (k *IngressService) Ensure(ctx context.Context, id router.InstanceID, o router.EnsureBackendOpts) error { 107 | span, ctx := opentracing.StartSpanFromContext(ctx, "ensureIngress") 108 | defer span.Finish() 109 | 110 | span.SetTag("cnames", o.CNames) 111 | 112 | ns, err := k.getAppNamespace(ctx, id.AppName) 113 | if err != nil { 114 | setSpanError(span, err) 115 | return err 116 | } 117 | ingressClient, err := k.ingressClient(ns) 118 | if err != nil { 119 | setSpanError(span, err) 120 | return err 121 | } 122 | isNew := false 123 | existingIngress, err := k.get(ctx, id) 124 | if err != nil { 125 | if !k8sErrors.IsNotFound(err) { 126 | setSpanError(span, err) 127 | return err 128 | } 129 | isNew = true 130 | } 131 | 132 | if !isNew && existingIngress != nil { 133 | if existingIngress.Annotations[AnnotationFreeze] == "true" { 134 | log.Printf("Ingress is frozen, skipping: %s/%s", existingIngress.Namespace, existingIngress.Name) 135 | return nil 136 | } 137 | } 138 | 139 | backendTargets, err := k.getBackendTargets(o.Prefixes, o.Opts.ExposeAllServices) 140 | if err != nil { 141 | setSpanError(span, err) 142 | return err 143 | } 144 | for k, v := range backendTargets { 145 | span.SetTag(fmt.Sprintf("%sTarget.service", k), v.Service) 146 | span.SetTag(fmt.Sprintf("%sTarget.namespace", k), v.Namespace) 147 | } 148 | 149 | backendServices := map[string]*v1.Service{} 150 | for key, target := range backendTargets { 151 | backendServices[key], err = k.getWebService(ctx, id.AppName, target) 152 | if err != nil { 153 | setSpanError(span, err) 154 | return err 155 | } 156 | } 157 | 158 | domainSuffix := o.Opts.DomainSuffix 159 | if k.DomainSuffix != "" { 160 | domainSuffix = k.DomainSuffix 161 | } 162 | 163 | vhosts := map[string]string{} 164 | for prefixString := range backendServices { 165 | prefix := "" 166 | if prefixString != "default" { 167 | prefix = prefixString + "." 168 | } 169 | if len(o.Opts.Domain) > 0 { 170 | vhosts[prefixString] = fmt.Sprintf("%s%s", prefix, o.Opts.Domain) 171 | } else if o.Opts.DomainPrefix == "" { 172 | vhosts[prefixString] = fmt.Sprintf("%s%s.%s", prefix, id.AppName, domainSuffix) 173 | } else { 174 | vhosts[prefixString] = fmt.Sprintf("%s%s.%s.%s", prefix, o.Opts.DomainPrefix, id.AppName, domainSuffix) 175 | } 176 | } 177 | 178 | ingress := &networkingV1.Ingress{ 179 | ObjectMeta: metav1.ObjectMeta{ 180 | Name: k.ingressName(id), 181 | Namespace: ns, 182 | Labels: map[string]string{ 183 | appBaseServiceNamespaceLabel: backendTargets["default"].Namespace, 184 | appBaseServiceNameLabel: backendTargets["default"].Service, 185 | }, 186 | OwnerReferences: []metav1.OwnerReference{ 187 | *metav1.NewControllerRef(backendServices["default"], schema.GroupVersionKind{ 188 | Group: v1.SchemeGroupVersion.Group, 189 | Version: v1.SchemeGroupVersion.Version, 190 | Kind: "Service", 191 | }), 192 | }, 193 | }, 194 | Spec: buildIngressSpec(vhosts, o.Opts.Route, backendServices, k), 195 | } 196 | k.fillIngressMeta(ingress, o.Opts, id, o.Team, o.Tags) 197 | if o.Opts.Acme { 198 | k.fillIngressTLS(ingress, id) 199 | ingress.ObjectMeta.Annotations[AnnotationsACMEKey] = "true" 200 | } else { 201 | k.cleanupCertManagerAnnotations(ingress) 202 | } 203 | if len(o.CNames) > 0 { 204 | ingress.Annotations[AnnotationsCNames] = strings.Join(o.CNames, ",") 205 | } 206 | 207 | if isNew { 208 | ingress, err = ingressClient.Create(ctx, ingress, metav1.CreateOptions{}) 209 | if err != nil { 210 | setSpanError(span, err) 211 | return err 212 | } 213 | } else if ingressHasChanges(span, existingIngress, ingress) { 214 | err = k.mergeIngressAndUpdate(ctx, ingress, existingIngress, id, ingressClient, span) 215 | if err != nil { 216 | setSpanError(span, err) 217 | return err 218 | } 219 | } else { 220 | ingress = existingIngress 221 | } 222 | 223 | var existingCNames []string 224 | if existingIngress != nil { 225 | existingCNames = strings.Split(existingIngress.Annotations[AnnotationsCNames], ",") 226 | } 227 | _, cnamesToRemove := diffCNames(existingCNames, o.CNames) 228 | 229 | for _, cname := range o.CNames { 230 | err = k.ensureCNameBackend(ctx, ensureCNameBackendOpts{ 231 | namespace: ns, 232 | id: id, 233 | parent: ingress, 234 | cname: cname, 235 | team: o.Team, 236 | certIssuer: o.CertIssuers[cname], 237 | service: backendServices["default"], 238 | routerOpts: o.Opts, 239 | tags: o.Tags, 240 | }) 241 | if err != nil { 242 | err = errors.Wrapf(err, "could not ensure CName: %q", cname) 243 | setSpanError(span, err) 244 | return err 245 | } 246 | } 247 | 248 | span.LogKV("cnamesToRemove", cnamesToRemove) 249 | for _, cname := range cnamesToRemove { 250 | err = k.removeCNameBackend(ctx, ensureCNameBackendOpts{ 251 | namespace: ns, 252 | id: id, 253 | cname: cname, 254 | team: o.Team, 255 | certIssuer: o.CertIssuers[cname], 256 | service: backendServices["default"], 257 | routerOpts: o.Opts, 258 | }) 259 | if err != nil { 260 | err = errors.Wrapf(err, "could not remove CName: %q", cname) 261 | setSpanError(span, err) 262 | return err 263 | } 264 | } 265 | 266 | return nil 267 | } 268 | 269 | func (k *IngressService) mergeIngressAndUpdate(ctx context.Context, ingress *networkingV1.Ingress, existingIngress *networkingV1.Ingress, id router.InstanceID, ingressClient networkingTypedV1.IngressInterface, span opentracing.Span) error { 270 | ingress.ObjectMeta.ResourceVersion = existingIngress.ObjectMeta.ResourceVersion 271 | ingress.ObjectMeta.UID = existingIngress.ObjectMeta.UID 272 | if existingIngress.Spec.DefaultBackend != nil { 273 | ingress.Spec.DefaultBackend = existingIngress.Spec.DefaultBackend 274 | } 275 | 276 | if existingIngress.Spec.TLS != nil && len(existingIngress.Spec.TLS) > 0 && !isManagedByCertManager(existingIngress.Annotations) { 277 | k.fillIngressTLS(ingress, id) 278 | } 279 | _, err := ingressClient.Update(ctx, ingress, metav1.UpdateOptions{}) 280 | if err != nil { 281 | setSpanError(span, err) 282 | return err 283 | } 284 | return nil 285 | } 286 | 287 | func buildIngressSpec(hosts map[string]string, path string, services map[string]*v1.Service, k *IngressService) networkingV1.IngressSpec { 288 | pathType := networkingV1.PathTypeImplementationSpecific 289 | rules := []networkingV1.IngressRule{} 290 | for k, service := range services { 291 | r := networkingV1.IngressRule{ 292 | Host: hosts[k], 293 | IngressRuleValue: networkingV1.IngressRuleValue{ 294 | HTTP: &networkingV1.HTTPIngressRuleValue{ 295 | Paths: []networkingV1.HTTPIngressPath{ 296 | { 297 | Path: path, 298 | PathType: &pathType, 299 | Backend: networkingV1.IngressBackend{ 300 | Service: &networkingV1.IngressServiceBackend{ 301 | Name: service.Name, 302 | Port: networkingV1.ServiceBackendPort{ 303 | Number: service.Spec.Ports[0].Port, 304 | }, 305 | }, 306 | }, 307 | }, 308 | }, 309 | }, 310 | }, 311 | } 312 | 313 | rules = append(rules, r) 314 | } 315 | 316 | if k.IngressClass != "" && k.UseIngressClassName { 317 | className := k.IngressClass 318 | return networkingV1.IngressSpec{ 319 | IngressClassName: &className, 320 | Rules: rules, 321 | } 322 | } 323 | 324 | return networkingV1.IngressSpec{ 325 | Rules: rules, 326 | } 327 | } 328 | 329 | func setSpanError(span opentracing.Span, err error) { 330 | span.SetTag("error", true) 331 | span.LogKV("error.message", err.Error()) 332 | } 333 | 334 | type ensureCNameBackendOpts struct { 335 | namespace string 336 | id router.InstanceID 337 | cname string 338 | team string 339 | certIssuer string 340 | parent *networkingV1.Ingress 341 | service *v1.Service 342 | routerOpts router.Opts 343 | tags []string 344 | } 345 | 346 | func (k *IngressService) ensureCNameBackend(ctx context.Context, opts ensureCNameBackendOpts) error { 347 | span, ctx := opentracing.StartSpanFromContext(ctx, "ensureIngressCName") 348 | defer span.Finish() 349 | 350 | span.SetTag("cname", opts.cname) 351 | 352 | ingressClient, err := k.ingressClient(opts.namespace) 353 | if err != nil { 354 | return err 355 | } 356 | isNew := false 357 | existingIngress, err := ingressClient.Get(ctx, k.ingressCName(opts.id, opts.cname), metav1.GetOptions{}) 358 | if err != nil { 359 | if !k8sErrors.IsNotFound(err) { 360 | return err 361 | 362 | } 363 | isNew = true 364 | } 365 | 366 | if !isNew && existingIngress != nil { 367 | if existingIngress.Annotations[AnnotationFreeze] == "true" { 368 | log.Printf("Ingress is frozen, skipping: %s/%s", existingIngress.Namespace, existingIngress.Name) 369 | return nil 370 | } 371 | } 372 | ingress := &networkingV1.Ingress{ 373 | ObjectMeta: metav1.ObjectMeta{ 374 | Name: k.ingressCName(opts.id, opts.cname), 375 | Namespace: opts.namespace, 376 | Labels: map[string]string{ 377 | appBaseServiceNamespaceLabel: opts.service.Namespace, 378 | appBaseServiceNameLabel: opts.service.Name, 379 | labelCNameIngress: "true", 380 | }, 381 | 382 | OwnerReferences: []metav1.OwnerReference{ 383 | *metav1.NewControllerRef(opts.parent, schema.GroupVersionKind{ 384 | Group: networkingV1.SchemeGroupVersion.Group, 385 | Version: networkingV1.SchemeGroupVersion.Version, 386 | Kind: "Ingress", 387 | }), 388 | }, 389 | }, 390 | Spec: buildIngressSpec(map[string]string{"ensureCnameBackend": opts.cname}, opts.routerOpts.Route, map[string]*v1.Service{"ensureCnameBackend": opts.service}, k), 391 | } 392 | 393 | k.fillIngressMeta(ingress, opts.routerOpts, opts.id, opts.team, opts.tags) 394 | 395 | if opts.routerOpts.HTTPOnly { 396 | k.cleanupCertManagerAnnotations(ingress) 397 | } else if opts.routerOpts.AcmeCName { 398 | k.fillIngressTLS(ingress, opts.id) 399 | ingress.ObjectMeta.Annotations[AnnotationsACMEKey] = "true" 400 | } else { 401 | err = k.ensureCNAMECertManagerIssuer(ctx, opts, ingress) 402 | if err != nil { 403 | return err 404 | } 405 | } 406 | 407 | if isNew { 408 | _, err = ingressClient.Create(ctx, ingress, metav1.CreateOptions{}) 409 | return err 410 | } 411 | 412 | if ingressHasChanges(span, existingIngress, ingress) { 413 | err = k.mergeIngressAndUpdate(ctx, ingress, existingIngress, opts.id, ingressClient, span) 414 | if err != nil { 415 | return err 416 | } 417 | } 418 | 419 | if len(ingress.Spec.TLS) == 0 { 420 | certificateName := k.secretName(opts.id, opts.cname) 421 | return k.ensureCertmanagerCertificateDeleted(ctx, opts.namespace, certificateName) 422 | } 423 | 424 | return nil 425 | } 426 | 427 | func (k *IngressService) ensureCertmanagerCertificateDeleted(ctx context.Context, namespace, certificateName string) error { 428 | certManagerClient, err := k.getCertManagerClient() 429 | if err != nil { 430 | return err 431 | } 432 | 433 | err = certManagerClient.CertmanagerV1().Certificates(namespace).Delete(ctx, certificateName, metav1.DeleteOptions{}) 434 | if err != nil && !k8sErrors.IsNotFound(err) { 435 | return err 436 | } 437 | 438 | return nil 439 | } 440 | 441 | func (k *IngressService) ensureCNAMECertManagerIssuer(ctx context.Context, opts ensureCNameBackendOpts, ingress *networkingV1.Ingress) error { 442 | if opts.certIssuer == "" { 443 | // If no cert issuer is provided, we should remove any existing cert issuer annotation 444 | k.cleanupCertManagerAnnotations(ingress) 445 | } else { 446 | // If a cert issuer is provided, we should add it to the ingress 447 | k.fillIngressTLS(ingress, opts.id) 448 | ingress.ObjectMeta.Annotations[certManagerClusterIssuerKey] = opts.certIssuer 449 | 450 | certIssuerData, err := k.getCertManagerIssuerData(ctx, opts.certIssuer, opts.namespace) 451 | if err != nil { 452 | log.Printf("Error getting cert manager issuer data: %v", err) 453 | return err 454 | } 455 | 456 | log.Printf("Cert manager issuer data: %v", certIssuerData) 457 | 458 | // Remove previous cermanager annotations if needed and 459 | // add cert-manager annotations to the ingress. 460 | k.cleanupCertManagerAnnotations(ingress) 461 | 462 | ingress.Annotations[certManagerCommonName] = opts.cname 463 | 464 | switch certIssuerData.issuerType { 465 | 466 | case certManagerIssuerTypeIssuer: 467 | ingress.ObjectMeta.Annotations[certManagerIssuerKey] = certIssuerData.name 468 | 469 | case certManagerIssuerTypeClusterIssuer: 470 | ingress.ObjectMeta.Annotations[certManagerClusterIssuerKey] = certIssuerData.name 471 | 472 | case certManagerIssuerTypeExternalIssuer: 473 | ingress.ObjectMeta.Annotations[certManagerIssuerKey] = certIssuerData.name 474 | ingress.ObjectMeta.Annotations[certManagerIssuerKindKey] = certIssuerData.kind 475 | ingress.ObjectMeta.Annotations[certManagerIssuerGroupKey] = certIssuerData.group 476 | } 477 | } 478 | 479 | return nil 480 | } 481 | 482 | func (k *IngressService) cleanupCertManagerAnnotations(ingress *networkingV1.Ingress) { 483 | for _, annotation := range certManagerAnnotations { 484 | delete(ingress.Annotations, annotation) 485 | } 486 | } 487 | 488 | func (k *IngressService) removeCNameBackend(ctx context.Context, opts ensureCNameBackendOpts) error { 489 | span, ctx := opentracing.StartSpanFromContext(ctx, "removeIngressCName") 490 | defer span.Finish() 491 | 492 | span.SetTag("cname", opts.cname) 493 | 494 | ingressClient, err := k.ingressClient(opts.namespace) 495 | if err != nil { 496 | return err 497 | } 498 | err = ingressClient.Delete(ctx, k.ingressCName(opts.id, opts.cname), metav1.DeleteOptions{}) 499 | if err != nil && !k8sErrors.IsNotFound(err) { 500 | return err 501 | } 502 | return nil 503 | } 504 | 505 | // Remove removes the Ingress resource associated with the app 506 | func (k *IngressService) Remove(ctx context.Context, id router.InstanceID) error { 507 | ns, err := k.getAppNamespace(ctx, id.AppName) 508 | if err != nil { 509 | return err 510 | } 511 | client, err := k.ingressClient(ns) 512 | if err != nil { 513 | return err 514 | } 515 | deletePropagation := metav1.DeletePropagationForeground 516 | err = client.Delete(ctx, k.ingressName(id), metav1.DeleteOptions{PropagationPolicy: &deletePropagation}) 517 | if k8sErrors.IsNotFound(err) { 518 | return nil 519 | } 520 | return err 521 | } 522 | 523 | // Get gets the address of the loadbalancer associated with 524 | // the app Ingress resource 525 | func (k *IngressService) GetAddresses(ctx context.Context, id router.InstanceID) ([]string, error) { 526 | ingress, err := k.get(ctx, id) 527 | if err != nil { 528 | if k8sErrors.IsNotFound(err) { 529 | return []string{""}, nil 530 | } 531 | return nil, err 532 | } 533 | hosts := []string{} 534 | urls := []string{} 535 | for _, rule := range ingress.Spec.Rules { 536 | if k.HTTPPort == 0 { 537 | hosts = append(hosts, rule.Host) 538 | } else { 539 | hostPort := net.JoinHostPort(rule.Host, strconv.Itoa(k.HTTPPort)) 540 | hosts = append(hosts, hostPort) 541 | } 542 | } 543 | for _, hostTLS := range ingress.Spec.TLS { 544 | for _, h := range hostTLS.Hosts { 545 | urls = append(urls, fmt.Sprintf("https://%v", h)) 546 | } 547 | } 548 | if len(urls) > 0 { 549 | return urls, nil 550 | } 551 | return hosts, nil 552 | } 553 | func (k *IngressService) GetStatus(ctx context.Context, id router.InstanceID) (router.BackendStatus, string, error) { 554 | ingress, err := k.get(ctx, id) 555 | if err != nil { 556 | if k8sErrors.IsNotFound(err) { 557 | return router.BackendStatusNotReady, "waiting for deploy", nil 558 | } 559 | return router.BackendStatusNotReady, "", err 560 | } 561 | if isIngressReady(ingress) { 562 | return router.BackendStatusReady, "", nil 563 | } 564 | detail, err := k.getStatusForRuntimeObject(ctx, ingress.Namespace, "Ingress", ingress.UID) 565 | if err != nil { 566 | return router.BackendStatusNotReady, "", err 567 | } 568 | 569 | return router.BackendStatusNotReady, detail, nil 570 | } 571 | 572 | func (k *IngressService) get(ctx context.Context, id router.InstanceID) (*networkingV1.Ingress, error) { 573 | ns, err := k.getAppNamespace(ctx, id.AppName) 574 | if err != nil { 575 | return nil, err 576 | } 577 | client, err := k.ingressClient(ns) 578 | if err != nil { 579 | return nil, err 580 | } 581 | ingress, err := client.Get(ctx, k.ingressName(id), metav1.GetOptions{}) 582 | if err != nil { 583 | return nil, err 584 | } 585 | return ingress, nil 586 | } 587 | 588 | func (k *IngressService) ingressClient(namespace string) (networkingTypedV1.IngressInterface, error) { 589 | client, err := k.getClient() 590 | if err != nil { 591 | return nil, err 592 | } 593 | return client.NetworkingV1().Ingresses(namespace), nil 594 | } 595 | 596 | func (k *IngressService) secretClient(namespace string) (typedV1.SecretInterface, error) { 597 | client, err := k.getClient() 598 | if err != nil { 599 | return nil, err 600 | } 601 | return client.CoreV1().Secrets(namespace), nil 602 | } 603 | 604 | func (s *IngressService) ingressName(id router.InstanceID) string { 605 | return s.hashedResourceName(id, "kubernetes-router-"+id.AppName+"-ingress", 253) 606 | } 607 | 608 | func (s *IngressService) ingressCName(id router.InstanceID, cname string) string { 609 | return s.hashedResourceName(id, "kubernetes-router-cname-"+cname, 253) 610 | } 611 | 612 | func (s *IngressService) secretName(id router.InstanceID, certName string) string { 613 | return s.hashedResourceName(id, "kr-"+id.AppName+"-"+certName, 253) 614 | } 615 | 616 | func (s *IngressService) annotationWithPrefix(suffix string) string { 617 | if s.AnnotationsPrefix == "" { 618 | return suffix 619 | } 620 | return fmt.Sprintf("%v/%v", s.AnnotationsPrefix, suffix) 621 | } 622 | 623 | // AddCertificate adds certificates to app ingress 624 | func (k *IngressService) AddCertificate(ctx context.Context, id router.InstanceID, certCname string, cert router.CertData) error { 625 | ns, err := k.getAppNamespace(ctx, id.AppName) 626 | if err != nil { 627 | return err 628 | } 629 | ingressClient, err := k.ingressClient(ns) 630 | if err != nil { 631 | return err 632 | } 633 | secret, err := k.secretClient(ns) 634 | if err != nil { 635 | return err 636 | } 637 | ingress, err := k.targetIngressForCertificate(ctx, id, certCname) 638 | if err != nil { 639 | return err 640 | } 641 | 642 | if isManagedByCertManager(ingress.Annotations) { 643 | return fmt.Errorf("cannot add certificate to ingress %s, it is managed by cert-manager", ingress.Name) 644 | } 645 | 646 | foundCname := false 647 | foundCNames := []string{} 648 | for _, rules := range ingress.Spec.Rules { 649 | foundCNames = append(foundCNames, rules.Host) 650 | 651 | if rules.Host == certCname { 652 | foundCname = true 653 | break 654 | } 655 | } 656 | 657 | if !foundCname { 658 | return fmt.Errorf("cname %s is not found in ingress %s, found cnames: %s", certCname, ingress.Name, strings.Join(foundCNames, ", ")) 659 | } 660 | 661 | if ingress.Annotations[AnnotationsACMEKey] == "true" { 662 | return fmt.Errorf("cannot add certificate to ingress %s, it is managed by ACME", ingress.Name) 663 | } 664 | 665 | secretName := k.secretName(id, certCname) 666 | tlsSecret := v1.Secret{ 667 | ObjectMeta: metav1.ObjectMeta{ 668 | Name: secretName, 669 | Namespace: ns, 670 | Labels: map[string]string{ 671 | appLabel: id.AppName, 672 | domainLabel: certCname, 673 | }, 674 | Annotations: make(map[string]string), 675 | }, 676 | Type: "kubernetes.io/tls", 677 | StringData: map[string]string{ 678 | "tls.key": cert.Key, 679 | "tls.crt": cert.Certificate, 680 | }, 681 | } 682 | _, err = secret.Create(ctx, &tlsSecret, metav1.CreateOptions{}) 683 | 684 | if k8sErrors.IsAlreadyExists(err) { 685 | var existingSecret *v1.Secret 686 | existingSecret, err = secret.Get(ctx, secretName, metav1.GetOptions{}) 687 | if err != nil { 688 | return err 689 | } 690 | tlsSecret.ResourceVersion = existingSecret.ResourceVersion 691 | _, err = secret.Update(ctx, &tlsSecret, metav1.UpdateOptions{}) 692 | } 693 | 694 | if err != nil { 695 | return err 696 | } 697 | 698 | tlsSpecExists := false 699 | for index, ingressTLS := range ingress.Spec.TLS { 700 | if ingressTLS.SecretName == tlsSecret.Name { 701 | ingress.Spec.TLS[index].Hosts = []string{certCname} 702 | tlsSpecExists = true 703 | break 704 | } 705 | } 706 | 707 | if !tlsSpecExists { 708 | ingress.Spec.TLS = append(ingress.Spec.TLS, 709 | []networkingV1.IngressTLS{ 710 | { 711 | Hosts: []string{certCname}, 712 | SecretName: tlsSecret.Name, 713 | }, 714 | }...) 715 | } 716 | _, err = ingressClient.Update(ctx, ingress, metav1.UpdateOptions{}) 717 | return err 718 | } 719 | 720 | func (k *IngressService) targetIngressForCertificate(ctx context.Context, id router.InstanceID, certCname string) (*networkingV1.Ingress, error) { 721 | ns, err := k.getAppNamespace(ctx, id.AppName) 722 | if err != nil { 723 | return nil, err 724 | } 725 | ingressClient, err := k.ingressClient(ns) 726 | if err != nil { 727 | return nil, err 728 | } 729 | ingressCName, err := ingressClient.Get(ctx, k.ingressCName(id, certCname), metav1.GetOptions{}) 730 | if err != nil { 731 | if !k8sErrors.IsNotFound(err) { 732 | return nil, err 733 | } 734 | } 735 | if ingressCName != nil && ingressCName.Labels[appLabel] == id.AppName { 736 | return ingressCName, nil 737 | } 738 | return k.get(ctx, id) 739 | } 740 | 741 | // GetCertificate get certificates from app ingress 742 | func (k *IngressService) GetCertificate(ctx context.Context, id router.InstanceID, certCname string) (*router.CertData, error) { 743 | ns, err := k.getAppNamespace(ctx, id.AppName) 744 | if err != nil { 745 | return nil, err 746 | } 747 | secret, err := k.secretClient(ns) 748 | if err != nil { 749 | return nil, err 750 | } 751 | 752 | secretName := k.secretName(id, certCname) 753 | retSecret, err := secret.Get(ctx, secretName, metav1.GetOptions{}) 754 | if err != nil { 755 | if k8sErrors.IsNotFound(err) { 756 | log.Printf("Secret %s/%s is not found\n", ns, secretName) 757 | return nil, router.ErrCertificateNotFound 758 | } 759 | return nil, err 760 | } 761 | 762 | certificate := string(retSecret.Data["tls.crt"]) 763 | key := string(retSecret.Data["tls.key"]) 764 | return &router.CertData{Certificate: certificate, Key: key}, err 765 | } 766 | 767 | // RemoveCertificate delete certificates from app ingress 768 | func (k *IngressService) RemoveCertificate(ctx context.Context, id router.InstanceID, certCname string) error { 769 | ns, err := k.getAppNamespace(ctx, id.AppName) 770 | if err != nil { 771 | return err 772 | } 773 | ingressClient, err := k.ingressClient(ns) 774 | if err != nil { 775 | return err 776 | } 777 | ingress, err := k.targetIngressForCertificate(ctx, id, certCname) 778 | if err != nil { 779 | return err 780 | } 781 | if ingress.Annotations[AnnotationsACMEKey] == "true" { 782 | return fmt.Errorf("cannot remove certificate from ingress %s, it is managed by ACME", ingress.Name) 783 | } 784 | 785 | if isManagedByCertManager(ingress.Annotations) { 786 | return fmt.Errorf("cannot remove certificate to ingress %s, it is managed by cert-manager", ingress.Name) 787 | } 788 | 789 | secret, err := k.secretClient(ns) 790 | if err != nil { 791 | return err 792 | } 793 | for k := range ingress.Spec.TLS { 794 | for _, host := range ingress.Spec.TLS[k].Hosts { 795 | if strings.Compare(certCname, host) == 0 { 796 | ingress.Spec.TLS = append(ingress.Spec.TLS[:k], ingress.Spec.TLS[k+1:]...) 797 | } 798 | } 799 | } 800 | _, err = ingressClient.Update(ctx, ingress, metav1.UpdateOptions{}) 801 | if err != nil { 802 | return err 803 | } 804 | err = secret.Delete(ctx, k.secretName(id, certCname), metav1.DeleteOptions{}) 805 | return err 806 | } 807 | 808 | // SupportedOptions returns the supported options 809 | func (s *IngressService) SupportedOptions(ctx context.Context) map[string]string { 810 | opts := map[string]string{ 811 | router.Domain: "", 812 | router.Acme: "", 813 | router.Route: "", 814 | router.AllPrefixes: "", 815 | } 816 | docs := mergeMaps(defaultOptsAsAnnotationsDocs, s.OptsAsAnnotationsDocs) 817 | for k, v := range mergeMaps(defaultOptsAsAnnotations, s.OptsAsAnnotations) { 818 | opts[k] = v 819 | if docs[k] != "" { 820 | opts[k] = docs[k] 821 | } 822 | } 823 | return opts 824 | } 825 | 826 | func (s *IngressService) fillIngressMeta(i *networkingV1.Ingress, routerOpts router.Opts, id router.InstanceID, team string, tags []string) { 827 | if i.ObjectMeta.Labels == nil { 828 | i.ObjectMeta.Labels = map[string]string{} 829 | } 830 | if i.ObjectMeta.Annotations == nil { 831 | i.ObjectMeta.Annotations = map[string]string{} 832 | } 833 | for k, v := range s.Labels { 834 | i.ObjectMeta.Labels[k] = v 835 | } 836 | for k, v := range s.Annotations { 837 | i.ObjectMeta.Annotations[k] = v 838 | } 839 | i.ObjectMeta.Labels[appLabel] = id.AppName 840 | i.ObjectMeta.Labels[teamLabel] = team 841 | 842 | additionalOpts := routerOpts.AdditionalOpts 843 | if s.IngressClass != "" && !s.UseIngressClassName { 844 | additionalOpts = mergeMaps(routerOpts.AdditionalOpts, map[string]string{ 845 | defaultClassOpt: s.IngressClass, 846 | }) 847 | } 848 | 849 | optsAsAnnotations := mergeMaps(defaultOptsAsAnnotations, s.OptsAsAnnotations) 850 | for optName, optValue := range additionalOpts { 851 | labelName, ok := optsAsAnnotations[optName] 852 | if !ok { 853 | if strings.Contains(optName, "/") { 854 | labelName = optName 855 | } else { 856 | labelName = s.annotationWithPrefix(optName) 857 | } 858 | } 859 | if strings.HasSuffix(labelName, "-") { 860 | delete(i.ObjectMeta.Annotations, strings.TrimSuffix(labelName, "-")) 861 | } else { 862 | i.ObjectMeta.Annotations[labelName] = optValue 863 | } 864 | } 865 | 866 | for _, tag := range tags { 867 | parts := strings.SplitN(tag, "=", 2) 868 | var key, value string 869 | if len(parts) != 2 { 870 | continue 871 | } 872 | 873 | key = parts[0] 874 | value = parts[1] 875 | 876 | if key == "" { 877 | continue 878 | } 879 | labelName := customTagPrefixLabel + key 880 | if len(validation.IsQualifiedName(labelName)) > 0 { 881 | // Ignoring tags that are not valid identifiers for labels or annotations 882 | continue 883 | } 884 | i.ObjectMeta.Labels[labelName] = value 885 | } 886 | } 887 | 888 | func (s *IngressService) validateCustomIssuer(ctx context.Context, resource CertManagerIssuerData, ns string) error { 889 | sigsClient, err := s.getSigsClient() 890 | if err != nil { 891 | return err 892 | } 893 | 894 | mapping, err := sigsClient.RESTMapper().RESTMapping(schema.GroupKind{ 895 | Group: resource.group, 896 | Kind: resource.kind, 897 | }) 898 | if err != nil { 899 | return err 900 | } 901 | 902 | u := &unstructured.Unstructured{} 903 | u.Object = map[string]interface{}{} 904 | u.SetGroupVersionKind(schema.GroupVersionKind{ 905 | Group: mapping.GroupVersionKind.Group, 906 | Kind: mapping.GroupVersionKind.Kind, 907 | Version: mapping.GroupVersionKind.Version, 908 | }) 909 | 910 | err = sigsClient.Get(ctx, types.NamespacedName{ 911 | Name: resource.name, 912 | Namespace: ns, 913 | }, u) 914 | if err != nil { 915 | return err 916 | } 917 | 918 | return nil 919 | } 920 | 921 | func (s *IngressService) getCertManagerIssuerData(ctx context.Context, issuerName, namespace string) (CertManagerIssuerData, error) { 922 | if strings.Contains(issuerName, ".") { 923 | // Treat as external issuer since it's more general 924 | parts := strings.SplitN(issuerName, ".", 3) 925 | if len(parts) != 3 { 926 | return CertManagerIssuerData{}, fmt.Errorf(errExternalIssuerInvalid, issuerName) 927 | } 928 | cmIssuerData := CertManagerIssuerData{ 929 | name: parts[0], 930 | kind: parts[1], 931 | group: parts[2], 932 | issuerType: certManagerIssuerTypeExternalIssuer, 933 | } 934 | 935 | if err := s.validateCustomIssuer(ctx, cmIssuerData, namespace); err != nil { 936 | return CertManagerIssuerData{}, fmt.Errorf(errExternalIssuerNotFound, issuerName, err.Error()) 937 | } 938 | 939 | return cmIssuerData, nil 940 | } 941 | 942 | // Treat as CertManager issuer 943 | cmClient, err := s.getCertManagerClient() 944 | if err != nil { 945 | return CertManagerIssuerData{}, err 946 | } 947 | 948 | _, err = cmClient.CertmanagerV1().Issuers(namespace).Get(ctx, issuerName, metav1.GetOptions{}) 949 | if err != nil && !k8sErrors.IsNotFound(err) { 950 | return CertManagerIssuerData{}, err 951 | } 952 | 953 | if err == nil { 954 | return CertManagerIssuerData{ 955 | name: issuerName, 956 | issuerType: certManagerIssuerTypeIssuer, 957 | }, nil 958 | } 959 | 960 | // Check if it's a cluster issuer 961 | _, err = cmClient.CertmanagerV1().ClusterIssuers().Get(ctx, issuerName, metav1.GetOptions{}) 962 | if err != nil && !k8sErrors.IsNotFound(err) { 963 | return CertManagerIssuerData{}, err 964 | } 965 | 966 | if err == nil { 967 | return CertManagerIssuerData{ 968 | name: issuerName, 969 | issuerType: certManagerIssuerTypeClusterIssuer, 970 | }, nil 971 | } 972 | 973 | // Issuer not found 974 | return CertManagerIssuerData{}, fmt.Errorf(errIssuerNotFound, issuerName) 975 | } 976 | 977 | func (s *IngressService) fillIngressTLS(i *networkingV1.Ingress, id router.InstanceID) { 978 | tlsRules := []networkingV1.IngressTLS{} 979 | if len(i.Spec.Rules) > 0 { 980 | for _, rule := range i.Spec.Rules { 981 | tlsRules = append(tlsRules, networkingV1.IngressTLS{ 982 | Hosts: []string{rule.Host}, 983 | SecretName: s.secretName(id, rule.Host), 984 | }) 985 | } 986 | } 987 | i.Spec.TLS = tlsRules 988 | } 989 | 990 | func ingressHasChanges(span opentracing.Span, existing *networkingV1.Ingress, ing *networkingV1.Ingress) (hasChanges bool) { 991 | if !reflect.DeepEqual(existing.Spec, ing.Spec) { 992 | span.LogKV( 993 | "message", "ingress has changed the spec", 994 | "ingress", existing.Name, 995 | ) 996 | return true 997 | } 998 | 999 | if !reflect.DeepEqual(existing.OwnerReferences, ing.OwnerReferences) { 1000 | span.LogKV( 1001 | "message", "ingress has changed the ownerReferences", 1002 | "ingress", existing.Name, 1003 | ) 1004 | return true 1005 | } 1006 | 1007 | if existing.Annotations[AnnotationsCNames] != ing.Annotations[AnnotationsCNames] { 1008 | return true 1009 | } 1010 | 1011 | for key, value := range ing.Annotations { 1012 | if existing.Annotations[key] != value { 1013 | span.LogKV( 1014 | "message", "ingress has changed the annotation", 1015 | "ingress", existing.Name, 1016 | "annotation", key, 1017 | "existingValue", existing.Annotations[key], 1018 | "newValue", value, 1019 | ) 1020 | 1021 | return true 1022 | } 1023 | } 1024 | for key, value := range ing.Labels { 1025 | if existing.Labels[key] != value { 1026 | span.LogKV( 1027 | "message", "ingress has changed the label", 1028 | "ingress", existing.Name, 1029 | "label", key, 1030 | "existingValue", existing.Labels[key], 1031 | "newValue", value, 1032 | ) 1033 | return true 1034 | } 1035 | } 1036 | span.LogKV( 1037 | "message", "ingress has no changes", 1038 | "ingress", existing.Name, 1039 | ) 1040 | return false 1041 | } 1042 | 1043 | func isIngressReady(ingress *networkingV1.Ingress) bool { 1044 | if len(ingress.Status.LoadBalancer.Ingress) == 0 { 1045 | return false 1046 | } 1047 | return ingress.Status.LoadBalancer.Ingress[0].IP != "" || ingress.Status.LoadBalancer.Ingress[0].Hostname != "" 1048 | } 1049 | 1050 | func isManagedByCertManager(annotations map[string]string) bool { 1051 | for _, annotation := range certManagerAnnotations { 1052 | if _, ok := annotations[annotation]; ok { 1053 | return true 1054 | } 1055 | } 1056 | return false 1057 | } 1058 | -------------------------------------------------------------------------------- /kubernetes/istiogateway.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package kubernetes 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "fmt" 11 | "sort" 12 | "strings" 13 | 14 | "github.com/tsuru/kubernetes-router/router" 15 | apiNetworking "istio.io/api/networking/v1beta1" 16 | networking "istio.io/client-go/pkg/apis/networking/v1beta1" 17 | networkingClientSet "istio.io/client-go/pkg/clientset/versioned/typed/networking/v1beta1" 18 | k8sErrors "k8s.io/apimachinery/pkg/api/errors" 19 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 20 | ) 21 | 22 | const ( 23 | hostsAnnotation = "tsuru.io/additional-hosts" 24 | ) 25 | 26 | var ( 27 | _ router.Router = &IstioGateway{} 28 | ) 29 | 30 | // IstioGateway manages gateways in a Kubernetes cluster with istio enabled. 31 | type IstioGateway struct { 32 | *BaseService 33 | istioClient networkingClientSet.NetworkingV1beta1Interface 34 | DomainSuffix string 35 | GatewaySelector map[string]string 36 | } 37 | 38 | func (k *IstioGateway) gatewayName(id router.InstanceID) string { 39 | return k.hashedResourceName(id, id.AppName, 63) 40 | } 41 | 42 | func (k *IstioGateway) vsName(id router.InstanceID) string { 43 | return k.hashedResourceName(id, id.AppName, 63) 44 | } 45 | 46 | func (k *IstioGateway) gatewayHost(id router.InstanceID) string { 47 | if id.InstanceName == "" { 48 | return fmt.Sprintf("%v.%v", id.AppName, k.DomainSuffix) 49 | } 50 | return fmt.Sprintf("%v.instance.%v.%v", id.InstanceName, id.AppName, k.DomainSuffix) 51 | } 52 | 53 | func (k *IstioGateway) updateObjectMeta(result *metav1.ObjectMeta, appName string, routerOpts router.Opts) { 54 | if result.Labels == nil { 55 | result.Labels = make(map[string]string) 56 | } 57 | if result.Annotations == nil { 58 | result.Annotations = make(map[string]string) 59 | } 60 | for k, v := range k.Labels { 61 | result.Labels[k] = v 62 | } 63 | result.Labels[appLabel] = appName 64 | for k, v := range k.Annotations { 65 | result.Annotations[k] = v 66 | } 67 | for k, v := range routerOpts.AdditionalOpts { 68 | result.Annotations[k] = v 69 | } 70 | } 71 | 72 | func (k *IstioGateway) getClient() (networkingClientSet.NetworkingV1beta1Interface, error) { 73 | if k.istioClient != nil { 74 | return k.istioClient, nil 75 | } 76 | var err error 77 | 78 | restConfig, err := k.getConfig() 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | k.istioClient, err = networkingClientSet.NewForConfig(restConfig) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | return k.istioClient, nil 89 | } 90 | 91 | func (k *IstioGateway) getVS(ctx context.Context, cli networkingClientSet.NetworkingV1beta1Interface, id router.InstanceID) (*networking.VirtualService, error) { 92 | ns, err := k.getAppNamespace(ctx, id.AppName) 93 | if err != nil { 94 | return nil, err 95 | } 96 | return cli.VirtualServices(ns).Get(ctx, k.vsName(id), metav1.GetOptions{}) 97 | } 98 | 99 | func addToSet(dst []string, toAdd ...string) []string { 100 | existingSet := map[string]struct{}{} 101 | for _, v := range dst { 102 | existingSet[v] = struct{}{} 103 | } 104 | for _, v := range toAdd { 105 | if _, in := existingSet[v]; !in { 106 | dst = append(dst, v) 107 | } 108 | } 109 | sort.Strings(dst) 110 | return dst 111 | } 112 | 113 | func removeFromSet(dst []string, toRemove ...string) []string { 114 | existingSet := map[string]struct{}{} 115 | for _, v := range dst { 116 | existingSet[v] = struct{}{} 117 | } 118 | for _, v := range toRemove { 119 | delete(existingSet, v) 120 | } 121 | dst = dst[:0] 122 | for h := range existingSet { 123 | dst = append(dst, h) 124 | } 125 | return dst 126 | } 127 | 128 | func hostsFromAnnotation(annotations map[string]string) []string { 129 | hostsRaw := annotations[hostsAnnotation] 130 | var hosts []string 131 | if hostsRaw != "" { 132 | hosts = strings.Split(hostsRaw, ",") 133 | } 134 | return hosts 135 | } 136 | 137 | func vsAddHost(v *networking.VirtualService, host string) { 138 | hosts := hostsFromAnnotation(v.Annotations) 139 | v.Spec.Hosts = removeFromSet(v.Spec.Hosts, hosts...) 140 | hosts = addToSet(hosts, host) 141 | v.Spec.Hosts = addToSet(v.Spec.Hosts, hosts...) 142 | sort.Strings(hosts) 143 | v.Annotations[hostsAnnotation] = strings.Join(hosts, ",") 144 | } 145 | 146 | func vsRemoveHost(v *networking.VirtualService, host string) { 147 | hosts := hostsFromAnnotation(v.Annotations) 148 | v.Spec.Hosts = removeFromSet(v.Spec.Hosts, hosts...) 149 | hosts = removeFromSet(hosts, host) 150 | v.Spec.Hosts = addToSet(v.Spec.Hosts, hosts...) 151 | sort.Strings(hosts) 152 | v.Annotations[hostsAnnotation] = strings.Join(hosts, ",") 153 | } 154 | 155 | func (k *IstioGateway) updateVirtualService(v *networking.VirtualService, id router.InstanceID, dstHost string) { 156 | v.Spec.Gateways = addToSet(v.Spec.Gateways, k.gatewayName(id)) 157 | v.Spec.Hosts = addToSet(v.Spec.Hosts, k.gatewayHost(id)) 158 | v.Spec.Hosts = addToSet(v.Spec.Hosts, dstHost) 159 | 160 | if len(v.Spec.Http) == 0 { 161 | v.Spec.Http = append(v.Spec.Http, &apiNetworking.HTTPRoute{}) 162 | } 163 | dstIdx := -1 164 | for i, dst := range v.Spec.Http[0].Route { 165 | if dst.Destination != nil && 166 | (dst.Destination.Host == dstHost) { 167 | dstIdx = i 168 | break 169 | } 170 | } 171 | if dstIdx == -1 { 172 | v.Spec.Http[0].Route = append(v.Spec.Http[0].Route, &apiNetworking.HTTPRouteDestination{}) 173 | dstIdx = len(v.Spec.Http[0].Route) - 1 174 | } 175 | v.Spec.Http[0].Route[dstIdx].Destination = &apiNetworking.Destination{ 176 | Host: dstHost, 177 | } 178 | } 179 | 180 | // Create adds a new gateway and a virtualservice for the app 181 | func (k *IstioGateway) Ensure(ctx context.Context, id router.InstanceID, o router.EnsureBackendOpts) error { 182 | cli, err := k.getClient() 183 | if err != nil { 184 | return err 185 | } 186 | namespace, err := k.getAppNamespace(ctx, id.AppName) 187 | if err != nil { 188 | return err 189 | } 190 | 191 | defaultTarget, err := k.getDefaultBackendTarget(o.Prefixes) 192 | if err != nil { 193 | return err 194 | } 195 | 196 | gateway := &networking.Gateway{ 197 | ObjectMeta: metav1.ObjectMeta{ 198 | Name: id.AppName, 199 | }, 200 | Spec: apiNetworking.Gateway{ 201 | Servers: []*apiNetworking.Server{ 202 | { 203 | Port: &apiNetworking.Port{ 204 | Number: 80, 205 | Name: "http2", 206 | Protocol: "HTTP2", 207 | }, 208 | Hosts: []string{"*"}, 209 | }, 210 | }, 211 | Selector: k.GatewaySelector, 212 | }, 213 | } 214 | 215 | k.updateObjectMeta(&gateway.ObjectMeta, id.AppName, o.Opts) 216 | 217 | _, err = cli.Gateways(namespace).Create(ctx, gateway, metav1.CreateOptions{}) 218 | isAlreadyExists := false 219 | if k8sErrors.IsAlreadyExists(err) { 220 | isAlreadyExists = true 221 | } else if err != nil { 222 | return err 223 | } 224 | 225 | existingSvc := true 226 | virtualSvc, err := k.getVS(ctx, cli, id) 227 | 228 | if err != nil && !k8sErrors.IsNotFound(err) { 229 | return err 230 | } 231 | 232 | if k8sErrors.IsNotFound(err) { 233 | existingSvc = false 234 | virtualSvc = &networking.VirtualService{ 235 | ObjectMeta: metav1.ObjectMeta{ 236 | Name: k.vsName(id), 237 | }, 238 | Spec: apiNetworking.VirtualService{ 239 | Gateways: []string{"mesh"}, 240 | }, 241 | } 242 | } 243 | 244 | k.updateObjectMeta(&virtualSvc.ObjectMeta, id.AppName, o.Opts) 245 | 246 | webService, err := k.getWebService(ctx, id.AppName, *defaultTarget) 247 | if err != nil { 248 | return err 249 | } 250 | 251 | k.updateVirtualService(virtualSvc, id, webService.Name) 252 | virtualSvc.Labels[appBaseServiceNamespaceLabel] = defaultTarget.Namespace 253 | virtualSvc.Labels[appBaseServiceNameLabel] = defaultTarget.Service 254 | 255 | existingCNames := hostsFromAnnotation(virtualSvc.Annotations) 256 | cnamesToAdd, cnamesToRemove := diffCNames(existingCNames, o.CNames) 257 | for _, cname := range cnamesToAdd { 258 | vsAddHost(virtualSvc, cname) 259 | } 260 | for _, cname := range cnamesToRemove { 261 | vsRemoveHost(virtualSvc, cname) 262 | } 263 | 264 | if existingSvc { 265 | _, err = cli.VirtualServices(namespace).Update(ctx, virtualSvc, metav1.UpdateOptions{}) 266 | } else { 267 | _, err = cli.VirtualServices(namespace).Create(ctx, virtualSvc, metav1.CreateOptions{}) 268 | } 269 | if err != nil { 270 | return err 271 | } 272 | 273 | if isAlreadyExists { 274 | return router.ErrIngressAlreadyExists 275 | } 276 | return nil 277 | } 278 | 279 | // Get returns the address in the gateway 280 | func (k *IstioGateway) GetAddresses(ctx context.Context, id router.InstanceID) ([]string, error) { 281 | return []string{k.gatewayHost(id)}, nil 282 | } 283 | 284 | // Swap is not implemented 285 | func (k *IstioGateway) Swap(ctx context.Context, srcApp, dstApp router.InstanceID) error { 286 | return errors.New("swap is not supported, the virtualservice should be edited manually") 287 | } 288 | 289 | // Remove removes the application gateway and removes it from the virtualservice 290 | func (k *IstioGateway) Remove(ctx context.Context, id router.InstanceID) error { 291 | cli, err := k.getClient() 292 | if err != nil { 293 | return err 294 | } 295 | virtualSvc, err := k.getVS(ctx, cli, id) 296 | if err != nil { 297 | return err 298 | } 299 | ns, err := k.getAppNamespace(ctx, id.AppName) 300 | if err != nil { 301 | return err 302 | } 303 | var gateways []string 304 | for _, g := range virtualSvc.Spec.Gateways { 305 | if g != k.gatewayName(id) { 306 | gateways = append(gateways, g) 307 | } 308 | } 309 | virtualSvc.Spec.Gateways = gateways 310 | _, err = cli.VirtualServices(ns).Update(ctx, virtualSvc, metav1.UpdateOptions{}) 311 | if err != nil { 312 | return err 313 | } 314 | return cli.Gateways(ns).Delete(ctx, k.gatewayName(id), metav1.DeleteOptions{}) 315 | } 316 | 317 | func diffCNames(existing []string, expected []string) (toAdd []string, toRemove []string) { 318 | mapExisting := map[string]bool{} 319 | mapExpected := map[string]bool{} 320 | 321 | for _, e := range existing { 322 | mapExisting[e] = true 323 | } 324 | 325 | for _, itemExpected := range expected { 326 | mapExpected[itemExpected] = true 327 | if !mapExisting[itemExpected] { 328 | toAdd = append(toAdd, itemExpected) 329 | } 330 | } 331 | 332 | for _, e := range existing { 333 | if !mapExpected[e] { 334 | toRemove = append(toRemove, e) 335 | } 336 | } 337 | 338 | return toAdd, toRemove 339 | } 340 | -------------------------------------------------------------------------------- /kubernetes/istiogateway_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package kubernetes 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | "github.com/tsuru/kubernetes-router/router" 14 | faketsuru "github.com/tsuru/tsuru/provision/kubernetes/pkg/client/clientset/versioned/fake" 15 | apiNetworking "istio.io/api/networking/v1beta1" 16 | networking "istio.io/client-go/pkg/apis/networking/v1beta1" 17 | fakeistio "istio.io/client-go/pkg/clientset/versioned/fake" 18 | networkingClientSet "istio.io/client-go/pkg/clientset/versioned/typed/networking/v1beta1" 19 | fakeapiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "k8s.io/client-go/kubernetes/fake" 22 | ) 23 | 24 | func fakeService() (IstioGateway, networkingClientSet.NetworkingV1beta1Interface) { 25 | fakeIstio := fakeistio.NewSimpleClientset().NetworkingV1beta1() 26 | return IstioGateway{ 27 | BaseService: &BaseService{ 28 | Namespace: "default", 29 | Client: fake.NewSimpleClientset(), 30 | TsuruClient: faketsuru.NewSimpleClientset(), 31 | ExtensionsClient: fakeapiextensions.NewSimpleClientset(), 32 | }, 33 | istioClient: fakeIstio, 34 | DomainSuffix: "my.domain", 35 | GatewaySelector: map[string]string{"istio": "ingress"}, 36 | }, fakeIstio 37 | } 38 | 39 | func TestIstioGateway_Ensure(t *testing.T) { 40 | svc, istio := fakeService() 41 | err := createAppWebService(svc.Client, svc.Namespace, "myapp") 42 | require.NoError(t, err) 43 | err = svc.Ensure(ctx, idForApp("myapp"), router.EnsureBackendOpts{ 44 | Prefixes: []router.BackendPrefix{ 45 | { 46 | Target: router.BackendTarget{ 47 | Service: "myapp-web", 48 | Namespace: svc.Namespace, 49 | }, 50 | }, 51 | }, 52 | }) 53 | require.NoError(t, err) 54 | gateway, err := istio.Gateways("default").Get(ctx, "myapp", metav1.GetOptions{}) 55 | require.NoError(t, err) 56 | virtualSvc, err := istio.VirtualServices("default").Get(ctx, "myapp", metav1.GetOptions{}) 57 | require.NoError(t, err) 58 | assert.Equal(t, map[string]string{"tsuru.io/app-name": "myapp"}, gateway.Labels) 59 | assert.Equal(t, map[string]string{}, gateway.Annotations) 60 | assert.Equal(t, apiNetworking.Gateway{ 61 | Servers: []*apiNetworking.Server{ 62 | { 63 | Port: &apiNetworking.Port{ 64 | Number: 80, 65 | Name: "http2", 66 | Protocol: "HTTP2", 67 | }, 68 | Hosts: []string{ 69 | "*", 70 | }, 71 | }, 72 | }, 73 | Selector: map[string]string{ 74 | "istio": "ingress", 75 | }, 76 | }, gateway.Spec) 77 | assert.Equal(t, map[string]string{ 78 | "tsuru.io/app-name": "myapp", 79 | "router.tsuru.io/base-service-name": "myapp-web", 80 | "router.tsuru.io/base-service-namespace": "default", 81 | }, virtualSvc.Labels) 82 | assert.Equal(t, map[string]string{}, virtualSvc.Annotations) 83 | assert.Equal(t, apiNetworking.VirtualService{ 84 | Gateways: []string{ 85 | "mesh", 86 | "myapp", 87 | }, 88 | Hosts: []string{ 89 | "myapp-web", 90 | "myapp.my.domain", 91 | }, 92 | Http: []*apiNetworking.HTTPRoute{ 93 | { 94 | Route: []*apiNetworking.HTTPRouteDestination{ 95 | { 96 | Destination: &apiNetworking.Destination{ 97 | Host: "myapp-web", 98 | }, 99 | }, 100 | }, 101 | }, 102 | }, 103 | }, virtualSvc.Spec) 104 | } 105 | 106 | func TestIstioGateway_EnsureWithCNames(t *testing.T) { 107 | svc, istio := fakeService() 108 | err := createAppWebService(svc.Client, svc.Namespace, "myapp") 109 | require.NoError(t, err) 110 | err = svc.Ensure(ctx, idForApp("myapp"), router.EnsureBackendOpts{ 111 | CNames: []string{"test.io", "www.test.io"}, 112 | Prefixes: []router.BackendPrefix{ 113 | { 114 | Target: router.BackendTarget{ 115 | Service: "myapp-web", 116 | Namespace: svc.Namespace, 117 | }, 118 | }, 119 | }, 120 | }) 121 | require.NoError(t, err) 122 | gateway, err := istio.Gateways("default").Get(ctx, "myapp", metav1.GetOptions{}) 123 | require.NoError(t, err) 124 | virtualSvc, err := istio.VirtualServices("default").Get(ctx, "myapp", metav1.GetOptions{}) 125 | require.NoError(t, err) 126 | assert.Equal(t, map[string]string{"tsuru.io/app-name": "myapp"}, gateway.Labels) 127 | assert.Equal(t, map[string]string{}, gateway.Annotations) 128 | assert.Equal(t, apiNetworking.Gateway{ 129 | Servers: []*apiNetworking.Server{ 130 | { 131 | Port: &apiNetworking.Port{ 132 | Number: 80, 133 | Name: "http2", 134 | Protocol: "HTTP2", 135 | }, 136 | Hosts: []string{ 137 | "*", 138 | }, 139 | }, 140 | }, 141 | Selector: map[string]string{ 142 | "istio": "ingress", 143 | }, 144 | }, gateway.Spec) 145 | assert.Equal(t, map[string]string{ 146 | "tsuru.io/app-name": "myapp", 147 | "router.tsuru.io/base-service-name": "myapp-web", 148 | "router.tsuru.io/base-service-namespace": "default", 149 | }, virtualSvc.Labels) 150 | assert.Equal(t, map[string]string{ 151 | "tsuru.io/additional-hosts": "test.io,www.test.io", 152 | }, virtualSvc.Annotations) 153 | assert.Equal(t, apiNetworking.VirtualService{ 154 | Gateways: []string{ 155 | "mesh", 156 | "myapp", 157 | }, 158 | Hosts: []string{ 159 | "myapp-web", 160 | "myapp.my.domain", 161 | "test.io", 162 | "www.test.io", 163 | }, 164 | Http: []*apiNetworking.HTTPRoute{ 165 | { 166 | Route: []*apiNetworking.HTTPRouteDestination{ 167 | { 168 | Destination: &apiNetworking.Destination{ 169 | Host: "myapp-web", 170 | }, 171 | }, 172 | }, 173 | }, 174 | }, 175 | }, virtualSvc.Spec) 176 | } 177 | 178 | func TestIstioGateway_Create_existingVirtualService(t *testing.T) { 179 | svc, istio := fakeService() 180 | err := createAppWebService(svc.Client, svc.Namespace, "myapp") 181 | require.NoError(t, err) 182 | 183 | _, err = istio.VirtualServices("default").Create(ctx, &networking.VirtualService{ 184 | ObjectMeta: metav1.ObjectMeta{ 185 | Name: "myapp", 186 | }, 187 | Spec: apiNetworking.VirtualService{ 188 | Hosts: []string{"older-host"}, 189 | Http: []*apiNetworking.HTTPRoute{ 190 | { 191 | Route: []*apiNetworking.HTTPRouteDestination{ 192 | { 193 | Destination: &apiNetworking.Destination{ 194 | Host: "to-be-keep", 195 | }, 196 | Weight: 100, 197 | }, 198 | }, 199 | }, 200 | }, 201 | }, 202 | }, metav1.CreateOptions{}) 203 | require.NoError(t, err) 204 | 205 | err = svc.Ensure(ctx, idForApp("myapp"), router.EnsureBackendOpts{ 206 | Prefixes: []router.BackendPrefix{ 207 | { 208 | Target: router.BackendTarget{ 209 | Service: "myapp-web", 210 | Namespace: svc.Namespace, 211 | }, 212 | }, 213 | }, 214 | }) 215 | require.NoError(t, err) 216 | 217 | gateway, err := istio.Gateways("default").Get(ctx, "myapp", metav1.GetOptions{}) 218 | require.NoError(t, err) 219 | virtualSvc, err := istio.VirtualServices("default").Get(ctx, "myapp", metav1.GetOptions{}) 220 | require.NoError(t, err) 221 | 222 | assert.Equal(t, map[string]string{"tsuru.io/app-name": "myapp"}, gateway.Labels) 223 | assert.Equal(t, map[string]string{}, gateway.Annotations) 224 | 225 | assert.Equal(t, apiNetworking.Gateway{ 226 | Servers: []*apiNetworking.Server{ 227 | { 228 | Port: &apiNetworking.Port{ 229 | Number: 80, 230 | Name: "http2", 231 | Protocol: "HTTP2", 232 | }, 233 | Hosts: []string{ 234 | "*", 235 | }, 236 | }, 237 | }, 238 | Selector: map[string]string{ 239 | "istio": "ingress", 240 | }, 241 | }, gateway.Spec) 242 | assert.Equal(t, map[string]string{ 243 | "tsuru.io/app-name": "myapp", 244 | "router.tsuru.io/base-service-name": "myapp-web", 245 | "router.tsuru.io/base-service-namespace": "default", 246 | }, virtualSvc.Labels) 247 | assert.Equal(t, map[string]string{}, virtualSvc.Annotations) 248 | assert.Equal(t, apiNetworking.VirtualService{ 249 | Gateways: []string{ 250 | "myapp", 251 | }, 252 | Hosts: []string{ 253 | "myapp-web", 254 | "myapp.my.domain", 255 | "older-host", 256 | }, 257 | Http: []*apiNetworking.HTTPRoute{ 258 | { 259 | Route: []*apiNetworking.HTTPRouteDestination{ 260 | { 261 | Destination: &apiNetworking.Destination{ 262 | Host: "to-be-keep", 263 | }, 264 | Weight: 100, 265 | }, 266 | { 267 | Destination: &apiNetworking.Destination{ 268 | Host: "myapp-web", 269 | }, 270 | }, 271 | }, 272 | }, 273 | }, 274 | }, virtualSvc.Spec) 275 | } 276 | 277 | func TestIstioGateway_CNameLifeCycle(t *testing.T) { 278 | tests := []struct { 279 | annotation string 280 | hosts []string 281 | ensureCNames []string 282 | expectedHosts []string 283 | expectedAnnotation string 284 | }{ 285 | { 286 | hosts: []string{"existing1"}, 287 | ensureCNames: []string{"myhost.com"}, 288 | expectedHosts: []string{ 289 | "existing1", 290 | "myapp.my.domain", 291 | "myapp-web", 292 | "myhost.com", 293 | }, 294 | expectedAnnotation: "myhost.com", 295 | }, 296 | { 297 | annotation: "my.other.addr", 298 | hosts: []string{"existing1"}, 299 | ensureCNames: []string{"myhost.com"}, 300 | expectedHosts: []string{ 301 | "existing1", 302 | "myapp.my.domain", 303 | "myapp-web", 304 | "myhost.com", 305 | }, 306 | expectedAnnotation: "myhost.com", 307 | }, 308 | { 309 | annotation: "my.other.addr", 310 | hosts: []string{"existing1", "my.other.addr"}, 311 | ensureCNames: []string{"my.other.addr", "myhost.com"}, 312 | expectedHosts: []string{ 313 | "myhost.com", 314 | "existing1", 315 | "myapp.my.domain", 316 | "myapp-web", 317 | "my.other.addr", 318 | }, 319 | expectedAnnotation: "my.other.addr,myhost.com", 320 | }, 321 | { 322 | annotation: "my.other.addr,myhost.com", 323 | hosts: []string{"existing1", "my.other.addr"}, 324 | ensureCNames: []string{"another.host.com", "my.other.addr", "myhost.com"}, 325 | expectedHosts: []string{ 326 | "myhost.com", "existing1", "my.other.addr", "another.host.com", 327 | "myapp.my.domain", 328 | "myapp-web", 329 | }, 330 | expectedAnnotation: "another.host.com,my.other.addr,myhost.com", 331 | }, 332 | { 333 | annotation: "my.other.addr,myhost.com", 334 | hosts: []string{"existing1", "my.other.addr"}, 335 | ensureCNames: []string{}, 336 | expectedHosts: []string{ 337 | "existing1", 338 | "myapp.my.domain", 339 | "myapp-web", 340 | }, 341 | expectedAnnotation: "", 342 | }, 343 | } 344 | for i, tt := range tests { 345 | t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { 346 | svc, istio := fakeService() 347 | err := createAppWebService(svc.Client, svc.Namespace, "myapp") 348 | require.NoError(t, err) 349 | _, err = istio.VirtualServices("default").Create(ctx, &networking.VirtualService{ 350 | ObjectMeta: metav1.ObjectMeta{ 351 | Name: "myapp", 352 | Labels: map[string]string{ 353 | "tsuru.io/app-name": "myapp", 354 | }, 355 | Annotations: map[string]string{ 356 | "tsuru.io/additional-hosts": tt.annotation, 357 | }, 358 | }, 359 | Spec: apiNetworking.VirtualService{ 360 | Hosts: tt.hosts, 361 | Http: []*apiNetworking.HTTPRoute{ 362 | { 363 | Route: []*apiNetworking.HTTPRouteDestination{ 364 | { 365 | Destination: &apiNetworking.Destination{ 366 | Host: "to-be-keep", 367 | }, 368 | }, 369 | }, 370 | }, 371 | }, 372 | }, 373 | }, metav1.CreateOptions{}) 374 | 375 | require.NoError(t, err) 376 | err = svc.Ensure(ctx, idForApp("myapp"), router.EnsureBackendOpts{ 377 | CNames: tt.ensureCNames, 378 | Prefixes: []router.BackendPrefix{ 379 | { 380 | Target: router.BackendTarget{ 381 | Service: "myapp-web", 382 | Namespace: svc.Namespace, 383 | }, 384 | }, 385 | }, 386 | }) 387 | require.NoError(t, err) 388 | virtualSvc, err := istio.VirtualServices("default").Get(ctx, "myapp", metav1.GetOptions{}) 389 | require.NoError(t, err) 390 | assert.ElementsMatch(t, tt.expectedHosts, virtualSvc.Spec.Hosts) 391 | assert.Equal(t, tt.expectedAnnotation, virtualSvc.Annotations["tsuru.io/additional-hosts"]) 392 | }) 393 | } 394 | } 395 | -------------------------------------------------------------------------------- /kubernetes/loadbalancer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package kubernetes 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "fmt" 11 | "reflect" 12 | "strconv" 13 | "strings" 14 | 15 | "github.com/opentracing/opentracing-go" 16 | "github.com/tsuru/kubernetes-router/router" 17 | v1 "k8s.io/api/core/v1" 18 | k8sErrors "k8s.io/apimachinery/pkg/api/errors" 19 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 20 | "k8s.io/apimachinery/pkg/util/intstr" 21 | ) 22 | 23 | const ( 24 | // defaultLBPort is the default exposed port to the LB 25 | defaultLBPort = 80 26 | 27 | // exposeAllPortsOpt is the flag used to expose all ports in the LB 28 | exposeAllPortsOpt = "expose-all-ports" 29 | 30 | annotationOptPrefix = "svc-annotation-" 31 | ) 32 | 33 | var ( 34 | // ErrLoadBalancerNotReady is returned when a given LB has no IP 35 | ErrLoadBalancerNotReady = errors.New("load balancer is not ready") 36 | ) 37 | 38 | var ( 39 | _ router.Router = &LBService{} 40 | _ router.RouterStatus = &LBService{} 41 | ) 42 | 43 | // LBService manages LoadBalancer services 44 | type LBService struct { 45 | *BaseService 46 | 47 | // OptsAsLabels maps router additional options to labels to be set on the service 48 | OptsAsLabels map[string]string 49 | 50 | // OptsAsLabelsDocs maps router additional options to user friendly help text 51 | OptsAsLabelsDocs map[string]string 52 | 53 | // PoolLabels maps router additional options for a given pool to be set on the service 54 | PoolLabels map[string]map[string]string 55 | } 56 | 57 | // Remove removes the LoadBalancer service 58 | func (s *LBService) Remove(ctx context.Context, id router.InstanceID) error { 59 | client, err := s.getClient() 60 | if err != nil { 61 | return err 62 | } 63 | service, err := s.getLBService(ctx, id) 64 | if err != nil { 65 | if k8sErrors.IsNotFound(err) { 66 | return nil 67 | } 68 | return err 69 | } 70 | ns, err := s.getAppNamespace(ctx, id.AppName) 71 | if err != nil { 72 | return err 73 | } 74 | err = client.CoreV1().Services(ns).Delete(ctx, service.Name, metav1.DeleteOptions{}) 75 | if k8sErrors.IsNotFound(err) { 76 | return nil 77 | } 78 | return err 79 | } 80 | 81 | // Get returns the LoadBalancer IP 82 | func (s *LBService) GetAddresses(ctx context.Context, id router.InstanceID) ([]string, error) { 83 | service, err := s.getLBService(ctx, id) 84 | if err != nil { 85 | if k8sErrors.IsNotFound(err) { 86 | return []string{""}, nil 87 | } 88 | return nil, err 89 | } 90 | var addr string 91 | lbs := service.Status.LoadBalancer.Ingress 92 | if service.Annotations[externalDNSHostnameLabel] != "" { 93 | hostnames := strings.Split(service.Annotations[externalDNSHostnameLabel], ",") 94 | return hostnames, nil 95 | } 96 | if len(lbs) != 0 { 97 | addr = lbs[0].IP 98 | ports := service.Spec.Ports 99 | if len(ports) != 0 { 100 | addr = fmt.Sprintf("%s:%d", addr, ports[0].Port) 101 | } 102 | if lbs[0].Hostname != "" { 103 | addr = lbs[0].Hostname 104 | } 105 | } 106 | return []string{addr}, nil 107 | } 108 | 109 | // SupportedOptions returns all the supported options 110 | func (s *LBService) SupportedOptions(ctx context.Context) map[string]string { 111 | opts := map[string]string{ 112 | router.ExposedPort: "", 113 | exposeAllPortsOpt: "Expose all ports used by application in the Load Balancer. Defaults to false.", 114 | } 115 | for k, v := range s.OptsAsLabels { 116 | opts[k] = v 117 | if s.OptsAsLabelsDocs[k] != "" { 118 | opts[k] = s.OptsAsLabelsDocs[k] 119 | } 120 | } 121 | return opts 122 | } 123 | 124 | func (s *LBService) GetStatus(ctx context.Context, id router.InstanceID) (router.BackendStatus, string, error) { 125 | service, err := s.getLBService(ctx, id) 126 | if err != nil { 127 | if k8sErrors.IsNotFound(err) { 128 | return router.BackendStatusNotReady, "waiting for deploy", nil 129 | } 130 | return router.BackendStatusNotReady, "", err 131 | } 132 | if isReady(service) { 133 | return router.BackendStatusReady, "", nil 134 | } 135 | detail, err := s.getStatusForRuntimeObject(ctx, service.Namespace, "Service", service.UID) 136 | if err != nil { 137 | return router.BackendStatusNotReady, "", err 138 | } 139 | 140 | return router.BackendStatusNotReady, detail, nil 141 | } 142 | 143 | func (s *LBService) getLBService(ctx context.Context, id router.InstanceID) (*v1.Service, error) { 144 | client, err := s.getClient() 145 | if err != nil { 146 | return nil, err 147 | } 148 | ns, err := s.getAppNamespace(ctx, id.AppName) 149 | if err != nil { 150 | return nil, err 151 | } 152 | return client.CoreV1().Services(ns).Get(ctx, s.serviceName(id), metav1.GetOptions{}) 153 | } 154 | 155 | func (s *LBService) serviceName(id router.InstanceID) string { 156 | return s.hashedResourceName(id, fmt.Sprintf("%s-router-lb", id.AppName), 63) 157 | } 158 | 159 | func isReady(service *v1.Service) bool { 160 | if len(service.Status.LoadBalancer.Ingress) == 0 { 161 | return false 162 | } 163 | // NOTE: aws load-balancers does not have IP 164 | return service.Status.LoadBalancer.Ingress[0].IP != "" || service.Status.LoadBalancer.Ingress[0].Hostname != "" 165 | } 166 | 167 | // Ensure creates or updates the LoadBalancer service copying the web service 168 | // labels, selectors, annotations and ports 169 | 170 | func (s *LBService) Ensure(ctx context.Context, id router.InstanceID, o router.EnsureBackendOpts) error { 171 | span, ctx := opentracing.StartSpanFromContext(ctx, "ensureLoadbalancer") 172 | defer span.Finish() 173 | 174 | app, err := s.getApp(ctx, id.AppName) 175 | if err != nil { 176 | return err 177 | } 178 | isNew := false 179 | existingLBService, err := s.getLBService(ctx, id) 180 | var lbService *v1.Service 181 | if err != nil { 182 | if !k8sErrors.IsNotFound(err) { 183 | return err 184 | } 185 | isNew = true 186 | ns := s.Namespace 187 | if app != nil { 188 | ns = app.Spec.NamespaceName 189 | } 190 | lbService = &v1.Service{ 191 | ObjectMeta: metav1.ObjectMeta{ 192 | Name: s.serviceName(id), 193 | Namespace: ns, 194 | }, 195 | Spec: v1.ServiceSpec{ 196 | Type: v1.ServiceTypeLoadBalancer, 197 | }, 198 | } 199 | } 200 | if !isNew { 201 | lbService = existingLBService.DeepCopy() 202 | } 203 | if isFrozenSvc(lbService) { 204 | return nil 205 | } 206 | 207 | if o.Opts.ExternalTrafficPolicy == "Cluster" || o.Opts.ExternalTrafficPolicy == "Local" { 208 | lbService.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(o.Opts.ExternalTrafficPolicy) 209 | } 210 | 211 | defaultTarget, err := s.getDefaultBackendTarget(o.Prefixes) 212 | if err != nil { 213 | return err 214 | } 215 | 216 | webService, err := s.getWebService(ctx, id.AppName, *defaultTarget) 217 | if err != nil { 218 | return err 219 | } 220 | 221 | lbService.Spec.Selector = webService.Spec.Selector 222 | 223 | err = s.fillLabelsAndAnnotations(ctx, lbService, id, webService, o.Opts, *defaultTarget) 224 | if err != nil { 225 | return err 226 | } 227 | 228 | ports, err := s.portsForService(lbService, o.Opts, webService) 229 | if err != nil { 230 | return err 231 | } 232 | lbService.Spec.Ports = ports 233 | client, err := s.getClient() 234 | if err != nil { 235 | return err 236 | } 237 | 238 | if isNew { 239 | _, err = client.CoreV1().Services(lbService.Namespace).Create(ctx, lbService, metav1.CreateOptions{}) 240 | return err 241 | } 242 | 243 | hasChanges := serviceHasChanges(span, existingLBService, lbService) 244 | 245 | if hasChanges { 246 | _, err = client.CoreV1().Services(lbService.Namespace).Update(ctx, lbService, metav1.UpdateOptions{}) 247 | return err 248 | } 249 | 250 | return nil 251 | } 252 | 253 | func (s *LBService) fillLabelsAndAnnotations(ctx context.Context, svc *v1.Service, id router.InstanceID, webService *v1.Service, opts router.Opts, backendTarget router.BackendTarget) error { 254 | optsLabels := make(map[string]string) 255 | registeredOpts := s.SupportedOptions(ctx) 256 | 257 | optsAnnotations, err := opts.ToAnnotations() 258 | if err != nil { 259 | return err 260 | } 261 | annotations := mergeMaps(s.Annotations, optsAnnotations) 262 | 263 | for optName, optValue := range opts.AdditionalOpts { 264 | if labelName, ok := s.OptsAsLabels[optName]; ok { 265 | optsLabels[labelName] = optValue 266 | continue 267 | } 268 | if _, ok := registeredOpts[optName]; ok { 269 | continue 270 | } 271 | 272 | if strings.HasPrefix(optName, annotationOptPrefix) { 273 | // Legacy tsuru versions do not support opt names with `.`. As a 274 | // workaround we accept opts with the prefix `svc-annotation-` to 275 | // use `:` instead of `.`. 276 | optName = strings.TrimPrefix(optName, annotationOptPrefix) 277 | optName = strings.ReplaceAll(optName, ":", ".") 278 | } 279 | 280 | if strings.HasSuffix(optName, "-") { 281 | delete(annotations, strings.TrimSuffix(optName, "-")) 282 | } else { 283 | annotations[optName] = optValue 284 | } 285 | } 286 | 287 | vhost := "" 288 | if len(opts.Domain) > 0 { 289 | vhost = opts.Domain 290 | } else if opts.DomainSuffix != "" { 291 | if opts.DomainPrefix == "" { 292 | vhost = fmt.Sprintf("%v.%v", id.AppName, opts.DomainSuffix) 293 | } else { 294 | vhost = fmt.Sprintf("%v.%v.%v", opts.DomainPrefix, id.AppName, opts.DomainSuffix) 295 | } 296 | } 297 | if vhost != "" { 298 | annotations[externalDNSHostnameLabel] = vhost 299 | } 300 | 301 | labels := []map[string]string{ 302 | svc.Labels, 303 | s.PoolLabels[opts.Pool], 304 | optsLabels, 305 | s.Labels, 306 | { 307 | appLabel: id.AppName, 308 | managedServiceLabel: "true", 309 | externalServiceLabel: "true", 310 | appPoolLabel: opts.Pool, 311 | }, 312 | } 313 | 314 | if webService != nil { 315 | labels = append(labels, webService.Labels) 316 | annotations = mergeMaps(annotations, webService.Annotations) 317 | } 318 | 319 | labels = append(labels, map[string]string{ 320 | appBaseServiceNamespaceLabel: backendTarget.Namespace, 321 | appBaseServiceNameLabel: backendTarget.Service, 322 | }) 323 | 324 | svc.Labels = mergeMaps(labels...) 325 | svc.Annotations = annotations 326 | return nil 327 | } 328 | 329 | func (s *LBService) portsForService(svc *v1.Service, opts router.Opts, baseSvc *v1.Service) ([]v1.ServicePort, error) { 330 | additionalPort, _ := strconv.Atoi(opts.ExposedPort) 331 | if additionalPort == 0 { 332 | additionalPort = defaultLBPort 333 | } 334 | 335 | existingPorts := map[int32]*v1.ServicePort{} 336 | for i, port := range svc.Spec.Ports { 337 | existingPorts[port.Port] = &svc.Spec.Ports[i] 338 | } 339 | 340 | exposeAllPorts, _ := strconv.ParseBool(opts.AdditionalOpts[exposeAllPortsOpt]) 341 | 342 | var basePorts, wantedPorts []v1.ServicePort 343 | if baseSvc != nil { 344 | basePorts = baseSvc.Spec.Ports 345 | } 346 | 347 | for _, basePort := range basePorts { 348 | if len(wantedPorts) == 0 { 349 | var name string 350 | if basePort.Name != "" { 351 | name = fmt.Sprintf("%s-extra", basePort.Name) 352 | } else { 353 | name = fmt.Sprintf("port-%d", additionalPort) 354 | } 355 | wantedPorts = append(wantedPorts, v1.ServicePort{ 356 | Name: name, 357 | Protocol: basePort.Protocol, 358 | Port: int32(additionalPort), 359 | TargetPort: basePort.TargetPort, 360 | }) 361 | } 362 | if !exposeAllPorts { 363 | break 364 | } 365 | 366 | if basePort.Port == int32(additionalPort) { 367 | // Skipping ports conflicting with additional port 368 | continue 369 | } 370 | basePort.NodePort = 0 371 | wantedPorts = append(wantedPorts, basePort) 372 | } 373 | 374 | if len(wantedPorts) == 0 { 375 | wantedPorts = append(wantedPorts, v1.ServicePort{ 376 | Name: fmt.Sprintf("port-%d", additionalPort), 377 | Protocol: v1.ProtocolTCP, 378 | Port: int32(additionalPort), 379 | TargetPort: intstr.FromInt(defaultServicePort), 380 | }) 381 | } 382 | 383 | for i := range wantedPorts { 384 | existingPort, ok := existingPorts[wantedPorts[i].Port] 385 | if ok { 386 | wantedPorts[i].NodePort = existingPort.NodePort 387 | } 388 | } 389 | 390 | return wantedPorts, nil 391 | } 392 | 393 | func serviceHasChanges(span opentracing.Span, existing *v1.Service, svc *v1.Service) (hasChanges bool) { 394 | if !reflect.DeepEqual(existing.Spec, svc.Spec) { 395 | span.LogKV( 396 | "message", "service has changed the spec", 397 | "service", existing.Name, 398 | ) 399 | return true 400 | } 401 | for key, value := range svc.Annotations { 402 | if existing.Annotations[key] != value { 403 | span.LogKV( 404 | "message", "service has changed the annotation", 405 | "service", existing.Name, 406 | "annotation", key, 407 | "existingValue", existing.Annotations[key], 408 | "newValue", value, 409 | ) 410 | return true 411 | } 412 | } 413 | for key, value := range svc.Labels { 414 | if existing.Labels[key] != value { 415 | span.LogKV( 416 | "message", "service has changed the label", 417 | "service", existing.Name, 418 | "label", key, 419 | "existingValue", existing.Labels[key], 420 | "newValue", value, 421 | ) 422 | return true 423 | } 424 | } 425 | span.LogKV( 426 | "message", "service has no changes", 427 | "service", existing.Name, 428 | ) 429 | return false 430 | } 431 | 432 | func mergeMaps(entries ...map[string]string) map[string]string { 433 | result := make(map[string]string) 434 | for _, entry := range entries { 435 | for k, v := range entry { 436 | if _, isSet := result[k]; !isSet { 437 | result[k] = v 438 | } 439 | } 440 | } 441 | return result 442 | } 443 | -------------------------------------------------------------------------------- /kubernetes/loadbalancer_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package kubernetes 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "reflect" 11 | "strconv" 12 | "testing" 13 | "time" 14 | 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | "github.com/tsuru/kubernetes-router/router" 18 | faketsuru "github.com/tsuru/tsuru/provision/kubernetes/pkg/client/clientset/versioned/fake" 19 | v1 "k8s.io/api/core/v1" 20 | fakeapiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "k8s.io/apimachinery/pkg/runtime" 23 | "k8s.io/apimachinery/pkg/util/intstr" 24 | "k8s.io/client-go/kubernetes/fake" 25 | ktesting "k8s.io/client-go/testing" 26 | ) 27 | 28 | var ctx = context.Background() 29 | 30 | func createFakeLBService() LBService { 31 | return LBService{ 32 | BaseService: &BaseService{ 33 | Namespace: "default", 34 | Client: fake.NewSimpleClientset(), 35 | TsuruClient: faketsuru.NewSimpleClientset(), 36 | ExtensionsClient: fakeapiextensions.NewSimpleClientset(), 37 | }, 38 | OptsAsLabels: make(map[string]string), 39 | OptsAsLabelsDocs: make(map[string]string), 40 | } 41 | } 42 | 43 | func TestLBEnsure(t *testing.T) { 44 | svc := createFakeLBService() 45 | err := createAppWebService(svc.Client, svc.Namespace, "test") 46 | require.NoError(t, err) 47 | svc.Labels = map[string]string{"label": "labelval"} 48 | svc.Annotations = map[string]string{"annotation": "annval"} 49 | svc.OptsAsLabels["my-opt"] = "my-opt-as-label" 50 | svc.PoolLabels = map[string]map[string]string{"mypool": {"pool-env": "dev"}, "otherpool": {"pool-env": "prod"}} 51 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 52 | Opts: router.Opts{Pool: "mypool", AdditionalOpts: map[string]string{"my-opt": "value"}, DomainSuffix: "myapps.io"}, 53 | Prefixes: []router.BackendPrefix{ 54 | { 55 | Target: router.BackendTarget{ 56 | Service: "test-web", 57 | Namespace: svc.Namespace, 58 | }, 59 | }, 60 | }, 61 | }) 62 | require.NoError(t, err) 63 | setIP(t, svc, "test") 64 | foundService, err := svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, "test-router-lb", metav1.GetOptions{}) 65 | require.NoError(t, err) 66 | 67 | svc.Labels[appPoolLabel] = "mypool" 68 | svc.Labels["my-opt-as-label"] = "value" 69 | svc.Labels["pool-env"] = "dev" 70 | expectedAnnotations := map[string]string{ 71 | "annotation": "annval", 72 | "external-dns.alpha.kubernetes.io/hostname": "test.myapps.io", 73 | "router.tsuru.io/opts": `{"Pool":"mypool","DomainSuffix":"myapps.io","AdditionalOpts":{"my-opt":"value"}}`, 74 | } 75 | expectedService := defaultService("test", "default", svc.Labels, expectedAnnotations, nil) 76 | assert.Equal(t, expectedService, foundService) 77 | } 78 | 79 | func TestLBEnsureWithExternalTrafficPolicy(t *testing.T) { 80 | svc := createFakeLBService() 81 | err := createAppWebService(svc.Client, svc.Namespace, "test") 82 | require.NoError(t, err) 83 | svc.Labels = map[string]string{"label": "labelval"} 84 | svc.Annotations = map[string]string{"annotation": "annval"} 85 | svc.PoolLabels = map[string]map[string]string{"mypool": {"pool-env": "dev"}, "otherpool": {"pool-env": "prod"}} 86 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 87 | Opts: router.Opts{Pool: "mypool", ExternalTrafficPolicy: "Local", AdditionalOpts: map[string]string{}, DomainSuffix: "myapps.io"}, 88 | Prefixes: []router.BackendPrefix{ 89 | { 90 | Target: router.BackendTarget{ 91 | Service: "test-web", 92 | Namespace: svc.Namespace, 93 | }, 94 | }, 95 | }, 96 | }) 97 | require.NoError(t, err) 98 | setIP(t, svc, "test") 99 | foundService, err := svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, "test-router-lb", metav1.GetOptions{}) 100 | require.NoError(t, err) 101 | 102 | svc.Labels[appPoolLabel] = "mypool" 103 | svc.Labels["pool-env"] = "dev" 104 | expectedAnnotations := map[string]string{ 105 | "annotation": "annval", 106 | "external-dns.alpha.kubernetes.io/hostname": "test.myapps.io", 107 | "router.tsuru.io/opts": `{"Pool":"mypool","DomainSuffix":"myapps.io","ExternalTrafficPolicy":"Local"}`, 108 | } 109 | expectedService := defaultService("test", "default", svc.Labels, expectedAnnotations, nil) 110 | expectedService.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal 111 | assert.Equal(t, expectedService, foundService) 112 | } 113 | 114 | func TestLBEnsureWithDomain(t *testing.T) { 115 | svc := createFakeLBService() 116 | svc.Labels = map[string]string{"label": "labelval"} 117 | svc.Annotations = map[string]string{"annotation": "annval"} 118 | svc.OptsAsLabels["my-opt"] = "my-opt-as-label" 119 | svc.PoolLabels = map[string]map[string]string{"mypool": {"pool-env": "dev"}, "otherpool": {"pool-env": "prod"}} 120 | 121 | err := createAppWebService(svc.Client, svc.Namespace, "test") 122 | require.NoError(t, err) 123 | 124 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 125 | Opts: router.Opts{ 126 | Pool: "mypool", 127 | AdditionalOpts: map[string]string{"my-opt": "value"}, 128 | Domain: "myappdomain.zone.io", 129 | }, 130 | Prefixes: []router.BackendPrefix{ 131 | { 132 | Target: router.BackendTarget{ 133 | Service: "test-web", 134 | Namespace: svc.Namespace, 135 | }, 136 | }, 137 | }, 138 | }) 139 | require.NoError(t, err) 140 | setIP(t, svc, "test") 141 | foundService, err := svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, "test-router-lb", metav1.GetOptions{}) 142 | require.NoError(t, err) 143 | svc.Labels[appPoolLabel] = "mypool" 144 | svc.Labels["my-opt-as-label"] = "value" 145 | svc.Labels["pool-env"] = "dev" 146 | expectedAnnotations := map[string]string{ 147 | "annotation": "annval", 148 | "external-dns.alpha.kubernetes.io/hostname": "myappdomain.zone.io", 149 | "router.tsuru.io/opts": `{"Pool":"mypool","Domain":"myappdomain.zone.io","AdditionalOpts":{"my-opt":"value"}}`, 150 | } 151 | expectedService := defaultService("test", "default", svc.Labels, expectedAnnotations, nil) 152 | assert.Equal(t, expectedService, foundService) 153 | } 154 | 155 | func TestLBEnsureCustomAnnotation(t *testing.T) { 156 | svc := createFakeLBService() 157 | svc.Labels = map[string]string{"label": "labelval"} 158 | svc.Annotations = map[string]string{"ann1": "val1", "ann2": "val2"} 159 | svc.OptsAsLabels["my-opt"] = "my-opt-as-label" 160 | svc.PoolLabels = map[string]map[string]string{"mypool": {"pool-env": "dev"}, "otherpool": {"pool-env": "prod"}} 161 | 162 | err := createAppWebService(svc.Client, svc.Namespace, "test") 163 | require.NoError(t, err) 164 | 165 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 166 | Opts: router.Opts{ 167 | Pool: "mypool", 168 | AdditionalOpts: map[string]string{ 169 | "my-opt": "value", 170 | "other-opt": "other-value", 171 | "svc-annotation-a:b/x:y": "true", 172 | "ann1-": "", 173 | }, 174 | }, 175 | Prefixes: []router.BackendPrefix{ 176 | { 177 | Target: router.BackendTarget{ 178 | Service: "test-web", 179 | Namespace: svc.Namespace, 180 | }, 181 | }, 182 | }, 183 | }) 184 | require.NoError(t, err) 185 | setIP(t, svc, "test") 186 | foundService, err := svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, "test-router-lb", metav1.GetOptions{}) 187 | require.NoError(t, err) 188 | svc.Labels[appPoolLabel] = "mypool" 189 | svc.Labels["my-opt-as-label"] = "value" 190 | svc.Labels["pool-env"] = "dev" 191 | expectedAnnotations := map[string]string{ 192 | "ann2": "val2", 193 | "other-opt": "other-value", 194 | "a.b/x.y": "true", 195 | "router.tsuru.io/opts": `{"Pool":"mypool","AdditionalOpts":{"ann1-":"","my-opt":"value","other-opt":"other-value","svc-annotation-a:b/x:y":"true"}}`, 196 | } 197 | expectedService := defaultService("test", "default", svc.Labels, expectedAnnotations, nil) 198 | assert.Equal(t, expectedService, foundService) 199 | } 200 | 201 | func TestLBEnsureDefaultPort(t *testing.T) { 202 | svc := createFakeLBService() 203 | err := createCRD(svc.BaseService, "myapp", "custom-namespace", nil) 204 | require.NoError(t, err) 205 | 206 | err = createAppWebService(svc.Client, svc.Namespace, "myapp") 207 | require.NoError(t, err) 208 | 209 | svc.BaseService.Client.(*fake.Clientset).PrependReactor("create", "services", func(action ktesting.Action) (bool, runtime.Object, error) { 210 | newSvc, ok := action.(ktesting.CreateAction).GetObject().(*v1.Service) 211 | if !ok { 212 | t.Errorf("Error creating service.") 213 | } 214 | ports := newSvc.Spec.Ports 215 | if len(ports) != 1 || ports[0].TargetPort != intstr.FromInt(8888) { 216 | t.Errorf("Expected service with targetPort 8888. Got %#v", ports) 217 | } 218 | return false, nil, nil 219 | }) 220 | err = svc.Ensure(ctx, idForApp("myapp"), router.EnsureBackendOpts{ 221 | Opts: router.Opts{ 222 | Pool: "mypool", 223 | AdditionalOpts: map[string]string{ 224 | "my-opt": "value", 225 | }, 226 | }, 227 | Prefixes: []router.BackendPrefix{ 228 | { 229 | Target: router.BackendTarget{ 230 | Service: "myapp-web", 231 | Namespace: svc.Namespace, 232 | }, 233 | }, 234 | }, 235 | }) 236 | require.NoError(t, err) 237 | } 238 | 239 | func TestLBSupportedOptions(t *testing.T) { 240 | svc := createFakeLBService() 241 | svc.OptsAsLabels["my-opt"] = "my-opt-as-label" 242 | svc.OptsAsLabels["my-opt2"] = "my-opt-as-label2" 243 | svc.OptsAsLabelsDocs["my-opt2"] = "User friendly option description." 244 | options := svc.SupportedOptions(ctx) 245 | expectedOptions := map[string]string{ 246 | "my-opt2": "User friendly option description.", 247 | "exposed-port": "", 248 | "my-opt": "my-opt-as-label", 249 | "expose-all-ports": "Expose all ports used by application in the Load Balancer. Defaults to false.", 250 | } 251 | if !reflect.DeepEqual(options, expectedOptions) { 252 | t.Errorf("Expected %v. Got %v", expectedOptions, options) 253 | } 254 | } 255 | 256 | func TestLBEnsureAppNamespace(t *testing.T) { 257 | svc := createFakeLBService() 258 | 259 | err := createAppWebService(svc.Client, svc.Namespace, "app") 260 | require.NoError(t, err) 261 | 262 | err = createCRD(svc.BaseService, "app", "custom-namespace", nil) 263 | require.NoError(t, err) 264 | 265 | err = svc.Ensure(ctx, idForApp("app"), router.EnsureBackendOpts{ 266 | Opts: router.Opts{}, 267 | Prefixes: []router.BackendPrefix{ 268 | { 269 | Target: router.BackendTarget{ 270 | Service: "app-web", 271 | Namespace: svc.Namespace, 272 | }, 273 | }, 274 | }, 275 | }) 276 | require.NoError(t, err) 277 | 278 | serviceList, err := svc.Client.CoreV1().Services("custom-namespace").List(ctx, metav1.ListOptions{}) 279 | require.NoError(t, err) 280 | 281 | if len(serviceList.Items) != 1 { 282 | t.Errorf("Expected 1 item. Got %d.", len(serviceList.Items)) 283 | } 284 | } 285 | 286 | func TestLBRemove(t *testing.T) { 287 | tt := []struct { 288 | testName string 289 | remove string 290 | expectedErr error 291 | expectedCount int 292 | }{ 293 | {"success", "test", nil, 1}, 294 | {"ignoresNotFound", "notfound", nil, 2}, 295 | } 296 | for _, tc := range tt { 297 | tc := tc 298 | t.Run(tc.testName, func(t *testing.T) { 299 | svc := createFakeLBService() 300 | 301 | err := createAppWebService(svc.Client, svc.Namespace, "test") 302 | require.NoError(t, err) 303 | 304 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 305 | Opts: router.Opts{}, 306 | Prefixes: []router.BackendPrefix{ 307 | { 308 | Target: router.BackendTarget{ 309 | Service: "test-web", 310 | Namespace: svc.Namespace, 311 | }, 312 | }, 313 | }, 314 | }) 315 | require.NoError(t, err) 316 | setIP(t, svc, "test") 317 | 318 | err = svc.Remove(ctx, idForApp(tc.remove)) 319 | 320 | assert.Equal(t, tc.expectedErr, err) 321 | serviceList, err := svc.Client.CoreV1().Services(svc.Namespace).List(ctx, metav1.ListOptions{}) 322 | require.NoError(t, err) 323 | assert.Len(t, serviceList.Items, tc.expectedCount) 324 | }) 325 | } 326 | } 327 | 328 | func TestLBUpdate(t *testing.T) { 329 | svc1 := v1.Service{ObjectMeta: metav1.ObjectMeta{ 330 | Name: "test-single", 331 | Namespace: "default", 332 | Labels: map[string]string{appLabel: "test"}, 333 | Annotations: map[string]string{"test-ann": "val-ann"}, 334 | }, 335 | Spec: v1.ServiceSpec{ 336 | Selector: map[string]string{"name": "test-single"}, 337 | Ports: []v1.ServicePort{{Protocol: "TCP", Port: int32(8899), TargetPort: intstr.FromInt(8899)}}, 338 | }, 339 | } 340 | svc2 := v1.Service{ 341 | ObjectMeta: metav1.ObjectMeta{ 342 | Name: "test-web", 343 | Namespace: "default", 344 | Labels: map[string]string{appLabel: "test", processLabel: "web", "custom1": "value1"}, 345 | }, 346 | Spec: v1.ServiceSpec{ 347 | Selector: map[string]string{"name": "test-web"}, 348 | Ports: []v1.ServicePort{{Protocol: "TCP", Port: int32(8890), TargetPort: intstr.FromInt(8890)}}, 349 | }, 350 | } 351 | svc3 := svc2 352 | svc3.ObjectMeta.Labels = svc1.ObjectMeta.Labels 353 | svc4 := svc2 354 | svc4.Spec.Ports = []v1.ServicePort{ 355 | {Protocol: "TCP", Port: int32(8890), TargetPort: intstr.FromInt(8890)}, 356 | {Protocol: "TCP", Port: int32(80), TargetPort: intstr.FromInt(8891)}, 357 | } 358 | svc5 := svc2 359 | svc5.Name = "test-web-v1" 360 | svc5.Labels = map[string]string{appLabel: "test", processLabel: "web", "custom2": "value2"} 361 | svc5.Spec.Selector = map[string]string{"name": "test-web", "version": "v1"} 362 | tt := []struct { 363 | name string 364 | services []v1.Service 365 | backendTarget router.BackendTarget 366 | expectedErr error 367 | expectedSelector map[string]string 368 | expectedPorts []v1.ServicePort 369 | expectedLabels map[string]string 370 | exposeAllPorts bool 371 | }{ 372 | { 373 | name: "noServices", 374 | services: []v1.Service{}, 375 | expectedErr: ErrNoService{App: "test"}, 376 | expectedLabels: map[string]string{ 377 | appLabel: "test", 378 | managedServiceLabel: "true", 379 | externalServiceLabel: "true", 380 | appPoolLabel: "", 381 | }, 382 | expectedPorts: []v1.ServicePort{ 383 | { 384 | Name: "port-80", 385 | Protocol: v1.ProtocolTCP, 386 | Port: int32(80), 387 | TargetPort: intstr.FromInt(8888), 388 | }, 389 | }, 390 | }, 391 | { 392 | name: "noServices with expose all", 393 | services: []v1.Service{}, 394 | exposeAllPorts: true, 395 | expectedErr: ErrNoService{App: "test"}, 396 | expectedLabels: map[string]string{ 397 | appLabel: "test", 398 | managedServiceLabel: "true", 399 | externalServiceLabel: "true", 400 | appPoolLabel: "", 401 | }, 402 | expectedPorts: []v1.ServicePort{ 403 | { 404 | Name: "port-80", 405 | Protocol: v1.ProtocolTCP, 406 | Port: int32(80), 407 | TargetPort: intstr.FromInt(8888), 408 | }, 409 | }, 410 | }, 411 | { 412 | name: "singleService with expose all", 413 | services: []v1.Service{svc1}, 414 | exposeAllPorts: true, 415 | backendTarget: router.BackendTarget{Service: svc1.Name, Namespace: svc1.Namespace}, 416 | expectedSelector: map[string]string{"name": "test-single"}, 417 | expectedLabels: map[string]string{ 418 | appLabel: "test", 419 | managedServiceLabel: "true", 420 | externalServiceLabel: "true", 421 | appPoolLabel: "", 422 | 423 | appBaseServiceNameLabel: svc1.Name, 424 | appBaseServiceNamespaceLabel: svc1.Namespace, 425 | }, 426 | expectedPorts: []v1.ServicePort{ 427 | { 428 | Name: "port-80", 429 | Protocol: v1.ProtocolTCP, 430 | Port: int32(80), 431 | TargetPort: intstr.FromInt(8899), 432 | }, 433 | { 434 | Protocol: v1.ProtocolTCP, 435 | Port: int32(8899), 436 | TargetPort: intstr.FromInt(8899), 437 | }, 438 | }, 439 | }, 440 | { 441 | name: "singleService", 442 | services: []v1.Service{svc1}, 443 | backendTarget: router.BackendTarget{Service: svc1.Name, Namespace: svc1.Namespace}, 444 | expectedSelector: map[string]string{"name": "test-single"}, 445 | expectedLabels: map[string]string{ 446 | appLabel: "test", 447 | managedServiceLabel: "true", 448 | externalServiceLabel: "true", 449 | appPoolLabel: "", 450 | 451 | appBaseServiceNameLabel: svc1.Name, 452 | appBaseServiceNamespaceLabel: svc1.Namespace, 453 | }, 454 | expectedPorts: []v1.ServicePort{ 455 | { 456 | Name: "port-80", 457 | Protocol: v1.ProtocolTCP, 458 | Port: int32(80), 459 | TargetPort: intstr.FromInt(8899), 460 | }, 461 | }, 462 | }, 463 | { 464 | name: "multiServiceWithWeb", 465 | services: []v1.Service{svc1, svc2}, 466 | exposeAllPorts: true, 467 | backendTarget: router.BackendTarget{Service: svc2.Name, Namespace: svc2.Namespace}, 468 | expectedSelector: map[string]string{"name": "test-web"}, 469 | expectedLabels: map[string]string{ 470 | appLabel: "test", 471 | processLabel: "web", 472 | managedServiceLabel: "true", 473 | externalServiceLabel: "true", 474 | appPoolLabel: "", 475 | "custom1": "value1", 476 | 477 | appBaseServiceNameLabel: svc2.Name, 478 | appBaseServiceNamespaceLabel: svc2.Namespace, 479 | }, 480 | expectedPorts: []v1.ServicePort{ 481 | { 482 | Name: "port-80", 483 | Protocol: v1.ProtocolTCP, 484 | Port: int32(80), 485 | TargetPort: intstr.FromInt(8890), 486 | }, 487 | { 488 | Protocol: v1.ProtocolTCP, 489 | Port: int32(8890), 490 | TargetPort: intstr.FromInt(8890), 491 | }, 492 | }, 493 | }, 494 | { 495 | name: "multiServiceWithoutWeb", 496 | services: []v1.Service{svc1, svc3}, 497 | expectedErr: ErrNoService{App: "test"}, 498 | expectedLabels: map[string]string{ 499 | appLabel: "test", 500 | managedServiceLabel: "true", 501 | externalServiceLabel: "true", 502 | appPoolLabel: "", 503 | }, 504 | expectedPorts: []v1.ServicePort{ 505 | { 506 | Name: "port-80", 507 | Protocol: v1.ProtocolTCP, 508 | Port: int32(80), 509 | TargetPort: intstr.FromInt(8888), 510 | }, 511 | }, 512 | }, 513 | { 514 | name: "service with conflicting port, port is ignored", 515 | services: []v1.Service{svc4}, 516 | exposeAllPorts: true, 517 | backendTarget: router.BackendTarget{Service: svc4.Name, Namespace: svc4.Namespace}, 518 | expectedSelector: map[string]string{"name": "test-web"}, 519 | expectedLabels: map[string]string{ 520 | appLabel: "test", 521 | processLabel: "web", 522 | managedServiceLabel: "true", 523 | externalServiceLabel: "true", 524 | appPoolLabel: "", 525 | "custom1": "value1", 526 | 527 | appBaseServiceNameLabel: svc4.Name, 528 | appBaseServiceNamespaceLabel: svc4.Namespace, 529 | }, 530 | expectedPorts: []v1.ServicePort{ 531 | { 532 | Name: "port-80", 533 | Protocol: v1.ProtocolTCP, 534 | Port: int32(80), 535 | TargetPort: intstr.FromInt(8890), 536 | }, 537 | { 538 | Protocol: v1.ProtocolTCP, 539 | Port: int32(8890), 540 | TargetPort: intstr.FromInt(8890), 541 | }, 542 | }, 543 | }, 544 | 545 | { 546 | name: "multiServiceWithWeb", 547 | services: []v1.Service{svc1, svc2}, 548 | backendTarget: router.BackendTarget{Namespace: svc2.Namespace, Service: svc2.Name}, 549 | exposeAllPorts: true, 550 | expectedSelector: map[string]string{"name": "test-web"}, 551 | expectedLabels: map[string]string{ 552 | appLabel: "test", 553 | processLabel: "web", 554 | managedServiceLabel: "true", 555 | externalServiceLabel: "true", 556 | appPoolLabel: "", 557 | appBaseServiceNamespaceLabel: "default", 558 | appBaseServiceNameLabel: "test-web", 559 | "custom1": "value1", 560 | }, 561 | expectedPorts: []v1.ServicePort{ 562 | { 563 | Name: "port-80", 564 | Protocol: v1.ProtocolTCP, 565 | Port: int32(80), 566 | TargetPort: intstr.FromInt(8890), 567 | }, 568 | { 569 | Protocol: v1.ProtocolTCP, 570 | Port: int32(8890), 571 | TargetPort: intstr.FromInt(8890), 572 | }, 573 | }, 574 | }, 575 | 576 | { 577 | name: "multiServiceWithConflictingWeb", 578 | services: []v1.Service{svc2, svc5}, 579 | backendTarget: router.BackendTarget{Namespace: svc2.Namespace, Service: svc2.Name}, 580 | exposeAllPorts: true, 581 | expectedSelector: map[string]string{"name": "test-web"}, 582 | expectedLabels: map[string]string{ 583 | appLabel: "test", 584 | processLabel: "web", 585 | managedServiceLabel: "true", 586 | externalServiceLabel: "true", 587 | appPoolLabel: "", 588 | appBaseServiceNamespaceLabel: "default", 589 | appBaseServiceNameLabel: "test-web", 590 | "custom1": "value1", 591 | }, 592 | expectedPorts: []v1.ServicePort{ 593 | { 594 | Name: "port-80", 595 | Protocol: v1.ProtocolTCP, 596 | Port: int32(80), 597 | TargetPort: intstr.FromInt(8890), 598 | }, 599 | { 600 | Protocol: v1.ProtocolTCP, 601 | Port: int32(8890), 602 | TargetPort: intstr.FromInt(8890), 603 | }, 604 | }, 605 | }, 606 | } 607 | 608 | for _, tc := range tt { 609 | tc := tc 610 | t.Run(tc.name, func(t *testing.T) { 611 | svc := createFakeLBService() 612 | 613 | for i := range tc.services { 614 | _, err := svc.Client.CoreV1().Services(svc.Namespace).Create(ctx, &tc.services[i], metav1.CreateOptions{}) 615 | require.NoError(t, err) 616 | } 617 | 618 | err := svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 619 | Opts: router.Opts{ 620 | AdditionalOpts: map[string]string{ 621 | exposeAllPortsOpt: strconv.FormatBool(tc.exposeAllPorts), 622 | }, 623 | }, 624 | Prefixes: []router.BackendPrefix{ 625 | { 626 | Target: tc.backendTarget, 627 | }, 628 | }, 629 | }) 630 | 631 | if tc.expectedErr != nil { 632 | assert.Equal(t, tc.expectedErr, err) 633 | return 634 | } 635 | 636 | require.NoError(t, err) 637 | setIP(t, svc, "test") 638 | service, err := svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, svc.serviceName(idForApp("test")), metav1.GetOptions{}) 639 | assert.NoError(t, err) 640 | assert.Equal(t, tc.expectedSelector, service.Spec.Selector) 641 | assert.Equal(t, tc.expectedPorts, service.Spec.Ports) 642 | assert.Equal(t, tc.expectedLabels, service.Labels) 643 | }) 644 | } 645 | } 646 | 647 | func TestLBUpdatePortDiffAndPreserveNodePort(t *testing.T) { 648 | svc := createFakeLBService() 649 | err := createAppWebService(svc.Client, svc.Namespace, "test") 650 | require.NoError(t, err) 651 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 652 | Opts: router.Opts{ 653 | AdditionalOpts: map[string]string{ 654 | exposeAllPortsOpt: "true", 655 | }, 656 | }, 657 | Prefixes: []router.BackendPrefix{ 658 | { 659 | Target: router.BackendTarget{ 660 | Service: "test-web", 661 | Namespace: svc.Namespace, 662 | }, 663 | }, 664 | }, 665 | }) 666 | require.NoError(t, err) 667 | service, err := svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, svc.serviceName(idForApp("test")), metav1.GetOptions{}) 668 | require.NoError(t, err) 669 | service.Spec.Ports = []v1.ServicePort{ 670 | { 671 | Name: "tcp-default-1", 672 | Protocol: v1.ProtocolTCP, 673 | Port: int32(22), 674 | TargetPort: intstr.FromInt(22), 675 | NodePort: 31999, 676 | }, 677 | { 678 | Name: "port-80", 679 | Protocol: v1.ProtocolTCP, 680 | Port: int32(80), 681 | TargetPort: intstr.FromInt(8888), 682 | NodePort: 31900, 683 | }, 684 | { 685 | Name: "http-default-1", 686 | Protocol: v1.ProtocolTCP, 687 | Port: int32(8080), 688 | TargetPort: intstr.FromInt(8080), 689 | NodePort: 31901, 690 | }, 691 | { 692 | Name: "to-be-removed", 693 | Protocol: v1.ProtocolTCP, 694 | Port: int32(8081), 695 | TargetPort: intstr.FromInt(8081), 696 | NodePort: 31902, 697 | }, 698 | } 699 | _, err = svc.Client.CoreV1().Services(svc.Namespace).Update(ctx, service, metav1.UpdateOptions{}) 700 | require.NoError(t, err) 701 | webSvc := v1.Service{ 702 | ObjectMeta: metav1.ObjectMeta{ 703 | Name: "test-web", 704 | Namespace: "default", 705 | Labels: map[string]string{appLabel: "test", processLabel: "web"}, 706 | }, 707 | Spec: v1.ServiceSpec{ 708 | Selector: map[string]string{"name": "test-web"}, 709 | Ports: []v1.ServicePort{ 710 | {Name: "http-default-1", Protocol: "TCP", Port: int32(8080), TargetPort: intstr.FromInt(8080), NodePort: 12000}, 711 | {Name: "tcp-default-1", Protocol: "TCP", Port: int32(22), TargetPort: intstr.FromInt(22), NodePort: 12001}, 712 | }, 713 | }, 714 | } 715 | _, err = svc.Client.CoreV1().Services(svc.Namespace).Update(ctx, &webSvc, metav1.UpdateOptions{}) 716 | require.NoError(t, err) 717 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 718 | Opts: router.Opts{ 719 | AdditionalOpts: map[string]string{ 720 | exposeAllPortsOpt: "true", 721 | }, 722 | }, 723 | Prefixes: []router.BackendPrefix{ 724 | { 725 | Target: router.BackendTarget{ 726 | Service: "test-web", 727 | Namespace: svc.Namespace, 728 | }, 729 | }, 730 | }, 731 | }) 732 | require.NoError(t, err) 733 | service, err = svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, svc.serviceName(idForApp("test")), metav1.GetOptions{}) 734 | require.NoError(t, err) 735 | assert.Equal(t, []v1.ServicePort{ 736 | { 737 | Name: "http-default-1-extra", 738 | Protocol: v1.ProtocolTCP, 739 | Port: int32(80), 740 | TargetPort: intstr.FromInt(8080), 741 | NodePort: 31900, 742 | }, 743 | { 744 | Name: "http-default-1", 745 | Protocol: v1.ProtocolTCP, 746 | Port: int32(8080), 747 | TargetPort: intstr.FromInt(8080), 748 | NodePort: 31901, 749 | }, 750 | { 751 | Name: "tcp-default-1", 752 | Protocol: v1.ProtocolTCP, 753 | Port: int32(22), 754 | TargetPort: intstr.FromInt(22), 755 | NodePort: 31999, 756 | }, 757 | }, service.Spec.Ports) 758 | 759 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 760 | Opts: router.Opts{ 761 | AdditionalOpts: map[string]string{ 762 | exposeAllPortsOpt: "true", 763 | }, 764 | }, 765 | Prefixes: []router.BackendPrefix{ 766 | { 767 | Target: router.BackendTarget{ 768 | Service: "test-web", 769 | Namespace: svc.Namespace, 770 | }, 771 | }, 772 | }, 773 | }) 774 | require.NoError(t, err) 775 | 776 | service, err = svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, svc.serviceName(idForApp("test")), metav1.GetOptions{}) 777 | require.NoError(t, err) 778 | assert.Equal(t, []v1.ServicePort{ 779 | { 780 | Name: "http-default-1-extra", 781 | Protocol: v1.ProtocolTCP, 782 | Port: int32(80), 783 | TargetPort: intstr.FromInt(8080), 784 | NodePort: 31900, 785 | }, 786 | { 787 | Name: "http-default-1", 788 | Protocol: v1.ProtocolTCP, 789 | Port: int32(8080), 790 | TargetPort: intstr.FromInt(8080), 791 | NodePort: 31901, 792 | }, 793 | { 794 | Name: "tcp-default-1", 795 | Protocol: v1.ProtocolTCP, 796 | Port: int32(22), 797 | TargetPort: intstr.FromInt(22), 798 | NodePort: 31999, 799 | }, 800 | }, service.Spec.Ports) 801 | 802 | } 803 | 804 | func TestLBUpdateNoChangeInFrozenService(t *testing.T) { 805 | svc := createFakeLBService() 806 | err := createAppWebService(svc.Client, svc.Namespace, "test") 807 | require.NoError(t, err) 808 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 809 | Opts: router.Opts{ 810 | AdditionalOpts: map[string]string{ 811 | exposeAllPortsOpt: "true", 812 | }, 813 | }, 814 | Prefixes: []router.BackendPrefix{ 815 | { 816 | Target: router.BackendTarget{ 817 | Service: "test-web", 818 | Namespace: svc.Namespace, 819 | }, 820 | }, 821 | }, 822 | }) 823 | require.NoError(t, err) 824 | service, err := svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, svc.serviceName(idForApp("test")), metav1.GetOptions{}) 825 | require.NoError(t, err) 826 | service.Labels = map[string]string{ 827 | routerFreezeLabel: "true", 828 | } 829 | service.Spec.Ports = []v1.ServicePort{ 830 | { 831 | Name: "tcp-default-1", 832 | Protocol: v1.ProtocolTCP, 833 | Port: int32(1234), 834 | TargetPort: intstr.FromInt(22), 835 | NodePort: 31999, 836 | }, 837 | } 838 | _, err = svc.Client.CoreV1().Services(svc.Namespace).Update(ctx, service, metav1.UpdateOptions{}) 839 | require.NoError(t, err) 840 | webSvc := v1.Service{ 841 | ObjectMeta: metav1.ObjectMeta{ 842 | Name: "test-web", 843 | Namespace: "default", 844 | Labels: map[string]string{appLabel: "test", processLabel: "web"}, 845 | }, 846 | Spec: v1.ServiceSpec{ 847 | Selector: map[string]string{"name": "test-web"}, 848 | Ports: []v1.ServicePort{ 849 | {Name: "tcp-default-1", Protocol: "TCP", Port: int32(22), TargetPort: intstr.FromInt(22), NodePort: 12001}, 850 | }, 851 | }, 852 | } 853 | _, err = svc.Client.CoreV1().Services(svc.Namespace).Update(ctx, &webSvc, metav1.UpdateOptions{}) 854 | require.NoError(t, err) 855 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 856 | Opts: router.Opts{ 857 | AdditionalOpts: map[string]string{ 858 | exposeAllPortsOpt: "true", 859 | }, 860 | }, 861 | Prefixes: []router.BackendPrefix{ 862 | { 863 | Target: router.BackendTarget{ 864 | Service: "test-web", 865 | Namespace: svc.Namespace, 866 | }, 867 | }, 868 | }, 869 | }) 870 | require.NoError(t, err) 871 | service, err = svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, svc.serviceName(idForApp("test")), metav1.GetOptions{}) 872 | require.NoError(t, err) 873 | assert.Equal(t, []v1.ServicePort{ 874 | { 875 | Name: "tcp-default-1", 876 | Protocol: v1.ProtocolTCP, 877 | Port: int32(1234), 878 | TargetPort: intstr.FromInt(22), 879 | NodePort: 31999, 880 | }, 881 | }, service.Spec.Ports) 882 | } 883 | 884 | func TestGetStatus(t *testing.T) { 885 | svc := createFakeLBService() 886 | 887 | err := createAppWebService(svc.Client, svc.Namespace, "test") 888 | require.NoError(t, err) 889 | 890 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 891 | Prefixes: []router.BackendPrefix{ 892 | { 893 | Target: router.BackendTarget{ 894 | Service: "test-web", 895 | Namespace: svc.Namespace, 896 | }, 897 | }, 898 | }, 899 | }) 900 | require.NoError(t, err) 901 | 902 | s, err := svc.getLBService(ctx, idForApp("test")) 903 | require.NoError(t, err) 904 | 905 | _, err = svc.BaseService.Client.CoreV1().Events("default").Create(ctx, &v1.Event{ 906 | ObjectMeta: metav1.ObjectMeta{ 907 | Name: "test-1234", 908 | CreationTimestamp: metav1.NewTime(time.Now()), 909 | }, 910 | InvolvedObject: v1.ObjectReference{ 911 | Name: s.Name, 912 | UID: s.UID, 913 | Kind: "Service", 914 | }, 915 | Type: "Warning", 916 | Reason: "Unknown reason", 917 | Message: "Failed to ensure loadbalancer", 918 | }, metav1.CreateOptions{}) 919 | require.NoError(t, err) 920 | 921 | status, detail, err := svc.GetStatus(ctx, idForApp("test")) 922 | require.NoError(t, err) 923 | 924 | assert.Equal(t, status, router.BackendStatusNotReady) 925 | assert.Contains(t, detail, "Warning - Failed to ensure loadbalancer") 926 | 927 | s.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{ 928 | { 929 | Hostname: "testing", 930 | IP: "66.66.66.66", 931 | }, 932 | } 933 | 934 | _, err = svc.BaseService.Client.CoreV1().Services("default").UpdateStatus(ctx, s, metav1.UpdateOptions{}) 935 | require.NoError(t, err) 936 | 937 | status, detail, err = svc.GetStatus(ctx, idForApp("test")) 938 | require.NoError(t, err) 939 | 940 | assert.Equal(t, status, router.BackendStatusReady) 941 | assert.Contains(t, detail, "") 942 | 943 | s.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{ 944 | { 945 | Hostname: "mylb.elb.ZONE.amazonaws.com", 946 | }, 947 | } 948 | 949 | _, err = svc.BaseService.Client.CoreV1().Services("default").UpdateStatus(ctx, s, metav1.UpdateOptions{}) 950 | require.NoError(t, err) 951 | 952 | status, detail, err = svc.GetStatus(ctx, idForApp("test")) 953 | require.NoError(t, err) 954 | 955 | assert.Equal(t, status, router.BackendStatusReady) 956 | assert.Contains(t, detail, "") 957 | } 958 | 959 | func TestGetAddresses(t *testing.T) { 960 | svc := createFakeLBService() 961 | 962 | err := createAppWebService(svc.Client, svc.Namespace, "test") 963 | require.NoError(t, err) 964 | 965 | err = svc.Ensure(ctx, idForApp("test"), router.EnsureBackendOpts{ 966 | Prefixes: []router.BackendPrefix{ 967 | { 968 | Target: router.BackendTarget{ 969 | Service: "test-web", 970 | Namespace: svc.Namespace, 971 | }, 972 | }, 973 | }, 974 | }) 975 | require.NoError(t, err) 976 | 977 | s, err := svc.getLBService(ctx, idForApp("test")) 978 | require.NoError(t, err) 979 | 980 | addresses, err := svc.GetAddresses(ctx, idForApp("test")) 981 | require.NoError(t, err) 982 | assert.Equal(t, []string{""}, addresses) 983 | 984 | s.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{ 985 | { 986 | Hostname: "testing.io", 987 | IP: "66.66.66.66", 988 | }, 989 | } 990 | _, err = svc.BaseService.Client.CoreV1().Services("default").UpdateStatus(ctx, s, metav1.UpdateOptions{}) 991 | require.NoError(t, err) 992 | 993 | addresses, err = svc.GetAddresses(ctx, idForApp("test")) 994 | require.NoError(t, err) 995 | assert.Equal(t, []string{"testing.io"}, addresses) 996 | 997 | s.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{ 998 | { 999 | Hostname: "mylb.elb.ZONE.amazonaws.com", 1000 | }, 1001 | } 1002 | _, err = svc.BaseService.Client.CoreV1().Services("default").UpdateStatus(ctx, s, metav1.UpdateOptions{}) 1003 | require.NoError(t, err) 1004 | 1005 | addresses, err = svc.GetAddresses(ctx, idForApp("test")) 1006 | require.NoError(t, err) 1007 | 1008 | assert.Equal(t, []string{"mylb.elb.ZONE.amazonaws.com"}, addresses) 1009 | 1010 | s.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{ 1011 | { 1012 | IP: "66.66.66.66", 1013 | }, 1014 | } 1015 | s.Annotations[externalDNSHostnameLabel] = "myapp.zone.io,myapp.com" 1016 | _, err = svc.BaseService.Client.CoreV1().Services("default").UpdateStatus(ctx, s, metav1.UpdateOptions{}) 1017 | require.NoError(t, err) 1018 | 1019 | addresses, err = svc.GetAddresses(ctx, idForApp("test")) 1020 | require.NoError(t, err) 1021 | 1022 | assert.Equal(t, []string{"myapp.zone.io", "myapp.com"}, addresses) 1023 | } 1024 | 1025 | func defaultService(app, namespace string, labels, annotations, selector map[string]string) *v1.Service { 1026 | if selector == nil { 1027 | selector = map[string]string{ 1028 | "tsuru.io/app-name": app, 1029 | "tsuru.io/app-process": "web", 1030 | } 1031 | } 1032 | svc := v1.Service{ 1033 | ObjectMeta: metav1.ObjectMeta{ 1034 | Name: app + "-router-lb", 1035 | Namespace: namespace, 1036 | Labels: map[string]string{ 1037 | appLabel: app, 1038 | managedServiceLabel: "true", 1039 | externalServiceLabel: "true", 1040 | appPoolLabel: "", 1041 | appBaseServiceNameLabel: app + "-web", 1042 | appBaseServiceNamespaceLabel: namespace, 1043 | }, 1044 | Annotations: annotations, 1045 | }, 1046 | Spec: v1.ServiceSpec{ 1047 | Selector: selector, 1048 | Type: v1.ServiceTypeLoadBalancer, 1049 | Ports: []v1.ServicePort{ 1050 | { 1051 | Name: fmt.Sprintf("port-%d", defaultLBPort), 1052 | Protocol: "TCP", 1053 | Port: int32(defaultLBPort), 1054 | TargetPort: intstr.FromInt(defaultServicePort), 1055 | }, 1056 | }, 1057 | }, 1058 | Status: v1.ServiceStatus{ 1059 | LoadBalancer: v1.LoadBalancerStatus{ 1060 | Ingress: []v1.LoadBalancerIngress{ 1061 | {IP: "127.0.0.1"}, 1062 | }, 1063 | }, 1064 | }, 1065 | } 1066 | for k, v := range labels { 1067 | svc.ObjectMeta.Labels[k] = v 1068 | } 1069 | return &svc 1070 | } 1071 | 1072 | func setIP(t *testing.T, svc LBService, appName string) { 1073 | service, err := svc.Client.CoreV1().Services(svc.Namespace).Get(ctx, svc.serviceName(idForApp(appName)), metav1.GetOptions{}) 1074 | if err != nil { 1075 | t.Fatalf("Expected err to be nil. Got %v", err) 1076 | } 1077 | service.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{IP: "127.0.0.1"}} 1078 | _, err = svc.Client.CoreV1().Services(svc.Namespace).Update(ctx, service, metav1.UpdateOptions{}) 1079 | if err != nil { 1080 | t.Fatalf("Expected err to be nil. Got %v", err) 1081 | } 1082 | } 1083 | -------------------------------------------------------------------------------- /kubernetes/service.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package kubernetes 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "crypto/sha256" 11 | "errors" 12 | "fmt" 13 | "net/http" 14 | "sort" 15 | "strconv" 16 | "strings" 17 | "time" 18 | 19 | certmanagerv1clientset "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned" 20 | "github.com/tsuru/kubernetes-router/observability" 21 | "github.com/tsuru/kubernetes-router/router" 22 | tsuruv1 "github.com/tsuru/tsuru/provision/kubernetes/pkg/apis/tsuru/v1" 23 | tsuruv1clientset "github.com/tsuru/tsuru/provision/kubernetes/pkg/client/clientset/versioned" 24 | corev1 "k8s.io/api/core/v1" 25 | apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 26 | k8sErrors "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/labels" 29 | "k8s.io/apimachinery/pkg/types" 30 | "k8s.io/client-go/kubernetes" 31 | "k8s.io/client-go/rest" 32 | "k8s.io/client-go/transport" 33 | sigsk8sclient "sigs.k8s.io/controller-runtime/pkg/client" 34 | ) 35 | 36 | const ( 37 | // managedServiceLabel is added to every service created by the router 38 | managedServiceLabel = "tsuru.io/router-lb" 39 | // externalServiceLabel should be added to every service with tsuru app 40 | // labels that are NOT created or managed by tsuru itself. 41 | externalServiceLabel = "tsuru.io/external-controller" 42 | 43 | appBaseServiceNamespaceLabel = "router.tsuru.io/base-service-namespace" 44 | appBaseServiceNameLabel = "router.tsuru.io/base-service-name" 45 | routerFreezeLabel = "router.tsuru.io/freeze" 46 | 47 | externalDNSHostnameLabel = "external-dns.alpha.kubernetes.io/hostname" 48 | 49 | defaultServicePort = 8888 50 | appLabel = "tsuru.io/app-name" 51 | teamLabel = "tsuru.io/app-team" 52 | domainLabel = "tsuru.io/domain-name" 53 | processLabel = "tsuru.io/app-process" 54 | appPoolLabel = "tsuru.io/app-pool" 55 | 56 | customTagPrefixLabel = "tsuru.io/custom-tag-" 57 | 58 | appCRDName = "apps.tsuru.io" 59 | ) 60 | 61 | var ( 62 | ErrNoBackendTarget = errors.New("No default backend target found") 63 | ) 64 | 65 | // ErrNoService indicates that the app has no service running 66 | type ErrNoService struct { 67 | App string 68 | Service string 69 | } 70 | 71 | func (e ErrNoService) Error() string { 72 | return fmt.Sprintf("service %q is not found for app %q", e.Service, e.App) 73 | } 74 | 75 | // BaseService has the base functionality needed by router.Service implementations 76 | // targeting kubernetes 77 | type BaseService struct { 78 | Namespace string 79 | Timeout time.Duration 80 | RestConfig *rest.Config 81 | Client kubernetes.Interface 82 | TsuruClient tsuruv1clientset.Interface 83 | CertManagerClient certmanagerv1clientset.Interface 84 | SigsClient sigsk8sclient.Client 85 | ExtensionsClient apiextensionsclientset.Interface 86 | Labels map[string]string 87 | Annotations map[string]string 88 | } 89 | 90 | // SupportedOptions returns the options supported by all services 91 | func (k *BaseService) SupportedOptions(ctx context.Context) map[string]string { 92 | return nil 93 | } 94 | 95 | // Healthcheck uses the kubernetes client to check the connectivity 96 | func (k *BaseService) Healthcheck(ctx context.Context) error { 97 | client, err := k.getClient() 98 | if err != nil { 99 | return err 100 | } 101 | _, err = client.CoreV1().Services(k.Namespace).List(ctx, metav1.ListOptions{}) 102 | return err 103 | } 104 | 105 | func (k *BaseService) getClient() (kubernetes.Interface, error) { 106 | if k.Client != nil { 107 | return k.Client, nil 108 | } 109 | config, err := k.getConfig() 110 | if err != nil { 111 | return nil, err 112 | } 113 | k.Client, err = kubernetes.NewForConfig(config) 114 | return k.Client, err 115 | } 116 | 117 | func (k *BaseService) getTsuruClient() (tsuruv1clientset.Interface, error) { 118 | if k.TsuruClient != nil { 119 | return k.TsuruClient, nil 120 | } 121 | config, err := k.getConfig() 122 | if err != nil { 123 | return nil, err 124 | } 125 | k.TsuruClient, err = tsuruv1clientset.NewForConfig(config) 126 | return k.TsuruClient, err 127 | } 128 | 129 | func (k *BaseService) getExtensionsClient() (apiextensionsclientset.Interface, error) { 130 | if k.ExtensionsClient != nil { 131 | return k.ExtensionsClient, nil 132 | } 133 | config, err := k.getConfig() 134 | if err != nil { 135 | return nil, err 136 | } 137 | k.ExtensionsClient, err = apiextensionsclientset.NewForConfig(config) 138 | return k.ExtensionsClient, err 139 | } 140 | 141 | func (k BaseService) getCertManagerClient() (certmanagerv1clientset.Interface, error) { 142 | if k.CertManagerClient != nil { 143 | return k.CertManagerClient, nil 144 | } 145 | config, err := k.getConfig() 146 | if err != nil { 147 | return nil, err 148 | } 149 | return certmanagerv1clientset.NewForConfig(config) 150 | } 151 | 152 | func (k *BaseService) getSigsClient() (sigsk8sclient.Client, error) { 153 | if k.SigsClient != nil { 154 | return k.SigsClient, nil 155 | } 156 | config, err := k.getConfig() 157 | if err != nil { 158 | return nil, err 159 | } 160 | k.SigsClient, err = sigsk8sclient.New(config, sigsk8sclient.Options{}) 161 | return k.SigsClient, err 162 | } 163 | 164 | func (k *BaseService) getConfig() (*rest.Config, error) { 165 | if k.RestConfig != nil { 166 | return k.RestConfig, nil 167 | } 168 | var err error 169 | k.RestConfig, err = rest.InClusterConfig() 170 | if err != nil { 171 | return nil, err 172 | } 173 | k.RestConfig.Timeout = k.Timeout 174 | k.RestConfig.WrapTransport = func(rt http.RoundTripper) http.RoundTripper { 175 | return transport.DebugWrappers(observability.WrapTransport(rt)) 176 | } 177 | return k.RestConfig, nil 178 | } 179 | 180 | func (k *BaseService) getWebService(ctx context.Context, appName string, target router.BackendTarget) (*corev1.Service, error) { 181 | client, err := k.getClient() 182 | if err != nil { 183 | return nil, err 184 | } 185 | 186 | svc, err := client.CoreV1().Services(target.Namespace).Get(ctx, target.Service, metav1.GetOptions{}) 187 | if err != nil { 188 | if k8sErrors.IsNotFound(err) { 189 | return nil, ErrNoService{App: appName, Service: target.Service} 190 | } 191 | return nil, err 192 | } 193 | return svc, nil 194 | } 195 | 196 | func (k *BaseService) getApp(ctx context.Context, app string) (*tsuruv1.App, error) { 197 | hasCRD, err := k.hasCRD(ctx) 198 | if err != nil { 199 | return nil, err 200 | } 201 | if !hasCRD { 202 | return nil, nil 203 | } 204 | tclient, err := k.getTsuruClient() 205 | if err != nil { 206 | return nil, err 207 | } 208 | return tclient.TsuruV1().Apps(k.Namespace).Get(ctx, app, metav1.GetOptions{}) 209 | } 210 | 211 | func (k *BaseService) getAppNamespace(ctx context.Context, appName string) (string, error) { 212 | app, err := k.getApp(ctx, appName) 213 | if err != nil { 214 | return "", err 215 | } 216 | if app == nil { 217 | return k.Namespace, nil 218 | } 219 | return app.Spec.NamespaceName, nil 220 | } 221 | 222 | func (k *BaseService) hasCRD(ctx context.Context) (bool, error) { 223 | eclient, err := k.getExtensionsClient() 224 | if err != nil { 225 | return false, err 226 | } 227 | _, err = eclient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, appCRDName, metav1.GetOptions{}) 228 | if err != nil { 229 | if k8sErrors.IsNotFound(err) { 230 | return false, nil 231 | } 232 | return false, err 233 | } 234 | return true, nil 235 | } 236 | 237 | func (s *BaseService) getDefaultBackendTarget(prefixes []router.BackendPrefix) (*router.BackendTarget, error) { 238 | for _, prefix := range prefixes { 239 | if prefix.Prefix == "" { 240 | return &prefix.Target, nil 241 | } 242 | } 243 | 244 | return nil, ErrNoBackendTarget 245 | } 246 | 247 | func addAllBackends(prefixes []router.BackendPrefix) map[string]router.BackendTarget { 248 | allTargets := map[string]router.BackendTarget{} 249 | for _, prefix := range prefixes { 250 | if prefix.Prefix == "" { 251 | allTargets["default"] = prefix.Target 252 | continue 253 | } 254 | prefixSanitized := strings.ReplaceAll(prefix.Prefix, "_", "-") 255 | allTargets[prefixSanitized] = prefix.Target 256 | } 257 | return allTargets 258 | } 259 | 260 | // getBackendTargets returns all targets pointed by the app services or only the base target according to the allBackends flag 261 | func (s *BaseService) getBackendTargets(prefixes []router.BackendPrefix, allBackends bool) (map[string]router.BackendTarget, error) { 262 | allTargets := map[string]router.BackendTarget{} 263 | if allBackends { 264 | allTargets = addAllBackends(prefixes) 265 | } else { 266 | baseTarget, err := s.getDefaultBackendTarget(prefixes) 267 | if err != nil { 268 | return nil, err 269 | } 270 | if baseTarget != nil { 271 | allTargets["default"] = *baseTarget 272 | } 273 | } 274 | if len(allTargets) <= 0 { 275 | return nil, ErrNoBackendTarget 276 | } 277 | 278 | return allTargets, nil 279 | } 280 | 281 | func (s *BaseService) hashedResourceName(id router.InstanceID, name string, limit int) string { 282 | if id.InstanceName != "" { 283 | name += "-" + id.InstanceName 284 | } 285 | if len(name) <= limit { 286 | return name 287 | } 288 | 289 | h := sha256.New() 290 | h.Write([]byte(name)) 291 | hash := fmt.Sprintf("%x", h.Sum(nil)) 292 | return fmt.Sprintf("%s-%s", name[:limit-17], hash[:16]) 293 | } 294 | 295 | func (s *BaseService) getStatusForRuntimeObject(ctx context.Context, ns string, kind string, uid types.UID) (string, error) { 296 | client, err := s.getClient() 297 | if err != nil { 298 | return "", err 299 | } 300 | selector := map[string]string{ 301 | "involvedObject.kind": kind, 302 | "involvedObject.uid": string(uid), 303 | } 304 | 305 | eventList, err := client.CoreV1().Events(ns).List(ctx, metav1.ListOptions{ 306 | FieldSelector: labels.SelectorFromSet(labels.Set(selector)).String(), 307 | }) 308 | if err != nil { 309 | return "", err 310 | } 311 | 312 | var buf bytes.Buffer 313 | reasonMap := map[string]bool{} 314 | sort.Slice(eventList.Items, func(i, j int) bool { 315 | return eventList.Items[i].CreationTimestamp.After(eventList.Items[j].CreationTimestamp.Time) 316 | }) 317 | 318 | for _, event := range eventList.Items { 319 | if reasonMap[event.Reason] { 320 | continue 321 | } 322 | reasonMap[event.Reason] = true 323 | 324 | fmt.Fprintf(&buf, "%s - %s - %s\n", event.CreationTimestamp.Format(time.RFC3339), event.Type, event.Message) 325 | } 326 | 327 | return buf.String(), nil 328 | } 329 | 330 | func isFrozenSvc(svc *corev1.Service) bool { 331 | if svc == nil || svc.Labels == nil { 332 | return false 333 | } 334 | frozen, _ := strconv.ParseBool(svc.Labels[routerFreezeLabel]) 335 | return frozen 336 | } 337 | -------------------------------------------------------------------------------- /kubernetes/service_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package kubernetes 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | "github.com/tsuru/kubernetes-router/router" 13 | tsuruv1 "github.com/tsuru/tsuru/provision/kubernetes/pkg/apis/tsuru/v1" 14 | faketsuru "github.com/tsuru/tsuru/provision/kubernetes/pkg/client/clientset/versioned/fake" 15 | "github.com/tsuru/tsuru/types/provision" 16 | v1 "k8s.io/api/core/v1" 17 | apiextensionsV1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" 18 | fakeapiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" 19 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 20 | "k8s.io/apimachinery/pkg/util/intstr" 21 | "k8s.io/client-go/kubernetes/fake" 22 | ) 23 | 24 | func TestGetWebService(t *testing.T) { 25 | svc := BaseService{ 26 | Namespace: "default", 27 | Client: fake.NewSimpleClientset(), 28 | TsuruClient: faketsuru.NewSimpleClientset(), 29 | ExtensionsClient: fakeapiextensions.NewSimpleClientset(), 30 | } 31 | 32 | _, err := svc.getWebService(ctx, "test", router.BackendTarget{Service: "test-not-found", Namespace: svc.Namespace}) 33 | assert.Equal(t, ErrNoService{App: "test", Service: "test-not-found"}, err) 34 | 35 | svc1 := v1.Service{ObjectMeta: metav1.ObjectMeta{ 36 | Name: "test-single", 37 | Namespace: "default", 38 | Labels: map[string]string{appLabel: "test"}, 39 | }, 40 | Spec: v1.ServiceSpec{ 41 | Selector: map[string]string{"name": "test-single"}, 42 | Ports: []v1.ServicePort{{Protocol: "TCP", Port: int32(8899), TargetPort: intstr.FromInt(8899)}}, 43 | }, 44 | } 45 | _, err = svc.Client.CoreV1().Services(svc.Namespace).Create(ctx, &svc1, metav1.CreateOptions{}) 46 | require.NoError(t, err) 47 | webService, err := svc.getWebService(ctx, "test", router.BackendTarget{Service: svc1.Name, Namespace: svc1.Namespace}) 48 | require.NoError(t, err) 49 | assert.Equal(t, "test-single", webService.Name) 50 | 51 | err = createCRD(&svc, "namespacedApp", "custom-namespace", nil) 52 | require.NoError(t, err) 53 | 54 | svc3 := v1.Service{ 55 | ObjectMeta: metav1.ObjectMeta{ 56 | Name: "namespacedApp-web", 57 | Namespace: "custom-namespace", 58 | Labels: map[string]string{appLabel: "namespacedApp", processLabel: "web"}, 59 | }, 60 | Spec: v1.ServiceSpec{ 61 | Selector: map[string]string{"name": "namespacedApp-web"}, 62 | Ports: []v1.ServicePort{{Protocol: "TCP", Port: int32(8890), TargetPort: intstr.FromInt(8890)}}, 63 | }, 64 | } 65 | _, err = svc.Client.CoreV1().Services(svc3.Namespace).Create(ctx, &svc3, metav1.CreateOptions{}) 66 | require.NoError(t, err) 67 | 68 | webService, err = svc.getWebService(ctx, "namespacedApp", router.BackendTarget{Service: svc3.Name, Namespace: svc3.Namespace}) 69 | require.NoError(t, err) 70 | assert.Equal(t, "namespacedApp-web", webService.Name) 71 | } 72 | 73 | func createCRD(svc *BaseService, app string, namespace string, configs *provision.TsuruYamlKubernetesConfig) error { 74 | _, err := svc.ExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &apiextensionsV1.CustomResourceDefinition{ 75 | ObjectMeta: metav1.ObjectMeta{Name: "apps.tsuru.io"}, 76 | Spec: apiextensionsV1.CustomResourceDefinitionSpec{ 77 | Group: "tsuru.io", 78 | Versions: []apiextensionsV1.CustomResourceDefinitionVersion{{Name: "v1"}}, 79 | Names: apiextensionsV1.CustomResourceDefinitionNames{ 80 | Plural: "apps", 81 | Singular: "app", 82 | Kind: "App", 83 | ListKind: "AppList", 84 | }, 85 | }, 86 | }, metav1.CreateOptions{}) 87 | if err != nil { 88 | return err 89 | } 90 | _, err = svc.TsuruClient.TsuruV1().Apps(svc.Namespace).Create(ctx, &tsuruv1.App{ 91 | ObjectMeta: metav1.ObjectMeta{Name: app}, 92 | Spec: tsuruv1.AppSpec{ 93 | NamespaceName: namespace, 94 | Configs: configs, 95 | }, 96 | }, metav1.CreateOptions{}) 97 | return err 98 | } 99 | 100 | func idForApp(appName string) router.InstanceID { 101 | return router.InstanceID{AppName: appName} 102 | } 103 | -------------------------------------------------------------------------------- /observability/middlware.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package observability 6 | 7 | import ( 8 | "net/http" 9 | 10 | "github.com/opentracing/opentracing-go" 11 | opentracingExt "github.com/opentracing/opentracing-go/ext" 12 | "github.com/uber/jaeger-client-go" 13 | "github.com/urfave/negroni" 14 | ) 15 | 16 | func Middleware() negroni.Handler { 17 | return &middleware{} 18 | } 19 | 20 | type middleware struct{} 21 | 22 | func (*middleware) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { 23 | tracer := opentracing.GlobalTracer() 24 | tags := []opentracing.StartSpanOption{ 25 | opentracingExt.SpanKindRPCServer, 26 | opentracing.Tag{Key: "component", Value: "api"}, 27 | opentracing.Tag{Key: "request_id", Value: r.Header.Get("X-Request-ID")}, 28 | opentracing.Tag{Key: "http.method", Value: r.Method}, 29 | opentracing.Tag{Key: "http.url", Value: r.RequestURI}, 30 | } 31 | wireContext, err := tracer.Extract( 32 | opentracing.HTTPHeaders, 33 | opentracing.HTTPHeadersCarrier(r.Header)) 34 | 35 | if err == nil { 36 | jaegerContext, isJaeger := wireContext.(*jaeger.SpanContext) 37 | if !isJaeger || jaegerContext.IsSampled() { 38 | // it's force all unsampled spans to be re-calculated and enforces use of const sampler with param 1 39 | tags = append(tags, opentracing.ChildOf(wireContext)) 40 | } 41 | } 42 | span := tracer.StartSpan(r.Method, tags...) 43 | defer span.Finish() 44 | ctx := opentracing.ContextWithSpan(r.Context(), span) 45 | newR := r.WithContext(ctx) 46 | 47 | next(rw, newR) 48 | statusCode := rw.(negroni.ResponseWriter).Status() 49 | if statusCode == 0 { 50 | statusCode = 200 51 | } 52 | span.SetTag("http.status_code", statusCode) 53 | if statusCode >= http.StatusInternalServerError { 54 | opentracingExt.Error.Set(span, true) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /observability/observability.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package observability 6 | 7 | import ( 8 | "log" 9 | 10 | opentracing "github.com/opentracing/opentracing-go" 11 | jaegerConfig "github.com/uber/jaeger-client-go/config" 12 | "github.com/uber/jaeger-client-go/zipkin" 13 | ) 14 | 15 | func init() { 16 | // We decided to use B3 Format, in the future plan to move to W3C context propagation 17 | // https://github.com/w3c/trace-context 18 | zipkinPropagator := zipkin.NewZipkinB3HTTPHeaderPropagator() 19 | 20 | // setup opentracing 21 | cfg, err := jaegerConfig.FromEnv() 22 | if err != nil { 23 | log.Fatal(err.Error()) 24 | } 25 | cfg.ServiceName = "kubernetes-router" 26 | 27 | tracer, _, err := cfg.NewTracer( 28 | jaegerConfig.Injector(opentracing.HTTPHeaders, zipkinPropagator), 29 | jaegerConfig.Extractor(opentracing.HTTPHeaders, zipkinPropagator), 30 | jaegerConfig.Injector(opentracing.TextMap, zipkinPropagator), 31 | jaegerConfig.Extractor(opentracing.TextMap, zipkinPropagator), 32 | ) 33 | if err == nil { 34 | opentracing.SetGlobalTracer(tracer) 35 | } else { 36 | // FIXME: we need to mark that traces are disabled 37 | log.Printf("Could not initialize jaeger tracer: %s", err.Error()) 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /observability/transport.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package observability 6 | 7 | import ( 8 | "io" 9 | "net/http" 10 | 11 | opentracingHTTP "github.com/opentracing-contrib/go-stdlib/nethttp" 12 | "github.com/opentracing/opentracing-go" 13 | ) 14 | 15 | func WrapTransport(rt http.RoundTripper) http.RoundTripper { 16 | return &AutoOpentracingTransport{RoundTripper: rt} 17 | } 18 | 19 | type AutoOpentracingTransport struct { 20 | http.RoundTripper 21 | } 22 | 23 | func (t *AutoOpentracingTransport) RoundTrip(req *http.Request) (*http.Response, error) { 24 | rt := t.RoundTripper 25 | tracer := opentracing.GlobalTracer() 26 | if rt == nil { 27 | rt = http.DefaultTransport 28 | } 29 | 30 | req, ht := opentracingHTTP.TraceRequest(tracer, req) 31 | 32 | transport := &opentracingHTTP.Transport{RoundTripper: rt} 33 | response, err := transport.RoundTrip(req) 34 | 35 | if err != nil { 36 | ht.Finish() 37 | return nil, err 38 | } 39 | response.Body = &autoCloseTracer{ht: ht, ReadCloser: response.Body} 40 | return response, nil 41 | } 42 | 43 | type autoCloseTracer struct { 44 | io.ReadCloser 45 | ht *opentracingHTTP.Tracer 46 | } 47 | 48 | func (a *autoCloseTracer) Close() error { 49 | err := a.ReadCloser.Close() 50 | a.ht.Finish() 51 | return err 52 | } 53 | -------------------------------------------------------------------------------- /router/mock/service.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package mock 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/tsuru/kubernetes-router/router" 11 | ) 12 | 13 | var _ router.Router = &RouterMock{} 14 | 15 | // RouterMock is a router.Router mock implementation to be 16 | // used by tests 17 | type RouterMock struct { 18 | EnsureFn func(router.InstanceID, router.EnsureBackendOpts) error 19 | RemoveFn func(router.InstanceID) error 20 | GetAddressesFn func(router.InstanceID) ([]string, error) 21 | GetStatusFn func(router.InstanceID) (router.BackendStatus, string, error) 22 | GetCertificateFn func(router.InstanceID, string) (*router.CertData, error) 23 | AddCertificateFn func(router.InstanceID, string, router.CertData) error 24 | RemoveCertificateFn func(router.InstanceID, string) error 25 | SupportedOptionsFn func() map[string]string 26 | RemoveInvoked bool 27 | EnsureInvoked bool 28 | GetAddressesInvoked bool 29 | AddCertificateInvoked bool 30 | GetCertificateInvoked bool 31 | RemoveCertificateInvoked bool 32 | SupportedOptionsInvoked bool 33 | GetStatusInvoked bool 34 | } 35 | 36 | // Remove calls RemoveFn 37 | func (s *RouterMock) Remove(ctx context.Context, id router.InstanceID) error { 38 | s.RemoveInvoked = true 39 | return s.RemoveFn(id) 40 | } 41 | 42 | // Update calls UpdateFn 43 | func (s *RouterMock) Ensure(ctx context.Context, id router.InstanceID, o router.EnsureBackendOpts) error { 44 | s.EnsureInvoked = true 45 | return s.EnsureFn(id, o) 46 | } 47 | 48 | // Get calls GetFn 49 | func (s *RouterMock) GetAddresses(ctx context.Context, id router.InstanceID) ([]string, error) { 50 | s.GetAddressesInvoked = true 51 | return s.GetAddressesFn(id) 52 | } 53 | 54 | func (s *RouterMock) GetStatus(ctx context.Context, id router.InstanceID) (router.BackendStatus, string, error) { 55 | s.GetStatusInvoked = true 56 | return s.GetStatusFn(id) 57 | } 58 | 59 | // GetCertificate calls GetCertificate 60 | func (s *RouterMock) GetCertificate(ctx context.Context, id router.InstanceID, certName string) (*router.CertData, error) { 61 | s.GetCertificateInvoked = true 62 | return s.GetCertificateFn(id, certName) 63 | } 64 | 65 | // AddCertificate calls AddCertificate 66 | func (s *RouterMock) AddCertificate(ctx context.Context, id router.InstanceID, certName string, cert router.CertData) error { 67 | s.AddCertificateInvoked = true 68 | return s.AddCertificateFn(id, certName, cert) 69 | } 70 | 71 | // RemoveCertificate calls RemoveCertificate 72 | func (s *RouterMock) RemoveCertificate(ctx context.Context, id router.InstanceID, certName string) error { 73 | s.RemoveCertificateInvoked = true 74 | return s.RemoveCertificateFn(id, certName) 75 | } 76 | 77 | // SupportedOptions calls SupportedOptionsFn 78 | func (s *RouterMock) SupportedOptions(ctx context.Context) map[string]string { 79 | s.SupportedOptionsInvoked = true 80 | return s.SupportedOptionsFn() 81 | } 82 | -------------------------------------------------------------------------------- /router/service.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package router 6 | 7 | import ( 8 | "context" 9 | "encoding/json" 10 | "errors" 11 | "strconv" 12 | "strings" 13 | 14 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 15 | ) 16 | 17 | const ( 18 | // ExposedPort is the exposed port option name 19 | ExposedPort = "exposed-port" 20 | 21 | // Domain is the domain option name 22 | Domain = "domain" 23 | 24 | // Domain suffix is used to append at name of app, ie: myapp. 25 | DomainSuffix = "domain-suffix" 26 | 27 | // Domain prefix is used to prepend at name of app, ie: .myapp. 28 | DomainPrefix = "domain-prefix" 29 | // Route is the route option name 30 | Route = "route" 31 | 32 | // Acme is the acme option name 33 | Acme = "tls-acme" 34 | 35 | HTTPOnly = "http-only" 36 | 37 | // AcmeCName is the acme option for cnames 38 | AcmeCName = "tls-acme-cname" 39 | 40 | ExternalTrafficPolicy = "external-traffic-policy" 41 | 42 | // optsAnnotation is the name of the annotation used to store opts. 43 | optsAnnotation = "router.tsuru.io/opts" 44 | 45 | AllPrefixes = "all-prefixes" 46 | ) 47 | 48 | // ErrIngressAlreadyExists is the error returned by the service when 49 | // trying to create a service that already exists 50 | var ( 51 | ErrIngressAlreadyExists = errors.New("ingress already exists") 52 | ErrCertificateNotFound = errors.New("certificate not found") 53 | ) 54 | 55 | type InstanceID struct { 56 | InstanceName string 57 | AppName string 58 | } 59 | 60 | type BackendStatus string 61 | 62 | var ( 63 | BackendStatusReady = BackendStatus("ready") 64 | BackendStatusNotReady = BackendStatus("not ready") 65 | ) 66 | 67 | // Router implements the basic functionally needed to 68 | // ingresses and/or loadbalancers. 69 | type Router interface { 70 | Ensure(ctx context.Context, id InstanceID, o EnsureBackendOpts) error 71 | Remove(ctx context.Context, id InstanceID) error 72 | GetAddresses(ctx context.Context, id InstanceID) ([]string, error) 73 | SupportedOptions(ctx context.Context) map[string]string 74 | } 75 | 76 | // RouterStatus could report status of backend 77 | type RouterStatus interface { 78 | Router 79 | GetStatus(ctx context.Context, id InstanceID) (status BackendStatus, detail string, err error) 80 | } 81 | 82 | // RouterTLS Certificates interface 83 | type RouterTLS interface { 84 | Router 85 | AddCertificate(ctx context.Context, id InstanceID, certName string, cert CertData) error 86 | GetCertificate(ctx context.Context, id InstanceID, certName string) (*CertData, error) 87 | RemoveCertificate(ctx context.Context, id InstanceID, certName string) error 88 | } 89 | 90 | // Opts used when creating/updating routers 91 | type Opts struct { 92 | Pool string `json:",omitempty"` 93 | ExposedPort string `json:",omitempty"` 94 | Domain string `json:",omitempty"` 95 | Route string `json:",omitempty"` 96 | DomainSuffix string `json:",omitempty"` 97 | DomainPrefix string `json:",omitempty"` 98 | ExternalTrafficPolicy string `json:",omitempty"` 99 | AdditionalOpts map[string]string `json:",omitempty"` 100 | HeaderOpts []string `json:",omitempty"` 101 | Acme bool `json:",omitempty"` 102 | HTTPOnly bool `json:",omitempty"` 103 | AcmeCName bool `json:",omitempty"` 104 | ExposeAllServices bool `json:",omitempty"` 105 | } 106 | 107 | // CertData user when adding certificates 108 | type CertData struct { 109 | Certificate string `json:"certificate"` 110 | Key string `json:"key"` 111 | } 112 | 113 | type BackendPrefix struct { 114 | Prefix string `json:"prefix"` 115 | Target BackendTarget `json:"target"` 116 | } 117 | 118 | type EnsureBackendOpts struct { 119 | Opts Opts `json:"opts"` 120 | CNames []string `json:"cnames"` 121 | Team string `json:"team"` 122 | Tags []string `json:"tags,omitempty"` 123 | CertIssuers map[string]string `json:"certIssuers"` 124 | Prefixes []BackendPrefix `json:"prefixes"` 125 | } 126 | 127 | type BackendTarget struct { 128 | Namespace string `json:"namespace"` 129 | Service string `json:"service"` 130 | } 131 | 132 | func (o *Opts) ToAnnotations() (map[string]string, error) { 133 | data, err := json.Marshal(o) 134 | if err != nil { 135 | return nil, err 136 | } 137 | return map[string]string{ 138 | optsAnnotation: string(data), 139 | }, nil 140 | } 141 | 142 | func OptsFromAnnotations(meta *metav1.ObjectMeta) (Opts, error) { 143 | if meta.Annotations == nil || meta.Annotations[optsAnnotation] == "" { 144 | return Opts{}, nil 145 | } 146 | type rawJsonOpts Opts 147 | var o rawJsonOpts 148 | err := json.Unmarshal([]byte(meta.Annotations[optsAnnotation]), &o) 149 | return Opts(o), err 150 | } 151 | 152 | // UnmarshalJSON unmarshals Opts from a byte array parsing known fields 153 | // and adding all other string fields to AdditionalOpts 154 | func (o *Opts) UnmarshalJSON(bs []byte) (err error) { 155 | m := make(map[string]interface{}) 156 | 157 | if err = json.Unmarshal(bs, &m); err != nil { 158 | return err 159 | } 160 | 161 | for _, headerOpt := range o.HeaderOpts { 162 | parts := strings.SplitN(headerOpt, "=", 2) 163 | if len(parts) == 0 { 164 | continue 165 | } 166 | key := parts[0] 167 | if _, ok := m[key]; ok { 168 | continue 169 | } 170 | var value string 171 | if len(parts) > 1 { 172 | value = parts[1] 173 | } 174 | m[key] = value 175 | } 176 | 177 | if o.AdditionalOpts == nil { 178 | o.AdditionalOpts = make(map[string]string) 179 | } 180 | 181 | for k, v := range m { 182 | strV, ok := v.(string) 183 | if !ok { 184 | continue 185 | } 186 | switch k { 187 | case "tsuru.io/app-pool": 188 | o.Pool = strV 189 | case ExposedPort: 190 | o.ExposedPort = strV 191 | case Domain: 192 | o.Domain = strV 193 | case DomainSuffix: 194 | o.DomainSuffix = strV 195 | case DomainPrefix: 196 | o.DomainPrefix = strV 197 | case Route: 198 | o.Route = strV 199 | case ExternalTrafficPolicy: 200 | o.ExternalTrafficPolicy = strV 201 | case HTTPOnly: 202 | o.HTTPOnly, err = strconv.ParseBool(strV) 203 | if err != nil { 204 | o.HTTPOnly = false 205 | } 206 | case Acme: 207 | o.Acme, err = strconv.ParseBool(strV) 208 | if err != nil { 209 | o.Acme = false 210 | } 211 | case AcmeCName: 212 | o.AcmeCName, err = strconv.ParseBool(strV) 213 | if err != nil { 214 | o.AcmeCName = false 215 | } 216 | case AllPrefixes: 217 | o.ExposeAllServices, err = strconv.ParseBool(strV) 218 | if err != nil { 219 | o.ExposeAllServices = false 220 | } 221 | default: 222 | o.AdditionalOpts[k] = strV 223 | } 224 | } 225 | 226 | return err 227 | } 228 | 229 | // DescribedOptions returns a map containing all the available options 230 | // and their description as values of the map 231 | func DescribedOptions() map[string]string { 232 | return map[string]string{ 233 | ExposedPort: "Port to be exposed by the Load Balancer. Defaults to 80.", 234 | Domain: "Domain used on Ingress.", 235 | Route: "Path used on Ingress rule.", 236 | Acme: "If set to true, adds ingress TLS options to Ingress. Defaults to false.", 237 | AcmeCName: "If set to true, adds ingress TLS options to CName Ingresses. Defaults to false.", 238 | AllPrefixes: "If set to true, exposes all of the services of the app, allowing them to be accessible from the router.", 239 | } 240 | } 241 | 242 | // HealthcheckableRouter is a Service that implements 243 | // a way to check of its health 244 | type HealthcheckableRouter interface { 245 | Healthcheck() error 246 | } 247 | -------------------------------------------------------------------------------- /router/service_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 tsuru authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package router 6 | 7 | import ( 8 | "encoding/json" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestUnmarshalOpts(t *testing.T) { 15 | js := `{"tsuru.io/app-pool": "pool","exposed-port": "80","tsuru.io/teams": ["teamA", "teamB"],"custom-opt": "val"}` 16 | routerOpts := Opts{} 17 | err := json.Unmarshal([]byte(js), &routerOpts) 18 | assert.NoError(t, err) 19 | expected := Opts{Pool: "pool", ExposedPort: "80", AdditionalOpts: map[string]string{"custom-opt": "val"}} 20 | assert.Equal(t, expected, routerOpts) 21 | } 22 | 23 | func TestUnmarshalOptsWithHeaderOpts(t *testing.T) { 24 | js := `{"tsuru.io/app-pool": "pool","exposed-port": "80","tsuru.io/teams": ["teamA", "teamB"],"custom-opt": "val"}` 25 | routerOpts := Opts{ 26 | HeaderOpts: []string{ 27 | "x", 28 | "a=b", 29 | "b=", 30 | "c=a=b=c", 31 | "d-", 32 | "domain=invalid.com", 33 | }, 34 | } 35 | err := json.Unmarshal([]byte(js), &routerOpts) 36 | assert.NoError(t, err) 37 | expected := Opts{ 38 | Pool: "pool", 39 | ExposedPort: "80", 40 | Domain: "invalid.com", 41 | AdditionalOpts: map[string]string{ 42 | "custom-opt": "val", 43 | "x": "", 44 | "a": "b", 45 | "b": "", 46 | "c": "a=b=c", 47 | "d-": "", 48 | }, 49 | HeaderOpts: []string{ 50 | "x", 51 | "a=b", 52 | "b=", 53 | "c=a=b=c", 54 | "d-", 55 | "domain=invalid.com", 56 | }, 57 | } 58 | assert.Equal(t, expected, routerOpts) 59 | } 60 | --------------------------------------------------------------------------------