├── VERSION ├── .dockerignore ├── test ├── secret.yaml ├── clusterconfig.yaml ├── istio-inject-config.yaml ├── test.sh ├── porterconfig_base.yaml ├── simpleconfig_base.yaml ├── porterconfig.yaml ├── porterconfig_aws.yaml ├── deprecated_hpa.yaml ├── current_hpa.yaml ├── deprecated_vs.yaml ├── simpleconfig_aws.yaml ├── simpleconfig.yaml ├── istio.yaml ├── hpa.yaml ├── sidecar.yaml ├── istio-mesh-config.yaml └── multiple_pod.yaml ├── Dockerfile-base ├── Dockerfile ├── pkg ├── vault │ ├── interface.go │ ├── client_v1.go │ ├── client.go │ ├── client_v2.go │ └── helpers.go └── porter2k8s │ ├── kubeobjects_test.go │ ├── kubeconfig_test.go │ ├── kubeconfig.go │ ├── porterconfig_test.go │ ├── podlogger.go │ ├── porterconfig.go │ ├── dynamic_watch_test.go │ ├── simpleconfig_test.go │ ├── updater.go │ ├── simpleconfig.go │ ├── kubeobjects.go │ ├── helper.go │ ├── helper_test.go │ ├── dynamic_watch.go │ ├── regionenv_test.go │ └── porter2k8s.go ├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── Dockerfile-dev ├── Dockerfile-build ├── .gitignore ├── Makefile ├── cmd └── porter2k8s │ └── porter2k8s.go ├── go.mod ├── CONTRIBUTING.md ├── CODE_OF_CONDUCT.md ├── LICENSE └── README.md /VERSION: -------------------------------------------------------------------------------- 1 | 0.5.8 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | /.git 2 | /vendor 3 | tags 4 | -------------------------------------------------------------------------------- /test/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | metadata: 3 | name: testapp-secret 4 | kind: Secret 5 | type: Opaque 6 | stringData: 7 | secretKey: "{{.TESTAPP_SECRET}}" 8 | -------------------------------------------------------------------------------- /test/clusterconfig.yaml: -------------------------------------------------------------------------------- 1 | clusters: 2 | - path: "secret/test/test/kubeconfig-1.yaml" 3 | context: "us-east-1" 4 | - path: "30-secret/test_test/k-2#(3.yaml" 5 | context: "other-region" 6 | -------------------------------------------------------------------------------- /Dockerfile-base: -------------------------------------------------------------------------------- 1 | FROM golang:1.16-alpine 2 | 3 | RUN apk add --no-cache \ 4 | bash \ 5 | build-base \ 6 | make \ 7 | git \ 8 | coreutils \ 9 | curl \ 10 | unzip 11 | -------------------------------------------------------------------------------- /test/istio-inject-config.yaml: -------------------------------------------------------------------------------- 1 | template: |- 2 | initContainers: 3 | - name: istio-init 4 | image: docker.io/istio/proxy_init:unittest 5 | containers: 6 | - name: istio-proxy 7 | image: docker.io/istio/proxy_debug:unittest 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | From alpine 2 | 3 | ADD porter2k8s /usr/local/bin/porter2k8s 4 | 5 | RUN apk --no-cache add ca-certificates curl python3 6 | RUN curl https://bootstrap.pypa.io/get-pip.py | python3 && pip install awscli 7 | 8 | ENTRYPOINT [ "porter2k8s" ] 9 | -------------------------------------------------------------------------------- /pkg/vault/interface.go: -------------------------------------------------------------------------------- 1 | package vault 2 | 3 | import vaultapi "github.com/hashicorp/vault/api" 4 | 5 | // Client - interface for reading / writing to vault 6 | type VaultClientInterface interface { 7 | Read(path string) (map[string]interface{}, error) 8 | 9 | Auth() *vaultapi.Auth 10 | } 11 | -------------------------------------------------------------------------------- /test/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | go get -u golang.org/x/lint/golint 3 | echo Linting 4 | golint -set_exit_status . || exit 1 5 | echo "Looking for go files with >120 character lines." 6 | LINES=$(grep -rn '.\{121\}' *.go) 7 | if [[ -n $LINES ]]; then 8 | echo $LINES 9 | exit 1 10 | fi 11 | echo Passed 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Expected Behaviour 2 | 3 | ### Actual Behaviour 4 | 5 | ### Reproduce Scenario (including but not limited to) 6 | 7 | #### Steps to Reproduce 8 | 9 | #### Platform and Version 10 | 11 | #### Sample Code that illustrates the problem 12 | 13 | #### Logs taken while reproducing problem 14 | -------------------------------------------------------------------------------- /Dockerfile-dev: -------------------------------------------------------------------------------- 1 | FROM golang:1.16-alpine 2 | 3 | # install runtime scripts 4 | ADD . $GOPATH/src/git.corp.adobe.com/EchoSign/porter2k8s 5 | 6 | WORKDIR $GOPATH/src/git.corp.adobe.com/EchoSign/porter2k8s 7 | 8 | RUN apk add --no-cache \ 9 | bash \ 10 | build-base \ 11 | make \ 12 | git \ 13 | coreutils 14 | 15 | CMD /bin/bash 16 | -------------------------------------------------------------------------------- /test/porterconfig_base.yaml: -------------------------------------------------------------------------------- 1 | --- # first document defines the bundle 2 | bundle: global 3 | 4 | --- 5 | # Microservices Secrets 6 | group: 1 7 | base_path: / 8 | packages: 9 | something: &test 10 | 11 | - path: different_path 12 | field: Cert 13 | name: BASE_SECRET 14 | 15 | - name: ENV_TEST 16 | value: should_be_overwritten 17 | 18 | container: *test 19 | 20 | -------------------------------------------------------------------------------- /Dockerfile-build: -------------------------------------------------------------------------------- 1 | FROM docker-dc-micro-release.dr.corp.adobe.com/porter2k8s:base 2 | 3 | ARG SHA 4 | 5 | COPY . $GOPATH/src/git.corp.adobe.com/EchoSign/porter2k8s 6 | 7 | WORKDIR $GOPATH/src/git.corp.adobe.com/EchoSign/porter2k8s 8 | 9 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X main.GitSha=$SHA -X main.Version=`cat VERSION` -X main.BuildDate=`date +%s`" -o porter2k8s cmd/porter2k8s/porter2k8s.go 10 | -------------------------------------------------------------------------------- /test/simpleconfig_base.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | base_path: / 3 | type: something 4 | vars: 5 | 6 | - path: different_path 7 | field: Cert 8 | name: BASE_SECRET 9 | 10 | - name: ENV_TEST 11 | value: should_be_overwritten 12 | 13 | --- 14 | base_path: / 15 | type: container 16 | vars: 17 | 18 | - path: different_path 19 | field: Cert 20 | name: BASE_SECRET 21 | 22 | - name: ENV_TEST 23 | value: should_be_overwritten 24 | -------------------------------------------------------------------------------- /test/porterconfig.yaml: -------------------------------------------------------------------------------- 1 | --- # first document defines the bundle 2 | bundle: dev-ue1 3 | includes: 4 | - ./porterconfig_base.yaml 5 | - ./porterconfig_aws.yaml 6 | 7 | --- 8 | # Porter Config 9 | group: 1 10 | base_path: /test_base 11 | packages: 12 | container: 13 | 14 | - name: ENV_TEST 15 | value: environment value 16 | 17 | - path: test_path 18 | field: Password 19 | name: SECRET_TEST 20 | 21 | - name: NO_VAL 22 | value: "" 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /porter2k8s 2 | 3 | # Binaries for programs and plugins 4 | *.exe 5 | *.dll 6 | *.so 7 | *.dylib 8 | 9 | # Test binary, build with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 16 | .glide/ 17 | 18 | # Do not check in vendor 19 | vendor/ 20 | 21 | # Ctags 22 | tags 23 | 24 | # vim 25 | *.sw[po] 26 | 27 | go.sum 28 | -------------------------------------------------------------------------------- /test/porterconfig_aws.yaml: -------------------------------------------------------------------------------- 1 | --- # first document defines the bundle 2 | bundle: global 3 | 4 | --- 5 | # Microservices Secrets 6 | group: 1 7 | base_path: / 8 | packages: 9 | host: 10 | 11 | - path: not_relevant 12 | field: irrelevant_field 13 | name: irrelevant_name 14 | 15 | - name: BUG 16 | value: Bug 17 | 18 | container: 19 | - path: aws_secret_path 20 | field: aws_field 21 | name: AWS_SECRET 22 | 23 | - name: AWS_ENV 24 | value: aws_environment_variable 25 | 26 | -------------------------------------------------------------------------------- /test/deprecated_hpa.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: autoscaling/v2beta1 3 | kind: HorizontalPodAutoscaler 4 | metadata: 5 | name: &name telemachus 6 | labels: 7 | app: *name 8 | spec: 9 | scaleTargetRef: 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | name: *name 13 | minReplicas: 2 # Don't set this value lower than 2. Just in case, a value less than 2 will be overridden to 2 in prod. 14 | maxReplicas: 10 15 | metrics: 16 | - type: Resource 17 | resource: 18 | name: cpu 19 | targetAverageUtilization: 70 20 | -------------------------------------------------------------------------------- /test/current_hpa.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: autoscaling/v2beta2 3 | kind: HorizontalPodAutoscaler 4 | metadata: 5 | name: &name laertes 6 | labels: 7 | app: *name 8 | spec: 9 | scaleTargetRef: 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | name: *name 13 | minReplicas: 2 # Don't set this value lower than 2. Just in case, a value less than 2 will be overridden to 2 in prod. 14 | maxReplicas: 10 15 | metrics: 16 | - resource: 17 | name: cpu 18 | target: 19 | averageUtilization: 70 20 | type: Utilization 21 | type: Resource 22 | -------------------------------------------------------------------------------- /test/deprecated_vs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: networking.istio.io/v1alpha3 3 | kind: VirtualService 4 | metadata: 5 | name: &name daedalus 6 | spec: 7 | hosts: 8 | - daedalus.dev0.lan 9 | gateways: 10 | - my-gateway 11 | http: 12 | - timeout: 15s # Total request time, including retries. Make sure this makes sense for your service! 13 | route: 14 | - destination: 15 | host: *name 16 | port: 17 | number: 80 18 | retries: 19 | attempts: 3 20 | perTryTimeout: 15s # Timeout for receiving response headers on retry. 21 | -------------------------------------------------------------------------------- /pkg/vault/client_v1.go: -------------------------------------------------------------------------------- 1 | package vault 2 | 3 | import vaultapi "github.com/hashicorp/vault/api" 4 | 5 | func NewClientV1(client *vaultapi.Client) *clientV1 { 6 | return &clientV1{client} 7 | } 8 | 9 | // vault API for KV V1 and cubbyhole mountpoints 10 | type clientV1 struct { 11 | *vaultapi.Client 12 | } 13 | 14 | func (c *clientV1) Read(path string) (map[string]interface{}, error) { 15 | s, err := c.Client.Logical().Read(path) 16 | reportWarnings(s) 17 | if err != nil { 18 | return nil, err 19 | } 20 | if s != nil { 21 | return s.Data, nil 22 | } 23 | return nil, nil 24 | } 25 | -------------------------------------------------------------------------------- /test/simpleconfig_aws.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | base_path: / 3 | type: host 4 | vars: 5 | - path: not_relevant 6 | field: irrelevant_field 7 | name: irrelevant_name 8 | 9 | - name: BUG 10 | value: Bug 11 | 12 | --- 13 | base_path: / 14 | type: container 15 | vars: 16 | - path: aws_secret_path 17 | field: aws_field 18 | name: AWS_SECRET 19 | 20 | - name: AWS_ENV 21 | value: aws_environment_variable 22 | 23 | --- 24 | base_path: /irrelevant 25 | type: porter2k8s 26 | vars: 27 | - name: ENVIRONMENT 28 | value: should_be_overriden 29 | 30 | - name: REGION 31 | value: westphalia 32 | -------------------------------------------------------------------------------- /pkg/vault/client.go: -------------------------------------------------------------------------------- 1 | package vault 2 | 3 | import ( 4 | vaultapi "github.com/hashicorp/vault/api" 5 | "log" 6 | ) 7 | 8 | func NewVaultClient(address, namespace, token string) (VaultClientInterface, error) { 9 | client, err := vaultapi.NewClient(&vaultapi.Config{ 10 | Address: address, 11 | }) 12 | 13 | if err != nil { 14 | return nil, err 15 | } 16 | client.SetToken(token) 17 | mountpointPrefix := "secret" 18 | if namespace != "" { 19 | client.SetNamespace(namespace) 20 | mountpointPrefix = namespace 21 | } 22 | 23 | mountPoint, version, err := GetMountPoint(mountpointPrefix, client) 24 | 25 | var versionedEngineClient VaultClientInterface 26 | if version == "2" { 27 | log.Printf("Determined KV version 2, mountpoint %s", mountPoint) 28 | versionedEngineClient = NewClientV2(client, mountPoint) 29 | } else { 30 | log.Printf("Determined KV version 1") 31 | versionedEngineClient = NewClientV1(client) 32 | } 33 | 34 | return versionedEngineClient, nil 35 | } 36 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | REPO=docker-dc-micro-release.dr.corp.adobe.com/porter2k8s 2 | SHA=$$(git rev-parse --short HEAD) 3 | DATE=$$(date +%Y%m%d) 4 | VERSION=$$(cat VERSION) 5 | 6 | clean: 7 | rm -rf vendor porter2k8s 8 | 9 | base: 10 | docker build --pull -t $(REPO):base . -f Dockerfile-base 11 | 12 | upload-base: 13 | docker push $(REPO):base 14 | 15 | build-container: 16 | docker build --pull -t $(REPO):build . -f Dockerfile-build --build-arg SHA=$(SHA) 17 | docker create --name extract $(REPO):build 18 | docker cp extract:/go/src/git.corp.adobe.com/EchoSign/porter2k8s/porter2k8s ./ 19 | docker rm -f extract 20 | docker build -t $(REPO):$(SHA)_$(DATE) . 21 | 22 | upload-current: 23 | docker push $(REPO):$(SHA)_$(DATE) 24 | docker tag $(REPO):$(SHA)_$(DATE) $(REPO):latest 25 | docker push $(REPO):latest 26 | docker tag $(REPO):latest $(REPO):$(VERSION) 27 | docker push $(REPO):$(VERSION) 28 | 29 | run-tests: 30 | docker build --pull -t $(REPO):build . -f Dockerfile-build 31 | docker run $(REPO):build go test -v ./... 32 | docker run $(REPO):build /bin/bash test/test.sh 33 | -------------------------------------------------------------------------------- /test/simpleconfig.yaml: -------------------------------------------------------------------------------- 1 | --- # first document defines the bundle 2 | includes: 3 | - ./simpleconfig_base.yaml 4 | - ./simpleconfig_aws.yaml 5 | 6 | --- 7 | base_path: /test_base 8 | type: container 9 | vars: 10 | 11 | - name: ENV_TEST 12 | value: environment value 13 | 14 | - path: test_path 15 | name: SECRET_TEST 16 | 17 | - name: NO_VAL 18 | value: "" 19 | 20 | - path: test_path 21 | name: NOT_VAULT_KEY 22 | key: vault_key 23 | 24 | - source: kubernetes 25 | name: FROM_K8S 26 | k8s_secret: k8s-secret 27 | key: k8s_key 28 | 29 | - source: object 30 | path: status.url 31 | name: FROM_OBJECT 32 | k8s_object: 33 | name: test-object 34 | kind: redis 35 | 36 | --- 37 | base_path: /extra_slash///more_slash/ 38 | type: container 39 | vars: 40 | 41 | - name: SLASHES 42 | path: /another_slash 43 | 44 | --- 45 | base_path: /kinda_matters_now 46 | type: porter2k8s 47 | vars: 48 | 49 | - name: ENVIRONMENT 50 | value: gorilla 51 | 52 | - name: P2K8S_SECRET 53 | path: secret_path 54 | key: Password 55 | -------------------------------------------------------------------------------- /test/istio.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.istio.io/v1alpha3 2 | kind: Gateway 3 | metadata: 4 | name: mobile-sig-capture-gateway 5 | spec: 6 | selector: 7 | istio: ingressgateway # use istio default controller 8 | servers: 9 | - port: 10 | number: 80 11 | name: http 12 | protocol: HTTP 13 | hosts: 14 | - mobilesigcapture.uswest2.micro.echosignpreview.com 15 | - port: 16 | number: 443 17 | name: https 18 | protocol: HTTPS 19 | hosts: 20 | - mobilesigcapture.uswest2.micro.echosignpreview.com 21 | tls: 22 | mode: SIMPLE 23 | privateKey: /etc/istio/ingressgateway-certs/tls.key 24 | serverCertificate: /etc/istio/ingressgateway-certs/tls.crt 25 | --- 26 | apiVersion: networking.istio.io/v1alpha3 27 | kind: VirtualService 28 | metadata: 29 | name: mobile-sig-capture 30 | spec: 31 | hosts: 32 | - mobilesigcapture.uswest2.micro.echosignpreview.com 33 | gateways: 34 | - mobile-sig-capture-gateway 35 | http: 36 | - timeout: 15s 37 | route: 38 | - destination: 39 | host: mobile-sig-capture 40 | port: 41 | number: 80 42 | retries: 43 | attempts: 3 44 | perTryTimeout: 10s # Timeout for receiving response headers on retry. 45 | 46 | -------------------------------------------------------------------------------- /pkg/vault/client_v2.go: -------------------------------------------------------------------------------- 1 | package vault 2 | 3 | import ( 4 | "fmt" 5 | vaultapi "github.com/hashicorp/vault/api" 6 | "path" 7 | "strings" 8 | ) 9 | 10 | func NewClientV2(client *vaultapi.Client, mountPoint string) *clientV2 { 11 | return &clientV2{Client: client, mountPoint: mountPoint} 12 | } 13 | 14 | // vault API for KV V2 mountpoints 15 | type clientV2 struct { 16 | *vaultapi.Client 17 | mountPoint string 18 | } 19 | 20 | func (c *clientV2) Read(vaultpath string) (map[string]interface{}, error) { 21 | internalPath, err := serverPath(vaultpath, c.mountPoint) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | s, err := c.Client.Logical().Read(internalPath) 27 | reportWarnings(s) 28 | 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | if s != nil { 34 | if data, ok := s.Data["data"].(map[string]interface{}); ok { 35 | return data, nil 36 | } 37 | } 38 | 39 | return nil, nil 40 | } 41 | 42 | func serverPath(vaultpath, mountPoint string) (string, error) { 43 | switch { 44 | case vaultpath == mountPoint, vaultpath == strings.TrimSuffix(mountPoint, "/"): 45 | return path.Join(mountPoint, "data"), nil 46 | default: 47 | subpath := strings.TrimPrefix(vaultpath, mountPoint) 48 | if subpath == vaultpath { 49 | return "", fmt.Errorf("illegal path: %s is not under mountpoint %s", vaultpath, mountPoint) 50 | } 51 | return path.Join(mountPoint, "data", subpath), nil 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pkg/vault/helpers.go: -------------------------------------------------------------------------------- 1 | package vault 2 | 3 | import ( 4 | "fmt" 5 | vaultapi "github.com/hashicorp/vault/api" 6 | "log" 7 | "strings" 8 | ) 9 | 10 | func reportWarnings(s *vaultapi.Secret) { 11 | if s != nil && len(s.Warnings) > 0 { 12 | log.Printf("vault warnings: %v", s.Warnings) 13 | } 14 | } 15 | 16 | func GetMountPoint(basePath string, client *vaultapi.Client) (string, string, error) { 17 | var mountpoint, version string 18 | s, err := client.Logical().Read("/sys/mounts") 19 | if err != nil { 20 | return mountpoint, version, fmt.Errorf("Error determining vault mountpoint for %s: %w", basePath, err) 21 | } 22 | 23 | basePath = EnsureSuffix(basePath, "/") 24 | for mountpoint, data := range s.Data { 25 | if strings.HasPrefix(basePath, mountpoint) { 26 | v := SafeGet(SafeGet(data, "options"), "version") 27 | if s, ok := v.(string); ok { 28 | version = s 29 | } 30 | log.Printf("Mountpoint found for %s: %s version: %s", basePath, mountpoint, version) 31 | return mountpoint, version, nil 32 | } 33 | } 34 | 35 | log.Printf("No mountpoint found for %s", basePath) 36 | return mountpoint, version, nil 37 | } 38 | 39 | func EnsureSuffix(s, suffix string) string { 40 | if !strings.HasSuffix(s, suffix) { 41 | return s + suffix 42 | } 43 | return s 44 | } 45 | 46 | func SafeGet(m interface{}, key string) interface{} { 47 | if mm, ok := m.(map[string]interface{}); ok { 48 | return mm[key] 49 | } 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /cmd/porter2k8s/porter2k8s.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | //Usage of : 14 | // -config-path string 15 | // Configuration root directory (default "/repo/.porter") 16 | // -config-type string 17 | // Configuration type, the only option is porter (default "porter") 18 | // -deployment-file string 19 | // Vault token (default "/repo/deployment.yaml") 20 | // -environment string 21 | // Environment of deployment. 22 | // -regions string 23 | // Regions of deployment. 24 | // -service-file string 25 | // Vault token (default "/repo/service.yaml") 26 | // -sha string 27 | // Deployment sha. 28 | // -vault-addr string 29 | // Vault server (default "https://vault.loc.adobe.net") 30 | // -vault-path string 31 | // Path in Vault (default "/") 32 | // -vault-namespace string 33 | // Vault namespace (default "") 34 | // -vault-token string 35 | // Vault token 36 | // 37 | // Example: 38 | // sha=6b875614b51bfab2f8b794279bda1598d555ed4b porter2k8s --regions "us-east-1" --environment dev --config-path $(pwd) 39 | // --vault-path secret/ethos/tenants/cloudtech_doc_cloud 40 | package main 41 | 42 | import ( 43 | "os" 44 | 45 | "git.corp.adobe.com/EchoSign/porter2k8s/pkg/porter2k8s" 46 | ) 47 | 48 | func main() { 49 | porter2k8s.Run(os.Args[1:]) 50 | } 51 | -------------------------------------------------------------------------------- /test/hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2beta2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: before 5 | namespace: test 6 | labels: 7 | app: &name testapp 8 | annotations: 9 | metric-config.object.test-istio-requests-total.prometheus/per-replica: "true" 10 | metric-config.object.test-istio-requests-total.prometheus/query: | 11 | sum(rate(istio_requests_total{destination_app="testapp"}[1m]))/sum(rate(istio_requests_total{destination_app="testapp"}[5m]))/count(kube_pod_container_status_ready{container="testapp"}) 12 | spec: 13 | scaleTargetRef: 14 | apiVersion: apps/v1 15 | kind: Deployment 16 | name: *name 17 | minReplicas: 2 18 | maxReplicas: 10 19 | metrics: 20 | - type: Object 21 | object: 22 | metric: 23 | name: test-istio-requests-total 24 | describedObject: 25 | apiVersion: v1 26 | kind: Pod 27 | name: testapp 28 | target: 29 | type: Value 30 | value: 1 31 | --- 32 | apiVersion: autoscaling/v2beta2 33 | kind: HorizontalPodAutoscaler 34 | metadata: 35 | name: after 36 | namespace: test 37 | labels: 38 | app: &name testapp 39 | annotations: 40 | metric-config.object.test-istio-requests-total.prometheus/per-replica: "true" 41 | metric-config.object.test-istio-requests-total.prometheus/query: | 42 | sum(rate(istio_requests_total{destination_app="testapp"}[1m]))/sum(rate(istio_requests_total{destination_app="testapp"}[5m]))/count(kube_pod_container_status_ready{container="testapp"}) 43 | spec: 44 | scaleTargetRef: 45 | apiVersion: apps/v1 46 | kind: Deployment 47 | name: *name 48 | minReplicas: 2 49 | maxReplicas: 10 50 | metrics: 51 | - type: Object 52 | object: 53 | metric: 54 | name: test-istio-requests-total 55 | describedObject: 56 | apiVersion: v1 57 | kind: Pod 58 | name: testapp 59 | target: 60 | type: Value 61 | value: 10 62 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Description 4 | 5 | 6 | 7 | ## Related Issue 8 | 9 | 10 | 11 | 12 | 13 | 14 | ## Motivation and Context 15 | 16 | 17 | 18 | ## How Has This Been Tested? 19 | 20 | 21 | 22 | 23 | 24 | ## Screenshots (if appropriate): 25 | 26 | ## Types of changes 27 | 28 | 29 | 30 | - [ ] Bug fix (non-breaking change which fixes an issue) 31 | - [ ] New feature (non-breaking change which adds functionality) 32 | - [ ] Breaking change (fix or feature that would cause existing functionality to change) 33 | 34 | ## Checklist: 35 | 36 | 37 | 38 | 39 | - [ ] I have signed the [Adobe Open Source CLA](http://opensource.adobe.com/cla.html). 40 | - [ ] My code follows the code style of this project. 41 | - [ ] My change requires a change to the documentation. 42 | - [ ] I have updated the documentation accordingly. 43 | - [ ] I have read the **CONTRIBUTING** document. 44 | - [ ] I have added tests to cover my changes. 45 | - [ ] All new and existing tests passed. 46 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module git.corp.adobe.com/EchoSign/porter2k8s 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/Masterminds/goutils v1.1.1 // indirect 7 | github.com/Masterminds/semver v1.5.0 // indirect 8 | github.com/Masterminds/sprig v2.22.0+incompatible 9 | github.com/fatih/color v1.10.0 // indirect 10 | github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 11 | github.com/go-test/deep v1.0.5 // indirect 12 | github.com/gogo/protobuf v1.3.2 // indirect 13 | github.com/golang/protobuf v1.5.2 // indirect 14 | github.com/golang/snappy v0.0.3 // indirect 15 | github.com/google/go-cmp v0.5.5 16 | github.com/google/gofuzz v1.2.0 // indirect 17 | github.com/googleapis/gnostic v0.5.4 // indirect 18 | github.com/hashicorp/errwrap v1.1.0 // indirect 19 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 20 | github.com/hashicorp/go-hclog v0.16.0 // indirect 21 | github.com/hashicorp/go-multierror v1.1.1 // indirect 22 | github.com/hashicorp/go-retryablehttp v0.6.8 // indirect 23 | github.com/hashicorp/vault/api v1.1.0 24 | github.com/hashicorp/vault/sdk v0.2.0 // indirect 25 | github.com/huandu/xstrings v1.3.2 // indirect 26 | github.com/imdario/mergo v0.3.12 // indirect 27 | github.com/itchyny/gojq v0.12.4 28 | github.com/mitchellh/mapstructure v1.4.1 // indirect 29 | github.com/onsi/ginkgo v1.14.1 // indirect 30 | github.com/onsi/gomega v1.10.2 // indirect 31 | github.com/pierrec/lz4 v2.6.0+incompatible // indirect 32 | github.com/sirupsen/logrus v1.8.1 33 | golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect 34 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect 35 | golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 // indirect 36 | golang.org/x/term v0.0.0-20210406210042-72f3dc4e9b72 // indirect 37 | golang.org/x/text v0.3.6 // indirect 38 | golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect 39 | google.golang.org/appengine v1.6.7 // indirect 40 | gopkg.in/yaml.v2 v2.4.0 41 | k8s.io/api v0.20.5 42 | k8s.io/apimachinery v0.20.5 43 | k8s.io/client-go v0.20.5 44 | k8s.io/klog/v2 v2.8.0 // indirect 45 | k8s.io/utils v0.0.0-20210305010621-2afb4311ab10 // indirect 46 | sigs.k8s.io/structured-merge-diff/v4 v4.1.1 // indirect 47 | ) 48 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thanks for choosing to contribute! 4 | 5 | The following are a set of guidelines to follow when contributing to this project. 6 | 7 | ## Code Of Conduct 8 | 9 | This project adheres to the Adobe [code of conduct](CODE_OF_CONDUCT.md). By participating, 10 | you are expected to uphold this code. Please report unacceptable behavior to 11 | [Grp-opensourceoffice@adobe.com](mailto:Grp-opensourceoffice@adobe.com). 12 | 13 | ## Have A Question? 14 | 15 | Start by filing an issue. The existing committers on this project work to reach 16 | consensus around project direction and issue solutions within issue threads 17 | (when appropriate). 18 | 19 | ## Contributor License Agreement 20 | 21 | All third-party contributions to this project must be accompanied by a signed contributor 22 | license agreement. This gives Adobe permission to redistribute your contributions 23 | as part of the project. [Sign our CLA](http://opensource.adobe.com/cla.html). You 24 | only need to submit an Adobe CLA one time, so if you have submitted one previously, 25 | you are good to go! 26 | 27 | ## Code Reviews 28 | 29 | All submissions should come in the form of pull requests and need to be reviewed 30 | by project committers. Read [GitHub's pull request documentation](https://help.github.com/articles/about-pull-requests/) 31 | for more information on sending pull requests. 32 | 33 | Lastly, please follow the [pull request template](PULL_REQUEST_TEMPLATE.md) when 34 | submitting a pull request! 35 | 36 | ## From Contributor To Committer 37 | 38 | We love contributions from our community! If you'd like to go a step beyond contributor 39 | and become a committer with full write access and a say in the project, you must 40 | be invited to the project. The existing committers employ an internal nomination 41 | process that must reach lazy consensus (silence is approval) before invitations 42 | are issued. If you feel you are qualified and want to get more deeply involved, 43 | feel free to reach out to existing committers to have a conversation about that. 44 | 45 | ## Security Issues 46 | 47 | Security issues shouldn't be reported on this issue tracker. Instead, [file an issue to our security experts](https://helpx.adobe.com/security/alertus.html) 48 | -------------------------------------------------------------------------------- /pkg/porter2k8s/kubeobjects_test.go: -------------------------------------------------------------------------------- 1 | package porter2k8s 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "testing" 9 | 10 | log "github.com/sirupsen/logrus" 11 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 12 | "k8s.io/apimachinery/pkg/runtime/schema" 13 | ) 14 | 15 | func TestProcessObjects(t *testing.T) { 16 | testCases := []struct { 17 | file []string 18 | gvk schema.GroupVersionKind 19 | kubeObject string // Empty if deprecated version 20 | err string 21 | }{ 22 | { 23 | []string{"deprecated_hpa"}, 24 | schema.GroupVersionKind{Group: "autoscaling", Version: "v2beta1", Kind: "HorizontalPodAutoscaler"}, 25 | "", 26 | "Object type autoscaling/v2beta1, Kind=HorizontalPodAutoscaler is unsupported", 27 | }, 28 | { 29 | []string{"deprecated_vs"}, 30 | schema.GroupVersionKind{Group: "networking.istio.io", Version: "v1alpha3", Kind: "VirtualService"}, 31 | "", 32 | "Object type networking.istio.io/v1alpha3, Kind=VirtualService is unsupported", 33 | }, 34 | { 35 | []string{"current_hpa"}, 36 | schema.GroupVersionKind{Group: "autoscaling", Version: "v2beta2", Kind: "HorizontalPodAutoscaler"}, 37 | "HPAutoscaler", 38 | "", 39 | }, 40 | { 41 | []string{"multiple_pod"}, 42 | schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, 43 | "", 44 | "One and only one pod containing object is allowed. Found 2", 45 | }, 46 | } 47 | for _, tc := range testCases { 48 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 49 | var kubeObjects KubeObjects 50 | kubeObjects.Unstructured = make(map[string][]*unstructured.Unstructured) 51 | 52 | pwd, _ := os.Getwd() 53 | filePath := fmt.Sprintf("%s/../../test", pwd) 54 | cfg := CmdConfig{ConfigPath: filePath} 55 | err := kubeObjects.processObjects(&cfg, tc.file) 56 | if err != nil { 57 | if !strings.Contains(err.Error(), tc.err) { 58 | t.Errorf("Unexpected message %s\nExpected: %s", err, tc.err) 59 | } 60 | } 61 | }) 62 | } 63 | } 64 | 65 | // No longer required since processObjects now returns an error. 66 | func captureOutput(f func()) string { 67 | var buf bytes.Buffer 68 | log.SetOutput(&buf) 69 | f() 70 | log.SetOutput(os.Stderr) 71 | return buf.String() 72 | } 73 | -------------------------------------------------------------------------------- /pkg/porter2k8s/kubeconfig_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "fmt" 17 | "os" 18 | "testing" 19 | 20 | "github.com/google/go-cmp/cmp" 21 | ) 22 | 23 | // Expected based on content of 'test/clusterconfig.yaml'. 24 | var expectedClusterConfig = ClusterConfigs{ 25 | Clusters: []ClusterReference{ 26 | ClusterReference{ 27 | VaultPath: "secret/test/test/kubeconfig-1.yaml", 28 | Context: "us-east-1", 29 | }, 30 | ClusterReference{ 31 | VaultPath: "30-secret/test_test/k-2#(3.yaml", 32 | Context: "other-region", 33 | }, 34 | }, 35 | } 36 | 37 | func TestReadReferences(t *testing.T) { 38 | testCases := []struct { 39 | filename string 40 | expected ClusterConfigs 41 | }{ 42 | {"../../test/clusterconfig.yaml", expectedClusterConfig}, 43 | } 44 | for _, tc := range testCases { 45 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 46 | pwd, _ := os.Getwd() 47 | filePath := fmt.Sprintf("%s/%s", pwd, tc.filename) 48 | readConfig, err := readReferences(filePath) 49 | if err != nil { 50 | t.Errorf("Error reading kubeconfig listing: \n%v\n", err) 51 | } 52 | equal := cmp.Equal(readConfig, tc.expected, cmp.Comparer(clusterConfigComparer)) 53 | if !equal { 54 | t.Errorf("Unexpected kubeconfig listing %+v, expected %+v\n", readConfig, tc.expected) 55 | } 56 | }) 57 | } 58 | } 59 | 60 | func clusterConfigComparer(x, y ClusterConfigs) bool { 61 | if len(x.Clusters) != len(y.Clusters) { 62 | return false 63 | } 64 | for i, v := range x.Clusters { 65 | if v.VaultPath != y.Clusters[i].VaultPath || v.Context != y.Clusters[i].Context { 66 | return false 67 | } 68 | } 69 | return true 70 | } 71 | -------------------------------------------------------------------------------- /test/sidecar.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: porter2k8s 5 | data: 6 | SIDECAR: | 7 | apiVersion: v1 8 | kind: Pod 9 | metadata: 10 | labels: 11 | app: sas 12 | service: sas 13 | name: sas-fluent-bit 14 | spec: 15 | containers: 16 | - env: 17 | - name: TAIL_BUF_LIMIT 18 | value: '{{ default "145MB" .LOGGING_TAIL_BUF_LIMIT }}' 19 | - name: SPLUNK_SOURCETYPE 20 | value: '{{ default "dc-k8s-asr" .LOGGING_SOURCETYPE }}' 21 | - name: SPLUNK_INDEX 22 | valueFrom: 23 | configMapKeyRef: 24 | key: SPLUNK_INDEX 25 | name: porter2k8s 26 | - name: SPLUNK_TOKEN 27 | valueFrom: 28 | secretKeyRef: 29 | key: token 30 | name: splunk 31 | - name: SPLUNK_HOST 32 | value: splunk-us-hec-external.adobelaas.com 33 | - name: SPLUNK_PORT 34 | value: "443" 35 | - name: SPLUNK_TLS 36 | value: "On" 37 | - name: SPLUNK_TLS_VERIFY 38 | value: "On" 39 | - name: SPLUNK_SEND_RAW 40 | value: "On" 41 | - name: POD_UID 42 | valueFrom: 43 | fieldRef: 44 | fieldPath: metadata.uid 45 | - name: POD_NAME 46 | valueFrom: 47 | fieldRef: 48 | fieldPath: metadata.name 49 | - name: POD_IP 50 | valueFrom: 51 | fieldRef: 52 | fieldPath: status.podIP 53 | - name: NODE_IP 54 | valueFrom: 55 | fieldRef: 56 | fieldPath: status.hostIP 57 | - name: POD_NAMESPACE 58 | valueFrom: 59 | fieldRef: 60 | fieldPath: metadata.namespace 61 | - name: NODE_NAME 62 | valueFrom: 63 | fieldRef: 64 | fieldPath: spec.nodeName 65 | - name: APP_LABEL 66 | valueFrom: 67 | fieldRef: 68 | fieldPath: metadata.labels['app'] 69 | - name: SERVICE_LABEL 70 | valueFrom: 71 | fieldRef: 72 | fieldPath: metadata.labels['service'] 73 | image: fluent/fluent-bit:1.4-debug 74 | name: fluent-bit 75 | ports: 76 | - containerPort: 2020 77 | resources: 78 | limits: 79 | cpu: 0.1 80 | memory: 500Mi 81 | requests: 82 | cpu: 0.1 83 | memory: 20Mi 84 | volumeMounts: 85 | - mountPath: /logging-volume 86 | name: logging-volume 87 | - mountPath: /var/fluent-bit 88 | name: fluent-data 89 | - mountPath: /fluent-bit/etc 90 | name: fluent-bit-config 91 | volumes: 92 | - emptyDir: {} 93 | name: logging-volume 94 | - emptyDir: {} 95 | name: fluent-data 96 | - configMap: 97 | name: fluent-bit-sidecar-config 98 | name: fluent-bit-config 99 | -------------------------------------------------------------------------------- /pkg/porter2k8s/kubeconfig.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "encoding/base64" 17 | "fmt" 18 | "io/ioutil" 19 | 20 | "git.corp.adobe.com/EchoSign/porter2k8s/pkg/vault" 21 | 22 | "github.com/ghodss/yaml" 23 | restclient "k8s.io/client-go/rest" 24 | "k8s.io/client-go/tools/clientcmd" 25 | ) 26 | 27 | // ClusterReference contains the path in vault and the name of the context, i.e. an identifier for the cluster. 28 | type ClusterReference struct { 29 | VaultPath string `json:"path"` 30 | Context string `json:"context"` 31 | } 32 | 33 | // ClusterConfigs contains all of the information for the Kubernetes client for the clusters. 34 | type ClusterConfigs struct { 35 | Clusters []ClusterReference `json:"clusters"` 36 | RegionMap map[string]*restclient.Config // map cluster context to config 37 | } 38 | 39 | // readReferences reads the cluster config file, which lists the clusters and their respective contexts. 40 | func readReferences(fileName string) (ClusterConfigs, error) { 41 | var clusterConfigs ClusterConfigs 42 | fileContent, readErr := ioutil.ReadFile(fileName) 43 | if readErr != nil { 44 | return clusterConfigs, readErr 45 | } 46 | if err := yaml.Unmarshal(fileContent, &clusterConfigs); err != nil { 47 | return clusterConfigs, fmt.Errorf("unable to read cluster reference file: %s\n%s", fileName, err) 48 | } 49 | return clusterConfigs, nil 50 | } 51 | 52 | // fetchConfigs fetches configs from vault. 53 | func (clusterConfigs *ClusterConfigs) fetchConfigs(client vault.VaultClientInterface) error { 54 | clusters := make(map[string]*restclient.Config) 55 | for _, clusterConfig := range clusterConfigs.Clusters { 56 | vaultSecret, err := client.Read(clusterConfig.VaultPath) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | if vaultSecret == nil { 62 | return fmt.Errorf("Vault secret not found for %s", clusterConfig.VaultPath) 63 | } 64 | 65 | encodedConfigValue, ok := vaultSecret["config"] 66 | if !ok { 67 | return fmt.Errorf("kubeconfig not found at location %s", clusterConfig.VaultPath) 68 | } 69 | 70 | encodedConfig, ok := encodedConfigValue.(string) 71 | if !ok { 72 | return fmt.Errorf("unable to assert Kubeconfig encoding to string \n%s", encodedConfigValue) 73 | } 74 | 75 | decodedConfig, err := base64.StdEncoding.DecodeString(encodedConfig) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | config, err := clientcmd.RESTConfigFromKubeConfig(decodedConfig) 81 | if err != nil { 82 | return err 83 | } 84 | // Update map of configs. 85 | clusters[clusterConfig.Context] = config 86 | } 87 | clusterConfigs.RegionMap = clusters 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /test/istio-mesh-config.yaml: -------------------------------------------------------------------------------- 1 | # Set the following variable to true to disable policy checks by the Mixer. 2 | # Note that metrics will still be reported to the Mixer. 3 | disablePolicyChecks: false 4 | # Set enableTracing to false to disable request tracing. 5 | enableTracing: true 6 | # 7 | # To disable the mixer completely (including metrics), comment out 8 | # the following lines 9 | mixerCheckServer: istio-policy.istio-system.svc.cluster.local:15004 10 | mixerReportServer: istio-telemetry.istio-system.svc.cluster.local:15004 11 | # This is the ingress service name, update if you used a different name 12 | ingressService: istio-ingress 13 | # 14 | # Along with discoveryRefreshDelay, this setting determines how 15 | # frequently should Envoy fetch and update its internal configuration 16 | # from istio Pilot. Lower refresh delay results in higher CPU 17 | # utilization and potential performance loss in exchange for faster 18 | # convergence. Tweak this value according to your setup. 19 | rdsRefreshDelay: 1s 20 | # 21 | defaultConfig: 22 | # NOTE: If you change any values in this section, make sure to make 23 | # the same changes in start up args in istio-ingress pods. 24 | # See rdsRefreshDelay for explanation about this setting. 25 | discoveryRefreshDelay: 1s 26 | # 27 | # TCP connection timeout between Envoy & the application, and between Envoys. 28 | connectTimeout: 1s 29 | # 30 | ### ADVANCED SETTINGS ############# 31 | # Where should envoy's configuration be stored in the istio-proxy container 32 | configPath: "/etc/istio/proxy" 33 | binaryPath: "/usr/local/bin/envoy" 34 | # The pseudo service name used for Envoy. 35 | serviceCluster: istio-proxy 36 | # These settings that determine how long an old Envoy 37 | # process should be kept alive after an occasional reload. 38 | drainDuration: 2s 39 | parentShutdownDuration: 3s 40 | # 41 | # The mode used to redirect inbound connections to Envoy. This setting 42 | # has no effect on outbound traffic: iptables REDIRECT is always used for 43 | # outbound connections. 44 | # If "REDIRECT", use iptables REDIRECT to NAT and redirect to Envoy. 45 | # The "REDIRECT" mode loses source addresses during redirection. 46 | # If "TPROXY", use iptables TPROXY to redirect to Envoy. 47 | # The "TPROXY" mode preserves both the source and destination IP 48 | # addresses and ports, so that they can be used for advanced filtering 49 | # and manipulation. 50 | # The "TPROXY" mode also configures the sidecar to run with the 51 | # CAP_NET_ADMIN capability, which is required to use TPROXY. 52 | #interceptionMode: REDIRECT 53 | # 54 | # Port where Envoy listens (on local host) for admin commands 55 | # You can exec into the istio-proxy container in a pod and 56 | # curl the admin port (curl http://localhost:15000/) to obtain 57 | # diagnostic information from Envoy. See 58 | # https://lyft.github.io/envoy/docs/operations/admin.html 59 | # for more details 60 | proxyAdminPort: 15000 61 | # 62 | # Zipkin trace collector 63 | zipkinAddress: "" 64 | # 65 | # Statsd metrics collector converts statsd metrics into Prometheus metrics. 66 | statsdUdpAddress: "" 67 | # 68 | # Mutual TLS authentication between sidecars and istio control plane. 69 | controlPlaneAuthPolicy: NONE 70 | # 71 | # Address where istio Pilot service is running 72 | discoveryAddress: istio-pilot:15007 73 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Adobe Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at Grp-opensourceoffice@adobe.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /test/multiple_pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 2 | kind: Deployment 3 | metadata: 4 | name: &name test 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: *name 9 | template: # create pods using pod definition in this template 10 | metadata: 11 | labels: 12 | app: *name 13 | spec: 14 | terminationGracePeriodSeconds: 32 15 | containers: 16 | - name: *name 17 | image: myregistry/echosign/test 18 | resources: 19 | requests: 20 | cpu: 0.75 21 | memory: 3G 22 | limits: 23 | cpu: 1 24 | memory: 3G 25 | ports: 26 | - containerPort: 8080 27 | lifecyle: 28 | preStop: 29 | exec: 30 | command: ["/bin/bash", "-c", "sleep 30"] 31 | livenessProbe: 32 | httpGet: 33 | path: /health 34 | port: 8080 35 | initialDelaySeconds: 60 36 | timeoutSeconds: 5 37 | periodSeconds: 10 38 | failureThreshold: 3 39 | readinessProbe: 40 | httpGet: 41 | path: /health 42 | port: 8080 43 | initialDelaySeconds: 30 44 | timeoutSeconds: 5 45 | periodSeconds: 10 46 | failureThreshold: 6 47 | envFrom: 48 | - configMapRef: 49 | name: cluster 50 | --- 51 | apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 52 | kind: StatefulSet 53 | metadata: 54 | name: &name smart # Must match image name for porter2k8s logic 55 | spec: 56 | replicas: 1 57 | selector: 58 | matchLabels: 59 | app: *name 60 | podManagementPolicy: Parallel 61 | template: # create pods using pod definition in this template 62 | metadata: 63 | labels: 64 | app: *name 65 | annotations: 66 | iam.amazonaws.com/role: arn:aws:iam::{{.ACCOUNT}}:role/recommend-{{.ENVIRONMENT}}-{{.REGION}} 67 | spec: 68 | terminationGracePeriodSeconds: 32 69 | containers: 70 | - name: *name 71 | image: myregistry/echosign/smart # Registry (up to first '/') is overriden by the cluster. 72 | resources: 73 | requests: 74 | cpu: 4 75 | memory: 50G 76 | nvidia.com/gpu: 1 77 | limits: 78 | cpu: 4 79 | memory: 50G 80 | nvidia.com/gpu: 1 81 | ports: 82 | - containerPort: 8080 83 | lifecycle: 84 | preStop: 85 | exec: 86 | command: ["/bin/bash", "-c", "sleep 30"] 87 | livenessProbe: 88 | exec: 89 | command: 90 | - /bin/sh 91 | - -c 92 | - pgrep "jupyter" > /dev/null || exit 1 93 | initialDelaySeconds: 90 94 | timeoutSeconds: 1 95 | periodSeconds: 10 96 | failureThreshold: 5 97 | readinessProbe: 98 | exec: 99 | command: 100 | - /bin/sh 101 | - -c 102 | - pgrep "jupyter" > /dev/null || exit 1 103 | initialDelaySeconds: 30 104 | timeoutSeconds: 30 105 | periodSeconds: 10 106 | failureThreshold: 5 107 | envFrom: 108 | - configMapRef: 109 | name: cluster 110 | volumeMounts: 111 | - mountPath: /storage 112 | name: smart-storage 113 | tolerations: 114 | - effect: NoSchedule 115 | key: nvidia.com/gpu 116 | operator: Exists 117 | volumeClaimTemplates: 118 | - metadata: 119 | name: smart-storage 120 | labels: 121 | app: smart 122 | spec: 123 | accessModes: 124 | - ReadWriteOnce 125 | storageClassName: ebs-sc 126 | resources: 127 | requests: 128 | storage: 2Ti 129 | 130 | -------------------------------------------------------------------------------- /pkg/porter2k8s/porterconfig_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "testing" 17 | 18 | log "github.com/sirupsen/logrus" 19 | ) 20 | 21 | var testFiles = []string{"../../test/porterconfig.yaml"} 22 | 23 | var vaultPath = "/secret/vault" 24 | 25 | // Expected values from test files in /test." 26 | var expectedReaderValues = map[string]Entry{ 27 | "ENV_TEST": { 28 | Path: "", 29 | Name: "ENV_TEST", 30 | Value: "environment value", 31 | }, 32 | "SECRET_TEST": { 33 | Path: "test_path", 34 | Field: "Password", 35 | Name: "SECRET_TEST", 36 | }, 37 | "BASE_SECRET": { 38 | Path: "different_path", 39 | Field: "Cert", 40 | Name: "BASE_SECRET", 41 | }, 42 | "AWS_ENV": { 43 | Path: "", 44 | Name: "AWS_ENV", 45 | Value: "aws_environment_variable", 46 | }, 47 | "AWS_SECRET": { 48 | Path: "aws_secret_path", 49 | Field: "aws_field", 50 | Name: "AWS_SECRET", 51 | }, 52 | "NO_VAL": { 53 | Name: "NO_VAL", 54 | Value: "", 55 | }, 56 | } 57 | 58 | var expectedRegionEnvVars = map[string]EnvVar{ 59 | "ENV_TEST": { 60 | Name: "ENV_TEST", 61 | Value: "environment value", 62 | }, 63 | "AWS_ENV": { 64 | Name: "AWS_ENV", 65 | Value: "aws_environment_variable", 66 | }, 67 | "NO_VAL": { 68 | Name: "NO_VAL", 69 | Value: "", 70 | }, 71 | } 72 | 73 | var expectedRegionSecretRefs = map[string]SecretRef{ 74 | "SECRET_TEST": { 75 | Path: vaultPath + "/test_base", 76 | Name: "SECRET_TEST", 77 | }, 78 | "BASE_SECRET": { 79 | Path: vaultPath, 80 | Name: "BASE_SECRET", 81 | }, 82 | "AWS_SECRET": { 83 | Path: vaultPath, 84 | Name: "AWS_SECRET", 85 | }, 86 | } 87 | 88 | func TestGetPorterEnv(t *testing.T) { 89 | testCases := []struct { 90 | files []string 91 | expectedResults map[string]Entry 92 | }{ 93 | { 94 | testFiles, 95 | expectedReaderValues, 96 | }, 97 | } 98 | log.SetLevel(log.DebugLevel) 99 | for _, tc := range testCases { 100 | t.Run("Test PorterConfig Reader", func(t *testing.T) { 101 | counter := 0 102 | porterConfig := &PorterEnv{} 103 | porterConfig.getPorterEnv(tc.files, map[string]bool{}) 104 | if len(porterConfig.Errors) != 0 { 105 | t.Errorf("Unexpected error %s\n", porterConfig.Errors) 106 | } 107 | for _, config := range porterConfig.PorterSecretConfigs { 108 | for _, entry := range config.Packages.Container { 109 | counter++ 110 | expectedEntry, ok := tc.expectedResults[entry.Name] 111 | if !ok { 112 | t.Errorf("Could not find entry: %+v\n", entry) 113 | } else if !entry.equal(expectedEntry) { 114 | t.Errorf("Unexpected Entry: %+v\nExpected: %+v\n", entry, expectedEntry) 115 | } else { 116 | t.Logf("Found %s\n", entry.Name) 117 | } 118 | } 119 | } 120 | if counter != len(expectedReaderValues) { 121 | t.Errorf("Found %d entries, expected %d\n", counter, len(expectedReaderValues)) 122 | } 123 | }) 124 | } 125 | } 126 | 127 | func TestParse(t *testing.T) { 128 | testCases := []struct { 129 | files []string 130 | expectedVars map[string]EnvVar 131 | expectedSecrets map[string]SecretRef 132 | }{ 133 | { 134 | testFiles, 135 | expectedRegionEnvVars, 136 | expectedRegionSecretRefs, 137 | }, 138 | } 139 | log.SetLevel(log.DebugLevel) 140 | for _, tc := range testCases { 141 | t.Run("Test PorterConfig parse", func(t *testing.T) { 142 | regionEnv := RegionEnv{ 143 | Cfg: &CmdConfig{ 144 | VaultBasePath: vaultPath, 145 | }, 146 | } 147 | porterConfig := &PorterEnv{} 148 | // Use output of first test. Probably not a great practice. 149 | porterConfig.getPorterEnv(tc.files, map[string]bool{}) 150 | porterConfig.parse(®ionEnv) 151 | 152 | if len(regionEnv.Vars) != len(tc.expectedVars) { 153 | t.Errorf("%d env vars found, expected %d", len(regionEnv.Vars), len(tc.expectedVars)) 154 | } 155 | if len(regionEnv.Secrets) != len(tc.expectedSecrets) { 156 | t.Errorf("%d secret references found, expected %d", len(regionEnv.Secrets), len(tc.expectedSecrets)) 157 | } 158 | 159 | for _, envVar := range regionEnv.Vars { 160 | expectedEnvVar, ok := tc.expectedVars[envVar.Name] 161 | if !ok { 162 | t.Errorf("Could not find EnvVar in expected list: %+v\n", envVar.Name) 163 | } 164 | if !envVar.equal(expectedEnvVar) { 165 | t.Errorf("Unexpected Entry: %+v\nExpected: %+v\n", envVar, expectedEnvVar) 166 | } 167 | } 168 | for _, secret := range regionEnv.Secrets { 169 | expectedSecret, ok := tc.expectedSecrets[secret.Name] 170 | if !ok { 171 | t.Errorf("Could not find Secret in expected list: %+v\n", secret.Name) 172 | } 173 | if !secret.equal(expectedSecret) { 174 | t.Errorf("Unexpected Entry: %+v\nExpected: %+v\n", secret, expectedSecret) 175 | } 176 | } 177 | }) 178 | } 179 | } 180 | 181 | func (expected Entry) equal(actual Entry) bool { 182 | if actual.Name != expected.Name || 183 | actual.Field != expected.Field || 184 | actual.Path != expected.Path || 185 | actual.Value != expected.Value { 186 | return false 187 | } 188 | return true 189 | } 190 | -------------------------------------------------------------------------------- /pkg/porter2k8s/podlogger.go: -------------------------------------------------------------------------------- 1 | package porter2k8s 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "os" 10 | "path/filepath" 11 | "sync" 12 | "time" 13 | 14 | log "github.com/sirupsen/logrus" 15 | apiv1 "k8s.io/api/core/v1" 16 | "k8s.io/apimachinery/pkg/api/errors" 17 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 18 | "k8s.io/apimachinery/pkg/watch" 19 | corev1 "k8s.io/client-go/kubernetes/typed/core/v1" 20 | ) 21 | 22 | // PodLogStreaming struct used for streaming pod logs from K8S 23 | type PodLogStreaming struct { 24 | watcher watch.Interface 25 | group sync.WaitGroup 26 | cancel context.CancelFunc 27 | } 28 | 29 | // PodLogSinker interface provides destination stream for pod logs 30 | // See LogrusSink(), DirectoryLogSink() 31 | type PodLogSinker interface { 32 | SinkForPod(podname string) (io.WriteCloser, error) 33 | } 34 | 35 | // streamPodLogs starts watching for pods inside the K8s cluster and streaming log data to specified destination. 36 | // Logs are retrieved for any pod matching the specified selector and that is not in phase 'pending'. 37 | // 38 | // ctx - context to cancel/timeout all operations 39 | // client - K8S client PodInterface 40 | // podselector - selector string for pod search 41 | // containerName - container to retrieve logs from 42 | // logDestination - callback for providing the log stream destination for a pod 43 | func streamPodLogs( 44 | ctx context.Context, 45 | client corev1.PodInterface, 46 | podselector string, 47 | containerName string, 48 | logDestination PodLogSinker) (*PodLogStreaming, error) { 49 | 50 | options := metav1.ListOptions{ 51 | LabelSelector: podselector, 52 | FieldSelector: "status.phase!=Pending,status.phase!=Unknown", 53 | } 54 | watcher, err := client.Watch(ctx, options) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | // query and attach to pod logs in the background 60 | logCtx, cancel := context.WithCancel(ctx) 61 | p := PodLogStreaming{watcher: watcher, cancel: cancel} 62 | p.group.Add(1) 63 | go func() { 64 | defer p.group.Done() 65 | 66 | watchCh := watcher.ResultChan() 67 | for { 68 | select { 69 | case <-logCtx.Done(): 70 | return 71 | 72 | case event, ok := <-watchCh: 73 | if !ok { 74 | return 75 | } 76 | 77 | switch event.Type { 78 | case watch.Added: 79 | pod, ok := event.Object.(*apiv1.Pod) 80 | if !ok { 81 | log.Warnf("%#v is not a pod event", event) 82 | return 83 | } 84 | 85 | p.group.Add(1) 86 | go func(podname string) { 87 | defer p.group.Done() 88 | 89 | log.Infof("Retrieving logs for pod %s", podname) 90 | 91 | err := attachToPod(logCtx, podname, containerName, client, logDestination) 92 | switch err { 93 | case nil, io.EOF: 94 | log.Infof("End of logs for pod %s", podname) 95 | case context.Canceled: 96 | log.Debugf("Canceled retrieving logs for pod %s", podname) 97 | default: 98 | log.Warnf("Error consuming logs from pod %s: %v", podname, err) 99 | } 100 | }(pod.Name) 101 | 102 | case watch.Error: 103 | err := errors.FromObject(event.Object) 104 | log.Warnf("error reported while watching pods: %v", err) 105 | } 106 | } 107 | } 108 | }() 109 | return &p, nil 110 | } 111 | 112 | // Stop ends the pod watching and log streaming operations. 113 | // gracePeriod - time to allow for any log streaming to finish. 114 | func (p *PodLogStreaming) Stop(gracePeriod time.Duration) bool { 115 | if p == nil { 116 | return true 117 | } 118 | defer p.cancel() // ultimately reel in any runaway log-streaming-gophers after the graceperiod 119 | 120 | p.watcher.Stop() // stop the main pod-watching-gopher 121 | c := make(chan struct{}) 122 | go func() { 123 | defer close(c) 124 | p.group.Wait() 125 | }() 126 | select { 127 | case <-c: 128 | return true 129 | case <-time.After(gracePeriod): 130 | return false // timed out 131 | } 132 | } 133 | 134 | func attachToPod( 135 | ctx context.Context, podname string, containerName string, client corev1.PodInterface, sinker PodLogSinker, 136 | ) error { 137 | podLogOptions := apiv1.PodLogOptions{ 138 | Container: containerName, 139 | Follow: true, 140 | } 141 | src, err := client.GetLogs(podname, &podLogOptions).Stream(ctx) 142 | if err != nil { 143 | return err 144 | } 145 | defer src.Close() 146 | 147 | dst, err := sinker.SinkForPod(podname) 148 | if err != nil { 149 | return err 150 | } 151 | defer dst.Close() 152 | 153 | _, err = io.Copy(dst, src) 154 | return err 155 | } 156 | 157 | type directoryLogSink struct { 158 | logDir string 159 | } 160 | 161 | func (dls directoryLogSink) SinkForPod(podname string) (io.WriteCloser, error) { 162 | path := filepath.Join(dls.logDir, podname+".log") 163 | return os.Create(path) 164 | } 165 | 166 | // DirectoryLogSink creates a new PodLogSinker that writes pod logs into the filesystem at the specified location. 167 | func DirectoryLogSink(logDir string) PodLogSinker { 168 | return directoryLogSink{logDir: logDir} 169 | } 170 | 171 | // LogrusSink creates a new PodLogSinker that streams pod logs to the logrus std logger 172 | func LogrusSink() PodLogSinker { 173 | return sinkForPodFunc(func(podname string) (io.WriteCloser, error) { 174 | return prefixedWriter(log.StandardLogger().Writer(), fmt.Sprintf("[%s]", podname)), nil 175 | }) 176 | } 177 | 178 | // NullLogSink sends to /dev/null 179 | func NullLogSink() PodLogSinker { 180 | return sinkForPodFunc(func(podname string) (io.WriteCloser, error) { 181 | return nopWriterCloser{Writer: ioutil.Discard}, nil 182 | }) 183 | } 184 | 185 | type nopWriterCloser struct { 186 | io.Writer 187 | } 188 | 189 | func (nopWriterCloser) Close() error { 190 | return nil 191 | } 192 | 193 | type sinkForPodFunc func(podname string) (io.WriteCloser, error) 194 | 195 | func (fn sinkForPodFunc) SinkForPod(podname string) (io.WriteCloser, error) { 196 | return fn(podname) 197 | } 198 | 199 | func prefixedWriter(dest io.Writer, prefix string) io.WriteCloser { 200 | reader, writer := io.Pipe() 201 | 202 | go func() { 203 | scanner := bufio.NewScanner(reader) 204 | for scanner.Scan() { 205 | line := fmt.Sprintf("%s %s\n", prefix, scanner.Bytes()) 206 | if _, err := dest.Write([]byte(line)); err != nil { 207 | break 208 | } 209 | } 210 | }() 211 | return writer 212 | } 213 | -------------------------------------------------------------------------------- /pkg/porter2k8s/porterconfig.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "fmt" 17 | "io" 18 | "os" 19 | "path/filepath" 20 | 21 | log "github.com/sirupsen/logrus" 22 | "gopkg.in/yaml.v2" 23 | ) 24 | 25 | // PorterEnv contains the env variables, and secrets for a specific region. 26 | type PorterEnv struct { 27 | Errors []error 28 | PorterSecretConfigs []PorterSecretYaml 29 | } 30 | 31 | // Entry in a porter config secret yaml. 32 | type Entry struct { 33 | Path string `yaml:"path"` 34 | Field string `yaml:"field"` 35 | Name string `yaml:"name"` 36 | Value string `yaml:"value"` 37 | } 38 | 39 | // PorterSecretYaml contains the format of a Porter secret yaml file. It is used for unmarshalling json. 40 | type PorterSecretYaml struct { 41 | Bundle string `yaml:"bundle"` 42 | Includes []string `yaml:"includes"` 43 | Group int `yaml:"group"` 44 | BasePath string `yaml:"base_path"` 45 | Packages struct { 46 | // Only concerned with container secrets and env variables. 47 | Container []Entry `yaml:"container"` 48 | } `yaml:"packages"` 49 | } 50 | 51 | // getEnv implements configReader interface 52 | func (porterEnv *PorterEnv) getEnv(environment *RegionEnv) { 53 | // Construct paths to secrets.yaml files 54 | // For example .porter/operation_data/dev/us-east-1/secrets.yaml 55 | file := []string{fmt.Sprintf( 56 | "%s/.porter/operation_data/%s/%s/secrets.yaml", 57 | environment.Cfg.ConfigPath, 58 | environment.Cfg.Environment, 59 | environment.Region, 60 | )} 61 | 62 | porterEnv.getPorterEnv(file, map[string]bool{}) 63 | log.Debugf("%+v\n", porterEnv) 64 | environment.Errors = porterEnv.Errors 65 | } 66 | 67 | // parse reads the PorterEnv and outputs EnvVar and SecretRef types. 68 | func (porterEnv *PorterEnv) parse(environment *RegionEnv) { 69 | // Segregate secret references from plain environment variables. 70 | for _, config := range porterEnv.PorterSecretConfigs { 71 | for _, entry := range config.Packages.Container { 72 | // Secrets have paths, environment variables do not. 73 | if entry.Path != "" { 74 | // Combine base path with path of secret, minus the final key. 75 | path := filepath.Dir(environment.Cfg.VaultBasePath + config.BasePath + "/" + entry.Path) 76 | // Only the container type is supported with the porterconfig. 77 | newSecret := SecretRef{Name: entry.Name, Path: path, Kind: "container"} 78 | environment.Secrets = append(environment.Secrets, newSecret) 79 | } else { 80 | newEnvVar := EnvVar{Name: entry.Name, Value: entry.Value} 81 | environment.Vars = append(environment.Vars, newEnvVar) 82 | } 83 | } 84 | } 85 | } 86 | 87 | // getPorterEnv unmarshals Porter yaml files into environment variables and secrets. 88 | // The files argument is a map of region or cluster name to file path. 89 | // Ex) "us-east-1": ".porter/operation_data/dev/us-east-1/secrets.yaml" 90 | func (porterEnv *PorterEnv) getPorterEnv(files []string, overridden map[string]bool) { 91 | // Process each yaml file. There will only be one at first, but there may be multiple through recursion. 92 | for _, file := range files { 93 | // Create a slice of configs just for this single yaml file. One for each yaml document. 94 | var referencedFiles []string 95 | 96 | // Open the file. 97 | fileReader, err := os.Open(file) 98 | if err != nil { 99 | porterEnv.Errors = append(porterEnv.Errors, err) 100 | return 101 | } 102 | 103 | // Yaml Decoder can handle multiple yaml documents in a single file, delimited by '---'. 104 | decoder := yaml.NewDecoder(fileReader) 105 | for { 106 | // Create a separate Porter config for each yaml document within the file. 107 | var porterSecretConfig PorterSecretYaml 108 | err = decoder.Decode(&porterSecretConfig) 109 | if err == io.EOF { 110 | // The file has been read to the end. 111 | break 112 | } else if err != nil { 113 | porterEnv.Errors = append(porterEnv.Errors, err) 114 | // The same yaml document will be re-read infinitely unless we break on error. 115 | break 116 | } 117 | // Filter out yaml documents without container env variables or secrets. 118 | if len(porterSecretConfig.Packages.Container) > 0 { 119 | // Env variables and secrets should override those in the referenced yaml files. 120 | // This allows setting a value for a specific cluster while all other clusters share a default. 121 | // Remove entries that are overridden or add them to the override map. 122 | // DO NOT HAVE THE SAME ENTRY IN TWO SIMULTANEOUSLY REFERENCED FILES!!! 123 | for i := len(porterSecretConfig.Packages.Container) - 1; i >= 0; i-- { 124 | if _, ok := overridden[porterSecretConfig.Packages.Container[i].Name]; ok { 125 | log.Infof("Env variable %s, overridden\n", porterSecretConfig.Packages.Container[i].Name) 126 | porterSecretConfig.Packages.Container = append( 127 | porterSecretConfig.Packages.Container[:i], 128 | porterSecretConfig.Packages.Container[i+1:]..., 129 | ) 130 | } else { 131 | overridden[porterSecretConfig.Packages.Container[i].Name] = true 132 | } 133 | } 134 | porterEnv.PorterSecretConfigs = append(porterEnv.PorterSecretConfigs, porterSecretConfig) 135 | } 136 | if porterSecretConfig.Bundle != "" { 137 | // Construct referenced file paths. 138 | // Referenced files are only in the first yaml section of the file, 139 | // which is the only section with "Bundle". 140 | for _, relativePath := range porterSecretConfig.Includes { 141 | // Reference files use relative paths. Filepath knows how to handle them. 142 | referencedFiles = append(referencedFiles, filepath.Join(filepath.Dir(file), relativePath)) 143 | } 144 | } 145 | } 146 | 147 | // Merge porter secret configs referenced in the includes section into the main config for the region. 148 | // Recurse on referenced yaml files. 149 | log.Infof("Recursing on files: %s\n", referencedFiles) 150 | referencedPorterEnv := &PorterEnv{} 151 | referencedPorterEnv.getPorterEnv(referencedFiles, overridden) 152 | porterEnv.PorterSecretConfigs = append( 153 | porterEnv.PorterSecretConfigs, 154 | referencedPorterEnv.PorterSecretConfigs..., 155 | ) 156 | porterEnv.Errors = append(porterEnv.Errors, referencedPorterEnv.Errors...) 157 | log.Debugf("Combined configs: %+v\n", porterEnv.PorterSecretConfigs) 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /pkg/porter2k8s/dynamic_watch_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "fmt" 17 | "testing" 18 | 19 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 20 | ) 21 | 22 | var redisWatch = unstructured.Unstructured{ 23 | Object: map[string]interface{}{ 24 | "apiVersion": "azure.microsoft.com/v1alpha1", 25 | "kind": "RedisCache", 26 | "metadata": map[string]interface{}{ 27 | "name": "myRedis", 28 | "namespace": "test", 29 | }, 30 | "spec": map[string]interface{}{ 31 | "location": "westeurope", 32 | }, 33 | "status": map[string]interface{}{ 34 | "message": "successfully provisioned", 35 | "output": "10.20.44.132", 36 | "provisioned": true, 37 | "requested": "2021-01-28T19:40:39Z", 38 | "resourceId": "/subscriptions/123/resourceGroups/rg/providers/Microsoft.Cache/Redis/myRedis", 39 | "state": "Succeeded", 40 | }, 41 | }, 42 | } 43 | 44 | var dynamoWatch = unstructured.Unstructured{ 45 | Object: map[string]interface{}{ 46 | "apiVersion": "dynamodb.services.k8s.aws/v1alpha1", 47 | "kind": "Table", 48 | "metadata": map[string]interface{}{ 49 | "name": "myDynamo", 50 | "namespace": "test", 51 | }, 52 | "spec": map[string]interface{}{ 53 | "attributeDefinitions": []map[string]interface{}{ 54 | { 55 | "attributeName": "agreement_id", 56 | "attributeType": "S", 57 | }, 58 | }, 59 | "keySchema": []map[string]interface{}{ 60 | { 61 | "attributeName": "agreement_id", 62 | "keyType": "HASH", 63 | }, 64 | }, 65 | "provisionedThroughput": map[string]interface{}{ 66 | "readCapacityUnits": 30, 67 | "writeCapacityUnits": 30, 68 | }, 69 | "tableName": "agreementmetadata", 70 | }, 71 | "status": map[string]interface{}{ 72 | "ackResourceMetadata": map[string]interface{}{ 73 | "arn": "arn:aws:dynamodb:us-east-1:12345:table/myDynamo", 74 | "ownerAccountID": "$(AWS_ACCOUNT_ID)", 75 | }, 76 | "conditions": []string{}, 77 | "creationDateTime": "2021-04-01T22:22:02Z", 78 | "itemCount": 0, 79 | "tableID": "0123456789", 80 | "tableSizeBytes": 0, 81 | "tableStatus": "ACTIVE", 82 | }, 83 | }, 84 | } 85 | 86 | func TestNewWatchConfigStatefulSet(t *testing.T) { 87 | testCases := []struct { 88 | previous bool 89 | expectedReplicas int64 90 | }{ 91 | {true, 4}, 92 | {false, 1}, 93 | } 94 | 95 | for _, tc := range testCases { 96 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 97 | service := statefulSet.DeepCopy() 98 | previous := &unstructured.Unstructured{} 99 | if tc.previous { 100 | previous = statefulSet.DeepCopy() 101 | unstructured.SetNestedField(previous.Object, tc.expectedReplicas, "spec", "replicas") 102 | } 103 | conf := NewWatchConfig(service, previous, 30) 104 | readyReplicas := conf.Success.Conditions[0].StateInt64 105 | if readyReplicas != tc.expectedReplicas { 106 | t.Errorf("Unexpected readyReplica count %d. Expected %d", readyReplicas, tc.expectedReplicas) 107 | } 108 | t.Logf("WatchConfig %+v", conf) 109 | }) 110 | } 111 | } 112 | 113 | func TestNewWatchConfig(t *testing.T) { 114 | testCases := []struct { 115 | version string 116 | kind string 117 | buffer int32 118 | expTimeout int64 119 | expSuccessCond int 120 | expFailureCond int 121 | expLogic ConditionLogic 122 | }{ 123 | // Deployment and StatefulSet timeouts are more complicated and tested as part of TestCalculateWatchTimeout 124 | {"dynamodb.services.k8s.aws/v1alpha1", "Table", 0, 600, 1, 1, AndLogic}, 125 | {"azure.microsoft.com/v1alpha1", "RedisCache", 0, 900, 1, 2, AndLogic}, 126 | {"azure.microsoft.com/v1alpha1", "CosmosDB", 0, 900, 1, 2, AndLogic}, 127 | {"azure.microsoft.com/v1alpha1", "ResourceGroup", 0, 120, 1, 2, AndLogic}, 128 | {"azure.microsoft.com/v1alpha2", "MySQLServer", 0, 1800, 1, 2, AndLogic}, 129 | {"monitoring.coreos.com/v1", "ServiceMonitor", 0, 0, 0, 0, AndLogic}, 130 | {"servicecatalog.k8s.io/v1beta1", "ServiceInstance", 0, 3000, 1, 2, AndLogic}, 131 | {"batch/v1", "Job", 30, 30, 1, 0, AndLogic}, 132 | } 133 | 134 | for _, tc := range testCases { 135 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 136 | service := unstructured.Unstructured{ 137 | Object: map[string]interface{}{ 138 | "apiVersion": tc.version, 139 | "kind": tc.kind, 140 | "metadata": map[string]interface{}{ 141 | "name": "testApp", 142 | }, 143 | }, 144 | } 145 | previous := unstructured.Unstructured{ 146 | Object: map[string]interface{}{ 147 | "apiVersion": tc.version, 148 | "kind": tc.kind, 149 | "metadata": map[string]interface{}{ 150 | "name": "testApp", 151 | }, 152 | "spec": map[string]interface{}{ 153 | "replicas": 5, 154 | }, 155 | }, 156 | } 157 | conf := NewWatchConfig(&service, &previous, tc.buffer) 158 | 159 | if *conf.TimeoutSeconds != tc.expTimeout { 160 | t.Errorf("Unexpected watch timeout %d, expected %d", *conf.TimeoutSeconds, tc.expTimeout) 161 | } 162 | if len(conf.Success.Conditions) != tc.expSuccessCond { 163 | t.Errorf("Unexpected number of success conditions %d, expected %d", len(conf.Success.Conditions), 164 | tc.expSuccessCond) 165 | } 166 | if len(conf.Failure.Conditions) != tc.expFailureCond { 167 | t.Errorf("Unexpected number of failure conditions %d, expected %d", len(conf.Failure.Conditions), 168 | tc.expFailureCond) 169 | } 170 | if conf.Success.Logic != tc.expLogic { 171 | t.Errorf("Unexpected logical operator %d, expected %d", conf.Success.Logic, tc.expLogic) 172 | } 173 | }) 174 | } 175 | } 176 | 177 | func TestCompareStatusMaps(t *testing.T) { 178 | testCases := []struct { 179 | status map[string]interface{} 180 | conditionMap map[string]interface{} 181 | expected bool 182 | }{ 183 | { 184 | map[string]interface{}{"message": "Irrelevant nonesense", "type": "Failed", "status": "True"}, 185 | map[string]interface{}{"type": "Failed", "status": "True"}, 186 | true, 187 | }, 188 | { 189 | map[string]interface{}{"message": "Irrelevant nonesense", "type": "Failed", "status": "True"}, 190 | map[string]interface{}{"type": "Ready", "status": "True"}, 191 | false, 192 | }, 193 | { 194 | map[string]interface{}{"message": "Irrelevant nonesense", "type": "Failed", "status": "True"}, 195 | map[string]interface{}{"type": "Failed", "missingKey": "True"}, 196 | false, 197 | }, 198 | } 199 | for _, tc := range testCases { 200 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 201 | result := compareStatusMaps(tc.status, tc.conditionMap) 202 | if result != tc.expected { 203 | t.Errorf("Unexpected comparison result. Expected %t", tc.expected) 204 | } 205 | }) 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /pkg/porter2k8s/simpleconfig_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "testing" 17 | 18 | log "github.com/sirupsen/logrus" 19 | ) 20 | 21 | var simpleTestFiles = []string{"../../test/simpleconfig.yaml"} 22 | 23 | var simpleVaultPath = "/secret/vault" 24 | 25 | // Expected values from test files in /test." 26 | var simpleExpectedReaderValues = map[string]SimpleEntry{ 27 | "ENV_TEST": { 28 | Path: "", 29 | Name: "ENV_TEST", 30 | Value: "environment value", 31 | }, 32 | "SECRET_TEST": { 33 | Path: "test_path", 34 | Name: "SECRET_TEST", 35 | }, 36 | "BASE_SECRET": { 37 | Path: "different_path", 38 | Name: "BASE_SECRET", 39 | }, 40 | "AWS_ENV": { 41 | Path: "", 42 | Name: "AWS_ENV", 43 | Value: "aws_environment_variable", 44 | }, 45 | "AWS_SECRET": { 46 | Path: "aws_secret_path", 47 | Name: "AWS_SECRET", 48 | }, 49 | "NO_VAL": { 50 | Name: "NO_VAL", 51 | Value: "", 52 | }, 53 | "NOT_VAULT_KEY": { 54 | Path: "test_path", 55 | Name: "NOT_VAULT_KEY", 56 | Key: "vault_key", 57 | }, 58 | "SLASHES": { 59 | Path: "/another_slash", 60 | Name: "SLASHES", 61 | }, 62 | "FROM_K8S": { 63 | Name: "FROM_K8S", 64 | K8sSecret: "k8s-secret", 65 | Source: "kubernetes", 66 | Key: "k8s_key", 67 | }, 68 | "FROM_OBJECT": { 69 | Name: "FROM_OBJECT", 70 | K8sObject: K8sObjectRef{ 71 | Name: "test_object", 72 | Kind: "redis", 73 | }, 74 | Source: "object", 75 | Path: "status.url", 76 | }, 77 | "ENVIRONMENT": { 78 | Name: "ENVIRONMENT", 79 | Value: "gorilla", 80 | }, 81 | "REGION": { 82 | Name: "REGION", 83 | Value: "westphalia", 84 | }, 85 | "P2K8S_SECRET": { 86 | Name: "P2K8S_SECRET", 87 | Path: "secret_path", 88 | Key: "Password", 89 | }, 90 | } 91 | 92 | var simpleExpectedRegionEnvVars = map[string]EnvVar{ 93 | "ENV_TEST": { 94 | Name: "ENV_TEST", 95 | Value: "environment value", 96 | }, 97 | "AWS_ENV": { 98 | Name: "AWS_ENV", 99 | Value: "aws_environment_variable", 100 | }, 101 | } 102 | 103 | var simpleExpectedRegionSecretRefs = map[string]SecretRef{ 104 | "SECRET_TEST": { 105 | Path: simpleVaultPath + "/test_base/test_path", 106 | Name: "SECRET_TEST", 107 | }, 108 | "BASE_SECRET": { 109 | Path: simpleVaultPath + "/different_path", 110 | Name: "BASE_SECRET", 111 | }, 112 | "AWS_SECRET": { 113 | Path: simpleVaultPath + "/aws_secret_path", 114 | Name: "AWS_SECRET", 115 | }, 116 | "NOT_VAULT_KEY": { 117 | Path: simpleVaultPath + "/test_base/test_path", 118 | Name: "NOT_VAULT_KEY", 119 | Key: "vault_key", 120 | }, 121 | "SLASHES": { 122 | Path: simpleVaultPath + "/extra_slash/more_slash/another_slash", 123 | Name: "SLASHES", 124 | }, 125 | "NO_VAL": { 126 | Name: "NO_VAL", 127 | Path: simpleVaultPath + "/test_base", 128 | }, 129 | "P2K8S_SECRET": { 130 | Name: "P2K8S_SECRET", 131 | Path: simpleVaultPath + "/kinda_matters_now/secret_path", 132 | Key: "Password", 133 | }, 134 | } 135 | 136 | var simpleExpectedRegionSecretKeyRefs = map[string]SecretKeyRef{ 137 | "FROM_K8S": { 138 | Name: "FROM_K8S", 139 | Key: "k8s_key", 140 | Secret: "k8s-secret", 141 | }, 142 | "FROM_OBJECT": { 143 | Name: "FROM_OBJECT", 144 | Key: "status.url", 145 | Secret: "test-object-redis", 146 | }, 147 | } 148 | 149 | var simpleExpectedClusterSettings = map[string]string{ 150 | "ENVIRONMENT": "gorilla", 151 | "REGION": "westphalia", 152 | } 153 | 154 | func TestGetSimpleEnv(t *testing.T) { 155 | testCases := []struct { 156 | files []string 157 | expectedResults map[string]SimpleEntry 158 | }{ 159 | { 160 | simpleTestFiles, 161 | simpleExpectedReaderValues, 162 | }, 163 | } 164 | log.SetLevel(log.DebugLevel) 165 | for _, tc := range testCases { 166 | t.Run("Test SimpleConfig Reader", func(t *testing.T) { 167 | counter := 0 168 | simpleConfig := &SimpleEnv{} 169 | simpleConfig.getSimpleEnv(tc.files, map[string]bool{}, 0) 170 | if len(simpleConfig.Errors) != 0 { 171 | t.Errorf("Unexpected error %s\n", simpleConfig.Errors) 172 | } 173 | for _, config := range simpleConfig.EnvConfigs { 174 | for _, entry := range config.Vars { 175 | counter++ 176 | expectedEntry, ok := tc.expectedResults[entry.Name] 177 | if !ok { 178 | t.Errorf("Unexpected Entry: %+v\n", entry) 179 | } else if !entry.equal(expectedEntry) { 180 | t.Errorf("Unexpected Entry values: %+v\nExpected: %+v\n", entry, expectedEntry) 181 | } else { 182 | t.Logf("Found %s\n", entry.Name) 183 | } 184 | } 185 | } 186 | if counter != len(simpleExpectedReaderValues) { 187 | t.Errorf("Found %d entries, expected %d\n", counter, len(simpleExpectedReaderValues)) 188 | } 189 | }) 190 | } 191 | } 192 | 193 | func TestSimpleParse(t *testing.T) { 194 | testCases := []struct { 195 | files []string 196 | expectedVars map[string]EnvVar 197 | expectedSecrets map[string]SecretRef 198 | expectedSecretKeys map[string]SecretKeyRef 199 | expectedSettings map[string]string 200 | }{ 201 | { 202 | simpleTestFiles, 203 | simpleExpectedRegionEnvVars, 204 | simpleExpectedRegionSecretRefs, 205 | simpleExpectedRegionSecretKeyRefs, 206 | simpleExpectedClusterSettings, 207 | }, 208 | } 209 | log.SetLevel(log.DebugLevel) 210 | for _, tc := range testCases { 211 | t.Run("Test SimpleConfig parse", func(t *testing.T) { 212 | regionEnv := RegionEnv{ 213 | Cfg: &CmdConfig{ 214 | VaultBasePath: simpleVaultPath, 215 | }, 216 | ClusterSettings: map[string]string{}, 217 | ObjectRefs: map[string][]string{}, 218 | } 219 | simpleConfig := &SimpleEnv{} 220 | // Use output of first test. Probably not a great practice. 221 | simpleConfig.getSimpleEnv(tc.files, map[string]bool{}, 0) 222 | simpleConfig.parse(®ionEnv) 223 | 224 | if len(regionEnv.Vars) != len(tc.expectedVars) { 225 | t.Errorf("%d env vars found, expected %d", len(regionEnv.Vars), len(tc.expectedVars)) 226 | } 227 | if len(regionEnv.Secrets) != len(tc.expectedSecrets) { 228 | t.Errorf("%d secret references found, expected %d", len(regionEnv.Secrets), len(tc.expectedSecrets)) 229 | } 230 | if len(regionEnv.SecretKeyRefs) != len(tc.expectedSecretKeys) { 231 | t.Errorf("%d secret key references found, expected %d", len(regionEnv.SecretKeyRefs), len(tc.expectedSecretKeys)) 232 | } 233 | if len(regionEnv.ClusterSettings) != len(tc.expectedSettings) { 234 | t.Errorf("%d cluster settings found, expected %d", len(regionEnv.ClusterSettings), len(tc.expectedSettings)) 235 | } 236 | 237 | for _, envVar := range regionEnv.Vars { 238 | expectedEnvVar, ok := tc.expectedVars[envVar.Name] 239 | if !ok { 240 | t.Errorf("Could not find EnvVar in expected list: %+v\n", envVar.Name) 241 | } 242 | if !envVar.equal(expectedEnvVar) { 243 | t.Errorf("Unexpected EnvVar: %+v\nExpected: %+v\n", envVar, expectedEnvVar) 244 | } 245 | } 246 | for _, secret := range regionEnv.Secrets { 247 | expectedSecret, ok := tc.expectedSecrets[secret.Name] 248 | if !ok { 249 | t.Errorf("Could not find Secret in expected list: %+v\n", secret.Name) 250 | } 251 | if !secret.equal(expectedSecret) { 252 | t.Errorf("Unexpected Secret: %+v\nExpected: %+v\n", secret, expectedSecret) 253 | } 254 | } 255 | for _, secretKeyRef := range regionEnv.SecretKeyRefs { 256 | expectedSecretKeyRef, ok := tc.expectedSecretKeys[secretKeyRef.Name] 257 | if !ok { 258 | t.Errorf("Could not find Secret Key Ref in expected list: %+v\n", secretKeyRef.Name) 259 | } 260 | if !secretKeyRef.equal(expectedSecretKeyRef) { 261 | t.Errorf("Unexpected Secret Key Ref: %+v\nExpected: %+v\n", secretKeyRef, expectedSecretKeyRef) 262 | } 263 | } 264 | for setting, value := range regionEnv.ClusterSettings { 265 | expectedValue, ok := tc.expectedSettings[setting] 266 | if !ok { 267 | t.Errorf("Could not find expected Setting: %s", setting) 268 | } 269 | if value != expectedValue { 270 | t.Errorf("Unexpected Setting value: %s Expected: %s", value, expectedValue) 271 | } 272 | } 273 | }) 274 | } 275 | } 276 | 277 | func (expected SimpleEntry) equal(actual SimpleEntry) bool { 278 | if actual.Name != expected.Name || 279 | actual.Key != expected.Key || 280 | actual.Path != expected.Path || 281 | actual.Value != expected.Value { 282 | return false 283 | } 284 | return true 285 | } 286 | -------------------------------------------------------------------------------- /pkg/porter2k8s/updater.go: -------------------------------------------------------------------------------- 1 | package porter2k8s 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | log "github.com/sirupsen/logrus" 9 | "k8s.io/apimachinery/pkg/api/errors" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 12 | "k8s.io/apimachinery/pkg/runtime/schema" 13 | "k8s.io/apimachinery/pkg/types" 14 | ) 15 | 16 | var createOpts metav1.CreateOptions = metav1.CreateOptions{FieldManager: "porter2k8s"} 17 | var updateOpts metav1.UpdateOptions = metav1.UpdateOptions{FieldManager: "porter2k8s"} 18 | 19 | // runUpdate Creates/Updates objects across regions. 20 | func runUpdate(regionEnvs []*RegionEnv, updateFunc UpdateFn) error { 21 | 22 | // See if all regions made it here. 23 | log.Debugf("Number of regions: %d", len(regionEnvs)) 24 | 25 | updater := func() <-chan *RegionEnv { 26 | // WaitGroup for all of the updates. 27 | var wg sync.WaitGroup 28 | wg.Add(len(regionEnvs)) 29 | objectStream := make(chan *RegionEnv) 30 | for _, regionEnv := range regionEnvs { 31 | go updateFunc(&wg, regionEnv, objectStream) 32 | } 33 | // Close the channel once all regionEnvs have been passed. 34 | // This needs to be inside the update function, because channels should be closed by the sender. 35 | go func() { 36 | wg.Wait() 37 | close(objectStream) 38 | }() 39 | return objectStream 40 | } 41 | 42 | updates := updater() 43 | for update := range updates { 44 | if len(update.Errors) > 0 { 45 | return fmt.Errorf("Update failed: %s", update.Errors) 46 | } 47 | } 48 | 49 | return nil 50 | } 51 | 52 | // updateDeploymentRegion creates/updates and monitors a deployment. 53 | func updatePodObjectRegion( 54 | wg *sync.WaitGroup, 55 | regionEnv *RegionEnv, 56 | deploymentStream chan *RegionEnv, 57 | ) { 58 | defer wg.Done() 59 | 60 | // Replace docker registry with regional value. 61 | regionEnv.updateRegistry() 62 | // Add sidecar if one exists in the cluster settings. 63 | regionEnv.addSidecarContainer() 64 | // Inject service binding secrets. 65 | regionEnv.injectSecretKeyRefs() 66 | 67 | podLogSinker := regionEnv.Cfg.PodLogSinker 68 | if regionEnv.Cfg.LogMode == "single" { 69 | regionEnv.Cfg.LogOnce.Do(func() { podLogSinker = LogrusSink() }) 70 | } 71 | 72 | podLogger, logErr := streamPodLogs( 73 | regionEnv.Context, 74 | regionEnv.Clientset.CoreV1().Pods(regionEnv.Cfg.Namespace), 75 | fmt.Sprintf("app=%s,sha=%s", regionEnv.PodObject.GetName(), regionEnv.PodObject.GetLabels()["sha"]), 76 | regionEnv.PodObject.GetName(), 77 | podLogSinker) 78 | if logErr != nil { 79 | regionEnv.Errors = append(regionEnv.Errors, logErr) 80 | } 81 | 82 | if len(regionEnv.Errors) > 1 { 83 | deploymentStream <- regionEnv 84 | return 85 | } 86 | defer podLogger.Stop(1 * time.Second) // cap pod log streaming 1 second after update returns. 87 | 88 | objectStream := make(chan *unstructured.Unstructured) 89 | defer close(objectStream) 90 | go dynamicUpdate(nil, regionEnv.PodObject, regionEnv, objectStream) 91 | update := <-objectStream 92 | 93 | if len(regionEnv.Errors) > 0 { 94 | deploymentStream <- regionEnv 95 | return 96 | } 97 | 98 | regionEnv.Logger.Debugf("Update of Pod Object completed: %+v", update) 99 | 100 | if cleanupErr := regionEnv.postDeployCleanup(); cleanupErr != nil { 101 | regionEnv.Errors = append(regionEnv.Errors, cleanupErr) 102 | } 103 | 104 | select { 105 | case <-regionEnv.Context.Done(): 106 | regionEnv.Logger.Infof("Received Done") 107 | return 108 | case deploymentStream <- regionEnv: 109 | } 110 | 111 | } 112 | 113 | func updateDynamicServiceRegion( 114 | wg *sync.WaitGroup, 115 | regionEnv *RegionEnv, 116 | serviceStream chan *RegionEnv, 117 | ) { 118 | defer wg.Done() 119 | 120 | // Replace domain name with regional value. 121 | // This affects all objects with domain names in them, not just services. 122 | // Since the dynamic update is mandatory, we do it here rather than in istio or ingress. 123 | regionEnv.updateDomainName() 124 | 125 | // Increase minimum replica number if required. 126 | regionEnv.updateHPAMinimum() 127 | 128 | regionEnv.Logger.Debugf("Provisioning Dynamic Objects: %+v", regionEnv.Unstructured) 129 | regionEnv.sortUnstructured() 130 | 131 | if regionEnv.Cfg.DynamicParallel { 132 | // Create/update all Kubernetes objects at once. 133 | // TODO: Use dependency metadata to defer creation of dependent objects, then make Dynamic Parallel the default. 134 | regionEnv.Logger.Info("Provisioning Dynamic Objects in parallel") 135 | updater := func() <-chan *unstructured.Unstructured { 136 | var dynamicWg sync.WaitGroup 137 | dynamicWg.Add(len(regionEnv.Unstructured)) 138 | objectStream := make(chan *unstructured.Unstructured) 139 | for _, serviceObject := range regionEnv.Unstructured { 140 | go dynamicUpdate(&dynamicWg, serviceObject, regionEnv, objectStream) 141 | } 142 | go func() { 143 | dynamicWg.Wait() 144 | close(objectStream) 145 | }() 146 | return objectStream 147 | } 148 | 149 | updates := updater() 150 | 151 | for update := range updates { 152 | regionEnv.Logger.Debugf("Update of Dynamic Objects completed: %+v", update) 153 | } 154 | } else { 155 | // Create/update all Kubernetes objects sequentially, 156 | regionEnv.Logger.Info("Provisioning Dynamic Objects sequentially") 157 | for _, serviceObject := range regionEnv.Unstructured { 158 | objectStream := make(chan *unstructured.Unstructured) 159 | go dynamicUpdate(nil, serviceObject, regionEnv, objectStream) 160 | update := <-objectStream 161 | close(objectStream) 162 | regionEnv.Logger.Debugf("Update of Dynamic Object completed: %+v", update) 163 | } 164 | 165 | select { 166 | case <-regionEnv.Context.Done(): 167 | regionEnv.Logger.Infof("Received Done") 168 | return 169 | case serviceStream <- regionEnv: 170 | } 171 | } 172 | } 173 | 174 | func dynamicUpdate( 175 | wg *sync.WaitGroup, 176 | service *unstructured.Unstructured, 177 | regionEnv *RegionEnv, 178 | serviceStream chan *unstructured.Unstructured, 179 | ) { 180 | if wg != nil { 181 | defer wg.Done() 182 | } 183 | 184 | var updateErr, getErr error 185 | var update *unstructured.Unstructured 186 | 187 | gvk := service.GetObjectKind().GroupVersionKind() 188 | // Token Substitution 189 | regionEnv.replaceDynamicParameters(service) 190 | 191 | name := service.GetName() 192 | logger := log.WithFields(log.Fields{"Region": regionEnv.Region, "Name": name, "Kind": gvk.Kind}) 193 | 194 | // Find GVR 195 | mapping, mapperErr := regionEnv.Mapper.RESTMapping(gvk.GroupKind(), gvk.Version) 196 | if mapperErr != nil { 197 | regionEnv.errf("Unable to get REST Mapping for gvk: %s\n%s", gvk, mapperErr) 198 | serviceStream <- service 199 | return 200 | } 201 | 202 | dynamicInterface := regionEnv.DynamicClient.Resource(mapping.Resource).Namespace(regionEnv.Cfg.Namespace) 203 | 204 | previous, getErr := dynamicInterface.Get(regionEnv.Context, name, metav1.GetOptions{}) 205 | 206 | if service.GetKind() == "Job" && getErr == nil { 207 | logger.Infof("Found existing job %s in %s - trying to delete", name, regionEnv.Region) 208 | if deleteErr := regionEnv.deleteJob(previous, true); deleteErr != nil { 209 | regionEnv.errf("Unable to delete preexisting job %s: %w", name, deleteErr) 210 | serviceStream <- service 211 | return 212 | } 213 | logger.Info("Job deleted") 214 | time.Sleep(5 * time.Second) 215 | // Dummy error to have the subsequent logic create a new job. 216 | getErr = errors.NewNotFound(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, name) 217 | } 218 | 219 | if getErr == nil { 220 | data, _ := service.MarshalJSON() 221 | // Dynamic Client allows for Server Side Apply (SSA) patching. 222 | update, updateErr = dynamicInterface.Patch(regionEnv.Context, name, types.ApplyPatchType, data, 223 | metav1.PatchOptions{FieldManager: "porter2k8s"}) 224 | if errors.IsConflict(updateErr) { 225 | forceApply := true 226 | logger.Debugf("Conflict failure detected on %s %s. Retrying with force option", gvk.Kind, name) 227 | update, updateErr = dynamicInterface.Patch(regionEnv.Context, name, types.ApplyPatchType, data, 228 | metav1.PatchOptions{FieldManager: "porter2k8s", Force: &forceApply}) 229 | } 230 | } else if errors.IsNotFound(getErr) { 231 | update, updateErr = dynamicInterface.Create(regionEnv.Context, service, createOpts) 232 | } else { 233 | regionEnv.errf("Unable to get service %s", gvk.Kind, name, getErr) 234 | serviceStream <- service 235 | return 236 | } 237 | 238 | if updateErr != nil { 239 | regionEnv.errf("Unable to create/update service %s %s\n%s", gvk.Kind, name, updateErr) 240 | serviceStream <- service 241 | return 242 | } 243 | logger.Debugf("update: %+v", update) 244 | 245 | watchConfig := NewWatchConfig(service, previous, int32(regionEnv.Cfg.Wait)) 246 | watchConfig.Watch(regionEnv, dynamicInterface) 247 | 248 | regionEnv.createObjectSecrets(update, 0) 249 | 250 | if regionEnv.PodObject.GetKind() == "Job" { 251 | if deleteErr := regionEnv.deleteJob(update, false); deleteErr != nil { 252 | regionEnv.Errors = append(regionEnv.Errors, deleteErr) 253 | } 254 | } 255 | 256 | select { 257 | case <-regionEnv.Context.Done(): 258 | logger.Info("Received Done") 259 | return 260 | case serviceStream <- service: 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /pkg/porter2k8s/simpleconfig.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "fmt" 17 | "io" 18 | "os" 19 | "path/filepath" 20 | "regexp" 21 | "strings" 22 | 23 | log "github.com/sirupsen/logrus" 24 | "gopkg.in/yaml.v2" 25 | ) 26 | 27 | // SimpleEnv contains the env variables, and secrets for a specific region. 28 | type SimpleEnv struct { 29 | EnvConfigs []SimpleSecretYaml 30 | Errors []error 31 | } 32 | 33 | // SimpleEntry in a simple config secret yaml. 34 | type SimpleEntry struct { 35 | K8sObject K8sObjectRef `yaml:"k8s_object"` 36 | K8sSecret string `yaml:"k8s_secret"` 37 | Key string `yaml:"key"` 38 | Name string `yaml:"name"` 39 | Path string `yaml:"path"` 40 | Source SourceType `yaml:"source"` 41 | Value string `yaml:"value"` 42 | } 43 | 44 | // SimpleSecretYaml contains the format of a Simple secret yaml file. It is used for unmarshalling json. 45 | type SimpleSecretYaml struct { 46 | BasePath string `yaml:"base_path"` 47 | Includes []string `yaml:"includes"` 48 | Type EntryType `yaml:"type"` 49 | Vars []SimpleEntry `yaml:"vars"` 50 | } 51 | 52 | // EntryType determines if the variable is for the container or porter2k8s templating. 53 | type EntryType string 54 | 55 | // Possible EntryType values. 56 | const ( 57 | ContainerType EntryType = "container" 58 | PorterType EntryType = "porter2k8s" // Old name for porter2ks8 type. 59 | ) 60 | 61 | // SourceType determines the source of a secret. 62 | type SourceType string 63 | 64 | // Secrets can come from vault, K8s secrets such as those created by the ASO, or Kubernetes objects. 65 | const ( 66 | K8sObjectSource SourceType = "object" 67 | K8sSecretSource SourceType = "kubernetes" 68 | VaultSource SourceType = "vault" 69 | ) 70 | 71 | // K8sObjectRef specifies the field of a Kubernetes object to retrieve. 72 | type K8sObjectRef struct { 73 | Kind string `yaml:"kind"` 74 | Name string `yaml:"name"` 75 | } 76 | 77 | // getEnv implements configReader interface 78 | func (simpleEnv *SimpleEnv) getEnv(environment *RegionEnv) { 79 | // Construct paths to environment.yaml files 80 | // For example environment/dev/us-east-1/environment.yaml 81 | file := []string{fmt.Sprintf("%s/environment/%s/%s/environment.yaml", 82 | environment.Cfg.ConfigPath, 83 | environment.Cfg.Environment, 84 | environment.Region)} 85 | 86 | simpleEnv.getSimpleEnv(file, map[string]bool{}, 0) 87 | log.Debugf("%+v\n", simpleEnv) 88 | environment.Errors = simpleEnv.Errors 89 | } 90 | 91 | // parse reads the SimpleEnv and outputs EnvVar and SecretRef types. 92 | func (simpleEnv *SimpleEnv) parse(environment *RegionEnv) { 93 | // Segregate secret references from plain environment variables. 94 | for _, config := range simpleEnv.EnvConfigs { 95 | switch config.Type { 96 | case PorterType: 97 | for _, entry := range config.Vars { 98 | if entry.Value != "" { 99 | environment.ClusterSettings[entry.Name] = entry.Value 100 | } else { 101 | entry.processSecret(environment, config) 102 | } 103 | } 104 | case ContainerType: 105 | for _, entry := range config.Vars { 106 | //log.Debugf("Entry: %+v", entry) 107 | switch entry.Source { 108 | case VaultSource: 109 | // Environment variables have a value, secrets do not. 110 | if entry.Value != "" { 111 | newEnvVar := EnvVar{Name: entry.Name, Value: entry.Value} 112 | environment.Vars = append(environment.Vars, newEnvVar) 113 | } else { 114 | entry.processSecret(environment, config) 115 | } 116 | case K8sSecretSource: 117 | // Container secrets injected from Kubernetes Secrets (e.g. from OSB service bindings) 118 | newSecretRef := SecretKeyRef{Name: entry.Name, Key: entry.Key, Secret: entry.K8sSecret} 119 | environment.SecretKeyRefs = append(environment.SecretKeyRefs, newSecretRef) 120 | case K8sObjectSource: 121 | // Container secrets injected from Kubernetes objects (e.g. from ACK objects) 122 | //The name of this secret will be `- 0 && (simpleSecretConfig.Type == ContainerType || 198 | simpleSecretConfig.Type == PorterType) { 199 | // Env variables and secrets should override those in the referenced yaml files. 200 | // This allows setting a value for a specific cluster and allowing all other clusters to share defaults. 201 | // Remove entries that are overridden or add them to the override map. 202 | // DO NOT HAVE THE SAME ENTRY IN TWO SIMULTANEOUSLY REFERENCED FILES!!! 203 | for i := len(simpleSecretConfig.Vars) - 1; i >= 0; i-- { 204 | overrideEntry := fmt.Sprintf("%s_%s", simpleSecretConfig.Vars[i].Name, simpleSecretConfig.Type) 205 | if _, ok := overridden[overrideEntry]; ok { 206 | log.Infof("Env variable %s, overridden\n", simpleSecretConfig.Vars[i].Name) 207 | simpleSecretConfig.Vars = append(simpleSecretConfig.Vars[:i], simpleSecretConfig.Vars[i+1:]...) 208 | } else { 209 | overridden[overrideEntry] = true 210 | } 211 | } 212 | simpleEnv.EnvConfigs = append(simpleEnv.EnvConfigs, simpleSecretConfig) 213 | } 214 | if len(simpleSecretConfig.Includes) != 0 { 215 | // Construct referenced file paths. 216 | // Referenced files are only in the first yaml section of the file, the only section with "includes". 217 | for _, relativePath := range simpleSecretConfig.Includes { 218 | // Reference files use relative paths. Filepath knows how to handle them. 219 | referencedFiles = append(referencedFiles, filepath.Join(filepath.Dir(file), relativePath)) 220 | } 221 | } 222 | } 223 | 224 | // Merge simple secret configs referenced in the includes section into the main config for the region. 225 | // Recurse on referenced yaml files. 226 | if depth < maxDepth { 227 | depth++ 228 | if len(referencedFiles) > 0 { 229 | log.Infof("Recursing on files: %s\n", referencedFiles) 230 | } 231 | referencedSimpleEnv := &SimpleEnv{} 232 | referencedSimpleEnv.getSimpleEnv(referencedFiles, overridden, depth) 233 | simpleEnv.EnvConfigs = append(simpleEnv.EnvConfigs, referencedSimpleEnv.EnvConfigs...) 234 | simpleEnv.Errors = append(simpleEnv.Errors, referencedSimpleEnv.Errors...) 235 | log.Debugf("Combined configs: %+v\n", simpleEnv.EnvConfigs) 236 | } else { 237 | log.Errorf("Reached Maximum Recursion Depth: %d", maxDepth) 238 | } 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /pkg/porter2k8s/kubeobjects.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | package porter2k8s 13 | 14 | import ( 15 | "fmt" 16 | "os" 17 | "regexp" 18 | 19 | "git.corp.adobe.com/EchoSign/porter2k8s/pkg/vault" 20 | 21 | log "github.com/sirupsen/logrus" 22 | v1 "k8s.io/api/core/v1" 23 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // KubeObjects is the global configuration for the service. 28 | type KubeObjects struct { 29 | ClusterConfigs ClusterConfigs 30 | PodObject *unstructured.Unstructured 31 | Unstructured map[string][]*unstructured.Unstructured 32 | } 33 | 34 | // getKubeConfigs fetches Kube Configs from vault. 35 | func (kubeObjects *KubeObjects) getKubeConfigs(cfg *CmdConfig) error { 36 | clusterListFileName := fmt.Sprintf("%s/%s", cfg.ConfigPath, clusterConfigFile) 37 | if _, err := os.Stat(clusterListFileName); err == nil { 38 | log.Infof("Retrieving kubeconfig from Vault using %s", clusterListFileName) 39 | client, err := vault.NewVaultClient(cfg.VaultAddress, cfg.VaultNamespace, cfg.VaultToken) 40 | if err != nil { 41 | return fmt.Errorf("vault client initialization error %s\n%s", cfg.ConfigType, err) 42 | } 43 | err = kubeObjects.KubeConfigFromVault(client, clusterListFileName) 44 | if err != nil { 45 | return fmt.Errorf("unable to pull cluster configuration from Vault %s", err) 46 | } 47 | } 48 | return nil 49 | } 50 | 51 | // KubeConfigFromVault reads a cluster config file, pulls from Vault and instantiates k8s client cluster configurations. 52 | func (kubeObjects *KubeObjects) KubeConfigFromVault(vaultClient vault.VaultClientInterface, fileName string) error { 53 | clusters, err := readReferences(fileName) 54 | if err != nil { 55 | return err 56 | } 57 | err = clusters.fetchConfigs(vaultClient) 58 | if err != nil { 59 | return err 60 | } 61 | kubeObjects.ClusterConfigs = clusters 62 | return nil 63 | } 64 | 65 | // Read objects from yaml files and populate kubeObjects. 66 | func (kubeObjects *KubeObjects) processObjects(cfg *CmdConfig, possibleFiles []string) error { 67 | var podObjects []*unstructured.Unstructured 68 | // Determine if kubernetes yaml files are in the config directory or the k8s subdirectory 69 | objectDirectory := fmt.Sprintf("%s/k8s", cfg.ConfigPath) 70 | if _, err := os.Stat(objectDirectory); os.IsNotExist(err) { 71 | objectDirectory = cfg.ConfigPath 72 | } 73 | log.Infof("Using %s as directory for Kubernetes yaml objects.", objectDirectory) 74 | 75 | for _, fileType := range possibleFiles { 76 | var foundFileName string 77 | fileNameEnv := fmt.Sprintf("%s/%s-%s.yaml", objectDirectory, fileType, cfg.Environment) 78 | fileNameGeneral := fmt.Sprintf("%s/%s.yaml", objectDirectory, fileType) 79 | for _, fileName := range []string{fileNameEnv, fileNameGeneral} { 80 | if _, err := os.Stat(fileName); os.IsNotExist(err) { 81 | log.Infof("%s not found.", fileName) 82 | } else { 83 | log.Infof("Using %s for %s.", fileName, fileType) 84 | foundFileName = fileName 85 | break 86 | } 87 | } 88 | // If no file name was found, try the next possible name. 89 | if foundFileName == "" { 90 | continue 91 | } 92 | objects, readErr := multiDecoder(foundFileName) 93 | if readErr != nil { 94 | return readErr 95 | } 96 | for _, decoded := range objects { 97 | object := decoded.Object 98 | unstructuredObject, err := serviceFromObject(object, nil) 99 | if err != nil { 100 | return err 101 | } 102 | switch unstructuredObject.GetAPIVersion() { 103 | case "apps/v1", "batch/v1": 104 | podObjects = append(podObjects, unstructuredObject) 105 | case "v1": 106 | if unstructuredObject.GetKind() == "Pod" { 107 | log.Warnf("Pods are not supported outside of deployments, jobs, etc.. %s will not be deployed", 108 | unstructuredObject.GetName()) 109 | } else { 110 | kubeObjects.Unstructured["all"] = append(kubeObjects.Unstructured["all"], unstructuredObject) 111 | } 112 | case "autoscaling/v2beta2", "contour.heptio.com/v1beta1", "monitoring.coreos.com/v1", 113 | "networking.istio.io/v1beta1", "policy/v1beta1": 114 | kubeObjects.Unstructured["all"] = append(kubeObjects.Unstructured["all"], unstructuredObject) 115 | case "dynamodb.services.k8s.aws/v1alpha1", "elasticache.services.k8s.aws/v1alpha1": 116 | kubeObjects.Unstructured["aws"] = append(kubeObjects.Unstructured["aws"], unstructuredObject) 117 | case "azure.microsoft.com/v1alpha1", "azure.microsoft.com/v1alpha2", "azure.microsoft.com/v1beta1": 118 | kubeObjects.Unstructured["azure"] = append(kubeObjects.Unstructured["azure"], unstructuredObject) 119 | case "servicecatalog.k8s.io/v1beta1": 120 | if fileType == "servicecatalog-aws" { 121 | kubeObjects.Unstructured["aws"] = append(kubeObjects.Unstructured["aws"], unstructuredObject) 122 | } else if fileType == "servicecatalog-azure" { 123 | kubeObjects.Unstructured["azure"] = append(kubeObjects.Unstructured["aws"], unstructuredObject) 124 | } else { 125 | log.Warnf( 126 | "Excluding Service Instance found in %s, not an AWS or Azure specific servicecatalog file", 127 | foundFileName, 128 | ) 129 | } 130 | default: 131 | return fmt.Errorf( 132 | "Object type %s is unsupported, please upgrade to current version", 133 | decoded.GVK, 134 | ) 135 | } 136 | } 137 | } 138 | 139 | if len(podObjects) != 1 { 140 | return fmt.Errorf("One and only one pod containing object is allowed. Found %d", len(podObjects)) 141 | } 142 | kubeObjects.PodObject = podObjects[0] 143 | 144 | return nil 145 | } 146 | 147 | // validate ensure that all kubernetes resources contain the service name. 148 | // This prevents different microservices from colliding. 149 | // It also adds the name to the tags of each resource. 150 | func (kubeObjects *KubeObjects) validateAndTag() error { 151 | 152 | // Additional Deployment checks are handled in prepareDeployment. 153 | // Order of preference: pod object annotation, name of service object, name of pod object. 154 | var serviceName string 155 | services := kubeObjects.findUnstructured("v1", "Service", "all") 156 | podObjectName := kubeObjects.PodObject.GetName() 157 | podObjectAnnotations := kubeObjects.PodObject.GetAnnotations() 158 | log.Infof("Annotations %+v", podObjectAnnotations) 159 | serviceNameOverride, ok := podObjectAnnotations["porter2k8s/service-name"] 160 | if ok { 161 | serviceName = serviceNameOverride 162 | } else if len(services) > 0 { 163 | serviceName = services[0].GetName() 164 | } else { 165 | serviceName = podObjectName 166 | } 167 | log.Infof("serviceName %s", serviceName) 168 | 169 | for _, cloud := range []string{"aws", "azure", "all"} { 170 | for _, object := range kubeObjects.Unstructured[cloud] { 171 | err := validateUnstructured(object, serviceName, podObjectName) 172 | if err != nil { 173 | return err 174 | } 175 | } 176 | } 177 | 178 | return nil 179 | } 180 | 181 | // findUnstructured returns all unstructured objects of matching an API Version and Kind, or just the APIVersion if 182 | // the Kind is empty. 183 | // key parameter is "all", "aws", or "azure" 184 | func (kubeObjects *KubeObjects) findUnstructured(apiVersion, kind string, key string) []*unstructured.Unstructured { 185 | var output []*unstructured.Unstructured 186 | for _, service := range kubeObjects.Unstructured[key] { 187 | if kind == "" { 188 | if service.GetAPIVersion() == apiVersion { 189 | log.Infof("Found %s", apiVersion) 190 | output = append(output, service) 191 | } 192 | } else { 193 | if service.GetAPIVersion() == apiVersion && service.GetKind() == kind { 194 | log.Infof("Found %s %s", apiVersion, kind) 195 | output = append(output, service) 196 | } 197 | } 198 | } 199 | if len(output) == 0 { 200 | log.Infof("Found no %s %s", apiVersion, kind) 201 | } 202 | 203 | return output 204 | } 205 | 206 | // prepareDeployment takes the deployment from the yaml file and applies image sha, labels, and configmap and secret 207 | // references. 208 | func (kubeObjects *KubeObjects) prepareDeployment(sha string) error { 209 | 210 | // Add sha to image, configMapRef, and secretRef 211 | // Find container in deployment that matches the deployment name 212 | applicationContainer, typedObject := findContainer(kubeObjects.PodObject) 213 | if applicationContainer == nil { 214 | return fmt.Errorf("unable to find application image in deployment spec") 215 | } 216 | 217 | // Remove docker tag, if it exists. 218 | // The image in the deployment.yaml should not have a tag. 219 | shaReplacement := fmt.Sprintf("$1:%s", sha) 220 | // Replace everything after the colon, if it exists. See tests for examples. 221 | regex := regexp.MustCompile(`(.*?)(:|\z).*`) 222 | applicationContainer.Image = regex.ReplaceAllString(applicationContainer.Image, shaReplacement) 223 | 224 | refName := fmt.Sprintf("%s-%s", kubeObjects.PodObject.GetName(), sha) 225 | // Stub out references to configmap and secret ref to be filled out for each region later. 226 | envSourceConfigMap := v1.EnvFromSource{ 227 | ConfigMapRef: &v1.ConfigMapEnvSource{ 228 | v1.LocalObjectReference{ 229 | Name: refName, 230 | }, 231 | boolRef(false), 232 | }, 233 | } 234 | envSourceSecret := v1.EnvFromSource{ 235 | SecretRef: &v1.SecretEnvSource{ 236 | v1.LocalObjectReference{ 237 | Name: refName, 238 | }, 239 | // Secrets may not be defined. 240 | boolRef(true), 241 | }, 242 | } 243 | applicationContainer.EnvFrom = append(applicationContainer.EnvFrom, envSourceConfigMap, envSourceSecret) 244 | 245 | // Add sha label to deployment and pod. 246 | appendLabel(typedObject, "sha", sha) 247 | appendLabel(findPodTemplateSpec(typedObject), "sha", sha) 248 | 249 | // Service Side Apply Bug fixed in 1.20. Need to add protocol to ports in the meantime. 250 | // https://github.com/kubernetes-sigs/structured-merge-diff/issues/130 251 | for i, port := range applicationContainer.Ports { 252 | if port.Protocol == "" { 253 | applicationContainer.Ports[i].Protocol = v1.ProtocolTCP 254 | } 255 | } 256 | 257 | // Back to unstructured 258 | newUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(typedObject) 259 | if err != nil { 260 | return err 261 | } 262 | kubeObjects.PodObject.Object = newUnstructured 263 | 264 | // Delete replicas if HPA exists. 265 | hpas := kubeObjects.findUnstructured("autoscaling/v2beta2", "HorizontalPodAutoscaler", "all") 266 | if len(hpas) > 0 { 267 | unstructured.RemoveNestedField(kubeObjects.PodObject.Object, "spec", "replicas") 268 | } 269 | 270 | return nil 271 | } 272 | -------------------------------------------------------------------------------- /pkg/porter2k8s/helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "bytes" 17 | "fmt" 18 | "io" 19 | "io/ioutil" 20 | "os" 21 | "regexp" 22 | "strconv" 23 | "strings" 24 | "text/template" 25 | 26 | "github.com/Masterminds/sprig" 27 | "github.com/google/go-cmp/cmp" 28 | log "github.com/sirupsen/logrus" 29 | v1 "k8s.io/api/core/v1" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 32 | "k8s.io/apimachinery/pkg/runtime" 33 | runtimeyaml "k8s.io/apimachinery/pkg/runtime/serializer/yaml" 34 | "k8s.io/apimachinery/pkg/util/yaml" 35 | ) 36 | 37 | // Decoded Kubernetes Object 38 | type Decoded struct { 39 | Object runtime.Object 40 | GVK string 41 | } 42 | 43 | // BoolRef returns a reference to the boolean parameter. 44 | func boolRef(b bool) *bool { 45 | return &b 46 | } 47 | 48 | // Int64Ref returns a reference to the int parameter. 49 | func int64Ref(i interface{}) *int64 { 50 | var i64 int64 51 | switch asserted := i.(type) { 52 | case int: 53 | i64 = int64(asserted) 54 | case int32: 55 | i64 = int64(asserted) 56 | case string: 57 | integer, err := strconv.Atoi(asserted) 58 | if err != nil { 59 | log.Errorf("Unable to convert %v to int64 reference\n", i) 60 | } else { 61 | i64 = int64(integer) 62 | } 63 | default: 64 | log.Errorf("Unable to convert %v to int64 reference. Argument must be string or int.\n", i) 65 | } 66 | return &i64 67 | } 68 | 69 | // Int32Ref returns a reference to the int parameter. 70 | func int32Ref(i interface{}) *int32 { 71 | var i32 int32 72 | switch asserted := i.(type) { 73 | case int: 74 | i32 = int32(asserted) 75 | case int64: 76 | i32 = int32(asserted) 77 | case string: 78 | integer, err := strconv.Atoi(asserted) 79 | if err != nil { 80 | log.Errorf("Unable to convert %v to int32 reference\n", i) 81 | } else { 82 | i32 = int32(integer) 83 | } 84 | default: 85 | log.Errorf("Unable to convert %v to int32 reference. Argument must be string or int.\n", i) 86 | } 87 | return &i32 88 | } 89 | 90 | // SetFromEnvStr sets parameter from environment, if available. 91 | func setFromEnvStr(envVariable string, defaultValue string) string { 92 | if value := os.Getenv(envVariable); value != "" { 93 | return value 94 | } 95 | return defaultValue 96 | } 97 | 98 | // SetFromEnvStr sets parameter from environment, if available. 99 | func setFromEnvBool(envVariable string) bool { 100 | if value := os.Getenv(envVariable); value != "" { 101 | return true 102 | } 103 | return false 104 | } 105 | 106 | // SetFromEnvInt sets parameter from environment, if available. 107 | func setFromEnvInt(envVariable string, defaultValue int) int { 108 | if value := os.Getenv(envVariable); value != "" { 109 | ret, err := strconv.Atoi(value) 110 | if err == nil { 111 | return ret 112 | } 113 | log.Errorf("Unexpected %s value: %s, using %d\n", envVariable, value, defaultValue) 114 | } 115 | return defaultValue 116 | } 117 | 118 | func setIfZeroInt32(value, defaultValue int32) int32 { 119 | if value == 0 { 120 | return defaultValue 121 | } 122 | return value 123 | } 124 | 125 | // tokenReplace replaces a []byte template with values from a dict. 126 | func tokenReplace(format []byte, args map[string]string, logger *log.Entry) []byte { 127 | var message bytes.Buffer 128 | strFormat := string(format) 129 | // Sprig functions may create maps or lists. If we don't remove the single quotes those will just be strings. 130 | regex := regexp.MustCompile(`'(.*{{.*}}.*)'`) 131 | strFormat = regex.ReplaceAllString(strFormat, "$1") 132 | 133 | tmpl, err := template.New("tokens").Funcs(sprig.TxtFuncMap()).Parse(strFormat) 134 | 135 | if err != nil { 136 | logger.Errorf("Error creating sprig template for replacing tokens: %s", err) 137 | return format 138 | } 139 | // Make the formatter enter a zero value if the token is not found. 140 | tmpl.Option("missingkey=zero") 141 | 142 | err = tmpl.Execute(&message, args) 143 | if err != nil { 144 | logger.Warnf("Token missing or is not applicable for this namespace, cloud, or region: %s", err) 145 | return format 146 | } 147 | return message.Bytes() 148 | } 149 | 150 | // multiDecoder reads multiple Kubernetes objects from the same file. 151 | func multiDecoder(filename string) (objects []Decoded, err error) { 152 | data, readErr := ioutil.ReadFile(filename) 153 | if readErr != nil { 154 | return objects, readErr 155 | } 156 | 157 | r := ioutil.NopCloser(bytes.NewReader(data)) 158 | decoder := yaml.NewDocumentDecoder(r) 159 | unstructuredDecoder := runtimeyaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) 160 | var chunk []byte 161 | for { 162 | chunk = make([]byte, len(data)) 163 | _, err = decoder.Read(chunk) 164 | if err != nil { 165 | break 166 | } 167 | chunk = bytes.Trim(chunk, "\x00") 168 | 169 | obj, gvk, err := unstructuredDecoder.Decode(chunk, nil, nil) 170 | if err != nil { 171 | log.Errorf("Unstructured Decoding Error: %v", err) 172 | return nil, err 173 | } 174 | log.Infof("gvk: %+v", gvk) 175 | objects = append(objects, Decoded{Object: obj, GVK: gvk.String()}) 176 | } 177 | // No need to pass EOF error to caller. 178 | if err == io.EOF { 179 | err = nil 180 | } 181 | return objects, err 182 | } 183 | 184 | // UnstructuredHeap is a heap in which each node is the oldest configmap in its subtree. 185 | type UnstructuredHeap []unstructured.Unstructured 186 | 187 | func (h UnstructuredHeap) Len() int { return len(h) } 188 | func (h UnstructuredHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } 189 | 190 | // Less will return the older timestamp. This way the oldest configmap is popped. 191 | func (h UnstructuredHeap) Less(i, j int) bool { 192 | iTimestamp := h[i].GetCreationTimestamp() 193 | jTimestamp := h[j].GetCreationTimestamp() 194 | return iTimestamp.Before(&jTimestamp) 195 | } 196 | 197 | // Push and Pop use pointer receivers because they modify the slice's length, 198 | // not just its contents. 199 | func (h *UnstructuredHeap) Push(x interface{}) { 200 | *h = append(*h, x.(unstructured.Unstructured)) 201 | } 202 | 203 | // Pop pops oldest configmap from the tree. 204 | func (h *UnstructuredHeap) Pop() interface{} { 205 | old := *h 206 | n := len(old) 207 | x := old[n-1] 208 | *h = old[0 : n-1] 209 | return x 210 | } 211 | 212 | // The TestEnv config implements the configReader interface. 213 | type TestEnv struct { 214 | Errors []error 215 | Vars []EnvVar 216 | } 217 | 218 | func (env *TestEnv) getEnv(environment *RegionEnv) { 219 | env.Vars = append(env.Vars, EnvVar{Name: "Region", Value: environment.Region}) 220 | return 221 | } 222 | 223 | func (env *TestEnv) parse(environment *RegionEnv) { 224 | environment.Vars = env.Vars 225 | return 226 | } 227 | 228 | // add missing elements of the second slice to the first. 229 | func addReferences(a, b []v1.ObjectReference) []v1.ObjectReference { 230 | result := a 231 | for _, reference := range b { 232 | if !containsReference(result, reference) { 233 | result = append(result, reference) 234 | } 235 | } 236 | return result 237 | } 238 | 239 | // Determine if ObjectReference exists in list of Object References. 240 | func containsReference(list []v1.ObjectReference, element v1.ObjectReference) bool { 241 | for _, reference := range list { 242 | if cmp.Equal(reference, element) { 243 | return true 244 | } 245 | } 246 | return false 247 | } 248 | 249 | // validateUnstructured ensures that all kubernetes resources contain the service name. 250 | // This prevents different microservices from colliding. 251 | // It also adds the name to the tags of each resource. 252 | func validateUnstructured(object *unstructured.Unstructured, serviceName, deploymentName string) error { 253 | // Specific Object validation. 254 | if object.GetKind() == "HorizontalPodAutoscaler" { 255 | targetName, _, _ := unstructured.NestedString(object.Object, "spec", "scaleTargetRef", "name") 256 | if targetName != deploymentName { 257 | return fmt.Errorf("HPAutoscaler references %s, not the deployment %s", targetName, deploymentName) 258 | } 259 | } 260 | 261 | // Add App Label to K8s objects. 262 | appendLabel(object, "app", serviceName) 263 | 264 | // Verify name matches service name. 265 | name := object.GetName() 266 | 267 | // Ensure the logging sidecar's ConfigMap is associated with the service before name validation 268 | if name == "fluent-bit-sidecar-config" { 269 | name = fmt.Sprintf("%s-fluent-bit-config", serviceName) 270 | } 271 | if !strings.Contains(name, serviceName) { 272 | return fmt.Errorf("object name '%s' does not contain service name, %s", name, serviceName) 273 | } 274 | log.Debugf("validated object: %+v", object) 275 | return nil 276 | } 277 | 278 | // serviceFromObject converts a runtime object to an unstructured 279 | func serviceFromObject(object runtime.Object, into *unstructured.Unstructured) (*unstructured.Unstructured, error) { 280 | var err error 281 | if into == nil { 282 | into = &unstructured.Unstructured{} 283 | } 284 | into.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(object) 285 | if err != nil { 286 | return into, err 287 | } 288 | return into, err 289 | } 290 | 291 | func appendLabel(kubeObject metav1.Object, key string, val string) { 292 | if kubeObject.GetLabels() == nil { 293 | kubeObject.SetLabels(map[string]string{key: val}) 294 | } else { 295 | labels := kubeObject.GetLabels() 296 | labels[key] = val 297 | kubeObject.SetLabels(labels) 298 | } 299 | } 300 | 301 | // Takes a domain string (e.g., "-dev.ethoscluster.ethos.adobe.net") 302 | // and injects a subdomain at a given domain index in the string 303 | // insertDomainPart("-dev.ethoscluster.ethos.adobe.net, 1, "corp") 304 | // returns "-dev.corp.ethoscluster.ethos.adobe.net" 305 | func insertDomainPart(domain string, at int, value string) string { 306 | domainSlice := strings.Split(domain, ".") 307 | // increase the slice capacity by 1 308 | domainSlice = append(domainSlice, "") 309 | // copy the contents of the slice at the given index one position to the right 310 | copy(domainSlice[at+1:], domainSlice[at:]) 311 | // set the value at the desired index 312 | domainSlice[at] = value 313 | newDomain := strings.Join(domainSlice, ".") 314 | return newDomain 315 | } 316 | 317 | func lessObjectReference(x, y v1.ObjectReference) bool { 318 | return x.Name > y.Name 319 | } 320 | 321 | // removeInvalidCharacters strips characters which cannot be used in a secret key. 322 | func removeInvalidCharacters(input string) string { 323 | regex := regexp.MustCompile(`[^a-zA-Z0-9.]*`) 324 | return regex.ReplaceAllString(input, "") 325 | } 326 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # porter2k8s 2 | Deploy projects on Kubernetes. 3 | 4 | ## Introduction 5 | Porter2k8s parses a configuration and deploys to kubernetes. 6 | 7 | ## Goals 8 | 1. Versatility 9 | 1. Environment variables and secrets are tied to a deployment. 10 | 2. Multi-cluster rollouts. 11 | 3. No intermediary secret storage. This reduces the operational overhead, but requires deployments to take place within Adobenet. Therefore they cannot run in Lambda. 12 | 4. Capitalize on the advantages of Kubernetes. 13 | 1. Deployments will only take a few minutes, rather than hours. 14 | 2. All microservices in the cluster are assumed to be part of the same application, reducing the need for increased security and compliance controls. 15 | 3. Allow developers to define their services and take advantage of all that Kubernetes can do. 16 | 5. As Vanilla as possible 17 | 1. A single set of YAML files should: 18 | 1. Cover all environments from local to production. 19 | 2. Be compatible with the `kubectl apply`. 20 | 21 | ## Tools 22 | 1. Kubernetes 23 | 2. Vault (Thycotic has no REST API) 24 | 25 | ## Method 26 | 1. Gather environment variables and secret references for a specific environment (ie dev, stage, prod). 27 | 2. Fetch secrets from Vault. 28 | 3. Create deployment specific configmap and secret object. Both are tagged with the deployment sha and name specified in the deployment.yaml. 29 | 4. Read kubernetes objects from files within the repository. 30 | 6. Modify the objects with cluster specific settings. 31 | 1. Update non-deployment objects if required. 32 | 1. Apply the kubernetes deployment, statefulset, or job with the following amendments. 33 | 1. Reference the previously created configmap and secret. 34 | 2. Set the sha passed to the command as the docker image tag. 35 | 6. Monitor the status of the objects as they are created. 36 | 37 | ## Assumptions 38 | 1. The region name, for example "us-east-1" or "va7", matches the following: 39 | 1. A context in the kubeconfig, or in `cluster-config.yaml` (see below). If cluster-config.yaml does not exist, the kubeconfig defaults to `~/.kube/config` but can be set by the KUBECONFIG env variable. 40 | 2. A directory within the `operation_data/` directory, which contains a `secret.yaml` file. 41 | 2. Kubernetes objects are kept in files within one or more of the following file names: service.yaml, deployment.yaml, istio.yaml, ingress.yaml, servicecatalog-aws, servicecatalog-azure, ethos-logging-sidecar. With the exception of the service catalog objects, The objects within these files do not need to correlate to the actual objects within them. Additionally, environment specific files will take precedence. For example if `deployment-dev.yaml` exists when deploying to the `dev` environment, then `deployment.yaml` will be ignored. These files may exist in $CONFIG_PATH or $CONFIG_PATH/k8s directory. Files in $CONFIG_PATH will be ignored if $CONFIG_PATH/k8s exists. 42 | 3. The docker image specified in the deployment.yaml: 43 | 1. Matches the "name" specified in the deployment metadata. 44 | 2. Has been built and exists within its repository with the sha passed to Porter2k8s in the command line argument or the SHA env variable. 45 | 46 | ## Kubeconfig 47 | Porter2k8s can deploy using multiple kubeconfig files stored in Vault. This feature is enabled if `cluster-config.yaml` is found in the root of the $CONFIG_PATH. 48 | 1. The file `cluster-config.yaml` is of the following form, with region name matching a directory within the porter configuration (see assumptions): 49 | ``` 50 | clusters: 51 | - path: "path/to/kubeconfig/in/vault" 52 | context: "region name" 53 | ``` 54 | 2. In Vault, the key must be `config` and the value the base64 encoded kubeconfig file. For example: 55 | ``` 56 | vault write secret/k8s/test/us-east-1/kubeconfig-201809261820 config=@config-ue1_encoded 57 | ``` 58 | 59 | ## Cluster specific settings 60 | In order to have service.yaml, ingress.yaml (or istio.yaml), and deployment.yaml work for all environments and be `kubectl apply` compatible, porter28ks will automatically substitute certain values. These substitutions will be based on a configmap named porter2k8s stored in the target kubernetes cluster in the same namespace as the service. The following key names are supported. 61 | * DOMAIN: All domain names in the ingress.yaml file will be changed to this domain. This may need to be refined further if multiple domains need to be supported. 62 | * REGISTRY: The registry of the image in the deployment.yaml will be changed to this value. 63 | * ISTIO: (true|false) Indicates whether istio objects should be created rather than ingress. 64 | * GATEWAYS: Name of Istio Gateway that should be used by the Virtual Service. Multiple space delimited gateways are allowed. 65 | * HPAMIN: Minimum value for horizontal pod autoscalers minimum allowed replicas. 66 | * HPAMINMAX: Maximum value for horizontal pod autoscalers minimum allowed replicas. 67 | * SIDECAR: Whole pods can be specified within the "SIDECAR" cluster setting. The pod will be injected into the pod containing object (ex Statefulset, Deployment, Job). 68 | * CLOUD: (aws|azure) Identifies the cloud provider. This is only required for Open Service Broker service instances. 69 | * CLOUD_LOCATION: The cloud provider region. This can be required for Open Service Broker service instances. 70 | 71 | In addition to specific keys, porter2k8s will replace Go template variables in the deployment spec annotations with values from the porter2k8s configmap. Sprig functions are also supported. http://masterminds.github.io/sprig/ 72 | ``` 73 | arn:aws:iam::{{.account}}:role/myapp-{{.environment}}-{{.region}} -> 74 | arn:aws:iam::9999999999:role/myapp-prod-us-east-2 75 | 76 | {{ .support_subnets | splitList \",\" }} -> 77 | [subnet-1932 subnet-9876] 78 | ``` 79 | 80 | ## Annotations 81 | 82 | ### porter2k8s/service-name 83 | Proteus2k8s is designed for multiple separate microservices to deploy to the same namespace. One of the perils of this model is microservices may overwrite the K8s objects of others. To help prevent that, porter2k8s will attempt to identify the name of the microservice and ensure all objects names contain that name. This microservice name can be specified explicitly with the annotation `porter2k8s/service-name` on the pod containing object (deployment, job, statefulset, etc). If that annotation is not present, it will use the name of the service object. If there is no service object, it will use the name of the pod containing object. 84 | 85 | ## Environment Files 86 | 87 | Environment files specify secrets and environment values. There are two categories of entries _container_ and _porter2k8s_ 88 | 89 | ### Container Entries 90 | Secrets and environment variables that should be injected into pods. 91 | #### Environment Variables 92 | Environment variables are be specified directly. 93 | ``` 94 | --- 95 | base_path: /test_base 96 | type: container 97 | vars: 98 | 99 | - name: DESIRED_ENV_VARIABLE 100 | value: value of DESIRED_ENV_VARIABLE 101 | ``` 102 | #### Secrets 103 | Secrets can have a number of sources. 104 | * Vault secrets are specified with a path and a key. The path is the concatenation of the *vault_path* arg, the *base_path* specified for the yaml document within the environment file, and the *path* specified for the entry. 105 | ``` 106 | --- 107 | base_path: /vault_base_path 108 | type: container 109 | vars: 110 | - path: vault_path 111 | name: VAULT_SECRET 112 | key: vault_key 113 | ``` 114 | 115 | * Values from Kubernetes secrets can also be requested. The name of the secret must contain the _service name_ as defined in the Annotations section. The *base_path* is ignored. 116 | ``` 117 | --- 118 | base_path: /vault_base_path 119 | type: container 120 | vars: 121 | - source: kubernetes 122 | name: FROM_K8S 123 | k8s_secret: k8s-secret 124 | key: k8s_key 125 | ``` 126 | * Values from Kubernetes objects may be requested directly. Proteus2k8s will find the value at the specified object path, create a secret with that value, and mount the secret within the pod object. The path form conforms to jq (https://stedolan.github.io/jq/manual/). The object must be created by porter2k8s directly. Once again, the *base_path* is ignored. 127 | ``` 128 | --- 129 | base_path: /vault_base_path 130 | type: container 131 | vars: 132 | - source: object 133 | name: FROM_K8S_OBJECT 134 | path: .status.nodeGroups[0].primaryEndpoint.address 135 | k8s_object: 136 | name: my_elasticache_redis 137 | kind: ReplicationGroup 138 | ``` 139 | 140 | ## Usage 141 | ``` 142 | $ porter2k8s --help 143 | Usage of : 144 | -config-path string 145 | Configuration root directory. Should include the '.porter' or 'environment' directory. Kubernetes object yaml files may be in the directory or in a subdirectory named 'k8s'. (default "/repo") 146 | -config-type string 147 | Configuration type, simple or porter. (default "porter") 148 | -environment string 149 | Environment of deployment. 150 | -log-dir string 151 | Directory to write pod logs into. (must already exist) (default "logs") 152 | -log-mode string 153 | Pod log streaming mode. One of 'inline' (print to porter2k8s log), 'single' (single region to stdout), 'file' (write to filesystem, see log-dir option), 'none' (disable log streaming) (default "inline") 154 | -max-cm int 155 | Maximum number of configmaps and secret objects to keep per app. (default 5) 156 | -namespace string 157 | Kubernetes namespace. (default "default") 158 | -regions string 159 | Regions of deployment. 160 | -sha string 161 | Deployment sha. 162 | -v Verbose log output. 163 | -vault-addr string 164 | Vault server. (default "https://vault.loc.adobe.net") 165 | -vault-path string 166 | Path in Vault. (default "/") 167 | -vault-token string 168 | Vault token. 169 | -wait int 170 | Extra time to wait for deployment to complete in seconds. (default 180) 171 | ``` 172 | 173 | ## Examples 174 | ``` 175 | $ porter2k8s --sha f69a8e348ca46e738c00027aba814feb56195819 --regions "va7 us-east-1" --environment dev --config-path $(pwd) --vault-path secret/ethos/tenants/cloudtech_doc_cloud 176 | ``` 177 | or 178 | ``` 179 | $ docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e VAULT_TOKEN -e VAULT_ADDR -e REGIONS -e ENVIRONMENT -e sha -e VAULT_PATH -v $HOME/.kube/:/root/.kube:z -v $(pwd)/:/repo:z docker-dc-micro-release.dr.corp.adobe.com/porter2k8s:1.0 180 | ``` 181 | 182 | ## Example File structure 183 | ``` 184 | k8s 185 | ├── deployment.yaml 186 | ├── ingress.yaml 187 | ├── service.yaml 188 | ├── istio.yaml 189 | ├── ingress-prod.yaml 190 | |── deployment-dev.yaml 191 | └── servicecatalog-azure.yaml 192 | environment 193 | ├── aws-environment.yaml 194 | ├── azure-environment.yaml 195 | ├── base-environment.yaml 196 | ├── dev 197 | │   ├── environment.yaml 198 | │   ├── eu-central-1 199 | │   │   └── environment.yaml 200 | │   ├── nld2 201 | │   │   └── environment.yaml 202 | │   └── us-east-1 203 | │   └── environment.yaml 204 | ├── prod 205 | │   ├── ap-northeast-1 206 | │   │   └── environment.yaml 207 | │   ├── ap-south-1 208 | │   │   └── environment.yaml 209 | │   ├── ap-southeast-2 210 | │   │   └── environment.yaml 211 | │   ├── environment.yaml 212 | │   ├── eu-central-1 213 | │   │   └── environment.yaml 214 | │   ├── eu-west-1 215 | │   │   └── environment.yaml 216 | │   ├── nld2 217 | │   │   └── environment.yaml 218 | │   ├── us-east-1 219 | │   │   └── environment.yaml 220 | │   ├── us-west-2 221 | │   │   └── environment.yaml 222 | │   └── va7 223 | │   └── environment.yaml 224 | └── stage 225 | ├── ap-northeast-1 226 | │   └── environment.yaml 227 | ├── ap-south-1 228 | │   └── environment.yaml 229 | ├── ap-southeast-2 230 | │   └── environment.yaml 231 | ├── environment.yaml 232 | ├── eu-central-1 233 | │   └── environment.yaml 234 | ├── eu-west-1 235 | │   └── environment.yaml 236 | ├── nld2 237 | │   └── environment.yaml 238 | ├── us-east-1 239 | │   └── environment.yaml 240 | ├── us-west-2 241 | │   └── environment.yaml 242 | └── va7 243 | └── environment.yaml 244 | ``` 245 | -------------------------------------------------------------------------------- /pkg/porter2k8s/helper_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "bytes" 17 | "fmt" 18 | "os" 19 | "reflect" 20 | "testing" 21 | 22 | "github.com/google/go-cmp/cmp" 23 | "github.com/google/go-cmp/cmp/cmpopts" 24 | v1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 26 | ) 27 | 28 | var serviceRunning = unstructured.Unstructured{ 29 | Object: map[string]interface{}{ 30 | "apiVersion": "v1", 31 | "kind": "Service", 32 | "metadata": map[string]interface{}{ 33 | "name": "running", 34 | "namespace": "test", 35 | "creationTimestamp": "2004-01-01T00:00:00Z", 36 | "labels": map[string]interface{}{ 37 | "app": "testapp", 38 | }, 39 | "resourceVersion": "612488", 40 | "selfLink": "/api/v1/namespaces/test/services/testapp", 41 | "uid": "871f9449-4f4e-11e9-a995-d2a4ff4cc534", 42 | }, 43 | "spec": map[string]interface{}{ 44 | "ports": []interface{}{ 45 | map[string]interface{}{ 46 | "name": "http", 47 | "protocol": "TCP", 48 | "targetPort": "8080", 49 | }, 50 | }, 51 | "selector": map[string]interface{}{ 52 | "app": "testapp", 53 | }, 54 | "sessionAffinity": "None", 55 | "type": "ClusterIP", 56 | "clusterIP": "127.0.0.1", 57 | }, 58 | }, 59 | } 60 | 61 | var serviceUpdateNeededValue = unstructured.Unstructured{ 62 | Object: map[string]interface{}{ 63 | "apiVersion": "v1", 64 | "kind": "Service", 65 | "metadata": map[string]interface{}{ 66 | "name": "no update", 67 | "namespace": "test", 68 | "labels": map[string]interface{}{ 69 | "app": "testapp", 70 | }, 71 | }, 72 | "spec": map[string]interface{}{ 73 | "ports": []interface{}{ 74 | map[string]interface{}{ 75 | "name": "http", 76 | "protocol": "TCP", 77 | "targetPort": 8083, 78 | }, 79 | }, 80 | "selector": map[string]interface{}{ 81 | "app": "testapp", 82 | }, 83 | "sessionAffinity": "None", 84 | "type": "ClusterIP", 85 | }, 86 | }, 87 | } 88 | 89 | var serviceUpdateNeededList = unstructured.Unstructured{ 90 | Object: map[string]interface{}{ 91 | "apiVersion": "v1", 92 | "kind": "Service", 93 | "metadata": map[string]interface{}{ 94 | "name": "needed list", 95 | "namespace": "test", 96 | "labels": map[string]interface{}{ 97 | "app": "testapp", 98 | }, 99 | }, 100 | "spec": map[string]interface{}{ 101 | "ports": []interface{}{ 102 | map[string]interface{}{ 103 | "name": "http", 104 | "protocol": "TCP", 105 | "targetPort": 8080, 106 | }, 107 | map[string]interface{}{ 108 | "name": "https", 109 | "protocol": "TCP", 110 | "targetPort": 8083, 111 | }, 112 | }, 113 | "selector": map[string]interface{}{ 114 | "app": "testapp", 115 | }, 116 | "sessionAffinity": "None", 117 | "type": "ClusterIP", 118 | }, 119 | }, 120 | } 121 | 122 | var serviceUpdateNeededMap = unstructured.Unstructured{ 123 | Object: map[string]interface{}{ 124 | "apiVersion": "v1", 125 | "kind": "Service", 126 | "metadata": map[string]interface{}{ 127 | "name": "needed map", 128 | "namespace": "test", 129 | "labels": map[string]interface{}{ 130 | "app": "testapp", 131 | }, 132 | }, 133 | "spec": map[string]interface{}{ 134 | "ports": []interface{}{ 135 | map[string]interface{}{ 136 | "name": "http", 137 | "protocol": "TCP", 138 | "targetPort": 8080, 139 | }, 140 | }, 141 | "selector": map[string]interface{}{ 142 | "app": "testapp", 143 | "new": "value", 144 | }, 145 | "sessionAffinity": "None", 146 | "type": "ClusterIP", 147 | }, 148 | }, 149 | } 150 | 151 | var serviceUpdateNotNeeded = unstructured.Unstructured{ 152 | Object: map[string]interface{}{ 153 | "apiVersion": "v1", 154 | "kind": "Service", 155 | "metadata": map[string]interface{}{ 156 | "name": "not needed", 157 | "namespace": "test", 158 | "labels": map[string]interface{}{ 159 | "app": "testapp", 160 | }, 161 | }, 162 | "spec": map[string]interface{}{ 163 | "ports": []interface{}{ 164 | map[string]interface{}{ 165 | "name": "http", 166 | "protocol": "TCP", 167 | "targetPort": 8080, 168 | }, 169 | }, 170 | "selector": map[string]interface{}{ 171 | "app": "testapp", 172 | "new": "value", 173 | }, 174 | "sessionAffinity": "None", 175 | "type": "ClusterIP", 176 | }, 177 | }, 178 | } 179 | 180 | var objectReferenceA = v1.ObjectReference{ 181 | Name: "snip", 182 | } 183 | var objectReferenceB = v1.ObjectReference{ 184 | Name: "snap", 185 | } 186 | var objectReferenceC = v1.ObjectReference{ 187 | Name: "snurr", 188 | } 189 | 190 | func TestMultiDecoder(t *testing.T) { 191 | testCases := []struct { 192 | filename string 193 | objectNumber int 194 | }{ 195 | {"../../test/istio.yaml", 2}, 196 | {"../../test/secret.yaml", 1}, 197 | } 198 | for _, tc := range testCases { 199 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 200 | pwd, _ := os.Getwd() 201 | filePath := fmt.Sprintf("%s/%s", pwd, tc.filename) 202 | objects, err := multiDecoder(filePath) 203 | t.Logf("objects %s\nerr %s\n", objects, err) 204 | numObjects := len(objects) 205 | if tc.objectNumber != numObjects { 206 | t.Errorf("Incorrect number of objects %d, expected %d\n", numObjects, tc.objectNumber) 207 | } 208 | }) 209 | } 210 | } 211 | 212 | func TestInt64Ref(t *testing.T) { 213 | testCases := []struct { 214 | input interface{} 215 | expected int64 216 | }{ 217 | {"2", int64(2)}, 218 | {2, int64(2)}, 219 | {int32(2), int64(2)}, 220 | {"a", int64(0)}, 221 | {false, int64(0)}, 222 | } 223 | for _, tc := range testCases { 224 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 225 | result := int64Ref(tc.input) 226 | if *result != tc.expected { 227 | t.Errorf("Unexpected result from input %d, expect %d \n", *result, tc.expected) 228 | } 229 | }) 230 | } 231 | } 232 | 233 | func TestInt32Ref(t *testing.T) { 234 | testCases := []struct { 235 | input interface{} 236 | expected int32 237 | }{ 238 | {"2", int32(2)}, 239 | {2, int32(2)}, 240 | {int64(2), int32(2)}, 241 | {"a", int32(0)}, 242 | {false, int32(0)}, 243 | } 244 | for _, tc := range testCases { 245 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 246 | result := int32Ref(tc.input) 247 | if *result != tc.expected { 248 | t.Errorf("Unexpected result from input %d, expect %d \n", *result, tc.expected) 249 | } 250 | }) 251 | } 252 | } 253 | 254 | func TestAppendLabel(t *testing.T) { 255 | var obj v1.PodTemplateSpec 256 | appendLabel(&obj, "testKey", "testVal") 257 | 258 | expected := map[string]string{"testKey": "testVal"} 259 | if !reflect.DeepEqual(obj.ObjectMeta.Labels, expected) { 260 | t.Fatalf("Labels map not initialized %v, expect %v\n", obj.ObjectMeta.Labels, expected) 261 | } 262 | 263 | appendLabel(&obj, "secondKey", "secondVal") 264 | expected2 := map[string]string{"testKey": "testVal", "secondKey": "secondVal"} 265 | if !reflect.DeepEqual(obj.ObjectMeta.Labels, expected2) { 266 | t.Fatalf("Labels map not initialized %v, expect %v\n", obj.ObjectMeta.Labels, expected2) 267 | } 268 | } 269 | 270 | func TestTokenReplace(t *testing.T) { 271 | testCases := []struct { 272 | format []byte // byte slice with tokens to be replaced. 273 | tokens map[string]string 274 | expected []byte 275 | }{ 276 | { 277 | []byte("arn:aws:iam::{{.account}}:role/mobilesigcapture-{{.environment}}-{{.region}}"), 278 | map[string]string{"account": "983073263818", "environment": "preview", "region": "us-west-2"}, 279 | []byte("arn:aws:iam::983073263818:role/mobilesigcapture-preview-us-west-2"), 280 | }, // Standard use. 281 | { 282 | []byte("{{.editor}} is better than {{.os}}."), 283 | map[string]string{"editor": "Vim", "os": "emacs"}, 284 | []byte("Vim is better than emacs."), 285 | }, // Random use. 286 | { 287 | []byte("arn:aws:iam::{{.account}}:role/mobilesigcapture-{{.environment}}-{{.region}}"), 288 | map[string]string{"account": "983073263818", "region": "us-west-2"}, 289 | []byte("arn:aws:iam::983073263818:role/mobilesigcapture--us-west-2"), 290 | }, // Missing environment. No longer returns original. 291 | // Missing keys must be treated as zero values for default function to work. 292 | { 293 | []byte("{{ .support_subnets | splitList \",\" }}"), 294 | map[string]string{"account": "983073263818", "support_subnets": "subnet-1932,subnet-9876"}, 295 | []byte("[subnet-1932 subnet-9876]"), 296 | }, // Sprig Function 297 | { 298 | []byte("\"{{.RDS_REGION}}-catmanagement\""), 299 | map[string]string{"account": "983073263818", "environment": "preview", "RDS_REGION": "us-west-2"}, 300 | []byte("\"us-west-2-catmanagement\""), 301 | }, // Standard use. 302 | } 303 | for _, tc := range testCases { 304 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 305 | result := tokenReplace(tc.format, tc.tokens, logger) 306 | if !bytes.Equal(result, tc.expected) { 307 | t.Errorf("Unexpected result: %s\n", result) 308 | } 309 | }) 310 | } 311 | } 312 | 313 | func TestContainsReference(t *testing.T) { 314 | testCases := []struct { 315 | reference v1.ObjectReference 316 | list []v1.ObjectReference 317 | expected bool 318 | }{ 319 | { 320 | objectReferenceA, 321 | []v1.ObjectReference{objectReferenceA, objectReferenceB, objectReferenceC}, 322 | true, 323 | }, 324 | { 325 | objectReferenceA, 326 | []v1.ObjectReference{objectReferenceB, objectReferenceC}, 327 | false, 328 | }, 329 | } 330 | for _, tc := range testCases { 331 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 332 | result := containsReference(tc.list, tc.reference) 333 | if result != tc.expected { 334 | t.Errorf("Unexpected result for %s in %s\n", tc.reference, tc.list) 335 | } 336 | }) 337 | } 338 | } 339 | 340 | func TestAddReferences(t *testing.T) { 341 | testCases := []struct { 342 | a []v1.ObjectReference 343 | b []v1.ObjectReference 344 | expected []v1.ObjectReference 345 | }{ 346 | { 347 | []v1.ObjectReference{objectReferenceB, objectReferenceC}, 348 | []v1.ObjectReference{objectReferenceA, objectReferenceB, objectReferenceC}, 349 | []v1.ObjectReference{objectReferenceA, objectReferenceB, objectReferenceC}, 350 | }, 351 | { 352 | []v1.ObjectReference{}, 353 | []v1.ObjectReference{objectReferenceA, objectReferenceB, objectReferenceC}, 354 | []v1.ObjectReference{objectReferenceA, objectReferenceB, objectReferenceC}, 355 | }, 356 | { 357 | []v1.ObjectReference{objectReferenceC}, 358 | []v1.ObjectReference{objectReferenceA, objectReferenceB}, 359 | []v1.ObjectReference{objectReferenceA, objectReferenceB, objectReferenceC}, 360 | }, 361 | } 362 | for _, tc := range testCases { 363 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 364 | result := addReferences(tc.a, tc.b) 365 | if !cmp.Equal(result, tc.expected, cmpopts.SortSlices(lessObjectReference)) { 366 | t.Errorf("Unexpected result %v expected %v\n", result, tc.expected) 367 | } 368 | }) 369 | } 370 | } 371 | 372 | func (expected EnvVar) equal(actual EnvVar) bool { 373 | if actual.Name != expected.Name || 374 | actual.Value != expected.Value { 375 | return false 376 | } 377 | return true 378 | } 379 | 380 | func (expected SecretRef) equal(actual SecretRef) bool { 381 | if actual.Name != expected.Name || 382 | actual.Key != expected.Key || 383 | actual.Path != expected.Path || 384 | actual.Value != expected.Value { 385 | return false 386 | } 387 | return true 388 | } 389 | 390 | func (expected SecretKeyRef) equal(actual SecretKeyRef) bool { 391 | if actual.Name != expected.Name || 392 | actual.Key != expected.Key || 393 | actual.Secret != expected.Secret { 394 | return false 395 | } 396 | return true 397 | } 398 | -------------------------------------------------------------------------------- /pkg/porter2k8s/dynamic_watch.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "strings" 17 | 18 | log "github.com/sirupsen/logrus" 19 | "k8s.io/apimachinery/pkg/api/errors" 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 22 | "k8s.io/apimachinery/pkg/fields" 23 | "k8s.io/apimachinery/pkg/watch" 24 | "k8s.io/client-go/dynamic" 25 | ) 26 | 27 | // WatchConfig is for customized watches of unstructured objects. 28 | type WatchConfig struct { 29 | Failure WatchMatch 30 | Kind string 31 | Name string 32 | Service *unstructured.Unstructured 33 | Success WatchMatch 34 | TimeoutSeconds *int64 35 | } 36 | 37 | // WatchMatch are combinations of conditions for watch success or failure. 38 | type WatchMatch struct { 39 | Conditions []WatchCondition 40 | Logic ConditionLogic 41 | } 42 | 43 | // WatchCondition are conditions for watch success or failure. 44 | type WatchCondition struct { 45 | Fields []string // Passed to unstructured helpers. 46 | StateBool bool 47 | StateInt64 int64 48 | StateString string 49 | StateMap map[string]interface{} 50 | Type ConditionType 51 | } 52 | 53 | // ConditionType type of comparison for WatchCondition 54 | type ConditionType int 55 | 56 | // Further comparison types can be added. 57 | // Anything in k8s.io/apimachinery@v0.17.2/pkg/apis/meta/v1/unstructured/helpers.go could be supported. 58 | const ( 59 | BoolCondition ConditionType = 0 60 | Int64Condition ConditionType = 1 61 | StringCondition ConditionType = 2 62 | StringContainsCondition ConditionType = 3 63 | StringSliceCondition ConditionType = 4 64 | MapCondition ConditionType = 5 65 | SliceMapCondition ConditionType = 6 // Slice of Maps 66 | ) 67 | 68 | // ConditionLogic type of comparison for WatchCondition 69 | type ConditionLogic int 70 | 71 | // Logical operators for combinations of Watch Conditions 72 | // Further logical operators types can be added. 73 | const ( 74 | AndLogic ConditionLogic = 0 75 | OrLogic ConditionLogic = 1 76 | ) 77 | 78 | // NewWatchConfig creates a WatchConfig on a service. Pod objects can wait an additional configurable period. 79 | // TODO: Check for configuration error and log it. 80 | func NewWatchConfig(service *unstructured.Unstructured, 81 | previous *unstructured.Unstructured, 82 | podWait int32) WatchConfig { 83 | watchConfig := WatchConfig{ 84 | Failure: WatchMatch{Conditions: []WatchCondition{}, Logic: OrLogic}, 85 | Kind: service.GetKind(), 86 | Name: service.GetName(), 87 | Service: service, 88 | Success: WatchMatch{Conditions: []WatchCondition{}, Logic: AndLogic}, 89 | TimeoutSeconds: int64Ref(0), // Timeout value of zero will indicate that no wait should be set on the object. 90 | } 91 | switch service.GetAPIVersion() { 92 | case "apps/v1": 93 | watchConfig.TimeoutSeconds = calculateWatchTimeout(service, podWait) 94 | switch watchConfig.Kind { 95 | case "Deployment": 96 | watchConfig.Success.Conditions = []WatchCondition{ 97 | WatchCondition{Fields: []string{"status", "conditions"}, 98 | StateMap: map[string]interface{}{"reason": "NewReplicaSetAvailable", "status": "True", "type": "Progressing"}, 99 | Type: SliceMapCondition}, 100 | } 101 | watchConfig.Failure.Conditions = []WatchCondition{ 102 | WatchCondition{Fields: []string{"status", "conditions"}, 103 | StateMap: map[string]interface{}{"reason": "ProgressDeadlineExceeded", "status": "False", "type": "Progressing"}, 104 | Type: SliceMapCondition}, 105 | } 106 | case "StatefulSet": 107 | if previous == nil { 108 | previous = service 109 | } 110 | currentReplicas, found, err := unstructured.NestedInt64(previous.Object, "spec", "replicas") 111 | if err != nil || !found { 112 | log.Warnf("Current replica count not found for statefulSet %s", previous.GetName()) 113 | currentReplicas = 1 114 | } 115 | log.Infof("Will wait for %d replicas to become ready", currentReplicas) 116 | watchConfig.Success.Conditions = []WatchCondition{ 117 | WatchCondition{Fields: []string{"status", "readyReplicas"}, StateInt64: currentReplicas, 118 | Type: Int64Condition}, 119 | WatchCondition{Fields: []string{"status", "updatedReplicas"}, StateInt64: currentReplicas, 120 | Type: Int64Condition}, 121 | WatchCondition{Fields: []string{"status", "unavailableReplicas"}, StateInt64: 0, 122 | Type: Int64Condition}, 123 | } 124 | } 125 | case "batch/v1": 126 | // This covers both deployments and stateful set. 127 | watchConfig.TimeoutSeconds = int64Ref(podWait) 128 | watchConfig.Success.Conditions = []WatchCondition{ 129 | WatchCondition{Fields: []string{"status", "succeeded"}, StateInt64: 1, 130 | Type: Int64Condition}, 131 | } 132 | case "dynamodb.services.k8s.aws/v1alpha1": 133 | switch watchConfig.Kind { 134 | case "Table": 135 | watchConfig.Success.Conditions = []WatchCondition{ 136 | WatchCondition{Fields: []string{"status", "tableStatus"}, StateString: "ACTIVE", Type: StringCondition}, 137 | } 138 | watchConfig.Failure.Conditions = []WatchCondition{ 139 | //TODO: Find better failure condition. 140 | WatchCondition{Fields: []string{"status", "tableStatus"}, StateString: "DELETING", 141 | Type: StringCondition}, 142 | } 143 | watchConfig.TimeoutSeconds = int64Ref(10 * 60) 144 | } 145 | case "elasticache.services.k8s.aws/v1alpha1": 146 | switch watchConfig.Kind { 147 | //CacheSubnetGroup has no conditions on successful provisioning. 148 | case "ReplicationGroup": 149 | watchConfig.Success.Conditions = []WatchCondition{ 150 | WatchCondition{Fields: []string{"status", "conditions"}, 151 | StateMap: map[string]interface{}{"type": "ACK.ResourceSynced", "status": "True"}, 152 | Type: SliceMapCondition}, 153 | WatchCondition{Fields: []string{"status", "status"}, StateString: "available", Type: StringCondition}, 154 | } 155 | watchConfig.TimeoutSeconds = int64Ref(2 * 60) 156 | } 157 | case "azure.microsoft.com/v1alpha1", "azure.microsoft.com/v1alpha2", "azure.microsoft.com/v1beta1": 158 | // ASO provides standard status messages, at least as of 4/21. 159 | watchConfig.Success.Conditions = []WatchCondition{ 160 | WatchCondition{Fields: []string{"status", "provisioned"}, StateBool: true, Type: BoolCondition}, 161 | } 162 | watchConfig.Failure.Conditions = []WatchCondition{ 163 | WatchCondition{Fields: []string{"status", "failedProvisioning"}, StateBool: true, Type: BoolCondition}, 164 | WatchCondition{Fields: []string{"status", "message"}, StateString: "Failure responding to request", 165 | Type: StringContainsCondition}, 166 | } 167 | switch watchConfig.Kind { 168 | case "MySQLServer": 169 | watchConfig.TimeoutSeconds = int64Ref(30 * 60) 170 | case "CosmosDB": 171 | watchConfig.TimeoutSeconds = int64Ref(15 * 60) 172 | case "RedisCache": 173 | watchConfig.TimeoutSeconds = int64Ref(15 * 60) 174 | default: 175 | watchConfig.TimeoutSeconds = int64Ref(2 * 60) 176 | } 177 | case "servicecatalog.k8s.io/v1beta1": 178 | watchConfig.Success.Conditions = []WatchCondition{ 179 | WatchCondition{Fields: []string{"status", "condition"}, StateString: "Ready", Type: StringSliceCondition}, 180 | } 181 | switch watchConfig.Kind { 182 | case "ServiceInstance": 183 | watchConfig.TimeoutSeconds = int64Ref(50 * 60) 184 | watchConfig.Failure.Conditions = []WatchCondition{ 185 | WatchCondition{Fields: []string{"status", "conditions"}, 186 | StateMap: map[string]interface{}{"type": "Failed", "status": "True"}, Type: SliceMapCondition}, 187 | WatchCondition{Fields: []string{"status", "conditions"}, 188 | StateMap: map[string]interface{}{"type": "OrphanMigration", "status": "True"}, Type: SliceMapCondition}, 189 | } 190 | watchConfig.Success.Conditions = []WatchCondition{ 191 | WatchCondition{Fields: []string{"status", "conditions"}, 192 | StateMap: map[string]interface{}{"type": "Ready", "status": "True"}, Type: SliceMapCondition}, 193 | } 194 | case "ServiceBinding": 195 | watchConfig.TimeoutSeconds = int64Ref(30) 196 | watchConfig.Failure.Conditions = []WatchCondition{ 197 | WatchCondition{Fields: []string{"status", "conditions"}, 198 | StateMap: map[string]interface{}{"type": "Failed", "status": "True"}, Type: SliceMapCondition}, 199 | } 200 | watchConfig.Success.Conditions = []WatchCondition{ 201 | WatchCondition{Fields: []string{"status", "conditions"}, 202 | StateMap: map[string]interface{}{"type": "Ready", "status": "True"}, Type: SliceMapCondition}, 203 | } 204 | } 205 | } 206 | return watchConfig 207 | } 208 | 209 | // Watch watches the unstructured objects. 210 | func (watchConfig *WatchConfig) Watch(regionEnv *RegionEnv, dynamicInterface dynamic.ResourceInterface) { 211 | logger := log.WithFields(log.Fields{"Region": regionEnv.Region, "Name": watchConfig.Name, "Kind": watchConfig.Kind}) 212 | if *watchConfig.TimeoutSeconds == 0 { 213 | logger.Infof("No watch for objects of type %s/%s, %s", watchConfig.Service.GetAPIVersion(), watchConfig.Kind, 214 | watchConfig.Name) 215 | return 216 | } 217 | listOptions := metav1.ListOptions{ 218 | FieldSelector: fields.OneTermEqualSelector("metadata.name", watchConfig.Name).String(), 219 | TimeoutSeconds: watchConfig.TimeoutSeconds, 220 | } 221 | logger.Infof("Waiting %d seconds for %s to be ready", *watchConfig.TimeoutSeconds, watchConfig.Name) 222 | 223 | watcher, watchErr := dynamicInterface.Watch(regionEnv.Context, listOptions) 224 | if watchErr != nil { 225 | regionEnv.errf("%s", watchErr) 226 | return 227 | } 228 | watchCh := watcher.ResultChan() 229 | for { 230 | select { 231 | case <-regionEnv.Context.Done(): 232 | logger.Info("Received Done") 233 | // Returning true to prevent an error from being reported. 234 | // If SIGTERM was sent, the user doesn't care about the serviceInstance status. 235 | return 236 | case event, ok := <-watchCh: 237 | if !ok { 238 | regionEnv.errf("timeout watching job") 239 | return 240 | } 241 | switch event.Type { 242 | case watch.Added, watch.Modified: 243 | update := event.Object.(*unstructured.Unstructured) 244 | logger.Debugf("Update: %+v", update) 245 | 246 | status, found, err := unstructured.NestedMap(update.Object, "status") 247 | if !found { 248 | log.Debugf("Unable to get service status, waiting for next event: error %s", err) 249 | continue 250 | } 251 | logger.Debugf("Status: %s", status) 252 | failed := watchConfig.Failure.check(update) 253 | if failed { 254 | regionEnv.errf("unable to create/update service %s %s\n%s", watchConfig.Kind, 255 | watchConfig.Name, status) 256 | return 257 | } 258 | succeeded := watchConfig.Success.check(update) 259 | if succeeded { 260 | logger.Infof("Dynamic object %s %s successfully provisioned", watchConfig.Kind, 261 | watchConfig.Name) 262 | return 263 | } 264 | 265 | case watch.Deleted: 266 | regionEnv.errf("service deleted while watching for completion") 267 | return 268 | 269 | case watch.Error: 270 | err := errors.FromObject(event.Object) 271 | regionEnv.errf("error reported while watching service: %s", err) 272 | return 273 | } 274 | } 275 | } 276 | } 277 | 278 | // check calls watchCondition's check and applies logic. 279 | func (match *WatchMatch) check(update *unstructured.Unstructured) bool { 280 | matches := 0 281 | for _, condition := range match.Conditions { 282 | if condition.check(update) { 283 | matches++ 284 | } 285 | } 286 | if match.Logic == OrLogic && matches > 0 { 287 | return true 288 | } 289 | if match.Logic == AndLogic && matches == len(match.Conditions) { 290 | return true 291 | } 292 | return false 293 | } 294 | 295 | // check returns true if the condition is met. 296 | func (condition *WatchCondition) check(update *unstructured.Unstructured) bool { 297 | switch condition.Type { 298 | case BoolCondition: 299 | status, _, _ := unstructured.NestedBool(update.Object, condition.Fields...) 300 | if status == condition.StateBool { 301 | return true 302 | } 303 | case Int64Condition: 304 | status, _, _ := unstructured.NestedInt64(update.Object, condition.Fields...) 305 | if status == condition.StateInt64 { 306 | return true 307 | } 308 | case StringCondition: 309 | status, _, _ := unstructured.NestedString(update.Object, condition.Fields...) 310 | if status == condition.StateString { 311 | return true 312 | } 313 | case StringContainsCondition: 314 | status, _, _ := unstructured.NestedString(update.Object, condition.Fields...) 315 | if strings.Contains(status, condition.StateString) { 316 | return true 317 | } 318 | case StringSliceCondition: 319 | status, _, _ := unstructured.NestedStringSlice(update.Object, condition.Fields...) 320 | for _, message := range status { 321 | if message == condition.StateString { 322 | return true 323 | } 324 | } 325 | case MapCondition: 326 | status, _, _ := unstructured.NestedMap(update.Object, condition.Fields...) 327 | return compareStatusMaps(status, condition.StateMap) 328 | case SliceMapCondition: 329 | statuses, _, _ := unstructured.NestedSlice(update.Object, condition.Fields...) 330 | for _, status := range statuses { 331 | statusMap := status.(map[string]interface{}) 332 | if compareStatusMaps(statusMap, condition.StateMap) { 333 | return true 334 | } 335 | } 336 | return false 337 | } 338 | return false 339 | } 340 | 341 | func compareStatusMaps(status, conditionMap map[string]interface{}) bool { 342 | for key, value := range conditionMap { 343 | if value != status[key] { 344 | return false 345 | } 346 | } 347 | return true 348 | } 349 | -------------------------------------------------------------------------------- /pkg/porter2k8s/regionenv_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | package porter2k8s 14 | 15 | import ( 16 | "context" 17 | "encoding/base64" 18 | "fmt" 19 | "strings" 20 | "testing" 21 | 22 | "github.com/google/go-cmp/cmp" 23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/runtime/schema" 27 | fakedynamic "k8s.io/client-go/dynamic/fake" 28 | "k8s.io/client-go/restmapper" 29 | ) 30 | 31 | var resourceGroup = unstructured.Unstructured{ 32 | Object: map[string]interface{}{ 33 | "apiVersion": "azure.microsoft.com/v1alpha1", 34 | "kind": "ResourceGroup", 35 | "metadata": map[string]interface{}{ 36 | "name": "resource group", 37 | "namespace": "test", 38 | }, 39 | "spec": map[string]interface{}{ 40 | "location": "{{.region}}", 41 | }, 42 | }, 43 | } 44 | 45 | var resourceGroup2 = unstructured.Unstructured{ 46 | Object: map[string]interface{}{ 47 | "apiVersion": "azure.microsoft.com/v1alpha1", 48 | "kind": "ResourceGroup", 49 | "metadata": map[string]interface{}{ 50 | "name": "resource group", 51 | "namespace": "test", 52 | }, 53 | "spec": map[string]interface{}{ 54 | "parameters": map[string]interface{}{ 55 | "location": "{{.region}}-catmanagement", 56 | }, 57 | "otherstuff": map[string]interface{}{ 58 | "cloud": "*-nimbus", 59 | }, 60 | }, 61 | }, 62 | } 63 | 64 | var cacheSubnetGroup = unstructured.Unstructured{ 65 | Object: map[string]interface{}{ 66 | "apiVersion": "dynamodb.services.k8s.aws/v1alpha1", 67 | "kind": "CacheSubnetGroup", 68 | "metadata": map[string]interface{}{ 69 | "name": "cacheSubnetGroup", 70 | "namespace": "test", 71 | }, 72 | "spec": map[string]interface{}{ 73 | "cacheSubnetGroupDescription": "Test Subnet Group", 74 | "cacheSubnetGroupName": "test", 75 | "subnetIDs": `{{ splitList "," .support_subnets | toJson }}`, 76 | }, 77 | }, 78 | } 79 | 80 | var redis = unstructured.Unstructured{ 81 | Object: map[string]interface{}{ 82 | "apiVersion": "azure.microsoft.com/v1alpha1", 83 | "kind": "RedisCache", 84 | "metadata": map[string]interface{}{ 85 | "name": "redis", 86 | "namespace": "test", 87 | }, 88 | "spec": map[string]interface{}{ 89 | "location": "{{.region}}", 90 | "resourceGroup": "resource group", 91 | "properties": map[string]interface{}{ 92 | "sku": map[string]interface{}{ 93 | "name": "Cloud", 94 | "family": "F", 95 | "capacity": "300", 96 | }, 97 | "enableNonSslPort": "false", 98 | "subnetID": "/subscriptions/{{.ACCOUNT}}/resourceGroups/{{.BASE_RESOURCE_GROUP}}/providers" + 99 | "/Microsoft.Network/virtualNetworks/{{.VNET}}/subnets/MicroSubnet", 100 | }, 101 | }, 102 | }, 103 | } 104 | 105 | var cosmosdb = unstructured.Unstructured{ 106 | Object: map[string]interface{}{ 107 | "apiVersion": "azure.microsoft.com/v1alpha1", 108 | "kind": "CosmosDB", 109 | "metadata": map[string]interface{}{ 110 | "name": "cosmosdb", 111 | "namespace": "test", 112 | }, 113 | "spec": map[string]interface{}{ 114 | "kind": "GlobalDocumentDB", 115 | "location": "{{.region}}", 116 | "resourceGroup": "resource group", 117 | "ipRules": []interface{}{ 118 | "104.42.195.92", 119 | "40.76.54.131", 120 | "52.176.6.30", 121 | "52.169.50.45", 122 | "52.187.184.26", 123 | }, 124 | "properties": map[string]interface{}{ 125 | "capabilities": []interface{}{ 126 | map[string]interface{}{ 127 | "name": "EnableTable", 128 | }, 129 | }, 130 | "databaseAccountOfferType": "Standard", 131 | "isVirtualNetworkFilterEnabled": true, 132 | }, 133 | "virtualNetworkRules": []interface{}{ 134 | map[string]interface{}{ 135 | "subnetID": "/subscriptions/{{.MS_ACCOUNT}}/resourceGroups/{{.MS_BASE_RESOURCE_GROUP}}/providers" + 136 | "/Microsoft.Network/virtualNetworks/{{.MS_VNET}}/subnets/cosmosdb", 137 | }, 138 | }, 139 | }, 140 | }, 141 | } 142 | 143 | var serviceAccount = unstructured.Unstructured{ 144 | Object: map[string]interface{}{ 145 | "apiVersion": "v1", 146 | "kind": "ServiceAccount", 147 | "metadata": map[string]interface{}{ 148 | "labels": map[string]interface{}{"app": "A"}, 149 | "name": "A", 150 | "annotations": map[string]interface{}{ 151 | "eks.amazonaws.com/role-arn": "arn:aws:iam::{{.account}}:role/test-{{.environment}}-{{.region}}", 152 | }, 153 | }, 154 | "secrets": []interface{}{"test-token-abdce"}, 155 | }, 156 | } 157 | 158 | func TestReplaceDynamicParameters(t *testing.T) { 159 | testCases := []struct { 160 | object *unstructured.Unstructured 161 | tokens map[string]string 162 | path []string 163 | subpath []string 164 | expected string 165 | }{ 166 | { 167 | &resourceGroup, 168 | map[string]string{"region": "europe"}, 169 | []string{"spec", "location"}, 170 | []string{}, 171 | "europe", 172 | }, 173 | { 174 | resourceGroup2.DeepCopy(), 175 | map[string]string{"region": "europe"}, 176 | []string{"spec", "parameters", "location"}, 177 | []string{}, 178 | "europe-catmanagement", 179 | }, 180 | { 181 | resourceGroup2.DeepCopy(), 182 | map[string]string{"region": "europe"}, 183 | []string{"spec", "otherstuff", "cloud"}, 184 | []string{}, 185 | "*-nimbus", 186 | }, 187 | { 188 | &redis, 189 | map[string]string{"region": "asia", "ACCOUNT": "0123456789", "BASE_RESOURCE_GROUP": "base_rg", 190 | "VNET": "my_vnet"}, 191 | []string{"spec", "properties", "subnetID"}, 192 | []string{}, 193 | "/subscriptions/0123456789/resourceGroups/base_rg/providers/Microsoft.Network/virtualNetworks/my_vnet/" + 194 | "subnets/MicroSubnet", 195 | }, 196 | { 197 | &cosmosdb, 198 | map[string]string{"region": "asia", "MS_ACCOUNT": "0123456789", "MS_BASE_RESOURCE_GROUP": "base_rg", 199 | "MS_VNET": "my_vnet"}, 200 | []string{"spec", "virtualNetworkRules"}, 201 | []string{"subnetID"}, 202 | "/subscriptions/0123456789/resourceGroups/base_rg/providers/Microsoft.Network/virtualNetworks/my_vnet/" + 203 | "subnets/cosmosdb", 204 | }, 205 | { 206 | &cacheSubnetGroup, 207 | map[string]string{"support_subnets": "subnet-1234,subnet-24500"}, 208 | []string{"spec", "subnetIDs"}, 209 | []string{}, 210 | "subnet-1234", 211 | }, 212 | { 213 | &serviceAccount, 214 | map[string]string{"account": "0123456789", "environment": "dev", "region": "east"}, 215 | []string{"metadata", "annotations", "eks.amazonaws.com/role-arn"}, 216 | []string{}, 217 | "arn:aws:iam::0123456789:role/test-dev-east", 218 | }, 219 | } 220 | for _, tc := range testCases { 221 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 222 | regionEnv := RegionEnv{ 223 | ClusterSettings: tc.tokens, 224 | Logger: logger, 225 | } 226 | regionEnv.replaceDynamicParameters(tc.object) 227 | result := findUnstructuredString(tc.object.Object, tc.path, tc.subpath) 228 | if result != tc.expected { 229 | t.Errorf("Unexpected result %s, expected %s\n", result, tc.expected) 230 | } 231 | }) 232 | } 233 | } 234 | 235 | func TestSortUnstructured(t *testing.T) { 236 | testCases := []struct { 237 | objs []string 238 | istio string 239 | expected []string 240 | }{ 241 | { 242 | []string{"v1/Service", "networking.istio.io/v1beta1/VirtualService"}, 243 | "true", 244 | []string{"v1/Service", "networking.istio.io/v1beta1/VirtualService"}, 245 | }, 246 | { 247 | []string{"v1/Service", "networking.istio.io/v1beta1/VirtualService"}, 248 | "false", 249 | []string{"v1/Service"}, 250 | }, 251 | } 252 | for _, tc := range testCases { 253 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 254 | clusterSettings := map[string]string{"ISTIO": tc.istio} 255 | regionEnv := RegionEnv{ 256 | ClusterSettings: clusterSettings, 257 | Unstructured: stringToUnstructured(tc.objs), 258 | } 259 | regionEnv.sortUnstructured() 260 | result := unstructuredToString(regionEnv.Unstructured) 261 | if !cmp.Equal(result, tc.expected) { 262 | t.Errorf("Unexpected result %v expected %v\n", result, tc.expected) 263 | } 264 | }) 265 | } 266 | } 267 | 268 | func TestAddSidecarContainer(t *testing.T) { 269 | testCases := []struct { 270 | clusterSettings map[string]string 271 | tailBufLimit string 272 | sourcetype string 273 | }{ 274 | { 275 | map[string]string{}, 276 | "145MB", 277 | "dc-k8s-asr", 278 | }, // No template values. 279 | { 280 | map[string]string{"LOGGING_TAIL_BUF_LIMIT": "100MB", "LOGGING_SOURCETYPE": "stage-source"}, 281 | "100MB", 282 | "stage-source", 283 | }, // Template values to validate token replacement. 284 | } 285 | // Read "test/sidcar.yaml" into the "SIDECAR" key. 286 | decoded, _ := multiDecoder("../../test/sidecar.yaml") 287 | porter2k8sCM, _ := serviceFromObject(decoded[0].Object, nil) 288 | porter2k8sData, _, _ := unstructured.NestedStringMap(porter2k8sCM.Object, "data") 289 | for _, tc := range testCases { 290 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 291 | tc.clusterSettings["SIDECAR"] = porter2k8sData["SIDECAR"] 292 | regionEnv := RegionEnv{ 293 | ClusterSettings: tc.clusterSettings, 294 | Logger: logger, 295 | PodObject: deployment.DeepCopy(), 296 | } 297 | regionEnv.addSidecarContainer() 298 | if len(regionEnv.Errors) > 0 { 299 | t.Error(regionEnv.Errors[0]) 300 | return 301 | } 302 | regionEnv.replaceDynamicParameters(regionEnv.PodObject) 303 | if len(regionEnv.Errors) > 0 { 304 | t.Error(regionEnv.Errors[0]) 305 | return 306 | } 307 | typedPodTemplate, _ := findPodTemplateSpecUnstructured(regionEnv.PodObject) 308 | // Ensure there are 2 containers and 3 volumes 309 | containers := typedPodTemplate.Spec.Containers 310 | if len(containers) != 2 { 311 | t.Errorf("Unexpected number of containers %d, expected 2", len(containers)) 312 | } 313 | // Logging container is the 2nd one. 314 | for _, envVar := range containers[1].Env { 315 | if envVar.Name == "TAIL_BUF_LIMIT" && envVar.Value != tc.tailBufLimit { 316 | t.Errorf("Unexpected value of TAIL_BUF_LIMIT, result %s expected %s", envVar.Value, tc.tailBufLimit) 317 | } else if envVar.Name == "SPLUNK_SOURCETYPE" && envVar.Value != tc.sourcetype { 318 | t.Errorf("Unexpected value of SPLUNK_SOURCETYPE, result %s expected %s", envVar.Value, tc.sourcetype) 319 | } 320 | } 321 | volumes := typedPodTemplate.Spec.Volumes 322 | if len(volumes) != 3 { 323 | t.Errorf("Unexpected number of volumes %d, expected 3", len(containers)) 324 | } 325 | }) 326 | } 327 | } 328 | 329 | func TestCreateObjectSecrets(t *testing.T) { 330 | testCases := []struct { 331 | paths []string 332 | keys []string 333 | objectMap map[string]interface{} 334 | attempt int 335 | expected []string 336 | expectErr bool 337 | }{ 338 | { // Basic case. 339 | []string{".status.url"}, 340 | []string{".status.url"}, 341 | map[string]interface{}{ 342 | "url": "www.example.com", 343 | }, 344 | 0, 345 | []string{"www.example.com"}, 346 | false, 347 | }, 348 | { // Multiple paths. 349 | []string{".status.url", ".status.key"}, 350 | []string{".status.url", ".status.key"}, 351 | map[string]interface{}{ 352 | "url": "www.example.com", 353 | "key": "1234", 354 | }, 355 | 0, 356 | []string{"www.example.com", "1234"}, 357 | false, 358 | }, 359 | { // path with slices 360 | []string{".status.nodeGroups[0].nodeGroupMembers[0].readEndpoint.address"}, 361 | []string{".status.nodeGroups0.nodeGroupMembers0.readEndpoint.address"}, 362 | map[string]interface{}{ 363 | "nodeGroups": []interface{}{ 364 | map[string]interface{}{ 365 | "nodeGroupID": "0001", 366 | "nodeGroupMembers": []interface{}{ 367 | map[string]interface{}{ 368 | "cacheClusterID": "test-001", 369 | "cacheNodeID": "0001", 370 | "readEndpoint": map[string]interface{}{ 371 | "address": "test.use1.cache.amazonaws.com", 372 | "port": "6379", 373 | }, 374 | }, 375 | }, 376 | }, 377 | }, 378 | }, 379 | 0, 380 | []string{"test.use1.cache.amazonaws.com"}, 381 | false, 382 | }, 383 | { // more complex jq query 384 | []string{`.status.nodeGroups[] | select(.nodeGroupID == "0001") | .nodeGroupMembers[] | ` + 385 | `select(.cacheNodeID == "0001") | .readEndpoint.address`}, 386 | []string{`.status.nodeGroupsselect.nodeGroupID0001.nodeGroupMembersselect.cacheNodeID0001.readEndpoint.address`}, 387 | map[string]interface{}{ 388 | "nodeGroups": []interface{}{ 389 | map[string]interface{}{ 390 | "nodeGroupID": "0001", 391 | "nodeGroupMembers": []interface{}{ 392 | map[string]interface{}{ 393 | "cacheClusterID": "test-001", 394 | "cacheNodeID": "0001", 395 | "readEndpoint": map[string]interface{}{ 396 | "address": "test.use1.cache.amazonaws.com", 397 | "port": "6379", 398 | }, 399 | }, 400 | }, 401 | }, 402 | }, 403 | }, 404 | 0, 405 | []string{"test.use1.cache.amazonaws.com"}, 406 | false, 407 | }, 408 | { // missing status item 409 | []string{`.status.nodeGroups[] | select(.nodeGroupID == "0001") | .nodeGroupMembers[] | ` + 410 | `select(.cacheNodeID == "0001") | .readEndpoint.address`}, 411 | []string{`.status.nodeGroupsselect.nodeGroupID0001.nodeGroupMembersselect.cacheNodeID0001.readEndpoint.address`}, 412 | map[string]interface{}{ 413 | "nodeGroups": []interface{}{ 414 | map[string]interface{}{ 415 | "nodeGroupID": "0001", 416 | }, 417 | }, 418 | }, 419 | 0, // Fail 420 | []string{""}, 421 | true, 422 | }, 423 | } 424 | ctx, cancel := context.WithCancel(context.Background()) 425 | gvrToListKind := map[schema.GroupVersionResource]string{ 426 | {Version: "v1", Resource: "secrets"}: "SecretList", 427 | } 428 | defer cancel() 429 | for _, tc := range testCases { 430 | t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { 431 | object := redis.DeepCopy() 432 | unstructured.SetNestedMap(object.Object, tc.objectMap, "status") 433 | refs := map[string][]string{"redis-rediscache": tc.paths} 434 | dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), 435 | gvrToListKind, &configMapList) 436 | mapper := restmapper.NewDiscoveryRESTMapper(resources) 437 | regionEnv := RegionEnv{ 438 | Cfg: &CmdConfig{ 439 | Namespace: "", 440 | }, 441 | Context: ctx, 442 | DynamicClient: dynamicClient, 443 | Logger: logger, 444 | Mapper: mapper, 445 | ObjectRefs: refs, 446 | PodObject: deployment.DeepCopy(), 447 | Unstructured: []*unstructured.Unstructured{object}, 448 | } 449 | gvk := schema.GroupVersionKind{Version: "v1", Kind: "Secret"} 450 | mapping, _ := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) 451 | regionEnv.createObjectSecrets(object, tc.attempt) 452 | 453 | if len(regionEnv.Errors) > 0 { 454 | if !tc.expectErr { 455 | t.Errorf("Unexpected Error %s", regionEnv.Errors[0]) 456 | } 457 | return 458 | } 459 | 460 | dynamicInterface := regionEnv.DynamicClient.Resource(mapping.Resource).Namespace("") 461 | resultSecret, getErr := dynamicInterface.Get(ctx, "redis-rediscache", metav1.GetOptions{}) 462 | 463 | if getErr != nil { 464 | t.Errorf("Secret missing %s", getErr.Error()) 465 | return 466 | } 467 | secretData, _, _ := unstructured.NestedStringMap(resultSecret.Object, "data") 468 | for i, key := range tc.keys { 469 | secretValue, _ := base64.StdEncoding.DecodeString(secretData[key]) 470 | if string(secretValue) != tc.expected[i] { 471 | t.Errorf("Unexpected result %v expected %v\n", string(secretValue), tc.expected[i]) 472 | } 473 | } 474 | }) 475 | } 476 | } 477 | 478 | func stringToUnstructured(objs []string) []*unstructured.Unstructured { 479 | var result []*unstructured.Unstructured 480 | for _, object := range objs { 481 | lastIndex := strings.LastIndex(object, "/") 482 | apiVersion := object[:lastIndex] 483 | kind := object[lastIndex+1:] 484 | unstruct := unstructured.Unstructured{ 485 | Object: map[string]interface{}{ 486 | "apiVersion": apiVersion, 487 | "kind": kind, 488 | }, 489 | } 490 | result = append(result, &unstruct) 491 | } 492 | return result 493 | } 494 | 495 | func unstructuredToString(objs []*unstructured.Unstructured) []string { 496 | var result []string 497 | for _, object := range objs { 498 | str := fmt.Sprintf("%s/%s", object.GetAPIVersion(), object.GetKind()) 499 | result = append(result, str) 500 | } 501 | return result 502 | } 503 | 504 | func findUnstructuredString(obj map[string]interface{}, path, subpath []string) string { 505 | result, ok, _ := unstructured.NestedString(obj, path...) 506 | if ok { 507 | return result 508 | } 509 | resultStringSlice, ok, _ := unstructured.NestedStringSlice(obj, path...) 510 | if ok { 511 | return resultStringSlice[0] 512 | } 513 | resultSlice, ok, _ := unstructured.NestedSlice(obj, path...) 514 | if ok { 515 | subObject := resultSlice[0].(map[string]interface{}) 516 | return findUnstructuredString(subObject, subpath, []string{}) 517 | } 518 | return "" 519 | } 520 | -------------------------------------------------------------------------------- /pkg/porter2k8s/porter2k8s.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | // Package porter2k8s parses a Porter configuration and deploys to kubernetes. 14 | // It creates a deployment specific configuration map and secret object from the config. 15 | // It then creates or updates the project's objects, and a deploys a sha specific image 16 | // (created elsewhere) for the project which references the configmap and secret object. 17 | // In the future Porter2k8s will also support more simplified configuration for services which no longer use Porter. 18 | package porter2k8s 19 | 20 | import ( 21 | "context" 22 | "flag" 23 | "fmt" 24 | "io" 25 | "os" 26 | "os/signal" 27 | "path/filepath" 28 | "strings" 29 | "sync" 30 | "syscall" 31 | "time" 32 | 33 | "git.corp.adobe.com/EchoSign/porter2k8s/pkg/vault" 34 | 35 | log "github.com/sirupsen/logrus" 36 | appsv1 "k8s.io/api/apps/v1" 37 | batchv1 "k8s.io/api/batch/v1" 38 | v1 "k8s.io/api/core/v1" 39 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 40 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 41 | "k8s.io/apimachinery/pkg/runtime" 42 | "k8s.io/client-go/discovery" 43 | "k8s.io/client-go/discovery/cached/memory" 44 | "k8s.io/client-go/dynamic" 45 | "k8s.io/client-go/kubernetes" 46 | restclient "k8s.io/client-go/rest" 47 | "k8s.io/client-go/restmapper" 48 | "k8s.io/client-go/tools/clientcmd" 49 | ) 50 | 51 | const clusterConfigFile = "cluster-config.yaml" 52 | 53 | // CmdConfig is contains the parsed args. 54 | type CmdConfig struct { 55 | ConfigPath string 56 | ConfigType string // Porter or Simple 57 | Environment string 58 | DynamicParallel bool 59 | LogDir string 60 | LogMode string 61 | LogOnce sync.Once 62 | MaxConfigMaps int 63 | Namespace string 64 | PodLogSinker PodLogSinker 65 | Regions string 66 | SHA string 67 | SecretPathWhiteList string 68 | VaultAddress string 69 | VaultBasePath string // Vault base path for secrets. 70 | VaultToken string 71 | VaultNamespace string 72 | Verbose bool 73 | Wait int 74 | } 75 | 76 | // SecretRef is a reference to an individual secret in Vault. 77 | type SecretRef struct { 78 | Key string 79 | Name string 80 | Path string 81 | Value string 82 | Kind EntryType 83 | } 84 | 85 | // SecretKeyRef is a reference to a key within a Kubernetes secret. 86 | type SecretKeyRef struct { 87 | Key string 88 | Name string 89 | Secret string 90 | } 91 | 92 | // EnvVar is an environment variable for the ConfigMap. 93 | type EnvVar struct { 94 | Name string 95 | Value string 96 | } 97 | 98 | type configReader interface { 99 | getEnv(*RegionEnv) 100 | parse(*RegionEnv) 101 | } 102 | 103 | // UpdateFn is a function that can update a single type of Kubernetes object in multiple clusters. 104 | type UpdateFn func(*sync.WaitGroup, *RegionEnv, chan *RegionEnv) 105 | 106 | // Run is the main function. 107 | func Run(args []string) { 108 | // Parse the arguments 109 | var cmdCfg CmdConfig 110 | if err := parse(args, &cmdCfg); err != nil { 111 | log.Errorf("%s", err) 112 | fmt.Fprintf(os.Stderr, "USAGE \n\n\t%s\n\n", os.Args[0]) 113 | fmt.Fprint(os.Stderr, "GLOBAL OPTIONS:\n\n") 114 | usage(os.Stderr, &cmdCfg) 115 | 116 | os.Exit(1) 117 | } 118 | 119 | // set up global context for signal interuption 120 | ctx, cancel := context.WithCancel(context.Background()) 121 | defer cancel() 122 | var stop = make(chan os.Signal) 123 | signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) 124 | go func() { 125 | sig := <-stop 126 | fmt.Printf("caught sig: %+v\n", sig) 127 | cancel() 128 | fmt.Println("Waiting up to 2 seconds to finish.") 129 | time.Sleep(2 * time.Second) 130 | os.Exit(0) 131 | }() 132 | 133 | // Global Configuration 134 | // Read deployment and service files into Kubernetes structs 135 | kubeServiceConfig, kubeServiceConfigErr := getServiceConfig(&cmdCfg) 136 | if kubeServiceConfigErr != nil { 137 | log.Errorf("%s", kubeServiceConfigErr) 138 | os.Exit(2) 139 | } 140 | 141 | // Regional configuration. 142 | // Gather environment variables and secret references. 143 | // Retrieve secrets from vault. 144 | // Create configmap and secret object. 145 | var regionEnvs []*RegionEnv 146 | for regionEnv := range createEnv(kubeServiceConfig, fetchSecrets(getConfig(ctx, &cmdCfg))) { 147 | log.Debugf("Retrieved Configuration %+v", regionEnv) 148 | if len(regionEnv.Errors) > 0 { 149 | log.Errorf("%s", regionEnv.Errors) 150 | os.Exit(2) 151 | } 152 | regionEnvs = append(regionEnvs, regionEnv) 153 | } 154 | 155 | // Run and monitor updates in this order. 156 | updateFns := []UpdateFn{ 157 | updateDynamicServiceRegion, 158 | updatePodObjectRegion, 159 | } 160 | for _, updateFn := range updateFns { 161 | err := runUpdate(regionEnvs, updateFn) 162 | if err != nil { 163 | log.Error(err) 164 | os.Exit(2) 165 | } 166 | } 167 | } 168 | 169 | // getServiceConfig reads the Kubernetes yaml files. 170 | func getServiceConfig(cfg *CmdConfig) (KubeObjects, error) { 171 | // Intialize KubeObjects 172 | var kubeObjects KubeObjects 173 | kubeObjects.Unstructured = make(map[string][]*unstructured.Unstructured) 174 | 175 | // Objects of all types are parsed from these files, they need not match the name. 176 | possibleFiles := []string{ 177 | "deployment", 178 | "service", 179 | "ingress", 180 | "istio", 181 | "job", 182 | "ethos-logging-sidecar", 183 | "servicecatalog-aws", 184 | "servicecatalog-azure", 185 | "hpa", 186 | "pdb", 187 | "secret", 188 | "service-operator", 189 | } 190 | 191 | // Pull down kube configs from Vault. 192 | if err := kubeObjects.getKubeConfigs(cfg); err != nil { 193 | return kubeObjects, err 194 | } 195 | 196 | // Decode Kubernetes Objects from files. 197 | if err := kubeObjects.processObjects(cfg, possibleFiles); err != nil { 198 | return kubeObjects, err 199 | } 200 | 201 | // Perform checks/namespacing to ensure resources will not conflict with those of other services. 202 | if validationErr := kubeObjects.validateAndTag(); validationErr != nil { 203 | return kubeObjects, validationErr 204 | } 205 | 206 | deploymentErr := kubeObjects.prepareDeployment(cfg.SHA) 207 | return kubeObjects, deploymentErr 208 | } 209 | 210 | // getReader gets the config reader for the type of env var, secret reference listing. 211 | func getReader(configType string) (configReader, error) { 212 | // Only one type of reader so far. 213 | if configType == "porter" { 214 | return &PorterEnv{}, nil 215 | } 216 | if configType == "simple" { 217 | return &SimpleEnv{}, nil 218 | } 219 | if configType == "test" { 220 | return &TestEnv{}, nil 221 | } 222 | return nil, fmt.Errorf("invalid config type %s", configType) 223 | } 224 | 225 | // getConfig fetches environment variables and secret references. 226 | func getConfig(ctx context.Context, cfg *CmdConfig) <-chan *RegionEnv { 227 | envStream := make(chan *RegionEnv) 228 | go func() { 229 | defer log.Debug("Closing getConfig channel") 230 | defer close(envStream) 231 | 232 | for _, region := range strings.Fields(cfg.Regions) { 233 | logger := log.WithFields(log.Fields{"Region": region}) 234 | // Create an environment for this region and populate it with information from the config files. 235 | environment := RegionEnv{ 236 | Region: region, 237 | Cfg: cfg, 238 | Context: ctx, 239 | Logger: logger, 240 | } 241 | 242 | reader, readerError := getReader(cfg.ConfigType) 243 | if readerError != nil { 244 | badEnv := RegionEnv{Errors: []error{readerError}} 245 | envStream <- &badEnv 246 | return 247 | } 248 | 249 | // getEnv adds values for the deployment specific configmap and secret. 250 | environment.ClusterSettings = map[string]string{} 251 | environment.ObjectRefs = map[string][]string{} 252 | reader.getEnv(&environment) 253 | reader.parse(&environment) 254 | environment.identifySecrets() 255 | log.Infof("Created region specific configuration for %s.", region) 256 | select { 257 | case <-ctx.Done(): 258 | log.Info("Received Done") 259 | return 260 | case envStream <- &environment: 261 | } 262 | } 263 | }() 264 | 265 | return envStream 266 | } 267 | 268 | // FetchSecrets from Vault. 269 | func fetchSecrets(regionEnvStream <-chan *RegionEnv) <-chan *RegionEnv { 270 | envWithSecretsStream := make(chan *RegionEnv) 271 | go func() { 272 | defer log.Debug("Closing fetchSecrets channel") 273 | defer close(envWithSecretsStream) 274 | var client vault.VaultClientInterface 275 | var clientErr error 276 | // Cache secrets. Map of vaultapi.Secret.Data. 277 | secretCache := make(map[string]map[string]interface{}) 278 | for regionEnv := range regionEnvStream { 279 | // Check for errors in the stream 280 | if len(regionEnv.Errors) > 0 { 281 | envWithSecretsStream <- regionEnv 282 | return 283 | } 284 | 285 | if client == nil { 286 | client, clientErr = vault.NewVaultClient( 287 | regionEnv.Cfg.VaultAddress, 288 | regionEnv.Cfg.VaultNamespace, 289 | regionEnv.Cfg.VaultToken, 290 | ) 291 | } 292 | if clientErr != nil { 293 | regionEnv.errf("vault client initialization error %s", clientErr) 294 | envWithSecretsStream <- regionEnv 295 | return 296 | } 297 | 298 | var secrets []SecretRef 299 | for _, secret := range regionEnv.Secrets { 300 | // Keys are optional in simple configs and do not exist in porter configs. 301 | if secret.Key == "" { 302 | secret.Key = secret.Name 303 | } 304 | vaultSecretData, ok := secretCache[secret.Path] 305 | if !ok { 306 | vaultSecret, readErr := client.Read(secret.Path) 307 | if readErr != nil { 308 | regionEnv.errf("unable to retrieve secrets for region %s\n%v", regionEnv.Region, readErr) 309 | continue 310 | } 311 | if vaultSecret == nil || vaultSecret[secret.Key] == nil { 312 | regionEnv.errf("vault path %s is valid but value for %s was not found", secret.Path, secret.Key) 313 | continue 314 | } 315 | vaultSecretData = vaultSecret 316 | } 317 | // Catch keys which do not exist in cache. 318 | if vaultSecretData[secret.Key] == nil { 319 | regionEnv.errf("vault path %s is valid but value for %s was not found", secret.Path, secret.Key) 320 | continue 321 | } 322 | secret.Value = vaultSecretData[secret.Key].(string) 323 | secretCache[secret.Path] = vaultSecretData 324 | secrets = append(secrets, secret) 325 | } 326 | regionEnv.Secrets = secrets 327 | 328 | log.Infof("Fetched secrets for %s.", regionEnv.Region) 329 | 330 | select { 331 | case <-regionEnv.Context.Done(): 332 | log.Info("Received Done") 333 | return 334 | case envWithSecretsStream <- regionEnv: 335 | } 336 | } 337 | }() 338 | return envWithSecretsStream 339 | } 340 | 341 | // createEnv creates Kubernetes configmap and secret 342 | func createEnv(kubeServiceConfig KubeObjects, regionEnvStream <-chan *RegionEnv) <-chan *RegionEnv { 343 | envKubernetesStream := make(chan *RegionEnv) 344 | // The secrets and config map take their name from the deployment. 345 | //deployment := kubeServiceConfig.Deployment.(*appsv1.Deployment) 346 | deploymentName := kubeServiceConfig.PodObject.GetName() 347 | go func() { 348 | defer log.Debug("Closing createEnv channel") 349 | defer close(envKubernetesStream) 350 | for regionEnv := range regionEnvStream { 351 | var clientConfig *restclient.Config 352 | var kubeConfigErr error 353 | 354 | // Check for errors in the stream 355 | if len(regionEnv.Errors) > 0 { 356 | envKubernetesStream <- regionEnv 357 | log.Info("Exiting on previous errors") 358 | return 359 | } 360 | 361 | if err := regionEnv.validateSecretPath(deploymentName); err != nil { 362 | regionEnv.Errors = append(regionEnv.Errors, err) 363 | envKubernetesStream <- regionEnv 364 | return 365 | } 366 | 367 | log.Infof("Modifying region specific Kubernetes structs for %s.", regionEnv.Region) 368 | 369 | if len(kubeServiceConfig.ClusterConfigs.Clusters) < 1 { 370 | // Follow kubectl convention. 371 | kubeConfigFile := setFromEnvStr("KUBECONFIG", filepath.Join(os.Getenv("HOME"), ".kube", "config")) 372 | log.Infof("Using kubeconfig file %s", kubeConfigFile) 373 | // Region name must be the same as the kubernetes context name. 374 | clientConfig, kubeConfigErr = buildConfigFromFlags(regionEnv.Region, kubeConfigFile) 375 | if kubeConfigErr != nil { 376 | regionEnv.Errors = append(regionEnv.Errors, kubeConfigErr) 377 | envKubernetesStream <- regionEnv 378 | log.Info("Exiting on kubeconfig construction errors") 379 | return 380 | } 381 | } else { 382 | ok := false 383 | clientConfig, ok = kubeServiceConfig.ClusterConfigs.RegionMap[regionEnv.Region] 384 | if !ok { 385 | regionEnv.errf("config for region %s not found in %s", regionEnv.Region, clusterConfigFile) 386 | envKubernetesStream <- regionEnv 387 | log.Info("Exiting on missing config in vault.") 388 | return 389 | } 390 | } 391 | 392 | clientset, clientsetErr := kubernetes.NewForConfig(clientConfig) 393 | if clientsetErr != nil { 394 | regionEnv.Errors = append(regionEnv.Errors, clientsetErr) 395 | envKubernetesStream <- regionEnv 396 | log.Info("Exiting on clientset creation error.") 397 | return 398 | } 399 | regionEnv.Clientset = clientset 400 | dynamic, dynamicErr := dynamic.NewForConfig(clientConfig) 401 | if dynamicErr != nil { 402 | regionEnv.Errors = append(regionEnv.Errors, dynamicErr) 403 | envKubernetesStream <- regionEnv 404 | log.Info("Exiting on dynamic client creation error.") 405 | return 406 | } 407 | regionEnv.DynamicClient = dynamic 408 | discoveryClient, discoveryErr := discovery.NewDiscoveryClientForConfig(clientConfig) 409 | if discoveryErr != nil { 410 | regionEnv.Errors = append(regionEnv.Errors, discoveryErr) 411 | envKubernetesStream <- regionEnv 412 | log.Info("Exiting on discovery client creation error.") 413 | return 414 | } 415 | regionEnv.Mapper = restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(discoveryClient)) 416 | 417 | // Retrieve cluster specific porter2k8s settings from "porter2k8s" configmap. 418 | regionEnv.porter2k8sConfigMap() 419 | log.Infof("Cluster settings %+v", regionEnv.ClusterSettings) 420 | 421 | // Load regionEnv.Secrets of Kind "porter2k8s" into the region's cluster settings 422 | for _, secret := range regionEnv.Secrets { 423 | if secret.Kind == PorterType { 424 | regionEnv.ClusterSettings[secret.Name] = secret.Value 425 | } 426 | } 427 | 428 | // Each RegionEnv needs its own copy of the kubernetes objects, since they are modified with region 429 | // specific settings. 430 | regionEnv.PodObject = kubeServiceConfig.PodObject.DeepCopy() 431 | 432 | dynamicObjects := append( 433 | kubeServiceConfig.Unstructured["all"], 434 | kubeServiceConfig.Unstructured[regionEnv.ClusterSettings["CLOUD"]]..., 435 | ) 436 | for _, dynamicObject := range dynamicObjects { 437 | regionEnv.Unstructured = append(regionEnv.Unstructured, dynamicObject.DeepCopy()) 438 | } 439 | 440 | configMapCreateSucceeded := regionEnv.createConfigMap(deploymentName) 441 | if !configMapCreateSucceeded { 442 | envKubernetesStream <- regionEnv 443 | log.Info("Exiting on configmap creation error.") 444 | return 445 | } 446 | 447 | secretCreateSucceeded := regionEnv.createSecret(deploymentName) 448 | if !secretCreateSucceeded { 449 | envKubernetesStream <- regionEnv 450 | log.Info("Exiting on secret creation failure.") 451 | return 452 | } 453 | 454 | select { 455 | case <-regionEnv.Context.Done(): 456 | log.Info("Received Done") 457 | return 458 | case envKubernetesStream <- regionEnv: 459 | } 460 | } 461 | }() 462 | return envKubernetesStream 463 | } 464 | 465 | // FlagSet - Set up the flags 466 | func flagSet(name string, cfg *CmdConfig) *flag.FlagSet { 467 | flags := flag.NewFlagSet(name, flag.ExitOnError) 468 | flags.StringVar( 469 | &cfg.ConfigPath, 470 | "config-path", 471 | setFromEnvStr("CONFIG_PATH", "/repo"), 472 | "Configuration root directory. Should include the '.porter' or 'environment' directory. "+ 473 | "Kubernetes object yaml files may be in the directory or in a subdirectory named 'k8s'.", 474 | ) 475 | flags.StringVar(&cfg.ConfigType, "config-type", setFromEnvStr("CONFIG_TYPE", "porter"), "Configuration type, "+ 476 | "simple or porter.") 477 | flags.StringVar(&cfg.Environment, "environment", setFromEnvStr("ENVIRONMENT", ""), "Environment of deployment.") 478 | flags.IntVar(&cfg.MaxConfigMaps, "max-cm", setFromEnvInt("MAX_CM", 5), "Maximum number of configmaps and secret "+ 479 | "objects to keep per app.") 480 | flags.StringVar(&cfg.Namespace, "namespace", setFromEnvStr("NAMESPACE", "default"), "Kubernetes namespace.") 481 | flags.StringVar(&cfg.Regions, "regions", setFromEnvStr("REGIONS", ""), "Regions"+ 482 | "of deployment. (Multiple Space delimited regions allowed)") 483 | flags.StringVar(&cfg.SHA, "sha", setFromEnvStr("sha", ""), "Deployment sha.") 484 | flags.StringVar(&cfg.VaultAddress, "vault-addr", setFromEnvStr("VAULT_ADDR", "https://vault.loc.adobe.net"), 485 | "Vault server.") 486 | flags.StringVar(&cfg.VaultBasePath, "vault-path", setFromEnvStr("VAULT_PATH", "/"), "Path in Vault.") 487 | flags.StringVar(&cfg.VaultToken, "vault-token", setFromEnvStr("VAULT_TOKEN", ""), "Vault token.") 488 | flags.StringVar(&cfg.VaultNamespace, "vault-namespace", setFromEnvStr("VAULT_NAMESPACE", ""), "Vault namespace.") 489 | flags.StringVar(&cfg.SecretPathWhiteList, "secret-path-whitelist", setFromEnvStr("SECRET_PATH_WHITELIST", ""), ""+ 490 | "Multiple Space delimited secret path whitelist allowed") 491 | flags.BoolVar(&cfg.Verbose, "v", setFromEnvBool("VERBOSE"), "Verbose log output.") 492 | flags.IntVar(&cfg.Wait, "wait", setFromEnvInt("WAIT", 180), "Extra time to wait for deployment to complete in "+ 493 | "seconds.") 494 | flags.StringVar(&cfg.LogMode, "log-mode", setFromEnvStr("LOG_MODE", "inline"), "Pod log streaming mode. "+ 495 | "One of 'inline' (print to STDOUT), 'single' (single region to stdout), "+ 496 | "'file' (write to filesystem, see log-dir option), 'none' (disable log streaming)") 497 | flags.StringVar(&cfg.LogDir, "log-dir", setFromEnvStr("LOG_DIR", "logs"), 498 | "Directory to write pod logs into. (must already exist)") 499 | flags.BoolVar(&cfg.DynamicParallel, "dynamic-parallel", setFromEnvBool("DYNAMIC_PARALLEL"), 500 | "Update Dynamic Objects in parallel.") 501 | 502 | return flags 503 | } 504 | 505 | // Parse - Process command line arguments 506 | func parse(args []string, cfg *CmdConfig) error { 507 | 508 | flags := flagSet(" ", cfg) 509 | if err := flags.Parse(args); err != nil { 510 | return err 511 | } 512 | if cfg.Regions == "" { 513 | return fmt.Errorf("no regions given") 514 | } 515 | if cfg.Environment == "" { 516 | return fmt.Errorf("no environment given") 517 | } 518 | if cfg.SHA == "" { 519 | return fmt.Errorf("no deployment SHA given") 520 | } 521 | if cfg.MaxConfigMaps < 3 { 522 | return fmt.Errorf("number of configmaps and secret objects to keep must be more than 2") 523 | } 524 | if cfg.Verbose { 525 | log.SetLevel(log.DebugLevel) 526 | log.Debug("Setting verbose logging") 527 | } 528 | switch cfg.LogMode { 529 | case "inline": 530 | cfg.PodLogSinker = LogrusSink() 531 | case "single", "none": 532 | cfg.PodLogSinker = NullLogSink() 533 | case "file": 534 | fileInfo, err := os.Stat(cfg.LogDir) 535 | if err == nil && !fileInfo.IsDir() { 536 | err = fmt.Errorf("not a directory") 537 | } 538 | if err != nil { 539 | return fmt.Errorf("invalid log-dir: %s (%w)", cfg.LogDir, err) 540 | } 541 | cfg.PodLogSinker = DirectoryLogSink(cfg.LogDir) 542 | default: 543 | return fmt.Errorf("invalid log-mode: %s", cfg.LogMode) 544 | } 545 | 546 | log.Infof("Regions: %s", cfg.Regions) 547 | return nil 548 | } 549 | 550 | // Usage - emit the usage 551 | func usage(writer io.Writer, cfg *CmdConfig) { 552 | flags := flagSet("", cfg) 553 | flags.SetOutput(writer) 554 | flags.PrintDefaults() 555 | } 556 | 557 | // findContainer finds the container in deployment/job that matches the deployment name. 558 | func findContainer(kubeObject *unstructured.Unstructured) (*v1.Container, metav1.Object) { 559 | name := kubeObject.GetName() 560 | templateSpec, typedObject := findPodTemplateSpecUnstructured(kubeObject) 561 | 562 | // Find container in deployment that matches the deployment name 563 | for i, container := range templateSpec.Spec.Containers { 564 | if container.Name == name { 565 | return &templateSpec.Spec.Containers[i], typedObject 566 | } 567 | } 568 | return nil, nil 569 | } 570 | 571 | // Retrieve Pod spec from either a deployment or a job. Returns typed pod template spec and typed object. 572 | func findPodTemplateSpecUnstructured(kubeObject *unstructured.Unstructured) (*v1.PodTemplateSpec, metav1.Object) { 573 | switch kubeObject.GetKind() { 574 | case "Deployment": 575 | v := appsv1.Deployment{} 576 | if err := runtime.DefaultUnstructuredConverter.FromUnstructured(kubeObject.Object, &v); err != nil { 577 | return nil, nil 578 | } 579 | return &v.Spec.Template, &v 580 | case "Job": 581 | v := batchv1.Job{} 582 | if err := runtime.DefaultUnstructuredConverter.FromUnstructured(kubeObject.Object, &v); err != nil { 583 | return nil, nil 584 | } 585 | return &v.Spec.Template, &v 586 | case "StatefulSet": 587 | v := appsv1.StatefulSet{} 588 | if err := runtime.DefaultUnstructuredConverter.FromUnstructured(kubeObject.Object, &v); err != nil { 589 | return nil, nil 590 | } 591 | return &v.Spec.Template, &v 592 | default: 593 | return nil, nil 594 | } 595 | } 596 | 597 | // Retrieve Pod spec from either a deployment or a job. 598 | func findPodTemplateSpec(kubeObject interface{}) *v1.PodTemplateSpec { 599 | switch asserted := kubeObject.(type) { 600 | case *appsv1.Deployment: 601 | return &asserted.Spec.Template 602 | case *batchv1.Job: 603 | return &asserted.Spec.Template 604 | case *appsv1.StatefulSet: 605 | return &asserted.Spec.Template 606 | default: 607 | return nil 608 | } 609 | } 610 | 611 | // buildConfigFromFlags creates Kube Config. 612 | // Context is assumed to be the region name. 613 | func buildConfigFromFlags(context, kubeconfigPath string) (*restclient.Config, error) { 614 | return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( 615 | &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, 616 | &clientcmd.ConfigOverrides{ 617 | CurrentContext: context, 618 | }).ClientConfig() 619 | } 620 | 621 | // calculateWatchTimeout calculates timeout for watch channel based on livenessProbe for primary pod container 622 | // The idea is for porter2k8s to fail if the containers fail to come up healthy. 623 | func calculateWatchTimeout(deployment *unstructured.Unstructured, buffer int32) *int64 { 624 | applicationContainer, _ := findContainer(deployment) 625 | if applicationContainer.LivenessProbe == nil { 626 | // Return default if liveness probe was initialized but all values left at defaults, plus buffer. 627 | return int64Ref(33 + buffer) 628 | } 629 | // Default settings as defined in k8s.io/api/core/v1/types.go 630 | initialDelay := applicationContainer.LivenessProbe.InitialDelaySeconds // Default is 0. 631 | timeout := setIfZeroInt32(applicationContainer.LivenessProbe.TimeoutSeconds, 1) 632 | period := setIfZeroInt32(applicationContainer.LivenessProbe.PeriodSeconds, 10) 633 | failureThreshold := setIfZeroInt32(applicationContainer.LivenessProbe.FailureThreshold, 3) 634 | return int64Ref(buffer + initialDelay + failureThreshold*(period+timeout)) 635 | } 636 | --------------------------------------------------------------------------------