├── hack ├── boilerplate.go.txt └── sample │ ├── cloudsql-celery │ ├── sqlproxy-secret.yaml │ ├── base.yaml │ └── cluster.yaml │ ├── postgres-celery-redis │ ├── redis-secret.yaml │ ├── cluster.yaml │ └── redis.yaml │ ├── mysql-celery │ ├── base.yaml │ └── cluster.yaml │ ├── postgres-celery │ ├── base.yaml │ └── cluster.yaml │ ├── cloudsql-local │ └── cluster.yaml │ ├── postgres-local │ └── cluster.yaml │ ├── cloudsql-k8s │ └── cluster.yaml │ ├── postgres-k8s │ └── cluster.yaml │ ├── mysql-celery-gcs │ └── cluster.yaml │ ├── mysql-local │ └── cluster.yaml │ ├── mysql-k8s │ └── cluster.yaml │ └── postgres-celery-memorystore │ └── cluster.yaml ├── config ├── webhook │ ├── manifests.yaml │ ├── kustomization.yaml │ ├── service.yaml │ └── kustomizeconfig.yaml ├── manager │ ├── kustomization.yaml │ └── manager.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── certmanager │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── certificate.yaml ├── samples │ ├── airflow_v1alpha1_airflowbase.yaml │ └── airflow_v1alpha1_airflowcluster.yaml ├── rbac │ ├── role_binding.yaml │ ├── auth_proxy_role_binding.yaml │ ├── leader_election_role_binding.yaml │ ├── auth_proxy_service.yaml │ ├── auth_proxy_role.yaml │ ├── kustomization.yaml │ ├── airflowbase_viewer_role.yaml │ ├── airflowcluster_viewer_role.yaml │ ├── airflowbase_editor_role.yaml │ ├── airflowcluster_editor_role.yaml │ ├── leader_election_role.yaml │ └── role.yaml ├── crd │ ├── patches │ │ ├── cainjection_in_airflowbases.yaml │ │ ├── cainjection_in_airflowclusters.yaml │ │ ├── webhook_in_airflowbases.yaml │ │ └── webhook_in_airflowclusters.yaml │ ├── kustomizeconfig.yaml │ └── kustomization.yaml └── default │ ├── manager_webhook_patch.yaml │ ├── webhookcainjection_patch.yaml │ ├── manager_auth_proxy_patch.yaml │ └── kustomization.yaml ├── docs ├── airflow-pod.png ├── airflow-base.png ├── airflow-cluster.png ├── airflow-multi-node.png ├── airflow-region-spread.png ├── airflow-zone-spread.png ├── userguide.md └── development.md ├── .gitignore ├── PROJECT ├── vendor └── sigs.k8s.io │ └── controller-reconciler │ ├── pkg │ ├── reconciler │ │ ├── testdata │ │ │ ├── unknown_rsrc.yaml │ │ │ └── sts.yaml │ │ ├── doc.go │ │ ├── manager │ │ │ ├── types.go │ │ │ ├── k8s │ │ │ │ └── patch │ │ │ │ │ ├── scheme.go │ │ │ │ │ └── patch_suite_test.go │ │ │ ├── interface.go │ │ │ ├── internal.go │ │ │ └── gcp │ │ │ │ └── utils.go │ │ ├── resource_suite_test.go │ │ ├── types.go │ │ └── resource.go │ ├── boilerplate.go.txt │ ├── status │ │ ├── doc.go │ │ ├── status_suite_test.go │ │ └── status.go │ ├── genericreconciler │ │ ├── doc.go │ │ ├── v1alpha1 │ │ │ ├── doc.go │ │ │ └── testutil.go │ │ ├── genericreconciler_suite_test.go │ │ ├── types.go │ │ └── genericreconciler_test.go │ ├── storage │ │ ├── doc.go │ │ ├── storage_suite_test.go │ │ ├── types.go │ │ ├── storage.go │ │ ├── zz_generated.deepcopy.go │ │ └── storage_test.go │ ├── finalizer │ │ ├── doc.go │ │ ├── zz_generated.deepcopy.go │ │ ├── finalizer_suite_test.go │ │ ├── finalizer.go │ │ └── finalizer_test.go │ ├── doc.go │ └── test │ │ └── helper.go │ └── go.mod ├── go.mod ├── license-templates └── LICENSE.txt ├── controllers ├── application │ ├── doc.go │ ├── application_suite_test.go │ ├── application_test.go │ └── application.go └── suite_test.go ├── yamllint-config.yaml ├── templates ├── serviceaccount.yaml ├── secret.yaml ├── storage.yaml ├── pdb.yaml ├── headlesssvc.yaml ├── svc.yaml ├── rolebinding.yaml ├── flower-sts.yaml ├── worker-sts.yaml ├── scheduler-sts.yaml ├── nfs-sts.yaml ├── ui-sts.yaml ├── sqlproxy-sts.yaml ├── redis-sts.yaml ├── postgres-sts.yaml └── mysql-sts.yaml ├── .asf.yaml ├── cloudbuild.yaml ├── api └── v1alpha1 │ └── groupversion_info.go ├── Dockerfile ├── .github └── workflows │ └── ci.yml ├── .pre-commit-config.yaml ├── main.go ├── README.md └── test └── e2e └── base └── base_test.go /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /config/webhook/manifests.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/airflow-pod.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/airflow-on-k8s-operator/master/docs/airflow-pod.png -------------------------------------------------------------------------------- /docs/airflow-base.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/airflow-on-k8s-operator/master/docs/airflow-base.png -------------------------------------------------------------------------------- /docs/airflow-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/airflow-on-k8s-operator/master/docs/airflow-cluster.png -------------------------------------------------------------------------------- /docs/airflow-multi-node.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/airflow-on-k8s-operator/master/docs/airflow-multi-node.png -------------------------------------------------------------------------------- /docs/airflow-region-spread.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/airflow-on-k8s-operator/master/docs/airflow-region-spread.png -------------------------------------------------------------------------------- /docs/airflow-zone-spread.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/airflow-on-k8s-operator/master/docs/airflow-zone-spread.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | cover.out 3 | pkg/controller/airflowbase/templates 4 | pkg/controller/airflowcluster/templates 5 | .idea/ 6 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: apache.org 2 | repo: github.com/apache/airflow-on-k8s-operator 3 | resources: 4 | - group: airflow 5 | kind: AirflowBase 6 | version: v1alpha1 7 | - group: airflow 8 | kind: AirflowCluster 9 | version: v1alpha1 10 | version: "2" 11 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/testdata/unknown_rsrc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Something 3 | metadata: 4 | name: {{.Name}} 5 | namespace: {{.Namespace}} 6 | labels: 7 | {{range $k,$v := .Labels }} 8 | {{$k}}: {{$v}} 9 | {{end}} 10 | spec: 11 | ports: 12 | - port: 8080 13 | name: http 14 | selector: 15 | {{range $k,$v := .Selector }} 16 | {{$k}}: {{$v}} 17 | {{end}} 18 | type: ClusterIP 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/apache/airflow-on-k8s-operator 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/go-logr/logr v0.1.0 7 | github.com/kr/pretty v0.2.0 // indirect 8 | github.com/kubernetes-sigs/application v0.8.1 9 | github.com/onsi/ginkgo v1.11.0 10 | github.com/onsi/gomega v1.8.1 11 | k8s.io/api v0.17.2 12 | k8s.io/apimachinery v0.17.2 13 | k8s.io/client-go v0.17.1 14 | k8s.io/utils v0.0.0-20200117235808-5f6fbceb4c31 // indirect 15 | sigs.k8s.io/controller-reconciler v0.0.0-00010101000000-000000000000 16 | sigs.k8s.io/controller-runtime v0.4.0 17 | ) 18 | 19 | replace sigs.k8s.io/controller-reconciler => ./vendor/sigs.k8s.io/controller-reconciler 20 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/go.mod: -------------------------------------------------------------------------------- 1 | module sigs.k8s.io/controller-reconciler 2 | 3 | go 1.13 4 | 5 | require ( 6 | cloud.google.com/go v0.39.0 7 | github.com/evanphx/json-patch v4.5.0+incompatible 8 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect 9 | github.com/onsi/ginkgo v1.11.0 10 | github.com/onsi/gomega v1.8.1 11 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 12 | google.golang.org/api v0.5.0 13 | google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69 // indirect 14 | k8s.io/api v0.17.1 15 | k8s.io/apiextensions-apiserver v0.17.1 16 | k8s.io/apimachinery v0.17.1 17 | k8s.io/client-go v0.17.1 18 | sigs.k8s.io/controller-runtime v0.4.0 19 | ) 20 | -------------------------------------------------------------------------------- /license-templates/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Licensed to the Apache Software Foundation (ASF) under one or more 2 | contributor license agreements. See the NOTICE file distributed with 3 | this work for additional information regarding copyright ownership. 4 | The ASF licenses this file to You under the Apache License, Version 2.0 5 | (the "License"); you may not use this file except in compliance with 6 | the License. You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | resources: 18 | - manager.yaml 19 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | resources: 18 | - monitor.yaml 19 | -------------------------------------------------------------------------------- /controllers/application/doc.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // Package application contains code for creating appcrd 17 | 18 | package application 19 | -------------------------------------------------------------------------------- /config/certmanager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | resources: 18 | - certificate.yaml 19 | 20 | configurations: 21 | - kustomizeconfig.yaml 22 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/status/doc.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // Package status contains status releated code 17 | package status 18 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/doc.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // Package reconciler contains resource methods 17 | package reconciler 18 | -------------------------------------------------------------------------------- /config/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | resources: 18 | - manifests.yaml 19 | - service.yaml 20 | 21 | configurations: 22 | - kustomizeconfig.yaml 23 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/genericreconciler/doc.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // Package genericreconciler contains generic reconciler loop 17 | package genericreconciler 18 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/storage/doc.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // Package storage contains storage releated code 17 | // +k8s:deepcopy-gen=package,register 18 | package storage 19 | -------------------------------------------------------------------------------- /hack/sample/cloudsql-celery/sqlproxy-secret.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: v1 18 | kind: Secret 19 | metadata: 20 | name: cc-base-sql 21 | type: Opaque 22 | data: 23 | rootpassword: cm9vdDEyMw== 24 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/finalizer/doc.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // Package finalizer contains storage releated code 17 | // +k8s:deepcopy-gen=package,register 18 | package finalizer 19 | -------------------------------------------------------------------------------- /hack/sample/postgres-celery-redis/redis-secret.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: v1 18 | kind: Secret 19 | metadata: 20 | name: pc-cluster-redis 21 | type: Opaque 22 | data: 23 | password: aGVsbG93b3JsZA== 24 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/finalizer/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // +build !ignore_autogenerated 16 | 17 | // Code generated by main. DO NOT EDIT. 18 | 19 | package finalizer 20 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/genericreconciler/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // Package v1alpha1 contains test custom resource 17 | // +k8s:deepcopy-gen=package,register 18 | package v1alpha1 19 | -------------------------------------------------------------------------------- /config/samples/airflow_v1alpha1_airflowbase.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowBase 19 | metadata: 20 | name: airflowbase-sample 21 | spec: 22 | # Add fields here 23 | foo: bar 24 | -------------------------------------------------------------------------------- /config/samples/airflow_v1alpha1_airflowcluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: airflowcluster-sample 21 | spec: 22 | # Add fields here 23 | foo: bar 24 | -------------------------------------------------------------------------------- /hack/sample/mysql-celery/base.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowBase 19 | metadata: 20 | name: mc-base 21 | spec: 22 | mysql: 23 | operator: false 24 | storage: 25 | version: "" 26 | -------------------------------------------------------------------------------- /yamllint-config.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | --- 18 | extends: default 19 | 20 | ignore: | 21 | /config/crd/bases/* 22 | /config/rbac/role.yaml 23 | /config/webhook/manifests.yaml 24 | 25 | rules: 26 | line-length: 27 | max: 200 28 | -------------------------------------------------------------------------------- /hack/sample/postgres-celery/base.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowBase 19 | metadata: 20 | name: pc-base 21 | spec: 22 | postgres: 23 | operator: false 24 | storage: 25 | version: "" 26 | -------------------------------------------------------------------------------- /templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: v1 18 | kind: ServiceAccount 19 | metadata: 20 | name: {{.Name}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/types.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package manager 17 | 18 | // ResourceManager is an interface for operating on CRs 19 | type ResourceManager struct { 20 | rsrcmgrs map[string]Manager 21 | } 22 | -------------------------------------------------------------------------------- /config/webhook/service.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: webhook-service 21 | namespace: system 22 | spec: 23 | ports: 24 | - port: 443 25 | targetPort: 9443 26 | selector: 27 | control-plane: controller-manager 28 | -------------------------------------------------------------------------------- /.asf.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # https://cwiki.apache.org/confluence/display/INFRA/.asf.yaml+features+for+git+repositories 17 | --- 18 | github: 19 | enabled_merge_buttons: 20 | squash: true 21 | merge: false 22 | rebase: false 23 | features: 24 | # Enable issues management 25 | issues: true 26 | -------------------------------------------------------------------------------- /templates/secret.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: v1 18 | kind: Secret 19 | metadata: 20 | name: {{.SecretName}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | data: 27 | {{range $k,$v := .Secret }} 28 | {{$k}}: {{$v}} 29 | {{end}} 30 | -------------------------------------------------------------------------------- /templates/storage.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | kind: StorageClass 18 | apiVersion: storage.k8s.io/v1 19 | metadata: 20 | name: ssd 21 | labels: 22 | {{range $k,$v := .Labels }} 23 | {{$k}}: {{$v}} 24 | {{end}} 25 | provisioner: kubernetes.io/gce-pd 26 | parameters: 27 | type: pd-ssd 28 | zone: asia-southeast1-a 29 | -------------------------------------------------------------------------------- /hack/sample/cloudsql-celery/base.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowBase 19 | metadata: 20 | name: cc-base 21 | spec: 22 | sqlproxy: 23 | project: kubeflow-193622 24 | region: us-central1 25 | instance: airflow-test1 26 | type: postgres 27 | storage: 28 | version: "" 29 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/genericreconciler/v1alpha1/testutil.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // Package v1alpha1 contains test custom resource 17 | //go:generate go run ../../../vendor/k8s.io/code-generator/cmd/deepcopy-gen/main.go -O zz_generated.deepcopy -i ./... -h ../../../hack/boilerplate.go.txt 18 | package v1alpha1 19 | -------------------------------------------------------------------------------- /hack/sample/cloudsql-local/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: cl-cluster 21 | spec: 22 | executor: Local 23 | scheduler: 24 | version: "1.10.2" 25 | ui: 26 | replicas: 1 27 | version: "1.10.2" 28 | airflowbase: 29 | name: cc-base 30 | -------------------------------------------------------------------------------- /hack/sample/postgres-local/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: pl-cluster 21 | spec: 22 | executor: Local 23 | ui: 24 | replicas: 1 25 | version: "1.10.2" 26 | scheduler: 27 | version: "1.10.2" 28 | airflowbase: 29 | name: pc-base 30 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/k8s/patch/scheme.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package patch 17 | 18 | import ( 19 | "k8s.io/apimachinery/pkg/runtime" 20 | ) 21 | 22 | // Scheme is the default instance of runtime.Scheme to which types in the Kubernetes API are already registered. 23 | var Scheme = runtime.NewScheme() 24 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | /* 18 | Package pkg provides libraries for building Controller reconciler. 19 | 20 | Reconciler 21 | 22 | Reconciler provides a library to implement reconciler for a controller built using controller-runtime 23 | 24 | Usage 25 | 26 | The following example shows creating a new Controller program which Reconciles a FooBar CRD 27 | 28 | */ 29 | //go:generate go run ../vendor/k8s.io/code-generator/cmd/deepcopy-gen/main.go -O zz_generated.deepcopy -i ./... -h boilerplate.go.txt 30 | package pkg 31 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: rbac.authorization.k8s.io/v1 18 | kind: ClusterRoleBinding 19 | metadata: 20 | name: manager-rolebinding 21 | roleRef: 22 | apiGroup: rbac.authorization.k8s.io 23 | kind: ClusterRole 24 | name: manager-role 25 | subjects: 26 | - kind: ServiceAccount 27 | name: default 28 | namespace: system 29 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: rbac.authorization.k8s.io/v1 18 | kind: ClusterRoleBinding 19 | metadata: 20 | name: proxy-rolebinding 21 | roleRef: 22 | apiGroup: rbac.authorization.k8s.io 23 | kind: ClusterRole 24 | name: proxy-role 25 | subjects: 26 | - kind: ServiceAccount 27 | name: default 28 | namespace: system 29 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: rbac.authorization.k8s.io/v1 18 | kind: RoleBinding 19 | metadata: 20 | name: leader-election-rolebinding 21 | roleRef: 22 | apiGroup: rbac.authorization.k8s.io 23 | kind: Role 24 | name: leader-election-role 25 | subjects: 26 | - kind: ServiceAccount 27 | name: default 28 | namespace: system 29 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | labels: 21 | control-plane: controller-manager 22 | name: controller-manager-metrics-service 23 | namespace: system 24 | spec: 25 | ports: 26 | - name: https 27 | port: 8443 28 | targetPort: https 29 | selector: 30 | control-plane: controller-manager 31 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: rbac.authorization.k8s.io/v1 18 | kind: ClusterRole 19 | metadata: 20 | name: proxy-role 21 | rules: 22 | - apiGroups: ["authentication.k8s.io"] 23 | resources: 24 | - tokenreviews 25 | verbs: ["create"] 26 | - apiGroups: ["authorization.k8s.io"] 27 | resources: 28 | - subjectaccessreviews 29 | verbs: ["create"] 30 | -------------------------------------------------------------------------------- /controllers/application/application_suite_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package application_test 17 | 18 | import ( 19 | . "github.com/onsi/ginkgo" 20 | . "github.com/onsi/gomega" 21 | "testing" 22 | ) 23 | 24 | // TestEventhandler exports tests 25 | func TestEventhandler(t *testing.T) { 26 | RegisterFailHandler(Fail) 27 | RunSpecsWithDefaultAndCustomReporters(t, "Application Suite", []Reporter{}) 28 | } 29 | -------------------------------------------------------------------------------- /cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Google Cloud Build script 17 | 18 | --- 19 | steps: 20 | - name: 'gcr.io/cloud-builders/docker' 21 | args: ['build', '.', '-t', 'gcr.io/airflow-operator/airflow-operator:v1alpha2', '-t', 'gcr.io/airflow-operator/airflow-operator:$REVISION_ID', '-f', 'Dockerfile'] 22 | 23 | images: ['gcr.io/airflow-operator/airflow-operator:v1alpha2', 'gcr.io/airflow-operator/airflow-operator:$REVISION_ID'] 24 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/status/status_suite_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package status_test 17 | 18 | import ( 19 | . "github.com/onsi/ginkgo" 20 | . "github.com/onsi/gomega" 21 | "testing" 22 | ) 23 | 24 | // TestEventhandler exports tests 25 | func TestEventhandler(t *testing.T) { 26 | RegisterFailHandler(Fail) 27 | RunSpecsWithDefaultAndCustomReporters(t, "Status Suite", []Reporter{}) 28 | } 29 | -------------------------------------------------------------------------------- /templates/pdb.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: policy/v1beta1 18 | kind: PodDisruptionBudget 19 | metadata: 20 | name: {{.Name}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | spec: 27 | minAvailable: {{.PDBMinAvail}} 28 | selector: 29 | matchLabels: 30 | {{range $k,$v := .Selector }} 31 | {{$k}}: {{$v}} 32 | {{end}} 33 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/storage/storage_suite_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package storage_test 17 | 18 | import ( 19 | . "github.com/onsi/ginkgo" 20 | . "github.com/onsi/gomega" 21 | "testing" 22 | ) 23 | 24 | // TestEventhandler exports tests 25 | func TestEventhandler(t *testing.T) { 26 | RegisterFailHandler(Fail) 27 | RunSpecsWithDefaultAndCustomReporters(t, "Status Suite", []Reporter{}) 28 | } 29 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Prometheus Monitor Service (Metrics) 17 | --- 18 | apiVersion: monitoring.coreos.com/v1 19 | kind: ServiceMonitor 20 | metadata: 21 | labels: 22 | control-plane: controller-manager 23 | name: controller-manager-metrics-monitor 24 | namespace: system 25 | spec: 26 | endpoints: 27 | - path: /metrics 28 | port: https 29 | selector: 30 | control-plane: controller-manager 31 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/finalizer/finalizer_suite_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package finalizer_test 17 | 18 | import ( 19 | . "github.com/onsi/ginkgo" 20 | . "github.com/onsi/gomega" 21 | "testing" 22 | ) 23 | 24 | // TestEventhandler exports tests 25 | func TestEventhandler(t *testing.T) { 26 | RegisterFailHandler(Fail) 27 | RunSpecsWithDefaultAndCustomReporters(t, "Status Suite", []Reporter{}) 28 | } 29 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/resource_suite_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package reconciler_test 17 | 18 | import ( 19 | . "github.com/onsi/ginkgo" 20 | . "github.com/onsi/gomega" 21 | "testing" 22 | ) 23 | 24 | // TestEventhandler exports tests 25 | func TestEventhandler(t *testing.T) { 26 | RegisterFailHandler(Fail) 27 | RunSpecsWithDefaultAndCustomReporters(t, "Resource Suite", []Reporter{}) 28 | } 29 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_airflowbases.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # The following patch adds a directive for certmanager to inject CA into the CRD 17 | # CRD conversion requires k8s 1.13 or later. 18 | --- 19 | apiVersion: apiextensions.k8s.io/v1beta1 20 | kind: CustomResourceDefinition 21 | metadata: 22 | annotations: 23 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 24 | name: airflowbases.airflow.apache.org 25 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_airflowclusters.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # The following patch adds a directive for certmanager to inject CA into the CRD 17 | # CRD conversion requires k8s 1.13 or later. 18 | --- 19 | apiVersion: apiextensions.k8s.io/v1beta1 20 | kind: CustomResourceDefinition 21 | metadata: 22 | annotations: 23 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 24 | name: airflowclusters.airflow.apache.org 25 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | resources: 18 | - role.yaml 19 | - role_binding.yaml 20 | - leader_election_role.yaml 21 | - leader_election_role_binding.yaml 22 | # Comment the following 3 lines if you want to disable 23 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 24 | # which protects your /metrics endpoint. 25 | - auth_proxy_service.yaml 26 | - auth_proxy_role.yaml 27 | - auth_proxy_role_binding.yaml 28 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/genericreconciler/genericreconciler_suite_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package genericreconciler_test 17 | 18 | import ( 19 | . "github.com/onsi/ginkgo" 20 | . "github.com/onsi/gomega" 21 | "testing" 22 | ) 23 | 24 | // TestEventhandler exports tests 25 | func TestEventhandler(t *testing.T) { 26 | RegisterFailHandler(Fail) 27 | RunSpecsWithDefaultAndCustomReporters(t, "Application Suite", []Reporter{}) 28 | } 29 | -------------------------------------------------------------------------------- /templates/headlesssvc.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: {{.SvcName}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | spec: 27 | ports: 28 | {{range $k,$v := .Ports }} 29 | - name: {{$k}} 30 | port: {{$v}} 31 | {{end}} 32 | selector: 33 | {{range $k,$v := .Selector }} 34 | {{$k}}: {{$v}} 35 | {{end}} 36 | clusterIP: None 37 | -------------------------------------------------------------------------------- /templates/svc.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: {{.SvcName}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | spec: 27 | ports: 28 | {{range $k,$v := .Ports }} 29 | - name: {{$k}} 30 | port: {{$v}} 31 | {{end}} 32 | selector: 33 | {{range $k,$v := .Selector }} 34 | {{$k}}: {{$v}} 35 | {{end}} 36 | type: ClusterIP # default can drop 37 | -------------------------------------------------------------------------------- /config/rbac/airflowbase_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # permissions to do viewer airflowbases. 17 | --- 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | kind: ClusterRole 20 | metadata: 21 | name: airflowbase-viewer-role 22 | rules: 23 | - apiGroups: 24 | - airflow.apache.org 25 | resources: 26 | - airflowbases 27 | verbs: 28 | - get 29 | - list 30 | - watch 31 | - apiGroups: 32 | - airflow.apache.org 33 | resources: 34 | - airflowbases/status 35 | verbs: 36 | - get 37 | -------------------------------------------------------------------------------- /templates/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: rbac.authorization.k8s.io/v1 18 | kind: RoleBinding 19 | metadata: 20 | name: {{.Name}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | roleRef: 27 | apiGroup: rbac.authorization.k8s.io 28 | kind: ClusterRole 29 | # TODO - restrict this to manager-role with pod creation 30 | name: cluster-admin 31 | subjects: 32 | - kind: ServiceAccount 33 | name: {{.Name}} 34 | namespace: {{.Namespace}} 35 | -------------------------------------------------------------------------------- /config/certmanager/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # This configuration is for teaching kustomize how to update name ref and var substitution 17 | --- 18 | nameReference: 19 | - kind: Issuer 20 | group: cert-manager.io 21 | fieldSpecs: 22 | - kind: Certificate 23 | group: cert-manager.io 24 | path: spec/issuerRef/name 25 | 26 | varReference: 27 | - kind: Certificate 28 | group: cert-manager.io 29 | path: spec/commonName 30 | - kind: Certificate 31 | group: cert-manager.io 32 | path: spec/dnsNames 33 | -------------------------------------------------------------------------------- /config/rbac/airflowcluster_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # permissions to do viewer airflowclusters. 17 | --- 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | kind: ClusterRole 20 | metadata: 21 | name: airflowcluster-viewer-role 22 | rules: 23 | - apiGroups: 24 | - airflow.apache.org 25 | resources: 26 | - airflowclusters 27 | verbs: 28 | - get 29 | - list 30 | - watch 31 | - apiGroups: 32 | - airflow.apache.org 33 | resources: 34 | - airflowclusters/status 35 | verbs: 36 | - get 37 | -------------------------------------------------------------------------------- /hack/sample/cloudsql-k8s/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: ck-cluster 21 | spec: 22 | executor: Kubernetes 23 | ui: 24 | replicas: 1 25 | version: "1.10.2" 26 | scheduler: 27 | version: "1.10.2" 28 | worker: 29 | version: "1.10.2" 30 | dags: 31 | subdir: "airflow/example_dags/" 32 | git: 33 | repo: "https://github.com/apache/airflow/" 34 | once: true 35 | branch: master 36 | airflowbase: 37 | name: cc-base 38 | -------------------------------------------------------------------------------- /hack/sample/postgres-k8s/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: pk-cluster 21 | spec: 22 | executor: Kubernetes 23 | ui: 24 | replicas: 1 25 | version: "1.10.2" 26 | scheduler: 27 | version: "1.10.2" 28 | worker: 29 | version: "1.10.2" 30 | dags: 31 | subdir: "airflow/example_dags/" 32 | git: 33 | repo: "https://github.com/apache/airflow/" 34 | once: true 35 | branch: master 36 | airflowbase: 37 | name: pc-base 38 | -------------------------------------------------------------------------------- /hack/sample/mysql-celery-gcs/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: mcg-cluster 21 | spec: 22 | executor: Celery 23 | redis: 24 | operator: false 25 | scheduler: 26 | version: "1.10.2" 27 | ui: 28 | replicas: 1 29 | version: "1.10.2" 30 | flower: 31 | replicas: 1 32 | version: "1.10.2" 33 | worker: 34 | replicas: 2 35 | version: "1.10.2" 36 | dags: 37 | subdir: "" 38 | gcs: 39 | bucket: "mydags" 40 | airflowbase: 41 | name: mc-base 42 | -------------------------------------------------------------------------------- /hack/sample/mysql-local/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: ml-cluster 21 | spec: 22 | executor: Local 23 | scheduler: 24 | version: "1.10.2" 25 | ui: 26 | replicas: 1 27 | version: "1.10.2" 28 | airflowbase: 29 | name: mc-base 30 | # config: 31 | # airflow: 32 | # AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "100" 33 | # AIRFLOW__WEBSERVER__AUTHENTICATE: "True" 34 | # AIRFLOW__WEBSERVER__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth" 35 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 17 | --- 18 | nameReference: 19 | - kind: Service 20 | version: v1 21 | fieldSpecs: 22 | - kind: CustomResourceDefinition 23 | group: apiextensions.k8s.io 24 | path: spec/conversion/webhookClientConfig/service/name 25 | 26 | namespace: 27 | - kind: CustomResourceDefinition 28 | group: apiextensions.k8s.io 29 | path: spec/conversion/webhookClientConfig/service/namespace 30 | create: false 31 | 32 | varReference: 33 | - path: metadata/annotations 34 | -------------------------------------------------------------------------------- /hack/sample/mysql-k8s/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: mk-cluster 21 | spec: 22 | executor: Kubernetes 23 | ui: 24 | replicas: 1 25 | version: "1.10.2" 26 | scheduler: 27 | version: "1.10.2" 28 | worker: 29 | # image: "gcr.io/airflow-development-225219/airflow" 30 | # version: "demo-nr4" 31 | version: "1.10.2" 32 | dags: 33 | subdir: "airflow/example_dags/" 34 | git: 35 | repo: "https://github.com/apache/airflow/" 36 | once: true 37 | branch: master 38 | airflowbase: 39 | name: mc-base 40 | -------------------------------------------------------------------------------- /config/rbac/airflowbase_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # permissions to do edit airflowbases. 17 | --- 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | kind: ClusterRole 20 | metadata: 21 | name: airflowbase-editor-role 22 | rules: 23 | - apiGroups: 24 | - airflow.apache.org 25 | resources: 26 | - airflowbases 27 | verbs: 28 | - create 29 | - delete 30 | - get 31 | - list 32 | - patch 33 | - update 34 | - watch 35 | - apiGroups: 36 | - airflow.apache.org 37 | resources: 38 | - airflowbases/status 39 | verbs: 40 | - get 41 | - patch 42 | - update 43 | -------------------------------------------------------------------------------- /hack/sample/cloudsql-celery/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: cc-cluster 21 | spec: 22 | executor: Celery 23 | redis: 24 | operator: false 25 | scheduler: 26 | version: "1.10.2" 27 | ui: 28 | replicas: 1 29 | version: "1.10.2" 30 | worker: 31 | replicas: 2 32 | version: "1.10.2" 33 | flower: 34 | replicas: 1 35 | version: "1.10.2" 36 | dags: 37 | subdir: "airflow/example_dags/" 38 | git: 39 | repo: "https://github.com/apache/airflow/" 40 | once: true 41 | airflowbase: 42 | name: cc-base 43 | -------------------------------------------------------------------------------- /hack/sample/postgres-celery/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: pc-cluster 21 | spec: 22 | executor: Celery 23 | redis: 24 | operator: false 25 | scheduler: 26 | version: "1.10.2" 27 | ui: 28 | replicas: 1 29 | version: "1.10.2" 30 | worker: 31 | replicas: 2 32 | version: "1.10.2" 33 | flower: 34 | replicas: 1 35 | version: "1.10.2" 36 | dags: 37 | subdir: "airflow/example_dags/" 38 | git: 39 | repo: "https://github.com/apache/airflow/" 40 | once: true 41 | airflowbase: 42 | name: pc-base 43 | -------------------------------------------------------------------------------- /config/rbac/airflowcluster_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # permissions to do edit airflowclusters. 17 | --- 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | kind: ClusterRole 20 | metadata: 21 | name: airflowcluster-editor-role 22 | rules: 23 | - apiGroups: 24 | - airflow.apache.org 25 | resources: 26 | - airflowclusters 27 | verbs: 28 | - create 29 | - delete 30 | - get 31 | - list 32 | - patch 33 | - update 34 | - watch 35 | - apiGroups: 36 | - airflow.apache.org 37 | resources: 38 | - airflowclusters/status 39 | verbs: 40 | - get 41 | - patch 42 | - update 43 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/genericreconciler/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package genericreconciler 15 | 16 | import ( 17 | "k8s.io/apimachinery/pkg/runtime" 18 | "k8s.io/apimachinery/pkg/runtime/schema" 19 | rm "sigs.k8s.io/controller-reconciler/pkg/reconciler/manager" 20 | "sigs.k8s.io/controller-runtime/pkg/manager" 21 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 22 | ) 23 | 24 | var _ reconcile.Reconciler = &Reconciler{} 25 | 26 | // Reconciler defines fields needed for all airflow controllers 27 | // +k8s:deepcopy-gen=false 28 | type Reconciler struct { 29 | gv schema.GroupVersion 30 | errorHandler func(interface{}, error, string) 31 | validate func(interface{}) error 32 | applyDefaults func(interface{}) 33 | resource runtime.Object 34 | manager manager.Manager 35 | rsrcMgr rm.ResourceManager 36 | using []Handler 37 | } 38 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/interface.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package manager 17 | 18 | import ( 19 | "sigs.k8s.io/controller-reconciler/pkg/reconciler" 20 | ) 21 | 22 | // Manager is an interface for operating on CRs 23 | type Manager interface { 24 | Observe(observables ...reconciler.Observable) ([]reconciler.Object, error) 25 | Update(item reconciler.Object) error 26 | Create(item reconciler.Object) error 27 | Delete(item reconciler.Object) error 28 | SpecDiffers(expected, observed *reconciler.Object) bool 29 | ObservablesFromObjects(bag []reconciler.Object, labels map[string]string) []reconciler.Observable 30 | } 31 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/internal.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package manager 17 | 18 | import () 19 | 20 | // Add Register resource manager 21 | func (rm *ResourceManager) Add(rsrcType string, m Manager) { 22 | if rm.rsrcmgrs == nil { 23 | rm.rsrcmgrs = make(map[string]Manager) 24 | } 25 | rm.rsrcmgrs[rsrcType] = m 26 | } 27 | 28 | // Get resource manager 29 | func (rm *ResourceManager) Get(rsrcType string) Manager { 30 | m, ok := rm.rsrcmgrs[rsrcType] 31 | if ok { 32 | return m 33 | } 34 | return nil 35 | } 36 | 37 | // All returns all managers 38 | func (rm *ResourceManager) All() map[string]Manager { 39 | return rm.rsrcmgrs 40 | } 41 | -------------------------------------------------------------------------------- /config/default/manager_webhook_patch.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: controller-manager 21 | namespace: system 22 | spec: 23 | template: 24 | spec: 25 | containers: 26 | - name: manager 27 | ports: 28 | - containerPort: 9443 29 | name: webhook-server 30 | protocol: TCP 31 | volumeMounts: 32 | - mountPath: /tmp/k8s-webhook-server/serving-certs 33 | name: cert 34 | readOnly: true 35 | volumes: 36 | - name: cert 37 | secret: 38 | defaultMode: 420 39 | secretName: webhook-server-cert 40 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # permissions to do leader election. 17 | --- 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | kind: Role 20 | metadata: 21 | name: leader-election-role 22 | rules: 23 | - apiGroups: 24 | - "" 25 | resources: 26 | - configmaps 27 | verbs: 28 | - get 29 | - list 30 | - watch 31 | - create 32 | - update 33 | - patch 34 | - delete 35 | - apiGroups: 36 | - "" 37 | resources: 38 | - configmaps/status 39 | verbs: 40 | - get 41 | - update 42 | - patch 43 | - apiGroups: 44 | - "" 45 | resources: 46 | - events 47 | verbs: 48 | - create 49 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/k8s/patch/patch_suite_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package patch_test 17 | 18 | import ( 19 | "testing" 20 | 21 | . "github.com/onsi/ginkgo" 22 | . "github.com/onsi/gomega" 23 | "sigs.k8s.io/controller-runtime/pkg/envtest" 24 | ) 25 | 26 | func TestSource(t *testing.T) { 27 | RegisterFailHandler(Fail) 28 | RunSpecsWithDefaultAndCustomReporters(t, "Patch Suite", []Reporter{envtest.NewlineReporter{}}) 29 | } 30 | 31 | var testenv *envtest.Environment 32 | 33 | var _ = BeforeSuite(func(done Done) { 34 | testenv = &envtest.Environment{} 35 | close(done) 36 | }, 60) 37 | 38 | var _ = AfterSuite(func() { 39 | testenv.Stop() 40 | }) 41 | -------------------------------------------------------------------------------- /hack/sample/postgres-celery-redis/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: pc-cluster 21 | spec: 22 | executor: Celery 23 | redis: 24 | operator: false 25 | redisHost: "redis" 26 | redisPassword: true 27 | scheduler: 28 | version: "1.10.2" 29 | ui: 30 | replicas: 1 31 | version: "1.10.2" 32 | worker: 33 | replicas: 2 34 | version: "1.10.2" 35 | flower: 36 | replicas: 1 37 | version: "1.10.2" 38 | dags: 39 | subdir: "airflow/example_dags/" 40 | git: 41 | repo: "https://github.com/apache/airflow/" 42 | once: true 43 | airflowbase: 44 | name: pc-base 45 | -------------------------------------------------------------------------------- /hack/sample/mysql-celery/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: mc-cluster 21 | spec: 22 | executor: Celery 23 | config: 24 | airflow: 25 | AIRFLOW_SOME_CONFIG: SomeValue 26 | redis: 27 | operator: false 28 | scheduler: 29 | version: "1.10.2" 30 | ui: 31 | replicas: 1 32 | version: "1.10.2" 33 | worker: 34 | replicas: 2 35 | version: "1.10.2" 36 | flower: 37 | replicas: 1 38 | version: "1.10.2" 39 | dags: 40 | subdir: "airflow/example_dags/" 41 | git: 42 | repo: "https://github.com/apache/airflow/" 43 | once: true 44 | airflowbase: 45 | name: mc-base 46 | -------------------------------------------------------------------------------- /config/default/webhookcainjection_patch.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # This patch add annotation to admission webhook config and 17 | # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. 18 | 19 | --- 20 | apiVersion: admissionregistration.k8s.io/v1beta1 21 | kind: MutatingWebhookConfiguration 22 | metadata: 23 | name: mutating-webhook-configuration 24 | annotations: 25 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 26 | 27 | --- 28 | apiVersion: admissionregistration.k8s.io/v1beta1 29 | kind: ValidatingWebhookConfiguration 30 | metadata: 31 | name: validating-webhook-configuration 32 | annotations: 33 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 34 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_airflowbases.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # The following patch enables conversion webhook for CRD 17 | # CRD conversion requires k8s 1.13 or later. 18 | --- 19 | apiVersion: apiextensions.k8s.io/v1beta1 20 | kind: CustomResourceDefinition 21 | metadata: 22 | name: airflowbases.airflow.apache.org 23 | spec: 24 | conversion: 25 | strategy: Webhook 26 | webhookClientConfig: 27 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 28 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 29 | caBundle: Cg== 30 | service: 31 | namespace: system 32 | name: webhook-service 33 | path: /convert 34 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_airflowclusters.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # The following patch enables conversion webhook for CRD 17 | # CRD conversion requires k8s 1.13 or later. 18 | --- 19 | apiVersion: apiextensions.k8s.io/v1beta1 20 | kind: CustomResourceDefinition 21 | metadata: 22 | name: airflowclusters.airflow.apache.org 23 | spec: 24 | conversion: 25 | strategy: Webhook 26 | webhookClientConfig: 27 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 28 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 29 | caBundle: Cg== 30 | service: 31 | namespace: system 32 | name: webhook-service 33 | path: /convert 34 | -------------------------------------------------------------------------------- /hack/sample/postgres-celery-memorystore/cluster.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: airflow.apache.org/v1alpha1 18 | kind: AirflowCluster 19 | metadata: 20 | name: pc-cluster 21 | spec: 22 | executor: Celery 23 | memoryStore: 24 | project: "project-id" 25 | region: "region-id" 26 | memorySizeGb: 1 27 | tier: "basic" 28 | maxMemoryPolicy: "allkeys-lru" 29 | scheduler: 30 | version: "1.10.2" 31 | ui: 32 | replicas: 1 33 | version: "1.10.2" 34 | worker: 35 | replicas: 2 36 | version: "1.10.2" 37 | flower: 38 | replicas: 1 39 | version: "1.10.2" 40 | dags: 41 | subdir: "airflow/example_dags/" 42 | git: 43 | repo: "https://github.com/apache/airflow/" 44 | once: true 45 | airflowbase: 46 | name: pc-base 47 | -------------------------------------------------------------------------------- /api/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // Package v1alpha1 contains API Schema definitions for the airflow v1alpha1 API group 17 | // +kubebuilder:object:generate=true 18 | // +groupName=airflow.apache.org 19 | package v1alpha1 20 | 21 | import ( 22 | "k8s.io/apimachinery/pkg/runtime/schema" 23 | "sigs.k8s.io/controller-runtime/pkg/scheme" 24 | ) 25 | 26 | var ( 27 | // GroupVersion is group version used to register these objects 28 | GroupVersion = schema.GroupVersion{Group: "airflow.apache.org", Version: "v1alpha1"} 29 | 30 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 31 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 32 | 33 | // AddToScheme adds the types in this group-version to the given scheme. 34 | AddToScheme = SchemeBuilder.AddToScheme 35 | ) 36 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/storage/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package storage 15 | 16 | import ( 17 | corev1 "k8s.io/api/core/v1" 18 | ) 19 | 20 | // const fileds 21 | const ( 22 | FieldGCS = "gcs" 23 | FieldNFS = "nfs" 24 | FieldFS = "fs" 25 | 26 | Optional = "optional" 27 | Expected = "expected" 28 | ) 29 | 30 | // Spec is a generic cwspec to define a storage destination 31 | // +k8s:deepcopy-gen=true 32 | type Spec struct { 33 | GCS *GCS `json:"gcs,omitempty"` 34 | NFS *corev1.NFSVolumeSource `json:"nfs,omitempty"` 35 | FS *FS `json:"fs,omitempty"` 36 | } 37 | 38 | // GCS hold gcs related fields 39 | type GCS struct { 40 | // Bucket name 41 | Bucket string `json:"bucket,omitempty"` 42 | // Secret to access bucket 43 | Secret *corev1.SecretReference `json:"secret,omitempty"` 44 | // ReadOnly bool 45 | ReadOnly bool `json:"readonly,omitempty"` 46 | // Path name 47 | Path string `json:"path,omitempty"` 48 | } 49 | 50 | // FS hold filesystem related fields 51 | type FS struct { 52 | // Path name 53 | Path string `json:"path,omitempty"` 54 | } 55 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Build the manager binary 17 | FROM golang:1.13 as builder 18 | 19 | WORKDIR /workspace 20 | COPY vendor/ vendor/ 21 | # Copy the Go Modules manifests 22 | COPY go.mod go.mod 23 | COPY go.sum go.sum 24 | # cache deps before building and copying source so that we don't need to re-download as much 25 | # and so that source changes don't invalidate our downloaded layer 26 | RUN go mod download 27 | 28 | # Copy the go source 29 | COPY main.go main.go 30 | COPY api/ api/ 31 | COPY controllers/ controllers/ 32 | 33 | # Build 34 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go 35 | 36 | # Use distroless as minimal base image to package the manager binary 37 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 38 | FROM gcr.io/distroless/static:nonroot 39 | WORKDIR / 40 | COPY --from=builder /workspace/manager . 41 | USER nonroot:nonroot 42 | 43 | ENTRYPOINT ["/manager"] 44 | -------------------------------------------------------------------------------- /config/default/manager_auth_proxy_patch.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # This patch inject a sidecar container which is a HTTP proxy for the controller manager, 17 | # it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. 18 | 19 | --- 20 | apiVersion: apps/v1 21 | kind: Deployment 22 | metadata: 23 | name: controller-manager 24 | namespace: system 25 | spec: 26 | template: 27 | spec: 28 | containers: 29 | - name: kube-rbac-proxy 30 | image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1 31 | args: 32 | - "--secure-listen-address=0.0.0.0:8443" 33 | - "--upstream=http://127.0.0.1:8080/" 34 | - "--logtostderr=true" 35 | - "--v=10" 36 | ports: 37 | - containerPort: 8443 38 | name: https 39 | - name: manager 40 | args: 41 | - "--metrics-addr=127.0.0.1:8080" 42 | - "--enable-leader-election" 43 | -------------------------------------------------------------------------------- /config/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # the following config is for teaching kustomize where to look at when substituting vars. 17 | # It requires kustomize v2.1.0 or newer to work properly. 18 | --- 19 | nameReference: 20 | - kind: Service 21 | version: v1 22 | fieldSpecs: 23 | - kind: MutatingWebhookConfiguration 24 | group: admissionregistration.k8s.io 25 | path: webhooks/clientConfig/service/name 26 | - kind: ValidatingWebhookConfiguration 27 | group: admissionregistration.k8s.io 28 | path: webhooks/clientConfig/service/name 29 | 30 | namespace: 31 | - kind: MutatingWebhookConfiguration 32 | group: admissionregistration.k8s.io 33 | path: webhooks/clientConfig/service/namespace 34 | create: true 35 | - kind: ValidatingWebhookConfiguration 36 | group: admissionregistration.k8s.io 37 | path: webhooks/clientConfig/service/namespace 38 | create: true 39 | 40 | varReference: 41 | - path: metadata/annotations 42 | -------------------------------------------------------------------------------- /docs/userguide.md: -------------------------------------------------------------------------------- 1 | 17 | 18 | # User Guide 19 | 20 | TODO 21 | 22 | # FAQs 23 | 24 | 1. How do we refresh DAGs ? 25 | Canonical way airflow supports refreshing DAGs is via `dag_dir_list_interval` config. 26 | https://cwiki.apache.org/confluence/display/AIRFLOW/Scheduler+Basics#Configuration 27 | You can set that config using `cluster.spec.config.airflow` 28 | Set the env `AIRFLOW__SCHEDULER__ DAG_DIR_LIST_INTERVAL` 29 | By default dags are refreshed every 5 minutes. 30 | To enable continuous sync, use git or gcs dag source with once disabled. 31 | 32 | ```yaml 33 | apiVersion: airflow.apache.org/v1alpha1 34 | kind: AirflowCluster 35 | ... 36 | spec: 37 | ... 38 | config: 39 | airflow: 40 | AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 100 # default is 300s 41 | ... 42 | dags: 43 | subdir: "" 44 | gcs: 45 | bucket: "mydags" 46 | # OR 47 | dags: 48 | subdir: "airflow/example_dags/" 49 | git: 50 | repo: "https://github.com/apache/incubator-airflow/" 51 | once: false 52 | ``` 53 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: v1 18 | kind: Namespace 19 | metadata: 20 | labels: 21 | control-plane: controller-manager 22 | name: system 23 | 24 | --- 25 | apiVersion: apps/v1 26 | kind: Deployment 27 | metadata: 28 | name: controller-manager 29 | namespace: system 30 | labels: 31 | control-plane: controller-manager 32 | spec: 33 | selector: 34 | matchLabels: 35 | control-plane: controller-manager 36 | replicas: 1 37 | template: 38 | metadata: 39 | labels: 40 | control-plane: controller-manager 41 | spec: 42 | containers: 43 | - command: 44 | - /manager 45 | args: 46 | - --enable-leader-election 47 | image: controller:latest 48 | name: manager 49 | resources: 50 | limits: 51 | cpu: 100m 52 | memory: 30Mi 53 | requests: 54 | cpu: 100m 55 | memory: 20Mi 56 | terminationGracePeriodSeconds: 10 57 | -------------------------------------------------------------------------------- /config/certmanager/certificate.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # The following manifests contain a self-signed issuer CR and a certificate CR. 17 | # More document can be found at https://docs.cert-manager.io 18 | # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes 19 | 20 | --- 21 | apiVersion: cert-manager.io/v1alpha2 22 | kind: Issuer 23 | metadata: 24 | name: selfsigned-issuer 25 | namespace: system 26 | spec: 27 | selfSigned: {} 28 | 29 | --- 30 | apiVersion: cert-manager.io/v1alpha2 31 | kind: Certificate 32 | metadata: 33 | name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml 34 | namespace: system 35 | spec: 36 | # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize 37 | dnsNames: 38 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc 39 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local 40 | issuerRef: 41 | kind: Issuer 42 | name: selfsigned-issuer 43 | secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize 44 | -------------------------------------------------------------------------------- /hack/sample/postgres-celery-redis/redis.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: redis 21 | spec: 22 | ports: 23 | - port: 6379 24 | name: redis 25 | selector: 26 | app: redis 27 | --- 28 | apiVersion: apps/v1beta2 29 | kind: StatefulSet 30 | metadata: 31 | name: redis 32 | spec: 33 | selector: 34 | matchLabels: 35 | app: redis # has to match .spec.template.metadata.labels 36 | serviceName: redis 37 | replicas: 1 38 | template: 39 | metadata: 40 | labels: 41 | app: redis # has to match .spec.selector.matchLabels 42 | spec: 43 | containers: 44 | - name: redis 45 | image: redis:4.0 46 | imagePullPolicy: Always 47 | args: 48 | - --requirepass 49 | - $(REDIS_PASSWORD) 50 | env: 51 | - name: REDIS_PASSWORD 52 | valueFrom: 53 | secretKeyRef: 54 | key: password 55 | name: pc-cluster-redis 56 | ports: 57 | - containerPort: 6379 58 | name: redis 59 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/testdata/sts.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: {{.Name}} 5 | namespace: {{.Namespace}} 6 | labels: 7 | {{range $k,$v := .Labels }} 8 | {{$k}}: {{$v}} 9 | {{end}} 10 | spec: 11 | serviceName: {{.Name}} 12 | # go template syntax replicas: with $n := index .Spec.NodeGroups 0 $n.Replicas end 13 | replicas: {{.Replicas}} 14 | selector: 15 | matchLabels: 16 | {{range $k,$v := .Selector }} 17 | {{$k}}: {{$v}} 18 | {{end}} 19 | updateStrategy: 20 | # https://www.elastic.co/guide/en/elasticsearch/reference/current/rolling-upgrades.html 21 | type: OnDelete 22 | podManagementPolicy: OrderedReady 23 | template: 24 | metadata: 25 | labels: 26 | {{range $k,$v := .Labels }} 27 | {{$k}}: {{$v}} 28 | {{end}} 29 | spec: 30 | containers: 31 | - name: foobar 32 | image: {{.Image}}:{{.Version}} 33 | ports: 34 | - containerPort: 8080 35 | name: http 36 | env: 37 | - name: NODENAME 38 | valueFrom: 39 | fieldRef: 40 | fieldPath: metadata.name 41 | - name: PROCESSORS 42 | valueFrom: 43 | resourceFieldRef: 44 | resource: limits.cpu 45 | readinessProbe: 46 | httpGet: 47 | path: /health?local=true 48 | port: 8080 49 | initialDelaySeconds: 5 50 | volumeMounts: 51 | - name: config 52 | mountPath: /etc/config/config.yml 53 | volumes: 54 | - name: config 55 | configMap: 56 | name: {{.Name}} 57 | volumeClaimTemplates: 58 | - metadata: 59 | name: data 60 | labels: 61 | {{range $k,$v := .Labels }} 62 | {{$k}}: {{$v}} 63 | {{end}} 64 | spec: 65 | accessModes: 66 | - ReadWriteOnce 67 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # This kustomization.yaml is not intended to be run by itself, 17 | # since it depends on service name and namespace that are out of this kustomize package. 18 | # It should be run by config/default 19 | --- 20 | resources: 21 | - bases/airflow.apache.org_airflowbases.yaml 22 | - bases/airflow.apache.org_airflowclusters.yaml 23 | # +kubebuilder:scaffold:crdkustomizeresource 24 | 25 | patchesStrategicMerge: 26 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 27 | # patches here are for enabling the conversion webhook for each CRD 28 | # - patches/webhook_in_airflowbases.yaml 29 | # - patches/webhook_in_airflowclusters.yaml 30 | # +kubebuilder:scaffold:crdkustomizewebhookpatch 31 | 32 | # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. 33 | # patches here are for enabling the CA injection for each CRD 34 | # - patches/cainjection_in_airflowbases.yaml 35 | # - patches/cainjection_in_airflowclusters.yaml 36 | # +kubebuilder:scaffold:crdkustomizecainjectionpatch 37 | 38 | # the following config is for teaching kustomize how to do kustomization for CRDs. 39 | configurations: 40 | - kustomizeconfig.yaml 41 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/test/helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package test 18 | 19 | import ( 20 | corev1 "k8s.io/api/core/v1" 21 | "k8s.io/apimachinery/pkg/api/resource" 22 | "os" 23 | "path/filepath" 24 | "strconv" 25 | ) 26 | 27 | // OpenFile - open file 28 | func OpenFile(path string) (*os.File, error) { 29 | path, err := filepath.Abs(path) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | f, err := os.Open(path) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | return f, nil 40 | } 41 | 42 | // AdjustCPU - increase/decrease cpu 43 | func AdjustCPU(r *corev1.ResourceRequirements, by int64) { 44 | newValue := r.Requests.Cpu().MilliValue() + by 45 | r.Requests[corev1.ResourceCPU] = *resource.NewMilliQuantity(newValue, resource.BinarySI) 46 | } 47 | 48 | // AddMemoryGB - increase Memory 49 | func AddMemoryGB(r *corev1.ResourceRequirements, req int, limit int) { 50 | mem := r.Requests[corev1.ResourceMemory] 51 | mmem := &mem 52 | add, _ := resource.ParseQuantity(strconv.Itoa(req) + "Gi") 53 | mmem.Add(add) 54 | r.Requests[corev1.ResourceMemory] = mem 55 | 56 | mem = r.Limits[corev1.ResourceMemory] 57 | mmem = &mem 58 | add, _ = resource.ParseQuantity(strconv.Itoa(limit) + "Gi") 59 | mmem.Add(add) 60 | r.Limits[corev1.ResourceMemory] = mem 61 | } 62 | 63 | //= corev1.ResourceList{ 64 | //corev1.ResourceCPU: *resource.NewMilliQuantity(newValue, resource.BinarySI), 65 | //corev1.ResourceMemory: *ng.Resources.Requests.Memory(), 66 | //} 67 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/types.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package reconciler 17 | 18 | import ( 19 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 20 | ) 21 | 22 | // Common const definitions 23 | const ( 24 | LifecycleManaged = "managed" // CRUD 25 | LifecycleReferred = "referred" // R 26 | LifecycleNoUpdate = "noupdate" // CRD 27 | LifecycleDecorate = "decorate" // RU 28 | ) 29 | 30 | // ObjectInterface - 31 | type ObjectInterface interface { 32 | GetName() string 33 | IsSameAs(interface{}) bool 34 | SetOwnerReferences(*metav1.OwnerReference) bool 35 | SetLabels(labels map[string]string) 36 | } 37 | 38 | // Object is a container to capture the k8s resource info to be used by controller 39 | type Object struct { 40 | // Lifecycle can be: managed, reference 41 | Lifecycle string 42 | // Type - object type 43 | Type string 44 | // Obj - object 45 | Obj ObjectInterface 46 | // Delete - marker for deletion 47 | Delete bool 48 | // Update - marker for update 49 | Update bool 50 | } 51 | 52 | // Observable captures the k8s resource info and selector to fetch child resources 53 | type Observable struct { 54 | // Type - object type 55 | Type string 56 | // Obj - object 57 | Obj interface{} 58 | } 59 | 60 | // KVMap is a map[string]string 61 | type KVMap map[string]string 62 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/finalizer/finalizer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package finalizer 15 | 16 | import ( 17 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 18 | ) 19 | 20 | // const fileds 21 | const ( 22 | Cleanup = "sigapps.k8s.io/cleanup" 23 | ) 24 | 25 | // Exists adds the finalizer string 26 | func Exists(o metav1.Object, new string) bool { 27 | exists := false 28 | existing := o.GetFinalizers() 29 | for _, f := range existing { 30 | if f == new { 31 | exists = true 32 | break 33 | } 34 | } 35 | 36 | return exists 37 | } 38 | 39 | // Add adds the finalizer string 40 | func Add(o metav1.Object, new string) { 41 | exists := false 42 | existing := o.GetFinalizers() 43 | for _, f := range existing { 44 | if f == new { 45 | exists = true 46 | break 47 | } 48 | } 49 | 50 | if !exists { 51 | existing = append(existing, new) 52 | o.SetFinalizers(existing) 53 | } 54 | } 55 | 56 | // Remove removes the finalizer string 57 | func Remove(o metav1.Object, new string) { 58 | foundat := -1 59 | existing := o.GetFinalizers() 60 | for i, f := range existing { 61 | if f == new { 62 | foundat = i 63 | break 64 | } 65 | } 66 | if foundat != -1 { 67 | existing[foundat] = existing[len(existing)-1] 68 | o.SetFinalizers(existing[:len(existing)-1]) 69 | } 70 | } 71 | 72 | // EnsureStandard adds standard finalizers 73 | func EnsureStandard(o metav1.Object) { 74 | Add(o, Cleanup) 75 | } 76 | 77 | // RemoveStandard removes standard finalizers 78 | func RemoveStandard(o metav1.Object) { 79 | Remove(o, Cleanup) 80 | } 81 | -------------------------------------------------------------------------------- /templates/flower-sts.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: {{.Name}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | annotations: 27 | {{range $k,$v := .Cluster.Spec.Annotations }} 28 | {{$k}}: {{$v}} 29 | {{end}} 30 | spec: 31 | replicas: {{.Cluster.Spec.Flower.Replicas}} 32 | selector: 33 | matchLabels: 34 | {{range $k,$v := .Selector }} 35 | {{$k}}: {{$v}} 36 | {{end}} 37 | updateStrategy: 38 | type: RollingUpdate 39 | podManagementPolicy: Parallel 40 | template: 41 | metadata: 42 | labels: 43 | {{range $k,$v := .Labels }} 44 | {{$k}}: {{$v}} 45 | {{end}} 46 | annotations: 47 | {{range $k,$v := .Cluster.Spec.Annotations }} 48 | {{$k}}: {{$v}} 49 | {{end}} 50 | spec: 51 | terminationGracePeriodSeconds: 30 52 | nodeSelector: 53 | {{range $k,$v := .Cluster.Spec.NodeSelector }} 54 | {{$k}}: {{$v}} 55 | {{end}} 56 | containers: 57 | - name: flower 58 | args: 59 | - flower 60 | image: {{.Cluster.Spec.Flower.Image}}:{{.Cluster.Spec.Flower.Version}} 61 | imagePullPolicy: IfNotPresent 62 | ports: 63 | - containerPort: 5555 64 | name: flower 65 | volumeMounts: 66 | - mountPath: /usr/local/airflow/dags/ 67 | name: dags-data 68 | volumes: 69 | - emptyDir: {} 70 | name: dags-data 71 | -------------------------------------------------------------------------------- /templates/worker-sts.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | apiVersion: apps/v1 17 | kind: StatefulSet 18 | metadata: 19 | name: {{.Name}} 20 | namespace: {{.Namespace}} 21 | labels: 22 | {{range $k,$v := .Labels }} 23 | {{$k}}: {{$v}} 24 | {{end}} 25 | annotations: 26 | {{range $k,$v := .Cluster.Spec.Annotations }} 27 | {{$k}}: {{$v}} 28 | {{end}} 29 | spec: 30 | replicas: {{.Cluster.Spec.Worker.Replicas}} 31 | serviceName: {{.Name}} 32 | selector: 33 | matchLabels: 34 | {{range $k,$v := .Selector }} 35 | {{$k}}: {{$v}} 36 | {{end}} 37 | updateStrategy: 38 | type: RollingUpdate 39 | podManagementPolicy: Parallel 40 | template: 41 | metadata: 42 | labels: 43 | {{range $k,$v := .Labels }} 44 | {{$k}}: {{$v}} 45 | {{end}} 46 | annotations: 47 | {{range $k,$v := .Cluster.Spec.Annotations }} 48 | {{$k}}: {{$v}} 49 | {{end}} 50 | spec: 51 | terminationGracePeriodSeconds: 30 52 | nodeSelector: 53 | {{range $k,$v := .Cluster.Spec.NodeSelector }} 54 | {{$k}}: {{$v}} 55 | {{end}} 56 | containers: 57 | - name: worker 58 | args: 59 | - worker 60 | image: {{.Cluster.Spec.Worker.Image}}:{{.Cluster.Spec.Worker.Version}} 61 | imagePullPolicy: IfNotPresent 62 | ports: 63 | - containerPort: 8793 64 | name: wlog 65 | volumeMounts: 66 | - mountPath: /usr/local/airflow/dags/ 67 | name: dags-data 68 | volumes: 69 | - emptyDir: {} 70 | name: dags-data 71 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/storage/storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package storage 15 | 16 | import ( 17 | "k8s.io/apimachinery/pkg/util/validation/field" 18 | ) 19 | 20 | // Validate validates the storage spec 21 | func (s *Spec) Validate(fp *field.Path, sfield string, errs field.ErrorList, noneok bool, one bool, smap map[string]string) field.ErrorList { 22 | fp = fp.Child(sfield) 23 | if s == nil { 24 | if !noneok { 25 | errs = append(errs, field.Required(fp, "Required storage spec missing.")) 26 | } 27 | return errs 28 | } 29 | 30 | found := 0 31 | if val, ok := smap[FieldGCS]; ok { 32 | if s.GCS != nil { 33 | found++ 34 | } else if val == Expected { 35 | errs = append(errs, field.Required(fp.Child("gcs"), "gcs spec missing")) 36 | } 37 | } else if s.GCS != nil { 38 | errs = append(errs, field.Invalid(fp.Child("gcs"), "", "gcs spec not supported")) 39 | } 40 | 41 | if val, ok := smap[FieldNFS]; ok { 42 | if s.NFS != nil { 43 | found++ 44 | } else if val == Expected { 45 | errs = append(errs, field.Required(fp.Child("nfs"), "nfs spec missing")) 46 | } 47 | } else if s.NFS != nil { 48 | errs = append(errs, field.Invalid(fp.Child("nfs"), "", "nfs spec not supported")) 49 | } 50 | 51 | if val, ok := smap[FieldFS]; ok { 52 | if s.FS != nil { 53 | found++ 54 | } else if val == Expected { 55 | errs = append(errs, field.Required(fp.Child("fs"), "fs spec missing")) 56 | } 57 | } else if s.FS != nil { 58 | errs = append(errs, field.Invalid(fp.Child("fs"), "", "fs spec not supported")) 59 | } 60 | 61 | if found == 0 { 62 | if !noneok { 63 | errs = append(errs, field.Invalid(fp, "", "No storage specs present.")) 64 | } 65 | } else if found > 1 && one { 66 | errs = append(errs, field.Invalid(fp, "", "Expecting only one storage. Multiple present.")) 67 | } 68 | return errs 69 | } 70 | -------------------------------------------------------------------------------- /templates/scheduler-sts.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: {{.Name}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | annotations: 27 | {{range $k,$v := .Cluster.Spec.Annotations }} 28 | {{$k}}: {{$v}} 29 | {{end}} 30 | spec: 31 | replicas: 1 32 | selector: 33 | matchLabels: 34 | {{range $k,$v := .Selector }} 35 | {{$k}}: {{$v}} 36 | {{end}} 37 | updateStrategy: 38 | type: RollingUpdate 39 | podManagementPolicy: Parallel 40 | template: 41 | metadata: 42 | labels: 43 | {{range $k,$v := .Labels }} 44 | {{$k}}: {{$v}} 45 | {{end}} 46 | annotations: 47 | {{range $k,$v := .Cluster.Spec.Annotations }} 48 | {{$k}}: {{$v}} 49 | {{end}} 50 | spec: 51 | terminationGracePeriodSeconds: 30 52 | nodeSelector: 53 | {{range $k,$v := .Cluster.Spec.NodeSelector }} 54 | {{$k}}: {{$v}} 55 | {{end}} 56 | containers: 57 | - name: scheduler 58 | args: 59 | - scheduler 60 | image: {{.Cluster.Spec.Scheduler.Image}}:{{.Cluster.Spec.Scheduler.Version}} 61 | imagePullPolicy: IfNotPresent 62 | volumeMounts: 63 | - mountPath: /usr/local/airflow/dags/ 64 | name: dags-data 65 | - name: metrics 66 | image: pbweb/airflow-prometheus-exporter:latest 67 | imagePullPolicy: IfNotPresent 68 | ports: 69 | - containerPort: 9112 70 | name: metrics 71 | volumes: 72 | - emptyDir: {} 73 | name: dags-data 74 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | name: "Airflow k8s operator" 18 | on: 19 | push: 20 | branches: 21 | - master 22 | pull_request: 23 | branches: 24 | - master 25 | jobs: 26 | statics: 27 | name: Static checks 28 | runs-on: ubuntu-latest 29 | steps: 30 | - uses: actions/checkout@v1 31 | - uses: actions/setup-python@v1 32 | - name: set PY 33 | run: echo "::set-env name=PY::$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" 34 | - uses: actions/cache@v1 35 | with: 36 | path: ~/.cache/pre-commit 37 | key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} 38 | - uses: pre-commit/action@v1.0.1 39 | 40 | build: 41 | name: Build docker 42 | runs-on: ubuntu-latest 43 | steps: 44 | - name: Check out code 45 | uses: actions/checkout@v2 46 | - name: Build 47 | run: docker build . -t "ci" 48 | 49 | tests: 50 | name: Tests 51 | runs-on: ubuntu-latest 52 | steps: 53 | - name: Set up Go 1.13 54 | uses: actions/setup-go@v1 55 | with: 56 | go-version: 1.13 57 | id: go 58 | - name: Check out code 59 | uses: actions/checkout@v2 60 | - name: Install kubebuilder 61 | run: | 62 | os=$(go env GOOS) && \ 63 | arch=$(go env GOARCH) && \ 64 | curl -L https://go.kubebuilder.io/dl/2.3.0/$os/$arch | tar -xz -C /tmp/ && \ 65 | sudo mv /tmp/kubebuilder_2.3.0_${os}_${arch} /usr/local/kubebuilder && \ 66 | export PATH=$PATH:/usr/local/kubebuilder/bin 67 | - name: Run test 68 | run: make test 69 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | creationTimestamp: null 7 | name: manager-role 8 | rules: 9 | - resources: 10 | - configmaps 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - resources: 20 | - secrets 21 | verbs: 22 | - create 23 | - delete 24 | - get 25 | - list 26 | - patch 27 | - update 28 | - watch 29 | - resources: 30 | - serviceaccounts 31 | verbs: 32 | - create 33 | - delete 34 | - get 35 | - list 36 | - patch 37 | - update 38 | - watch 39 | - resources: 40 | - services 41 | verbs: 42 | - create 43 | - delete 44 | - get 45 | - list 46 | - patch 47 | - update 48 | - watch 49 | - apiGroups: 50 | - airflow.apache.org 51 | resources: 52 | - airflowbases 53 | verbs: 54 | - create 55 | - delete 56 | - get 57 | - list 58 | - patch 59 | - update 60 | - watch 61 | - apiGroups: 62 | - airflow.apache.org 63 | resources: 64 | - airflowbases/status 65 | verbs: 66 | - get 67 | - patch 68 | - update 69 | - apiGroups: 70 | - airflow.apache.org 71 | resources: 72 | - airflowclusters 73 | verbs: 74 | - create 75 | - delete 76 | - get 77 | - list 78 | - patch 79 | - update 80 | - watch 81 | - apiGroups: 82 | - airflow.apache.org 83 | resources: 84 | - airflowclusters/status 85 | verbs: 86 | - get 87 | - patch 88 | - update 89 | - apiGroups: 90 | - app.k8s.io 91 | resources: 92 | - applications 93 | verbs: 94 | - create 95 | - delete 96 | - get 97 | - list 98 | - patch 99 | - update 100 | - watch 101 | - apiGroups: 102 | - apps 103 | resources: 104 | - statefulsets 105 | verbs: 106 | - create 107 | - delete 108 | - get 109 | - list 110 | - patch 111 | - update 112 | - watch 113 | - apiGroups: 114 | - policy 115 | resources: 116 | - poddisruptionbudgets 117 | verbs: 118 | - create 119 | - delete 120 | - get 121 | - list 122 | - patch 123 | - update 124 | - watch 125 | - apiGroups: 126 | - rbac.authorization.k8s.io 127 | resources: 128 | - rolebindings 129 | verbs: 130 | - create 131 | - delete 132 | - get 133 | - list 134 | - patch 135 | - update 136 | - watch 137 | - apiGroups: 138 | - storage.k8s.io 139 | resources: 140 | - storageclasses 141 | verbs: 142 | - create 143 | - delete 144 | - get 145 | - list 146 | - patch 147 | - update 148 | - watch 149 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/finalizer/finalizer_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package finalizer_test 17 | 18 | import ( 19 | . "github.com/onsi/ginkgo" 20 | . "github.com/onsi/gomega" 21 | appsv1 "k8s.io/api/apps/v1" 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | "sigs.k8s.io/controller-reconciler/pkg/finalizer" 24 | ) 25 | 26 | var _ = Describe("Finalizer", func() { 27 | 28 | deployment := &appsv1.Deployment{} 29 | BeforeEach(func() { 30 | deployment = &appsv1.Deployment{ 31 | ObjectMeta: metav1.ObjectMeta{ 32 | Name: "n-deploy", 33 | Namespace: "ns", 34 | }, 35 | } 36 | }) 37 | 38 | Describe("Finalizer", func() { 39 | It("Add finalizer", func(done Done) { 40 | Expect(len(deployment.ObjectMeta.Finalizers)).To(Equal(0)) 41 | finalizer.Add(deployment, finalizer.Cleanup) 42 | Expect(len(deployment.ObjectMeta.Finalizers)).To(Equal(1)) 43 | finalizer.Add(deployment, finalizer.Cleanup) 44 | Expect(len(deployment.ObjectMeta.Finalizers)).To(Equal(1)) 45 | finalizer.Add(deployment, "second") 46 | Expect(len(deployment.ObjectMeta.Finalizers)).To(Equal(2)) 47 | close(done) 48 | }) 49 | It("Remove finalizer", func(done Done) { 50 | Expect(len(deployment.ObjectMeta.Finalizers)).To(Equal(0)) 51 | finalizer.Add(deployment, finalizer.Cleanup) 52 | finalizer.Add(deployment, "second") 53 | Expect(len(deployment.ObjectMeta.Finalizers)).To(Equal(2)) 54 | finalizer.Remove(deployment, "second") 55 | Expect(len(deployment.ObjectMeta.Finalizers)).To(Equal(1)) 56 | finalizer.Remove(deployment, "second") 57 | Expect(len(deployment.ObjectMeta.Finalizers)).To(Equal(1)) 58 | finalizer.Remove(deployment, finalizer.Cleanup) 59 | Expect(len(deployment.ObjectMeta.Finalizers)).To(Equal(0)) 60 | close(done) 61 | }) 62 | }) 63 | }) 64 | -------------------------------------------------------------------------------- /templates/nfs-sts.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: {{.Name}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | annotations: 27 | {{range $k,$v := .Base.Spec.Annotations }} 28 | {{$k}}: {{$v}} 29 | {{end}} 30 | spec: 31 | replicas: 1 32 | selector: 33 | matchLabels: 34 | {{range $k,$v := .Selector }} 35 | {{$k}}: {{$v}} 36 | {{end}} 37 | updateStrategy: 38 | type: OnDelete 39 | podManagementPolicy: Parallel 40 | serviceName: {{.SvcName}} 41 | template: 42 | metadata: 43 | labels: 44 | {{range $k,$v := .Labels }} 45 | {{$k}}: {{$v}} 46 | {{end}} 47 | annotations: 48 | {{range $k,$v := .Base.Spec.Annotations }} 49 | {{$k}}: {{$v}} 50 | {{end}} 51 | spec: 52 | terminationGracePeriodSeconds: 30 ## check this change to 180 ? 53 | nodeSelector: 54 | {{range $k,$v := .Base.Spec.NodeSelector }} 55 | {{$k}}: {{$v}} 56 | {{end}} 57 | containers: 58 | - name: nfs-server 59 | image: {{.Base.Spec.Storage.Image}}:{{.Base.Spec.Storage.Version}} 60 | imagePullPolicy: IfNotPresent 61 | ports: 62 | - containerPort: 2049 63 | name: nfs 64 | protocol: TCP 65 | - containerPort: 20048 66 | name: mountd 67 | protocol: TCP 68 | - containerPort: 111 69 | name: rpcbind 70 | protocol: TCP 71 | resources: {} 72 | volumeMounts: 73 | - mountPath: /exports 74 | name: data 75 | restartPolicy: Always 76 | {{if .Base.Spec.Storage.Volume}} 77 | {{else}} 78 | volumes: 79 | - emptyDir: {} 80 | name: data 81 | {{end}} 82 | -------------------------------------------------------------------------------- /templates/ui-sts.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: {{.Name}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | annotations: 27 | {{range $k,$v := .Cluster.Spec.Annotations }} 28 | {{$k}}: {{$v}} 29 | {{end}} 30 | spec: 31 | replicas: {{.Cluster.Spec.UI.Replicas}} 32 | selector: 33 | matchLabels: 34 | {{range $k,$v := .Selector }} 35 | {{$k}}: {{$v}} 36 | {{end}} 37 | updateStrategy: 38 | type: RollingUpdate 39 | podManagementPolicy: Parallel 40 | template: 41 | metadata: 42 | labels: 43 | {{range $k,$v := .Labels }} 44 | {{$k}}: {{$v}} 45 | {{end}} 46 | annotations: 47 | {{range $k,$v := .Cluster.Spec.Annotations }} 48 | {{$k}}: {{$v}} 49 | {{end}} 50 | spec: 51 | terminationGracePeriodSeconds: 30 52 | nodeSelector: 53 | {{range $k,$v := .Cluster.Spec.NodeSelector }} 54 | {{$k}}: {{$v}} 55 | {{end}} 56 | containers: 57 | - name: airflow-ui 58 | args: 59 | - webserver 60 | image: {{.Cluster.Spec.UI.Image}}:{{.Cluster.Spec.UI.Version}} 61 | imagePullPolicy: IfNotPresent 62 | livenessProbe: 63 | failureThreshold: 5 64 | httpGet: 65 | path: /health 66 | port: web 67 | scheme: HTTP 68 | initialDelaySeconds: 100 69 | periodSeconds: 60 70 | successThreshold: 1 71 | timeoutSeconds: 2 72 | ports: 73 | - containerPort: 8080 74 | name: web 75 | protocol: TCP 76 | volumeMounts: 77 | - mountPath: /usr/local/airflow/dags/ 78 | name: dags-data 79 | volumes: 80 | - emptyDir: {} 81 | name: dags-data 82 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/resource.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package reconciler 17 | 18 | import ( 19 | "math/rand" 20 | "time" 21 | ) 22 | 23 | // Password char space 24 | const ( 25 | PasswordCharNumSpace = "abcdefghijklmnopqrstuvwxyz0123456789" 26 | PasswordCharSpace = "abcdefghijklmnopqrstuvwxyz" 27 | ) 28 | 29 | var ( 30 | random = rand.New(rand.NewSource(time.Now().UnixNano())) 31 | ) 32 | 33 | // RandomAlphanumericString generates a random password of some fixed length. 34 | func RandomAlphanumericString(strlen int) string { 35 | result := make([]byte, strlen) 36 | for i := range result { 37 | result[i] = PasswordCharNumSpace[random.Intn(len(PasswordCharNumSpace))] 38 | } 39 | result[0] = PasswordCharSpace[random.Intn(len(PasswordCharSpace))] 40 | return string(result[:strlen]) 41 | } 42 | 43 | // NoUpdate - set lifecycle to noupdate 44 | // Manage by Create Only, Delete. 45 | func NoUpdate(o *Object, v interface{}) { 46 | o.Lifecycle = LifecycleNoUpdate 47 | } 48 | 49 | // DecorateOnly - set lifecycle to decorate 50 | // Manage by update only. Dont create or delete 51 | func DecorateOnly(o *Object, v interface{}) { 52 | o.Lifecycle = LifecycleDecorate 53 | } 54 | 55 | // Merge is used to merge multiple maps into the target map 56 | func (out KVMap) Merge(kvmaps ...KVMap) { 57 | for _, kvmap := range kvmaps { 58 | for k, v := range kvmap { 59 | out[k] = v 60 | } 61 | } 62 | } 63 | 64 | // ObjectsByType get items from the Object bag 65 | func ObjectsByType(in []Object, t string) []Object { 66 | var out []Object 67 | for _, item := range in { 68 | if item.Type == t { 69 | out = append(out, item) 70 | } 71 | } 72 | return out 73 | } 74 | 75 | // DeleteAt object at index 76 | func DeleteAt(in []Object, index int) []Object { 77 | var out []Object 78 | in[index] = in[len(in)-1] 79 | out = in[:len(in)-1] 80 | return out 81 | } 82 | -------------------------------------------------------------------------------- /templates/sqlproxy-sts.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: {{.Name}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | annotations: 27 | {{range $k,$v := .Base.Spec.Annotations }} 28 | {{$k}}: {{$v}} 29 | {{end}} 30 | spec: 31 | replicas: 1 32 | selector: 33 | matchLabels: 34 | {{range $k,$v := .Selector }} 35 | {{$k}}: {{$v}} 36 | {{end}} 37 | updateStrategy: 38 | type: RollingUpdate 39 | podManagementPolicy: OrderedReady 40 | serviceName: {{.SvcName}} 41 | template: 42 | metadata: 43 | labels: 44 | {{range $k,$v := .Labels }} 45 | {{$k}}: {{$v}} 46 | {{end}} 47 | annotations: 48 | {{range $k,$v := .Base.Spec.Annotations }} 49 | {{$k}}: {{$v}} 50 | {{end}} 51 | spec: 52 | ## TODO affinity in code 53 | terminationGracePeriodSeconds: 30 ## check this change to 180 ? 54 | #priorityClassName: something 55 | #securityContext: 56 | # fsGroup: 1000 57 | nodeSelector: 58 | {{range $k,$v := .Base.Spec.NodeSelector }} 59 | {{$k}}: {{$v}} 60 | {{end}} 61 | #tolerations: 62 | #initContainers: 63 | containers: 64 | - name: sqlproxy 65 | command: 66 | - /cloud_sql_proxy 67 | - -instances 68 | - $(SQL_INSTANCE) 69 | env: 70 | - name: SQL_INSTANCE 71 | value: {{.Base.Spec.SQLProxy.Project}}:{{.Base.Spec.SQLProxy.Region}}:{{.Base.Spec.SQLProxy.Instance}}=tcp:0.0.0.0:{{index .Ports "sqlproxy"}} 72 | image: {{.Base.Spec.SQLProxy.Image}}:{{.Base.Spec.SQLProxy.Version}} 73 | imagePullPolicy: IfNotPresent 74 | ports: 75 | - containerPort: {{index .Ports "sqlproxy"}} 76 | name: sqlproxy 77 | protocol: TCP 78 | resources: {} 79 | -------------------------------------------------------------------------------- /controllers/suite_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package controllers 17 | 18 | import ( 19 | "path/filepath" 20 | "testing" 21 | 22 | . "github.com/onsi/ginkgo" 23 | . "github.com/onsi/gomega" 24 | 25 | airflowv1alpha1 "github.com/apache/airflow-on-k8s-operator/api/v1alpha1" 26 | "k8s.io/client-go/kubernetes/scheme" 27 | "k8s.io/client-go/rest" 28 | "sigs.k8s.io/controller-runtime/pkg/client" 29 | "sigs.k8s.io/controller-runtime/pkg/envtest" 30 | logf "sigs.k8s.io/controller-runtime/pkg/log" 31 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 32 | // +kubebuilder:scaffold:imports 33 | ) 34 | 35 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 36 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 37 | 38 | var cfg *rest.Config 39 | var k8sClient client.Client 40 | var testEnv *envtest.Environment 41 | 42 | func TestAPIs(t *testing.T) { 43 | RegisterFailHandler(Fail) 44 | 45 | RunSpecsWithDefaultAndCustomReporters(t, 46 | "Controller Suite", 47 | []Reporter{envtest.NewlineReporter{}}) 48 | } 49 | 50 | var _ = BeforeSuite(func(done Done) { 51 | logf.SetLogger(zap.LoggerTo(GinkgoWriter, true)) 52 | 53 | By("bootstrapping test environment") 54 | testEnv = &envtest.Environment{ 55 | CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, 56 | } 57 | 58 | var err error 59 | cfg, err = testEnv.Start() 60 | Expect(err).ToNot(HaveOccurred()) 61 | Expect(cfg).ToNot(BeNil()) 62 | 63 | err = airflowv1alpha1.AddToScheme(scheme.Scheme) 64 | Expect(err).NotTo(HaveOccurred()) 65 | 66 | err = airflowv1alpha1.AddToScheme(scheme.Scheme) 67 | Expect(err).NotTo(HaveOccurred()) 68 | 69 | // +kubebuilder:scaffold:scheme 70 | 71 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) 72 | Expect(err).ToNot(HaveOccurred()) 73 | Expect(k8sClient).ToNot(BeNil()) 74 | 75 | close(done) 76 | }, 60) 77 | 78 | var _ = AfterSuite(func() { 79 | By("tearing down the test environment") 80 | err := testEnv.Stop() 81 | Expect(err).ToNot(HaveOccurred()) 82 | }) 83 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/storage/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | // Licensed to the Apache Software Foundation (ASF) under one or more 4 | // contributor license agreements. See the NOTICE file distributed with 5 | // this work for additional information regarding copyright ownership. 6 | // The ASF licenses this file to You under the Apache License, Version 2.0 7 | // (the "License"); you may not use this file except in compliance with 8 | // the License. You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | 18 | // Code generated by main. DO NOT EDIT. 19 | 20 | package storage 21 | 22 | import ( 23 | v1 "k8s.io/api/core/v1" 24 | ) 25 | 26 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 27 | func (in *FS) DeepCopyInto(out *FS) { 28 | *out = *in 29 | return 30 | } 31 | 32 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FS. 33 | func (in *FS) DeepCopy() *FS { 34 | if in == nil { 35 | return nil 36 | } 37 | out := new(FS) 38 | in.DeepCopyInto(out) 39 | return out 40 | } 41 | 42 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 43 | func (in *GCS) DeepCopyInto(out *GCS) { 44 | *out = *in 45 | if in.Secret != nil { 46 | in, out := &in.Secret, &out.Secret 47 | *out = new(v1.SecretReference) 48 | **out = **in 49 | } 50 | return 51 | } 52 | 53 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCS. 54 | func (in *GCS) DeepCopy() *GCS { 55 | if in == nil { 56 | return nil 57 | } 58 | out := new(GCS) 59 | in.DeepCopyInto(out) 60 | return out 61 | } 62 | 63 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 64 | func (in *Spec) DeepCopyInto(out *Spec) { 65 | *out = *in 66 | if in.GCS != nil { 67 | in, out := &in.GCS, &out.GCS 68 | *out = new(GCS) 69 | (*in).DeepCopyInto(*out) 70 | } 71 | if in.NFS != nil { 72 | in, out := &in.NFS, &out.NFS 73 | *out = new(v1.NFSVolumeSource) 74 | **out = **in 75 | } 76 | if in.FS != nil { 77 | in, out := &in.FS, &out.FS 78 | *out = new(FS) 79 | **out = **in 80 | } 81 | return 82 | } 83 | 84 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Spec. 85 | func (in *Spec) DeepCopy() *Spec { 86 | if in == nil { 87 | return nil 88 | } 89 | out := new(Spec) 90 | in.DeepCopyInto(out) 91 | return out 92 | } 93 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | repos: 18 | - repo: git://github.com/dnephin/pre-commit-golang 19 | rev: master 20 | hooks: 21 | - id: go-fmt 22 | name: Run Gofmt 23 | exclude: ^vendor/.*$ 24 | - repo: local 25 | hooks: 26 | - id: yamllint 27 | name: Check yaml files with yamllint 28 | entry: yamllint -c yamllint-config.yaml 29 | language: python 30 | additional_dependencies: ['yamllint'] 31 | types: [yaml] 32 | exclude: ^templates/.*$|^vendor/.*$ 33 | - repo: https://github.com/pre-commit/pre-commit-hooks 34 | rev: v2.5.0 35 | hooks: 36 | - id: check-merge-conflict 37 | - id: detect-private-key 38 | - id: end-of-file-fixer 39 | exclude: ^vendor/.*$ 40 | - id: mixed-line-ending 41 | - id: check-executables-have-shebangs 42 | - id: trailing-whitespace 43 | exclude: ^vendor/.*$ 44 | - repo: https://github.com/Lucas-C/pre-commit-hooks 45 | rev: v1.1.7 46 | hooks: 47 | - id: insert-license 48 | name: Add license for all yaml files 49 | exclude: ^vendor/.*$|^config/crd/bases/.*$|^config/rbac/role.yaml$|^config/webhook/manifests.yaml$ 50 | types: [yaml] 51 | args: 52 | - --comment-style 53 | - "|#|" 54 | - --license-filepath 55 | - license-templates/LICENSE.txt 56 | - --fuzzy-match-generates-todo 57 | - id: insert-license 58 | name: Add license for all md files 59 | files: \.md$ 60 | exclude: ^vendor/.*$ 61 | args: 62 | - --comment-style 63 | - "" 64 | - --license-filepath 65 | - license-templates/LICENSE.txt 66 | - --fuzzy-match-generates-todo 67 | - id: insert-license 68 | name: Add license for all go files 69 | files: \.go$ 70 | exclude: ^vendor/.*$|api/v1alpha1/zz_generated.deepcopy.go 71 | args: 72 | - --comment-style 73 | - "//" 74 | - --license-filepath 75 | - license-templates/LICENSE.txt 76 | - --fuzzy-match-generates-todo 77 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package main 17 | 18 | import ( 19 | "flag" 20 | "os" 21 | "time" 22 | 23 | airflowv1alpha1 "github.com/apache/airflow-on-k8s-operator/api/v1alpha1" 24 | "github.com/apache/airflow-on-k8s-operator/controllers" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 27 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 28 | ctrl "sigs.k8s.io/controller-runtime" 29 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 30 | // +kubebuilder:scaffold:imports 31 | ) 32 | 33 | var ( 34 | scheme = runtime.NewScheme() 35 | setupLog = ctrl.Log.WithName("setup") 36 | ) 37 | 38 | func init() { 39 | _ = clientgoscheme.AddToScheme(scheme) 40 | 41 | _ = airflowv1alpha1.AddToScheme(scheme) 42 | 43 | // +kubebuilder:scaffold:scheme 44 | } 45 | 46 | func main() { 47 | var metricsAddr string 48 | var enableLeaderElection bool 49 | flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") 50 | flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, 51 | "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") 52 | flag.Parse() 53 | 54 | ctrl.SetLogger(zap.New(func(o *zap.Options) { 55 | o.Development = true 56 | })) 57 | 58 | syncperiod := time.Minute * 2 59 | 60 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 61 | Scheme: scheme, 62 | MetricsBindAddress: metricsAddr, 63 | LeaderElection: enableLeaderElection, 64 | Port: 9443, 65 | SyncPeriod: &syncperiod, 66 | }) 67 | if err != nil { 68 | setupLog.Error(err, "unable to start manager") 69 | os.Exit(1) 70 | } 71 | 72 | if err = (&controllers.AirflowBaseReconciler{ 73 | Client: mgr.GetClient(), 74 | Log: ctrl.Log.WithName("controllers").WithName("AirflowBase"), 75 | Scheme: mgr.GetScheme(), 76 | }).SetupWithManager(mgr); err != nil { 77 | setupLog.Error(err, "unable to create controller", "controller", "AirflowBase") 78 | os.Exit(1) 79 | } 80 | if err = (&controllers.AirflowClusterReconciler{ 81 | Client: mgr.GetClient(), 82 | Log: ctrl.Log.WithName("controllers").WithName("AirflowCluster"), 83 | Scheme: mgr.GetScheme(), 84 | }).SetupWithManager(mgr); err != nil { 85 | setupLog.Error(err, "unable to create controller", "controller", "AirflowCluster") 86 | os.Exit(1) 87 | } 88 | // +kubebuilder:scaffold:builder 89 | 90 | setupLog.Info("starting manager") 91 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { 92 | setupLog.Error(err, "problem running manager") 93 | os.Exit(1) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /controllers/application/application_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package application_test 17 | 18 | import ( 19 | "github.com/apache/airflow-on-k8s-operator/controllers/application" 20 | app "github.com/kubernetes-sigs/application/pkg/apis/app/v1beta1" 21 | . "github.com/onsi/ginkgo" 22 | . "github.com/onsi/gomega" 23 | appsv1 "k8s.io/api/apps/v1" 24 | corev1 "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "sigs.k8s.io/controller-reconciler/pkg/reconciler" 27 | "sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/k8s" 28 | ) 29 | 30 | var _ = Describe("Application", func() { 31 | BeforeEach(func() { 32 | }) 33 | labels := map[string]string{ 34 | "k1": "v1", 35 | "k2": "v2", 36 | "k3": "v3", 37 | } 38 | var resources []reconciler.Object = []reconciler.Object{ 39 | { 40 | Lifecycle: reconciler.LifecycleManaged, 41 | Type: k8s.Type, 42 | Obj: &k8s.Object{ 43 | ObjList: &appsv1.DeploymentList{}, 44 | Obj: &appsv1.Deployment{ 45 | TypeMeta: metav1.TypeMeta{ 46 | Kind: "k3", 47 | APIVersion: "v4", 48 | }, 49 | ObjectMeta: metav1.ObjectMeta{ 50 | Name: "n-deploy", 51 | Namespace: "ns", 52 | Labels: labels, 53 | }, 54 | }, 55 | }, 56 | }, 57 | { 58 | Lifecycle: reconciler.LifecycleManaged, 59 | Type: k8s.Type, 60 | Obj: &k8s.Object{ 61 | ObjList: &corev1.ConfigMapList{}, 62 | Obj: &corev1.ConfigMap{ 63 | TypeMeta: metav1.TypeMeta{ 64 | Kind: "k1", 65 | APIVersion: "v2", 66 | }, 67 | ObjectMeta: metav1.ObjectMeta{ 68 | Name: "n-cm", 69 | Namespace: "ns", 70 | Labels: labels, 71 | }, 72 | Data: map[string]string{ 73 | "test-key": "test-value", 74 | }, 75 | }, 76 | }, 77 | }, 78 | } 79 | a := application.NewApplication(&app.Application{}) 80 | Describe("Status", func() { 81 | It("Sets Component Group Kind", func(done Done) { 82 | a.SetComponentGK(resources) 83 | Expect(len(a.Spec.ComponentGroupKinds)).To(Equal(len(resources))) 84 | close(done) 85 | }) 86 | It("Sets Selector", func(done Done) { 87 | a.SetSelector(labels) 88 | Expect(a.Spec.Selector.MatchLabels).To(Equal(labels)) 89 | close(done) 90 | }) 91 | It("Sets Meta Name Namespace Labels", func(done Done) { 92 | a.SetName("somename").SetNamespace("somens").SetLabels(labels) 93 | Expect(a.ObjectMeta.Name).To(Equal("somename")) 94 | Expect(a.ObjectMeta.Namespace).To(Equal("somens")) 95 | Expect(a.ObjectMeta.Labels).To(Equal(labels)) 96 | close(done) 97 | }) 98 | }) 99 | }) 100 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 17 | 18 | # Airflow On K8S Operator 19 | ![Airflow k8s operator](https://github.com/apache/airflow-on-k8s-operator/workflows/Airflow%20k8s%20operator/badge.svg?branch=master) 20 | [![Go Report Card](https://goreportcard.com/badge/github.com/apache/airflow-on-k8s-operator)](https://goreportcard.com/report/github.com/apache/airflow-on-k8s-operator) 21 | 22 | 23 | ## Community 24 | 25 | * Join [Airflow Slack](https://apache-airflow-slack.herokuapp.com) and the dedicated #sig-kubernetes channel. 26 | 27 | ## Project Status 28 | 29 | *Alpha* 30 | 31 | The Airflow Operator is still under active development and has not been extensively tested in production environment. Backward compatibility of the APIs is not guaranteed for alpha releases. 32 | 33 | ## Prerequisites 34 | * Version >= 1.9 of Kubernetes. 35 | * Uses 1.9 of Airflow (1.10.1+ for k8s executor) 36 | * Uses 4.0.x of Redis (for celery operator) 37 | * Uses 5.7 of MySQL 38 | 39 | ## Get Started 40 | 41 | [One Click Deployment](https://console.cloud.google.com/marketplace/details/google/airflow-operator) from Google Cloud Marketplace to your [GKE cluster](https://cloud.google.com/kubernetes-engine/) 42 | 43 | Get started quickly with the Airflow Operator using the [Quick Start Guide](docs/quickstart.md) 44 | 45 | For more information check the [Design](docs/design.md) and detailed [User Guide](docs/userguide.md) 46 | 47 | ## Airflow Operator Overview 48 | Airflow Operator is a custom [Kubernetes operator](https://coreos.com/blog/introducing-operators.html) that makes it easy to deploy and manage [Apache Airflow](https://airflow.apache.org/) on Kubernetes. Apache Airflow is a platform to programmatically author, schedule and monitor workflows. Using the Airflow Operator, an Airflow cluster is split into 2 parts represented by the `AirflowBase` and `AirflowCluster` custom resources. 49 | The Airflow Operator performs these jobs: 50 | * Creates and manages the necessary Kubernetes resources for an Airflow deployment. 51 | * Updates the corresponding Kubernetes resources when the `AirflowBase` or `AirflowCluster` specification changes. 52 | * Restores managed Kubernetes resources that are deleted. 53 | * Supports creation of Airflow schedulers with different Executors 54 | * Supports sharing of the `AirflowBase` across mulitple `AirflowClusters` 55 | 56 | Checkout out the [Design](docs/design.md) 57 | 58 | ![Airflow Cluster](docs/airflow-cluster.png) 59 | 60 | 61 | ## Development 62 | 63 | Refer to the [Design](docs/design.md) and [Development Guide](docs/development.md). 64 | 65 | ## History 66 | This repo has been donated to Apache foundation. 67 | It was originally developed here at [GoogleCloud repo](https://github.com/GoogleCloudPlatform/airflow-operator) 68 | -------------------------------------------------------------------------------- /test/e2e/base/base_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package e2etest 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/apache/airflow-on-k8s-operator/api/v1alpha1" 22 | . "github.com/onsi/ginkgo" 23 | . "github.com/onsi/gomega" 24 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 25 | "sigs.k8s.io/controller-reconciler/pkg/test" 26 | ) 27 | 28 | const ( 29 | CRName = "AirflowBase" 30 | SampleDir = "../../../hack/sample/" 31 | ) 32 | 33 | var f *test.Framework 34 | var ctx *test.Context 35 | 36 | func airflowBase(file string) *v1alpha1.AirflowBase { 37 | cr := &v1alpha1.AirflowBase{} 38 | if err := f.LoadFromFile(file, cr); err != nil { 39 | return nil 40 | } 41 | return cr 42 | } 43 | 44 | func Test(t *testing.T) { 45 | RegisterFailHandler(Fail) 46 | RunSpecs(t, CRName+" Suite") 47 | } 48 | 49 | var _ = BeforeSuite(func() { 50 | f = test.New(CRName) 51 | err := v1alpha1.SchemeBuilder.AddToScheme(f.GetScheme()) 52 | Expect(err).NotTo(HaveOccurred(), "failed to initialize the Framework: %v", err) 53 | f.Start() 54 | }) 55 | 56 | var _ = AfterSuite(func() { 57 | if ctx != nil { 58 | ctx.DeleteCR() 59 | } 60 | if f != nil { 61 | f.Stop() 62 | } 63 | }) 64 | 65 | func isBaseReady(cr interface{}) bool { 66 | stts := cr.(*v1alpha1.AirflowBase).Status 67 | return stts.IsReady() 68 | } 69 | 70 | var _ = Describe(CRName+" controller tests", func() { 71 | AfterEach(func() { 72 | ctx.DeleteCR() 73 | ctx = nil 74 | }) 75 | 76 | It("creating a "+CRName+" with mysql", func() { 77 | ctx = f.NewContext().WithCR(airflowBase(SampleDir + "mysql-celery/base.yaml")) 78 | cr := ctx.CR.(*v1alpha1.AirflowBase) 79 | By("creating a new " + CRName + ": " + cr.Name) 80 | ctx.CreateCR() 81 | ctx.WithTimeout(200).CheckStatefulSet(cr.Name+"-mysql", 1, 1) 82 | ctx.WithTimeout(10).CheckService(cr.Name+"-sql", map[string]int32{"mysql": 3306}) 83 | //ctx.WithTimeout(10).CheckSecret(name) 84 | ctx.WithTimeout(200).CheckStatefulSet(cr.Name+"-nfs", 1, 1) 85 | ctx.WithTimeout(200).CheckCR(isBaseReady) 86 | }) 87 | 88 | It("creating a "+CRName+" with postgres", func() { 89 | ctx = f.NewContext().WithCR(airflowBase(SampleDir + "postgres-celery/base.yaml")) 90 | cr := ctx.CR.(*v1alpha1.AirflowBase) 91 | By("creating a new " + CRName + ": " + cr.Name) 92 | ctx.CreateCR() 93 | ctx.WithTimeout(200).CheckStatefulSet(cr.Name+"-postgres", 1, 1) 94 | ctx.WithTimeout(10).CheckService(cr.Name+"-sql", map[string]int32{"postgres": 5432}) 95 | //ctx.WithTimeout(10).CheckSecret(name) 96 | ctx.WithTimeout(200).CheckStatefulSet(cr.Name+"-nfs", 1, 1) 97 | ctx.WithTimeout(200).CheckCR(isBaseReady) 98 | }) 99 | }) 100 | -------------------------------------------------------------------------------- /templates/redis-sts.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | --- 18 | apiVersion: apps/v1 19 | kind: StatefulSet 20 | metadata: 21 | name: {{.Name}} 22 | namespace: {{.Namespace}} 23 | labels: 24 | {{range $k,$v := .Labels }} 25 | {{$k}}: {{$v}} 26 | {{end}} 27 | annotations: 28 | {{range $k,$v := .Cluster.Spec.Annotations }} 29 | {{$k}}: {{$v}} 30 | {{end}} 31 | spec: 32 | replicas: 1 33 | selector: 34 | matchLabels: 35 | {{range $k,$v := .Selector }} 36 | {{$k}}: {{$v}} 37 | {{end}} 38 | updateStrategy: 39 | type: OnDelete 40 | podManagementPolicy: OrderedReady 41 | template: 42 | metadata: 43 | labels: 44 | {{range $k,$v := .Labels }} 45 | {{$k}}: {{$v}} 46 | {{end}} 47 | annotations: 48 | {{range $k,$v := .Cluster.Spec.Annotations }} 49 | {{$k}}: {{$v}} 50 | {{end}} 51 | spec: 52 | terminationGracePeriodSeconds: 30 53 | nodeSelector: 54 | {{range $k,$v := .Cluster.Spec.NodeSelector }} 55 | {{$k}}: {{$v}} 56 | {{end}} 57 | containers: 58 | - name: redis 59 | args: 60 | - --requirepass 61 | - $(REDIS_PASSWORD) 62 | {{if .Cluster.Spec.Redis.AdditionalArgs}} 63 | - {{.Cluster.Spec.Redis.AdditionalArgs}} 64 | {{end}} 65 | env: 66 | - name: REDIS_EXTRA_FLAGS 67 | - name: REDIS_PASSWORD 68 | valueFrom: 69 | secretKeyRef: 70 | key: password 71 | name: {{.SecretName}} 72 | image: {{.Cluster.Spec.Redis.Image}}:{{.Cluster.Spec.Redis.Version}} 73 | imagePullPolicy: IfNotPresent 74 | livenessProbe: 75 | exec: 76 | command: 77 | - redis-cli 78 | - ping 79 | failureThreshold: 3 80 | initialDelaySeconds: 30 81 | periodSeconds: 20 82 | successThreshold: 1 83 | timeoutSeconds: 5 84 | ports: 85 | - containerPort: 6379 86 | name: redis 87 | protocol: TCP 88 | readinessProbe: 89 | exec: 90 | command: 91 | - redis-cli 92 | - ping 93 | failureThreshold: 3 94 | initialDelaySeconds: 10 95 | periodSeconds: 5 96 | successThreshold: 1 97 | timeoutSeconds: 2 98 | volumeMounts: 99 | - mountPath: /data 100 | name: data 101 | restartPolicy: Always 102 | {{if .Cluster.Spec.Redis.VolumeClaimTemplate}} 103 | {{else}} 104 | volumes: 105 | - emptyDir: {} 106 | name: data 107 | {{end}} 108 | -------------------------------------------------------------------------------- /templates/postgres-sts.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: {{.Name}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | annotations: 27 | {{range $k,$v := .Base.Spec.Annotations }} 28 | {{$k}}: {{$v}} 29 | {{end}} 30 | spec: 31 | replicas: {{.Base.Spec.Postgres.Replicas}} 32 | selector: 33 | matchLabels: 34 | {{range $k,$v := .Selector }} 35 | {{$k}}: {{$v}} 36 | {{end}} 37 | updateStrategy: 38 | type: OnDelete 39 | podManagementPolicy: OrderedReady 40 | template: 41 | metadata: 42 | labels: 43 | {{range $k,$v := .Labels }} 44 | {{$k}}: {{$v}} 45 | {{end}} 46 | annotations: 47 | {{range $k,$v := .Base.Spec.Annotations }} 48 | {{$k}}: {{$v}} 49 | {{end}} 50 | spec: 51 | terminationGracePeriodSeconds: 30 52 | nodeSelector: 53 | {{range $k,$v := .Base.Spec.NodeSelector }} 54 | {{$k}}: {{$v}} 55 | {{end}} 56 | containers: 57 | - name: postgres 58 | env: 59 | - name: POSTGRES_DB 60 | value: testdb 61 | - name: POSTGRES_USER 62 | value: postgres 63 | - name: POSTGRES_PASSWORD 64 | valueFrom: 65 | secretKeyRef: 66 | key: rootpassword 67 | name: {{.SecretName}} 68 | image: {{.Base.Spec.Postgres.Image}}:{{.Base.Spec.Postgres.Version}} 69 | imagePullPolicy: IfNotPresent 70 | livenessProbe: 71 | exec: 72 | command: 73 | - bash 74 | - -c 75 | - psql -w -U $POSTGRES_USER -d $POSTGRES_DB -c SELECT 1 76 | failureThreshold: 3 77 | initialDelaySeconds: 30 78 | periodSeconds: 20 79 | successThreshold: 1 80 | timeoutSeconds: 5 81 | ports: 82 | - containerPort: 5432 83 | name: postgres 84 | readinessProbe: 85 | exec: 86 | command: 87 | - bash 88 | - -c 89 | - psql -w -U $POSTGRES_USER -d $POSTGRES_DB -c SELECT 1 90 | failureThreshold: 3 91 | initialDelaySeconds: 10 92 | periodSeconds: 5 93 | successThreshold: 1 94 | timeoutSeconds: 2 95 | volumeMounts: 96 | - name: data 97 | mountPath: /var/lib/postgres/data 98 | restartPolicy: Always 99 | {{if .Base.Spec.Postgres.VolumeClaimTemplate}} 100 | {{else}} 101 | volumes: 102 | - emptyDir: {} 103 | name: data 104 | {{end}} 105 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/status/status.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package status 15 | 16 | import ( 17 | //"fmt" 18 | appsv1 "k8s.io/api/apps/v1" 19 | policyv1 "k8s.io/api/policy/v1beta1" 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "k8s.io/apimachinery/pkg/runtime" 22 | "sigs.k8s.io/controller-reconciler/pkg/reconciler" 23 | "sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/k8s" 24 | ) 25 | 26 | // Constants defining labels 27 | const ( 28 | StatusReady = "Ready" 29 | StatusInProgress = "InProgress" 30 | StatusDisabled = "Disabled" 31 | ) 32 | 33 | func (s *Statefulset) update(rsrc *appsv1.StatefulSet) string { 34 | s.Replicas = rsrc.Status.Replicas 35 | s.ReadyReplicas = rsrc.Status.ReadyReplicas 36 | s.CurrentReplicas = rsrc.Status.CurrentReplicas 37 | if rsrc.Status.ReadyReplicas == *rsrc.Spec.Replicas && rsrc.Status.CurrentReplicas == *rsrc.Spec.Replicas { 38 | return StatusReady 39 | } 40 | return StatusInProgress 41 | } 42 | 43 | func (s *ObjectStatus) update(rsrc metav1.Object) { 44 | ro := rsrc.(runtime.Object) 45 | gvk := ro.GetObjectKind().GroupVersionKind() 46 | s.Link = rsrc.GetSelfLink() 47 | s.Name = rsrc.GetName() 48 | s.Group = gvk.GroupVersion().String() 49 | s.Kind = gvk.GroupKind().Kind 50 | s.Status = StatusReady 51 | } 52 | 53 | // Pdb is a generic status holder for pdb 54 | func (s *Pdb) update(rsrc *policyv1.PodDisruptionBudget) string { 55 | s.CurrentHealthy = rsrc.Status.CurrentHealthy 56 | s.DesiredHealthy = rsrc.Status.DesiredHealthy 57 | if s.CurrentHealthy >= s.DesiredHealthy { 58 | return StatusReady 59 | } 60 | return StatusInProgress 61 | } 62 | 63 | // ResetComponentList - reset component list objects 64 | func (cm *ComponentMeta) ResetComponentList() { 65 | cm.ComponentList.Objects = []ObjectStatus{} 66 | } 67 | 68 | // UpdateStatus the component status 69 | func (cm *ComponentMeta) UpdateStatus(items []reconciler.Object) bool { 70 | var ready = true 71 | for _, item := range items { 72 | r := item.Obj.(*k8s.Object) 73 | os := ObjectStatus{} 74 | os.update(r.Obj) 75 | switch r.Obj.(type) { 76 | case *appsv1.StatefulSet: 77 | os.ExtendedStatus.STS = &Statefulset{} 78 | os.Status = os.ExtendedStatus.STS.update(r.Obj.(*appsv1.StatefulSet)) 79 | case *policyv1.PodDisruptionBudget: 80 | os.ExtendedStatus.PDB = &Pdb{} 81 | os.Status = os.ExtendedStatus.PDB.update(r.Obj.(*policyv1.PodDisruptionBudget)) 82 | } 83 | cm.ComponentList.Objects = append(cm.ComponentList.Objects, os) 84 | } 85 | for _, os := range cm.ComponentList.Objects { 86 | if os.Status != StatusReady { 87 | ready = false 88 | } 89 | } 90 | 91 | return ready 92 | } 93 | 94 | // UpdateStatus the component status 95 | func (m *Meta) UpdateStatus(componentready *bool, err error) { 96 | if componentready != nil { 97 | if *componentready { 98 | m.Ready("ComponentsReady", "all components ready") 99 | } else { 100 | m.NotReady("ComponentsNotReady", "some components not ready") 101 | } 102 | } 103 | if err != nil { 104 | m.SetCondition(Error, "ErrorSeen", err.Error()) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Adds namespace to all resources. 17 | --- 18 | namespace: airflow-on-k8s-operator-system 19 | 20 | # Value of this field is prepended to the 21 | # names of all resources, e.g. a deployment named 22 | # "wordpress" becomes "alices-wordpress". 23 | # Note that it should also match with the prefix (text before '-') of the namespace 24 | # field above. 25 | namePrefix: airflow-on-k8s-operator- 26 | 27 | # Labels to add to all resources and selectors. 28 | # commonLabels: 29 | # someName: someValue 30 | 31 | bases: 32 | - ../crd 33 | - ../rbac 34 | - ../manager 35 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml 36 | # - ../webhook 37 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. 38 | # - ../certmanager 39 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 40 | # - ../prometheus 41 | 42 | patchesStrategicMerge: 43 | # Protect the /metrics endpoint by putting it behind auth. 44 | # Only one of manager_auth_proxy_patch.yaml and 45 | # manager_prometheus_metrics_patch.yaml should be enabled. 46 | - manager_auth_proxy_patch.yaml 47 | # If you want your controller-manager to expose the /metrics 48 | # endpoint w/o any authn/z, uncomment the following line and 49 | # comment manager_auth_proxy_patch.yaml. 50 | # Only one of manager_auth_proxy_patch.yaml and 51 | # manager_prometheus_metrics_patch.yaml should be enabled. 52 | # - manager_prometheus_metrics_patch.yaml 53 | 54 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml 55 | # - manager_webhook_patch.yaml 56 | 57 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 58 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. 59 | # 'CERTMANAGER' needs to be enabled to use ca injection 60 | # - webhookcainjection_patch.yaml 61 | 62 | # the following config is for teaching kustomize how to do var substitution 63 | vars: 64 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. 65 | # - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR 66 | # objref: 67 | # kind: Certificate 68 | # group: cert-manager.io 69 | # version: v1alpha2 70 | # name: serving-cert # this name should match the one in certificate.yaml 71 | # fieldref: 72 | # fieldpath: metadata.namespace 73 | # - name: CERTIFICATE_NAME 74 | # objref: 75 | # kind: Certificate 76 | # group: cert-manager.io 77 | # version: v1alpha2 78 | # name: serving-cert # this name should match the one in certificate.yaml 79 | # - name: SERVICE_NAMESPACE # namespace of the service 80 | # objref: 81 | # kind: Service 82 | # version: v1 83 | # name: webhook-service 84 | # fieldref: 85 | # fieldpath: metadata.namespace 86 | # - name: SERVICE_NAME 87 | # objref: 88 | # kind: Service 89 | # version: v1 90 | # name: webhook-service 91 | -------------------------------------------------------------------------------- /controllers/application/application.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package application 17 | 18 | import ( 19 | app "github.com/kubernetes-sigs/application/pkg/apis/app/v1beta1" 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "k8s.io/apimachinery/pkg/runtime" 22 | "k8s.io/apimachinery/pkg/runtime/schema" 23 | "sigs.k8s.io/controller-reconciler/pkg/reconciler" 24 | "sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/k8s" 25 | ) 26 | 27 | // Application obj to attach methods 28 | type Application struct { 29 | *app.Application 30 | } 31 | 32 | // NewApplication - return Application object from runtime Object 33 | func NewApplication(obj metav1.Object) Application { 34 | return Application{obj.(*app.Application)} 35 | } 36 | 37 | // SetSelector attaches selectors to Application object 38 | func (a *Application) SetSelector(labels map[string]string) *Application { 39 | a.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} 40 | return a 41 | } 42 | 43 | // SetName sets name 44 | func (a *Application) SetName(value string) *Application { 45 | a.ObjectMeta.Name = value 46 | return a 47 | } 48 | 49 | // SetNamespace asets namespace 50 | func (a *Application) SetNamespace(value string) *Application { 51 | a.ObjectMeta.Namespace = value 52 | return a 53 | } 54 | 55 | // AddLabels adds more labels 56 | func (a *Application) AddLabels(value reconciler.KVMap) *Application { 57 | value.Merge(a.ObjectMeta.Labels) 58 | a.ObjectMeta.Labels = value 59 | return a 60 | } 61 | 62 | // Observable returns resource object 63 | func (a *Application) Observable() *reconciler.Observable { 64 | return &reconciler.Observable{ 65 | Obj: k8s.Observable{ 66 | Obj: a.Application, 67 | ObjList: &app.ApplicationList{}, 68 | Labels: a.GetLabels(), 69 | }, 70 | Type: k8s.Type, 71 | } 72 | } 73 | 74 | // Item returns resource object 75 | func (a *Application) Item() *reconciler.Object { 76 | return &reconciler.Object{ 77 | Lifecycle: reconciler.LifecycleManaged, 78 | Type: k8s.Type, 79 | Obj: &k8s.Object{ 80 | Obj: a.Application, 81 | ObjList: &app.ApplicationList{}, 82 | }, 83 | } 84 | } 85 | 86 | // AddToScheme return AddToScheme of application crd 87 | func AddToScheme(sb *runtime.SchemeBuilder) { 88 | *sb = append(*sb, app.AddToScheme) 89 | } 90 | 91 | // SetComponentGK attaches component GK to Application object 92 | func (a *Application) SetComponentGK(bag []reconciler.Object) *Application { 93 | a.Spec.ComponentGroupKinds = []metav1.GroupKind{} 94 | gkmap := map[schema.GroupKind]struct{}{} 95 | for _, item := range reconciler.ObjectsByType(bag, k8s.Type) { 96 | obj := item.Obj.(*k8s.Object) 97 | if obj.ObjList != nil { 98 | ro := obj.Obj.(runtime.Object) 99 | gk := ro.GetObjectKind().GroupVersionKind().GroupKind() 100 | if _, ok := gkmap[gk]; !ok { 101 | gkmap[gk] = struct{}{} 102 | mgk := metav1.GroupKind{ 103 | Group: gk.Group, 104 | Kind: gk.Kind, 105 | } 106 | a.Spec.ComponentGroupKinds = append(a.Spec.ComponentGroupKinds, mgk) 107 | } 108 | } 109 | } 110 | return a 111 | } 112 | -------------------------------------------------------------------------------- /docs/development.md: -------------------------------------------------------------------------------- 1 | 17 | 18 | # Development 19 | You should have kubeconfig setup to point to your cluster. 20 | In case you want to build the Airflow Operator from the source code, e.g., 21 | to test a fix or a feature you write, you can do so following the instructions below. 22 | 23 | ## Cloning the repo: 24 | ```bash 25 | $ mkdir -p $GOPATH/src/apache 26 | $ cd $GOPATH/src/apache 27 | $ git clone git@github.com:apache/airflow-on-k8s-operator.git 28 | ``` 29 | 30 | ## Install prerequisites 31 | You have to install the [kubebuilder](https://book.kubebuilder.io/quick-start.html): 32 | ```shell script 33 | os=$(go env GOOS) 34 | arch=$(go env GOARCH) 35 | 36 | # download kubebuilder and extract it to tmp 37 | curl -L https://go.kubebuilder.io/dl/2.3.0/${os}/${arch} | tar -xz -C /tmp/ 38 | 39 | # move to a long-term location and put it on your path 40 | # (you'll need to set the KUBEBUILDER_ASSETS env var if you put it somewhere else) 41 | sudo mv /tmp/kubebuilder_2.3.0_${os}_${arch} /usr/local/kubebuilder 42 | export PATH=$PATH:/usr/local/kubebuilder/bin 43 | ``` 44 | 45 | Also the [kustomize](https://github.com/kubernetes-sigs/kustomize) is required. On MacOS: 46 | ```shell script 47 | brew install kustomize 48 | ``` 49 | For other installation options check https://github.com/kubernetes-sigs/kustomize/blob/master/docs/INSTALL.md 50 | 51 | 52 | ## Building and running locally: 53 | ```bash 54 | # build 55 | make build 56 | 57 | # run locally 58 | make run 59 | ``` 60 | 61 | ## Building docker image 62 | #### GCP 63 | When working with GCP ensure that gcloud is setup and gcr(container registry) is enabled for the current project. 64 | If not set IMG env to point to the desired registry image. 65 | ```bash 66 | # building docker image 67 | make docker-build 68 | 69 | # push docker image 70 | make docker-push 71 | ``` 72 | 73 | 74 | ### Non GCP 75 | Set IMG env to point to the desired registry image. 76 | ```bash 77 | # building docker image 78 | make docker-build NOTGCP=true 79 | 80 | # push docker image 81 | make docker-push NOTGCP=true 82 | ``` 83 | ## Running in cluster 84 | ```bash 85 | # assumes kubeconfig is setup correctly 86 | make deploy 87 | ``` 88 | 89 | ## Static checks 90 | This project uses the [pre-commit](https://pre-commit.com) framework. 91 | The pre-commit hooks use several external linters that need to be installed 92 | before pre-commit is run. Each of the checks installs its own environment, 93 | so you do not need to install those. 94 | 95 | ### Enabling Pre-commit Hooks 96 | 97 | To turn on pre-commit checks for ``commit`` operations in git, enter: 98 | 99 | ```sh 100 | pre-commit install 101 | ``` 102 | 103 | ## Tests 104 | 105 | ### Running local tests 106 | Runs unit-tests locally 107 | 108 | ```bash 109 | make test 110 | ``` 111 | 112 | ### Running e2e tests 113 | Before running e2e tests ensure that the desrired version is running on the cluster or locally. 114 | 115 | ```bash 116 | # Start controller in cluster: 117 | # make docker-push 118 | # make deploy 119 | # OR locally: 120 | # make install 121 | # make run 122 | # and then run the tests 123 | 124 | make e2e-test 125 | ``` 126 | -------------------------------------------------------------------------------- /templates/mysql-sts.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: {{.Name}} 21 | namespace: {{.Namespace}} 22 | labels: 23 | {{range $k,$v := .Labels }} 24 | {{$k}}: {{$v}} 25 | {{end}} 26 | annotations: 27 | {{range $k,$v := .Base.Spec.Annotations }} 28 | {{$k}}: {{$v}} 29 | {{end}} 30 | spec: 31 | replicas: {{.Base.Spec.MySQL.Replicas}} 32 | selector: 33 | matchLabels: 34 | {{range $k,$v := .Selector }} 35 | {{$k}}: {{$v}} 36 | {{end}} 37 | updateStrategy: 38 | type: OnDelete 39 | podManagementPolicy: OrderedReady 40 | template: 41 | metadata: 42 | labels: 43 | {{range $k,$v := .Labels }} 44 | {{$k}}: {{$v}} 45 | {{end}} 46 | annotations: 47 | {{range $k,$v := .Base.Spec.Annotations }} 48 | {{$k}}: {{$v}} 49 | {{end}} 50 | spec: 51 | ## TODO affinity in code 52 | terminationGracePeriodSeconds: 30 ## check this change to 180 ? 53 | #priorityClassName: something 54 | #securityContext: 55 | # fsGroup: 1000 56 | nodeSelector: 57 | {{range $k,$v := .Base.Spec.NodeSelector }} 58 | {{$k}}: {{$v}} 59 | {{end}} 60 | #tolerations: 61 | #initContainers: 62 | containers: 63 | - name: mysql 64 | args: 65 | - --explicit-defaults-for-timestamp=ON 66 | env: 67 | - name: MYSQL_DATABASE 68 | value: testdb 69 | - name: MYSQL_USER 70 | value: airflow 71 | - name: MYSQL_PASSWORD 72 | valueFrom: 73 | secretKeyRef: 74 | key: password 75 | name: {{.SecretName}} 76 | - name: MYSQL_ROOT_PASSWORD 77 | valueFrom: 78 | secretKeyRef: 79 | key: rootpassword 80 | name: {{.SecretName}} 81 | image: {{.Base.Spec.MySQL.Image}}:{{.Base.Spec.MySQL.Version}} 82 | imagePullPolicy: IfNotPresent 83 | livenessProbe: 84 | exec: 85 | command: 86 | - bash 87 | - -c 88 | - mysqladmin -p$MYSQL_ROOT_PASSWORD ping 89 | failureThreshold: 3 90 | initialDelaySeconds: 30 91 | periodSeconds: 20 92 | successThreshold: 1 93 | timeoutSeconds: 5 94 | ports: 95 | - containerPort: 3306 96 | name: mysql 97 | protocol: TCP 98 | readinessProbe: 99 | exec: 100 | command: 101 | - bash 102 | - -c 103 | - mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e "use testdb" 104 | failureThreshold: 3 105 | initialDelaySeconds: 10 106 | periodSeconds: 5 107 | successThreshold: 1 108 | timeoutSeconds: 2 109 | resources: {} 110 | volumeMounts: 111 | - name: data 112 | mountPath: /var/lib/mysql 113 | restartPolicy: Always 114 | {{if .Base.Spec.MySQL.VolumeClaimTemplate}} 115 | {{else}} 116 | volumes: 117 | - emptyDir: {} 118 | name: data 119 | {{end}} 120 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/storage/storage_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package storage_test 17 | 18 | import ( 19 | "fmt" 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | corev1 "k8s.io/api/core/v1" 23 | "k8s.io/apimachinery/pkg/util/validation/field" 24 | "sigs.k8s.io/controller-reconciler/pkg/storage" 25 | ) 26 | 27 | var _ = Describe("Resource", func() { 28 | gcs := storage.GCS{ 29 | Bucket: "asdasd", 30 | ReadOnly: true, 31 | } 32 | nfs := corev1.NFSVolumeSource{ 33 | Server: "asdasd", 34 | Path: "/asd", 35 | } 36 | gcsonly := storage.Spec{GCS: &gcs} 37 | nfsonly := storage.Spec{NFS: &nfs} 38 | both := storage.Spec{GCS: &gcs, NFS: &nfs} 39 | none := storage.Spec{} 40 | fp := field.NewPath("spec") 41 | 42 | BeforeEach(func() { 43 | }) 44 | 45 | Describe("Validate", func() { 46 | fmt.Printf(".") 47 | It("validate expected specs exist", func(done Done) { 48 | var e field.ErrorList 49 | e = gcsonly.Validate(fp, "storage", e, false, true, map[string]string{storage.FieldGCS: storage.Expected}) 50 | Expect(len(e)).To(Equal(0)) 51 | e = nfsonly.Validate(fp, "storage", e, false, true, map[string]string{storage.FieldNFS: storage.Expected}) 52 | Expect(len(e)).To(Equal(0)) 53 | e = both.Validate(fp, "storage", e, false, false, map[string]string{storage.FieldNFS: storage.Expected, storage.FieldGCS: storage.Expected}) 54 | Expect(len(e)).To(Equal(0)) 55 | close(done) 56 | }) 57 | It("validate missing expected specs", func(done Done) { 58 | var e field.ErrorList 59 | e = none.Validate(fp, "storage", e, false, false, map[string]string{storage.FieldNFS: storage.Expected}) 60 | //fmt.Printf("%s", e.ToAggregate().Error()) 61 | Expect(len(e)).To(Equal(2)) 62 | 63 | e = gcsonly.Validate(fp, "storage", e, false, true, map[string]string{storage.FieldNFS: storage.Expected}) 64 | //fmt.Printf("%s", e.ToAggregate().Error()) 65 | Expect(len(e)).To(Equal(5)) 66 | 67 | e = nfsonly.Validate(fp, "storage", e, false, true, map[string]string{storage.FieldGCS: storage.Expected}) 68 | //fmt.Printf("%s", e.ToAggregate().Error()) 69 | Expect(len(e)).To(Equal(8)) 70 | 71 | close(done) 72 | }) 73 | It("validate only one exist", func(done Done) { 74 | var e field.ErrorList 75 | e = nfsonly.Validate(fp, "storage", e, false, true, map[string]string{storage.FieldNFS: storage.Optional, storage.FieldGCS: storage.Optional}) 76 | Expect(len(e)).To(Equal(0)) 77 | 78 | e = both.Validate(fp, "storage", e, false, true, map[string]string{storage.FieldNFS: storage.Optional, storage.FieldGCS: storage.Optional}) 79 | //fmt.Printf("%s", e.ToAggregate().Error()) 80 | Expect(len(e)).To(Equal(1)) 81 | 82 | e = none.Validate(fp, "storage", e, false, true, map[string]string{storage.FieldNFS: storage.Optional, storage.FieldGCS: storage.Optional}) 83 | fmt.Printf("%s", e.ToAggregate().Error()) 84 | Expect(len(e)).To(Equal(2)) 85 | 86 | close(done) 87 | }) 88 | It("validate no storage spec errors", func(done Done) { 89 | var e field.ErrorList 90 | e = none.Validate(fp, "storage", e, true, false, map[string]string{storage.FieldNFS: storage.Optional, storage.FieldGCS: storage.Optional}) 91 | Expect(len(e)).To(Equal(0)) 92 | 93 | close(done) 94 | }) 95 | }) 96 | }) 97 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/gcp/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package gcp 15 | 16 | import ( 17 | "cloud.google.com/go/compute/metadata" 18 | "flag" 19 | "google.golang.org/api/googleapi" 20 | "net/http" 21 | "regexp" 22 | "sigs.k8s.io/controller-reconciler/pkg/reconciler" 23 | "strings" 24 | ) 25 | 26 | var ( 27 | projectID, zone string 28 | ) 29 | 30 | func init() { 31 | // TODO: Fix this to allow double vendoring this library but still register flags on behalf of users 32 | flag.StringVar(&projectID, "gcpproject", "", 33 | "GCP Project ID. Required for outside cluster.") 34 | flag.StringVar(&zone, "gcpzone", "", 35 | "GCP Zone. Required for outside cluster.") 36 | } 37 | 38 | // CompliantLabelString - convert to GCP compliant string 39 | // https://cloud.google.com/resource-manager/docs/creating-managing-labels 40 | func CompliantLabelString(s string) string { 41 | s = strings.ToLower(s) 42 | reg, _ := regexp.Compile("[^a-z0-9_-]+") 43 | s = reg.ReplaceAllString(s, "-") 44 | return s 45 | } 46 | 47 | // CompliantLabelMap - convert to GCP compliant map 48 | // https://cloud.google.com/resource-manager/docs/creating-managing-labels 49 | func CompliantLabelMap(in map[string]string) map[string]string { 50 | out := make(map[string]string) 51 | for k, v := range in { 52 | out[CompliantLabelString(k)] = CompliantLabelString(v) 53 | } 54 | return out 55 | } 56 | 57 | // IsNotAuthorized - true if not-authorized error is returned 58 | func IsNotAuthorized(err error) bool { 59 | return isGoogleErrorWithCode(err, http.StatusForbidden) 60 | } 61 | 62 | // IsNotFound - true if not-found error is returned 63 | func IsNotFound(err error) bool { 64 | return isGoogleErrorWithCode(err, http.StatusNotFound) 65 | } 66 | 67 | func isGoogleErrorWithCode(err error, code int) bool { 68 | if err == nil { 69 | return false 70 | } 71 | if ge, ok := err.(*googleapi.Error); ok { 72 | return ge.Code == code 73 | } 74 | return false 75 | } 76 | 77 | // GetProjectFromMetadata - get metadata 78 | func GetProjectFromMetadata() (string, error) { 79 | if projectID != "" { 80 | return projectID, nil 81 | } 82 | return metadata.ProjectID() 83 | } 84 | 85 | // GetZoneFromMetadata - get metadata 86 | func GetZoneFromMetadata() (string, error) { 87 | if zone != "" { 88 | return zone, nil 89 | } 90 | return metadata.Zone() 91 | } 92 | 93 | // GetFilterStringFromLabels returns filter string 94 | func GetFilterStringFromLabels(map[string]string) string { 95 | return "" 96 | } 97 | 98 | // Objects internal 99 | type Objects struct { 100 | err error 101 | context interface{} 102 | bag []reconciler.Object 103 | labels map[string]string 104 | } 105 | 106 | // NewObjects returns nag 107 | func NewObjects() *Objects { 108 | return &Objects{ 109 | bag: []reconciler.Object{}, 110 | } 111 | } 112 | 113 | //WithContext injects context for mutators 114 | func (b *Objects) WithContext(context interface{}) *Objects { 115 | b.context = context 116 | return b 117 | } 118 | 119 | //WithLabels injects labels 120 | func (b *Objects) WithLabels(labels map[string]string) *Objects { 121 | b.labels = labels 122 | return b 123 | } 124 | 125 | //Build - process 126 | func (b *Objects) Build() ([]reconciler.Object, error) { 127 | return b.bag, b.err 128 | } 129 | 130 | // Add - add obj 131 | func (b *Objects) Add(o *reconciler.Object, err error) *Objects { 132 | if err != nil { 133 | b.err = err 134 | } else { 135 | o.Obj.SetLabels(b.labels) 136 | b.bag = append(b.bag, *o) 137 | } 138 | return b 139 | } 140 | -------------------------------------------------------------------------------- /vendor/sigs.k8s.io/controller-reconciler/pkg/genericreconciler/genericreconciler_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package genericreconciler_test 17 | 18 | import ( 19 | "context" 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | appsv1 "k8s.io/api/apps/v1" 23 | corev1 "k8s.io/api/core/v1" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/types" 27 | "k8s.io/client-go/kubernetes/scheme" 28 | "sigs.k8s.io/controller-reconciler/pkg/genericreconciler" 29 | test "sigs.k8s.io/controller-reconciler/pkg/genericreconciler/v1alpha1" 30 | "sigs.k8s.io/controller-reconciler/pkg/reconciler" 31 | "sigs.k8s.io/controller-reconciler/pkg/reconciler/manager/k8s" 32 | "sigs.k8s.io/controller-runtime/pkg/client" 33 | "sigs.k8s.io/controller-runtime/pkg/client/fake" 34 | ) 35 | 36 | var AddToSchemes runtime.SchemeBuilder 37 | var _ = Describe("Reconciler", func() { 38 | cl := fake.NewFakeClient() 39 | AddToSchemes.AddToScheme(scheme.Scheme) 40 | gr := genericreconciler.Reconciler{} 41 | // TODO gr.CR = customresource.CustomResource{Handle: &test.Foo{}} 42 | gr.For(&test.Foo{}, test.SchemeGroupVersion). 43 | WithResourceManager(k8s.Getter(context.TODO(), cl, scheme.Scheme)). 44 | Using(&test.FooHandler{}) 45 | m1 := reconciler.KVMap{"k1": "v1"} 46 | m2 := reconciler.KVMap{"k2": "v2"} 47 | m3 := reconciler.KVMap{"k3": "v3"} 48 | 49 | BeforeEach(func() {}) 50 | 51 | Describe("Status", func() { 52 | It("should be able to Merge KVmap", func(done Done) { 53 | m1.Merge(m2, m3, reconciler.KVMap{"k3": "v3"}) 54 | Expect(len(m1)).To(Equal(3)) 55 | close(done) 56 | }) 57 | It("should not be able to Get non-existing deployment", func() { 58 | By("Getting a deployment") 59 | namespacedName := types.NamespacedName{ 60 | Name: "test-deployment", 61 | Namespace: "ns1", 62 | } 63 | obj := &appsv1.Deployment{} 64 | err := cl.Get(nil, namespacedName, obj) 65 | Expect(err).NotTo(BeNil()) 66 | }) 67 | It("should able to List Service", func() { 68 | By("Listing service") 69 | opts := client.ListOptions{Raw: &metav1.ListOptions{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}} 70 | err := cl.List(context.TODO(), &corev1.ServiceList{}, &opts) 71 | Expect(err).To(BeNil()) 72 | }) 73 | It("should not be able to Reconcile missing custom resource", func() { 74 | By("Getting a deployment") 75 | namespacedName := types.NamespacedName{ 76 | Name: "crA", 77 | Namespace: "ns1", 78 | } 79 | _, err := gr.ReconcileResource(namespacedName) 80 | Expect(err).To(BeNil()) 81 | }) 82 | It("should be able to Reconcile custom resource", func() { 83 | By("Getting a deployment") 84 | cl.Create(context.TODO(), &test.Foo{ 85 | ObjectMeta: metav1.ObjectMeta{ 86 | Name: "crA", 87 | Namespace: "ns1", 88 | }, 89 | Spec: test.FooSpec{ 90 | Version: "", 91 | }, 92 | }) 93 | namespacedName := types.NamespacedName{ 94 | Name: "crA", 95 | Namespace: "ns1", 96 | } 97 | _, err := gr.ReconcileResource(namespacedName) 98 | Expect(err).To(BeNil()) 99 | sts := &appsv1.Deployment{} 100 | err = cl.Get(nil, types.NamespacedName{Name: "crA-deploy", Namespace: "ns1"}, sts) 101 | Expect(err).To(BeNil()) 102 | cm := &corev1.ConfigMap{} 103 | err = cl.Get(nil, types.NamespacedName{Name: "crA-cm", Namespace: "ns1"}, cm) 104 | Expect(err).To(BeNil()) 105 | }) 106 | 107 | }) 108 | }) 109 | --------------------------------------------------------------------------------