├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── TODO.md ├── chart ├── .helmignore ├── Chart.yaml ├── README.md ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── clusterrole.yaml │ ├── clusterrolebinding.yaml │ ├── configmap.yaml │ ├── crd.yaml │ ├── deployment.yaml │ └── serviceaccount.yaml └── values.yaml ├── controller.py ├── example-config.yaml ├── examples ├── multi-instance.yaml ├── simple.yaml ├── with-extensions.yaml └── with-sql.yaml ├── functions.py ├── img ├── k8s-logo.png └── postgres-logo.png └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | kubeconfig 107 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7.0-alpine3.8 2 | 3 | COPY requirements.txt / 4 | 5 | RUN apk --update add postgresql-dev \ 6 | && apk --update add --virtual build-dependencies gcc libffi-dev musl-dev \ 7 | && pip install -r requirements.txt \ 8 | && apk del build-dependencies 9 | 10 | COPY controller.py functions.py / 11 | 12 | CMD ["python", "-u", "controller.py"] 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Max Williams 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes postgres-controller 2 | 3 | + 4 | 5 | A simple k8s controller to create PostgresSQL databases. Once you install the controller and point it at your existing PostgresSQL database instance, you can create `PostgresDatabase` resource in k8s and the controller will create a database in your PostgresSQL instance, create a role that with access to this database and optionally install any extensions and run extra SQL commands. 6 | 7 | Example resource: 8 | 9 | ```yaml 10 | apiVersion: postgresql.org/v1 11 | kind: PostgresDatabase 12 | metadata: 13 | name: app1 14 | spec: 15 | dbName: db1 16 | dbRoleName: user1 17 | dbRolePassword: swordfish 18 | ``` 19 | 20 | Pull requests welcome. 21 | 22 | ### Installation 23 | 24 | Use the included [Helm](https://helm.sh/) chart and set the host, username and password for your default PostgresSQL instance: 25 | 26 | ``` 27 | helm install ./chart --set config.postgres_instances.default.host=my-rds-instance.rds.amazonaws.com --set config.postgres_instances.default.user=root --set config.postgres_instances.default.password=admin_password 28 | ``` 29 | 30 | Or use the docker image: [maxrocketinternet/postgres-controller](https://hub.docker.com/r/maxrocketinternet/postgres-controller) 31 | 32 | ### Examples 33 | 34 | See [examples](examples) to for how to add extensions, extra SQL commands and also how to drop databases when the k8s resource is deleted. 35 | 36 | See [example-config.yaml](example-config.yaml) for example chart values file. 37 | 38 | ### Testing 39 | 40 | To test locally, start a postgres container: 41 | 42 | ``` 43 | docker run -d -p 127.0.0.1:5432:5432 -e POSTGRES_PASSWORD=postgres postgres:9.6 44 | ``` 45 | 46 | Start the controller, it will use your default kubectl configuration/context: 47 | 48 | ``` 49 | ./controller.py --log-level=debug --config-file=example-config.yaml 50 | ``` 51 | 52 | Create or change some resources: 53 | 54 | ``` 55 | kubectl apply -f examples/simple.yaml 56 | ``` 57 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # To do 2 | 3 | - Fix `openAPIV3Schema` validation in CRD definition so that it matches Postgres standards 4 | - Fix `additionalPrinterColumns` in CRD definition 5 | - Add update events of `PostgresDatabase` resources (only beta in k8s 1.11) 6 | - Error checking and exception catching x1000 7 | - Reduce docker image size 8 | - Handle SIGTERM 9 | - Write a class for handling events 10 | 11 | Features: 12 | 13 | - Add CRD for managing Postgres roles 14 | - Allow granular permissions for dbRoleName 15 | - Handle `MODIFIED` type of events? 16 | -------------------------------------------------------------------------------- /chart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: "0.5" 3 | description: A controller for managing PostgreSQL databases, roles and more 4 | name: postgres-controller 5 | version: 1.1 6 | -------------------------------------------------------------------------------- /chart/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes PostgreSQL Controller 2 | 3 | This chart creates a [postgres-controller](https://github.com/max-rocket-internet/postgres-controller) deployment and a `PostgresDatabase` Custom Resource Definition. 4 | 5 | ## TL;DR; 6 | 7 | You will need to set at least 3 parameters when installing. These options set the default database hostname, username and password: 8 | 9 | ```console 10 | $ helm install --name my-release ./chart --set config.postgres_instances.default.host=my-rds-instance.rds.amazonaws.com --set config.postgres_instances.default.user=root --set config.postgres_instances.default.password=swordfish 11 | ``` 12 | 13 | ## Prerequisites 14 | 15 | - Kubernetes 1.8+ 16 | 17 | ## Uninstalling the Chart 18 | 19 | To delete the chart: 20 | 21 | ```console 22 | $ helm delete --purge my-release 23 | ``` 24 | 25 | ## Configuration 26 | 27 | The following table lists the configurable parameters for this chart and their default values. 28 | 29 | | Parameter | Description | Default | 30 | |--------------------|------------------------------------------------------------------|--------------------------------| 31 | | `annotations` | Optional deployment annotations | `{}` | 32 | | `affinity` | Map of node/pod affinities | `{}` | 33 | | `image.repository` | Image | `maxrocketinternet/postgres-controller` | 34 | | `image.tag` | Image tag | `0.5` | 35 | | `image.pullPolicy` | Image pull policy | `IfNotPresent` | 36 | | `rbac.create` | RBAC | `true` | 37 | | `resources` | Pod resource requests and limits | `{}` | 38 | | `tolerations` | Optional deployment tolerations | `[]` | 39 | | `config` | Map containing Postgres instances. See `values.yaml` for example | `{}` | 40 | | `log_level` | Log level for controller (`info`|`debug`) | `info` | 41 | 42 | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install` or provide a YAML file containing the values for the above parameters: 43 | 44 | ```console 45 | $ helm install --name my-release ./chart --values my-values.yaml 46 | ``` 47 | -------------------------------------------------------------------------------- /chart/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | To verify that the postgres-controller pods have started, run: 2 | 3 | kubectl --namespace={{ .Release.Namespace }} get pods -l "app={{ template "postgres-controller.name" . }},release={{ .Release.Name }}" 4 | -------------------------------------------------------------------------------- /chart/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | 3 | {{/* 4 | Expand the name of the chart. 5 | */}} 6 | {{- define "postgres-controller.name" -}} 7 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 8 | {{- end -}} 9 | 10 | {{/* 11 | Create a default fully qualified app name. 12 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 13 | If release name contains chart name it will be used as a full name. 14 | */}} 15 | {{- define "postgres-controller.fullname" -}} 16 | {{- if .Values.fullnameOverride -}} 17 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 18 | {{- else -}} 19 | {{- $name := default .Chart.Name .Values.nameOverride -}} 20 | {{- if contains $name .Release.Name -}} 21 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 22 | {{- else -}} 23 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 24 | {{- end -}} 25 | {{- end -}} 26 | {{- end -}} 27 | 28 | {{/* 29 | Create chart name and version as used by the chart label. 30 | */}} 31 | {{- define "postgres-controller.chart" -}} 32 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 33 | {{- end -}} 34 | 35 | {{/* Generate basic labels */}} 36 | {{- define "postgres-controller.labels" }} 37 | app: {{ template "postgres-controller.name" . }} 38 | heritage: {{.Release.Service }} 39 | release: {{.Release.Name }} 40 | {{- if .Values.podLabels }} 41 | {{ toYaml .Values.podLabels }} 42 | {{- end }} 43 | {{- end }} 44 | 45 | {{/* Create the name of the service account to use */}} 46 | {{- define "postgres-controller.serviceAccountName" -}} 47 | {{- if .Values.serviceAccount.create -}} 48 | {{ default (include "postgres-controller.fullname" .) .Values.serviceAccount.name }} 49 | {{- else -}} 50 | {{ default "default" .Values.serviceAccount.name }} 51 | {{- end -}} 52 | {{- end -}} 53 | -------------------------------------------------------------------------------- /chart/templates/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create -}} 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | labels: 6 | app: {{ template "postgres-controller.name" . }} 7 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 8 | heritage: {{ .Release.Service }} 9 | release: {{ .Release.Name }} 10 | app.kubernetes.io/name: {{ include "postgres-controller.name" . }} 11 | helm.sh/chart: {{ include "postgres-controller.chart" . }} 12 | app.kubernetes.io/instance: {{ .Release.Name }} 13 | app.kubernetes.io/managed-by: {{ .Release.Service }} 14 | name: {{ template "postgres-controller.fullname" . }} 15 | rules: 16 | - apiGroups: ["postgresql.org"] 17 | resources: ["pgdatabases"] 18 | verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] 19 | {{- end -}} 20 | -------------------------------------------------------------------------------- /chart/templates/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create -}} 2 | apiVersion: rbac.authorization.k8s.io/v1beta1 3 | kind: ClusterRoleBinding 4 | metadata: 5 | labels: 6 | app: {{ template "postgres-controller.name" . }} 7 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 8 | heritage: {{ .Release.Service }} 9 | release: {{ .Release.Name }} 10 | app.kubernetes.io/name: {{ include "postgres-controller.name" . }} 11 | helm.sh/chart: {{ include "postgres-controller.chart" . }} 12 | app.kubernetes.io/instance: {{ .Release.Name }} 13 | app.kubernetes.io/managed-by: {{ .Release.Service }} 14 | name: {{ template "postgres-controller.fullname" . }} 15 | roleRef: 16 | apiGroup: rbac.authorization.k8s.io 17 | kind: ClusterRole 18 | name: {{ template "postgres-controller.fullname" . }} 19 | subjects: 20 | - kind: ServiceAccount 21 | name: {{ template "postgres-controller.serviceAccountName" . }} 22 | namespace: {{ .Release.Namespace }} 23 | {{- end -}} 24 | -------------------------------------------------------------------------------- /chart/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | labels: 5 | app: {{ template "postgres-controller.name" . }} 6 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 7 | heritage: {{ .Release.Service }} 8 | release: {{ .Release.Name }} 9 | app.kubernetes.io/name: {{ include "postgres-controller.name" . }} 10 | helm.sh/chart: {{ include "postgres-controller.chart" . }} 11 | app.kubernetes.io/instance: {{ .Release.Name }} 12 | app.kubernetes.io/managed-by: {{ .Release.Service }} 13 | name: {{ template "postgres-controller.fullname" . }} 14 | data: 15 | postgres-controller.yaml: | 16 | {{ .Values.config | toYaml | indent 4 }} 17 | -------------------------------------------------------------------------------- /chart/templates/crd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1beta1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | name: pgdatabases.postgresql.org 5 | labels: 6 | app: {{ template "postgres-controller.name" . }} 7 | chart: {{ template "postgres-controller.chart" . }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | app.kubernetes.io/name: {{ include "postgres-controller.name" . }} 11 | helm.sh/chart: {{ include "postgres-controller.chart" . }} 12 | app.kubernetes.io/instance: {{ .Release.Name }} 13 | app.kubernetes.io/managed-by: {{ .Release.Service }} 14 | spec: 15 | group: postgresql.org 16 | {{- if semverCompare ">=1.11" .Capabilities.KubeVersion.GitVersion }} 17 | versions: 18 | - name: v1 19 | served: true 20 | storage: true 21 | {{- else }} 22 | version: v1 23 | {{- end }} 24 | scope: Namespaced 25 | names: 26 | plural: pgdatabases 27 | singular: pgdatabase 28 | kind: PostgresDatabase 29 | shortNames: 30 | - pgdb 31 | additionalPrinterColumns: 32 | - name: DBNAME 33 | type: string 34 | description: The name of the database 35 | JSONPath: .spec.dbName 36 | validation: 37 | openAPIV3Schema: 38 | properties: 39 | spec: 40 | required: 41 | - dbName 42 | - dbRoleName 43 | - dbRolePassword 44 | properties: 45 | dbName: 46 | type: string 47 | dbRoleName: 48 | type: string 49 | dbRolePassword: 50 | type: string 51 | dbInstanceId: 52 | type: string 53 | dbExtensions: 54 | type: array 55 | items: 56 | type: string 57 | extraSQL: 58 | type: string 59 | -------------------------------------------------------------------------------- /chart/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta2 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "postgres-controller.fullname" . }} 5 | labels: 6 | app: {{ template "postgres-controller.name" . }} 7 | chart: {{ template "postgres-controller.chart" . }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | app.kubernetes.io/name: {{ include "postgres-controller.name" . }} 11 | helm.sh/chart: {{ include "postgres-controller.chart" . }} 12 | app.kubernetes.io/instance: {{ .Release.Name }} 13 | app.kubernetes.io/managed-by: {{ .Release.Service }} 14 | {{- if .Values.labels }} 15 | {{ toYaml .Values.labels | indent 4 }} 16 | {{- end }} 17 | spec: 18 | replicas: {{ .Values.replicaCount }} 19 | selector: 20 | matchLabels: 21 | app.kubernetes.io/name: {{ include "postgres-controller.name" . }} 22 | app.kubernetes.io/instance: {{ .Release.Name }} 23 | template: 24 | metadata: 25 | annotations: 26 | checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} 27 | labels: 28 | app.kubernetes.io/name: {{ include "postgres-controller.name" . }} 29 | app.kubernetes.io/instance: {{ .Release.Name }} 30 | app: {{ template "postgres-controller.name" . }} 31 | release: {{ .Release.Name }} 32 | {{- if .Values.pod_labels }} 33 | {{ toYaml .Values.pod_labels | indent 8 }} 34 | {{- end }} 35 | spec: 36 | serviceAccountName: {{ template "postgres-controller.serviceAccountName" . }} 37 | containers: 38 | - name: {{ .Chart.Name }} 39 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 40 | imagePullPolicy: {{ .Values.image.pullPolicy }} 41 | env: 42 | - name: CONFIG_FILE 43 | value: "/config/postgres-controller.yaml" 44 | - name: LOG_LEVEL 45 | value: "{{ .Values.log_level }}" 46 | resources: 47 | {{ toYaml .Values.resources | indent 12 }} 48 | volumeMounts: 49 | - name: config 50 | mountPath: /config 51 | volumes: 52 | - name: config 53 | configMap: 54 | name: {{ include "postgres-controller.fullname" . }} 55 | {{- if .Values.nodeSelector }} 56 | nodeSelector: 57 | {{ toYaml .Values.nodeSelector | indent 8 }} 58 | {{- end }} 59 | {{- if .Values.tolerations }} 60 | tolerations: 61 | {{ toYaml .Values.tolerations | indent 8 }} 62 | {{- end }} 63 | {{- if .Values.affinity }} 64 | affinity: 65 | {{ toYaml .Values.affinity | indent 8 }} 66 | {{- end }} 67 | -------------------------------------------------------------------------------- /chart/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if or .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | app: {{ template "postgres-controller.name" . }} 7 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 8 | heritage: {{ .Release.Service }} 9 | release: {{ .Release.Name }} 10 | app.kubernetes.io/name: {{ include "postgres-controller.name" . }} 11 | helm.sh/chart: {{ include "postgres-controller.chart" . }} 12 | app.kubernetes.io/instance: {{ .Release.Name }} 13 | app.kubernetes.io/managed-by: {{ .Release.Service }} 14 | name: {{ template "postgres-controller.serviceAccountName" . }} 15 | {{- end -}} 16 | -------------------------------------------------------------------------------- /chart/values.yaml: -------------------------------------------------------------------------------- 1 | # Controller configuration 2 | # Contains list of Postgres instances. Should have at least a `default` instance 3 | config: 4 | postgres_instances: 5 | # default: 6 | # host: "" 7 | # user: "" 8 | # password: "" 9 | # port: 5432 10 | 11 | log_level: 'info' 12 | 13 | replicaCount: 1 14 | 15 | image: 16 | repository: maxrocketinternet/postgres-controller 17 | tag: 0.5 18 | pullPolicy: IfNotPresent 19 | 20 | nameOverride: "" 21 | fullnameOverride: "" 22 | 23 | resources: {} 24 | 25 | nodeSelector: {} 26 | 27 | tolerations: [] 28 | 29 | affinity: {} 30 | 31 | rbac: 32 | create: true 33 | 34 | serviceAccount: 35 | # Specifies whether a ServiceAccount should be created 36 | create: true 37 | # The name of the ServiceAccount to use. 38 | # If not set and create is true, a name is generated using the fullname template 39 | name: 40 | -------------------------------------------------------------------------------- /controller.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | from kubernetes import client, watch 5 | from functions import PostgresControllerConfig, process_event, create_logger, parse_too_old_failure 6 | 7 | 8 | runtime_config = PostgresControllerConfig() 9 | crds = client.CustomObjectsApi() 10 | logger = create_logger(log_level=runtime_config.args.log_level) 11 | 12 | 13 | resource_version = '' 14 | 15 | 16 | if __name__ == "__main__": 17 | logger.info('postgres-controller initializing') 18 | while True: 19 | stream = watch.Watch().stream(crds.list_cluster_custom_object, 'postgresql.org', 'v1', 'pgdatabases', resource_version=resource_version) 20 | try: 21 | for event in stream: 22 | event_type = event["type"] 23 | obj = event["object"] 24 | metadata = obj.get('metadata') 25 | spec = obj.get('spec') 26 | code = obj.get('code') 27 | 28 | if code == 410: 29 | logger.debug('Error code 410') 30 | new_version = parse_too_old_failure(obj.get('message')) 31 | if new_version == None: 32 | logger.error('Failed to parse 410 error code') 33 | resource_version = '' 34 | break 35 | else: 36 | resource_version = new_version 37 | logger.debug('Updating resource version to {0} due to "too old" error'.format(new_version)) 38 | break 39 | 40 | if not metadata or not spec: 41 | logger.error('No metadata or spec in object, skipping: {0}'.format(json.dumps(obj, indent=1))) 42 | break 43 | 44 | if metadata['resourceVersion'] is not None: 45 | resource_version = metadata['resourceVersion'] 46 | logger.debug('resourceVersion now: {0}'.format(resource_version)) 47 | 48 | process_event(crds, obj, event_type, runtime_config) 49 | 50 | except client.rest.ApiException as e: 51 | if e.status == 404: 52 | logger.error('Custom Resource Definition not created in cluster') 53 | break 54 | elif e.status == 401: 55 | logger.error('Unauthorized') 56 | sys.exit(1) 57 | else: 58 | raise e 59 | except KeyboardInterrupt: 60 | break 61 | -------------------------------------------------------------------------------- /example-config.yaml: -------------------------------------------------------------------------------- 1 | # Example config file for controller 2 | postgres_instances: 3 | default: 4 | host: localhost 5 | user: postgres 6 | password: postgres 7 | port: 5432 8 | instance2: 9 | host: localhost 10 | user: postgres 11 | password: postgres 12 | port: 5432 13 | -------------------------------------------------------------------------------- /examples/multi-instance.yaml: -------------------------------------------------------------------------------- 1 | # If dbInstanceId is not in the resource, the controller will look for a `default` key 2 | apiVersion: postgresql.org/v1 3 | kind: PostgresDatabase 4 | metadata: 5 | name: app1 6 | spec: 7 | dbName: db1 8 | dbRoleName: user1 9 | dbRolePassword: swordfish 10 | # This string must correspond to a key in one of `postgres_instances` in the config file 11 | dbInstanceId: default 12 | --- 13 | apiVersion: postgresql.org/v1 14 | kind: PostgresDatabase 15 | metadata: 16 | name: app2 17 | spec: 18 | dbName: db2 19 | dbRoleName: user2 20 | dbRolePassword: swordfish 21 | # This string must correspond to a key in one of `postgres_instances` in the config file 22 | dbInstanceId: instance2 23 | -------------------------------------------------------------------------------- /examples/simple.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: postgresql.org/v1 2 | kind: PostgresDatabase 3 | metadata: 4 | name: app3 5 | spec: 6 | dbName: db3 7 | dbRoleName: user3 8 | dbRolePassword: swordfish 9 | onDeletion: 10 | # Whether to drop the databse when the resource is deleted 11 | dropDB: false 12 | # Whether to drop the role when the resource is deleted 13 | dropRole: false 14 | -------------------------------------------------------------------------------- /examples/with-extensions.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: postgresql.org/v1 2 | kind: PostgresDatabase 3 | metadata: 4 | name: app4 5 | spec: 6 | dbName: db4 7 | dbRoleName: user4 8 | dbRolePassword: swordfish 9 | dbExtensions: 10 | - hstore 11 | onDeletion: 12 | dropDB: true 13 | dropRole: true 14 | -------------------------------------------------------------------------------- /examples/with-sql.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: postgresql.org/v1 2 | kind: PostgresDatabase 3 | metadata: 4 | name: app5 5 | spec: 6 | dbName: db5 7 | dbRoleName: user5 8 | dbRolePassword: swordfish 9 | extraSQL: | 10 | CREATE TABLE example ( 11 | user_id serial PRIMARY KEY, 12 | username VARCHAR (50) UNIQUE NOT NULL, 13 | email VARCHAR (355) UNIQUE NOT NULL 14 | ); 15 | INSERT INTO example (user_id, username, email) VALUES (1, 'user1', 'user1@email.com'); 16 | -------------------------------------------------------------------------------- /functions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import sys 5 | import json 6 | import re 7 | from kubernetes import config 8 | import logging 9 | import psycopg2 10 | import logging.handlers 11 | import argparse 12 | import yaml 13 | 14 | 15 | logger = logging.getLogger() 16 | 17 | 18 | class K8sLoggingFilter(logging.Filter): 19 | ''' 20 | A small filter to add add extra logging data if not present 21 | ''' 22 | def filter(self, record): 23 | if not hasattr(record, 'resource_name'): 24 | record.resource_name = '-' 25 | return True 26 | 27 | 28 | def create_logger(log_level): 29 | ''' 30 | Creates logging object 31 | ''' 32 | json_format = logging.Formatter('{"time":"%(asctime)s", "level":"%(levelname)s", "resource_name":"%(resource_name)s", "message":"%(message)s"}') 33 | filter = K8sLoggingFilter() 34 | logger = logging.getLogger() 35 | stdout_handler = logging.StreamHandler() 36 | stdout_handler.setLevel(logging.DEBUG) 37 | stdout_handler.setFormatter(json_format) 38 | logger.addHandler(stdout_handler) 39 | logger.addFilter(filter) 40 | 41 | if log_level == 'debug': 42 | logger.setLevel(logging.DEBUG) 43 | elif log_level == 'info': 44 | logger.setLevel(logging.INFO) 45 | else: 46 | raise Exception('Unsupported log_level {0}'.format(log_level)) 47 | 48 | return logger 49 | 50 | 51 | class PostgresControllerConfig(object): 52 | ''' 53 | Manages run time configuration 54 | ''' 55 | def __init__(self): 56 | if 'KUBERNETES_PORT' in os.environ: 57 | config.load_incluster_config() 58 | else: 59 | config.load_kube_config() 60 | 61 | parser = argparse.ArgumentParser(description='A simple k8s controller to create PostgresSQL databases.') 62 | parser.add_argument('-c', '--config-file', help='Path to config file.', default=os.environ.get('CONFIG_FILE', None)) 63 | parser.add_argument('-l', '--log-level', help='Log level.', choices=['info', 'debug'], default=os.environ.get('LOG_LEVEL', 'info')) 64 | self.args = parser.parse_args() 65 | 66 | if not self.args.config_file: 67 | parser.print_usage() 68 | sys.exit() 69 | 70 | with open(self.args.config_file) as fp: 71 | self.yaml_config = yaml.safe_load(fp) 72 | if 'postgres_instances' not in self.yaml_config or len(self.yaml_config['postgres_instances'].keys()) < 1: 73 | raise Exception('No postgres instances in configuration') 74 | 75 | def get_credentials(self, instance_id): 76 | ''' 77 | Returns the correct instance credentials from current list in configuration 78 | ''' 79 | creds = None 80 | 81 | if instance_id == None: 82 | instance_id = 'default' 83 | 84 | for id, data in self.yaml_config['postgres_instances'].items(): 85 | if id == instance_id: 86 | creds = data.copy() 87 | if 'dbname' not in creds: 88 | creds['dbname'] = 'postgres' 89 | break 90 | 91 | return creds 92 | 93 | 94 | def parse_too_old_failure(message): 95 | ''' 96 | Parse an error from watch API when resource version is too old 97 | ''' 98 | regex = r"too old resource version: .* \((.*)\)" 99 | 100 | result = re.search(regex, message) 101 | if result == None: 102 | return None 103 | 104 | match = result.group(1) 105 | if match == None: 106 | return None 107 | 108 | try: 109 | return int(match) 110 | except: 111 | return None 112 | 113 | 114 | def create_db_if_not_exists(cur, db_name): 115 | ''' 116 | A function to create a database if it does not already exist 117 | ''' 118 | cur.execute("SELECT 1 FROM pg_database WHERE datname = '{}';".format(db_name)) 119 | if not cur.fetchone(): 120 | cur.execute("CREATE DATABASE {};".format(db_name)) 121 | return True 122 | else: 123 | return False 124 | 125 | 126 | def create_role_not_exists(cur, role_name, role_password): 127 | ''' 128 | A function to create a role if it does not already exist 129 | ''' 130 | cur.execute("SELECT 1 FROM pg_roles WHERE rolname = '{}';".format(role_name)) 131 | if not cur.fetchone(): 132 | cur.execute("CREATE ROLE {0} PASSWORD '{1}' LOGIN;".format(role_name, role_password)) 133 | return True 134 | else: 135 | return False 136 | 137 | 138 | def process_event(crds, obj, event_type, runtime_config): 139 | ''' 140 | Processes events in order to create or drop databases 141 | ''' 142 | spec = obj.get('spec') 143 | metadata = obj.get('metadata') 144 | k8s_resource_name = metadata.get('name') 145 | 146 | logger = logging.LoggerAdapter(logging.getLogger(), {'resource_name': k8s_resource_name}) 147 | 148 | logger.debug('Processing event {0}: {1}'.format(event_type, json.dumps(obj, indent=1))) 149 | 150 | if event_type == 'MODIFIED': 151 | logger.debug('Ignoring modification for DB {0}, not supported'.format(spec['dbName'])) 152 | return 153 | 154 | db_credentials = runtime_config.get_credentials(instance_id=spec.get('dbInstanceId')) 155 | 156 | if db_credentials == None: 157 | logger.error('No corresponding postgres instance in configuration for instance id {0}'.format(spec.get('dbInstanceId'))) 158 | return 159 | 160 | try: 161 | conn = psycopg2.connect(**db_credentials) 162 | cur = conn.cursor() 163 | conn.set_session(autocommit=True) 164 | except Exception as e: 165 | logger.error('Error when connecting to DB instance {0}: {1}'.format(spec.get('dbInstanceId'), e)) 166 | return 167 | 168 | 169 | if event_type == 'DELETED': 170 | try: 171 | drop_db = spec['onDeletion']['dropDB'] 172 | except KeyError: 173 | drop_db = False 174 | 175 | if drop_db == True: 176 | try: 177 | cur.execute("DROP DATABASE {0};".format(spec['dbName'])) 178 | except psycopg2.OperationalError as e: 179 | logger.error('Dropping of dbName {0} failed: {1}'.format(spec['dbName'], e)) 180 | else: 181 | logger.info('Dropped dbName {0}'.format(spec['dbName'])) 182 | else: 183 | logger.info('Ignoring deletion for dbName {0}, onDeletion setting not enabled'.format(spec['dbName'])) 184 | 185 | try: 186 | drop_role = spec['onDeletion']['dropRole'] 187 | except KeyError: 188 | drop_role = False 189 | 190 | if drop_role == True: 191 | try: 192 | cur.execute("DROP ROLE {0};".format(spec['dbRoleName'])) 193 | except Exception as e: 194 | logger.error('Error when dropping role {0}: {1}'.format(spec['dbRoleName'], e)) 195 | else: 196 | logger.info('Dropped role {0}'.format(spec['dbRoleName'])) 197 | else: 198 | logger.info('Ignoring deletion of role {0}, onDeletion setting not enabled'.format(spec['dbRoleName'])) 199 | 200 | logger.info('Deleted') 201 | 202 | 203 | elif event_type == 'ADDED': 204 | logger.info('Adding dbName {0}'.format(spec['dbName'])) 205 | 206 | db_created = create_db_if_not_exists(cur, spec['dbName']) 207 | if db_created: 208 | logger.info('Database {0} created'.format(spec['dbName'])) 209 | else: 210 | logger.info('Database {0} already exists'.format(spec['dbName'])) 211 | 212 | role_created = create_role_not_exists(cur, spec['dbRoleName'], spec['dbRolePassword']) 213 | if role_created: 214 | logger.info('Role {0} created'.format(spec['dbRoleName'])) 215 | else: 216 | logger.info('Role {0} already exists'.format(spec['dbRoleName'])) 217 | 218 | cur.execute("GRANT ALL PRIVILEGES ON DATABASE {0} to {1};".format(spec['dbName'], spec['dbRoleName'])) 219 | 220 | if ('dbExtensions' in spec or 'extraSQL' in spec) and not db_created: 221 | logger.info('Ingoring extra SQL commands dbName {0} as it is already created'.format(spec['dbName'])) 222 | 223 | elif ('dbExtensions' in spec or 'extraSQL' in spec) and db_created: 224 | user_credentials = { 225 | **db_credentials, 226 | **{ 227 | 'dbname': spec['dbName'], 228 | 'user': spec['dbRoleName'], 229 | 'password': spec['dbRolePassword'], 230 | } 231 | } 232 | 233 | admin_credentials = { 234 | **db_credentials, 235 | **{ 236 | 'dbname': spec['dbName'] 237 | }, 238 | } 239 | 240 | if 'dbExtensions' in spec: 241 | db_conn = psycopg2.connect(**admin_credentials) 242 | db_cur = db_conn.cursor() 243 | db_conn.set_session(autocommit=True) 244 | for ext in spec['dbExtensions']: 245 | logger.info('Creating extension {0} in dbName {1}'.format(ext, spec['dbName'])) 246 | db_cur.execute('CREATE EXTENSION IF NOT EXISTS "{0}";'.format(ext)) 247 | 248 | if 'extraSQL' in spec: 249 | db_conn = psycopg2.connect(**user_credentials) 250 | db_cur = db_conn.cursor() 251 | db_conn.set_session(autocommit=False) 252 | logger.info('Running extra SQL commands for in dbName {0}'.format(spec['dbName'])) 253 | try: 254 | db_cur.execute(spec['extraSQL']) 255 | db_conn.commit() 256 | except psycopg2.OperationalError as e: 257 | logger.error('OperationalError when running extraSQL for dbName {0}: {1}'.format(spec['dbName'], e)) 258 | except psycopg2.ProgrammingError as e: 259 | logger.error('ProgrammingError when running extraSQL for dbName {0}: {1}'.format(spec['dbName'], e)) 260 | 261 | db_cur.close() 262 | 263 | logger.info('Added PostgresDatabase dbName {0}'.format(spec['dbName'])) 264 | 265 | cur.close() 266 | -------------------------------------------------------------------------------- /img/k8s-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/max-rocket-internet/postgres-controller/87b5846c1a92922486a31023de0b98de338c02cf/img/k8s-logo.png -------------------------------------------------------------------------------- /img/postgres-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/max-rocket-internet/postgres-controller/87b5846c1a92922486a31023de0b98de338c02cf/img/postgres-logo.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | kubernetes==11.0.0 2 | psycopg2-binary==2.7.5 3 | --------------------------------------------------------------------------------