├── LICENSE ├── README.md ├── dags ├── cloudsql_to_bigquery.py ├── copy_cloudsql_to_bigquery_v2.py └── mssql_to_bigquery.py ├── media ├── example1.png └── example2.png └── yaml ├── sql_proxy_dvh_service.yaml ├── sql_proxy_mysql_deployment.yaml ├── sql_proxy_postgresql_deployment_with_sa.yaml └── sql_proxy_postgresql_service.yaml /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Anders Elton 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gcp_cloudsql_airflow_bigquery 2 | Example on how to export from cloud sql into bigquery, using airflow 3 | 4 | In this example 3 tables are synced to bigquery 5 | 6 | 2 dimension tables, and one fact. 7 | 8 | Each sync is a full table copy. (this can be improved to use time/id in the sync select) 9 | 10 | 11 | ## Install 12 | 1. Copy content of dag folder into your own 13 | 2. Alter any credentials / etc you might need 14 | 15 | ### Sample of the job 16 | Version2 17 | ![alt text](media/example2.png "airflow job") 18 | 19 | Version 1 (old) 20 | ![alt text](media/example1.png "airflow job") 21 | 22 | -------------------------------------------------------------------------------- /dags/cloudsql_to_bigquery.py: -------------------------------------------------------------------------------- 1 | """ 2 | ### Copy cloudsql tables to bigquery 3 | 4 | Flow is this 5 | 6 | 1. Export table from cloud sql to storage bucket 7 | -> do some fixes on the export. 8 | 2. Export schema to storage bucket 9 | -> Create bigwuery schema from it 10 | 2. Import from storage bucket to bigquery stage location 11 | 3. Run a query in bigquery that will join the stage table with the existing table and overwrite. 12 | """ 13 | import os 14 | import logging 15 | from airflow import DAG 16 | from airflow.operators.bash_operator import BashOperator 17 | from airflow.operators.python_operator import PythonOperator 18 | from airflow.operators.dummy_operator import DummyOperator 19 | from datetime import datetime, timedelta 20 | 21 | 22 | default_args = { 23 | 'owner': 'airflow', 24 | 'depends_on_past': False, 25 | 'start_date': datetime.utcnow(), # CHANGEME! 26 | 'email': ['airflow@airflow.com'], 27 | 'email_on_failure': False, 28 | 'email_on_retry': False, 29 | 'retries': 2, 30 | 'retry_delay': timedelta(minutes=1), 31 | } 32 | 33 | dag = DAG('copy_cloudsql_to_bigquery', default_args=default_args) 34 | dag.doc_md = __doc__ 35 | 36 | 37 | # Just for visual completness 38 | start_task = DummyOperator(task_id="start", dag=dag) 39 | end_task = DummyOperator(task_id="end", dag=dag) 40 | 41 | 42 | class TableConfig: 43 | """ 44 | Holds config for the export/import job we are creating. 45 | """ 46 | STANDARD_EXPORT_QUERY = None 47 | _STANDARD_EXPORT_QUERY = "SELECT * from {}" 48 | 49 | def __init__(self, 50 | cloud_sql_instance, 51 | export_bucket, 52 | export_database, 53 | export_table, 54 | export_query, 55 | gcp_project, 56 | stage_dataset, 57 | stage_table, 58 | stage_final_query, 59 | bq_location 60 | ): 61 | 62 | self.params = { 63 | 'export_table': export_table, 64 | 'export_bucket': export_bucket, 65 | 'export_database': export_database, 66 | 'export_query': export_query or self._STANDARD_EXPORT_QUERY.format(export_table), 67 | 'gcp_project': gcp_project, 68 | 'stage_dataset': stage_dataset, 69 | 'stage_table': stage_table or export_table, 70 | 'stage_final_query': stage_final_query, 71 | 'cloud_sql_instance': cloud_sql_instance, 72 | 'bq_location': bq_location or "EU", 73 | } 74 | 75 | 76 | def get_tables(): 77 | """ 78 | return a list of tables that should go from cloud sql to bigquery 79 | In this example all 3 tables reside in the same cloud sql instance. 80 | :return: 81 | """ 82 | dim_tables = ["DimAge", "DimPerson"] 83 | fact_tables = ["FactPerson"] 84 | export_tables = dim_tables + fact_tables 85 | tables = [] 86 | for dim in export_tables: 87 | tables.append(TableConfig(cloud_sql_instance='CLOUD_SQL_INSTANCE_NAME', 88 | export_table=dim, 89 | export_bucket='YOUR_STAGING_BUCKET', 90 | export_database='prod', 91 | export_query=TableConfig.STANDARD_EXPORT_QUERY, 92 | gcp_project="YOUR_PROJECT_ID", 93 | stage_dataset="YOUR_STAGING_DATASET", 94 | stage_table=None, 95 | stage_final_query=None, 96 | bq_location="EU")) 97 | return tables 98 | 99 | 100 | def gen_export_table_task(table_config): 101 | export_script = BashOperator( 102 | task_id='export_{}_with_gcloud'.format(table_config.params['export_table']), 103 | params=table_config.params, 104 | bash_command=""" 105 | set -e 106 | rm -f link.txt 107 | 108 | gcloud --project {{ params.gcp_project }} sql export csv {{ params.cloud_sql_instance }} \ 109 | gs://{{ params.export_bucket }}/{{ params.export_table }}_{{ ds_nodash }} \ 110 | --database={{ params.export_database }} --query="{{ params.export_query }}" \ 111 | --async > link.txt 112 | 113 | echo "Export started, job id is:" 114 | cat link.txt 115 | link=$(cat link.txt) 116 | echo "Waiting for export operation to finish: (timeout=3600 seconds)" 117 | gcloud beta sql operations wait --timeout=3600 --project {{ params.gcp_project }} ${link##*/} 118 | echo "Table exported" 119 | 120 | """, 121 | dag=dag) 122 | export_script.doc_md = """\ 123 | #### Export table from cloudsql to cloud storage 124 | task documentation 125 | """ 126 | 127 | return export_script 128 | 129 | 130 | def gen_export_schema_task(table_config): 131 | """ 132 | Generate an export for the myssql schema 133 | """ 134 | export_script = BashOperator( 135 | task_id='export_schema_{}_with_gcloud'.format(table_config.params['export_table']), 136 | params=table_config.params, 137 | bash_command=""" 138 | gcloud --project {{ params.gcp_project }} sql export csv {{ params.cloud_sql_instance }} \ 139 | gs://{{ params.export_bucket }}/{{ params.export_table }}_{{ ds_nodash }}_schema_raw \ 140 | --database={{ params.export_database }} \ 141 | --query="SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{{ params.export_database }}' AND TABLE_NAME = '{{ params.export_table }}' order by ORDINAL_POSITION;" 142 | """, 143 | dag=dag) 144 | export_script.doc_md = """\ 145 | #### Export table from cloudsql to cloud storage 146 | task documentation 147 | """ 148 | 149 | return export_script 150 | 151 | 152 | def datatype_to_bq(datatype): 153 | """ 154 | Simple attempt to do some basic datatypes. 155 | Fallback to String for unknowns and then you can fix it later in bigquery. 156 | """ 157 | if "DATETIME" in datatype: 158 | return "DATETIME" 159 | if "DATE" in datatype: 160 | return "DATE" 161 | if "INT" in datatype: 162 | return "INTEGER" 163 | if "FLOAT" in datatype or "DOUBLE" in datatype or "DECIMAL" in datatype: 164 | return "FLOAT" 165 | return "STRING" 166 | 167 | 168 | def create_bigquery_schema_from_kv(**kwargs): 169 | """ 170 | Translates csv schema file to a bigquery json file. 171 | ID:INT 172 | foo:Text 173 | 174 | Will become 175 | 176 | [ {"name": "id", "type": "INTEGER", "mode": "nullable"}, 177 | {"name": "foo", "type": "STRING", "mode": "nullable"} 178 | ] 179 | """ 180 | bucket = "gs://{}/".format(kwargs['export_bucket']) 181 | raw_file = "{}_{}_schema_raw".format(kwargs['export_table'], kwargs['ds_nodash']) 182 | os.system("gsutil cp {}{} {}".format(bucket, raw_file, raw_file)) 183 | logging.info("Grabbed raw schmea file to {}".format(raw_file)) 184 | bigquery_schema = [] 185 | with open(raw_file) as f: 186 | for line in f.readlines(): 187 | name,datatype = line.replace(" ", "_").replace("/", "_and_").replace('"', "").split(",") 188 | bq_datatype = datatype_to_bq(datatype.upper()) 189 | bigquery_schema.append(' {{\n "name": "{}",\n "type": "{}",\n "mode": "NULLABLE"\n }}\n'.format(name, bq_datatype)) 190 | 191 | schema = "[\n {}]".format(",".join(bigquery_schema)) 192 | filename = "{}_json".format(raw_file[:-4]) 193 | with open(filename, "w") as text_file: 194 | text_file.write(schema) 195 | 196 | os.system("gsutil cp {} {}{}".format(filename, bucket,filename)) 197 | 198 | 199 | def gen_create_bigquery_schema_from_export(table_config): 200 | task = PythonOperator( 201 | task_id='create_bigquery_schema_for_{}'.format(table_config.params['export_table']), 202 | python_callable=create_bigquery_schema_from_kv, 203 | provide_context=True, 204 | op_kwargs=table_config.params, 205 | dag=dag) 206 | return task 207 | 208 | 209 | def gen_import_table_task(table_config): 210 | """ 211 | First copies json schema file from stage bucket (created by previous tasks) 212 | cat content for debugging 213 | the try to load into bigquery. 214 | :param table_config: 215 | :return: 216 | """ 217 | import_script = BashOperator( 218 | task_id='import_{}_to_bigquery'.format(table_config.params['export_table']), 219 | params=table_config.params, 220 | bash_command=""" 221 | gsutil cp gs://{{ params.export_bucket }}/{{ params.export_table }}_{{ ds_nodash }}_schema_json {{ params.export_table }}_{{ ds_nodash }}_schema.json 222 | 223 | cat {{ params.export_table }}_{{ ds_nodash }}_schema.json 224 | 225 | bq --project {{ params.gcp_project }} --location={{ params.bq_location }} load --replace \ 226 | --source_format=CSV {{ params.stage_dataset }}.{{ params.stage_table }}_{{ ds_nodash }} \ 227 | gs://{{ params.export_bucket }}/{{ params.export_table }}_{{ ds_nodash }} {{ params.export_table }}_{{ ds_nodash }}_schema.json 228 | """, 229 | dag=dag) 230 | 231 | import_script.doc_md = """\ 232 | #### Import table from storage to bigquery 233 | task documentation 234 | """ 235 | 236 | return import_script 237 | 238 | 239 | def gen_fix_null_hack(table_config): 240 | """ 241 | How silly it is that google has had this bug for months! 242 | 243 | Found a script in this tracker that has been modified slightly to work here. 244 | https://issuetracker.google.com/issues/64579566 post #22 245 | Changed sed to produce "" instead of N 246 | 247 | UPDATE: 248 | also fixes invalid '\r' export. if this character is encountered it will export '\r"' breaking the csv, 249 | This process also patches this to become just '\\r' 250 | 251 | Once google fixes the export we should not need this function anymore. 252 | :param table_config: 253 | :return: 254 | """ 255 | import_script = BashOperator( 256 | task_id='fix_csv_broken_null_values_{}'.format(table_config.params['export_table']), 257 | params=table_config.params, 258 | bash_command=""" 259 | gsutil cp gs://{{ params.export_bucket }}/{{ params.export_table }}_{{ ds_nodash }} - \ 260 | | sed 's/,"N,/,"",/g' | sed 's/,"N,/,"",/g' | sed 's/^"N,/"",/g' | sed 's/,"N$/,""/g' | sed 's/\\r"$/\\\\r/' \ 261 | | gsutil cp - gs://{{ params.export_bucket }}/{{ params.export_table }}_{{ ds_nodash }} 262 | """, 263 | dag=dag) 264 | 265 | import_script.doc_md = """\ 266 | #### Fix broken NULL values in csv export because google dont know how to do it. 267 | task documentation 268 | """ 269 | 270 | return import_script 271 | 272 | 273 | last_export = None 274 | for table_config in get_tables(): 275 | export_script = gen_export_table_task(table_config) 276 | export_schema_script = gen_export_schema_task(table_config) 277 | bq_schema_task = gen_create_bigquery_schema_from_export(table_config) 278 | import_script = gen_import_table_task(table_config) 279 | nullfix_script = gen_fix_null_hack(table_config) 280 | 281 | start_task >> export_schema_script >> export_script >> nullfix_script >> import_script >> end_task 282 | export_schema_script >> bq_schema_task >> import_script 283 | 284 | # only one export can run at a time 285 | if last_export: 286 | last_export >> export_schema_script 287 | last_export = export_script 288 | -------------------------------------------------------------------------------- /dags/copy_cloudsql_to_bigquery_v2.py: -------------------------------------------------------------------------------- 1 | """ 2 | ### Copy cloudsql tables to bigquery 3 | 4 | superstision 5 | """ 6 | import os 7 | import logging 8 | from airflow import DAG 9 | from datetime import datetime, timedelta 10 | from airflow.contrib.operators.mysql_to_gcs import MySqlToGoogleCloudStorageOperator 11 | from airflow.contrib.operators.gcs_to_bq import GoogleCloudStorageToBigQueryOperator 12 | 13 | default_args = { 14 | 'owner': 'anders', 15 | 'depends_on_past': False, 16 | 'start_date': datetime.now(), # Change this to a literal date to make scheduler work. 17 | 'email': ['anders@...'], 18 | 'email_on_failure': False, 19 | 'email_on_retry': False, 20 | 'retries': 2, 21 | 'retry_delay': timedelta(minutes=1), 22 | } 23 | 24 | dag = DAG('copy_cloudsql_to_bigquery_v2', 25 | default_args=default_args, 26 | schedule_interval='0 6 * * *') 27 | dag.doc_md = __doc__ 28 | 29 | 30 | class TableConfig: 31 | def __init__(self, 32 | cloud_sql_instance, 33 | export_bucket, 34 | export_database, 35 | export_table, 36 | export_query, 37 | gcp_project, 38 | stage_dataset, 39 | stage_table, 40 | stage_final_query, 41 | bq_location 42 | ): 43 | 44 | self.params = { 45 | 'export_table': export_table, 46 | 'export_bucket': export_bucket, 47 | 'export_database': export_database, 48 | 'export_query': export_query, 49 | 'gcp_project': gcp_project, 50 | 'stage_dataset': stage_dataset, 51 | 'stage_table': stage_table or export_table, 52 | 'stage_final_query': stage_final_query, 53 | 'cloud_sql_instance': cloud_sql_instance, 54 | 'bq_location': bq_location or "EU", 55 | } 56 | 57 | 58 | def get_tables(): 59 | """ 60 | return a list of tables that should go from cloud sql to bigquery 61 | In this example all 3 tables reside in the same cloud sql instance. 62 | :return: 63 | """ 64 | dim_tables = ["dim.DimAge", "dim.DimPerson"] 65 | fact_tables = ["facts.FactPerson"] 66 | export_tables = dim_tables + fact_tables 67 | tables = [] 68 | for dim in export_tables: 69 | cfg = TableConfig(cloud_sql_instance='CLOUD_SQL_INSTANCE_NAME', 70 | export_table=dim.split(".")[-1], 71 | export_bucket='YOUR_STAGING_BUCKET', 72 | export_database=dim.split(".")[0], 73 | export_query="SELECT * from {}".format(dim), 74 | gcp_project="YOUR_PROJECT_ID", 75 | stage_dataset="YOUR_STAGING_DATASET", 76 | stage_table=None, 77 | stage_final_query=None, 78 | bq_location="EU") 79 | tables.append(cfg) 80 | return tables 81 | 82 | def gen_export_table_task(table_config): 83 | export_task = MySqlToGoogleCloudStorageOperator(task_id='export_{}'.format(table_config.params['export_table']), 84 | dag=dag, 85 | sql=table_config.params['export_query'], 86 | bucket=table_config.params['export_bucket'], 87 | filename="cloudsql_to_bigquery/{}/{}".format(table_config.params['export_database'], 88 | table_config.params['export_table']) + "_{}", 89 | schema_filename="cloudsql_to_bigquery/schema/{}/schema_raw".format(table_config.params['export_table']), 90 | mysql_conn_id="gcp_dvh_cloudsql") 91 | export_task.doc_md = """\ 92 | #### Export table from cloudsql to cloud storage 93 | task documentation 94 | """ 95 | return export_task 96 | 97 | 98 | def gen_import_table_task(table_config): 99 | import_task = GoogleCloudStorageToBigQueryOperator( 100 | task_id='{}_to_bigquery'.format(table_config.params['export_table']), 101 | bucket=table_config.params['export_bucket'], 102 | source_objects=["cloudsql_to_bigquery/{}/{}*".format(table_config.params['export_database'], 103 | table_config.params['export_table'])], 104 | destination_project_dataset_table="{}.{}.{}".format(table_config.params['gcp_project'], 105 | table_config.params['stage_dataset'], 106 | table_config.params['stage_table']), 107 | schema_object="cloudsql_to_bigquery/schema/{}/schema_raw".format(table_config.params['export_table']), 108 | write_disposition='WRITE_TRUNCATE', 109 | source_format="NEWLINE_DELIMITED_JSON", 110 | dag=dag) 111 | 112 | import_task.doc_md = """\ 113 | #### Import table from storage to bigquery 114 | task documentation 115 | """ 116 | return import_task 117 | 118 | 119 | """ 120 | The code that follows setups the dependencies between the tasks 121 | """ 122 | 123 | for table_config in get_tables(): 124 | export_script = gen_export_table_task(table_config) 125 | import_script = gen_import_table_task(table_config) 126 | 127 | export_script >> import_script 128 | -------------------------------------------------------------------------------- /dags/mssql_to_bigquery.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | your_db_TABLES = ["database.dbo.thetable"] 3 | 4 | dag = DAG('your_db_to_bigquery', default_args=default_args) 5 | start_task = DummyOperator(task_id="start", dag=dag) 6 | end_task = DummyOperator(task_id="end", dag=dag) 7 | 8 | 9 | def gen_filename(tbl): 10 | return tbl.split(".")[-1] 11 | 12 | 13 | def gen_export_task(tbl): 14 | filename = gen_filename(tbl) 15 | export_tbl = MySqlToGoogleCloudStorageOperator( 16 | task_id='extract_data_from_{}'.format(filename), 17 | mysql_conn_id='your_db', 18 | google_cloud_storage_conn_id='your-connection-id', 19 | sql='SELECT * FROM {}'.format(tbl), 20 | table_schema=tbl, 21 | bucket='your-bucket', 22 | filename='your_db/tables/{}'.format(filename) + '/{}.json', 23 | schema_filename='your_db/schemas/{}.json'.format(filename), 24 | dag=dag) 25 | return export_tbl 26 | 27 | 28 | def gen_import_task(filename): 29 | import_task = GoogleCloudStorageToBigQueryOperator( 30 | task_id='{}_to_bigquery'.format(filename), 31 | bucket='your-bucket', 32 | source_objects=['your_db/tables/{}/*'.format(filename)], 33 | destination_project_dataset_table='your-bigquery-project.your_db_raw.{}'.format(filename), 34 | schema_object='your_db/schemas/{}.json'.format(filename), 35 | write_disposition='WRITE_TRUNCATE', 36 | source_format="NEWLINE_DELIMITED_JSON", 37 | dag=dag) 38 | return import_task 39 | 40 | 41 | for table in your_db_TABLES: 42 | export_table = gen_export_task(table) 43 | import_file = gen_import_task(gen_filename(table)) 44 | start_task >> export_table >> import_file >> end_task -------------------------------------------------------------------------------- /media/example1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ael-computas/gcp_cloudsql_airflow_bigquery/143c836b94f5693114a4e460aac4b72cbfd31233/media/example1.png -------------------------------------------------------------------------------- /media/example2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ael-computas/gcp_cloudsql_airflow_bigquery/143c836b94f5693114a4e460aac4b72cbfd31233/media/example2.png -------------------------------------------------------------------------------- /yaml/sql_proxy_dvh_service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | labels: 5 | run: mysql-dvh-sqlproxy 6 | name: mysql-dvh-sqlproxy-service 7 | spec: 8 | ports: 9 | - port: 3306 10 | protocol: TCP 11 | targetPort: 3306 12 | selector: 13 | run: mysql-dvh-sqlproxy 14 | -------------------------------------------------------------------------------- /yaml/sql_proxy_mysql_deployment.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: extensions/v1beta1 16 | kind: Deployment 17 | metadata: 18 | labels: 19 | run: mysql-dvh-sqlproxy 20 | name: mysql-dvh-sqlproxy 21 | namespace: default 22 | spec: 23 | replicas: 1 24 | selector: 25 | matchLabels: 26 | run: mysql-dvh-sqlproxy 27 | strategy: 28 | rollingUpdate: 29 | maxSurge: 1 30 | maxUnavailable: 1 31 | type: RollingUpdate 32 | template: 33 | metadata: 34 | creationTimestamp: null 35 | labels: 36 | run: mysql-dvh-sqlproxy 37 | spec: 38 | volumes: 39 | - name: ssl-certs 40 | hostPath: 41 | path: /etc/ssl/certs 42 | containers: 43 | - image: gcr.io/cloudsql-docker/gce-proxy:1.11 44 | volumeMounts: 45 | - name: ssl-certs 46 | mountPath: /etc/ssl/certs 47 | command: ["/cloud_sql_proxy", 48 | "-instances=PROJECT_ID:REGION:DB_INSTANCE=tcp:0.0.0.0:3306"] 49 | imagePullPolicy: Always 50 | livenessProbe: 51 | exec: 52 | command: 53 | - /bin/sh 54 | - -c 55 | - netstat -tlnp | grep -i cloud_sql_proxy 56 | failureThreshold: 3 57 | periodSeconds: 10 58 | successThreshold: 1 59 | timeoutSeconds: 1 60 | name: mysql-dvh-sqlproxy 61 | ports: 62 | - containerPort: 3306 63 | protocol: TCP 64 | resources: {} 65 | terminationMessagePath: /dev/termination-log 66 | dnsPolicy: ClusterFirst 67 | restartPolicy: Always 68 | securityContext: {} 69 | terminationGracePeriodSeconds: 30 -------------------------------------------------------------------------------- /yaml/sql_proxy_postgresql_deployment_with_sa.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # kubectl create secret --namespace=ut1 generic postgresql-ut1-key --from-file=key.json=sa.json 15 | apiVersion: extensions/v1beta1 16 | kind: Deployment 17 | metadata: 18 | labels: 19 | run: postgresql-sqlproxy-ut1 20 | name: postgresql-sqlproxy-ut1 21 | namespace: ut1 22 | spec: 23 | replicas: 1 24 | selector: 25 | matchLabels: 26 | run: postgresql-sqlproxy-ut1 27 | strategy: 28 | rollingUpdate: 29 | maxSurge: 1 30 | maxUnavailable: 1 31 | type: RollingUpdate 32 | template: 33 | metadata: 34 | creationTimestamp: null 35 | labels: 36 | run: postgresql-sqlproxy-ut1 37 | spec: 38 | volumes: 39 | - name: google-cloud-key 40 | secret: 41 | defaultMode: 420 42 | secretName: postgresql-ut1-key # dynamic (ex: postgresql-ut1-key) 43 | - name: ssl-certs 44 | hostPath: 45 | path: /etc/ssl/certs 46 | containers: 47 | - env: 48 | - name: GOOGLE_APPLICATION_CREDENTIALS 49 | value: /var/secrets/google/key.json 50 | image: gcr.io/cloudsql-docker/gce-proxy:1.11 51 | volumeMounts: 52 | - name: ssl-certs 53 | mountPath: /etc/ssl/certs 54 | - name: google-cloud-key 55 | mountPath: /var/secrets/google 56 | command: ["/cloud_sql_proxy", 57 | "-instances=PROJECT_ID:REGION:DATABASE=tcp:0.0.0.0:5432"] 58 | imagePullPolicy: Always 59 | livenessProbe: 60 | exec: 61 | command: 62 | - /bin/sh 63 | - -c 64 | - netstat -tlnp | grep -i cloud_sql_proxy 65 | failureThreshold: 3 66 | periodSeconds: 10 67 | successThreshold: 1 68 | timeoutSeconds: 1 69 | name: postgresql-sqlproxy-ut1 70 | ports: 71 | - containerPort: 5432 72 | protocol: TCP 73 | resources: {} 74 | terminationMessagePath: /dev/termination-log 75 | dnsPolicy: ClusterFirst 76 | restartPolicy: Always 77 | securityContext: {} 78 | terminationGracePeriodSeconds: 30 -------------------------------------------------------------------------------- /yaml/sql_proxy_postgresql_service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | labels: 5 | run: postgresql-sqlproxy-ut1 6 | name: postgresql-sqlproxy-ut1-service 7 | namespace: ut1 8 | spec: 9 | ports: 10 | - port: 5432 11 | protocol: TCP 12 | targetPort: 5432 13 | selector: 14 | run: postgresql-sqlproxy-ut1 15 | --------------------------------------------------------------------------------