├── .gitignore ├── LICENSE ├── README.md ├── cleanup-db.sh ├── connect-db.sh ├── delete-db.sh ├── infrastructure ├── cleanup_db │ ├── .dockerignore │ ├── Dockerfile │ ├── README.md │ ├── build.sh │ ├── cleanup_db.py │ └── requirements.txt ├── metadata_generator │ ├── .dockerignore │ ├── Dockerfile │ ├── README.md │ ├── build.sh │ ├── metadata_generator.py │ └── requirements.txt └── terraform │ ├── main.tf │ ├── outputs.tf │ ├── provider.tf │ ├── variables.tf │ └── versions.tf └── init-db.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | # lib/ -- replaced by above line while installing the client library from a zip file is required. 18 | lib/tmp/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | db.sqlite3 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # PyCharm 74 | .DS_Store 75 | .idea 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # pyenv 81 | .python-version 82 | 83 | # celery beat schedule file 84 | celerybeat-schedule 85 | 86 | # SageMath parsed files 87 | *.sage.py 88 | 89 | # Environments 90 | .env 91 | .venv 92 | env/ 93 | venv/ 94 | ENV/ 95 | env.bak/ 96 | venv.bak/ 97 | 98 | # Spyder project settings 99 | .spyderproject 100 | .spyproject 101 | 102 | # Rope project settings 103 | .ropeproject 104 | 105 | # mkdocs documentation 106 | /site 107 | 108 | # mypy 109 | .mypy_cache/ 110 | 111 | # Application logs 112 | log/ 113 | 114 | instructions.txt 115 | 116 | bin/ 117 | 118 | build_info.txt 119 | *.tar.gz 120 | 121 | # Service accounts 122 | *credentials.json 123 | 124 | # Terraform 125 | *.terraform/ 126 | *.tfstate 127 | tfplan -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Marcelo Costa 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cloudsql-sqlserver-tooling 2 | 3 | Scripts with the goal to enable easy usage of some SQLServer operations. 4 | 5 | ## INIT database 6 | ```bash 7 | ./init-db.sh 8 | ``` 9 | 10 | ## Creating Schemas and Tables in SQLServer 11 | ```bash 12 | ./connect-db.sh 13 | ``` 14 | Provide your password when prompted, then execute: 15 | ```bash 16 | use "test-db"; 17 | CREATE SCHEMA MY_SCHEMA; 18 | CREATE TABLE MY_SCHEMA.MY_TABLE (name INT, address TEXT); 19 | exit 20 | ``` 21 | If you receive errors please run: 22 | ```bash 23 | sudo pip install mssql-cli --ignore-installed six 24 | ``` 25 | To update the sql server driver used by gcloud. 26 | 27 | ## Clean up SQLServer Schemas and Tables 28 | ```bash 29 | ./cleanup-db.sh 30 | ``` 31 | 32 | ## Delete the SQLServer database 33 | ```bash 34 | ./delete-db.sh 35 | ``` 36 | 37 | -------------------------------------------------------------------------------- /cleanup-db.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | root_dir=$(pwd) 3 | cd infrastructure/terraform 4 | 5 | public_ip_address=$(cat terraform.tfstate | jq '.outputs.public_ip_address.value') 6 | username=$(cat terraform.tfstate | jq '.outputs.username.value') 7 | password=$(cat terraform.tfstate | jq '.outputs.password.value') 8 | database=$(cat terraform.tfstate | jq '.outputs.db_name.value') 9 | 10 | # Remove quotes 11 | public_ip_address=${public_ip_address//\"/} 12 | username=${username//\"/} 13 | password=${password//\"/} 14 | database=${database//\"/} 15 | 16 | docker run --rm --tty -v \ 17 | "$PWD":/data mesmacosta/sqlserver-db-cleaner:stable \ 18 | --sqlserver-host $public_ip_address \ 19 | --sqlserver-user $username \ 20 | --sqlserver-pass $password \ 21 | --sqlserver-database $database 22 | 23 | cd $root_dir -------------------------------------------------------------------------------- /connect-db.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | root_dir=$(pwd) 3 | cd infrastructure/terraform 4 | 5 | instance_id=$(cat terraform.tfstate | jq '.outputs.instance_id.value') 6 | username=$(cat terraform.tfstate | jq '.outputs.username.value') 7 | 8 | # Remove quotes 9 | username=${username//\"/} 10 | instance_id=${instance_id//\"/} 11 | 12 | cd $root_dir 13 | 14 | gcloud sql connect $instance_id --user=$username 15 | -------------------------------------------------------------------------------- /delete-db.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | root_dir=$(pwd) 3 | cd infrastructure/terraform 4 | 5 | instance_id=$(cat terraform.tfstate | jq '.outputs.instance_id.value') 6 | project_id=$(cat terraform.tfstate | jq '.outputs.project_id.value') 7 | 8 | # Remove quotes 9 | instance_id=${instance_id//\"/} 10 | project_id=${project_id//\"/} 11 | 12 | # 1 option - Delete using gcloud 13 | gcloud sql instances delete $instance_id --quiet 14 | 15 | # 2 option - Delete using terraform 16 | # terraform destroy 17 | 18 | # Clean up TF state files if they exist. 19 | rm terraform.tfstate > /dev/null 2>&1 20 | rm terraform.tfstate.backup > /dev/null 2>&1 21 | rm tfplan > /dev/null 2>&1 22 | rm .tfvars > /dev/null 2>&1 23 | 24 | echo -e "\033[1;42m Cloud SQL Instance deleted \033[0m" 25 | 26 | cd $root_dir 27 | echo -e "\033[1;42m COMPLETED \033[0m" -------------------------------------------------------------------------------- /infrastructure/cleanup_db/.dockerignore: -------------------------------------------------------------------------------- 1 | # Git 2 | .git 3 | .gitignore 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | #lib/ -- replaced by above line while installing the client library from a zip file is required. 22 | #lib/tmp/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # PyCharm 78 | .DS_Store 79 | .idea 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # celery beat schedule file 88 | celerybeat-schedule 89 | 90 | # SageMath parsed files 91 | *.sage.py 92 | 93 | # Environments 94 | .env 95 | .venv 96 | env/ 97 | venv/ 98 | ENV/ 99 | env.bak/ 100 | venv.bak/ 101 | 102 | # Spyder project settings 103 | .spyderproject 104 | .spyproject 105 | 106 | # Rope project settings 107 | .ropeproject 108 | 109 | # mkdocs documentation 110 | /site 111 | 112 | # mypy 113 | .mypy_cache/ 114 | 115 | # Application logs 116 | log/ 117 | 118 | # Docker 119 | .dockerignore 120 | Dockerfile 121 | 122 | # Shared volume 123 | data/ -------------------------------------------------------------------------------- /infrastructure/cleanup_db/Dockerfile: -------------------------------------------------------------------------------- 1 | # docker build -t sqlserver-db-cleaner . 2 | FROM python:3.7 3 | 4 | # install FreeTDS and dependencies 5 | RUN apt-get update \ 6 | && apt-get install unixodbc -y \ 7 | && apt-get install unixodbc-dev -y \ 8 | && apt-get install --reinstall build-essential -y 9 | 10 | # Debian 10 ODBC DRIVER 11 | RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - 12 | RUN curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list 13 | RUN apt-get update 14 | RUN ACCEPT_EULA=Y apt-get install msodbcsql17 15 | 16 | WORKDIR /lib 17 | COPY requirements.txt . 18 | RUN pip install -r requirements.txt 19 | 20 | WORKDIR /app 21 | 22 | # Copy project files (see .dockerignore). 23 | COPY . . 24 | 25 | 26 | ENTRYPOINT ["python", "cleanup_db.py"] 27 | -------------------------------------------------------------------------------- /infrastructure/cleanup_db/README.md: -------------------------------------------------------------------------------- 1 | # sqlserver-cleanup-db 2 | Script to clear all user defined schemas. 3 | 4 | ## Activate your virtualenv if it’s not up 5 | ```bash 6 | pip install --upgrade virtualenv 7 | python3 -m virtualenv --python python3 env 8 | source ./env/bin/activate 9 | ``` 10 | 11 | ## Install the requirements for the metadata generator 12 | ```bash 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## Run the script 17 | ```bash 18 | python cleanup_db.py 19 | ``` 20 | 21 | ## Developer environment 22 | 23 | ### Install and run Yapf formatter 24 | 25 | ```bash 26 | pip install --upgrade yapf 27 | 28 | # Auto update files 29 | yapf --in-place metadata_generator.py 30 | 31 | # Show diff 32 | yapf --diff metadata_generator.py 33 | 34 | # Set up pre-commit hook 35 | # From the root of your git project. 36 | curl -o pre-commit.sh https://raw.githubusercontent.com/google/yapf/master/plugins/pre-commit.sh 37 | chmod a+x pre-commit.sh 38 | mv pre-commit.sh .git/hooks/pre-commit 39 | ``` 40 | 41 | ### Install and run Flake8 linter 42 | 43 | ```bash 44 | pip install --upgrade flake8 45 | flake8 metadata_generator.py 46 | ``` -------------------------------------------------------------------------------- /infrastructure/cleanup_db/build.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env bash 3 | docker build -t sqlserver-db-cleaner . 4 | docker tag sqlserver-db-cleaner mesmacosta/sqlserver-db-cleaner:stable 5 | docker push mesmacosta/sqlserver-db-cleaner:stable 6 | -------------------------------------------------------------------------------- /infrastructure/cleanup_db/cleanup_db.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import sys 4 | import uuid 5 | 6 | from pyodbc import connect 7 | 8 | QUERY = """ 9 | SELECT DISTINCT t.table_schema as schema_name, 10 | t.table_name as table_name 11 | FROM information_schema.tables t 12 | WHERE t.table_schema NOT IN ('dbo') 13 | ORDER BY t.table_schema; 14 | """ 15 | 16 | 17 | def get_conn(connection_args): 18 | return connect( 19 | build_connection(database=connection_args['database'], 20 | host=connection_args['host'], 21 | user=connection_args['user'], 22 | password=connection_args['pass'])) 23 | 24 | 25 | def build_connection(database, host, user, password): 26 | return 'DRIVER={ODBC Driver 17 for SQL Server};' + \ 27 | 'SERVER={};DATABASE={};UID={};PWD={}'.format( 28 | host, database, user, password) 29 | 30 | 31 | def cleanup_metadata(connection_args): 32 | conn = get_conn(connection_args) 33 | 34 | cursor = conn.cursor() 35 | cursor.execute(QUERY) 36 | rows = cursor.fetchall() 37 | for row in rows: 38 | schema_name = row[0] 39 | table_name = row[1] 40 | table_stmt = build_drop_table_statement(schema_name, table_name) 41 | cursor.execute(table_stmt) 42 | print('Cleaned table: {}.{}'.format(schema_name, table_name)) 43 | conn.commit() 44 | 45 | cursor.close() 46 | 47 | 48 | def build_drop_table_statement(schema_name, table_name): 49 | table_stmt = 'DROP TABLE {}.{};'.format(schema_name, table_name) 50 | return table_stmt 51 | 52 | 53 | def parse_args(): 54 | parser = argparse.ArgumentParser( 55 | description='Command line to cleanup metadata from sqlserver') 56 | parser.add_argument('--sqlserver-host', 57 | help='Your sqlserver server host', 58 | required=True) 59 | parser.add_argument('--sqlserver-user', 60 | help='Your sqlserver credentials user', 61 | required=True) 62 | parser.add_argument('--sqlserver-pass', 63 | help='Your sqlserver credentials password', 64 | required=True) 65 | parser.add_argument('--sqlserver-database', 66 | help='Your sqlserver database name', 67 | required=True) 68 | return parser.parse_args() 69 | 70 | 71 | if __name__ == "__main__": 72 | args = parse_args() 73 | # Enable logging 74 | logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) 75 | 76 | cleanup_metadata({ 77 | 'database': args.sqlserver_database, 78 | 'host': args.sqlserver_host, 79 | 'user': args.sqlserver_user, 80 | 'pass': args.sqlserver_pass 81 | }) 82 | -------------------------------------------------------------------------------- /infrastructure/cleanup_db/requirements.txt: -------------------------------------------------------------------------------- 1 | pyodbc -------------------------------------------------------------------------------- /infrastructure/metadata_generator/.dockerignore: -------------------------------------------------------------------------------- 1 | # Git 2 | .git 3 | .gitignore 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | #lib/ -- replaced by above line while installing the client library from a zip file is required. 22 | #lib/tmp/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # PyCharm 78 | .DS_Store 79 | .idea 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # celery beat schedule file 88 | celerybeat-schedule 89 | 90 | # SageMath parsed files 91 | *.sage.py 92 | 93 | # Environments 94 | .env 95 | .venv 96 | env/ 97 | venv/ 98 | ENV/ 99 | env.bak/ 100 | venv.bak/ 101 | 102 | # Spyder project settings 103 | .spyderproject 104 | .spyproject 105 | 106 | # Rope project settings 107 | .ropeproject 108 | 109 | # mkdocs documentation 110 | /site 111 | 112 | # mypy 113 | .mypy_cache/ 114 | 115 | # Application logs 116 | log/ 117 | 118 | # Docker 119 | .dockerignore 120 | Dockerfile 121 | 122 | # Shared volume 123 | data/ -------------------------------------------------------------------------------- /infrastructure/metadata_generator/Dockerfile: -------------------------------------------------------------------------------- 1 | # docker build -t sqlserver-metadata-generator . 2 | FROM python:3.7 3 | 4 | # install FreeTDS and dependencies 5 | RUN apt-get update \ 6 | && apt-get install unixodbc -y \ 7 | && apt-get install unixodbc-dev -y \ 8 | && apt-get install --reinstall build-essential -y 9 | 10 | # Debian 10 ODBC DRIVER 11 | RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - 12 | RUN curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list 13 | RUN apt-get update 14 | RUN ACCEPT_EULA=Y apt-get install msodbcsql17 15 | 16 | WORKDIR /lib 17 | COPY requirements.txt . 18 | RUN pip install -r requirements.txt 19 | 20 | WORKDIR /app 21 | 22 | # Copy project files (see .dockerignore). 23 | COPY . . 24 | 25 | ENTRYPOINT ["python", "metadata_generator.py"] 26 | -------------------------------------------------------------------------------- /infrastructure/metadata_generator/README.md: -------------------------------------------------------------------------------- 1 | # sqlserver-metadata-generator 2 | 3 | To test some SQLServer capabilities, it’s good to have a good number of tables with different complex column types. This script generates random metadata for SQLServer. 4 | 5 | ## Activate your virtualenv if it’s not up 6 | ```bash 7 | pip install --upgrade virtualenv 8 | python3 -m virtualenv --python python3 env 9 | source ./env/bin/activate 10 | ``` 11 | 12 | ## Install the requirements for the metadata generator 13 | ```bash 14 | pip install -r requirements.txt 15 | ``` 16 | 17 | # Install ODBC Driver 17 for SQL Server 18 | https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-2017 19 | 20 | ## Run the script 21 | ```bash 22 | python metadata_generator.py 23 | ``` 24 | 25 | ## Developer environment 26 | 27 | ### Install and run Yapf formatter 28 | 29 | ```bash 30 | pip install --upgrade yapf 31 | 32 | # Auto update files 33 | yapf --in-place metadata_generator.py 34 | 35 | # Show diff 36 | yapf --diff metadata_generator.py 37 | 38 | # Set up pre-commit hook 39 | # From the root of your git project. 40 | curl -o pre-commit.sh https://raw.githubusercontent.com/google/yapf/master/plugins/pre-commit.sh 41 | chmod a+x pre-commit.sh 42 | mv pre-commit.sh .git/hooks/pre-commit 43 | ``` 44 | 45 | ### Install and run Flake8 linter 46 | 47 | ```bash 48 | pip install --upgrade flake8 49 | flake8 metadata_generator.py 50 | ``` 51 | -------------------------------------------------------------------------------- /infrastructure/metadata_generator/build.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env bash 3 | docker build -t sqlserver-metadata-generator . 4 | docker tag sqlserver-metadata-generator mesmacosta/sqlserver-metadata-generator:stable 5 | docker push mesmacosta/sqlserver-metadata-generator:stable 6 | -------------------------------------------------------------------------------- /infrastructure/metadata_generator/metadata_generator.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import random 4 | import sys 5 | import uuid 6 | 7 | from pyodbc import connect 8 | 9 | _DATA_TYPES = [ 10 | 'INT', 'TINYINT', 'SMALLINT', 'FLOAT', 'REAL', 'CHAR(5)', 'VARCHAR(25)', 11 | 'TEXT', 'BINARY', 'DATE', 'TIME', 'DATETIME' 12 | ] 13 | 14 | _COLUMN_NAMES = [ 15 | 'name', 'address', 'city', 'state', 'date_time', 'paragraph', 'randomdata', 16 | 'person', 'credit_card', 'size', 'reason', 'school', 'food', 'location', 17 | 'house', 'price', 'cpf', 'cnpj', 'passport', 'security_number', 18 | 'phone_number', 'bank_account_number', 'ip_address', 'stocks' 19 | ] 20 | 21 | _TABLE_NAMES = [ 22 | 'school_info', 'personal_info', 'persons', 'employees', 'companies', 23 | 'store', 'home' 24 | ] 25 | 26 | _SCHEMA_NAMES = [ 27 | 'school_warehouse', 'company_warehouse', 'on_prem_warehouse', 28 | 'factory_warehouse', 'organization_warehouse' 29 | ] 30 | 31 | 32 | def build_connection(database, host, user, password): 33 | return 'DRIVER={ODBC Driver 17 for SQL Server};' + \ 34 | 'SERVER={};DATABASE={};UID={};PWD={}'.format( 35 | host, database, user, password) 36 | 37 | 38 | def get_conn(connection_args): 39 | return connect( 40 | build_connection(database=connection_args['database'], 41 | host=connection_args['host'], 42 | user=connection_args['user'], 43 | password=connection_args['pass'])) 44 | 45 | 46 | def create_random_metadata(connection_args): 47 | conn = get_conn(connection_args) 48 | 49 | cursor = conn.cursor() 50 | 51 | for x in range(connection_args['number_schemas']): 52 | schema_name, schema_stmt = build_create_schema_statement() 53 | cursor.execute(schema_stmt) 54 | for y in range(connection_args['number_tables']): 55 | query = build_create_table_statement(schema_name) 56 | print('\n' + query) 57 | cursor.execute(query) 58 | conn.commit() 59 | 60 | cursor.close() 61 | 62 | 63 | def get_random_schema_name(): 64 | return random.choice(_SCHEMA_NAMES) 65 | 66 | 67 | def build_create_schema_statement(): 68 | schema_name = '{}{}'.format(get_random_schema_name(), 69 | str(random.randint(1, 100000))) 70 | schema_stmt = 'CREATE SCHEMA {} '.format(schema_name) 71 | return schema_name, schema_stmt 72 | 73 | 74 | def get_random_data_type(): 75 | return random.choice(_DATA_TYPES) 76 | 77 | 78 | def get_random_column_name(): 79 | return random.choice(_COLUMN_NAMES) 80 | 81 | 82 | def get_random_table_name(): 83 | return random.choice(_TABLE_NAMES) 84 | 85 | 86 | def build_create_table_statement(schema_name): 87 | table_stmt = 'CREATE TABLE {}.{}{} ( '.format(schema_name, 88 | get_random_table_name(), 89 | uuid.uuid4().hex[:8]) 90 | table_stmt = '{}{}{} {}'.format(table_stmt, get_random_column_name(), 91 | str(random.randint(1, 100000)), 92 | get_random_data_type()) 93 | for x in range(random.randint(1, 15)): 94 | table_stmt += ', {}{}'.format(get_random_column_name(), 95 | str(random.randint(1, 100000))) + \ 96 | ' {}'.format(get_random_data_type()) 97 | 98 | table_stmt = '{} )'.format(table_stmt) 99 | return table_stmt 100 | 101 | 102 | def parse_args(): 103 | parser = argparse.ArgumentParser( 104 | description='Command line generate random metadata into sqlserver') 105 | parser.add_argument( 106 | '--sqlserver-host', 107 | help='Your sqlserver server host, this is required even' 108 | ' for the raw_metadata_csv,' 109 | ' so we are able to map the created entries' 110 | ' resource with the sqlserver host', 111 | required=True) 112 | parser.add_argument('--sqlserver-user', 113 | help='Your sqlserver credentials user') 114 | parser.add_argument('--sqlserver-pass', 115 | help='Your sqlserver credentials password') 116 | parser.add_argument('--sqlserver-database', 117 | help='Your sqlserver database name') 118 | parser.add_argument('--number-schemas', 119 | help='Number of schemas to create', 120 | type=int, 121 | default=4) 122 | parser.add_argument('--number-tables', 123 | help='Number of tables to create', 124 | type=int, 125 | default=250) 126 | return parser.parse_args() 127 | 128 | 129 | if __name__ == "__main__": 130 | args = parse_args() 131 | # Enable logging 132 | logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) 133 | 134 | create_random_metadata({ 135 | 'database': args.sqlserver_database, 136 | 'host': args.sqlserver_host, 137 | 'user': args.sqlserver_user, 138 | 'pass': args.sqlserver_pass, 139 | 'number_schemas': args.number_schemas, 140 | 'number_tables': args.number_tables 141 | }) 142 | -------------------------------------------------------------------------------- /infrastructure/metadata_generator/requirements.txt: -------------------------------------------------------------------------------- 1 | pyodbc -------------------------------------------------------------------------------- /infrastructure/terraform/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | instance_name = "${var.db_name}-${random_id.name.hex}" 3 | generated_password = "${var.user_password_prefix}_${random_id.pass.hex}" 4 | } 5 | 6 | resource "google_sql_database_instance" "sqlserver" { 7 | provider = google-beta 8 | name = local.instance_name 9 | database_version = var.database_version 10 | region = var.db_region 11 | root_password = local.generated_password 12 | project = var.project_id 13 | 14 | settings { 15 | ip_configuration { 16 | ipv4_enabled = true 17 | private_network = null 18 | require_ssl = false 19 | 20 | authorized_networks { 21 | name = "test-machine-ip" 22 | value = var.test_machine_ip 23 | } 24 | } 25 | 26 | tier = var.tier 27 | disk_size = var.disk_size 28 | replication_type = var.replication_type 29 | activation_policy = var.activation_policy 30 | } 31 | } 32 | 33 | resource "google_sql_database" "test_db" { 34 | provider = google-beta 35 | name = var.db_name 36 | instance = google_sql_database_instance.sqlserver.name 37 | project = var.project_id 38 | } 39 | 40 | -------------------------------------------------------------------------------- /infrastructure/terraform/outputs.tf: -------------------------------------------------------------------------------- 1 | 2 | output "project_id" { 3 | value = var.project_id 4 | description = "The project to run tests against" 5 | } 6 | 7 | output "name" { 8 | value = local.instance_name 9 | description = "The name for Cloud SQL instance" 10 | } 11 | 12 | output "self_link" { 13 | description = "link for your sql database instance" 14 | value = google_sql_database_instance.sqlserver.self_link 15 | } 16 | 17 | output "public_ip_address" { 18 | description = "Public ip address to connect to this Database" 19 | value = google_sql_database_instance.sqlserver.public_ip_address 20 | } 21 | 22 | output "instance_id" { 23 | description = "Id of the Cloud SQL instance" 24 | value = google_sql_database_instance.sqlserver.id 25 | } 26 | 27 | output "username" { 28 | description = "Username to connect to this Database" 29 | value = "sqlserver" 30 | } 31 | 32 | output "password" { 33 | description = "Password to connect to this Database" 34 | value = google_sql_database_instance.sqlserver.root_password 35 | } 36 | 37 | output "db_name" { 38 | value = google_sql_database.test_db.name 39 | description = "The name of the Database to connect to" 40 | } 41 | 42 | -------------------------------------------------------------------------------- /infrastructure/terraform/provider.tf: -------------------------------------------------------------------------------- 1 | provider "google" { 2 | version = "~> 3.8" 3 | project = var.project_id 4 | region = var.project_region 5 | } 6 | 7 | provider "google-beta" { 8 | version = "~> 3.8" 9 | } 10 | 11 | provider "null" { 12 | version = "~> 2.1" 13 | } 14 | 15 | provider "random" { 16 | version = "~> 2.2" 17 | } 18 | 19 | resource "random_id" "name" { 20 | byte_length = 2 21 | } 22 | 23 | resource "random_id" "pass" { 24 | byte_length = 16 25 | } -------------------------------------------------------------------------------- /infrastructure/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "project_id" { 2 | description = "The ID of the project in which resources will be provisioned." 3 | type = string 4 | } 5 | 6 | variable "test_machine_ip" { 7 | description = "IP of the machine that will be authorized to connect to Cloud SQL." 8 | type = string 9 | } 10 | 11 | variable "tier" { default = "db-custom-2-13312" } 12 | variable "db_name" { default = "test-db" } 13 | variable "db_region" { default = "us-central1" } 14 | variable "project_region" { default = "us-central1" } 15 | variable "disk_size" { default = "20" } 16 | variable "database_version" { default = "SQLSERVER_2017_STANDARD" } 17 | variable "user_host" { default = "%" } 18 | variable "user_name" { default = "admin" } 19 | variable "user_password_prefix" { default = "MyPASSDB" } 20 | variable "replication_type" { default = "SYNCHRONOUS" } 21 | variable "activation_policy" { default = "always" } -------------------------------------------------------------------------------- /infrastructure/terraform/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 0.12.6" 3 | } -------------------------------------------------------------------------------- /init-db.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | root_dir=$(pwd) 3 | cd infrastructure/terraform 4 | 5 | project=$(gcloud config get-value project) 6 | 7 | test_machine_ip=$(curl https://diagnostic.opendns.com/myip) 8 | 9 | if [ -z "$project" ]; then 10 | echo "GCloud project must be set! Run: gcloud config set project [MY_PROJECT]" 11 | exit 2 12 | fi 13 | 14 | # Create TF environment file 15 | cat > .tfvars << EOF 16 | project_id="$project" 17 | test_machine_ip="$test_machine_ip" 18 | EOF 19 | 20 | echo -e "\033[1;42m [STEP 1] Enable required APIs \033[0m" 21 | 22 | gcloud services enable datacatalog.googleapis.com sqladmin.googleapis.com --project $project 23 | 24 | echo -e "\033[1;42m [STEP 2] Create Cloud SQL - SQLServer environment \033[0m" 25 | # download utility tootls directly into ~/ 26 | curl http://stedolan.github.io/jq/download/linux64/jq -o ~/jq 27 | # give it executable permissions 28 | chmod a+x ~/jq 29 | 30 | # Initialise the configuration 31 | terraform init -input=false 32 | 33 | # Plan and deploy 34 | terraform plan -input=false -out=tfplan -var-file=".tfvars" > /dev/null 2>&1 35 | terraform apply tfplan 36 | 37 | public_ip_address=$(cat terraform.tfstate | jq '.outputs.public_ip_address.value') 38 | username=$(cat terraform.tfstate | jq '.outputs.username.value') 39 | password=$(cat terraform.tfstate | jq '.outputs.password.value') 40 | database=$(cat terraform.tfstate | jq '.outputs.db_name.value') 41 | 42 | # Remove quotes 43 | public_ip_address=${public_ip_address//\"/} 44 | username=${username//\"/} 45 | password=${password//\"/} 46 | database=${database//\"/} 47 | 48 | export public_ip_address=$public_ip_address 49 | export username=$username 50 | export password=$password 51 | export database=$database 52 | 53 | if [ -z "$public_ip_address" ]; then 54 | echo "Cloud SQL instance creation failed" 55 | exit 3 56 | fi 57 | 58 | echo $public_ip_address $username $password 59 | 60 | echo -e "\033[1;42m [STEP 3] POPULATE DATABASE \033[0m" 61 | 62 | # Generate Metadata 63 | docker run --rm --tty \ 64 | mesmacosta/sqlserver-metadata-generator:stable \ 65 | --sqlserver-host=$public_ip_address \ 66 | --sqlserver-user=$username \ 67 | --sqlserver-pass=$password \ 68 | --sqlserver-database=$database \ 69 | --number-schemas=5 \ 70 | --number-tables=5 71 | 72 | cd $root_dir 73 | echo -e "\033[1;42m COMPLETED \033[0m" 74 | --------------------------------------------------------------------------------