├── .github └── workflows │ ├── onMainPullRequest.yaml │ ├── onPR.yaml │ ├── onPullRequest.yml │ ├── onPushRequest.yml │ └── onRelease.yml ├── .vscode └── settings.json ├── README.md ├── bundle.yml ├── bundle_config_schema.json ├── deployment ├── deploy-job-dab.yaml └── deploy-job-dbx.yaml ├── images ├── .gitignore ├── Dbx.png ├── GitRepos.png ├── TestingPyramid.png └── cicd_design ├── src └── main │ ├── python │ ├── bronze │ │ ├── load_bronze_tables.py │ │ └── load_data_into_bronze.py │ ├── gold │ │ ├── gold_layer_etl.py │ │ └── gold_layer_etl_main.py │ ├── setup │ │ ├── cleanup.py │ │ ├── create_ddl.py │ │ ├── generate_retail_data.py │ │ └── initiate_setup.py │ └── silver │ │ ├── call_standardise_retail_dataset.py │ │ ├── call_transform_to_scd2.py │ │ ├── standardise_retail_dataset.py │ │ └── transform_to_scd2.py │ └── tests │ ├── bronze │ └── test_load_data_into_bronze.py │ ├── cleanup_tests.py │ ├── gold │ └── test_gold_layer_etl.py │ ├── integration_suite │ ├── cleanup_integration_suite.py │ ├── integration_tests.py │ ├── test_integration_bronze_layer_etl.py │ ├── test_integration_gold_layer_etl.py │ └── test_integration_silver_layer_etl.py │ ├── silver │ ├── test_standardise_retail_dataset.py │ └── test_transform_to_scd2.py │ └── test_all.py └── unit-testing-requirements.txt /.github/workflows/onMainPullRequest.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/.github/workflows/onMainPullRequest.yaml -------------------------------------------------------------------------------- /.github/workflows/onPR.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/.github/workflows/onPR.yaml -------------------------------------------------------------------------------- /.github/workflows/onPullRequest.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/.github/workflows/onPullRequest.yml -------------------------------------------------------------------------------- /.github/workflows/onPushRequest.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/.github/workflows/onPushRequest.yml -------------------------------------------------------------------------------- /.github/workflows/onRelease.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/.github/workflows/onRelease.yml -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/.vscode/settings.json -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/README.md -------------------------------------------------------------------------------- /bundle.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/bundle.yml -------------------------------------------------------------------------------- /bundle_config_schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/bundle_config_schema.json -------------------------------------------------------------------------------- /deployment/deploy-job-dab.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/deployment/deploy-job-dab.yaml -------------------------------------------------------------------------------- /deployment/deploy-job-dbx.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/deployment/deploy-job-dbx.yaml -------------------------------------------------------------------------------- /images/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /images/Dbx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/images/Dbx.png -------------------------------------------------------------------------------- /images/GitRepos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/images/GitRepos.png -------------------------------------------------------------------------------- /images/TestingPyramid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/images/TestingPyramid.png -------------------------------------------------------------------------------- /images/cicd_design: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/images/cicd_design -------------------------------------------------------------------------------- /src/main/python/bronze/load_bronze_tables.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/bronze/load_bronze_tables.py -------------------------------------------------------------------------------- /src/main/python/bronze/load_data_into_bronze.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/bronze/load_data_into_bronze.py -------------------------------------------------------------------------------- /src/main/python/gold/gold_layer_etl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/gold/gold_layer_etl.py -------------------------------------------------------------------------------- /src/main/python/gold/gold_layer_etl_main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/gold/gold_layer_etl_main.py -------------------------------------------------------------------------------- /src/main/python/setup/cleanup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/setup/cleanup.py -------------------------------------------------------------------------------- /src/main/python/setup/create_ddl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/setup/create_ddl.py -------------------------------------------------------------------------------- /src/main/python/setup/generate_retail_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/setup/generate_retail_data.py -------------------------------------------------------------------------------- /src/main/python/setup/initiate_setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/setup/initiate_setup.py -------------------------------------------------------------------------------- /src/main/python/silver/call_standardise_retail_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/silver/call_standardise_retail_dataset.py -------------------------------------------------------------------------------- /src/main/python/silver/call_transform_to_scd2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/silver/call_transform_to_scd2.py -------------------------------------------------------------------------------- /src/main/python/silver/standardise_retail_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/silver/standardise_retail_dataset.py -------------------------------------------------------------------------------- /src/main/python/silver/transform_to_scd2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/python/silver/transform_to_scd2.py -------------------------------------------------------------------------------- /src/main/tests/bronze/test_load_data_into_bronze.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/bronze/test_load_data_into_bronze.py -------------------------------------------------------------------------------- /src/main/tests/cleanup_tests.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/cleanup_tests.py -------------------------------------------------------------------------------- /src/main/tests/gold/test_gold_layer_etl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/gold/test_gold_layer_etl.py -------------------------------------------------------------------------------- /src/main/tests/integration_suite/cleanup_integration_suite.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/integration_suite/cleanup_integration_suite.py -------------------------------------------------------------------------------- /src/main/tests/integration_suite/integration_tests.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/integration_suite/integration_tests.py -------------------------------------------------------------------------------- /src/main/tests/integration_suite/test_integration_bronze_layer_etl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/integration_suite/test_integration_bronze_layer_etl.py -------------------------------------------------------------------------------- /src/main/tests/integration_suite/test_integration_gold_layer_etl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/integration_suite/test_integration_gold_layer_etl.py -------------------------------------------------------------------------------- /src/main/tests/integration_suite/test_integration_silver_layer_etl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/integration_suite/test_integration_silver_layer_etl.py -------------------------------------------------------------------------------- /src/main/tests/silver/test_standardise_retail_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/silver/test_standardise_retail_dataset.py -------------------------------------------------------------------------------- /src/main/tests/silver/test_transform_to_scd2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/silver/test_transform_to_scd2.py -------------------------------------------------------------------------------- /src/main/tests/test_all.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shivampanicker/cicd_with_databricks/HEAD/src/main/tests/test_all.py -------------------------------------------------------------------------------- /unit-testing-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | coverage 3 | flake8 4 | pyspark --------------------------------------------------------------------------------