├── .env ├── .gitignore ├── LICENSE ├── README.md ├── airflow_dockerfile ├── Dockerfile └── requirements.txt ├── bash_control ├── down.sh └── up.sh ├── clickhouse └── init │ └── init.sql ├── dags ├── Create_Postgres__app_installs.py ├── Create_Postgres__order_events.py ├── Load_API__earthquake.py ├── Load_API__github.py ├── Load_API__weather.py ├── Load_JDBC__app_installs.py ├── Load_JDBC__regions.py ├── Load_Kafka__order_events.py └── Transform_ObjStore__earthquake_regions.py ├── dbt_click ├── .gitignore ├── README.md ├── analyses │ └── .gitkeep ├── dbt_project.yml ├── macros │ └── .gitkeep ├── models │ ├── datamart │ │ ├── debit_cards.sql │ │ └── schema.yaml │ └── monitoring │ │ ├── logs_datamart.sql │ │ └── schema.yaml ├── profiles.yml ├── seeds │ └── .gitkeep ├── snapshots │ └── .gitkeep └── tests │ └── .gitkeep ├── debezium-grafana ├── Dockerfile ├── dashboard.yml ├── datasource.yml └── debezium-dashboard.json ├── debezium-jmx-exporter ├── Dockerfile └── config.yml ├── debezium-prometheus └── prometheus.yml ├── docker-compose.yaml ├── docker_related_config.xml ├── jupyter_dockerfile ├── Dockerfile └── requirements.txt ├── jupyter_notebook └── spark_submit.ipynb ├── pdf ├── .DS_Store └── dashboard_debit_cards.pdf ├── plugins ├── __ini__.py ├── __pycache__ │ ├── __ini__.cpython-39.pyc │ └── db_utils.cpython-39.pyc └── db_utils.py ├── png ├── .DS_Store └── main.png ├── scripts ├── legacy │ ├── data_quality.py │ ├── data_stream_synth.py │ ├── data_synth.py │ ├── debit_cards.py │ └── extract_data.py ├── load │ ├── load__app_installs.py │ ├── load__full_refresh.py │ └── load__order_events.py └── transform │ └── transform__earthquake_regions.py ├── src ├── init_db │ └── init.sql └── kafka_connectors │ └── backend__order_events.http └── superset_dockerfile ├── Dockerfile ├── superset-init.sh ├── superset.db └── superset_config.py /.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/.env -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/README.md -------------------------------------------------------------------------------- /airflow_dockerfile/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/airflow_dockerfile/Dockerfile -------------------------------------------------------------------------------- /airflow_dockerfile/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/airflow_dockerfile/requirements.txt -------------------------------------------------------------------------------- /bash_control/down.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker compose down -v -------------------------------------------------------------------------------- /bash_control/up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker compose up -d -------------------------------------------------------------------------------- /clickhouse/init/init.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/clickhouse/init/init.sql -------------------------------------------------------------------------------- /dags/Create_Postgres__app_installs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dags/Create_Postgres__app_installs.py -------------------------------------------------------------------------------- /dags/Create_Postgres__order_events.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dags/Create_Postgres__order_events.py -------------------------------------------------------------------------------- /dags/Load_API__earthquake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dags/Load_API__earthquake.py -------------------------------------------------------------------------------- /dags/Load_API__github.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dags/Load_API__github.py -------------------------------------------------------------------------------- /dags/Load_API__weather.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dags/Load_API__weather.py -------------------------------------------------------------------------------- /dags/Load_JDBC__app_installs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dags/Load_JDBC__app_installs.py -------------------------------------------------------------------------------- /dags/Load_JDBC__regions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dags/Load_JDBC__regions.py -------------------------------------------------------------------------------- /dags/Load_Kafka__order_events.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dags/Load_Kafka__order_events.py -------------------------------------------------------------------------------- /dags/Transform_ObjStore__earthquake_regions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dags/Transform_ObjStore__earthquake_regions.py -------------------------------------------------------------------------------- /dbt_click/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dbt_click/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dbt_click/README.md -------------------------------------------------------------------------------- /dbt_click/analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_click/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dbt_click/dbt_project.yml -------------------------------------------------------------------------------- /dbt_click/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_click/models/datamart/debit_cards.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dbt_click/models/datamart/debit_cards.sql -------------------------------------------------------------------------------- /dbt_click/models/datamart/schema.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: debit_cards -------------------------------------------------------------------------------- /dbt_click/models/monitoring/logs_datamart.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dbt_click/models/monitoring/logs_datamart.sql -------------------------------------------------------------------------------- /dbt_click/models/monitoring/schema.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: logs_datamart -------------------------------------------------------------------------------- /dbt_click/profiles.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/dbt_click/profiles.yml -------------------------------------------------------------------------------- /dbt_click/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_click/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_click/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /debezium-grafana/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/debezium-grafana/Dockerfile -------------------------------------------------------------------------------- /debezium-grafana/dashboard.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/debezium-grafana/dashboard.yml -------------------------------------------------------------------------------- /debezium-grafana/datasource.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/debezium-grafana/datasource.yml -------------------------------------------------------------------------------- /debezium-grafana/debezium-dashboard.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/debezium-grafana/debezium-dashboard.json -------------------------------------------------------------------------------- /debezium-jmx-exporter/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/debezium-jmx-exporter/Dockerfile -------------------------------------------------------------------------------- /debezium-jmx-exporter/config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/debezium-jmx-exporter/config.yml -------------------------------------------------------------------------------- /debezium-prometheus/prometheus.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/debezium-prometheus/prometheus.yml -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/docker-compose.yaml -------------------------------------------------------------------------------- /docker_related_config.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/docker_related_config.xml -------------------------------------------------------------------------------- /jupyter_dockerfile/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/jupyter_dockerfile/Dockerfile -------------------------------------------------------------------------------- /jupyter_dockerfile/requirements.txt: -------------------------------------------------------------------------------- 1 | pyspark -------------------------------------------------------------------------------- /jupyter_notebook/spark_submit.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/jupyter_notebook/spark_submit.ipynb -------------------------------------------------------------------------------- /pdf/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/pdf/.DS_Store -------------------------------------------------------------------------------- /pdf/dashboard_debit_cards.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/pdf/dashboard_debit_cards.pdf -------------------------------------------------------------------------------- /plugins/__ini__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/__pycache__/__ini__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/plugins/__pycache__/__ini__.cpython-39.pyc -------------------------------------------------------------------------------- /plugins/__pycache__/db_utils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/plugins/__pycache__/db_utils.cpython-39.pyc -------------------------------------------------------------------------------- /plugins/db_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/plugins/db_utils.py -------------------------------------------------------------------------------- /png/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/png/.DS_Store -------------------------------------------------------------------------------- /png/main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/png/main.png -------------------------------------------------------------------------------- /scripts/legacy/data_quality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/scripts/legacy/data_quality.py -------------------------------------------------------------------------------- /scripts/legacy/data_stream_synth.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/scripts/legacy/data_stream_synth.py -------------------------------------------------------------------------------- /scripts/legacy/data_synth.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/scripts/legacy/data_synth.py -------------------------------------------------------------------------------- /scripts/legacy/debit_cards.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/scripts/legacy/debit_cards.py -------------------------------------------------------------------------------- /scripts/legacy/extract_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/scripts/legacy/extract_data.py -------------------------------------------------------------------------------- /scripts/load/load__app_installs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/scripts/load/load__app_installs.py -------------------------------------------------------------------------------- /scripts/load/load__full_refresh.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/scripts/load/load__full_refresh.py -------------------------------------------------------------------------------- /scripts/load/load__order_events.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/scripts/load/load__order_events.py -------------------------------------------------------------------------------- /scripts/transform/transform__earthquake_regions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/scripts/transform/transform__earthquake_regions.py -------------------------------------------------------------------------------- /src/init_db/init.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/src/init_db/init.sql -------------------------------------------------------------------------------- /src/kafka_connectors/backend__order_events.http: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/src/kafka_connectors/backend__order_events.http -------------------------------------------------------------------------------- /superset_dockerfile/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/superset_dockerfile/Dockerfile -------------------------------------------------------------------------------- /superset_dockerfile/superset-init.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/superset_dockerfile/superset-init.sh -------------------------------------------------------------------------------- /superset_dockerfile/superset.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/superset_dockerfile/superset.db -------------------------------------------------------------------------------- /superset_dockerfile/superset_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/halltape/HalltapeETL/HEAD/superset_dockerfile/superset_config.py --------------------------------------------------------------------------------