├── .gitattributes ├── .github ├── README-config.md └── workflows │ └── upload-spark-script-to-bucket.yml ├── .gitignore ├── LICENSE ├── README.md ├── airflow ├── .airflowignore ├── .env ├── .gitignore ├── Dockerfile ├── README.md ├── __init__.py ├── dags │ ├── common_package │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-38.pyc │ │ │ └── utils_module.cpython-38.pyc │ │ └── utils_module.py │ └── custom_dags │ │ ├── __init__.py │ │ ├── reviews_ingest_dag.py │ │ ├── store_and_reviews_ingest_dag.py │ │ └── store_ingest_dag.py ├── docker-compose.yaml ├── requirements.txt └── scripts │ └── entrypoint.sh ├── dbt ├── .gitignore ├── README.md ├── analyses │ └── .gitkeep ├── dbt_project.yml ├── macros │ ├── .gitkeep │ ├── fix_bools.sql │ └── fix_strings.sql ├── models │ ├── analysis │ │ ├── devs_metacritic.sql │ │ ├── dlcs_by_game.sql │ │ ├── games_by_genre.sql │ │ └── steam_games_analysis.sql │ ├── core │ │ ├── steam_dlc_data.sql │ │ ├── steam_games.sql │ │ └── steam_reviews.sql │ ├── schema.yml │ └── staging │ │ ├── categories.sql │ │ ├── developers.sql │ │ ├── genres.sql │ │ ├── publishers.sql │ │ ├── steam_spy_scrap.sql │ │ └── steam_store_data.sql ├── seeds │ └── .gitkeep ├── snapshots │ └── .gitkeep └── tests │ ├── .gitkeep │ └── assert_release_year_is_valid.sql ├── img ├── airflow_gant.png ├── airflow_graph.png ├── airflow_grid.png ├── dashboard.png ├── dbt_linege.png ├── lineage_dbt_1.png ├── lineage_dbt_2.png ├── lineage_dbt_3.png ├── owners_new.png ├── owners_old.png ├── schema_denorm.png ├── steam.jpg └── steam_logo.png ├── spark ├── .gitignore ├── README.md ├── all_games_reviews_gcp.ipynb └── spark_all_games_reviews.py └── terraform ├── .gitignore ├── README.md ├── operator-workspace ├── main.tf └── variables.tf └── vault-admin-workspace └── main.tf /.gitattributes: -------------------------------------------------------------------------------- 1 | *.sql linguist-detectable=true -------------------------------------------------------------------------------- /.github/README-config.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/.github/README-config.md -------------------------------------------------------------------------------- /.github/workflows/upload-spark-script-to-bucket.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/.github/workflows/upload-spark-script-to-bucket.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/README.md -------------------------------------------------------------------------------- /airflow/.airflowignore: -------------------------------------------------------------------------------- 1 | airflow/dags/common_package -------------------------------------------------------------------------------- /airflow/.env: -------------------------------------------------------------------------------- 1 | AIRFLOW_UID=1000 2 | -------------------------------------------------------------------------------- /airflow/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/.gitignore -------------------------------------------------------------------------------- /airflow/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/Dockerfile -------------------------------------------------------------------------------- /airflow/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/README.md -------------------------------------------------------------------------------- /airflow/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /airflow/dags/common_package/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /airflow/dags/common_package/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/dags/common_package/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /airflow/dags/common_package/__pycache__/utils_module.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/dags/common_package/__pycache__/utils_module.cpython-38.pyc -------------------------------------------------------------------------------- /airflow/dags/common_package/utils_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/dags/common_package/utils_module.py -------------------------------------------------------------------------------- /airflow/dags/custom_dags/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /airflow/dags/custom_dags/reviews_ingest_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/dags/custom_dags/reviews_ingest_dag.py -------------------------------------------------------------------------------- /airflow/dags/custom_dags/store_and_reviews_ingest_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/dags/custom_dags/store_and_reviews_ingest_dag.py -------------------------------------------------------------------------------- /airflow/dags/custom_dags/store_ingest_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/dags/custom_dags/store_ingest_dag.py -------------------------------------------------------------------------------- /airflow/docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/docker-compose.yaml -------------------------------------------------------------------------------- /airflow/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/requirements.txt -------------------------------------------------------------------------------- /airflow/scripts/entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/airflow/scripts/entrypoint.sh -------------------------------------------------------------------------------- /dbt/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dbt/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/README.md -------------------------------------------------------------------------------- /dbt/analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/dbt_project.yml -------------------------------------------------------------------------------- /dbt/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/macros/fix_bools.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/macros/fix_bools.sql -------------------------------------------------------------------------------- /dbt/macros/fix_strings.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/macros/fix_strings.sql -------------------------------------------------------------------------------- /dbt/models/analysis/devs_metacritic.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/analysis/devs_metacritic.sql -------------------------------------------------------------------------------- /dbt/models/analysis/dlcs_by_game.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/analysis/dlcs_by_game.sql -------------------------------------------------------------------------------- /dbt/models/analysis/games_by_genre.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/analysis/games_by_genre.sql -------------------------------------------------------------------------------- /dbt/models/analysis/steam_games_analysis.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/analysis/steam_games_analysis.sql -------------------------------------------------------------------------------- /dbt/models/core/steam_dlc_data.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/core/steam_dlc_data.sql -------------------------------------------------------------------------------- /dbt/models/core/steam_games.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/core/steam_games.sql -------------------------------------------------------------------------------- /dbt/models/core/steam_reviews.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/core/steam_reviews.sql -------------------------------------------------------------------------------- /dbt/models/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/schema.yml -------------------------------------------------------------------------------- /dbt/models/staging/categories.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/staging/categories.sql -------------------------------------------------------------------------------- /dbt/models/staging/developers.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/staging/developers.sql -------------------------------------------------------------------------------- /dbt/models/staging/genres.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/staging/genres.sql -------------------------------------------------------------------------------- /dbt/models/staging/publishers.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/staging/publishers.sql -------------------------------------------------------------------------------- /dbt/models/staging/steam_spy_scrap.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/staging/steam_spy_scrap.sql -------------------------------------------------------------------------------- /dbt/models/staging/steam_store_data.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/models/staging/steam_store_data.sql -------------------------------------------------------------------------------- /dbt/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/tests/assert_release_year_is_valid.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/dbt/tests/assert_release_year_is_valid.sql -------------------------------------------------------------------------------- /img/airflow_gant.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/airflow_gant.png -------------------------------------------------------------------------------- /img/airflow_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/airflow_graph.png -------------------------------------------------------------------------------- /img/airflow_grid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/airflow_grid.png -------------------------------------------------------------------------------- /img/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/dashboard.png -------------------------------------------------------------------------------- /img/dbt_linege.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/dbt_linege.png -------------------------------------------------------------------------------- /img/lineage_dbt_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/lineage_dbt_1.png -------------------------------------------------------------------------------- /img/lineage_dbt_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/lineage_dbt_2.png -------------------------------------------------------------------------------- /img/lineage_dbt_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/lineage_dbt_3.png -------------------------------------------------------------------------------- /img/owners_new.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/owners_new.png -------------------------------------------------------------------------------- /img/owners_old.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/owners_old.png -------------------------------------------------------------------------------- /img/schema_denorm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/schema_denorm.png -------------------------------------------------------------------------------- /img/steam.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/steam.jpg -------------------------------------------------------------------------------- /img/steam_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/img/steam_logo.png -------------------------------------------------------------------------------- /spark/.gitignore: -------------------------------------------------------------------------------- 1 | lib/ 2 | -------------------------------------------------------------------------------- /spark/README.md: -------------------------------------------------------------------------------- 1 | # Spark 2 | 3 | [In progess ...] 4 | -------------------------------------------------------------------------------- /spark/all_games_reviews_gcp.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/spark/all_games_reviews_gcp.ipynb -------------------------------------------------------------------------------- /spark/spark_all_games_reviews.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/spark/spark_all_games_reviews.py -------------------------------------------------------------------------------- /terraform/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/terraform/.gitignore -------------------------------------------------------------------------------- /terraform/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/terraform/README.md -------------------------------------------------------------------------------- /terraform/operator-workspace/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/terraform/operator-workspace/main.tf -------------------------------------------------------------------------------- /terraform/operator-workspace/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/terraform/operator-workspace/variables.tf -------------------------------------------------------------------------------- /terraform/vault-admin-workspace/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VicenteYago/steam-data-engineering/HEAD/terraform/vault-admin-workspace/main.tf --------------------------------------------------------------------------------