├── .env ├── airflow.env ├── minio.env ├── nessie.env ├── postgres.env └── superset.env ├── README.md ├── airflow ├── airflow-scheduler.Dockerfile ├── airflow.Dockerfile └── entrypoint.sh ├── docker-compose.yaml ├── ingestion ├── ingestion.Dockerfile └── requirements.txt ├── minio_data └── minio.Dockerfile ├── postgres ├── init-db.sh └── postgres.Dockerfile ├── superset ├── predefined-database.json ├── superset-init.sh └── superset.Dockerfile ├── transformation ├── java_opts.txt ├── spark-defaults.conf └── transformation.Dockerfile └── trino └── etc ├── catalog └── nessie.properties ├── config.properties ├── jvm.config ├── log.properties └── node.properties /.env/airflow.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/.env/airflow.env -------------------------------------------------------------------------------- /.env/minio.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/.env/minio.env -------------------------------------------------------------------------------- /.env/nessie.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/.env/nessie.env -------------------------------------------------------------------------------- /.env/postgres.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/.env/postgres.env -------------------------------------------------------------------------------- /.env/superset.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/.env/superset.env -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/README.md -------------------------------------------------------------------------------- /airflow/airflow-scheduler.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/airflow/airflow-scheduler.Dockerfile -------------------------------------------------------------------------------- /airflow/airflow.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/airflow/airflow.Dockerfile -------------------------------------------------------------------------------- /airflow/entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/airflow/entrypoint.sh -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/docker-compose.yaml -------------------------------------------------------------------------------- /ingestion/ingestion.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/ingestion/ingestion.Dockerfile -------------------------------------------------------------------------------- /ingestion/requirements.txt: -------------------------------------------------------------------------------- 1 | beautifulsoup4==4.12.3 2 | boto3==1.35.14 3 | pandas==2.2.3 4 | -------------------------------------------------------------------------------- /minio_data/minio.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/minio_data/minio.Dockerfile -------------------------------------------------------------------------------- /postgres/init-db.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/postgres/init-db.sh -------------------------------------------------------------------------------- /postgres/postgres.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/postgres/postgres.Dockerfile -------------------------------------------------------------------------------- /superset/predefined-database.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/superset/predefined-database.json -------------------------------------------------------------------------------- /superset/superset-init.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/superset/superset-init.sh -------------------------------------------------------------------------------- /superset/superset.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/superset/superset.Dockerfile -------------------------------------------------------------------------------- /transformation/java_opts.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /transformation/spark-defaults.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/transformation/spark-defaults.conf -------------------------------------------------------------------------------- /transformation/transformation.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/transformation/transformation.Dockerfile -------------------------------------------------------------------------------- /trino/etc/catalog/nessie.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/trino/etc/catalog/nessie.properties -------------------------------------------------------------------------------- /trino/etc/config.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/trino/etc/config.properties -------------------------------------------------------------------------------- /trino/etc/jvm.config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/trino/etc/jvm.config -------------------------------------------------------------------------------- /trino/etc/log.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/trino/etc/log.properties -------------------------------------------------------------------------------- /trino/etc/node.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Armaan1Gohil/dataengineering-tech-stack/HEAD/trino/etc/node.properties --------------------------------------------------------------------------------