├── .gitignore ├── README.md ├── docker-compose.yml ├── imgs ├── HLA.png ├── add_conn.png └── connections.png └── src ├── Dockerfile ├── dags ├── hello_spark.py ├── otrium_daily.py └── spark-postgres.py ├── requirements.txt ├── script └── entrypoint.sh └── spark ├── applications ├── hello-world-spark.py ├── hello-world.py ├── load-postgres.py └── read-postgres.py └── assets ├── data ├── movies.csv └── ratings.csv └── jars └── postgresql-42.2.6.jar /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/README.md -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /imgs/HLA.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/imgs/HLA.png -------------------------------------------------------------------------------- /imgs/add_conn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/imgs/add_conn.png -------------------------------------------------------------------------------- /imgs/connections.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/imgs/connections.png -------------------------------------------------------------------------------- /src/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/Dockerfile -------------------------------------------------------------------------------- /src/dags/hello_spark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/dags/hello_spark.py -------------------------------------------------------------------------------- /src/dags/otrium_daily.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/dags/otrium_daily.py -------------------------------------------------------------------------------- /src/dags/spark-postgres.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/dags/spark-postgres.py -------------------------------------------------------------------------------- /src/requirements.txt: -------------------------------------------------------------------------------- 1 | celery 2 | kombu 3 | tornado 4 | werkzeug 5 | pyspark==3.2.1 6 | SQLAlchemy 7 | wtforms -------------------------------------------------------------------------------- /src/script/entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/script/entrypoint.sh -------------------------------------------------------------------------------- /src/spark/applications/hello-world-spark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/spark/applications/hello-world-spark.py -------------------------------------------------------------------------------- /src/spark/applications/hello-world.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/spark/applications/hello-world.py -------------------------------------------------------------------------------- /src/spark/applications/load-postgres.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/spark/applications/load-postgres.py -------------------------------------------------------------------------------- /src/spark/applications/read-postgres.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/spark/applications/read-postgres.py -------------------------------------------------------------------------------- /src/spark/assets/data/movies.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/spark/assets/data/movies.csv -------------------------------------------------------------------------------- /src/spark/assets/data/ratings.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/spark/assets/data/ratings.csv -------------------------------------------------------------------------------- /src/spark/assets/jars/postgresql-42.2.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrejnevesjr/airflow-spark-minio-postgres/HEAD/src/spark/assets/jars/postgresql-42.2.6.jar --------------------------------------------------------------------------------