├── .env ├── .gitignore ├── README.md ├── dags ├── Final_app.py ├── branchOperator.py ├── infromation_scrapy_app.py ├── insert_data.py ├── scrapy_develop.ipynb ├── telegram.py └── time_zone.py ├── database └── sql │ └── create_table.sql ├── docker-compose.yml └── python_package ├── Dockerfile └── requirements.txt /.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/.env -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | logs/ 2 | __pycache__ 3 | pg/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/README.md -------------------------------------------------------------------------------- /dags/Final_app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/dags/Final_app.py -------------------------------------------------------------------------------- /dags/branchOperator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/dags/branchOperator.py -------------------------------------------------------------------------------- /dags/infromation_scrapy_app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/dags/infromation_scrapy_app.py -------------------------------------------------------------------------------- /dags/insert_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/dags/insert_data.py -------------------------------------------------------------------------------- /dags/scrapy_develop.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/dags/scrapy_develop.ipynb -------------------------------------------------------------------------------- /dags/telegram.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/dags/telegram.py -------------------------------------------------------------------------------- /dags/time_zone.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/dags/time_zone.py -------------------------------------------------------------------------------- /database/sql/create_table.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/database/sql/create_table.sql -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /python_package/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/python_package/Dockerfile -------------------------------------------------------------------------------- /python_package/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChickenBenny/Airflow-scraping-ETL-tutorial/HEAD/python_package/requirements.txt --------------------------------------------------------------------------------