├── Main ├── .idea │ └── workspace.xml ├── Dashboard.pbix ├── Lambda │ ├── .idea │ │ ├── .gitignore │ │ ├── .name │ │ ├── inspectionProfiles │ │ │ └── profiles_settings.xml │ │ ├── misc.xml │ │ ├── modules.xml │ │ ├── price prediction (big data envirnment).iml │ │ └── vcs.xml │ ├── Batch_layer │ │ ├── HDFS_consumer.py │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── HDFS_consumer.cpython-310.pyc │ │ │ ├── __init__.cpython-310.pyc │ │ │ ├── batch_layer.cpython-310.pyc │ │ │ ├── put_data_hdfs.cpython-310.pyc │ │ │ ├── save_data_postgresql.cpython-310.pyc │ │ │ └── spark_tranformation.cpython-310.pyc │ │ ├── batch_layer.py │ │ ├── batch_pipeline.py │ │ ├── dags │ │ │ ├── __init__.py │ │ │ └── syc_with_Airflow.py │ │ ├── put_data_hdfs.py │ │ ├── save_data_postgresql.py │ │ └── spark_tranformation.py │ ├── ML_operations │ │ └── xgb_model.pkl │ ├── Stream_data │ │ ├── stream_data.csv │ │ └── stream_data.py │ ├── Stream_layer │ │ ├── ML_consumer.py │ │ ├── __init__.py │ │ ├── insert_data_hbase.py │ │ └── stream_pipeline.py │ ├── __pycache__ │ │ ├── producer.cpython-310.pyc │ │ └── transform.cpython-310.pyc │ ├── docker-compose.yaml │ ├── producer.py │ ├── real_time_web_app(Flask) │ │ ├── __pycache__ │ │ │ └── get_Data_from_hbase.cpython-310.pyc │ │ ├── app.py │ │ ├── get_Data_from_hbase.py │ │ ├── static │ │ │ ├── css │ │ │ │ └── style.css │ │ │ └── js │ │ │ │ └── script.js │ │ └── templates │ │ │ └── index.html │ └── transform.py ├── commands.sh └── real_time_app(Spring boot) │ ├── .gitignore │ ├── .mvn │ └── wrapper │ │ ├── maven-wrapper.jar │ │ └── maven-wrapper.properties │ ├── mvnw │ ├── mvnw.cmd │ ├── pom.xml │ └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── demo │ │ │ ├── RealTimeAppApplication.java │ │ │ ├── controller │ │ │ └── IndexController.java │ │ │ └── service │ │ │ └── HbaseService.java │ └── resources │ │ ├── application.properties │ │ ├── static │ │ ├── css │ │ │ └── style.css │ │ └── js │ │ │ └── script.js │ │ └── templates │ │ └── index.html │ └── test │ └── java │ └── com │ └── example │ └── demo │ └── RealTimeAppApplicationTests.java ├── README.md └── images ├── architecture.png ├── dashboard_phone.png ├── run_web_app.png └── spring_boot_web_app.png /Main/.idea/workspace.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/.idea/workspace.xml -------------------------------------------------------------------------------- /Main/Dashboard.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Dashboard.pbix -------------------------------------------------------------------------------- /Main/Lambda/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /Main/Lambda/.idea/.name: -------------------------------------------------------------------------------- 1 | ML_consumer.py -------------------------------------------------------------------------------- /Main/Lambda/.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/.idea/inspectionProfiles/profiles_settings.xml -------------------------------------------------------------------------------- /Main/Lambda/.idea/misc.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/.idea/misc.xml -------------------------------------------------------------------------------- /Main/Lambda/.idea/modules.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/.idea/modules.xml -------------------------------------------------------------------------------- /Main/Lambda/.idea/price prediction (big data envirnment).iml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/.idea/price prediction (big data envirnment).iml -------------------------------------------------------------------------------- /Main/Lambda/.idea/vcs.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/.idea/vcs.xml -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/HDFS_consumer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/HDFS_consumer.py -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/__pycache__/HDFS_consumer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/__pycache__/HDFS_consumer.cpython-310.pyc -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/__pycache__/batch_layer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/__pycache__/batch_layer.cpython-310.pyc -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/__pycache__/put_data_hdfs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/__pycache__/put_data_hdfs.cpython-310.pyc -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/__pycache__/save_data_postgresql.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/__pycache__/save_data_postgresql.cpython-310.pyc -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/__pycache__/spark_tranformation.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/__pycache__/spark_tranformation.cpython-310.pyc -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/batch_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/batch_layer.py -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/batch_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/batch_pipeline.py -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/dags/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/dags/syc_with_Airflow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/dags/syc_with_Airflow.py -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/put_data_hdfs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/put_data_hdfs.py -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/save_data_postgresql.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/save_data_postgresql.py -------------------------------------------------------------------------------- /Main/Lambda/Batch_layer/spark_tranformation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Batch_layer/spark_tranformation.py -------------------------------------------------------------------------------- /Main/Lambda/ML_operations/xgb_model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/ML_operations/xgb_model.pkl -------------------------------------------------------------------------------- /Main/Lambda/Stream_data/stream_data.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Stream_data/stream_data.csv -------------------------------------------------------------------------------- /Main/Lambda/Stream_data/stream_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Stream_data/stream_data.py -------------------------------------------------------------------------------- /Main/Lambda/Stream_layer/ML_consumer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Stream_layer/ML_consumer.py -------------------------------------------------------------------------------- /Main/Lambda/Stream_layer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Main/Lambda/Stream_layer/insert_data_hbase.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Stream_layer/insert_data_hbase.py -------------------------------------------------------------------------------- /Main/Lambda/Stream_layer/stream_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/Stream_layer/stream_pipeline.py -------------------------------------------------------------------------------- /Main/Lambda/__pycache__/producer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/__pycache__/producer.cpython-310.pyc -------------------------------------------------------------------------------- /Main/Lambda/__pycache__/transform.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/__pycache__/transform.cpython-310.pyc -------------------------------------------------------------------------------- /Main/Lambda/docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/docker-compose.yaml -------------------------------------------------------------------------------- /Main/Lambda/producer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/producer.py -------------------------------------------------------------------------------- /Main/Lambda/real_time_web_app(Flask)/__pycache__/get_Data_from_hbase.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/real_time_web_app(Flask)/__pycache__/get_Data_from_hbase.cpython-310.pyc -------------------------------------------------------------------------------- /Main/Lambda/real_time_web_app(Flask)/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/real_time_web_app(Flask)/app.py -------------------------------------------------------------------------------- /Main/Lambda/real_time_web_app(Flask)/get_Data_from_hbase.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/real_time_web_app(Flask)/get_Data_from_hbase.py -------------------------------------------------------------------------------- /Main/Lambda/real_time_web_app(Flask)/static/css/style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/real_time_web_app(Flask)/static/css/style.css -------------------------------------------------------------------------------- /Main/Lambda/real_time_web_app(Flask)/static/js/script.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/real_time_web_app(Flask)/static/js/script.js -------------------------------------------------------------------------------- /Main/Lambda/real_time_web_app(Flask)/templates/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/real_time_web_app(Flask)/templates/index.html -------------------------------------------------------------------------------- /Main/Lambda/transform.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/Lambda/transform.py -------------------------------------------------------------------------------- /Main/commands.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/commands.sh -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/.gitignore -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/.mvn/wrapper/maven-wrapper.properties -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/mvnw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/mvnw -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/mvnw.cmd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/mvnw.cmd -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/pom.xml -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/src/main/java/com/example/demo/RealTimeAppApplication.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/src/main/java/com/example/demo/RealTimeAppApplication.java -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/src/main/java/com/example/demo/controller/IndexController.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/src/main/java/com/example/demo/controller/IndexController.java -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/src/main/java/com/example/demo/service/HbaseService.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/src/main/java/com/example/demo/service/HbaseService.java -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.application.name=real_time_app 2 | -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/src/main/resources/static/css/style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/src/main/resources/static/css/style.css -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/src/main/resources/static/js/script.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/src/main/resources/static/js/script.js -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/src/main/resources/templates/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/src/main/resources/templates/index.html -------------------------------------------------------------------------------- /Main/real_time_app(Spring boot)/src/test/java/com/example/demo/RealTimeAppApplicationTests.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/Main/real_time_app(Spring boot)/src/test/java/com/example/demo/RealTimeAppApplicationTests.java -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/README.md -------------------------------------------------------------------------------- /images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/images/architecture.png -------------------------------------------------------------------------------- /images/dashboard_phone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/images/dashboard_phone.png -------------------------------------------------------------------------------- /images/run_web_app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/images/run_web_app.png -------------------------------------------------------------------------------- /images/spring_boot_web_app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aymane-maghouti/Big-Data-Project/HEAD/images/spring_boot_web_app.png --------------------------------------------------------------------------------