├── .gitignore ├── LICENSE ├── README.md ├── config-hadoop ├── docker-compose.yml ├── docker ├── consumer │ ├── Dockerfile │ ├── hdfs.py │ ├── kafka_consumer.py │ └── requirements.txt └── producer │ ├── Dockerfile │ ├── hb-data.json │ ├── kafka_producer.py │ └── requirements.txt └── images ├── hadoop.png ├── hdfs_data.png ├── hdfs_datanode.png ├── ingress.png ├── instance.png ├── kafka_ui.png └── map.png /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/README.md -------------------------------------------------------------------------------- /config-hadoop: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/config-hadoop -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /docker/consumer/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/docker/consumer/Dockerfile -------------------------------------------------------------------------------- /docker/consumer/hdfs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/docker/consumer/hdfs.py -------------------------------------------------------------------------------- /docker/consumer/kafka_consumer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/docker/consumer/kafka_consumer.py -------------------------------------------------------------------------------- /docker/consumer/requirements.txt: -------------------------------------------------------------------------------- 1 | kafka-python 2 | PyHDFS -------------------------------------------------------------------------------- /docker/producer/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/docker/producer/Dockerfile -------------------------------------------------------------------------------- /docker/producer/hb-data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/docker/producer/hb-data.json -------------------------------------------------------------------------------- /docker/producer/kafka_producer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/docker/producer/kafka_producer.py -------------------------------------------------------------------------------- /docker/producer/requirements.txt: -------------------------------------------------------------------------------- 1 | kafka-python -------------------------------------------------------------------------------- /images/hadoop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/images/hadoop.png -------------------------------------------------------------------------------- /images/hdfs_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/images/hdfs_data.png -------------------------------------------------------------------------------- /images/hdfs_datanode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/images/hdfs_datanode.png -------------------------------------------------------------------------------- /images/ingress.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/images/ingress.png -------------------------------------------------------------------------------- /images/instance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/images/instance.png -------------------------------------------------------------------------------- /images/kafka_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/images/kafka_ui.png -------------------------------------------------------------------------------- /images/map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhmetFurkanDEMIR/Data-Engineering-Project-with-HDFS-and-Kafka/HEAD/images/map.png --------------------------------------------------------------------------------