├── log-store ├── consumer.js ├── package.json ├── producer.js ├── system_logs.json └── topic.js ├── ornek-1 ├── consumer.js ├── package.json ├── producer.js └── topic.js └── pub-sub ├── consumer_1k_2k.js ├── consumer_4k_8k.js ├── consumer_mobile.js ├── package-lock.json ├── package.json ├── producer.js └── topic.js /log-store/consumer.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | createConsumer(); 4 | 5 | async function createConsumer() { 6 | try { 7 | const kafka = new Kafka({ 8 | clientId: "kafka_log_store_client", 9 | brokers: ["192.168.8.148:9092"] 10 | }); 11 | 12 | const consumer = kafka.consumer({ 13 | groupId: "log_store_consumer_group" 14 | }); 15 | 16 | console.log("Consumer'a bağlanılıyor.."); 17 | await consumer.connect(); 18 | console.log("Bağlantı başarılı."); 19 | 20 | // Consumer Subscribe.. 21 | await consumer.subscribe({ 22 | topic: "LogStoreTopic", 23 | fromBeginning: true 24 | }); 25 | 26 | await consumer.run({ 27 | eachMessage: async result => { 28 | console.log( 29 | `Gelen Mesaj ${result.message.value}, Par => ${result.partition}` 30 | ); 31 | } 32 | }); 33 | } catch (error) { 34 | console.log("Bir Hata Oluştu", error); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /log-store/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ornek-1", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "kafkajs": "^1.12.0" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /log-store/producer.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | const log_data = require("./system_logs.json"); 3 | 4 | createProducer(); 5 | 6 | async function createProducer() { 7 | try { 8 | const kafka = new Kafka({ 9 | clientId: "kafka_log_store_client", 10 | brokers: ["192.168.8.148:9092"] 11 | }); 12 | 13 | const producer = kafka.producer(); 14 | console.log("Producer'a bağlanılıyor.."); 15 | await producer.connect(); 16 | console.log("Bağlantı başarılı."); 17 | let messages = log_data.map(item => { 18 | return { 19 | value: JSON.stringify(item), 20 | partition: item.type == "system" ? 0 : 1 21 | }; 22 | }); 23 | 24 | const message_result = await producer.send({ 25 | topic: "LogStoreTopic", 26 | messages: messages 27 | }); 28 | console.log("Gonderim işlemi başarılıdır", JSON.stringify(message_result)); 29 | await producer.disconnect(); 30 | } catch (error) { 31 | console.log("Bir Hata Oluştu", error); 32 | } finally { 33 | process.exit(0); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /log-store/topic.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | createTopic(); 4 | 5 | async function createTopic() { 6 | try { 7 | // Admin Stuff.. 8 | const kafka = new Kafka({ 9 | clientId: "kafka_log_store_client", 10 | brokers: ["192.168.8.148:9092"] 11 | }); 12 | 13 | const admin = kafka.admin(); 14 | console.log("Kafka Broker'a bağlanılıyor..."); 15 | await admin.connect(); 16 | console.log("Kafka Broker'a bağlantı başarılı, Topic üretilecek.."); 17 | await admin.createTopics({ 18 | topics: [ 19 | { 20 | topic: "LogStoreTopic", 21 | numPartitions: 2 22 | } 23 | ] 24 | }); 25 | console.log("Topic Başarılı bir şekilde oluşturulmuştur..."); 26 | await admin.disconnect(); 27 | } catch (error) { 28 | console.log("Bir Hata Oluştu", error); 29 | } finally { 30 | process.exit(0); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ornek-1/consumer.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | // node consumer.js Logs || Logs2 4 | const topic_name = process.argv[2] || "Logs2"; 5 | 6 | createConsumer(); 7 | 8 | async function createConsumer() { 9 | try { 10 | const kafka = new Kafka({ 11 | clientId: "kafka_ornek_1", 12 | brokers: ["192.168.8.148:9092"] 13 | }); 14 | 15 | const consumer = kafka.consumer({ 16 | groupId: "ornek_1_cg_1" 17 | }); 18 | 19 | console.log("Consumer'a bağlanılıyor.."); 20 | await consumer.connect(); 21 | console.log("Bağlantı başarılı."); 22 | 23 | // Consumer Subscribe.. 24 | await consumer.subscribe({ 25 | topic: topic_name, 26 | fromBeginning: true 27 | }); 28 | 29 | await consumer.run({ 30 | eachMessage: async result => { 31 | console.log( 32 | `Gelen Mesaj ${result.message.value}, Par => ${result.partition}` 33 | ); 34 | } 35 | }); 36 | } catch (error) { 37 | console.log("Bir Hata Oluştu", error); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /ornek-1/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ornek-1", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "kafkajs": "^1.12.0" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /ornek-1/producer.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | const topic_name = process.argv[2] || "Logs2"; 4 | const partition = process.argv[3] || 0; 5 | 6 | createProducer(); 7 | 8 | async function createProducer() { 9 | try { 10 | const kafka = new Kafka({ 11 | clientId: "kafka_ornek_1", 12 | brokers: ["192.168.8.148:9092"] 13 | }); 14 | 15 | const producer = kafka.producer(); 16 | console.log("Producer'a bağlanılıyor.."); 17 | await producer.connect(); 18 | console.log("Bağlantı başarılı."); 19 | 20 | const message_result = await producer.send({ 21 | topic: topic_name, 22 | messages: [ 23 | { 24 | value: "Bu bir test Log Mesajıdır...", 25 | partition: partition 26 | } 27 | ] 28 | }); 29 | console.log("Gonderim işlemi başarılıdır", JSON.stringify(message_result)); 30 | await producer.disconnect(); 31 | } catch (error) { 32 | console.log("Bir Hata Oluştu", error); 33 | } finally { 34 | process.exit(0); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /ornek-1/topic.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | createTopic(); 4 | 5 | async function createTopic() { 6 | try { 7 | // Admin Stuff.. 8 | const kafka = new Kafka({ 9 | clientId: "kafka_ornek_1", 10 | brokers: ["192.168.8.148:9092"] 11 | }); 12 | 13 | const admin = kafka.admin(); 14 | console.log("Kafka Broker'a bağlanılıyor..."); 15 | await admin.connect(); 16 | console.log("Kafka Broker'a bağlantı başarılı, Topic üretilecek.."); 17 | await admin.createTopics({ 18 | topics: [ 19 | { 20 | topic: "Logs", 21 | numPartitions: 1 22 | }, 23 | { 24 | topic: "Logs2", 25 | numPartitions: 2 26 | } 27 | ] 28 | }); 29 | console.log("Topic Başarılı bir şekilde oluşturulmuştur..."); 30 | await admin.disconnect(); 31 | } catch (error) { 32 | console.log("Bir Hata Oluştu", error); 33 | } finally { 34 | process.exit(0); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pub-sub/consumer_1k_2k.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | createConsumer(); 4 | 5 | async function createConsumer() { 6 | try { 7 | const kafka = new Kafka({ 8 | clientId: "kafka_pub_sub_client", 9 | brokers: ["192.168.8.148:9092"] 10 | }); 11 | 12 | const consumer = kafka.consumer({ 13 | groupId: "hd_1k_2k_encoder_consumer_group" 14 | }); 15 | 16 | console.log("Consumer'a bağlanılıyor.."); 17 | await consumer.connect(); 18 | console.log("Bağlantı başarılı."); 19 | 20 | // Consumer Subscribe.. 21 | await consumer.subscribe({ 22 | topic: "raw_video_topic", 23 | fromBeginning: true 24 | }); 25 | 26 | await consumer.run({ 27 | eachMessage: async result => { 28 | console.log(`İşlenen Video ${result.message.value}_1k_2k_encoder`); 29 | } 30 | }); 31 | } catch (error) { 32 | console.log("Bir Hata Oluştu", error); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /pub-sub/consumer_4k_8k.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | createConsumer(); 4 | 5 | async function createConsumer() { 6 | try { 7 | const kafka = new Kafka({ 8 | clientId: "kafka_pub_sub_client", 9 | brokers: ["192.168.8.148:9092"] 10 | }); 11 | 12 | const consumer = kafka.consumer({ 13 | groupId: "hd_4k_8k_encoder_consumer_group" 14 | }); 15 | 16 | console.log("Consumer'a bağlanılıyor.."); 17 | await consumer.connect(); 18 | console.log("Bağlantı başarılı."); 19 | 20 | // Consumer Subscribe.. 21 | await consumer.subscribe({ 22 | topic: "raw_video_topic", 23 | fromBeginning: true 24 | }); 25 | 26 | await consumer.run({ 27 | eachMessage: async result => { 28 | console.log(`İşlenen Video ${result.message.value}_4k_8k_encoder`); 29 | } 30 | }); 31 | } catch (error) { 32 | console.log("Bir Hata Oluştu", error); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /pub-sub/consumer_mobile.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | createConsumer(); 4 | 5 | async function createConsumer() { 6 | try { 7 | const kafka = new Kafka({ 8 | clientId: "kafka_pub_sub_client", 9 | brokers: ["192.168.8.148:9092"] 10 | }); 11 | 12 | const consumer = kafka.consumer({ 13 | groupId: "mobile_encoder_consumer_group" 14 | }); 15 | 16 | console.log("Consumer'a bağlanılıyor.."); 17 | await consumer.connect(); 18 | console.log("Bağlantı başarılı."); 19 | 20 | // Consumer Subscribe.. 21 | await consumer.subscribe({ 22 | topic: "raw_video_topic", 23 | fromBeginning: true 24 | }); 25 | 26 | await consumer.run({ 27 | eachMessage: async result => { 28 | console.log(`İşlenen Video ${result.message.value}_mobile_encoder`); 29 | } 30 | }); 31 | } catch (error) { 32 | console.log("Bir Hata Oluştu", error); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /pub-sub/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ornek-1", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "kafkajs": { 8 | "version": "1.12.0", 9 | "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-1.12.0.tgz", 10 | "integrity": "sha512-Izkd9iFRgeeKaHEgVpGQH08ygzCbHSxTbnu8W3G3uiNaVjGibUTmTwjv1Qf2M8NORXcPfzwVyg6bBlVj4SKr9g==", 11 | "requires": { 12 | "long": "^4.0.0" 13 | } 14 | }, 15 | "long": { 16 | "version": "4.0.0", 17 | "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", 18 | "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pub-sub/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ornek-1", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "kafkajs": "^1.12.0" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /pub-sub/producer.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | createProducer(); 4 | 5 | async function createProducer() { 6 | try { 7 | const kafka = new Kafka({ 8 | clientId: "kafka_pub_sub_client", 9 | brokers: ["192.168.8.148:9092"] 10 | }); 11 | 12 | const producer = kafka.producer(); 13 | console.log("Producer'a bağlanılıyor.."); 14 | await producer.connect(); 15 | console.log("Bağlantı başarılı."); 16 | const message_result = await producer.send({ 17 | topic: "raw_video_topic", 18 | messages: [ 19 | { 20 | value: "Yeni Video İçeriği", 21 | partition: 0 22 | } 23 | ] 24 | }); 25 | console.log("Gonderim işlemi başarılıdır", JSON.stringify(message_result)); 26 | await producer.disconnect(); 27 | } catch (error) { 28 | console.log("Bir Hata Oluştu", error); 29 | } finally { 30 | process.exit(0); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pub-sub/topic.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | createTopic(); 4 | 5 | async function createTopic() { 6 | try { 7 | // Admin Stuff.. 8 | const kafka = new Kafka({ 9 | clientId: "kafka_pub_sub_client", 10 | brokers: ["192.168.8.148:9092"] 11 | }); 12 | 13 | const admin = kafka.admin(); 14 | console.log("Kafka Broker'a bağlanılıyor..."); 15 | await admin.connect(); 16 | console.log("Kafka Broker'a bağlantı başarılı, Topic üretilecek.."); 17 | await admin.createTopics({ 18 | topics: [ 19 | { 20 | topic: "raw_video_topic", 21 | numPartitions: 1 22 | } 23 | ] 24 | }); 25 | console.log("Topic Başarılı bir şekilde oluşturulmuştur..."); 26 | await admin.disconnect(); 27 | } catch (error) { 28 | console.log("Bir Hata Oluştu", error); 29 | } finally { 30 | process.exit(0); 31 | } 32 | } 33 | --------------------------------------------------------------------------------