├── .gitignore ├── static └── message.png ├── fixtures └── payload.json ├── src ├── consumer │ ├── consumer.js │ └── index.js ├── server │ ├── app.js │ └── index.js └── config.js ├── package.json ├── docker-compose.yml ├── bin ├── create-topic └── send-event └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ -------------------------------------------------------------------------------- /static/message.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nevon/demo-npm-publish-slack-notifier/HEAD/static/message.png -------------------------------------------------------------------------------- /fixtures/payload.json: -------------------------------------------------------------------------------- 1 | { 2 | "event": "package:publish", 3 | "name": "@kafkajs/zstd", 4 | "type": "package", 5 | "version": "1.0.0", 6 | "hookOwner": { "username": "nevon"}, 7 | "payload": { 8 | "name": "@kafkajs/zstd" 9 | }, 10 | "change": { 11 | "version": "1.0.0" 12 | }, 13 | "time": 1603444214995 14 | } -------------------------------------------------------------------------------- /src/consumer/consumer.js: -------------------------------------------------------------------------------- 1 | module.exports = async ({ kafka, config, slack }) => { 2 | const consumer = kafka.consumer(config.consumer); 3 | 4 | await consumer.connect(); 5 | await consumer.subscribe({ topic: config.app.topic, fromBeginning: true }); 6 | 7 | await consumer.run({ 8 | eachMessage: async ({ message }) => { 9 | const { package, version } = JSON.parse(message.value.toString()); 10 | 11 | const text = `:package: ${package}@${version} released\n`; 12 | 13 | await slack.send({ 14 | text, 15 | username: "Package bot", 16 | }); 17 | }, 18 | }); 19 | 20 | return consumer; 21 | }; 22 | -------------------------------------------------------------------------------- /src/server/app.js: -------------------------------------------------------------------------------- 1 | const createHookReceiver = require("npm-hook-receiver"); 2 | 3 | module.exports = ({ producer, config }) => { 4 | const server = createHookReceiver({ 5 | secret: config.secret, 6 | mount: config.mount, 7 | }); 8 | 9 | server.on( 10 | "package:publish", 11 | async ({ name: package, version, time: timestamp }) => { 12 | console.log("Received webhook event", { 13 | package, 14 | version, 15 | timestamp, 16 | }); 17 | 18 | try { 19 | await producer.send({ 20 | topic: config.topic, 21 | messages: [ 22 | { 23 | key: package, 24 | value: JSON.stringify({ 25 | package, 26 | version, 27 | timestamp, 28 | }), 29 | }, 30 | ], 31 | }); 32 | } catch (error) { 33 | console.error(`Failed to publish webhook message`, error); 34 | } 35 | } 36 | ); 37 | 38 | return server; 39 | }; 40 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "npm-slack-notifier", 3 | "version": "0.1.0", 4 | "description": "Demo KafkaJS application to notify Slack webhook on NPM package releases", 5 | "main": "src/index.js", 6 | "private": true, 7 | "scripts": { 8 | "start": "(trap 'kill 0' SIGINT; npm run start:server & npm run start:consumer)", 9 | "start:server": "node src/server/index.js", 10 | "start:consumer": "node src/consumer/index.js", 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "git+https://github.com/Nevon/demo-slack-npm-notifier.git" 16 | }, 17 | "author": "Tommy Brunn ", 18 | "license": "MIT", 19 | "bugs": { 20 | "url": "https://github.com/Nevon/demo-slack-npm-notifier/issues" 21 | }, 22 | "homepage": "https://github.com/Nevon/demo-slack-npm-notifier#readme", 23 | "dependencies": { 24 | "@slack/webhook": "^5.0.3", 25 | "kafkajs": "^1.14.0", 26 | "npm-hook-receiver": "^1.0.0" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2' 3 | services: 4 | zookeeper: 5 | image: confluentinc/cp-zookeeper:latest 6 | environment: 7 | ZOOKEEPER_CLIENT_PORT: 2181 8 | ZOOKEEPER_TICK_TIME: 2000 9 | 10 | kafka: 11 | image: confluentinc/cp-kafka:latest 12 | labels: 13 | - 'custom.project=kafkajs-zstd' 14 | - 'custom.service=kafka' 15 | depends_on: 16 | - zookeeper 17 | ports: 18 | - 9092:9092 19 | environment: 20 | KAFKA_BROKER_ID: 1 21 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 22 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 23 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 24 | KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT 25 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 26 | KAFKA_LOG4J_ROOT_LOGLEVEL: INFO 27 | KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO' 28 | CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' -------------------------------------------------------------------------------- /src/config.js: -------------------------------------------------------------------------------- 1 | const server = { port: process.env.PORT || 3000 }; 2 | 3 | const kafka = { 4 | clientId: "npm-slack-notifier", 5 | brokers: [process.env.BOOTSTRAP_BROKER || "localhost:9092"], 6 | ssl: process.env.KAFKA_SSL ? JSON.parse(process.env.KAFKA_SSL) : false, 7 | sasl: 8 | process.env.KAFKA_USERNAME && process.env.KAFKA_PASSWORD 9 | ? { 10 | username: process.env.KAFKA_USERNAME, 11 | password: process.env.KAFKA_PASSWORD, 12 | mechanism: 'plain' 13 | } 14 | : null, 15 | }; 16 | 17 | const consumer = { 18 | groupId: process.env.KAFKA_GROUP_ID || "npm-slack-notifier", 19 | }; 20 | 21 | const app = { 22 | secret: process.env.HOOK_SECRET, 23 | topic: process.env.TOPIC || "npm-package-published", 24 | mount: "/hook", 25 | }; 26 | 27 | const processor = { 28 | topic: app.topic, 29 | }; 30 | 31 | const slack = { 32 | webhookUrl: process.env.SLACK_WEBHOOK_URL, 33 | }; 34 | 35 | module.exports = { 36 | server, 37 | kafka, 38 | consumer, 39 | app, 40 | processor, 41 | slack, 42 | }; 43 | -------------------------------------------------------------------------------- /src/consumer/index.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | const { IncomingWebhook } = require("@slack/webhook"); 3 | const config = require("../config"); 4 | const createConsumer = require("./consumer"); 5 | 6 | const kafka = new Kafka(config.kafka); 7 | const slack = new IncomingWebhook(config.slack.webhookUrl); 8 | 9 | const main = async () => { 10 | const consumer = await createConsumer({ kafka, config, slack }); 11 | 12 | const shutdown = async () => { 13 | await consumer.disconnect(); 14 | }; 15 | 16 | return shutdown; 17 | }; 18 | 19 | const signalTraps = ["SIGTERM", "SIGINT", "SIGUSR2"]; 20 | 21 | main() 22 | .then(async (shutdown) => { 23 | signalTraps.forEach((signal) => { 24 | process.on(signal, async () => { 25 | console.log(`Received ${signal} signal. Shutting down.`); 26 | try { 27 | await shutdown(); 28 | process.exit(0); 29 | } catch (error) { 30 | console.error("Error during shutdown", error); 31 | process.exit(1); 32 | } 33 | }); 34 | }); 35 | }) 36 | .catch((error) => { 37 | console.error("Error during startup", error); 38 | process.exit(1); 39 | }); 40 | -------------------------------------------------------------------------------- /src/server/index.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | const config = require("../config"); 3 | const createApp = require("./app"); 4 | 5 | const client = new Kafka(config.kafka); 6 | const producer = client.producer(); 7 | 8 | const main = async () => { 9 | await producer.connect(); 10 | 11 | const app = createApp({ producer, config: config.app }); 12 | 13 | const server = app.listen(config.server.port, (error) => { 14 | if (error != null) { 15 | throw error; 16 | } 17 | 18 | console.log(`Server is listening on port ${config.server.port}`); 19 | }); 20 | 21 | const shutdown = async () => { 22 | await server.close(); 23 | await producer.disconnect(); 24 | }; 25 | 26 | return shutdown; 27 | }; 28 | 29 | const signalTraps = ["SIGTERM", "SIGINT", "SIGUSR2"]; 30 | 31 | main() 32 | .then(async (shutdown) => { 33 | signalTraps.forEach((signal) => { 34 | process.on(signal, async () => { 35 | console.log(`Received ${signal} signal. Shutting down.`); 36 | try { 37 | await shutdown(); 38 | process.exit(0); 39 | } catch (error) { 40 | console.error("Error during shutdown", error); 41 | process.exit(1); 42 | } 43 | }); 44 | }); 45 | }) 46 | .catch((error) => { 47 | console.error("Error during startup", error); 48 | process.exit(1); 49 | }); 50 | -------------------------------------------------------------------------------- /bin/create-topic: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const path = require('path') 4 | const { Kafka } = require('kafkajs') 5 | 6 | const config = { 7 | clientId: 'npm-slack-notifier-admin', 8 | brokers: [process.env.BOOTSTRAP_BROKER || 'localhost:9092'], 9 | ssl: process.env.KAFKA_SSL ? JSON.parse(process.env.KAFKA_SSL) : false, 10 | sasl: process.env.KAFKA_USERNAME && process.env.KAFKA_PASSWORD ? { 11 | username: process.env.KAFKA_USERNAME, 12 | password: process.env.KAFKA_PASSWORD 13 | } : null 14 | } 15 | 16 | const kafka = new Kafka(config) 17 | const admin = kafka.admin() 18 | 19 | const topic = process.argv[2] 20 | const numPartitions = process.argv[3] && !isNaN(process.argv[3]) 21 | ? parseInt(process.argv[3], 10) 22 | : undefined 23 | 24 | const help = () => { 25 | console.log(`${path.basename(__filename)} [numPartitions]`) 26 | console.log() 27 | console.log('Ensure the following environment variables are set:') 28 | console.log(' * BOOTSTRAP_BROKER') 29 | console.log(' * KAFKA_SSL') 30 | console.log(' * KAFKA_USERNAME') 31 | console.log(' * KAFKA_PASSWORD') 32 | } 33 | 34 | if (!topic) { 35 | help() 36 | process.exit(1) 37 | } 38 | 39 | admin.connect().then(() => { 40 | return admin.createTopics({ 41 | topics: [{ 42 | topic, 43 | numPartitions 44 | }] 45 | }) 46 | }).then(() => { 47 | admin.disconnect() 48 | }).then(() => { 49 | process.exit(0) 50 | }) 51 | .catch(error => { 52 | console.error(error) 53 | console.log() 54 | help() 55 | process.exit(1) 56 | }) -------------------------------------------------------------------------------- /bin/send-event: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const http = require('http') 4 | const path = require('path') 5 | const fs = require('fs') 6 | const crypto = require('crypto') 7 | 8 | const url = process.argv[2] 9 | const secret = process.argv[3] 10 | const payloadPath = process.argv[4] 11 | 12 | const help = () => { 13 | console.log(`${path.basename(__filename)} ./path/to/payload.json`) 14 | } 15 | 16 | if (!secret || !payloadPath) { 17 | help() 18 | process.exit(1) 19 | } 20 | 21 | const file = fs.readFileSync(payloadPath) 22 | const signature = crypto.createHmac('sha256', secret).update(file).digest('hex') 23 | const send = async () => { 24 | return new Promise((resolve, reject) => { 25 | const req = http.request(url, { 26 | headers: { 27 | 'Content-Type': 'application/json', 28 | 'x-npm-signature': `sha256=${signature}` 29 | }, 30 | method: 'POST' 31 | }, res => { 32 | if (res.statusCode >= 400) { 33 | reject(new Error(`Server returned statusCode ${res.statusCode}: ${res.statusMessage}`)) 34 | } else { 35 | res.setEncoding('utf-8') 36 | let body = '' 37 | res.on('data', chunk => body += chunk) 38 | res.on('end', () => { 39 | console.log(body) 40 | resolve() 41 | }) 42 | } 43 | }) 44 | 45 | req.on('error', error => { 46 | reject(error) 47 | }) 48 | 49 | req.write(file) 50 | 51 | req.end() 52 | }) 53 | } 54 | 55 | send().then(() => { 56 | process.exit(0) 57 | }).catch(error => { 58 | console.error('Error making request', error) 59 | process.exit(1) 60 | }) 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Demo NPM package release Slack notification 2 | 3 | This is a sample application developed for an article written for Confluent's blog. It is not intended to be production-ready, but rather to serve as a basic example of how to use KafkaJS. 4 | 5 | This application sets up a server that receives NPM webhook events on package publishing and publishes those messages to Kafka. A consumer then consumes those messages and posts them to a Slack channel. 6 | 7 | ![Picture of a Slack message sent from the bot](./static/message.png) 8 | 9 | ## Getting started 10 | 11 | This application requires a reasonably recent version of Node.js (tested on v14). In order to run a local Kafka cluster, Docker is also required. 12 | 13 | ```sh 14 | # Install dependencies 15 | $ npm install 16 | 17 | # Start docker containers 18 | $ docker-compose up -d 19 | 20 | # Run server and consumer 21 | $ npm start 22 | 23 | # If you prefer to run the server and consumer separately 24 | $ npm run start:server 25 | $ npm run start:consumer 26 | ``` 27 | 28 | ## Configuration 29 | 30 | Configuration is done by setting environment variables: 31 | 32 | * `HOOK_SECRET` - Secret used when registering the hook with NPM **REQUIRED** 33 | * `SLACK_WEBHOOK_URL` - [Slack incoming webhook URL](https://api.slack.com/messaging/webhooks) to post messages to **REQUIRED** 34 | * `TOPIC` - The topic to produce to and consume from (default: `npm-package-published`) 35 | * `PORT` - Port for the HTTP server to listen on (default: `3000`) 36 | * `BOOTSTRAP_BROKER` - Initial broker to connect to (default: `localhost:9092`) 37 | * `KAFKA_SSL` - [TLS SecureContext](https://nodejs.org/api/tls.html#tls_tls_connect_options_callback) options. Can be set to `true` to use TLS with default configuration, else a JSON object that gets passed to `JSON.parse`. `false` will connect without encryption (default: `false`) 38 | * `KAFKA_USERNAME` & `KAFKA_PASSWORD` - SASL credentials. Will connect without authentication if left empty (default: `undefined`) 39 | 40 | ### Creating a topic 41 | 42 | When using the provided `docker-compose.yml` file, the topic will be created automatically when producing the first message. If not using topic auto-creation, the script in `bin/create-topic` can be used to create the topic ahead of time: 43 | 44 | ```sh 45 | # Create the topic 'npm-package-published' with 3 partitions 46 | $ ./bin/create-topic "npm-package-published" 3 47 | ``` 48 | 49 | ## Sending a test message 50 | 51 | To see that the application is working, start the server and consumer and then send a test message: 52 | 53 | ```sh 54 | $ ./bin/send-event 55 | send-event ./path/to/payload.json 56 | 57 | $ ./bin/send-event http://localhost:3000/hook "very-secret-string" ./fixtures/payload.json 58 | "OK" 59 | ``` 60 | 61 | ## Registering hook with NPM 62 | 63 | `npm` has the ability to add hooks directly in the CLI. In this example, I'm using [ngrok](https://ngrok.com/) to establish a tunnel to expose my IP on the public internet. 64 | 65 | ```sh 66 | # Setting up the tunnel to forward from my ngrok.io address to port 3000 on my localhost 67 | $ ngrok http 3000 68 | 69 | Session Status online 70 | Account (Plan: Free) 71 | Version 2.3.35 72 | Region Europe (eu) 73 | Web Interface http://127.0.0.1:4040 74 | Forwarding http://4d6d3b9e3ed3.eu.ngrok.io -> http://localhost:3000 75 | Forwarding https://4d6d3b9e3ed3.eu.ngrok.io -> http://localhost:3000 76 | 77 | Connections ttl opn rt1 rt5 p50 p90 78 | 0 0 0.00 0.00 0.00 0.00 79 | 80 | # Registering webhook with NPM 81 | # Note that you need to login first with `npm adduser` 82 | $ npm hook add kafkajs https://4d6d3b9e3ed3.eu.ngrok.io "super-secret-string" 83 | + kafkajs ➜ https://4d6d3b9e3ed3.eu.ngrok.io 84 | 85 | # Check your existing hooks 86 | $ npm hook ls 87 | You have one hook configured. 88 | ┌──────────┬─────────┬──────────────────────────────────┐ 89 | │ id │ target │ endpoint │ 90 | ├──────────┼─────────┼──────────────────────────────────┤ 91 | │ 1hog35p1 │ kafkajs │ https://4d6d3b9e3ed3.eu.ngrok.io │ 92 | │ ├─────────┴──────────────────────────────────┤ 93 | │ │ never triggered │ 94 | └──────────┴────────────────────────────────────────────┘ 95 | 96 | # Delete the hook once you don't need it anymore 97 | $ npm hook rm 1hog35p1 98 | - kafkajs ✘ https://4d6d3b9e3ed3.eu.ngrok.io 99 | ``` --------------------------------------------------------------------------------