├── .gitignore ├── subscriber ├── package.json └── index.js ├── endpoint ├── package.json └── index.js └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /subscriber/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "request": "^2.88.0" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /endpoint/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "@google-cloud/pubsub": "^1.7.0" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /endpoint/index.js: -------------------------------------------------------------------------------- 1 | const {PubSub} = require('@google-cloud/pubsub'); 2 | const topicName = process.env.TOPIC_NAME; 3 | 4 | const pubsub = new PubSub(); 5 | const topic = pubsub.topic(topicName); // .setPublishOptions(...); 6 | 7 | /** 8 | * HTTP function called by webhook and publishes post data to Pub/Sub. 9 | * 10 | * @param {Object} req Cloud Function request context. 11 | * More info: https://expressjs.com/en/api.html#req 12 | * @param {Object} res Cloud Function response context. 13 | * More info: https://expressjs.com/en/api.html#res 14 | */ 15 | exports.endpoint = (req, res) => { 16 | if (req.method != 'POST') { 17 | res.status(405).send('Method not allowed'); 18 | return; 19 | } 20 | 21 | //if (req.ip != '1.2.3.4') { 22 | // res.status(403).send('Forbidden'); 23 | // return; 24 | //} 25 | 26 | //if (req.get('X-Webhook-Token') != credentials) { 27 | // res.status(403).send('Forbidden'); 28 | // return; 29 | //} 30 | 31 | topic.publish(req.rawBody, (err, messageId) => { 32 | if (err) { 33 | console.error(err); 34 | res.status(500).end(); 35 | return; 36 | } 37 | res.status(200).end(); 38 | }); 39 | }; 40 | -------------------------------------------------------------------------------- /subscriber/index.js: -------------------------------------------------------------------------------- 1 | const request = require('request'); 2 | const forwardURL = process.env.FORWARD_URL; 3 | 4 | /** 5 | * Background Cloud Function to be triggered by Pub/Sub. 6 | * This function is exported by index.js, and executed when 7 | * the trigger topic receives a message. 8 | * 9 | * @param {object} pubSubEvent The event payload. 10 | * @param {object} context The event metadata. 11 | */ 12 | exports.subscriber = (pubSubEvent, context) => { 13 | 14 | // https://cloud.google.com/functions/docs/bestpractices/retries#set_an_end_condition_to_avoid_infinite_retry_loops 15 | const age = Date.now() - Date.parse(context.timestamp); 16 | if (age > 60*60*1000) { 17 | return; 18 | } 19 | 20 | // https://cloud.google.com/functions/docs/calling/pubsub#event_structure 21 | let body = Buffer.from(pubSubEvent.data, 'base64').toString('utf-8'); 22 | 23 | // If the endpoint only needs certain event types: 24 | // 25 | // const types = ['dropped', 'bounce', 'delivered', 'spamreport']; 26 | // const events = JSON.parse(body).filter(e => types.includes(e.event)); 27 | // if (events.length == 0) { 28 | // return; 29 | // } 30 | // body = JSON.stringify(events); 31 | 32 | const options = { 33 | // https://cloud.google.com/functions/docs/env-var#accessing_environment_variables_at_runtime 34 | url: forwardURL, 35 | headers: { 36 | 'Content-Type': 'application/json; charset=utf-8' 37 | //'Authorization': 'Basic ' + Buffer.from('username:password').toString('base64') 38 | }, 39 | body: body, 40 | }; 41 | 42 | req = request.post(options, (err, res, body) => { 43 | if (err) { 44 | console.error(err); 45 | throw new Error(err); 46 | } 47 | if (res.statusCode != 200) { 48 | console.error(body); 49 | throw new Error(res.StatusMessage); 50 | } 51 | }); 52 | }; 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Webhook Forwarder 2 | 3 | Does your webhook provider support one endpoint, and you need multiple? Use this template to deploy a webhook forwarder on Google Cloud Platform (GCP) with minimal effort. 4 | 5 | The template leverages [Google Cloud Functions](https://cloud.google.com/functions/docs/concepts/overview) and [Google Cloud Pub/Sub](https://cloud.google.com/pubsub/docs/overview) to provide scalability, reliability, and separation with minimal code. 6 | 7 | The endpoint function receives HTTPS POST requests from a webhook and publishes the messages to a Pub/Sub topic. The subscriber function is subscribed to the Pub/Sub topic and forwards the messages to a specific URL. Multiple subscriber functions with different URLs can be deployed, which all get the same messages. 8 | 9 | Through Pub/Sub the endpoint is decoupled from the subscribers. Incoming messages are immediately acknowledged as soon as the message is saved in the queue. If forwarding to a receiver fails, the message remains in the queue and the subscriber function is retried. 10 | 11 | ## Deployment 12 | 13 | ### Create cloud project 14 | 15 | Login to the [Google Cloud Console](https://console.cloud.google.com/). 16 | 17 | Create a new project. The project ID will be part of the endpoint URL. Make sure billing is enabled for the project. 18 | 19 | Navigate to [Cloud Functions](https://console.cloud.google.com/functions). 20 | 21 | Enable the [Cloud Functions API](https://console.cloud.google.com/flows/enableapi?apiid=cloudfunctions) if needed. 22 | 23 | ### Deploy endpoint 24 | 25 | Review the source in endpoint/index.js. Add an authentication mechanism if needed. If the webhook provider does not support authentication use a name for the endpoint that is difficult to guess. 26 | 27 | The function code can be changed after deployment using the code editor in the console. 28 | 29 | The Pub/Sub topic and subscriptions will be automatically created. This can be checked in the Pub/Sub section of the console after some data is received. You should see the topic and a subscription for each subscriber function. Refresh the page if you don't. 30 | 31 | #### Using console 32 | 33 | Click *Create Function*. Enable required APIs when asked. 34 | 35 | Enter the following details: 36 | 37 | * Basics 38 | * Environment: 1st gen 39 | * Function name: Enter "sendgrid-endpoint" or another name 40 | * Region: Select a region close to the webhook provider 41 | * Trigger 42 | * Trigger type: Select HTTP 43 | * URL: Note the URL, which is needed by the webhook provider 44 | * Authentication: 45 | * Check Allow unauthenticated invocations 46 | * Check Require HTTPS 47 | * Click Save 48 | * Runtime, build, connections and security settings 49 | * RUNTIME 50 | * Runtime environment variables 51 | * Click Add variable 52 | * Add variable "TOPIC_NAME" with name for Pub/Sub topic, e.g. "sendgrid-events" 53 | * Click Next 54 | * Code 55 | * Runtime: Select Node.js 10 56 | * Source Code: Inline Editor 57 | * index.js: Copy the code from endpoint/index.js 58 | * package.json: Copy the code from endpoint/package.json 59 | * Entry point: Enter "endpoint" 60 | * Click Deploy 61 | 62 | #### Using gcloud 63 | 64 | Make sure the default project is properly set or add --project to the glcloud commands below. 65 | 66 | cd endpoint 67 | gcloud functions deploy endpoint --runtime nodejs10 --trigger-http --allow-unauthenticated 68 | 69 | ### Deploy subscribers 70 | 71 | Review the source in subscriber/index.js. The forwarding URL is set using an environment variable. Authentication may be added as needed. 72 | 73 | For multiple subscribers, just add more subscriber functions. Use a different function name and change the forwarding URL. Other options, including the Pub/Sub topic should be the same. 74 | 75 | #### Using console 76 | 77 | Click *Create function*. 78 | 79 | Enter the following details: 80 | 81 | * Basics 82 | * Function name: Enter "postmastery-webhook" or another name 83 | * Region: Select a region close to the endpoint provider 84 | * Trigger 85 | * Trigger type: Select Cloud Pub/Sub 86 | * Select a topic: Create a new topic, e.g. "sendgrid-events" or select it when already created 87 | * Retry on failure: Yes 88 | * Click Save 89 | * Runtime, build, connections and security settings 90 | * RUNTIME 91 | * Runtime environment variables 92 | * Click Add variable 93 | * Add variable "FORWARD_URL" with the destination URL 94 | * Click Next 95 | * Code 96 | * Runtime: Select Node.js 10 97 | * Source Code: Inline Editor 98 | * index.js: Copy the code from subscriber/index.js 99 | * package.json: Copy the code from subscriber/package.json 100 | * Entry point: Enter "subscriber" 101 | * Click Deploy 102 | 103 | To create another subscriber, in the list of functions select Actions on the first subscriber and click Copy. Then change the function name and optionally the region. Click Save to accept the Trigger settings. Open Runtime environment variables and set the FORWARD_URL to the destination URL. 104 | 105 | #### Using gcloud 106 | 107 | Make sure the default project is properly set or add --project to the glcloud commands below. 108 | 109 | cd subscriber 110 | gcloud functions deploy analytics --runtime nodejs10 --entry-point=subscriber --set-env-vars FORWARD_URL=https://path/to/endpoint --trigger-topic=webhook --retry 111 | 112 | ## Testing 113 | 114 | Use cURL to submit a test message. Use the endpoint URL shown in the function properties. Below is an example of a Sendgrid webhook request: 115 | 116 | curl -X POST -i -H "Content-Type: application/json" -d '[{"email":"john.doe@sendgrid.com","timestamp":1588777534,"smtp-id":"<4FB4041F.6080505@sendgrid.com>","event":"processed"},{"email":"john.doe@sendgrid.com","timestamp":1588777600,"category":"newuser","event":"click","url":"https://sendgrid.com"},{"email":"john.doe@sendgrid.com","timestamp":1588777692,"smtp-id":"<20120525181309.C1A9B40405B3@Example-Mac.local>","event":"processed"}]' https://us-central1-my-project-id.cloudfunctions.net/sendgrid-endpoint 117 | 118 | You can check the logs of each function in the console. 119 | --------------------------------------------------------------------------------