├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ ├── go.yml │ └── release.yml ├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.md ├── README.md ├── ROADMAP.md ├── SECURITY.md ├── config-example.json ├── example-project ├── api │ ├── Dockerfile │ ├── app.py │ └── requirements.txt ├── config.json └── docker-compose.yaml └── sendhooks ├── .dockerignore ├── .env.example ├── Dockerfile ├── adapter ├── adapter.go ├── adapter_manager │ └── adapter_manager.go └── redis_adapter │ └── redis_adapter.go ├── go.mod ├── go.sum ├── logging └── logging.go ├── main.go ├── queue └── worker.go ├── sender ├── sender_test.go ├── utils.go ├── webhook.go └── webhook_test.go └── utils ├── redis_tls_config.go ├── redis_tls_config_test.go └── tests └── certificates.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Summary 11 | Briefly describe the feature you are proposing. Summarize the key points of your request. 12 | 13 | ## Motivation 14 | Explain why this feature is necessary or beneficial to the project. Describe the problem you are trying to solve or the enhancement you want to make. 15 | 16 | ## Proposed Solution (if any) 17 | If you have a proposed solution or ideas on how to implement this feature, outline them here. This could include technical details, sketches, or code snippets. 18 | 19 | ## Alternatives Considered 20 | Describe any alternative solutions or features you have considered. Why is your proposed solution preferable? 21 | 22 | ## Additional Context (Optiona) 23 | Add any other context or screenshots about the feature request here. This could include use cases or examples from other projects that illustrate your point. 24 | 25 | ## Potential Impact (Optiona) 26 | Discuss the potential impact of this feature on the existing project. This includes any possible side effects or changes in current behavior. 27 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | jobs: 13 | 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v3 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v4 21 | with: 22 | go-version: '1.20' 23 | 24 | - name: Build 25 | run: cd sendhooks && go build -o sendhooks . 26 | 27 | - name: Test 28 | run: cd sendhooks && go test -v ./... 29 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Build and Release 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | 8 | jobs: 9 | 10 | build-n-release: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | 15 | - name: Set up Go 16 | uses: actions/setup-go@v4 17 | with: 18 | go-version: '1.20' 19 | 20 | - name: Build 21 | run: cd sendhooks && go build -o sendhooks . 22 | 23 | - name: Test 24 | run: cd sendhooks && go test -v ./... 25 | 26 | - name: Upload Release Asset 27 | id: upload-release-asset 28 | uses: actions/upload-release-asset@v1 29 | with: 30 | upload_url: https://uploads.github.com/repos/${{ github.repository }}/releases/${{ github.event.release.id }}/assets?name=sendhooks 31 | asset_path: ./sendhooks/sendhooks 32 | asset_name: sendhooks 33 | asset_content_type: application/octet-stream 34 | env: 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | 37 | build-n-release-docker: 38 | name: Push Docker image to Docker Hub 39 | runs-on: ubuntu-latest 40 | steps: 41 | - name: Set up QEMU 42 | uses: docker/setup-qemu-action@v3 43 | 44 | - name: Checkout 45 | uses: actions/checkout@v4 46 | 47 | - name: Set up Docker Buildx 48 | uses: docker/setup-buildx-action@v3 49 | with: 50 | install: true 51 | 52 | - name: Login to Docker Hub 53 | uses: docker/login-action@v3 54 | with: 55 | username: ${{ secrets.DOCKERHUB_USERNAME }} 56 | password: ${{ secrets.DOCKERHUB_TOKEN }} 57 | 58 | # Extract metadata (tags, labels) for Docker 59 | # https://github.com/docker/metadata-action 60 | - name: Extract Docker metadata 61 | id: meta 62 | uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 63 | with: 64 | images: transfa/sendhooks 65 | 66 | - name: Build and push Docker image 67 | uses: docker/build-push-action@v5 68 | with: 69 | context: ./sendhooks 70 | file: ./sendhooks/Dockerfile 71 | push: true 72 | tags: ${{ steps.meta.outputs.tags }} 73 | labels: ${{ steps.meta.outputs.labels }} 74 | cache-from: type=gha 75 | cache-to: type=gha,mode=max 76 | platforms: | 77 | linux/amd64 78 | linux/arm64 79 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Go 2 | *.exe 3 | *.exe~ 4 | *.test 5 | *.out 6 | *.o 7 | *.a 8 | *.so 9 | *~ 10 | # Compiled Object files 11 | *.S 12 | *.rdb 13 | 14 | # Log files 15 | *.log 16 | 17 | # Dependency directories 18 | vendor/ 19 | Godeps/ 20 | 21 | # IDE and editor files 22 | .vscode/ 23 | .idea/ 24 | .fleet/ 25 | 26 | # binary files 27 | webhook 28 | 29 | # sendhooks api 30 | node_modules/ 31 | dist/ 32 | *.rdb 33 | 34 | # sendhooks settings 35 | config.json 36 | 37 | #sendhooks-engine-api 38 | api/.env 39 | 40 | .DS_Store 41 | 42 | sendhooks 43 | sendhooks/config.json -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](http://keepachangelog.com/) 6 | and this project adheres to [Semantic Versioning](http://semver.org/). 7 | 8 | ## [Unreleased] - yyyy-mm-dd 9 | 10 | ### Added 11 | 12 | - Create Adapter for Redis Integration [#76](https://github.com/Transfa/sendhooks-engine/issues/76) 13 | - Add Data Size and Number of Tries to Payload Sent to Redis Status Stream [#75](https://github.com/Transfa/sendhooks-engine/issues/75) 14 | - Add Configuration Parameters for Number of Workers and Channel Size [#79](https://github.com/Transfa/sendhooks-engine/issues/79) 15 | 16 | ### Fixed 17 | 18 | ## [v0.3.3-beta] - 2024-06-01 19 | 20 | - Wrong status sent in case of errors (#71) 21 | - Configure Automatic Push to Docker Registry using GitHub Actions (#49) 22 | 23 | ### Added 24 | 25 | ### Fixed 26 | 27 | 28 | ## [0.3.2-beta] - 2024-05-29 29 | 30 | - Fix Delivery Error message (#68) 31 | - Race condition on log file writing (#67) 32 | 33 | ### Added 34 | 35 | ### Fixed 36 | 37 | ## [0.3.1-beta] - 2024-05-24 38 | 39 | - Convert WebhookDeliveryStatus to JSON String for Redis Stream (#65) 40 | - Rename webhook directory and binary to sendhooks (#62) 41 | 42 | ### Added 43 | 44 | ### Fixed 45 | 46 | 47 | ## [0.3.0-beta] - 2024-02-06 48 | 49 | ### Added 50 | 51 | - Remove the sendhooks image from the API (#58) [migrate to https://github.com/Transfa/sendhooks-monitoring] 52 | - Persisting data about the webhooks (#42) 53 | - Integration of Logrus for Improved Logging (#55) 54 | - Send response details in case of webhook failed (#39) 55 | 56 | ## [0.2.0-beta] - 2023-11-23 57 | 58 | ### Added 59 | 60 | - Using JSON for configuration instead of environment variables (#44) 61 | - Migrate to Redis streams (#45) 62 | 63 | ### Fixed 64 | 65 | ## [0.1.0] - 2023-10-31 66 | 67 | ### Added 68 | 69 | - Make Header Name Dynamic through Environment Variable (#40) 70 | - Implement Redis Channel for Webhook Delivery Status Updates (#11) 71 | - Add a .env.example file (#36) 72 | - Add Conditional SSL Support for Redis Connection (#21) 73 | - Adding a CONTRIBUTING.md file (#27) 74 | - Adding a SECURITY.md file( #29) 75 | 76 | 77 | ## [0.0.1] - 2023-09-10 78 | 79 | ### Added 80 | 81 | - Dockerize Project for Distribution on DockerHub (#8) 82 | - Add tests suite for the Webhook sender package (#19) 83 | - Refactor ProcessWebhooks for clarity, safety, and testability (#3 ) 84 | - Implement Payload Signing with webhook-signature Header in SendWebhook (#6) 85 | - Implement File Logging Module for Enhanced Error Handling (#7) 86 | - Refactor Subscribe function in the redis package for better error handling and modularity( #4 ) 87 | - Improve Payload Flexibility to Accept Various JSON Formats( #1 ) 88 | - Refactor SendWebhook function in the sender package for enhanced modularity, error handling, and logging ( #5 ) 89 | 90 | ### Fixed 91 | 92 | - Fix calculateBackoff redaclarring function (#16 ) 93 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | . 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to SendHooks Engine 2 | 3 | Thank you for considering contributing to SendHooks Engine! We appreciate your help in improving and maintaining this project. 4 | 5 | ## Table of Contents 6 | 7 | - [Introduction](#introduction) 8 | - [Development Environment Setup](#development-environment-setup) 9 | - [Submitting Pull Requests](#submitting-pull-requests) 10 | - [Reporting Bugs or Proposing Features](#reporting-bugs-or-proposing-features) 11 | - [Running Tests](#running-tests) 12 | - [Project-Specific Guidelines and Expectations](#project-specific-guidelines-and-expectations) 13 | 14 | ## Introduction 15 | 16 | This guide outlines the process for contributing to SendHooks Engine. We encourage contributions from the community to make our project better. 17 | 18 | ## Development Environment Setup 19 | 20 | Before you start contributing, make sure you have set up your development environment: 21 | 22 | ```bash 23 | git clone https://github.com/Transfa/sendhooks-engine.git && cd sendhooks-engine/sendhooks 24 | 25 | go mod download 26 | ``` 27 | 28 | At the moment, we are working with Go 1.20. 29 | 30 | ## Submitting Pull Requests 31 | 32 | If you want to contribute code changes or bug fixes, please follow these steps: 33 | 34 | 1. Fork the SendHooks Engine repository. 35 | 2. Create a new branch for your changes: `git checkout -b feature/your-feature-name`. 36 | 3. Make your changes and commit them. 37 | 4. Push your changes to your forked repository. 38 | 5. Submit a pull request to the `main` branch of the SendHooks Engine repository. 39 | 40 | ## Reporting Bugs or Proposing Features 41 | 42 | If you find a bug or want to propose a new feature, please: 43 | 44 | 1. Check if the issue already exists in our [issue tracker](https://github.com/Transfa/sendhooks-engine/issues). 45 | 2. If not, create a new issue and provide detailed information. 46 | 47 | ## Docker 48 | 49 | If you prefer using Docker for development, you can set up the project as follows: 50 | 51 | 1. **Pull the Docker Image**: 52 | 53 | ```bash 54 | cd webhooks 55 | 56 | docker build . -t sendhooks 57 | ``` 58 | 59 | Then run the container. 60 | 61 | ```bash 62 | docker run -t sendhooks --env REDIS_ADDRESS= sendhooks 63 | ``` 64 | 65 | ## Running Tests 66 | 67 | To ensure the reliability and correctness of your code changes, it's important to run tests before submitting your pull request. 68 | 69 | ```bash 70 | go test -v ./... 71 | ``` 72 | 73 | ## Project-Specific Guidelines and Expectations 74 | 75 | For any project-specific guidelines or expectations, please refer to our [README](README.md). 76 | 77 | We appreciate your contributions and look forward to working with you to improve SendHooks Engine! 78 | 79 | --- 80 | **Note**: This contributing guide is subject to updates and changes. We encourage you to check this document periodically for any updates. 81 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [2023] [Kolawole Mangabo] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SendHooks Engine 2 | ![GitHub Tag](https://img.shields.io/github/v/tag/transfa/sendhooks-engine) 3 | [![Build and Release](https://github.com/Transfa/sendhooks-engine/actions/workflows/release.yml/badge.svg)](https://github.com/Transfa/sendhooks-engine/actions/workflows/release.yml) 4 | [![Go](https://github.com/Transfa/sendhooks-engine/actions/workflows/go.yml/badge.svg)](https://github.com/Transfa/sendhooks-engine/actions/workflows/go.yml) 5 | [![CodeQL](https://github.com/Transfa/sendhooks-engine/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/Transfa/sendhooks-engine/actions/workflows/github-code-scanning/codeql) 6 | 7 | > ⚠️ **Warning**: This project is not stable yet. Feel free to join the Discord so we can build together [https://discord.gg/w2fBSz3j](https://discord.gg/2mHxEgxm5r) 8 | 9 | ## Introduction 10 | SendHooks Engine is a high-performance, scalable tool for managing and delivering webhooks. It uses modern architectural patterns to efficiently handle webhook processing, making it an ideal choice for applications requiring reliable and quick webhook delivery. Written in Go, SendHooks takes full advantage of the language's concurrency and performance features. 11 | 12 | ## Key Features 13 | - **High Performance**: Utilizes Golang's efficient concurrency model for fast webhook processing. 14 | - **Scalable Architecture**: Designed to handle varying loads with a queue-based approach. 15 | - **Secure**: Supports SSL/TLS for Redis connections and authenticates webhook messages for security. 16 | 17 | ## Installation 18 | 19 | ### Using Docker 20 | To run SendHooks using Docker, a `config.json` file is required. This file contains necessary configurations including the Redis server address. 21 | 22 | 1. **Pull the Docker Image**: 23 | ```bash 24 | docker pull transfa/sendhooks:latest 25 | ``` 26 | 27 | 2. **Run the Docker Image with `config.json`**: 28 | ```bash 29 | docker run -v /path/to/config.json:/app/config.json -t transfa/sendhooks 30 | ``` 31 | Replace `/path/to/config.json` with the path to your configuration file. 32 | 33 | ### Using the Compiled Binary (macOS and Linux) 34 | 35 | Ensure that the `config.json` file is located in the same directory as the binary. 36 | 37 | 1. **Download and Run the Binary**: 38 | ```bash 39 | curl -LO https://github.com/Transfa/sendhooks-engine/releases/latest/download/sendhooks 40 | chmod +x sendhooks 41 | ./sendhooks 42 | ``` 43 | 44 | ## Components 45 | - **Redis Client**: Interacts with Redis streams to manage incoming webhook messages. 46 | - **Queue**: Buffers messages ensuring smooth flow and handling of sudden influxes. 47 | - **HTTP Client**: Processes each message, sending it as an HTTP POST request to the intended URL. 48 | 49 | ## Concurrency Handling 50 | - Utilizes Go routines for simultaneous operations, enhancing efficiency and responsiveness. 51 | - Employs a buffered channel (queue) for concurrent message handling. 52 | 53 | ## Security 54 | - Configurable for SSL/TLS encryption with Redis, ensuring secure message transmission. 55 | - Includes a secret hash in HTTP headers for message authenticity verification. 56 | 57 | ## Contributors 58 | We welcome contributions from the community. If you'd like to contribute, please check out our [list of issues](https://github.com/Transfa/sendhooks-engine/issues) to see how you can help. 59 | 60 | You can find how to contribute in the [CONTRIBUTING](CONTRIBUTING.md) file. 61 | 62 | ## Security 63 | 64 | You can find our security policies on [SECURITY](SECURITY.md). 65 | -------------------------------------------------------------------------------- /ROADMAP.md: -------------------------------------------------------------------------------- 1 | # sendhooks-engine 2 | 3 | ## Roadmap 4 | 5 | ## 1. Core Functionality 6 | 7 | ### Data Integration 8 | 9 | - [X] Integration with Redis for data input. 10 | - [X] Set up a Redis channel for ingesting the data. 11 | 12 | ### Sending Data 13 | 14 | - [X] HTTP client setup to send data to the endpoint. 15 | - [X] Header configuration for the HTTP client. 16 | 17 | ### Header Signing 18 | 19 | - [X] Algorithm selection for header signing (e.g., HMAC, RSA). 20 | - [X] Implementation of the signing process using a secret key. 21 | 22 | ## 2. Features and Enhancements 23 | 24 | ### Exponential Backoff 25 | 26 | - [X] Implement an exponential backoff mechanism. 27 | - [X] Tests to validate exponential backoff behavior. 28 | 29 | ### Queuing 30 | 31 | - [X] Implement or integrate a queuing mechanism. 32 | 33 | ### Retry Mechanism 34 | 35 | - [X] Implement a retry mechanism upon failure. 36 | - [X] Define max retry count and intervals. 37 | 38 | ### Logging 39 | 40 | - [X] Logger setup for the project. 41 | - [X] Implement file-based persistent logging. 42 | 43 | ## 3. Additional Considerations 44 | 45 | ### Security 46 | 47 | - [X] Implementation of password security or other authentication methods for Redis. 48 | 49 | ### Scalability 50 | 51 | - [X] Strategy for handling high concurrency using goroutines and worker pools. 52 | 53 | ## Documentation and Community Building 54 | 55 | - [X] Comprehensive documentation for setup, features, and usage. 56 | - [X] Contribution guidelines for the community. 57 | - [X] Discord community 58 | 59 | ## Testing and Release 60 | 61 | - [X] Comprehensive test coverage for all features. 62 | - [X] Continuous Integration (CI) setup. 63 | - [X] First major release with all the initial features. 64 | - [X] Distribution in Dockerhub 65 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy for Sendhooks 2 | 3 | ## Reporting a Security Vulnerability 4 | 5 | At Sendhooks, we take security seriously and welcome the responsible disclosure of any security vulnerabilities. If you have discovered a security issue, we appreciate your cooperation in disclosing it to us in a responsible manner. 6 | 7 | To report a security vulnerability, please follow these steps: 8 | 9 | 1. Send an email to [kolawole.mangabo@transfapp.com](mailto:kolawole.mangabo@transfapp.com) with the subject line: **[Sendhooks Security Vulnerability]**. 10 | 11 | 2. Provide a detailed description of the vulnerability, including: 12 | - A clear and concise summary of the issue. 13 | - Steps to reproduce the vulnerability. 14 | - Information about the affected versions of Sendhooks, if applicable. 15 | 16 | 17 | ## Expectations 18 | 19 | Upon receiving your report, we will acknowledge your email within 48 hours to confirm that we have received your submission. We will then work to validate and address the reported vulnerability. 20 | 21 | We kindly request that you do not publicly disclose the vulnerability until we have had a chance to review and address it. 22 | 23 | ## Disclosure Policy 24 | 25 | We are committed to coordinating the disclosure of security vulnerabilities responsibly, and we aim to provide timely updates and resolutions to security issues. 26 | 27 | ## Acknowledgment and Reward 28 | 29 | To encourage responsible disclosure, Sendhooks may acknowledge and provide rewards to individuals or organizations that report security vulnerabilities, subject to our discretion. Rewards may be monetary or non-monetary, depending on the severity and impact of the vulnerability. 30 | 31 | ## Additional Information 32 | 33 | Thank you for helping us keep Sendhooks secure. Your efforts in reporting security vulnerabilities are greatly appreciated, and they contribute to the safety and trust of our community. 34 | 35 | This security policy is subject to updates and changes. We encourage you to check this document periodically for any updates. 36 | 37 | Last Updated: 04/06/2024 38 | -------------------------------------------------------------------------------- /config-example.json: -------------------------------------------------------------------------------- 1 | { 2 | "Redis": { 3 | "redisAddress": "127.0.0.1:6379", 4 | "redisPassword": "your_password_here", 5 | "redisDb": "0", 6 | "redisSsl": "false", 7 | "redisCaCert": "/path/to/ca_cert.pem", 8 | "redisClientCert": "/path/to/client_cert.pem", 9 | "redisClientKey": "/path/to/client_key.pem", 10 | "redisStreamName": "example_stream", 11 | "redisStreamStatusName": "status_stream" 12 | }, 13 | "SecretHashHeaderName": "dump_value", 14 | "Broker": "redis", 15 | "NumWorkers": 1, 16 | "ChannelSize": 1 17 | } 18 | -------------------------------------------------------------------------------- /example-project/api/Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile.flask 2 | FROM python:3.11-slim 3 | 4 | WORKDIR /app 5 | COPY . . 6 | 7 | RUN pip install --no-cache-dir -r requirements.txt 8 | 9 | CMD ["python", "app.py"] 10 | -------------------------------------------------------------------------------- /example-project/api/app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request, jsonify 2 | import redis 3 | import json 4 | 5 | app = Flask(__name__) 6 | r = redis.Redis(host='redis', port=6379, db=0) 7 | 8 | 9 | @app.route('/api/send', methods=['POST']) 10 | def send_data(): 11 | payload = request.json 12 | # Use xadd to add the message to the Redis Stream named 'hooks' 13 | r.xadd('hooks', {'data': json.dumps(payload)}) 14 | return jsonify({"status": "sent to stream"}) 15 | 16 | 17 | if __name__ == '__main__': 18 | app.run(debug=True, host='0.0.0.0') 19 | -------------------------------------------------------------------------------- /example-project/api/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask 2 | redis 3 | -------------------------------------------------------------------------------- /example-project/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "redisAddress": "redis:6379", 3 | "redisPassword": "", 4 | "redisDb": "0", 5 | "redisSsl": "false", 6 | "redisStreamName": "hooks", 7 | "redisStreamStatusName": "hooks-status" 8 | } 9 | -------------------------------------------------------------------------------- /example-project/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.9' 2 | 3 | services: 4 | redis: 5 | image: redis:latest 6 | hostname: redis 7 | networks: 8 | - sendhooks 9 | ports: 10 | - "6379:6379" # Exposes Redis on localhost via port 6379 11 | 12 | webhook-sender: 13 | build: ../sendhooks 14 | networks: 15 | - sendhooks 16 | depends_on: 17 | - redis 18 | volumes: 19 | - ./config.json:/root/config.json # Mounts config.json from host to container 20 | 21 | flask-api: 22 | build: ./api/ 23 | networks: 24 | - sendhooks 25 | ports: 26 | - "5001:5000" # Exposes Flask API on localhost via port 5001, internal port 5000 27 | depends_on: 28 | - webhook-sender 29 | 30 | networks: 31 | sendhooks: 32 | driver: bridge -------------------------------------------------------------------------------- /sendhooks/.dockerignore: -------------------------------------------------------------------------------- 1 | .editorconfig 2 | .git 3 | AUTHORS.md 4 | CONTRIBUTING.md 5 | LICENSE 6 | Makefile 7 | NOTICE 8 | README.md 9 | logs -------------------------------------------------------------------------------- /sendhooks/.env.example: -------------------------------------------------------------------------------- 1 | # Redis Configuration 2 | REDIS_ADDRESS=localhost:6379 3 | REDIS_DB=0 4 | REDIS_PASSWORD=your_password 5 | REDIS_SSL=false 6 | REDIS_CA_CERT=path_to_ca_cert 7 | REDIS_CLIENT_CERT=path_to_client_cert 8 | REDIS_CLIENT_KEY=path_to_client_key 9 | REDIS_CHANNEL_NAME=hooks 10 | -------------------------------------------------------------------------------- /sendhooks/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use the official Go image from the DockerHub 2 | FROM golang:1.20 as builder 3 | 4 | # We set the Current Working Directory inside the container 5 | WORKDIR /app 6 | 7 | # We copy everything from the current directory to the Working Directory inside the container 8 | COPY . . 9 | 10 | # Build the Go app 11 | RUN CGO_ENABLED=0 GOOS=linux go build -o sendhooks . 12 | 13 | #### Start a new stage from scratch #### 14 | FROM alpine:latest 15 | 16 | # Install CA certificates as we will need it when adding the Redis SSL configuration 17 | RUN apk --no-cache add ca-certificates 18 | 19 | WORKDIR /root/ 20 | 21 | # Copy the Pre-built binary file from the previous stage 22 | COPY --from=builder /app/sendhooks . 23 | 24 | # Command to run the executable 25 | CMD ["./sendhooks"] 26 | -------------------------------------------------------------------------------- /sendhooks/adapter/adapter.go: -------------------------------------------------------------------------------- 1 | package adapter 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | // WebhookPayload represents the structure of the data from Redis. 8 | type WebhookPayload struct { 9 | URL string `json:"url"` 10 | WebhookID string `json:"webhookId"` 11 | MessageID string `json:"messageId"` 12 | Data map[string]interface{} `json:"data"` 13 | SecretHash string `json:"secretHash"` 14 | MetaData map[string]interface{} `json:"metaData"` 15 | } 16 | 17 | type WebhookDeliveryStatus struct { 18 | WebhookID string `json:"webhookId"` 19 | Status string `json:"status"` 20 | DeliveryError string `json:"deliveryError"` 21 | URL string `json:"url"` 22 | Created string `json:"created"` 23 | PayloadSize int `json:"payloadSize"` 24 | NumberOfTries int `json:"numberOfTries"` 25 | Delivered string `json:"delivered"` 26 | } 27 | 28 | type RedisConfig struct { 29 | RedisAddress string `json:"redisAddress"` 30 | RedisPassword string `json:"redisPassword"` 31 | RedisDb string `json:"redisDb"` 32 | RedisSsl string `json:"redisSsl"` 33 | RedisCaCert string `json:"redisCaCert"` 34 | RedisClientCert string `json:"redisClientCert"` 35 | RedisClientKey string `json:"redisClientKey"` 36 | RedisStreamName string `json:"redisStreamName"` 37 | RedisStreamStatusName string `json:"redisStreamStatusName"` 38 | } 39 | 40 | type Configuration struct { 41 | Redis RedisConfig `json:"redis"` 42 | SecretHashHeaderName string `json:"secretHashHeaderName"` 43 | Broker string `json:"broker"` 44 | NumWorkers int `json:"numWorkers"` 45 | ChannelSize int `json:"channelSize"` 46 | } 47 | 48 | // Adapter defines methods for interacting with different queue systems. 49 | type Adapter interface { 50 | Connect() error 51 | SubscribeToQueue(ctx context.Context, queue chan<- WebhookPayload) error 52 | ProcessWebhooks(ctx context.Context, queue chan WebhookPayload, queueAdapter Adapter) 53 | PublishStatus(ctx context.Context, webhookID, url, created, delivered, status, deliveryError string, payloadSize int, numberOfTries int) error 54 | } 55 | -------------------------------------------------------------------------------- /sendhooks/adapter/adapter_manager/adapter_manager.go: -------------------------------------------------------------------------------- 1 | package adapter_manager 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "os" 7 | "sendhooks/adapter" 8 | "sync" 9 | 10 | redisadapter "sendhooks/adapter/redis_adapter" 11 | ) 12 | 13 | var ( 14 | instance adapter.Adapter 15 | once sync.Once 16 | ) 17 | 18 | var ( 19 | config adapter.Configuration 20 | ) 21 | 22 | // LoadConfiguration loads the configuration from a file 23 | func LoadConfiguration(filename string) { 24 | once.Do(func() { 25 | file, err := os.Open(filename) 26 | if err != nil { 27 | log.Fatalf("Failed to open config file: %v", err) 28 | } 29 | defer file.Close() 30 | 31 | decoder := json.NewDecoder(file) 32 | err = decoder.Decode(&config) 33 | if err != nil { 34 | log.Fatalf("Failed to decode config file: %v", err) 35 | } 36 | }) 37 | } 38 | 39 | // GetConfig returns the loaded configuration 40 | func GetConfig() adapter.Configuration { 41 | return config 42 | } 43 | 44 | // Initialize initializes the appropriate adapter based on the configuration. 45 | func Initialize() { 46 | once.Do(func() { 47 | conf := GetConfig() 48 | switch conf.Broker { 49 | case "redis": 50 | instance = redisadapter.NewRedisAdapter(conf) 51 | default: 52 | log.Fatalf("Unsupported broker type: %v", conf.Broker) 53 | } 54 | 55 | err := instance.Connect() 56 | if err != nil { 57 | log.Fatalf("Failed to connect to broker: %v", err) 58 | } 59 | }) 60 | } 61 | 62 | // GetAdapter returns the singleton instance of the adapter. 63 | func GetAdapter() adapter.Adapter { 64 | return instance 65 | } 66 | -------------------------------------------------------------------------------- /sendhooks/adapter/redis_adapter/redis_adapter.go: -------------------------------------------------------------------------------- 1 | package redisadapter 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "encoding/json" 7 | "fmt" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "sendhooks/adapter" 13 | "sendhooks/logging" 14 | worker "sendhooks/queue" 15 | "sendhooks/utils" 16 | 17 | "github.com/go-redis/redis/v8" 18 | ) 19 | 20 | // RedisAdapter implements the Adapter interface for Redis. 21 | type RedisAdapter struct { 22 | client *redis.Client 23 | config adapter.Configuration 24 | queueName string 25 | statusQueue string 26 | lastID string 27 | } 28 | 29 | // NewRedisAdapter creates a new RedisAdapter instance. 30 | func NewRedisAdapter(config adapter.Configuration) *RedisAdapter { 31 | return &RedisAdapter{ 32 | config: config, 33 | queueName: config.Redis.RedisStreamName, 34 | statusQueue: config.Redis.RedisStreamStatusName, 35 | lastID: "0", 36 | } 37 | } 38 | 39 | // Connect initializes the Redis client and establishes a connection. 40 | func (r *RedisAdapter) Connect() error { 41 | redisAddress := r.config.Redis.RedisAddress 42 | if redisAddress == "" { 43 | redisAddress = "localhost:6379" // Default address 44 | } 45 | 46 | redisDB := r.config.Redis.RedisDb 47 | if redisDB == "" { 48 | redisDB = "0" // Default database 49 | } 50 | 51 | redisDBInt, _ := strconv.Atoi(redisDB) 52 | redisPassword := r.config.Redis.RedisPassword 53 | 54 | useSSL := strings.ToLower(r.config.Redis.RedisSsl) == "true" 55 | var tlsConfig *tls.Config 56 | 57 | if useSSL { 58 | caCertPath := r.config.Redis.RedisCaCert 59 | clientCertPath := r.config.Redis.RedisClientCert 60 | clientKeyPath := r.config.Redis.RedisClientKey 61 | 62 | var err error 63 | tlsConfig, err = utils.CreateTLSConfig(caCertPath, clientCertPath, clientKeyPath) 64 | if err != nil { 65 | return err 66 | } 67 | } 68 | 69 | r.client = redis.NewClient(&redis.Options{ 70 | Addr: redisAddress, 71 | Password: redisPassword, 72 | DB: redisDBInt, 73 | TLSConfig: tlsConfig, 74 | PoolSize: r.config.NumWorkers, 75 | }) 76 | 77 | return nil 78 | } 79 | 80 | // SubscribeToQueue subscribes to the specified Redis queue and processes messages. 81 | func (r *RedisAdapter) SubscribeToQueue(ctx context.Context, queue chan<- adapter.WebhookPayload) error { 82 | for { 83 | select { 84 | case <-ctx.Done(): 85 | return ctx.Err() 86 | default: 87 | if err := r.processQueueMessages(ctx, queue); err != nil { 88 | return err 89 | } 90 | } 91 | } 92 | } 93 | 94 | // processQueueMessages retrieves, decodes, and dispatches messages from the Redis queue. 95 | func (r *RedisAdapter) processQueueMessages(ctx context.Context, queue chan<- adapter.WebhookPayload) error { 96 | messages, err := r.readMessagesFromQueue(ctx) 97 | if err != nil { 98 | return err 99 | } 100 | 101 | for _, payload := range messages { 102 | select { 103 | case queue <- payload: 104 | _, delErr := r.client.XDel(ctx, r.queueName, payload.MessageID).Result() 105 | if delErr != nil { 106 | logging.WebhookLogger(logging.ErrorType, fmt.Errorf("failed to delete message %s: %v", payload.MessageID, delErr)) 107 | } 108 | case <-ctx.Done(): 109 | return ctx.Err() 110 | default: 111 | logging.WebhookLogger(logging.WarningType, fmt.Errorf("dropped webhook due to channel overflow. Webhook ID: %s", payload.WebhookID)) 112 | } 113 | } 114 | 115 | return nil 116 | } 117 | 118 | // readMessagesFromQueue reads messages from the Redis queue. 119 | func (r *RedisAdapter) readMessagesFromQueue(ctx context.Context) ([]adapter.WebhookPayload, error) { 120 | entries, err := r.client.XRead(ctx, &redis.XReadArgs{ 121 | Streams: []string{r.queueName, r.lastID}, 122 | Count: 5, 123 | }).Result() 124 | 125 | if err != nil { 126 | if err == redis.Nil { 127 | time.Sleep(time.Second) 128 | return r.readMessagesFromQueue(ctx) 129 | } 130 | return nil, err 131 | } 132 | 133 | var messages []adapter.WebhookPayload 134 | for _, entry := range entries[0].Messages { 135 | var payload adapter.WebhookPayload 136 | 137 | if data, ok := entry.Values["data"].(string); ok { 138 | if err := json.Unmarshal([]byte(data), &payload); err != nil { 139 | logging.WebhookLogger(logging.ErrorType, fmt.Errorf("error unmarshalling message data: %w", err)) 140 | r.lastID = entry.ID 141 | return nil, err 142 | } 143 | } else { 144 | logging.WebhookLogger(logging.ErrorType, fmt.Errorf("expected string for 'data' field but got %T", entry.Values["data"])) 145 | r.lastID = entry.ID 146 | return nil, err 147 | } 148 | 149 | payload.MessageID = entry.ID 150 | messages = append(messages, payload) 151 | r.lastID = entry.ID 152 | } 153 | 154 | return messages, nil 155 | } 156 | 157 | // ProcessWebhooks processes webhooks from the specified queue. 158 | func (r *RedisAdapter) ProcessWebhooks(ctx context.Context, queue chan adapter.WebhookPayload, queueAdapter adapter.Adapter) { 159 | 160 | worker.ProcessWebhooks(ctx, queue, r.config, queueAdapter) 161 | } 162 | 163 | // PublishStatus publishes the status of a webhook delivery. 164 | func (r *RedisAdapter) PublishStatus(ctx context.Context, webhookID, url, created, delivered, status, deliveryError string, payloadSize int, numberOfTries int) error { 165 | message := adapter.WebhookDeliveryStatus{ 166 | WebhookID: webhookID, 167 | Status: status, 168 | DeliveryError: deliveryError, 169 | URL: url, 170 | Created: created, 171 | Delivered: delivered, 172 | PayloadSize: payloadSize, 173 | NumberOfTries: numberOfTries, 174 | } 175 | 176 | jsonString, err := json.Marshal(message) 177 | if err != nil { 178 | return err 179 | } 180 | 181 | _, err = r.client.XAdd(ctx, &redis.XAddArgs{ 182 | Stream: r.statusQueue, 183 | Values: map[string]interface{}{"data": jsonString}, 184 | }).Result() 185 | 186 | return err 187 | } 188 | -------------------------------------------------------------------------------- /sendhooks/go.mod: -------------------------------------------------------------------------------- 1 | module sendhooks 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/go-redis/redis/v8 v8.11.5 7 | github.com/sirupsen/logrus v1.9.3 8 | ) 9 | 10 | require ( 11 | github.com/davecgh/go-spew v1.1.1 // indirect 12 | github.com/pmezard/go-difflib v1.0.0 // indirect 13 | golang.org/x/net v0.23.0 // indirect 14 | golang.org/x/sys v0.18.0 // indirect 15 | gopkg.in/yaml.v3 v3.0.1 // indirect 16 | ) 17 | 18 | require ( 19 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 20 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 21 | github.com/stretchr/testify v1.8.4 22 | ) 23 | -------------------------------------------------------------------------------- /sendhooks/go.sum: -------------------------------------------------------------------------------- 1 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 2 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 7 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 8 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 9 | github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= 10 | github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= 11 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 12 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 13 | github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= 14 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 17 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 18 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 19 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 20 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 21 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 22 | golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= 23 | golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= 24 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 25 | golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= 26 | golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 27 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 28 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 29 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 30 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 31 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 32 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 33 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 34 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 35 | -------------------------------------------------------------------------------- /sendhooks/logging/logging.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "sync" 7 | "time" 8 | 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | const ( 13 | ErrorType = "ERROR" 14 | WarningType = "WARNING" 15 | EventType = "EVENT" 16 | ) 17 | 18 | // Logger setup 19 | var ( 20 | logger = logrus.New() 21 | logFile *os.File 22 | logFileName string 23 | logMutex sync.Mutex 24 | ) 25 | 26 | func init() { 27 | setupLogFile() 28 | go rotateLogFileDaily() 29 | } 30 | 31 | // setupLogFile initializes the log file. 32 | func setupLogFile() { 33 | var err error 34 | logFileName = currentDate() + ".log" 35 | logFile, err = os.OpenFile(logFileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 36 | if err != nil { 37 | fmt.Printf("Failed to open log file: %v\n", err) 38 | return 39 | } 40 | logger.SetOutput(logFile) 41 | logger.SetFormatter(&logrus.TextFormatter{}) 42 | } 43 | 44 | // currentDate retrieves the current date in "YYYY-MM-DD" format. 45 | func currentDate() string { 46 | return time.Now().Format("2006-01-02") 47 | } 48 | 49 | // currentDateTime retrieves the current date and time in "YYYY-MM-DD HH:MM:SS" format. 50 | func currentDateTime() string { 51 | return time.Now().Format("2006-01-02 15:04:05") 52 | } 53 | 54 | // rotateLogFileDaily handles daily log rotation. 55 | func rotateLogFileDaily() { 56 | for { 57 | time.Sleep(24 * time.Hour) 58 | logMutex.Lock() 59 | if currentDate() != logFileName[:10] { 60 | logFile.Close() 61 | setupLogFile() 62 | } 63 | logMutex.Unlock() 64 | } 65 | } 66 | 67 | // WebhookLogger logs messages with different types (Error, Warning, Event). 68 | var WebhookLogger = func(errorType string, message interface{}) error { 69 | var messageString string 70 | 71 | switch v := message.(type) { 72 | case error: 73 | messageString = v.Error() 74 | case string: 75 | messageString = v 76 | default: 77 | return fmt.Errorf("unsupported message type: %T", message) 78 | } 79 | 80 | logMutex.Lock() 81 | defer logMutex.Unlock() 82 | 83 | // Log the entry 84 | entry := logger.WithFields(logrus.Fields{ 85 | "date": currentDateTime(), 86 | }) 87 | switch errorType { 88 | case ErrorType: 89 | entry.Error(messageString) 90 | case WarningType: 91 | entry.Warning(messageString) 92 | case EventType: 93 | entry.Info(messageString) 94 | } 95 | 96 | return nil 97 | } 98 | -------------------------------------------------------------------------------- /sendhooks/main.go: -------------------------------------------------------------------------------- 1 | // main.go 2 | package main 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "log" 8 | "runtime" 9 | "sync" 10 | 11 | "sendhooks/adapter" 12 | "sendhooks/adapter/adapter_manager" 13 | redisadapter "sendhooks/adapter/redis_adapter" 14 | "sendhooks/logging" 15 | ) 16 | 17 | func main() { 18 | runtime.GOMAXPROCS(runtime.NumCPU()) 19 | adapter_manager.LoadConfiguration("config.json") 20 | 21 | ctx, cancel := context.WithCancel(context.Background()) 22 | defer cancel() 23 | 24 | err := logging.WebhookLogger(logging.EventType, "starting sendhooks engine") 25 | if err != nil { 26 | log.Fatalf("Failed to log sendhooks event: %v", err) 27 | } 28 | 29 | var queueAdapter adapter.Adapter 30 | conf := adapter_manager.GetConfig() 31 | 32 | if conf.Broker == "redis" { 33 | queueAdapter = redisadapter.NewRedisAdapter(conf) 34 | } 35 | 36 | err = queueAdapter.Connect() 37 | if err != nil { 38 | log.Fatalf("Failed to connect to Redis: %v", err) 39 | } 40 | 41 | // Define the size of the channel and the number of workers 42 | webhookQueue := make(chan adapter.WebhookPayload, conf.ChannelSize) 43 | numWorkers := conf.NumWorkers 44 | 45 | // Start the worker pool 46 | var wg sync.WaitGroup 47 | wg.Add(numWorkers) 48 | for i := 0; i < numWorkers; i++ { 49 | go func() { 50 | defer wg.Done() 51 | queueAdapter.ProcessWebhooks(ctx, webhookQueue, queueAdapter) 52 | }() 53 | } 54 | 55 | err = queueAdapter.SubscribeToQueue(ctx, webhookQueue) 56 | if err != nil { 57 | logging.WebhookLogger(logging.ErrorType, fmt.Errorf("error initializing connection: %s", err)) 58 | log.Fatalf("error initializing connection: %v", err) 59 | return 60 | } 61 | 62 | // Wait for all workers to finish 63 | wg.Wait() 64 | 65 | select {} 66 | } 67 | -------------------------------------------------------------------------------- /sendhooks/queue/worker.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | /* 4 | * This package is used for queuing the webhooks to send using golang channels. In this package, 5 | we also handle the retries and the exponential backoff logic for sending the hooks. 6 | */ 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "sendhooks/adapter" 12 | "sendhooks/logging" 13 | "sendhooks/sender" 14 | "time" 15 | "unsafe" 16 | ) 17 | 18 | // Function to measure the size of a map in bytes 19 | func SizeofMap(m map[string]interface{}) int { 20 | size := int(unsafe.Sizeof(m)) 21 | for k, v := range m { 22 | size += int(unsafe.Sizeof(k)) + len(k) 23 | size += SizeofValue(v) 24 | } 25 | return size 26 | } 27 | 28 | // Function to measure the size of a value in bytes 29 | func SizeofValue(v interface{}) int { 30 | switch v := v.(type) { 31 | case int: 32 | return int(unsafe.Sizeof(v)) 33 | case float64: 34 | return int(unsafe.Sizeof(v)) 35 | case string: 36 | return int(unsafe.Sizeof(v)) + len(v) 37 | case []int: 38 | return int(unsafe.Sizeof(v)) + len(v)*int(unsafe.Sizeof(v[0])) 39 | // Add more cases as needed for other types 40 | default: 41 | return 0 // Unsupported type 42 | } 43 | } 44 | 45 | const ( 46 | maxRetries int = 5 47 | ) 48 | 49 | const ( 50 | initialBackoff time.Duration = time.Second 51 | maxBackoff time.Duration = time.Hour 52 | ) 53 | 54 | func ProcessWebhooks(ctx context.Context, webhookQueue chan adapter.WebhookPayload, configuration adapter.Configuration, queueAdapter adapter.Adapter) { 55 | 56 | for payload := range webhookQueue { 57 | go sendWebhookWithRetries(ctx, payload, configuration, queueAdapter) 58 | } 59 | } 60 | 61 | func sendWebhookWithRetries(ctx context.Context, payload adapter.WebhookPayload, configuration adapter.Configuration, queueAdapter adapter.Adapter) { 62 | 63 | if err, created, retries := retryWithExponentialBackoff(ctx, payload, configuration, queueAdapter); err != nil { 64 | logging.WebhookLogger(logging.WarningType, fmt.Errorf("failed to send sendhooks after maximum retries. WebhookID : %s", payload.WebhookID)) 65 | err := queueAdapter.PublishStatus(ctx, payload.WebhookID, payload.URL, created, "", "failed", err.Error(), SizeofMap(payload.Data), retries) 66 | if err != nil { 67 | logging.WebhookLogger(logging.WarningType, fmt.Errorf("error publishing status update: WebhookID : %s ", payload.WebhookID)) 68 | } 69 | } 70 | } 71 | 72 | func calculateBackoff(currentBackoff time.Duration) time.Duration { 73 | 74 | nextBackoff := currentBackoff * 2 75 | 76 | if nextBackoff > maxBackoff { 77 | return maxBackoff 78 | } 79 | 80 | return nextBackoff 81 | } 82 | 83 | func retryWithExponentialBackoff(context context.Context, payload adapter.WebhookPayload, configuration adapter.Configuration, queueAdapter adapter.Adapter) (error, string, int) { 84 | retries := 0 85 | backoffTime := initialBackoff 86 | var requestError error 87 | 88 | created := time.Now().String() 89 | 90 | for retries < maxRetries { 91 | err := sender.SendWebhook(payload.Data, payload.URL, payload.WebhookID, payload.SecretHash, configuration) 92 | 93 | if err == nil { 94 | if err != nil { 95 | logging.WebhookLogger(logging.WarningType, fmt.Errorf("error publishing status update WebhookID : %s ", payload. 96 | WebhookID)) 97 | } 98 | // Break the loop if the request has been delivered successfully. 99 | requestError = nil 100 | break 101 | } 102 | 103 | logging.WebhookLogger(logging.ErrorType, fmt.Errorf("error sending sendhooks: %s", err)) 104 | 105 | backoffTime = calculateBackoff(backoffTime) 106 | retries++ 107 | requestError = err 108 | time.Sleep(backoffTime) 109 | } 110 | 111 | logging.WebhookLogger(logging.WarningType, fmt.Errorf("maximum retries reached: %d", retries)) 112 | 113 | if requestError != nil { 114 | return requestError, created, retries 115 | } 116 | 117 | delivered := time.Now().String() 118 | 119 | err := queueAdapter.PublishStatus(context, payload.WebhookID, payload.URL, created, delivered, "success", "", SizeofMap(payload.Data), retries) 120 | if err != nil { 121 | logging.WebhookLogger(logging.WarningType, fmt.Errorf("error publishing status update: WebhookID : %s ", payload.WebhookID)) 122 | } 123 | 124 | return nil, "", retries 125 | } 126 | -------------------------------------------------------------------------------- /sendhooks/sender/sender_test.go: -------------------------------------------------------------------------------- 1 | package sender 2 | 3 | /* 4 | This package contains mostly test functions for the utils used to send webhooks. If these utils works, 5 | we can ensure that the send sendhooks function will function normally too. For a whole test suite for the sender function, 6 | check out the webhook_test.go file. 7 | */ 8 | 9 | import ( 10 | "bytes" 11 | "io" 12 | "net/http" 13 | "sendhooks/adapter" 14 | "testing" 15 | 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | type MockClient struct { 20 | MockDo func(req *http.Request) (*http.Response, error) 21 | } 22 | 23 | func (m *MockClient) Do(req *http.Request) (*http.Response, error) { 24 | return m.MockDo(req) 25 | } 26 | 27 | // Testing the MarshalJSON method 28 | func TestMarshalJSON(t *testing.T) { 29 | data := map[string]string{ 30 | "key": "value", 31 | } 32 | expectedJSON := `{"key":"value"}` 33 | 34 | jsonBytes, err := marshalJSON(data) 35 | 36 | assert.NoError(t, err) 37 | 38 | assert.Equal(t, expectedJSON, string(jsonBytes)) 39 | } 40 | 41 | // Test for prepareRequest 42 | func TestPrepareRequest(t *testing.T) { 43 | url := "http://example.com/webhook" 44 | jsonBytes := []byte(`{"key":"value"}`) 45 | secretHash := "secret123" 46 | 47 | req, err := prepareRequest(url, jsonBytes, secretHash, adapter.Configuration{}) 48 | 49 | assert.NoError(t, err) 50 | 51 | assert.Equal(t, "application/json", req.Header.Get("Content-Type")) 52 | 53 | assert.Equal(t, secretHash, req.Header.Get("X-Secret-Hash")) 54 | } 55 | 56 | func TestSendRequest(t *testing.T) { 57 | HTTPClient = &MockClient{ 58 | MockDo: func(req *http.Request) (*http.Response, error) { 59 | return &http.Response{ 60 | StatusCode: 200, 61 | Body: io.NopCloser(bytes.NewBufferString("OK")), 62 | }, nil 63 | }, 64 | } 65 | 66 | req, _ := http.NewRequest("GET", "http://example.com", nil) 67 | resp, err := sendRequest(req) 68 | 69 | assert.NoError(t, err) 70 | 71 | body, _ := io.ReadAll(resp.Body) 72 | 73 | assert.Equal(t, 200, resp.StatusCode) 74 | assert.Equal(t, "OK", string(body)) 75 | } 76 | -------------------------------------------------------------------------------- /sendhooks/sender/utils.go: -------------------------------------------------------------------------------- 1 | package sender 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "sendhooks/adapter" 10 | "sendhooks/logging" 11 | ) 12 | 13 | type HTTPDoer interface { 14 | Do(req *http.Request) (*http.Response, error) 15 | } 16 | 17 | var HTTPClient HTTPDoer = &http.Client{} 18 | 19 | var marshalJSON = func(data interface{}) ([]byte, error) { 20 | jsonBytes, err := json.Marshal(data) 21 | if err != nil { 22 | logging.WebhookLogger(logging.ErrorType, fmt.Errorf("error marshaling JSON: %s", err)) 23 | return nil, err 24 | } 25 | return jsonBytes, nil 26 | } 27 | 28 | var prepareRequest = func(url string, jsonBytes []byte, secretHash string, configuration adapter.Configuration) (*http.Request, error) { 29 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonBytes)) 30 | if err != nil { 31 | logging.WebhookLogger(logging.ErrorType, fmt.Errorf("error during the sendhooks request preparation")) 32 | return nil, err 33 | } 34 | 35 | req.Header.Set("Content-Type", "application/json") 36 | 37 | secretHashHeaderName := configuration.SecretHashHeaderName 38 | if secretHashHeaderName == "" { 39 | secretHashHeaderName = "X-Secret-Hash" 40 | } 41 | 42 | if secretHash != "" { 43 | req.Header.Set(secretHashHeaderName, secretHash) 44 | } 45 | 46 | return req, nil 47 | } 48 | 49 | var sendRequest = func(req *http.Request) (*http.Response, error) { 50 | resp, err := HTTPClient.Do(req) 51 | 52 | if err != nil { 53 | return nil, err 54 | } 55 | return resp, nil 56 | } 57 | 58 | var closeResponse = func(body io.ReadCloser) { 59 | if err := body.Close(); err != nil { 60 | logging.WebhookLogger(logging.ErrorType, fmt.Errorf("error closing response body: %s", err)) 61 | } 62 | } 63 | 64 | var processResponse = func(resp *http.Response) (string, []byte, int, error) { 65 | respBody, err := io.ReadAll(resp.Body) 66 | if err != nil { 67 | logging.WebhookLogger(logging.ErrorType, fmt.Errorf("error reading response body: %s", err)) 68 | return "failed", nil, 0, err 69 | } 70 | 71 | status := "failed" 72 | if resp.StatusCode == http.StatusOK { 73 | status = "delivered" 74 | } 75 | 76 | if status == "failed" { 77 | logging.WebhookLogger(logging.ErrorType, fmt.Errorf("HTTP request failed with status code: %d, response body: %s", resp.StatusCode, string(respBody))) 78 | } 79 | 80 | return status, respBody, resp.StatusCode, nil 81 | } 82 | -------------------------------------------------------------------------------- /sendhooks/sender/webhook.go: -------------------------------------------------------------------------------- 1 | package sender 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sendhooks/adapter" 7 | "sendhooks/logging" 8 | ) 9 | 10 | // SendWebhook sends a JSON POST request to the specified URL 11 | func SendWebhook(data interface{}, url string, webhookId string, secretHash string, configuration adapter.Configuration) error { 12 | jsonBytes, err := marshalJSON(data) 13 | if err != nil { 14 | return err 15 | } 16 | 17 | req, err := prepareRequest(url, jsonBytes, secretHash, configuration) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | resp, err := sendRequest(req) 23 | if err != nil { 24 | 25 | return err 26 | } 27 | 28 | defer closeResponse(resp.Body) 29 | 30 | status, respBody, statusCode, err := processResponse(resp) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | message := fmt.Sprintf("webhook sending failed with status: %d, response body: %s", statusCode, string(respBody)) 36 | 37 | if status == "failed" { 38 | logging.WebhookLogger(logging.WarningType, fmt.Errorf(message)) 39 | return errors.New(message) 40 | } 41 | 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /sendhooks/sender/webhook_test.go: -------------------------------------------------------------------------------- 1 | package sender 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | "sendhooks/adapter" 7 | "sendhooks/logging" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var ( 14 | marshalJSONOrig = marshalJSON 15 | prepareRequestOrig = prepareRequest 16 | sendRequestOrig = sendRequest 17 | processResponseOrig = processResponse 18 | webhookLoggerInvoked = false 19 | ) 20 | 21 | func TestSendWebhook(t *testing.T) { 22 | logging.WebhookLogger = func(errorType string, errorMessage interface{}) error { 23 | webhookLoggerInvoked = true 24 | return nil 25 | } 26 | 27 | t.Run("Successful sendhooks sending", func(t *testing.T) { 28 | resetMocks() // Reset all mocks to original functions 29 | 30 | err := SendWebhook(nil, "http://dummy.com", "webhookId", "secretHash", adapter.Configuration{}) 31 | 32 | assert.NoError(t, err) 33 | }) 34 | 35 | t.Run("Failed sendhooks due to marshaling errors", func(t *testing.T) { 36 | resetMocks() 37 | marshalJSON = func(data interface{}) ([]byte, error) { 38 | return nil, errors.New("marshaling error") 39 | } 40 | 41 | err := SendWebhook(nil, "http://dummy.com", "webhookId", "secretHash", adapter.Configuration{}) 42 | 43 | assert.EqualError(t, err, "marshaling error") 44 | }) 45 | 46 | t.Run("Failed sendhooks due to request preparation errors", func(t *testing.T) { 47 | resetMocks() 48 | prepareRequest = func(url string, jsonBytes []byte, secretHash string, configuration adapter.Configuration) (*http.Request, error) { 49 | return nil, errors.New("request preparation error") 50 | } 51 | 52 | err := SendWebhook(nil, "http://dummy.com", "webhookId", "secretHash", adapter.Configuration{}) 53 | 54 | assert.EqualError(t, err, "request preparation error") 55 | }) 56 | 57 | t.Run("Failed sendhooks due to response processing errors", func(t *testing.T) { 58 | resetMocks() 59 | processResponse = func(resp *http.Response) (string, []byte, int, error) { 60 | return "failed", nil, 0, errors.New("response processing error") 61 | } 62 | 63 | err := SendWebhook(nil, "http://dummy.com", "webhookId", "secretHash", adapter.Configuration{}) 64 | 65 | assert.EqualError(t, err, "response processing error") 66 | }) 67 | 68 | t.Run("Logging on failed sendhooks delivery", func(t *testing.T) { 69 | resetMocks() 70 | processResponse = func(resp *http.Response) (string, []byte, int, error) { 71 | return "failed", []byte("error body"), 0, nil 72 | } 73 | 74 | SendWebhook(nil, "http://dummy\t\tassert.EqualError(t, err, \"failed\")\n.com", "webhookId", "secretHash", adapter.Configuration{}) 75 | if !webhookLoggerInvoked { 76 | assert.Fail(t, "Expected WebhookLogger to be invoked") 77 | } 78 | 79 | }) 80 | } 81 | 82 | func resetMocks() { 83 | marshalJSON = marshalJSONOrig 84 | prepareRequest = prepareRequestOrig 85 | sendRequest = sendRequestOrig 86 | processResponse = processResponseOrig 87 | } 88 | -------------------------------------------------------------------------------- /sendhooks/utils/redis_tls_config.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "os" 8 | ) 9 | 10 | func CreateTLSConfig(caCertPath, clientCertPath, clientKeyPath string) (*tls.Config, error) { 11 | // Load CA cert 12 | caCert, err := os.ReadFile(caCertPath) 13 | if err != nil { 14 | return nil, fmt.Errorf("failed to load CA certificate: %v", err) 15 | } 16 | 17 | caCertPool := x509.NewCertPool() 18 | caCertPool.AppendCertsFromPEM(caCert) 19 | 20 | tlsConfig := &tls.Config{ 21 | RootCAs: caCertPool, 22 | } 23 | 24 | // Load client cert and key if they're provided 25 | if clientCertPath != "" && clientKeyPath != "" { 26 | cert, err := tls.LoadX509KeyPair(clientCertPath, clientKeyPath) 27 | if err != nil { 28 | return nil, fmt.Errorf("failed to load client cert and key: %v", err) 29 | } 30 | 31 | tlsConfig.Certificates = []tls.Certificate{cert} 32 | } 33 | 34 | return tlsConfig, nil 35 | } 36 | -------------------------------------------------------------------------------- /sendhooks/utils/redis_tls_config_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | /* 4 | This is a test to ensure that given path of certificates required to run Redis with SSL, the files are read and the 5 | configuration can work. For this purpose, we use a custom certificates generator for our tests. 6 | */ 7 | 8 | import ( 9 | "fmt" 10 | "github.com/stretchr/testify/assert" 11 | "os" 12 | certificates "sendhooks/utils/tests" 13 | "testing" 14 | ) 15 | 16 | func FormatErrorBody(errorType string) string { 17 | 18 | return fmt.Sprintf("error creating temp %s file", errorType) 19 | } 20 | 21 | func TestCreateTLSConfig(t *testing.T) { 22 | // Generate test certificates 23 | caCertPEM, certPEM, keyPEM, err := certificates.GenerateTestCertificates() 24 | assert.NoError(t, err, "Error generating test certificates") 25 | 26 | // Create temporary files to store these certificates and keys 27 | caCertFile, err := os.CreateTemp("", "caCert") 28 | assert.NoError(t, err, FormatErrorBody("CA cert")) 29 | defer os.Remove(caCertFile.Name()) 30 | caCertFile.Write(caCertPEM) 31 | caCertFile.Close() 32 | 33 | clientCertFile, err := os.CreateTemp("", "clientCert") 34 | assert.NoError(t, err, FormatErrorBody("client cert")) 35 | 36 | defer os.Remove(clientCertFile.Name()) 37 | clientCertFile.Write(certPEM) 38 | clientCertFile.Close() 39 | 40 | clientKeyFile, err := os.CreateTemp("", "clientKey") 41 | assert.NoError(t, err, FormatErrorBody("client key")) 42 | 43 | defer os.Remove(clientKeyFile.Name()) 44 | clientKeyFile.Write(keyPEM) 45 | clientKeyFile.Close() 46 | 47 | // Optimist Test: valid CA certificate 48 | tlsConfig, err := CreateTLSConfig(caCertFile.Name(), "", "") 49 | assert.NoError(t, err, "Expected no error creating TLS config with only CA cert") 50 | 51 | //Subjects is deprecated for the moment, but it looks like there is no other alternatives. https://github.com/golang/go/issues/46287 52 | assert.NotEmpty(t, tlsConfig.RootCAs.Subjects(), "Expected CA certificates to be loaded") 53 | 54 | // Optimist Test: valid CA certificate with valid client certificate and key 55 | tlsConfig, err = CreateTLSConfig(caCertFile.Name(), clientCertFile.Name(), clientKeyFile.Name()) 56 | assert.NoError(t, err, "Expected no error creating TLS config with client cert and key") 57 | assert.NotEmpty(t, tlsConfig.Certificates, "Expected client certificates to be loaded") 58 | 59 | // Pessimist Test: invalid file paths 60 | _, err = CreateTLSConfig("invalidPath", "", "") 61 | assert.Error(t, err, "Expected error creating TLS config with invalid CA cert path") 62 | 63 | _, err = CreateTLSConfig(caCertFile.Name(), "invalidPath", "invalidPath") 64 | assert.Error(t, err, "Expected error creating TLS config with invalid client cert and key paths") 65 | } 66 | -------------------------------------------------------------------------------- /sendhooks/utils/tests/certificates.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | /* 4 | This utils is used to generate dummy certificates for SSL connection to Redis. 5 | */ 6 | 7 | import ( 8 | "crypto/ecdsa" 9 | "crypto/elliptic" 10 | "crypto/rand" 11 | "crypto/x509" 12 | "crypto/x509/pkix" 13 | "encoding/pem" 14 | "math/big" 15 | "time" 16 | ) 17 | 18 | // GenerateTestCertificates generates a dummy CA certificate and a client certificate 19 | // signed by that CA for testing purposes. It returns the PEM-encoded certificates and key. 20 | func GenerateTestCertificates() (caCertPEM, certPEM, keyPEM []byte, err error) { 21 | // Generate CA certificate: 22 | 23 | // Create a new private key using elliptic curve cryptography 24 | caKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) 25 | if err != nil { 26 | return nil, nil, nil, err 27 | } 28 | 29 | // Define the template for the CA certificate 30 | caTemplate := &x509.Certificate{ 31 | SerialNumber: big.NewInt(1), // Unique serial number for the certificate 32 | Subject: pkix.Name{Organization: []string{"Test CA"}}, // Define details of the entity the certificate represents 33 | NotBefore: time.Now(), // Certificate validity start time 34 | NotAfter: time.Now().Add(24 * time.Hour), // Certificate expiry time (24 hours from now) 35 | KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, // Define how the certificate can be used 36 | IsCA: true, // Indicate that this is a CA certificate 37 | } 38 | 39 | // Create the CA certificate using the template and private key 40 | caCertDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) 41 | if err != nil { 42 | return nil, nil, nil, err 43 | } 44 | // Convert the DER-encoded certificate into PEM format 45 | caCertPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caCertDER}) 46 | 47 | // Generate client certificate: 48 | 49 | // Create a new private key for the client certificate 50 | clientKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) 51 | if err != nil { 52 | return nil, nil, nil, err 53 | } 54 | 55 | // Define the template for the client certificate 56 | clientTemplate := &x509.Certificate{ 57 | SerialNumber: big.NewInt(2), 58 | Subject: pkix.Name{Organization: []string{"Test Client"}}, 59 | NotBefore: time.Now(), 60 | NotAfter: time.Now().Add(24 * time.Hour), 61 | KeyUsage: x509.KeyUsageDigitalSignature, 62 | } 63 | 64 | // Create the client certificate using the template, CA certificate, and private key 65 | clientCertDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, caTemplate, &clientKey.PublicKey, caKey) 66 | if err != nil { 67 | return nil, nil, nil, err 68 | } 69 | // Convert the DER-encoded client certificate into PEM format 70 | certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: clientCertDER}) 71 | 72 | // Convert the private key into DER format 73 | keyDER, err := x509.MarshalECPrivateKey(clientKey) 74 | if err != nil { 75 | return nil, nil, nil, err 76 | } 77 | // Convert the DER-encoded key into PEM format 78 | keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) 79 | 80 | return 81 | } 82 | --------------------------------------------------------------------------------