├── .devcontainer └── devcontainer.json ├── .github ├── ISSUE_TEMPLATE │ └── scrum.md ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ ├── db.yml │ ├── deploy.yml │ ├── linter.yml │ └── scrum.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── demos.md ├── docker-compose.yml ├── images ├── db-query.png ├── flyway-logs.png └── port-forwarding.png ├── python ├── app.py └── requirements.txt ├── sql ├── db.env ├── flyway.conf ├── migrations │ ├── repeatable │ │ └── R__seed-testwebhook-data.sql │ └── versioned │ │ ├── V01__init.sql │ │ ├── V02__add-collab-changelog.sql │ │ └── V03__add-indexes.sql └── queries │ └── webhook-events.sql └── systemd ├── README.md ├── app-queue.service ├── app.py ├── app.service ├── requirements.txt └── worker.py /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: 2 | // https://github.com/microsoft/vscode-dev-containers/tree/v0.209.6/containers/docker-existing-docker-compose 3 | // If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml. 4 | { 5 | "name": "Webhook receiver", 6 | 7 | // Update the 'dockerComposeFile' list if you have more compose files or use different names. 8 | // The .devcontainer/docker-compose.yml file contains any overrides you need/want to make. 9 | "dockerComposeFile": [ 10 | "../docker-compose.yml" 11 | ], 12 | 13 | // The 'service' property is the name of the service for the container that VS Code should 14 | // use. Update this value and .devcontainer/docker-compose.yml to the real service name. 15 | "service": "web", 16 | 17 | // The optional 'workspaceFolder' property is the path VS Code should open by default when 18 | // connected. This is typically a file mount in .devcontainer/docker-compose.yml 19 | // "workspaceFolder": "/workspace", 20 | 21 | // Set *default* container specific settings.json values on container create. 22 | "settings": { 23 | "sqltools.connections": [ 24 | { 25 | "name": "Container database", 26 | "driver": "PostgreSQL", 27 | "previewLimit": 50, 28 | "server": "database", 29 | "port": 5432, 30 | "database": "webhooks", 31 | "username": "postgres", 32 | "password": "mysecretpassword" 33 | } 34 | ] 35 | }, 36 | 37 | // Add the IDs of extensions you want installed when the container is created. 38 | "extensions": [ 39 | "mtxr.sqltools", 40 | "mtxr.sqltools-driver-pg" 41 | ], 42 | 43 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 44 | "forwardPorts": [ 45 | 5000, 46 | 5432 47 | ], 48 | 49 | "portsAttributes": { 50 | "5000": { 51 | "label": "webhook receiver" 52 | }, 53 | "5432": { 54 | "label": "postgres" 55 | } 56 | }, 57 | 58 | // Uncomment the next line if you want start specific services in your Docker Compose config. 59 | "runServices": [ 60 | "database", 61 | "redis_cache", 62 | "flyway" 63 | ], 64 | 65 | // Uncomment the next line if you want to keep your containers running after VS Code shuts down. 66 | // "shutdownAction": "none", 67 | 68 | // Uncomment the next line to run commands after the container is created - for example installing curl. 69 | // "postCreateCommand": "apt-get update && apt-get install -y curl", 70 | 71 | // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root. 72 | // "remoteUser": "vscode" 73 | } 74 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/scrum.md: -------------------------------------------------------------------------------- 1 | # Weekly standup 2 | 3 | ## Attendees 4 | 5 | - [ ] team member 6 | - [ ] team member 7 | - [ ] team member, scrum master 8 | 9 | ## Retrospective 10 | 11 | - ✅ What worked well this week? 12 | - things 13 | - ❌ What didn't work well this week? 14 | - more things 15 | - 📋 What do we want to work on for next week? 16 | - even more things 17 | 18 | ## Admin items 19 | 20 | Example items that would go here would be like what's below 👇 21 | 22 | - Upcoming meetings, off-sites, etc. 23 | - HR tasks like open enrollment, new hires, etc. 24 | - Other admin items 25 | 26 | ## Planning 27 | 28 | Given everything going on this week, what's the goal(s)? (pick issue(s)) 29 | 30 | - @mention teammate here = 31 | - @mention teammate here = 32 | 33 | Is everything ☝️ assigned to this sprint in [GitHub](https://github.com/users/some-natalie/projects/3/views/15)? 34 | 35 | Do we have any blockers or potential unplanned events? 36 | 37 | ## Other tasks 38 | 39 | - [ ] one-off tasks that don't warrant an issue, but also need to be done go here 40 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | - package-ecosystem: "pip" 8 | directory: "/python" 9 | schedule: 10 | interval: "weekly" 11 | - package-ecosystem: "pip" 12 | directory: "/systemd" 13 | schedule: 14 | interval: "weekly" 15 | - package-ecosystem: "docker" 16 | directory: "/" 17 | schedule: 18 | interval: "weekly" 19 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | workflow_dispatch: 21 | 22 | jobs: 23 | analyze: 24 | name: Analyze 25 | runs-on: ubuntu-latest 26 | permissions: 27 | actions: read 28 | contents: read 29 | security-events: write 30 | 31 | strategy: 32 | fail-fast: false 33 | matrix: 34 | language: [ 'python' ] 35 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 36 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 37 | 38 | steps: 39 | - name: Checkout repository 40 | uses: actions/checkout@v3 41 | 42 | # Initializes the CodeQL tools for scanning. 43 | - name: Initialize CodeQL 44 | uses: github/codeql-action/init@v2 45 | with: 46 | languages: ${{ matrix.language }} 47 | setup-python-dependencies: true 48 | queries: security-and-quality 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v2 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v2 72 | -------------------------------------------------------------------------------- /.github/workflows/db.yml: -------------------------------------------------------------------------------- 1 | name: Test database schema migrations 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | branches: 7 | - "main" 8 | push: 9 | paths: 10 | - 'sql/migrations/**' 11 | 12 | jobs: 13 | flyway-check: 14 | name: Validate the Flyway migrations on an empty database 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Check out the code 18 | uses: actions/checkout@v3 19 | - name: Apply all migrations on empty DB 20 | run: docker compose up --exit-code-from flyway 21 | - name: Print the flyway logs, if needed later 22 | run: docker compose logs flyway 23 | if: always() 24 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy database changes 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | flyway-deploy: 8 | name: Deploy 9 | runs-on: database 10 | environment: Test 11 | steps: 12 | - name: Check out the code 13 | uses: actions/checkout@v3 14 | - name: Run migrate 15 | run: docker-compose run flyway 16 | - name: Print the flyway logs, if needed later 17 | run: docker compose logs flyway 18 | if: always() 19 | -------------------------------------------------------------------------------- /.github/workflows/linter.yml: -------------------------------------------------------------------------------- 1 | ################################# 2 | ################################# 3 | ## Super Linter GitHub Actions ## 4 | ################################# 5 | ################################# 6 | name: Lint Code Base 7 | 8 | # 9 | # Documentation: 10 | # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions 11 | # 12 | 13 | ############################# 14 | # Start the job on all push # 15 | ############################# 16 | on: 17 | push: 18 | branches-ignore: [master, main] 19 | # Remove the line above to run when pushing to master 20 | pull_request: 21 | branches: [master, main] 22 | 23 | ############### 24 | # Set the Job # 25 | ############### 26 | jobs: 27 | build: 28 | # Name the Job 29 | name: Lint Code Base 30 | # Set the agent to run on 31 | runs-on: ubuntu-latest 32 | 33 | ################## 34 | # Load all steps # 35 | ################## 36 | steps: 37 | ########################## 38 | # Checkout the code base # 39 | ########################## 40 | - name: Checkout Code 41 | uses: actions/checkout@v3 42 | with: 43 | # Full git history is needed to get a proper list of changed files within `super-linter` 44 | fetch-depth: 0 45 | 46 | ################################ 47 | # Run Linter against code base # 48 | ################################ 49 | - name: Lint Code Base 50 | uses: super-linter/super-linter/slim@v5 51 | env: 52 | VALIDATE_ALL_CODEBASE: true 53 | DEFAULT_BRANCH: main 54 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 55 | # VALIDATE_GITHUB_ACTIONS: true 56 | VALIDATE_PYTHON_BLACK: true 57 | -------------------------------------------------------------------------------- /.github/workflows/scrum.yml: -------------------------------------------------------------------------------- 1 | name: Weekly scrum 2 | 3 | on: 4 | schedule: 5 | - cron: "0 12 * * 2" # Tuesday at 12 pm UTC (which is early morning in the US) 6 | workflow_dispatch: 7 | 8 | jobs: 9 | weekly_meeting: 10 | name: create new issue 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Set Date 14 | run: echo "date=$(date -u '+%B %d %Y')" >> $GITHUB_ENV 15 | 16 | - name: Checkout 17 | uses: actions/checkout@v3 18 | 19 | - name: Create issue 20 | id: issue-bot 21 | uses: imjohnbo/issue-bot@v2 22 | with: 23 | title: "Weekly scrum for ${{ env.date }}" 24 | # rotate-assignees: true 25 | assignees: "some-natalie" 26 | labels: "standup" 27 | pinned: true 28 | close-previous: true 29 | linked-comments: true 30 | template: ".github/ISSUE_TEMPLATE/scrum.md" 31 | env: 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | WORKDIR /app 4 | 5 | # Set environment variables 6 | ENV FLASK_APP=app.py 7 | ENV FLASK_RUN_HOST=0.0.0.0 8 | ENV PYTHONUNBUFFERED=1 9 | 10 | # Install dependencies 11 | COPY ./python/requirements.txt requirements.txt 12 | RUN pip install --no-cache-dir -r requirements.txt 13 | 14 | # Expose the app 15 | EXPOSE 5000 16 | 17 | # Copy the files in 18 | COPY ./python . 19 | 20 | # Start it up 21 | CMD ["flask", "run"] 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Natalie Somersall 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Webhook receiver 2 | 3 | An example of a webhook receiver using [GitHub Codespaces](https://github.com/features/codespaces)! Check out the [demo](demos.md) guide for a short guided tour. 4 | 5 | > **Warning** 6 | > Not for production. May contain out of date dependencies in `requirements.txt`. Also, it doesn't do anything useful - but you can probably modify it to do useful stuff on webhook receipt. 7 | 8 | ## Real world 9 | 10 | Change the `app.py` function to do something useful ... whatever it was that you were really wanting to accomplish. Update the requirements in `requirements.txt` first. Consider moving off Werkzeug. 11 | 12 | :information_source: Yes, the Redis container in `docker-compose.yml` does nothing at the moment, but it does in the [systemd](systemd) directory. I had originally used this sort of program to kick off longer running tasks that couldn't be finished in time for the 10s webhook timeout in GitHub, so it'll return `201 - created` once the info from the webhook is put into Redis. 13 | -------------------------------------------------------------------------------- /demos.md: -------------------------------------------------------------------------------- 1 | # Demos run from this repo 2 | 3 | - [Codespaces](#codespaces) 4 | - [Database schema management](#database-schema-management) 5 | - [GHAS code quality](#code-quality) 6 | - [Dependabot](#dependabot) 7 | - [Scrum](#scrum) 8 | 9 | --- 10 | 11 | ## Codespaces 12 | 13 | First, intro Codespaces with some cool slides! Next, let's open the repo and walk through a couple things before launching a Codespace. 14 | 15 | This repo has a custom [devcontainer.json](.devcontainer/devcontainer.json) file, so you won't have the default [kitchen sink](https://docs.github.com/en/codespaces/setting-up-your-project-for-codespaces/introduction-to-dev-containers#using-the-default-dev-container-configuration) image. If you want, the source code to the default image is [here](https://github.com/microsoft/vscode-dev-containers/tree/main/containers/codespaces-linux). It's a lot to look at, so this one is both simple and custom. :) 16 | 17 | It installs and configures some custom extensions in VS Code ([here](.devcontainer/devcontainer.json#L21-L41)), spins up all the containers defined in [`docker-compose.yml`](docker-compose.yml), and starts the Flask app. The ports are labelled and privately exposed by default. 18 | 19 | Now launch a new Codespace from the main branch. Expose port 5000 publicly, then copy "Local Address" URL. Send a repo webhook to that URL (type `application/json`), appended by `/webhook`. It'll look something like this `https://your-codespace-url-5000.githubpreview.dev/webhook`. 20 | 21 | ![images/port-forwarding.png](images/port-forwarding.png) 22 | 23 | Note how it populates the `test_webhook` table with the repo owner, repo name, and timestamp with [this SQL query](sql/queries/webhook-events.sql), pasted below. 24 | 25 | ```sql 26 | SELECT * FROM test_webhook; 27 | ``` 28 | 29 | ![images/db-query.png](images/db-query.png) 30 | 31 | Note how fast it launched? It's prebuilt off the main branch. The setting is controlled [here](https://github.com/octodemo/webhook-demo/settings/codespaces). Prebuilds are run automatically from GitHub Actions and you can see the output of the latest builds [here](https://github.com/octodemo/webhook-demo/actions/workflows/codespaces/create_codespaces_prebuilds). This is a pretty small image, so builds don't take long, but for larger or more complicated repositories this can save 20+ minutes on custom image builds! 32 | 33 | No chat is complete without policy and governance. You should walk through this at the [organization](https://github.com/organizations/octodemo/settings/codespaces) level and check out the [documentation](https://docs.github.com/en/enterprise-cloud@latest/codespaces/managing-codespaces-for-your-organization). :heart: 34 | 35 | ## Database schema management 36 | 37 | This repo uses [FlywayDB](https://flywaydb.org/) to manage the database schema in git. The schema and config files are all in the [sql](sql) directory. Docker Compose will run `flyway migrate` on the empty database, which is great for development and testing. FlywayDB will apply the [versioned](sql/migrations/versioned/) migrations, then seed a table with some data in a [repeatable](sql/migrations/repeatable/) migration. It's also a PR check run by a GitHub Actions workflow [here](.github/workflows/db.yml), and it will always print the logs in case it's needed for troubleshooting. 38 | 39 | ![images/flyway-logs.png](images/flyway-logs.png) 40 | 41 | An example of it failing because of an invalid schema change is in this [pull request](https://github.com/octodemo/webhook-demo/pull/23). Note how it's unable to merge because this is a required check. 42 | 43 | ## GitHub Advanced Security 44 | 45 | Don't use this as your main demo for security stuff. It's not supposed to replace more mature demos, only to highlight a couple things that have come up as edge cases where I didn't have an easy answer. This is a really REALLY simple Flask app. 46 | 47 | ### Code quality 48 | 49 | It has some "code smells" found through the `security-and-quality` query pack ([here](.github/workflows/codeql-analysis.yml#L48)), visible in the Security dashboard. It should find some unused imports and variables that are declared and not used. 50 | 51 | Not part of Advanced Security, but perhaps worth mentioning is that the [super-linter](https://github.com/github/super-linter) helps us maintain clean code by linting the Python code with [this workflow](.github/workflows/linter.yml). The [blog post](https://github.blog/2020-06-18-introducing-github-super-linter-one-linter-to-rule-them-all/) does a great job summarizing how awesome this Action truly is. 52 | 53 | ### Dependabot 54 | 55 | There's nothing much to see here other than some [pull requests](https://github.com/octodemo/webhook-demo/pulls?q=is%3Apr+is%3Aclosed+author%3Aapp%2Fdependabot) for out of date dependencies and an overview of the [security alerts](https://github.com/octodemo/webhook-demo/security/dependabot?q=is%3Aclosedt) for it. The [`dependabot.yml`](.github/dependabot.yml) file is a nice example of it checking for dependencies in multiple languages and the [documentation](https://docs.github.com/en/enterprise-cloud@latest/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) is excellent. 56 | 57 | ## Scrum 58 | 59 | There's some scrum-y things in this repo too. The first is an [issue template](.github/ISSUE_TEMPLATE/scrum.md) and [workflow](.github/workflows/scrum.yml) that opens a weekly planning issue to run sprint planning, using [this Action](https://github.com/imjohnbo/issue-bot). The schedule can be changed as needed in the workflow. The workflow is currently disabled because this is a demo repo and I don't want it to be annoying anyone. 60 | 61 | Next is a pair of shiny project boards. Both have a bit of automation built in to explore too. :) 62 | 63 | 1. The legacy Projects kanban-style board is [here](https://github.com/octodemo/webhook-demo/projects/1) 64 | 2. The new Projects board is [here](https://github.com/orgs/octodemo/projects/62) 65 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | flyway: 4 | image: flyway/flyway:8 5 | command: migrate 6 | volumes: 7 | - ./sql/migrations:/flyway/sql 8 | - ./sql:/flyway/conf 9 | depends_on: 10 | database: 11 | condition: service_healthy 12 | 13 | database: 14 | image: postgres:14 15 | restart: always 16 | env_file: 17 | ./sql/db.env 18 | ports: 19 | - 5432:5432 20 | healthcheck: 21 | test: ["CMD-SHELL", "pg_isready"] 22 | interval: 15s 23 | timeout: 5s 24 | retries: 10 25 | 26 | redis_cache: 27 | image: redis:6 28 | restart: always 29 | ports: 30 | - 6379:6379 31 | 32 | web: 33 | build: . 34 | ports: 35 | - 5000:5000 36 | depends_on: 37 | - redis_cache 38 | - database 39 | - flyway 40 | -------------------------------------------------------------------------------- /images/db-query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/octodemo/webhook-demo/b709d055f215545f4d1e6a390597a3a3076c958e/images/db-query.png -------------------------------------------------------------------------------- /images/flyway-logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/octodemo/webhook-demo/b709d055f215545f4d1e6a390597a3a3076c958e/images/flyway-logs.png -------------------------------------------------------------------------------- /images/port-forwarding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/octodemo/webhook-demo/b709d055f215545f4d1e6a390597a3a3076c958e/images/port-forwarding.png -------------------------------------------------------------------------------- /python/app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | This is a quick and dirty webhook receiver that will add an entry to a database 5 | based on the webhook payload. 6 | """ 7 | 8 | # Imports 9 | import psycopg2 10 | from datetime import datetime 11 | from flask import Flask, request, Response 12 | from redis import Redis 13 | from rq import Queue 14 | 15 | 16 | # Make instance of flask app 17 | app = Flask(__name__) 18 | 19 | # Make instance of redis queue 20 | q = Queue(connection=Redis(host="redis_cache", port=6379)) 21 | 22 | 23 | # Write the username and repo to the test table 24 | def write_to_db(username, repo): 25 | """ 26 | Writes the username, repo, and time to the test table 27 | """ 28 | conn = psycopg2.connect( 29 | "host=database dbname=webhooks user=postgres password=mysecretpassword" 30 | ) 31 | cur = conn.cursor() 32 | try: 33 | cur.execute( 34 | """ 35 | INSERT INTO test_webhook (username, target_repo, event_timestamp) 36 | VALUES (%s, %s, %s); 37 | """, 38 | (username, repo, datetime.now()), 39 | ) 40 | except psycopg2.Error as e: 41 | print("Error: %s", e) 42 | conn.commit() 43 | cur.close() 44 | conn.close() 45 | 46 | 47 | # Do the things 48 | @app.route("/webhook", methods=["POST"]) 49 | def respond(): 50 | username = request.json["repository"]["owner"]["login"] 51 | repo = request.json["repository"]["name"] 52 | write_to_db(username, repo) 53 | return Response(status=201) 54 | # TODO: add some business logic to do something else with the data, maybe 55 | # put it into redis to queue up something bigger? 56 | -------------------------------------------------------------------------------- /python/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==2.2.1 2 | psycopg2-binary==2.9.3 3 | redis==4.3.4 4 | rq==1.15.1 5 | -------------------------------------------------------------------------------- /sql/db.env: -------------------------------------------------------------------------------- 1 | POSTGRES_USER=postgres 2 | POSTGRES_PASSWORD=mysecretpassword 3 | POSTGRES_DB=webhooks -------------------------------------------------------------------------------- /sql/flyway.conf: -------------------------------------------------------------------------------- 1 | flyway.url=jdbc:postgresql://database/webhooks 2 | flyway.user=postgres 3 | flyway.password=mysecretpassword 4 | flyway.schemas=public 5 | flyway.connectRetries=60 6 | flyway.baselineOnMigrate=true 7 | flyway.sqlMigrationSuffixes=.pgsql,.sql -------------------------------------------------------------------------------- /sql/migrations/repeatable/R__seed-testwebhook-data.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO test_webhook (username, event_timestamp, target_repo) 2 | VALUES ( 3 | 'octodemo', 4 | '2022-02-22 20:22:06.473408', 5 | 'seed-data' 6 | ); 7 | INSERT INTO test_webhook (username, event_timestamp, target_repo) 8 | VALUES ( 9 | 'octodemo', 10 | '2022-02-22 20:23:17.073389', 11 | 'seed-data' 12 | ); 13 | INSERT INTO test_webhook (username, event_timestamp, target_repo) 14 | VALUES ( 15 | 'octodemo', 16 | '2022-02-22 20:23:25.557446', 17 | 'seed-data' 18 | ); 19 | INSERT INTO test_webhook (username, event_timestamp, target_repo) 20 | VALUES ( 21 | 'octodemo', 22 | '2022-04-20 20:23:09.915988', 23 | 'seed-data' 24 | ); 25 | INSERT INTO test_webhook (username, event_timestamp, target_repo) 26 | VALUES ( 27 | 'octodemo', 28 | '2022-04-20 20:33:05.863319', 29 | 'seed-data' 30 | ); 31 | INSERT INTO test_webhook (username, event_timestamp, target_repo) 32 | VALUES ( 33 | 'octodemo', 34 | '2022-04-20 20:33:05.863738', 35 | 'seed-data' 36 | ); -------------------------------------------------------------------------------- /sql/migrations/versioned/V01__init.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Create a table to store basic webhook payload info for demo 3 | */ 4 | 5 | CREATE TABLE public.test_webhook ( 6 | event_id INT GENERATED ALWAYS AS IDENTITY, 7 | username VARCHAR NOT NULL, 8 | event_timestamp TIMESTAMP NOT NULL, 9 | target_repo VARCHAR NOT NULL, 10 | PRIMARY KEY (event_id) 11 | ); 12 | -------------------------------------------------------------------------------- /sql/migrations/versioned/V02__add-collab-changelog.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Create a table for a collab changelog proof of concept 3 | */ 4 | 5 | CREATE TABLE public.collab_changelog ( 6 | event_id INT GENERATED ALWAYS AS IDENTITY, 7 | event_type TEXT NOT NULL, 8 | username VARCHAR NOT NULL, 9 | event_timestamp TIMESTAMP NOT NULL, 10 | target_repo VARCHAR NOT NULL, 11 | PRIMARY KEY (event_id) 12 | ); 13 | -------------------------------------------------------------------------------- /sql/migrations/versioned/V03__add-indexes.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Create indexes 3 | */ 4 | CREATE INDEX IF NOT EXISTS idx_test_webhook ON public.test_webhook (target_repo); 5 | CREATE INDEX IF NOT EXISTS idx_collabs ON public.collab_changelog (username); -------------------------------------------------------------------------------- /sql/queries/webhook-events.sql: -------------------------------------------------------------------------------- 1 | SELECT * 2 | FROM test_webhook; -------------------------------------------------------------------------------- /systemd/README.md: -------------------------------------------------------------------------------- 1 | # SystemD implementation 2 | 3 | ## Usage 4 | 5 | Pretty simple 6 | 7 | 1. Clone the repo and `cd` into the directory. 8 | 2. Start a vanilla redis server (or use one you already have). Docker Desktop is fine for a demo. 9 | 3. Make sure the port number needed in :point_up: is in `worker.py`. 10 | 4. (Mac only) Set the environment variable needed for process forking. 11 | 12 | ```shell 13 | export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES 14 | ``` 15 | 16 | 5. Start `worker.py` in the background. 17 | 6. Start `app.py` by launching flask. 18 | 19 | ```shell 20 | /usr/bin/env python3 -m flask run --host=0.0.0.0 --port=5000 21 | ``` 22 | 23 | 7. If needed for external connectivity, launch `ngrok` and map it to whatever you have `app.py` listening on. 24 | 8. Look at the logs of the Docker container once it's done to see a nice "Hello world" message customized to your username! :tada: 25 | 26 | There are 2 systemd unit files in this directory. They launch `worker.py` and `app.py` in the background once redis is up. You should check the name of the redis service in [app-queue.service](systemd/app-queue.service) to make sure it's correct to your distribution. 27 | -------------------------------------------------------------------------------- /systemd/app-queue.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Redis queue worker 3 | Requires=rh-redis5-redis.service 4 | 5 | [Service] 6 | Type=simple 7 | User=automation 8 | ExecStart=/opt/glue/worker.py 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /systemd/app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | This is a quick and dirty webhook receiver that will put a job into Redis for a 5 | Docker container app to run. 6 | """ 7 | 8 | # Imports 9 | import docker 10 | from flask import Flask, request, Response 11 | import requests 12 | from rq import Queue 13 | from systemd.worker import conn 14 | 15 | # Docker instance 16 | client = docker.from_env() 17 | 18 | # Make instance of flask app 19 | app = Flask(__name__) 20 | 21 | # Make instance of redis queue 22 | q = Queue(connection=conn) 23 | 24 | 25 | # How to run that Docker container 26 | def run_container(who_to_greet): 27 | """ 28 | Runs a Docker container that greets a named user 29 | """ 30 | container = client.containers.run( 31 | image="python:3.9", 32 | command="python3 -c \"print('Hello, {}!')\"".format(who_to_greet), 33 | detach=True, 34 | ) 35 | 36 | 37 | # Do the things 38 | @app.route("/webhook", methods=["POST"]) 39 | def respond(): 40 | who_to_greet = request.json["repository"]["owner"]["login"] 41 | job = q.enqueue_call(func=run_container, args=(), result_ttl=5000) 42 | return Response(status=201) 43 | -------------------------------------------------------------------------------- /systemd/app.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Docker glue app 3 | Requires=app-queue.service 4 | 5 | [Service] 6 | Type=simple 7 | User=automation 8 | WorkingDirectory=/opt/glue 9 | ExecStart=/usr/bin/env python3 -m flask run --host=0.0.0.0 --port=5000 10 | Restart=always 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /systemd/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==2.1.2 2 | psycopg2-binary==2.9.3 3 | redis==4.3.4 4 | rq==1.10.1 5 | -------------------------------------------------------------------------------- /systemd/worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import redis 4 | from rq import Worker, Queue, Connection 5 | 6 | listen = ["default"] 7 | 8 | redis_url = "redis://localhost:16379" 9 | 10 | conn = redis.from_url(redis_url) 11 | 12 | if __name__ == "__main__": 13 | with Connection(conn): 14 | worker = Worker(list(map(Queue, listen))) 15 | worker.work() 16 | --------------------------------------------------------------------------------