├── .gitignore ├── .travis.yml ├── Dockerfile ├── Dockerfile-docker ├── LICENSE ├── README.md ├── docker-requirements.txt ├── manifest-docker.yml ├── manifest.yml ├── requirements.txt ├── sample.yml ├── src ├── actions │ ├── __init__.py │ ├── action_docker.py │ ├── action_docker_compose.py │ ├── action_docker_swarm.py │ ├── action_evaluate.py │ ├── action_execute.py │ ├── action_github_verify.py │ ├── action_http.py │ ├── action_log.py │ ├── action_metrics.py │ ├── action_sleep.py │ └── replay_helper.py ├── app.py ├── endpoints.py ├── server.py └── util.py └── tests ├── github └── webhook.json ├── imports ├── actions_to_load.py ├── invalid.py ├── test1 │ └── action.py └── test2 │ └── action.py ├── integrationtest_helper.py ├── it_docker.py ├── it_docker_compose.py ├── it_docker_swarm.py ├── it_execute.py ├── it_import.py ├── it_log.py ├── it_metrics.py ├── test_action.py ├── test_docker_action.py ├── test_docker_compose_action.py ├── test_docker_swarm_action.py ├── test_execute_action.py ├── test_github_verify_action.py ├── test_http_action.py ├── test_import.py ├── test_metrics_action.py ├── test_replay_helper.py ├── test_server.py └── unittest_helper.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.iml 3 | .idea/ 4 | .coverage 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | dist: xenial 3 | python: 4 | - '2.7' 5 | - '3.4' 6 | - '3.5' 7 | - '3.6' 8 | - '3.7' 9 | 10 | services: 11 | - docker 12 | install: skip 13 | script: 14 | # prepare dependencies 15 | - pip install -r requirements.txt 16 | - pip install -r docker-requirements.txt 17 | - pip install coveralls 18 | # prepare reporter 19 | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter 20 | - chmod +x ./cc-test-reporter 21 | - ./cc-test-reporter before-build 22 | # python tests 23 | - PYTHONPATH=src python -m coverage run --branch --source=src -m unittest discover -s tests -v 24 | # coverage reports 25 | - coveralls 26 | - python -m coverage report -m 27 | - python -m coverage xml 28 | - | 29 | if [[ "$(python --version 2>&1)" = *2.7* ]]; then 30 | coveralls || exit 0 31 | ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT || exit 0 32 | fi 33 | 34 | jobs: 35 | include: 36 | - &integration-stage 37 | stage: integration 38 | script: 39 | - pip install -r requirements.txt 40 | - pip install -r docker-requirements.txt 41 | - PYTHONPATH=tests python -m unittest discover -s tests -v -p it_*.py 42 | env: DIND_VERSION=18.09 43 | 44 | - <<: *integration-stage 45 | env: DIND_VERSION=18.02 46 | 47 | - <<: *integration-stage 48 | env: DIND_VERSION=17.12 49 | 50 | - <<: *integration-stage 51 | env: DIND_VERSION=17.09 52 | 53 | - <<: *integration-stage 54 | env: DIND_VERSION=17.06 55 | 56 | - <<: *integration-stage 57 | env: DIND_VERSION=1.13 58 | 59 | - &deploy-stage 60 | stage: deploy 61 | if: branch = master AND type = push 62 | script: 63 | - docker run --rm --privileged multiarch/qemu-user-static:register --reset 64 | - | 65 | docker build -t webhook-proxy:$DOCKER_TAG \ 66 | --build-arg BASE_IMAGE=$BASE_IMAGE \ 67 | --build-arg GIT_COMMIT=$TRAVIS_COMMIT \ 68 | --build-arg BUILD_TIMESTAMP=$(date +%s) \ 69 | -f Dockerfile${FLAVOR} \ 70 | . 71 | - docker tag webhook-proxy:$DOCKER_TAG rycus86/webhook-proxy:$DOCKER_TAG 72 | - echo ${DOCKER_PASSWORD} | docker login --username "rycus86" --password-stdin 73 | after_success: 74 | - docker push rycus86/webhook-proxy:$DOCKER_TAG 75 | env: 76 | - DOCKER_TAG=amd64 77 | - BASE_IMAGE=alpine 78 | 79 | - <<: *deploy-stage 80 | env: 81 | - DOCKER_TAG=armhf 82 | - BASE_IMAGE=rycus86/armhf-alpine-qemu 83 | 84 | - <<: *deploy-stage 85 | env: 86 | - DOCKER_TAG=aarch64 87 | - BASE_IMAGE=rycus86/arm64v8-alpine-qemu 88 | 89 | - <<: *deploy-stage 90 | env: 91 | - FLAVOR=-docker 92 | - DOCKER_TAG=amd64-docker 93 | - BASE_IMAGE=alpine 94 | 95 | - <<: *deploy-stage 96 | env: 97 | - FLAVOR=-docker 98 | - DOCKER_TAG=armhf-docker 99 | - BASE_IMAGE=rycus86/armhf-alpine-qemu 100 | 101 | - <<: *deploy-stage 102 | env: 103 | - FLAVOR=-docker 104 | - DOCKER_TAG=aarch64-docker 105 | - BASE_IMAGE=rycus86/arm64v8-alpine-qemu 106 | 107 | - stage: manifest 108 | if: branch = master AND type = push 109 | script: 110 | - echo ${DOCKER_PASSWORD} | docker login --username "rycus86" --password-stdin 111 | - curl -fsSL https://github.com/estesp/manifest-tool/releases/download/v0.7.0/manifest-tool-linux-amd64 > ./manifest-tool 112 | - chmod +x ./manifest-tool 113 | - ./manifest-tool push from-spec manifest.yml 114 | - ./manifest-tool push from-spec manifest-docker.yml 115 | 116 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE="alpine" 2 | 3 | FROM $BASE_IMAGE 4 | 5 | LABEL maintainer "Viktor Adam " 6 | 7 | RUN apk --no-cache add python py2-pip 8 | 9 | ADD requirements.txt /tmp/requirements.txt 10 | RUN pip install -r /tmp/requirements.txt 11 | 12 | RUN adduser -S webapp 13 | USER webapp 14 | 15 | ADD src /app 16 | WORKDIR /app 17 | 18 | ENV PYTHONUNBUFFERED=1 19 | 20 | ENTRYPOINT [ "python", "app.py"] 21 | 22 | # add app info as environment variables 23 | ARG GIT_COMMIT 24 | ENV GIT_COMMIT $GIT_COMMIT 25 | ARG BUILD_TIMESTAMP 26 | ENV BUILD_TIMESTAMP $BUILD_TIMESTAMP 27 | -------------------------------------------------------------------------------- /Dockerfile-docker: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE="alpine" 2 | 3 | FROM $BASE_IMAGE 4 | 5 | LABEL maintainer "Viktor Adam " 6 | 7 | RUN apk --no-cache add python py2-pip 8 | 9 | ADD requirements.txt /tmp/requirements.txt 10 | RUN pip install -r /tmp/requirements.txt 11 | 12 | ADD docker-requirements.txt /tmp/docker-requirements.txt 13 | RUN pip install -r /tmp/docker-requirements.txt 14 | 15 | ADD src /app 16 | WORKDIR /app 17 | 18 | ENV PYTHONUNBUFFERED=1 19 | 20 | ENTRYPOINT [ "python", "app.py"] 21 | 22 | # add app info as environment variables 23 | ARG GIT_COMMIT 24 | ENV GIT_COMMIT $GIT_COMMIT 25 | ARG BUILD_TIMESTAMP 26 | ENV BUILD_TIMESTAMP $BUILD_TIMESTAMP 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Viktor Adam 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Webhook Proxy 2 | 3 | A simple `Python` [Flask](http://flask.pocoo.org) *REST* server to 4 | accept *JSON* webhooks and run actions as a result. 5 | 6 | [![Build Status](https://travis-ci.org/rycus86/webhook-proxy.svg?branch=master)](https://travis-ci.org/rycus86/webhook-proxy) 7 | [![Build Status](https://img.shields.io/docker/build/rycus86/webhook-proxy.svg)](https://hub.docker.com/r/rycus86/webhook-proxy) 8 | [![Coverage Status](https://coveralls.io/repos/github/rycus86/webhook-proxy/badge.svg?branch=master)](https://coveralls.io/github/rycus86/webhook-proxy?branch=master) 9 | [![Code Climate](https://codeclimate.com/github/rycus86/webhook-proxy/badges/gpa.svg)](https://codeclimate.com/github/rycus86/webhook-proxy) 10 | 11 | ## Usage 12 | 13 | To start the server, run: 14 | 15 | ```shell 16 | python app.py [server.yml] 17 | ``` 18 | 19 | If the parameter is omitted, the configuration file is expected to be `server.yml` 20 | in the current directory (see configuration details below). 21 | 22 | The application can be run using Python 2 or 3. 23 | 24 | ## Configuration 25 | 26 | The configuration for the server and its endpoints is described in a *YAML* file. 27 | 28 | A short example: 29 | 30 | ```yaml 31 | server: 32 | host: '127.0.0.1' 33 | port: '5000' 34 | 35 | endpoints: 36 | - /endpoint/path: 37 | method: 'POST' 38 | 39 | headers: 40 | X-Sender: 'regex for X-Sender HTTP header' 41 | 42 | body: 43 | project: 44 | name: 'regex for project.name in the JSON payload' 45 | items: 46 | name: '^example_[0-9]+' 47 | 48 | actions: 49 | - log: 50 | message: 'Processing {{ request.path }} ...' 51 | ``` 52 | 53 | ### server 54 | 55 | The `server` section defines settings for the HTTP server receiving the webhook requests. 56 | 57 | | key | description | default | required | 58 | | --- | ----------- | ------- | -------- | 59 | | host | The host name or address for the server to listen on | `127.0.0.1` | no | 60 | | port | The port number to accept incoming connections on | `5000` | no | 61 | | imports | Python modules (as list of file paths) to import for registering additional actions | `None` | no | 62 | 63 | Set the `host` to `0.0.0.0` to accept connections from any hosts. 64 | 65 | The `imports` property has to be a `list` and should point to the `.py` files. 66 | They will be copied temporarily into the `TMP_IMPORT_DIR` folder (`/tmp` by default, 67 | override with the environment variable) then renamed to a random filename and 68 | finally imported as a module so that we can load multiple modules with the same 69 | filename from different paths. 70 | Also note that because of this, we cannot rely on the module `__name__`. 71 | 72 | ### endpoints 73 | 74 | The `endpoints` section configures the list of endpoints exposed on the server. 75 | 76 | Each endpoint supports the following configuration (all optional): 77 | 78 | | key | description | default | 79 | | --- | ----------- | ------- | 80 | | method | HTTP method supported on the endpoint | `POST` | 81 | | headers | HTTP header validation rules as a dictionary of names to regular expressions | `empty` | 82 | | body | Validation rules for the JSON payload in the request body | `empty` | 83 | | async | Execute the action asynchronously | `False` | 84 | | actions | List of actions to execute for valid requests. | `empty` | 85 | 86 | The message body validation supports lists too, the `project.item.name` in the example would accept 87 | `{"project": {"name": "...", "items": [{"name": "example_12"}]}}` as an incoming body. 88 | 89 | ### actions 90 | 91 | Action definitions support variables for most properties using _Jinja2_ templates. 92 | By default, these receive the following objects in their context: 93 | 94 | - `request` : the incoming _Flask_ request being handled 95 | - `timestamp` : the Epoch timestamp as `time.time()` 96 | - `datetime` : human-readable timestamp as `time.ctime()` 97 | - `own_container_id`: the ID of the container the app is running in or otherwise `None` 98 | - `read_config`: helper for reading configuration parameters from key-value files 99 | or environment variables and also full configuration files (certificates for example), 100 | see [docker_helper](https://github.com/rycus86/docker_helper) for more information and usage 101 | - `error(..)` : a function with an optional `message` argument to raise errors when evaluating templates 102 | - `context` : a thread-local object for passing information from one action to another 103 | 104 | _Jinja2_ does not let you execute code in the templates directly, so to use 105 | the `error` and `context` objects you need to do something like this: 106 | 107 | ``` 108 | {% if 'something' is 'wrong' %} 109 | 110 | {# treat it as literal (will display None) #} 111 | {{ error('Something is not right }} 112 | 113 | {# or use the assignment block with a dummy variable #} 114 | {% set _ = error() %} 115 | 116 | {% else %} 117 | 118 | {% set _ = context.set('verdict', 'All good') %} 119 | 120 | {% endif %} 121 | 122 | ## In another action's template: 123 | 124 | Previously we said {{ context.verdict }} 125 | ``` 126 | 127 | The following actions are supported (given their dependencies are met). 128 | 129 | #### log 130 | 131 | The `log` action prints a message on the standard output. 132 | 133 | | key | description | default | templated | required | 134 | | --- | ----------- | ------- | --------- | -------- | 135 | | message | The log message template | `Processing {{ request.path }} ...` | yes | no | 136 | 137 | #### eval 138 | 139 | The `eval` action evaluates a _Jinja2_ template block. 140 | This can be useful to work with objects passed through from previous actions using 141 | the `context` for example. 142 | 143 | | key | description | default | templated | required | 144 | | --- | ----------- | ------- | --------- | -------- | 145 | | block | The template block to evaluate | | yes | yes | 146 | 147 | #### execute 148 | 149 | The `execute` action executes an external command using `subprocess.check_output`. 150 | The output (string) of the invocation is passed to the _Jinja2_ template as `result`. 151 | 152 | | key | description | default | templated | required | 153 | | --- | ----------- | ------- | --------- | -------- | 154 | | command | The command to execute as a string or list | | no | yes | 155 | | shell | Configuration for the shell used (see below) | `True` | no | no | 156 | | output | Output template for printing the result on the standard output | `{{ result }}` | yes | no | 157 | 158 | The `shell` parameter accepts: 159 | 160 | - boolean : whether to use the default shell or run the command directly 161 | - string : a shell command that supports `-c` 162 | - list : for the complete shell prefix, like `['bash', '-c']` 163 | 164 | #### http 165 | 166 | The `http` action sends an HTTP request to a target and requires the __requests__ Python module. 167 | The HTTP response object (from the _requests_ module) is available to the 168 | _Jinja2_ template as `response`. 169 | 170 | | key | description | default | templated | required | 171 | | --- | ----------- | ------- | --------- | -------- | 172 | | target | The target endpoint as `://[:][/]` | | no | yes | 173 | | method | The HTTP method to use for the request | `POST` | no | no | 174 | | headers | The HTTP headers (as dictionary) to add to the request | `empty` | yes | no | 175 | | json | Whether to dump `body` YAML subtree as json | `False` | no | no | 176 | | body | The HTTP body to send with the request. String (or YAML tree, if `json` is `True`) | `empty` | yes | no | 177 | | output | Output template for printing the response on the standard output | `HTTP {{ response.status_code }} : {{ response.content }}` | yes | no | 178 | | verify | SSL certificate check behavior, set to `False` to ignore certificate errors (not recomended) or provide path to custom CA | `True` | no | no 179 | 180 | #### github-verify 181 | 182 | The `github-verify` is a convenience action to validate incoming _GitHub_ webhooks. 183 | It requires the webhook to be signed with a secret. 184 | 185 | | key | description | default | templated | required | 186 | | --- | ----------- | ------- | --------- | -------- | 187 | | secret | The webhook secret configured in _GitHub_ | | yes | yes | 188 | | output | Output template for printing a message on the standard output | `{{ result }}` | yes | no | 189 | 190 | The action will raise an `ActionInvocationException` on failure. 191 | If that happens, the actions defined after this one will not be executed. 192 | 193 | #### docker 194 | 195 | The `docker` action interacts with the _Docker_ daemon and requires the __docker__ Python module. 196 | It also needs access to the _Docker_ UNIX socket at `/var/run/docker.sock`. 197 | 198 | The action supports _exactly one_ invocation on the _Docker_ client (per action). 199 | Invocations (or properties) are keys starting with `$` in the configuration, 200 | for example listing the containers would use `$containers` with `$list` as a sub-item. 201 | The result of the invocation (as an object from the _Docker_ client) is available to the 202 | _Jinja2_ templates as `result`. 203 | 204 | | key | description | default | templated | required | 205 | | --- | ----------- | ------- | --------- | -------- | 206 | | `$invocation` | Exactly one invocation supported by the _Docker_ client (see examples below) | | yes (for values) | yes | 207 | | output | Output template for printing the result on the standard output | `{{ result }}` | yes | no | 208 | 209 | Examples: 210 | 211 | ```yaml 212 | ... 213 | actions: 214 | - docker: 215 | $containers: 216 | $list: 217 | filters: 218 | name: '{{ request.json.repo.name }}' 219 | output: | 220 | Containers matching "{{ request.json.name }}": 221 | {% for container in result %} 222 | - {{ container.name }} @ {{ container.short_id }} 223 | {% endfor %} 224 | 225 | - docker: 226 | $info: 227 | output: 'Docker version: {{ result.ServerVersion }} on {{ result.OperatingSystem }}' 228 | 229 | - docker: 230 | $images: 231 | $pull: 232 | repository: '{{ request.json.namespace }}/{{ request.json.name }}' 233 | tag: '{{ request.json.get('tag', 'latest') }}' 234 | 235 | - docker: 236 | $containers: 237 | $run: 238 | image: 'alpine' 239 | command: 'echo "Hello {{ request.json.message }}!"' 240 | remove: true 241 | ``` 242 | 243 | #### docker-compose 244 | 245 | The `docker-compose` action interacts with _Docker Compose_ and requires the `docker-compose` Python module. 246 | 247 | The action supports _exactly one_ invocation on the _Docker Compose_ project (per action). 248 | The invocations are in the same format as with the `docker` action and the 249 | result is available for _Jinja2_ templates as `result` that is the return object 250 | from the _Docker Compose_ invocation. 251 | 252 | | key | description | default | templated | required | 253 | | --- | ----------- | ------- | --------- | -------- | 254 | | project\_name | The _Compose_ project name | | no | yes | 255 | | directory | The directory of the _Compose_ project | | no | yes | 256 | | composefile | The filename of the _Composefile_ within the directory | `docker-compose.yml` | no | no | 257 | | `$invocation` | Exactly one invocation supported by the _Docker Compose_ client (see examples below) | | yes (for values) | yes | 258 | | output | Output template for printing the result on the standard output | `{{ result }}` | yes | no | 259 | 260 | Examples: 261 | 262 | ```yaml 263 | ... 264 | actions: 265 | - docker-compose: 266 | project_name: 'web' 267 | directory: '/opt/projects/web' 268 | $get_services: 269 | output: | 270 | Compose services: 271 | {% for service in result %} 272 | - service: {{ service.name }} 273 | {% endfor %} 274 | 275 | - docker-compose: 276 | project_name: 'backend' 277 | directory: '/opt/projects/compose_project' 278 | $up: 279 | detached: true 280 | output: | 281 | Containers started: 282 | {% for container in result %} 283 | - {{ container.name }} 284 | {% endfor %} 285 | 286 | - docker-compose: 287 | project_name: 'backend' 288 | directory: '/opt/projects/compose_project' 289 | $down: 290 | remove_image_type: false 291 | include_volumes: true 292 | output: 'Compose project stopped' 293 | ``` 294 | 295 | #### docker-swarm (deprecated) 296 | 297 | *Since the merge of [docker-py#1807](https://github.com/docker/docker-py/pull/1807), 298 | this convenience action is no longer necessary. 299 | The official Docker SDK can handle Swarm service updates nicely.* 300 | 301 | The `docker-swarm` action exposes convenience _Docker_ actions for _Swarm_ related operations 302 | that might require quite a bit of manual work to replicate with the `docker` action. 303 | 304 | The action supports _exactly one_ invocation (per action) on its own action object. 305 | The invocations are in the same format as with the `docker` action and the available 306 | ones are: 307 | 308 | - `$restart`: restarts (force updates) a _Swarm_ service matching the `service_id` parameter 309 | (this can be a service name or ID) 310 | - `$scale`: updates a __replicated__ service matched by `service_id` to have `replicas` number 311 | of instances 312 | - `$update`: updates a service matched by `service_id` 313 | 314 | The update invocation uses the current service spec and updates them with the following 315 | parameters if they are present: 316 | 317 | - `image`, `command`, `args`, `hostname`, `env`, `dir`, `user`, `mounts`, `stop_grace_period`, `tty` 318 | for the container specification (see `docker.types.services.ContainerSpec`) 319 | - `container_labels` for container labels 320 | - `secrets` for secret references as a list of dictionaries 321 | (see `docker.types.services.SecretReference`) 322 | - `resources`, `restart_policy`, `placement` for the task template specification 323 | (see `docker.types.services.TaskTemplate`) 324 | - `labels` for service labels 325 | - `replicas` for number of instances for __replicated__ services 326 | - `update_config` for the service update configuration 327 | (see `docker.types.services.UpdateConfig`) 328 | - `networks` as a list of network IDs or names 329 | - `endpoint_spec` for the endpoint specification 330 | (see `docker.types.services.EndpointSpec`) 331 | 332 | The result of the invocations will be the service object if the service update was successful. 333 | 334 | | key | description | default | templated | required | 335 | | --- | ----------- | ------- | --------- | -------- | 336 | | `$invocation` | Exactly one invocation supported by the action (see examples below) | | yes (for values) | yes | 337 | | output | Output template for printing the result on the standard output | `{{ result }}` | yes | no | 338 | 339 | Examples: 340 | 341 | ```yaml 342 | ... 343 | actions: 344 | - docker-swarm: 345 | $restart: 346 | service_id: '{{ request.json.service }}' 347 | output: > 348 | Service restarted: {{ result.name }} 349 | 350 | - docker-swarm: 351 | $scale: 352 | service_id: '{{ request.json.service }}' 353 | replicas: '{{ request.json.replicas }}' 354 | 355 | - docker-swarm: 356 | $update: 357 | service_id: '{{ request.json.service }}' 358 | command: '{{ request.json.command }}' 359 | labels: 360 | label_1: 'sample' 361 | label_2: '{{ request.json.label }}' 362 | ``` 363 | 364 | #### sleep 365 | 366 | The `sleep` action waits for a given time period. 367 | It may be useful if an action has executed something asynchronous and another action 368 | relies on the outcome that would only happen a little bit later. 369 | 370 | | key | description | default | templated | required | 371 | | --- | ----------- | ------- | --------- | -------- | 372 | | seconds | Number of seconds to sleep for | | yes | yes | 373 | | message | The message template to print on the standard output | `Waiting {{ seconds }} seconds before continuing ...` | yes | no | 374 | 375 | #### metrics 376 | 377 | The application exposes [Prometheus](https://prometheus.io/) metrics 378 | about the number of calls and the execution times of the endpoints. 379 | 380 | The `metrics` action registers a new metric in addition that 381 | tracks the entire execution of the endpoint (not only the action). 382 | Apart from the optional `output` configuration it has to contain 383 | one metric registration from the table below. 384 | 385 | | key | description | default | templated | required | 386 | | --- | ----------- | ------- | --------- | -------- | 387 | | histogram | Registers a Histogram | | yes (labels) | yes (one) | 388 | | summary | Registers a Summary | | yes (labels) | yes (one) | 389 | | gauge | Registers a Gauge | | yes (labels) | yes (one) | 390 | | counter | Registers a Counter | | yes (labels) | yes (one) | 391 | | message | The message template to print on the standard output | `Waiting {{ seconds }} seconds before continuing ...` | yes | no | 392 | | output | Output template for printing the result on the standard output | `Tracking metrics: {{ metric }}` | yes | no | 393 | 394 | Note that the `name` configuration is mandatory for metrics. 395 | Also note that metric labels are accepted as a dictionary where 396 | the value can be templated and will be evaluated within 397 | the Flask request context. 398 | The templates also have access to the Flask `response` object 399 | (with the `gauge` being the exception as it is also evaluated 400 | before the request to track in-progress executions). 401 | 402 | For example: 403 | 404 | ```yaml 405 | ... 406 | actions: 407 | - metrics: 408 | gauge: 409 | name: requests_in_progress 410 | help: Tracks current requests in progress 411 | 412 | - metrics: 413 | summary: 414 | name: request_summary 415 | labels: 416 | path: '{{ request.path }}' 417 | ... 418 | ``` 419 | 420 | ## Docker 421 | 422 | The application can be run in *Docker* containers using images based on *Alpine Linux* 423 | for 3 processor architectures with the following tags: 424 | 425 | - `latest`: for *x86* hosts 426 | [![Layers](https://images.microbadger.com/badges/image/rycus86/webhook-proxy.svg)](https://microbadger.com/images/rycus86/webhook-proxy "Get your own image badge on microbadger.com") 427 | - `armhf`: for *32-bits ARM* hosts 428 | [![Layers](https://images.microbadger.com/badges/image/rycus86/webhook-proxy:armhf.svg)](https://microbadger.com/images/rycus86/webhook-proxy:armhf "Get your own image badge on microbadger.com") 429 | - `aarch64`: for *64-bits ARM* hosts 430 | [![Layers](https://images.microbadger.com/badges/image/rycus86/webhook-proxy:aarch64.svg)](https://microbadger.com/images/rycus86/webhook-proxy:aarch64 "Get your own image badge on microbadger.com") 431 | 432 | `latest` is auto-built on [Docker Hub](https://hub.docker.com/r/rycus86/webhook-proxy) 433 | while the *ARM* builds are uploaded from [Travis](https://travis-ci.org/rycus86/webhook-proxy). 434 | 435 | The containers run as a non-root user. 436 | 437 | To start the server: 438 | 439 | ```shell 440 | docker run -d --name=webhook-proxy -p 5000:5000 \ 441 | -v $PWD/server.yml:/etc/conf/webhook-server.yml \ 442 | rycus86/webhook-proxy:latest \ 443 | /etc/conf/webhook-server.yml 444 | ``` 445 | 446 | Or put the configuration file at the default location: 447 | 448 | ```shell 449 | docker run -d --name=webhook-proxy -p 5000:5000 \ 450 | -v $PWD/server.yml:/app/server.yml \ 451 | rycus86/webhook-proxy:latest 452 | ``` 453 | 454 | There are 3 more tags available for images that can use the `docker` and `docker-compose` 455 | actions which are running as `root` user: 456 | 457 | - `docker`: for *x86* hosts 458 | [![Layers](https://images.microbadger.com/badges/image/rycus86/webhook-proxy:docker.svg)](https://microbadger.com/images/rycus86/webhook-proxy:docker "Get your own image badge on microbadger.com") 459 | - `armhf-docker`: for *32-bits ARM* hosts 460 | [![Layers](https://images.microbadger.com/badges/image/rycus86/webhook-proxy:armhf-docker.svg)](https://microbadger.com/images/rycus86/webhook-proxy:armhf-docker "Get your own image badge on microbadger.com") 461 | - `aarch64-docker`: for *64-bits ARM* hosts 462 | [![Layers](https://images.microbadger.com/badges/image/rycus86/webhook-proxy:aarch64-docker.svg)](https://microbadger.com/images/rycus86/webhook-proxy:aarch64-docker "Get your own image badge on microbadger.com") 463 | 464 | Each of these are built on [Travis](https://travis-ci.org/rycus86/webhook-proxy) and 465 | pushed to [Docker Hub](https://hub.docker.com/r/rycus86/webhook-proxy). 466 | 467 | To run these, the _Docker_ daemon's UNIX socket needs to be mounted into the container 468 | too apart from the configuration file: 469 | 470 | ```shell 471 | docker run -d --name=webhook-proxy -p 5000:5000 \ 472 | -v $PWD/server.yml:/app/server.yml \ 473 | -v /var/run/docker.sock:/var/run/docker.sock:ro \ 474 | rycus86/webhook-proxy:docker 475 | ``` 476 | 477 | In _Docker Compose_ on a 64-bit ARM machine the service definition could look like this: 478 | 479 | ```yaml 480 | version: '2' 481 | services: 482 | 483 | webhooks: 484 | image: rycus86/webhook-proxy:aarch64 485 | ports: 486 | - 8080:5000 487 | volumes: 488 | - ./webhook-server.yml:/app/server.yml:ro 489 | ``` 490 | 491 | ## Examples 492 | 493 | Have a look at the [sample.yml](https://github.com/rycus86/webhook-proxy/blob/master/sample.yml) included in this repo to get 494 | a better idea of the configuration. 495 | 496 | You can also find some examples with short explanation below. 497 | 498 | - An externally available server listening on port `7000` and printing 499 | details about a _GitHub_ push webhook 500 | 501 | ```yaml 502 | server: 503 | host: '0.0.0.0' 504 | port: '7000' 505 | 506 | endpoints: 507 | - /github: 508 | method: 'POST' 509 | 510 | headers: 511 | X-GitHub-Delivery: '^[0-9a-f\-]+$' 512 | X-GitHub-Event: 'push' 513 | 514 | body: 515 | ref: 'refs/heads/.+' 516 | before: '^[0-9a-f]{40}' 517 | after: '^[0-9a-f]{40}' 518 | repository: 519 | id: '^[0-9]+$' 520 | full_name: 'sample/.+' 521 | owner: 522 | email: '.+@.+\..+' 523 | commits: 524 | id: '^[0-9a-f]{40}' 525 | message: '.+' 526 | author: 527 | name: '.+' 528 | added: '^(src/.+)?' 529 | removed: '^(src/.+)?' 530 | pusher: 531 | name: '.+' 532 | email: '.+@.+\..+' 533 | 534 | actions: 535 | - log: 536 | message: | 537 | Received a GitHub push from the {{ request.json.repository.full_name }} repo: 538 | - Pushed by {{ request.json.pusher.name }} <{{ request.json.pusher.email }}> 539 | - Commits included: 540 | {% for commit in request.json.commits %} 541 | + {{ commit.id }} 542 | + {{ commit.committer.name }} at {{ commit.timestamp }} 543 | + {{ commit.message }} 544 | 545 | {% endfor %} 546 | Check this change out at {{ request.json.compare }} 547 | 548 | # verify the webhook signature 549 | - github-verify: 550 | secret: '{{ read_config("GITHUB_SECRET", "/var/run/secrets/github") }}' 551 | ``` 552 | 553 | The validators for the `/github` endpoint require that 554 | the `X-GitHub-Delivery` header is hexadecimal separated by dashes and 555 | the `X-GitHub-Event` header has the `push` value. 556 | The event also has to come from one of the repos under the `sample` namespace. 557 | Some of the commit hashes are checked that they are 40 character long 558 | hexadecimal values and the commit author's name has to be non-empty. 559 | The `commits` field is actually a list in the _GitHub_ webhook so 560 | the validation is applied to each commit data individually. 561 | The `added` and `removed` checks for example accept if the commit has 562 | not added or removed anything but if it did it has to be in the `src` folder. 563 | 564 | For valid webhooks the repository's name, the pushers name and emails are 565 | printed to the standard output followed by the ID, committer name, timestamp 566 | and message of each commit in the push. 567 | The last line displays the URL for the _GitHub_ compare page for the change. 568 | 569 | 570 | For more information about using the _Jinja2_ templates have a look 571 | at the [official documentation](http://jinja.pocoo.org). 572 | 573 | The `github-verify` action will make sure that the webhook is signed as appropriate. 574 | The _secret_ for this is read either from the `/var/run/secrets/github` file or 575 | the `GITHUB_SECRET` environment variable. 576 | 577 | > In case it is in a file, that file should contain key-value pairs, like `GITHUB_SECRET=TopSecret` 578 | 579 | - Update a _Docker Compose_ project on image changes 580 | 581 | Let's assume we have a _Compose_ project with a few services. 582 | When their image is updated in _Docker Hub_ we want to pull it 583 | and get _Compose_ to restart the related containers. 584 | 585 | ```yaml 586 | server: 587 | host: '0.0.0.0' 588 | port: '5000' 589 | 590 | endpoints: 591 | - /webhook/dockerhub: 592 | method: 'POST' 593 | 594 | body: 595 | repository: 596 | repo_name: 'somebody/.+' 597 | owner: 'somebody' 598 | push_data: 599 | tag: 'latest' 600 | 601 | actions: 602 | - docker: 603 | $containers: 604 | $list: 605 | output: | 606 | {% for container in result if request.json.repository.repo_name in container.image.tags %} 607 | Found {{ container.name }} with {{ container.image }} 608 | {% else %} 609 | {% set _ = error('No containers found using %s'|filter(request.json.repo_name)) %} 610 | {% endfor %} 611 | - docker: 612 | $images: 613 | $pull: 614 | repository: '{{ request.json.repo_name }}' 615 | tag: '{{ request.json.tag }}' 616 | - docker-compose: 617 | project_name: 'autoupdate' 618 | directory: '/var/compose/project' 619 | $up: 620 | detached: true 621 | output: | 622 | Containers affected: 623 | {% for container in result %} 624 | {{ container.name }} <{{ container.short_id }}> 625 | ``` 626 | 627 | The `/webhook/dockerhub` endpoint will accept webhooks from `somebody/*` repos 628 | when an image's `latest` tag is updated. 629 | First a `docker` action checks that we already have containers running that 630 | use the image then another `docker` action pulls the updated image and 631 | finally the `docker-compose` action applies the changes by restarting 632 | any related containers. 633 | -------------------------------------------------------------------------------- /docker-requirements.txt: -------------------------------------------------------------------------------- 1 | docker 2 | docker-compose 3 | -------------------------------------------------------------------------------- /manifest-docker.yml: -------------------------------------------------------------------------------- 1 | image: rycus86/webhook-proxy:docker 2 | manifests: 3 | - 4 | image: rycus86/webhook-proxy:amd64-docker 5 | platform: 6 | architecture: amd64 7 | os: linux 8 | - 9 | image: rycus86/webhook-proxy:armhf-docker 10 | platform: 11 | architecture: arm 12 | variant: v7 13 | os: linux 14 | - 15 | image: rycus86/webhook-proxy:aarch64-docker 16 | platform: 17 | architecture: arm64 18 | variant: v8 19 | os: linux 20 | -------------------------------------------------------------------------------- /manifest.yml: -------------------------------------------------------------------------------- 1 | image: rycus86/webhook-proxy:latest 2 | manifests: 3 | - 4 | image: rycus86/webhook-proxy:amd64 5 | platform: 6 | architecture: amd64 7 | os: linux 8 | - 9 | image: rycus86/webhook-proxy:armhf 10 | platform: 11 | architecture: arm 12 | variant: v7 13 | os: linux 14 | - 15 | image: rycus86/webhook-proxy:aarch64 16 | platform: 17 | architecture: arm64 18 | variant: v8 19 | os: linux 20 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | six 2 | flask 3 | jinja2 4 | pyYAML 5 | requests 6 | prometheus-client 7 | prometheus-flask-exporter 8 | docker-helper 9 | certifi>=2022.12.7 # not directly required, pinned by Snyk to avoid a vulnerability 10 | -------------------------------------------------------------------------------- /sample.yml: -------------------------------------------------------------------------------- 1 | server: 2 | host: '0.0.0.0' 3 | port: '9999' 4 | 5 | endpoints: 6 | - /echo: 7 | method: 'POST' 8 | 9 | actions: 10 | - log: 11 | message: 'Incoming request on {{ request.path }}: 12 | request headers: 13 | {{ request.headers }} 14 | request payload: 15 | {{ request.json }}' 16 | 17 | - /test: 18 | method: 'POST' 19 | 20 | actions: 21 | - log: 22 | message: 'Running tests at {{ timestamp }} ...' 23 | - docker-compose: 24 | project_name: 'abc' 25 | directory: '/tmp/compose' 26 | $get_services: 27 | output: | 28 | Compose: 29 | {% for service in result %} 30 | - service: {{ service.name }} 31 | {% endfor %} 32 | - docker-compose: 33 | project_name: 'abc' 34 | directory: '/tmp/compose' 35 | $up: 36 | detached: true 37 | output: | 38 | Containers started: 39 | {% for container in result %} 40 | - {{ container.name }} 41 | {% endfor %} 42 | - docker-compose: 43 | project_name: 'abc' 44 | directory: '/tmp/compose' 45 | $down: 46 | remove_image_type: false 47 | include_volumes: true 48 | output: 'Compose project stopped and removed' 49 | - execute: 50 | command: 'echo "Hello from the shell"' 51 | - docker: 52 | $containers: 53 | $list: 54 | filters: 55 | name: '{{ request.json.name }}' 56 | output: > 57 | Containers matching "{{ request.json.name }}": 58 | {% for container in result %} 59 | - {{ container.name }} @ {{ container.short_id }} 60 | {% endfor %} 61 | - docker: 62 | $info: 63 | output: 'Docker version: {{ result.ServerVersion }} on {{ result.OperatingSystem }}' 64 | - docker: 65 | $containers: 66 | $list: 67 | output: '{% for container in result %} 68 | Container: {{ container.name }} @{{ container.short_id }} 69 | {% endfor %}' 70 | - docker: 71 | $containers: 72 | $run: 73 | image: alpine 74 | command: 'echo "Hello from a new alpine container"' 75 | remove: true 76 | - log: 77 | message: 'Tests have finished at {{ timestamp }}' 78 | 79 | - /docker: 80 | method: 'POST' 81 | 82 | headers: 83 | X-Test: '123' 84 | 85 | body: 86 | callback_url: 'https://.+' 87 | repository: 88 | repo_name: '^[0-9a-z_\\-]+/[0-9a-z_\\-]+$' 89 | 90 | actions: 91 | - log: 92 | message: '[{{ datetime }}] Processing {{ request.method }} {{ request.path }} ...' 93 | - http: 94 | target: 'http://localhost:9999/echo' 95 | method: 'POST' 96 | headers: 97 | Content-Type: 'application/json' 98 | X-From: 'webhook-proxy' 99 | X-Source: '{{ request.path }}' 100 | body: '{ "repo": "{{ request.json.repository.repo_name }}" }' 101 | - log: 102 | message: '[{{ datetime }}] Finished {{ request.method }} {{ request.path }}' 103 | 104 | -------------------------------------------------------------------------------- /src/actions/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import time 5 | import threading 6 | import traceback 7 | 8 | from flask import request 9 | from jinja2 import Template 10 | 11 | import docker_helper 12 | 13 | from actions.replay_helper import replay 14 | from actions.replay_helper import initialize as _initialize_replays 15 | from util import ActionInvocationException 16 | from util import ConfigurationException 17 | from util import ReplayRequested 18 | 19 | 20 | def _safe_import(): 21 | class SafeImportContext(object): 22 | def __enter__(self): 23 | pass 24 | 25 | def __exit__(self, exc_type, exc_val, exc_tb): 26 | if exc_tb: 27 | error_file = traceback.extract_tb(exc_tb)[1][0] 28 | name, _ = os.path.splitext(os.path.basename(error_file)) 29 | 30 | if name.startswith('action_'): 31 | name = name[len('action_'):].replace('_', '-') 32 | 33 | print('The "%s" action is not available' % name) 34 | 35 | return True 36 | 37 | return SafeImportContext() 38 | 39 | 40 | def _register_available_actions(): 41 | from actions.action_log import LogAction 42 | from actions.action_execute import ExecuteAction 43 | from actions.action_evaluate import EvaluateAction 44 | from actions.action_github_verify import GitHubVerifyAction 45 | from actions.action_sleep import SleepAction 46 | from actions.action_metrics import MetricsAction 47 | 48 | with _safe_import(): 49 | from actions.action_http import HttpAction 50 | with _safe_import(): 51 | from actions.action_docker import DockerAction 52 | with _safe_import(): 53 | from actions.action_docker_compose import DockerComposeAction 54 | with _safe_import(): 55 | from actions.action_docker_swarm import DockerSwarmAction 56 | 57 | 58 | class _ContextHelper(object): 59 | _context = threading.local() 60 | 61 | def __getattr__(self, item): 62 | return getattr(self._context, item) 63 | 64 | def set(self, name, value): 65 | setattr(self._context, name, value) 66 | 67 | 68 | class _CauseTraceback(object): 69 | def __init__(self): 70 | self.content = list() 71 | 72 | def write(self, data): 73 | self.content.append(' %s' % data) 74 | 75 | def __str__(self): 76 | return ''.join(self.content) 77 | 78 | 79 | class Action(object): 80 | action_name = None 81 | _registered_actions = dict() 82 | 83 | def run(self): 84 | try: 85 | return self._run() 86 | 87 | except ReplayRequested as rr: 88 | replay(request.path, request.method, request.headers, request.json, rr.at) 89 | 90 | except Exception as ex: 91 | cause = _CauseTraceback() 92 | traceback.print_exc(file=cause) 93 | 94 | raise ActionInvocationException('Failed to invoke %s.run:\n' 95 | ' Reason (%s): %s\n' 96 | 'Cause:\n%s' % 97 | (type(self).__name__, type(ex).__name__, ex, cause)) 98 | 99 | def _run(self): 100 | raise ActionInvocationException('%s.run not implemented' % type(self).__name__) 101 | 102 | def _render_with_template(self, template, **kwargs): 103 | template = Template(template) 104 | return template.render(request=request, 105 | timestamp=time.time(), 106 | datetime=time.ctime(), 107 | context=_ContextHelper(), 108 | error=self.error, 109 | replay=self.request_replay, 110 | own_container_id=docker_helper.get_current_container_id(), 111 | read_config=docker_helper.read_configuration, 112 | **kwargs) 113 | 114 | def error(self, message=''): 115 | if not message: 116 | message = 'The "%s" action threw an error' % self.action_name 117 | 118 | raise ActionInvocationException(message) 119 | 120 | @staticmethod 121 | def request_replay(after): 122 | after = float(after) 123 | 124 | if after <= 0: 125 | raise ActionInvocationException('Illegal replay interval: %.2f' % after) 126 | 127 | raise ReplayRequested(at=time.time() + after) 128 | 129 | @classmethod 130 | def register(cls, name, action_type): 131 | if name in cls._registered_actions: 132 | raise ConfigurationException('Action already registered: %s' % name) 133 | 134 | cls._registered_actions[name] = action_type 135 | 136 | @classmethod 137 | def create(cls, name, **settings): 138 | if name not in cls._registered_actions: 139 | raise ConfigurationException('Unkown action: %s (registered: %s)' % 140 | (name, cls._registered_actions.keys())) 141 | 142 | try: 143 | return cls._registered_actions[name](**settings) 144 | 145 | except Exception as ex: 146 | raise ConfigurationException('Failed to create action: %s (settings = %s)\n' 147 | ' Reason (%s): %s' % 148 | (name, settings, type(ex).__name__, ex)) 149 | 150 | 151 | def action(name): 152 | def invoke(cls): 153 | cls.action_name = name 154 | 155 | Action.register(name, cls) 156 | 157 | return cls 158 | 159 | return invoke 160 | 161 | 162 | def _safe_initialize_replays(): 163 | try: 164 | _initialize_replays() 165 | except Exception as ex: 166 | print('Failed to initialize replays, the functionality won\'t be available! Cause: %s' % ex) 167 | 168 | 169 | _register_available_actions() 170 | _safe_initialize_replays() 171 | -------------------------------------------------------------------------------- /src/actions/action_docker.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import docker 4 | import six 5 | 6 | from actions import action, Action 7 | from util import ConfigurationException 8 | 9 | 10 | @action('docker') 11 | class DockerAction(Action): 12 | client = docker.from_env(version='auto') 13 | 14 | def __init__(self, output='{{ result }}', **invocations): 15 | if len(invocations) != 1: 16 | raise ConfigurationException('The "%s" action has to have one invocation' % self.action_name) 17 | 18 | self.output_format = output 19 | self.command, self.arguments = self._split_invocation(invocations, self._target()) 20 | 21 | def _target(self): 22 | return self.client 23 | 24 | def _split_invocation(self, invocation, target): 25 | if invocation is None or not (any(key.startswith('$') for key in invocation)): 26 | return target, invocation if invocation else dict() 27 | 28 | prop, value = next(iter(invocation.items())) 29 | 30 | return self._split_invocation(value, getattr(target, prop[1:])) 31 | 32 | def _run(self): 33 | arguments = self._process_arguments(self.arguments.copy()) 34 | 35 | result = self.command(**arguments) 36 | 37 | if result is not None and not isinstance(result, str) and hasattr(result, 'decode'): 38 | result = result.decode() 39 | 40 | print(self._render_with_template(self.output_format, result=result)) 41 | 42 | def _process_arguments(self, current): 43 | for key, value in current.items(): 44 | current[key] = self._process_value(value) 45 | 46 | return current 47 | 48 | def _process_value(self, value): 49 | if isinstance(value, dict): 50 | return self._process_arguments(value.copy()) 51 | 52 | elif isinstance(value, list): 53 | return [self._process_value(item) for item in value] 54 | 55 | elif isinstance(value, six.string_types): 56 | return self._render_with_template(value) 57 | 58 | return value 59 | -------------------------------------------------------------------------------- /src/actions/action_docker_compose.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from compose.config.config import ConfigFile, ConfigDetails 4 | from compose.config.config import load as load_config 5 | from compose.project import Project 6 | 7 | from actions import action 8 | from actions.action_docker import DockerAction 9 | 10 | 11 | @action('docker-compose') 12 | class DockerComposeAction(DockerAction): 13 | def __init__(self, project_name, directory, composefile='docker-compose.yml', output='{{ result }}', **invocations): 14 | config = ConfigFile.from_filename('%s/%s' % (directory, composefile)) 15 | details = ConfigDetails(directory, [config]) 16 | self.project = Project.from_config(project_name, load_config(details), self.client.api) 17 | 18 | super(DockerComposeAction, self).__init__(output, **invocations) 19 | 20 | def _target(self): 21 | return self.project 22 | -------------------------------------------------------------------------------- /src/actions/action_docker_swarm.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from actions import action 4 | from actions.action_docker import DockerAction 5 | 6 | 7 | @action('docker-swarm') 8 | class DockerSwarmAction(DockerAction): 9 | def __init__(self, output='{{ result }}', **invocations): 10 | super(DockerSwarmAction, self).__init__(output, **invocations) 11 | print( 12 | 'The `docker-swarm` is now deprecated, use the `docker` action ' 13 | 'with `$services.$get` and `service.update(..)`' 14 | ) 15 | 16 | def _target(self): 17 | return self 18 | 19 | def restart(self, service_id): 20 | print( 21 | 'Instead of `docker-swarm.$restart` please use `docker.$services.$get(id)` ' 22 | 'and `service.update(force_update=num)`' 23 | ) 24 | return self._update_service(service_id, force_update=True) 25 | 26 | def scale(self, service_id, replicas): 27 | print( 28 | 'Instead of `docker-swarm.$scale` please use `docker.$services.$get(id)` ' 29 | 'and `service.update(mode={"replicated": {"Replicas": num}})`' 30 | ) 31 | return self._update_service(service_id, mode={'replicated': {'Replicas': int(replicas)}}) 32 | 33 | def update(self, service_id, **kwargs): 34 | print( 35 | 'Instead of `docker-swarm.$update` please use `docker.$services.$get(id)` ' 36 | 'and `service.update(**kwargs)`' 37 | ) 38 | return self._update_service(service_id, **kwargs) 39 | 40 | def _update_service(self, service_id, **kwargs): 41 | service = self.client.services.get(self._render_with_template(service_id)) 42 | 43 | if 'force_update' in kwargs and isinstance(kwargs['force_update'], bool): 44 | current = service.attrs['Spec']['TaskTemplate'].get('ForceUpdate', 0) 45 | kwargs['force_update'] = (current + 1) % 100 46 | 47 | if service.update(**self._process_arguments(kwargs)): 48 | service.reload() 49 | 50 | return service 51 | -------------------------------------------------------------------------------- /src/actions/action_evaluate.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from actions import action, Action 4 | 5 | 6 | @action('eval') 7 | class EvaluateAction(Action): 8 | def __init__(self, block): 9 | self.block = block 10 | 11 | def _run(self): 12 | print(self._render_with_template(self.block)) 13 | -------------------------------------------------------------------------------- /src/actions/action_execute.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from subprocess import check_output as invoke_command 4 | 5 | from actions import action, Action 6 | 7 | 8 | @action('execute') 9 | class ExecuteAction(Action): 10 | def __init__(self, command, shell=True, output='{{ result }}'): 11 | self.command = command if isinstance(command, list) else [command] 12 | self.output_format = output 13 | 14 | if isinstance(shell, bool): 15 | self.shell = 'sh' if shell else None 16 | 17 | elif shell: 18 | self.shell = shell 19 | 20 | def _run(self): 21 | if self.shell: 22 | command = self._render_with_template(' '.join(self.command)) 23 | 24 | if isinstance(self.shell, list): 25 | output = invoke_command(self.shell + [command]) 26 | 27 | else: 28 | output = invoke_command([self.shell, '-c', command]) 29 | 30 | else: 31 | command = map(self._render_with_template, self.command) 32 | 33 | output = invoke_command(command) 34 | 35 | if not isinstance(output, str) and hasattr(output, 'decode'): 36 | output = output.decode() 37 | 38 | print(self._render_with_template(self.output_format, result=output)) 39 | -------------------------------------------------------------------------------- /src/actions/action_github_verify.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import six 4 | import hmac 5 | from hashlib import sha1 6 | from flask import request 7 | 8 | from actions import action, Action 9 | 10 | 11 | @action('github-verify') 12 | class GitHubVerifyAction(Action): 13 | def __init__(self, secret, output='{{ result }}'): 14 | self.secret = secret 15 | self.output_format = output 16 | 17 | def _run(self): 18 | # based on https://github.com/carlos-jenkins/python-github-webhooks/blob/master/webhooks.py 19 | secret = str(self._render_with_template(self.secret)) 20 | if six.PY3: 21 | secret = secret.encode('utf-8') 22 | 23 | header_signature = request.headers.get('X-Hub-Signature') 24 | if header_signature is None: 25 | self.error('Missing X-Hub-Signature header') 26 | 27 | sha_name, signature = header_signature.split('=') 28 | if sha_name != 'sha1': 29 | self.error('Invalid hashing algorithm') 30 | 31 | mac = hmac.new(secret, msg=request.data, digestmod=sha1) 32 | 33 | if not hmac.compare_digest(str(mac.hexdigest()), str(signature)): 34 | self.error('GitHub webhook validation failed') 35 | 36 | print(self._render_with_template(self.output_format, result='GitHub webhook successfully validated')) 37 | -------------------------------------------------------------------------------- /src/actions/action_http.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | 4 | from actions import action, Action 5 | 6 | 7 | @action('http') 8 | class HttpAction(Action): 9 | def __init__(self, target, method='POST', headers=None, body=None, json=False, fail_on_error=False, 10 | output='HTTP {{ response.status_code }} : {{ response.content }}', verify=True): 11 | 12 | self.target = target 13 | self.method = method 14 | self.headers = headers 15 | self.body = body 16 | self.json = json 17 | self.fail_on_error = fail_on_error 18 | self.output_format = output 19 | self.verify = verify 20 | 21 | def _run(self): 22 | headers = self._headers.copy() 23 | 24 | if self.body and 'Content-Length' not in headers: 25 | headers['Content-Length'] = str(len(self.body)) 26 | 27 | response = requests.request(self.method, self._target, headers=headers, data=self._body, verify=self.verify) 28 | 29 | if self.fail_on_error and response.status_code // 100 != 2: 30 | self.error('HTTP call failed (HTTP %d)' % response.status_code) 31 | 32 | print(self._render_with_template(self.output_format, response=response)) 33 | 34 | @property 35 | def _target(self): 36 | return self._render_with_template(self.target) 37 | 38 | @property 39 | def _headers(self): 40 | headers = dict() 41 | 42 | if self.headers: 43 | for name, value in self.headers.items(): 44 | headers[name] = self._render_with_template(value) 45 | 46 | return headers 47 | 48 | @property 49 | def _body(self): 50 | if self.body: 51 | if self.json: 52 | return self._render_json(self.body) 53 | else: 54 | return self._render_with_template(self.body) 55 | else: 56 | return self.body 57 | 58 | def _render_json(self, body): 59 | return json.dumps(self._render_json_item(body)) 60 | 61 | def _render_json_item(self, item): 62 | if isinstance(item, dict): 63 | rendered = {} 64 | for key, value in item.items(): 65 | rendered[key] = self._render_json_item(value) 66 | return rendered 67 | 68 | if isinstance(item, list): 69 | return [self._render_json_item(x) for x in item] 70 | 71 | return self._render_with_template(item).strip() 72 | -------------------------------------------------------------------------------- /src/actions/action_log.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from actions import action, Action 4 | 5 | 6 | @action('log') 7 | class LogAction(Action): 8 | def __init__(self, message='Processing {{ request.path }} ...'): 9 | self.message = message 10 | 11 | def _run(self): 12 | print(self._render_with_template(self.message)) 13 | -------------------------------------------------------------------------------- /src/actions/action_metrics.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from timeit import default_timer 4 | 5 | from flask import request 6 | from prometheus_client import Histogram, Summary, Gauge, Counter 7 | 8 | from actions import action, Action 9 | from util import ConfigurationException 10 | 11 | 12 | @action('metrics') 13 | class MetricsAction(Action): 14 | def __init__(self, output='Tracking metrics: {{ metric }}', **kwargs): 15 | from server import Server 16 | from endpoints import Endpoint 17 | 18 | self._app = Server.app 19 | self._endpoint = Endpoint.current 20 | self._name = None 21 | 22 | if len(kwargs) != 1: 23 | raise ConfigurationException('The metrics action has to configure exactly one metric') 24 | 25 | for metric_type, configuration in kwargs.items(): 26 | handler = getattr(self, metric_type) 27 | 28 | if not handler: 29 | raise ConfigurationException('Invalid metric type: %s' % metric_type) 30 | 31 | handler(**configuration) 32 | 33 | self._output_format = output 34 | 35 | def histogram(self, name, help=None, labels=None, **kwargs): 36 | self._register( 37 | Histogram, 38 | lambda m, t: m.observe(t), 39 | name, help or name, 40 | labels or dict(), 41 | **kwargs 42 | ) 43 | 44 | def summary(self, name, help=None, labels=None, **kwargs): 45 | self._register( 46 | Summary, 47 | lambda m, t: m.observe(t), 48 | name, help or name, 49 | labels or dict(), 50 | **kwargs 51 | ) 52 | 53 | def gauge(self, name, help=None, labels=None, **kwargs): 54 | self._register( 55 | Gauge, 56 | [lambda m: m.inc(), lambda m, _: m.dec()], 57 | name, help or name, 58 | labels or dict(), 59 | **kwargs 60 | ) 61 | 62 | def counter(self, name, help=None, labels=None, **kwargs): 63 | self._register( 64 | Counter, 65 | lambda m, _: m.inc(), 66 | name, help or name, 67 | labels or dict(), 68 | **kwargs 69 | ) 70 | 71 | def _register(self, metric_type, metric_calls, name, help, labels, **kwargs): 72 | label_names = tuple(labels.keys()) 73 | 74 | metric = metric_type(name, help, labelnames=label_names, **kwargs) 75 | 76 | try: 77 | before_request_call, after_request_call = metric_calls 78 | except TypeError: 79 | before_request_call, after_request_call = None, metric_calls 80 | 81 | def target_metric(response): 82 | if label_names: 83 | return metric.labels( 84 | *map(lambda key: self._render_with_template( 85 | labels.get(key), response=response 86 | ).strip(), label_names) 87 | ) 88 | 89 | else: 90 | return metric 91 | 92 | def before_request(): 93 | if request.path != self._endpoint.route: 94 | return 95 | 96 | if before_request_call: 97 | before_request_call(target_metric(response=None)) 98 | 99 | request.whp_start_time = default_timer() 100 | 101 | def after_request(response): 102 | if request.path != self._endpoint.route: 103 | return response 104 | 105 | total_time = max(default_timer() - request.whp_start_time, 0) 106 | 107 | after_request_call(target_metric(response=response), total_time) 108 | 109 | return response 110 | 111 | self._app.before_request(before_request) 112 | self._app.after_request(after_request) 113 | 114 | self._name = name 115 | 116 | def _run(self): 117 | print(self._render_with_template(self._output_format, metric=self._name)) 118 | -------------------------------------------------------------------------------- /src/actions/action_sleep.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from actions import action, Action 4 | 5 | 6 | @action('sleep') 7 | class SleepAction(Action): 8 | def __init__(self, seconds, output='Waiting {{ seconds }} seconds before continuing ...'): 9 | self.seconds = seconds 10 | self.output_format = output 11 | 12 | def _run(self): 13 | seconds = float(self._render_with_template(str(self.seconds))) 14 | 15 | print(self._render_with_template(self.output_format, seconds=seconds)) 16 | 17 | time.sleep(seconds) 18 | 19 | -------------------------------------------------------------------------------- /src/actions/replay_helper.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import json 4 | import time 5 | import sqlite3 6 | import threading 7 | import traceback 8 | 9 | import requests 10 | 11 | import docker_helper 12 | 13 | 14 | _database_path = docker_helper.read_configuration( 15 | 'REPLAY_DATABASE', '/var/config/webhooks/replay', 'webhooks-replay.db' 16 | ) 17 | 18 | _schedule_condition = threading.Condition() 19 | _shutdown = [False] 20 | 21 | 22 | def initialize(): 23 | _initialize_schema() 24 | 25 | thread = threading.Thread(target=_schedule) 26 | thread.setDaemon(True) 27 | thread.start() 28 | 29 | 30 | def shutdown(): 31 | _shutdown[:] = [True] 32 | 33 | with _schedule_condition: 34 | _schedule_condition.notify() 35 | 36 | 37 | def _schedule(): 38 | from server import Server 39 | 40 | # wait for server initialization 41 | while not Server.http_port: 42 | time.sleep(1) 43 | 44 | while not _shutdown[0]: 45 | # wait for the next task or until notified 46 | with _schedule_condition: 47 | _schedule_condition.wait(timeout=_until_next_scheduled()) 48 | 49 | _next = _next_scheduled() 50 | 51 | if not _next: 52 | continue 53 | 54 | _id, _path, _method, _headers, _body, _time = _next 55 | 56 | if _time > time.time(): 57 | continue 58 | 59 | print('Replaying request on %s' % _path) 60 | 61 | try: 62 | url = 'http://localhost:%d%s' % (Server.http_port, _path) 63 | requests.request( 64 | method=_method, url=url, 65 | headers=json.loads(_headers), json=json.loads(_body) 66 | ) 67 | 68 | except Exception: 69 | traceback.print_exc() 70 | 71 | finally: 72 | with read_write_db() as db: 73 | db.execute('DELETE FROM requests WHERE id = :id', {'id': _id}) 74 | db.commit() 75 | 76 | 77 | def _next_scheduled(): 78 | with read_only_db() as db: 79 | return db.execute(''' 80 | SELECT id, path, method, headers, body, next 81 | FROM requests 82 | ORDER BY next ASC 83 | LIMIT 1 84 | ''').fetchone() 85 | 86 | 87 | def _until_next_scheduled(): 88 | _next = _next_scheduled() 89 | if _next: 90 | _, _, _, _, _, scheduled_time = _next 91 | return max(0.1, scheduled_time - time.time()) 92 | 93 | 94 | def replay(path, method, headers, body, at): 95 | print('Replay requested on %s' % path) 96 | 97 | headers = { 98 | key: value for key, value in headers.items() 99 | } 100 | 101 | with read_write_db() as db: 102 | db.execute(''' 103 | INSERT INTO requests (path, method, headers, body, next) 104 | VALUES (:path, :method, :headers, :body, :at) 105 | ''', { 106 | 'path': path, 107 | 'method': method, 108 | 'headers': json.dumps(headers), 109 | 'body': json.dumps(body), 110 | 'at': at 111 | }) 112 | db.commit() 113 | 114 | with _schedule_condition: 115 | _schedule_condition.notify() 116 | 117 | 118 | class _DatabaseContext(object): 119 | write_lock = threading.RLock() 120 | 121 | def __init__(self, path, read_only=True): 122 | self.path = path 123 | self.read_only = read_only 124 | 125 | def __enter__(self): 126 | if not self.read_only: 127 | self.write_lock.acquire() 128 | 129 | try: 130 | self.connection = sqlite3.connect(self.path) 131 | return self.connection 132 | 133 | except Exception: 134 | if not self.read_only: 135 | self.write_lock.release() 136 | 137 | raise 138 | 139 | def __exit__(self, exc_type, exc_val, exc_tb): 140 | try: 141 | self.connection.close() 142 | 143 | finally: 144 | if not self.read_only: 145 | self.write_lock.release() 146 | 147 | 148 | def read_only_db(path=_database_path): 149 | return _DatabaseContext(path, read_only=True) 150 | 151 | 152 | def read_write_db(path=_database_path): 153 | return _DatabaseContext(path, read_only=False) 154 | 155 | 156 | def _initialize_schema(): 157 | with read_write_db() as db: 158 | cursor = db.cursor() 159 | 160 | cursor.execute(''' 161 | CREATE TABLE IF NOT EXISTS requests ( 162 | id INTEGER PRIMARY KEY, 163 | path TEXT, 164 | method TEXT, 165 | headers TEXT, 166 | body TEXT, 167 | next TIMESTAMP 168 | ) 169 | ''') 170 | 171 | cursor.execute(''' 172 | CREATE INDEX IF NOT EXISTS idx_next_date ON requests(next ASC) 173 | ''') 174 | -------------------------------------------------------------------------------- /src/app.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import signal 3 | 4 | import yaml 5 | 6 | from server import Server 7 | 8 | 9 | def parse_settings(source='server.yml'): 10 | with open(source, 'r') as source_file: 11 | return yaml.load(source_file) 12 | 13 | 14 | def handle_signal(num, _): 15 | if num == signal.SIGTERM: 16 | exit(0) 17 | 18 | else: 19 | exit(1) 20 | 21 | 22 | if __name__ == '__main__': 23 | settings = parse_settings(sys.argv[1]) if len(sys.argv) == 2 else parse_settings() 24 | 25 | signal.signal(signal.SIGTERM, handle_signal) 26 | signal.signal(signal.SIGINT, handle_signal) 27 | 28 | server = Server(endpoint_configurations=settings.get('endpoints'), **settings.get('server', dict())) 29 | server.run() 30 | -------------------------------------------------------------------------------- /src/endpoints.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import re 4 | import threading 5 | import traceback 6 | 7 | import six 8 | from flask import request 9 | from jinja2 import Template 10 | 11 | import docker_helper 12 | 13 | from actions import Action 14 | from util import ConfigurationException, classproperty 15 | 16 | 17 | class Endpoint(object): 18 | _current = threading.local() 19 | 20 | def __init__(self, route, settings, action_metrics): 21 | if not route: 22 | raise ConfigurationException('An endpoint must have its route defined') 23 | 24 | if settings is None: 25 | settings = dict() 26 | 27 | self._route = route 28 | self._method = settings.get('method', 'POST') 29 | self._async = settings.get('async', False) 30 | self._headers = settings.get('headers', dict()) 31 | self._body = settings.get('body', dict()) 32 | 33 | with Endpoint.in_context(self): 34 | self._actions = list(Action.create(name, **(action_settings if action_settings else dict())) 35 | for action_item in settings.get('actions', list()) 36 | for name, action_settings in action_item.items()) 37 | 38 | self._action_metrics = action_metrics 39 | 40 | @property 41 | def route(self): 42 | return self._route 43 | 44 | @property 45 | def method(self): 46 | return self._method 47 | 48 | @property 49 | def is_async(self): 50 | return self._async 51 | 52 | @property 53 | def headers(self): 54 | return dict(self._headers) 55 | 56 | @property 57 | def body(self): 58 | return dict(self._body) 59 | 60 | @classmethod 61 | def in_context(cls, endpoint): 62 | class EndpointSetupContext(object): 63 | def __enter__(self): 64 | cls._current.instance = endpoint 65 | 66 | def __exit__(self, *args, **kwargs): 67 | cls._current.instance = None 68 | 69 | return EndpointSetupContext() 70 | 71 | @classproperty 72 | def current(self): 73 | return self._current.instance 74 | 75 | def setup(self, app): 76 | @app.route(self._route, endpoint=self._route[1:], methods=[self._method]) 77 | def receive(**kwargs): 78 | if not request.json: 79 | if self._body: 80 | return self._make_response(400, 'No payload') 81 | 82 | if not self.accept(): 83 | return self._make_response(409, 'Invalid payload') 84 | 85 | if self._async: 86 | args = (app, request.environ.copy(), request.get_json()) 87 | 88 | threading.Thread(target=self._safe_run_actions, args=args).start() 89 | 90 | else: 91 | try: 92 | self._run_actions() 93 | 94 | except Exception: 95 | traceback.print_exc() 96 | return self._make_response(500, 'Internal Server Error') 97 | 98 | return self._make_response(200, 'OK\n') 99 | 100 | def _run_actions(self): 101 | for idx, action in enumerate(self._actions): 102 | labels = (self._route, self._method, action.action_name, idx) 103 | 104 | with self._action_metrics.labels(*labels).time(): 105 | action.run() 106 | 107 | def _safe_run_actions(self, app, request_environment, json): 108 | with app.request_context(request_environment): 109 | # reassigning the JSON body of the request on a different thread 110 | setattr(request, '_cached_json', (json, json)) 111 | 112 | try: 113 | with Endpoint.in_context(self): 114 | self._run_actions() 115 | 116 | except Exception: 117 | traceback.print_exc() 118 | 119 | @staticmethod 120 | def _make_response(status, message): 121 | return message, status, {'Content-Type': 'text/plain'} 122 | 123 | def accept(self): 124 | return self._accept_headers(request.headers, self._headers) and self._accept_body(request.json, self._body) 125 | 126 | @staticmethod 127 | def _accept_headers(headers, rules): 128 | for key, rule in rules.items(): 129 | value = headers.get(key, '') 130 | 131 | translated_rule = Template(rule).render(read_config=docker_helper.read_configuration) 132 | translated_value = Template(value).render(read_config=docker_helper.read_configuration) 133 | 134 | if not re.match(translated_rule, translated_value): 135 | print('Failed to validate the "%s" header: "%s" does not match "%s"' % 136 | (key, translated_value, translated_rule)) 137 | return False 138 | 139 | return True 140 | 141 | def _accept_body(self, data, rules, prefix=''): 142 | for key, rule in rules.items(): 143 | value = data.get(key, dict() if isinstance(rule, dict) else '') 144 | 145 | if isinstance(value, list): 146 | for idx, item in enumerate(value): 147 | if not self._check_body(item, rule, '%s.%s[%d]' % (prefix, key, idx)): 148 | return False 149 | 150 | else: 151 | if not self._check_body(value, rule, '%s.%s' % (prefix, key)): 152 | return False 153 | 154 | return True 155 | 156 | def _check_body(self, value, rule, property_path): 157 | if isinstance(rule, six.string_types): 158 | rule = Template(rule).render( 159 | read_config=docker_helper.read_configuration 160 | ) 161 | 162 | if isinstance(value, six.string_types): 163 | value = Template(value).render( 164 | read_config=docker_helper.read_configuration 165 | ) 166 | 167 | if isinstance(rule, dict) and isinstance(value, dict): 168 | if not self._accept_body(value, rule, property_path): 169 | return False 170 | 171 | elif not isinstance(rule, six.string_types) or not re.match(rule, str(value)): 172 | print('Failed to validate "%s": "%s" does not match "%s"' % 173 | (property_path[1:], value, rule)) 174 | return False 175 | 176 | return True 177 | -------------------------------------------------------------------------------- /src/server.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from flask import Flask 4 | from prometheus_client import Summary 5 | from prometheus_flask_exporter import PrometheusMetrics 6 | 7 | from endpoints import Endpoint 8 | from util import ConfigurationException, import_action_module 9 | 10 | 11 | class Server(object): 12 | app = None 13 | http_port = None 14 | 15 | def __init__(self, endpoint_configurations, host='127.0.0.1', port=5000, imports=None): 16 | self.host = host 17 | self.port = int(port) 18 | 19 | if not endpoint_configurations: 20 | raise ConfigurationException('No endpoints defined') 21 | 22 | Server.http_port = self.port 23 | 24 | if imports: 25 | for path in imports: 26 | import_action_module(path) 27 | 28 | Server.app = Flask(__name__) 29 | 30 | action_metrics = self._setup_metrics() 31 | 32 | endpoints = [Endpoint(route, settings, action_metrics) 33 | for config in endpoint_configurations 34 | for route, settings in config.items()] 35 | 36 | for endpoint in endpoints: 37 | endpoint.setup(self.app) 38 | 39 | def _setup_metrics(self): 40 | metrics = PrometheusMetrics(self.app) 41 | 42 | metrics.info('flask_app_info', 'Application info', 43 | version=os.environ.get('GIT_COMMIT') or 'unknown') 44 | 45 | metrics.info( 46 | 'flask_app_built_at', 'Application build timestamp' 47 | ).set( 48 | float(os.environ.get('BUILD_TIMESTAMP') or '0') 49 | ) 50 | 51 | action_summary = Summary( 52 | 'webhook_proxy_actions', 53 | 'Action invocation metrics', 54 | labelnames=('http_route', 'http_method', 'action_type', 'action_index') 55 | ) 56 | 57 | return action_summary 58 | 59 | def run(self): 60 | self.app.run(host=self.host, port=self.port, threaded=True) 61 | -------------------------------------------------------------------------------- /src/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import random 5 | 6 | import six 7 | 8 | 9 | class ConfigurationException(Exception): 10 | pass 11 | 12 | 13 | class ActionInvocationException(Exception): 14 | pass 15 | 16 | 17 | class ReplayRequested(Exception): 18 | def __init__(self, at): 19 | super(ReplayRequested, self).__init__() 20 | self.at = at 21 | 22 | 23 | def import_action_module(file_path): 24 | directory = os.environ.get('TMP_IMPORT_DIR', '/tmp') 25 | 26 | module_name = 'action_%s_%s' % (int(1000.0 * time.time()), random.randint(1000, 9999)) 27 | filename = '%s.py' % module_name 28 | 29 | tmp_file_path = os.path.join(directory, filename) 30 | 31 | try: 32 | with open(tmp_file_path, 'w') as tmp_file: 33 | with open(file_path, 'r') as input_file: 34 | tmp_file.write(input_file.read()) 35 | 36 | sys_path = list(sys.path) 37 | sys.path.insert(0, directory) 38 | 39 | try: 40 | if six.PY34: 41 | import importlib.machinery 42 | loader = importlib.machinery.SourceFileLoader(module_name, tmp_file_path) 43 | loader.load_module() 44 | 45 | elif six.PY3: 46 | import importlib 47 | 48 | spec = importlib.util.spec_from_file_location(module_name, tmp_file_path) 49 | module = importlib.util.module_from_spec(spec) 50 | spec.loader.exec_module(module) 51 | 52 | else: 53 | __import__(module_name) 54 | 55 | finally: 56 | sys.path[:] = sys_path 57 | 58 | except Exception as ex: 59 | raise ConfigurationException( 60 | 'Failed to import %s\nReason: (%s) %s' % ( 61 | file_path, type(ex).__name__, ex 62 | ) 63 | ) 64 | 65 | finally: 66 | os.remove(tmp_file_path) 67 | 68 | # remove .pyc 69 | tmp_file_path += 'c' 70 | 71 | if os.path.exists(tmp_file_path): 72 | os.remove(tmp_file_path) 73 | 74 | 75 | class classproperty(property): 76 | def __init__(self, fget): 77 | super(classproperty, self).__init__(classmethod(fget)) 78 | 79 | def __get__(self, instance, owner): 80 | return self.fget.__get__(None, owner)() 81 | -------------------------------------------------------------------------------- /tests/github/webhook.json: -------------------------------------------------------------------------------- 1 | {"zen":"Speak like a human.","hook_id":20238401,"hook":{"type":"Repository","id":20238401,"name":"web","active":true,"events":["push"],"config":{"content_type":"json","insecure_ssl":"0","secret":"********","url":"https://requestb.in/rjlvxarj"},"updated_at":"2018-01-17T23:27:53Z","created_at":"2018-01-17T23:27:53Z","url":"https://api.github.com/repos/rycus86/webhook-proxy/hooks/20238401","test_url":"https://api.github.com/repos/rycus86/webhook-proxy/hooks/20238401/test","ping_url":"https://api.github.com/repos/rycus86/webhook-proxy/hooks/20238401/pings","last_response":{"code":null,"status":"unused","message":null}},"repository":{"id":101429044,"name":"webhook-proxy","full_name":"rycus86/webhook-proxy","owner":{"login":"rycus86","id":3105242,"avatar_url":"https://avatars3.githubusercontent.com/u/3105242?v=4","gravatar_id":"","url":"https://api.github.com/users/rycus86","html_url":"https://github.com/rycus86","followers_url":"https://api.github.com/users/rycus86/followers","following_url":"https://api.github.com/users/rycus86/following{/other_user}","gists_url":"https://api.github.com/users/rycus86/gists{/gist_id}","starred_url":"https://api.github.com/users/rycus86/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rycus86/subscriptions","organizations_url":"https://api.github.com/users/rycus86/orgs","repos_url":"https://api.github.com/users/rycus86/repos","events_url":"https://api.github.com/users/rycus86/events{/privacy}","received_events_url":"https://api.github.com/users/rycus86/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/rycus86/webhook-proxy","description":"Simple web server for JSON webhooks","fork":false,"url":"https://api.github.com/repos/rycus86/webhook-proxy","forks_url":"https://api.github.com/repos/rycus86/webhook-proxy/forks","keys_url":"https://api.github.com/repos/rycus86/webhook-proxy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rycus86/webhook-proxy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rycus86/webhook-proxy/teams","hooks_url":"https://api.github.com/repos/rycus86/webhook-proxy/hooks","issue_events_url":"https://api.github.com/repos/rycus86/webhook-proxy/issues/events{/number}","events_url":"https://api.github.com/repos/rycus86/webhook-proxy/events","assignees_url":"https://api.github.com/repos/rycus86/webhook-proxy/assignees{/user}","branches_url":"https://api.github.com/repos/rycus86/webhook-proxy/branches{/branch}","tags_url":"https://api.github.com/repos/rycus86/webhook-proxy/tags","blobs_url":"https://api.github.com/repos/rycus86/webhook-proxy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rycus86/webhook-proxy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rycus86/webhook-proxy/git/refs{/sha}","trees_url":"https://api.github.com/repos/rycus86/webhook-proxy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rycus86/webhook-proxy/statuses/{sha}","languages_url":"https://api.github.com/repos/rycus86/webhook-proxy/languages","stargazers_url":"https://api.github.com/repos/rycus86/webhook-proxy/stargazers","contributors_url":"https://api.github.com/repos/rycus86/webhook-proxy/contributors","subscribers_url":"https://api.github.com/repos/rycus86/webhook-proxy/subscribers","subscription_url":"https://api.github.com/repos/rycus86/webhook-proxy/subscription","commits_url":"https://api.github.com/repos/rycus86/webhook-proxy/commits{/sha}","git_commits_url":"https://api.github.com/repos/rycus86/webhook-proxy/git/commits{/sha}","comments_url":"https://api.github.com/repos/rycus86/webhook-proxy/comments{/number}","issue_comment_url":"https://api.github.com/repos/rycus86/webhook-proxy/issues/comments{/number}","contents_url":"https://api.github.com/repos/rycus86/webhook-proxy/contents/{+path}","compare_url":"https://api.github.com/repos/rycus86/webhook-proxy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rycus86/webhook-proxy/merges","archive_url":"https://api.github.com/repos/rycus86/webhook-proxy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rycus86/webhook-proxy/downloads","issues_url":"https://api.github.com/repos/rycus86/webhook-proxy/issues{/number}","pulls_url":"https://api.github.com/repos/rycus86/webhook-proxy/pulls{/number}","milestones_url":"https://api.github.com/repos/rycus86/webhook-proxy/milestones{/number}","notifications_url":"https://api.github.com/repos/rycus86/webhook-proxy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rycus86/webhook-proxy/labels{/name}","releases_url":"https://api.github.com/repos/rycus86/webhook-proxy/releases{/id}","deployments_url":"https://api.github.com/repos/rycus86/webhook-proxy/deployments","created_at":"2017-08-25T17:56:48Z","updated_at":"2017-08-25T17:57:39Z","pushed_at":"2018-01-15T22:48:09Z","git_url":"git://github.com/rycus86/webhook-proxy.git","ssh_url":"git@github.com:rycus86/webhook-proxy.git","clone_url":"https://github.com/rycus86/webhook-proxy.git","svn_url":"https://github.com/rycus86/webhook-proxy","homepage":null,"size":88,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"},"sender":{"login":"rycus86","id":3105242,"avatar_url":"https://avatars3.githubusercontent.com/u/3105242?v=4","gravatar_id":"","url":"https://api.github.com/users/rycus86","html_url":"https://github.com/rycus86","followers_url":"https://api.github.com/users/rycus86/followers","following_url":"https://api.github.com/users/rycus86/following{/other_user}","gists_url":"https://api.github.com/users/rycus86/gists{/gist_id}","starred_url":"https://api.github.com/users/rycus86/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rycus86/subscriptions","organizations_url":"https://api.github.com/users/rycus86/orgs","repos_url":"https://api.github.com/users/rycus86/repos","events_url":"https://api.github.com/users/rycus86/events{/privacy}","received_events_url":"https://api.github.com/users/rycus86/received_events","type":"User","site_admin":false}} -------------------------------------------------------------------------------- /tests/imports/actions_to_load.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import json 4 | 5 | from actions import action, Action 6 | 7 | 8 | @action('sample') 9 | class SampleAction(Action): 10 | def __init__(self, **kwargs): 11 | self.params = kwargs 12 | 13 | def _run(self): 14 | print('\n'.join('%s=%s' % (key, value) for key, value in self.params.items())) 15 | 16 | 17 | @action('json') 18 | class JsonAction(Action): 19 | def __init__(self, **kwargs): 20 | self.params = kwargs 21 | 22 | def _run(self): 23 | print(json.dumps(self.params, indent=2)) 24 | 25 | -------------------------------------------------------------------------------- /tests/imports/invalid.py: -------------------------------------------------------------------------------- 1 | raise Exception('Failed to initialize') 2 | -------------------------------------------------------------------------------- /tests/imports/test1/action.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from actions import action, Action 4 | 5 | 6 | @action('test1') 7 | class CustomAction(Action): 8 | def __init__(self, **kwargs): 9 | self.params = kwargs 10 | 11 | def _run(self): 12 | print('\n'.join('%s=%s' % (key, value) for key, value in self.params.items())) 13 | 14 | -------------------------------------------------------------------------------- /tests/imports/test2/action.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from actions import action, Action 4 | 5 | 6 | @action('test2') 7 | class CustomAction(Action): 8 | def __init__(self, **kwargs): 9 | self.params = kwargs 10 | 11 | def _run(self): 12 | print('\n'.join('%s=%s' % (key, value) for key, value in self.params.items())) 13 | 14 | -------------------------------------------------------------------------------- /tests/integrationtest_helper.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import unittest 4 | 5 | import docker 6 | import requests 7 | 8 | 9 | class IntegrationTestBase(unittest.TestCase): 10 | DIND_HOST = os.environ.get('DIND_HOST', 'localhost') 11 | DIND_VERSION = os.environ.get('DIND_VERSION') 12 | 13 | local_client = None 14 | remote_client = None 15 | dind_container = None 16 | 17 | @classmethod 18 | def setUpClass(cls): 19 | assert cls.DIND_VERSION is not None 20 | 21 | cls.local_client = docker.DockerClient(os.environ.get('DOCKER_ADDRESS')) 22 | 23 | assert cls.local_client.version() is not None 24 | 25 | cls.build_project() 26 | 27 | cls.dind_container = cls.start_dind_container() 28 | 29 | cls.remote_client = cls.dind_client(cls.dind_container) 30 | 31 | cls.prepare_images('webhook-testing') 32 | 33 | @classmethod 34 | def tearDownClass(cls): 35 | cls.remote_client.api.close() 36 | 37 | cls.dind_container.remove(force=True, v=True) 38 | 39 | cls.local_client.api.close() 40 | 41 | @classmethod 42 | def start_dind_container(cls): 43 | container = cls.local_client.containers.run('docker:%s-dind' % cls.DIND_VERSION, 44 | name='webhook-dind-%s' % int(time.time()), 45 | ports={'2375': None, '9002': '9003'}, 46 | privileged=True, detach=True) 47 | 48 | try: 49 | for _ in range(10): 50 | container.reload() 51 | 52 | if container.status == 'running': 53 | if container.id in (c.id for c in cls.local_client.containers.list()): 54 | break 55 | 56 | time.sleep(0.2) 57 | 58 | port = cls.dind_port(container) 59 | 60 | for _ in range(25): 61 | try: 62 | response = requests.get('http://%s:%s/version' % (cls.DIND_HOST, port)) 63 | if response and response.status_code == 200: 64 | break 65 | 66 | except requests.exceptions.RequestException: 67 | pass 68 | 69 | time.sleep(0.2) 70 | 71 | remote_client = cls.dind_client(container) 72 | 73 | assert remote_client.version() is not None 74 | 75 | return container 76 | 77 | except Exception: 78 | container.remove(force=True, v=True) 79 | 80 | raise 81 | 82 | @classmethod 83 | def dind_port(cls, container): 84 | return container.attrs['NetworkSettings']['Ports']['2375/tcp'][0]['HostPort'] 85 | 86 | @classmethod 87 | def dind_client(cls, container): 88 | return docker.DockerClient('tcp://%s:%s' % (cls.DIND_HOST, cls.dind_port(container)), 89 | version='auto') 90 | 91 | @classmethod 92 | def wait_for_startup(cls, container, extra=0, timeout=30): 93 | container.reload() 94 | 95 | for _ in range(timeout * 2): 96 | if container.status == 'running': 97 | if extra > 0: 98 | time.sleep(extra) 99 | 100 | return 101 | 102 | time.sleep(0.5) 103 | 104 | @classmethod 105 | def prepare_images(cls, *images): 106 | for tag in images: 107 | image = cls.local_client.images.get(tag) 108 | 109 | cls.remote_client.images.load(image.save()) 110 | 111 | if ':' in tag: 112 | name, tag = tag.split(':') 113 | 114 | else: 115 | name, tag = tag, None 116 | 117 | cls.remote_client.images.get(image.id).tag(name, tag=tag) 118 | 119 | @classmethod 120 | def build_project(cls, tag='webhook-testing'): 121 | cls.local_client.images.build( 122 | path=os.path.join(os.path.dirname(__file__), '..'), 123 | dockerfile='Dockerfile-docker', 124 | tag=tag, 125 | rm=True) 126 | 127 | @classmethod 128 | def prepare_file(cls, filename, contents): 129 | cls.dind_container.exec_run(['mkdir', '-p', os.path.dirname('/tmp/%s' % filename)]) 130 | cls.dind_container.exec_run( 131 | ['tee', '/tmp/%s' % filename], stdin=True, socket=True 132 | ).output.sendall(contents) 133 | 134 | @classmethod 135 | def request(cls, uri, **json): 136 | return requests.post('http://%s:9003%s' % (cls.DIND_HOST, uri), json=json) 137 | 138 | @classmethod 139 | def metrics(cls): 140 | return requests.get('http://%s:9003/metrics' % cls.DIND_HOST) 141 | 142 | def setUp(self): 143 | self.started_containers = list() 144 | 145 | def tearDown(self): 146 | for container in self.started_containers: 147 | container.remove(force=True, v=True) 148 | 149 | def start_app_container(self, config_filename): 150 | container = self.remote_client.containers.run('webhook-testing', 151 | command='/var/tmp/%s' % config_filename, 152 | ports={'9001': '9002'}, 153 | volumes=['/var/run/docker.sock:/var/run/docker.sock:ro', 154 | '/tmp:/var/tmp:ro'], 155 | detach=True) 156 | 157 | self.started_containers.append(container) 158 | 159 | self.wait_for_startup(container, extra=1) 160 | 161 | return container 162 | -------------------------------------------------------------------------------- /tests/it_docker.py: -------------------------------------------------------------------------------- 1 | from integrationtest_helper import IntegrationTestBase 2 | 3 | 4 | class DockerIntegrationTest(IntegrationTestBase): 5 | 6 | def test_docker_info(self): 7 | config = """ 8 | server: 9 | host: 0.0.0.0 10 | port: 9001 11 | 12 | endpoints: 13 | - /info: 14 | actions: 15 | - docker: 16 | $info: 17 | output: 'version={{ result.ServerVersion }}' 18 | """ 19 | 20 | self.prepare_file('test-21.yml', config) 21 | 22 | container = self.start_app_container('test-21.yml') 23 | 24 | response = self.request('/info', data='none') 25 | 26 | self.assertEqual(response.status_code, 200) 27 | 28 | output = container.logs(stdout=True, stderr=False) 29 | 30 | self.assertIn('version=%s' % self.DIND_VERSION, output) 31 | 32 | def test_list_containers(self): 33 | config = """ 34 | server: 35 | host: 0.0.0.0 36 | port: 9001 37 | 38 | endpoints: 39 | - /docker/list: 40 | actions: 41 | - docker: 42 | $containers: 43 | $list: 44 | filters: 45 | name: '{{ request.json.name }}' 46 | output: | 47 | {% for container in result %} 48 | - {{ container.id }} 49 | {% endfor %} 50 | """ 51 | 52 | self.prepare_file('test-22.yml', config) 53 | 54 | container = self.start_app_container('test-22.yml') 55 | 56 | response = self.request('/docker/list', name=container.name) 57 | 58 | self.assertEqual(response.status_code, 200) 59 | 60 | output = container.logs(stdout=True, stderr=False) 61 | 62 | self.assertEqual(output.strip().splitlines()[-1], '- %s' % container.id) 63 | 64 | def test_run_container(self): 65 | self.prepare_images('alpine') 66 | 67 | config = """ 68 | server: 69 | host: 0.0.0.0 70 | port: 9001 71 | 72 | endpoints: 73 | - /run: 74 | actions: 75 | - docker: 76 | $containers: 77 | $run: 78 | image: alpine 79 | command: 'echo "Alpine says: {{ request.json.message }}"' 80 | remove: true 81 | """ 82 | 83 | self.prepare_file('test-23.yml', config) 84 | 85 | container = self.start_app_container('test-23.yml') 86 | 87 | response = self.request('/run', message='testing') 88 | 89 | self.assertEqual(response.status_code, 200) 90 | 91 | response = self.request('/run', message='sample') 92 | 93 | self.assertEqual(response.status_code, 200) 94 | 95 | output = container.logs(stdout=True, stderr=False) 96 | 97 | self.assertIn('Alpine says: testing', output) 98 | self.assertIn('Alpine says: sample', output) 99 | 100 | def test_log_container_status(self): 101 | config = """ 102 | server: 103 | host: 0.0.0.0 104 | port: 9001 105 | 106 | endpoints: 107 | - /log/status: 108 | actions: 109 | - docker: 110 | $containers: 111 | $get: 112 | container_id: '{{ request.json.target }}' 113 | output: '{{ context.set("container", result) }}' 114 | - log: 115 | message: > 116 | status={{ context.container.status }} 117 | """ 118 | 119 | self.prepare_file('test-24.yml', config) 120 | 121 | container = self.start_app_container('test-24.yml') 122 | 123 | response = self.request('/log/status', target=container.id) 124 | 125 | self.assertEqual(response.status_code, 200) 126 | 127 | output = container.logs(stdout=True, stderr=False) 128 | 129 | self.assertIn('status=running', output) 130 | 131 | def test_restart_container(self): 132 | self.prepare_images('alpine') 133 | 134 | config = """ 135 | server: 136 | host: 0.0.0.0 137 | port: 9001 138 | 139 | endpoints: 140 | - /docker/restart: 141 | actions: 142 | - docker: 143 | $containers: 144 | $run: 145 | image: alpine 146 | command: 'sh -c "echo \"{{ request.json.message }}\" && sleep 3600"' 147 | detach: true 148 | output: '{% set _ = context.set("target", result) %}' 149 | - eval: 150 | block: | 151 | {{ context.target.restart(timeout=1) }} 152 | {{ context.target.logs(stdout=true, stderr=false) }} 153 | """ 154 | 155 | self.prepare_file('test-25.yml', config) 156 | 157 | container = self.start_app_container('test-25.yml') 158 | 159 | response = self.request('/docker/restart', message='Starting...') 160 | 161 | self.assertEqual(response.status_code, 200) 162 | 163 | output = container.logs(stdout=True, stderr=False) 164 | 165 | self.assertIn('Starting...\nStarting...', output) 166 | -------------------------------------------------------------------------------- /tests/it_docker_compose.py: -------------------------------------------------------------------------------- 1 | from integrationtest_helper import IntegrationTestBase 2 | 3 | 4 | class DockerComposeIntegrationTest(IntegrationTestBase): 5 | def setUp(self): 6 | super(DockerComposeIntegrationTest, self).setUp() 7 | 8 | self.prepare_images('alpine') 9 | 10 | def test_service_names(self): 11 | project_yaml = """ 12 | version: '2' 13 | services: 14 | app: 15 | image: alpine 16 | command: sleep 10 17 | web: 18 | image: alpine 19 | command: echo "hello" 20 | """ 21 | 22 | self.prepare_file('project/docker-compose.yml', project_yaml) 23 | 24 | config = """ 25 | server: 26 | host: 0.0.0.0 27 | port: 9001 28 | 29 | endpoints: 30 | - /compose/info: 31 | actions: 32 | - docker-compose: 33 | project_name: sample 34 | directory: /var/tmp/project 35 | $get_services: 36 | output: | 37 | {% for service in result %} 38 | name={{ service.name }} 39 | {% endfor %} 40 | """ 41 | 42 | self.prepare_file('test-31.yml', config) 43 | 44 | container = self.start_app_container('test-31.yml') 45 | 46 | response = self.request('/compose/info', data='none') 47 | 48 | self.assertEqual(response.status_code, 200) 49 | 50 | output = container.logs(stdout=True, stderr=False) 51 | 52 | self.assertIn('name=app', output) 53 | self.assertIn('name=web', output) 54 | 55 | def test_scale_service(self): 56 | project_yaml = """ 57 | version: '2' 58 | services: 59 | app: 60 | image: alpine 61 | command: sh -c 'echo "app is running" && sleep 3600' 62 | web: 63 | image: alpine 64 | command: sh -c 'echo "web is running" && sleep 3600' 65 | """ 66 | 67 | self.prepare_file('scaling_project/docker-compose.yml', project_yaml) 68 | 69 | config = """ 70 | server: 71 | host: 0.0.0.0 72 | port: 9001 73 | 74 | endpoints: 75 | - /compose/scaling: 76 | actions: 77 | - docker-compose: 78 | project_name: scaling 79 | directory: /var/tmp/scaling_project 80 | $up: 81 | detached: true 82 | - docker-compose: 83 | project_name: scaling 84 | directory: /var/tmp/scaling_project 85 | $get_service: 86 | name: '{{ request.json.service }}' 87 | output: > 88 | {{ context.set("service", result) }} 89 | - eval: 90 | block: > 91 | {% set target_num = request.json.num %} 92 | {{ context.service.scale(desired_num=target_num) }} 93 | - docker-compose: 94 | project_name: scaling 95 | directory: /var/tmp/scaling_project 96 | $containers: 97 | output: | 98 | {% for container in result %} 99 | name={{ container.name }} 100 | logs={{ container.logs(stdout=true) }} 101 | {% endfor %} 102 | """ 103 | 104 | self.prepare_file('test-32.yml', config) 105 | 106 | container = self.start_app_container('test-32.yml') 107 | 108 | response = self.request('/compose/scaling', service='web', num=2) 109 | 110 | self.assertEqual(response.status_code, 200) 111 | 112 | output = container.logs(stdout=True, stderr=False) 113 | 114 | self.assertIn('name=scaling_app_1\nlogs=app is running', output) 115 | self.assertIn('name=scaling_web_1\nlogs=web is running', output) 116 | self.assertIn('name=scaling_web_2\nlogs=web is running', output) 117 | 118 | def test_restart_service(self): 119 | project_yaml = """ 120 | version: '2' 121 | services: 122 | app: 123 | image: alpine 124 | command: sh -c 'echo "app started at $$(date +%s)" && sleep 3600' 125 | web: 126 | image: alpine 127 | command: sh -c 'echo "web started at $$(date +%s)" && sleep 3600' 128 | """ 129 | 130 | self.prepare_file('restarts/docker-compose.yml', project_yaml) 131 | 132 | config = """ 133 | server: 134 | host: 0.0.0.0 135 | port: 9001 136 | 137 | endpoints: 138 | - /compose/restart: 139 | actions: 140 | - docker-compose: 141 | project_name: restarts 142 | directory: /var/tmp/restarts 143 | $up: 144 | detached: true 145 | scale_override: 146 | web: 2 147 | - docker-compose: 148 | project_name: restarts 149 | directory: /var/tmp/restarts 150 | $containers: 151 | output: | 152 | {% for container in result %} 153 | Logging for {{ container.name }} 154 | {{ container.logs(stdout=true) }} 155 | {% endfor %} 156 | - log: 157 | message: '--- separator ---' 158 | - docker-compose: 159 | project_name: restarts 160 | directory: /var/tmp/restarts 161 | $up: 162 | detached: true 163 | scale_override: 164 | web: 2 165 | service_names: 166 | - web 167 | strategy: 2 168 | - docker-compose: 169 | project_name: restarts 170 | directory: /var/tmp/restarts 171 | $containers: 172 | service_names: 173 | - app 174 | output: > 175 | {{ context.set('containers', result) }} 176 | - eval: 177 | block: > 178 | {% for container in context.containers %} 179 | {{ container.restart(timeout=request.json.timeout) }} 180 | {% endfor %} 181 | - docker-compose: 182 | project_name: restarts 183 | directory: /var/tmp/restarts 184 | $containers: 185 | output: | 186 | {% for container in result %} 187 | Logging for {{ container.name }} 188 | {{ container.logs(stdout=true) }} 189 | {% endfor %} 190 | """ 191 | 192 | self.prepare_file('test-33.yml', config) 193 | 194 | container = self.start_app_container('test-33.yml') 195 | 196 | response = self.request('/compose/restart', timeout=1) 197 | 198 | self.assertEqual(response.status_code, 200) 199 | 200 | output = container.logs(stdout=True, stderr=False) 201 | 202 | initial, after_restart = output.split('--- separator ---') 203 | 204 | self.assertEqual(initial.count('app started'), 1) 205 | self.assertEqual(initial.count('web started'), 2) 206 | 207 | self.assertEqual(after_restart.count('app started'), 2) # from project.up and from its own restart 208 | self.assertEqual(after_restart.count('web started'), 2) 209 | -------------------------------------------------------------------------------- /tests/it_docker_swarm.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from integrationtest_helper import IntegrationTestBase 4 | 5 | 6 | def skip_below_version(version): 7 | def decorator(f): 8 | def wrapper(self, *args, **kwargs): 9 | if map(int, self.DIND_VERSION.split('.')) < map(int, version.split('.')): 10 | self.skipTest(reason='Skipping %s on version %s (< %s)' % (f.__name__, self.DIND_VERSION, version)) 11 | else: 12 | f(self, *args, **kwargs) 13 | 14 | return wrapper 15 | 16 | return decorator 17 | 18 | 19 | class DockerSwarmIntegrationTest(IntegrationTestBase): 20 | 21 | @classmethod 22 | def setUpClass(cls): 23 | super(DockerSwarmIntegrationTest, cls).setUpClass() 24 | 25 | cls.prepare_images('alpine') 26 | cls.dind_container.exec_run('docker swarm init') 27 | 28 | def tearDown(self): 29 | super(DockerSwarmIntegrationTest, self).tearDown() 30 | 31 | for service in self.remote_client.services.list(): 32 | service.remove() 33 | 34 | def test_list_services_using_docker_action(self): 35 | config = """ 36 | server: 37 | host: 0.0.0.0 38 | port: 9001 39 | 40 | endpoints: 41 | - /docker/services/list: 42 | actions: 43 | - docker: 44 | $services: 45 | $list: 46 | output: | 47 | {% for service in result %} 48 | s={{ service.name }}#{{ service.id }} 49 | {% endfor %} 50 | """ 51 | 52 | self.prepare_file('test-41.yml', config) 53 | 54 | service = self.remote_client.services.create('alpine', 55 | name='sample-app', 56 | command='sh -c "date +%s ; sleep 3600"', 57 | stop_grace_period=1) 58 | 59 | self.wait_for_service_start(service, num_tasks=1) 60 | 61 | self.assertGreater(len(self.get_service_logs(service)), 0) 62 | 63 | container = self.start_app_container('test-41.yml') 64 | 65 | response = self.request('/docker/services/list', data='none') 66 | 67 | self.assertEqual(response.status_code, 200) 68 | 69 | output = container.logs(stdout=True, stderr=False) 70 | 71 | self.assertIn('s=%s#%s' % (service.name, service.id), output) 72 | 73 | @skip_below_version('1.13') 74 | def test_restart_service(self): 75 | config = """ 76 | server: 77 | host: 0.0.0.0 78 | port: 9001 79 | 80 | endpoints: 81 | - /docker/swarm/restart: 82 | actions: 83 | - docker-swarm: 84 | $restart: 85 | service_id: '{{ request.json.service }}' 86 | """ 87 | 88 | self.prepare_file('test-42.yml', config) 89 | 90 | service = self.remote_client.services.create('alpine', 91 | name='sample-app', 92 | command='sh -c "echo \"Starting\" ; sleep 3600"', 93 | stop_grace_period=1) 94 | 95 | self.wait_for_service_start(service, num_tasks=1) 96 | 97 | logs = self.get_service_logs(service) 98 | 99 | self.assertEqual(logs.count('Starting'), 1) 100 | 101 | self.start_app_container('test-42.yml') 102 | 103 | response = self.request('/docker/swarm/restart', service='sample-app') 104 | 105 | self.assertEqual(response.status_code, 200) 106 | 107 | self.wait_for_service_start(service, num_tasks=2) 108 | 109 | logs = self.get_service_logs(service) 110 | 111 | self.assertEqual(logs.count('Starting'), 2) 112 | 113 | def test_scale_service(self): 114 | config = """ 115 | server: 116 | host: 0.0.0.0 117 | port: 9001 118 | 119 | endpoints: 120 | - /docker/swarm/scale: 121 | actions: 122 | - docker-swarm: 123 | $scale: 124 | service_id: '{{ request.json.service }}' 125 | replicas: '{{ request.json.replicas }}' 126 | """ 127 | 128 | self.prepare_file('test-43.yml', config) 129 | 130 | service = self.remote_client.services.create('alpine', 131 | name='sample-app', 132 | command='sh -c "echo \"Starting\" ; sleep 3600"', 133 | stop_grace_period=1) 134 | 135 | self.wait_for_service_start(service, num_tasks=1) 136 | 137 | self.assertEqual(len(service.tasks()), 1) 138 | 139 | self.start_app_container('test-43.yml') 140 | 141 | response = self.request('/docker/swarm/scale', service='sample-app', replicas=2) 142 | 143 | self.assertEqual(response.status_code, 200) 144 | 145 | self.wait_for_service_start(service, num_tasks=2) 146 | 147 | self.assertGreaterEqual(len(service.tasks(filters={'desired-state': 'running'})), 2) 148 | 149 | def test_update_service(self): 150 | config = """ 151 | server: 152 | host: 0.0.0.0 153 | port: 9001 154 | 155 | endpoints: 156 | - /docker/swarm/update: 157 | actions: 158 | - docker-swarm: 159 | $update: 160 | service_id: '{{ request.json.service }}' 161 | command: '{{ request.json.command }}' 162 | labels: 163 | label_1: 'sample' 164 | label_2: '{{ request.json.label }}' 165 | """ 166 | 167 | self.prepare_file('test-44.yml', config) 168 | 169 | service = self.remote_client.services.create('alpine', 170 | name='sample-app', 171 | command='sh -c "echo \"Starting\" ; sleep 3600"', 172 | stop_grace_period=1) 173 | 174 | self.wait_for_service_start(service, num_tasks=1) 175 | 176 | self.start_app_container('test-44.yml') 177 | 178 | response = self.request('/docker/swarm/update', 179 | service='sample-app', 180 | command='sh -c "echo \"Updated\" ; sleep 300"', 181 | label='testing') 182 | 183 | self.assertEqual(response.status_code, 200) 184 | 185 | self.wait_for_service_start(service, num_tasks=2) 186 | 187 | service.reload() 188 | 189 | self.assertEqual(service.attrs.get('Spec').get('Labels', dict()).get('label_1'), 'sample') 190 | self.assertEqual(service.attrs.get('Spec').get('Labels', dict()).get('label_2'), 'testing') 191 | 192 | logs = self.get_service_logs(service) 193 | 194 | self.assertIn('Starting', logs) 195 | self.assertIn('Updated', logs) 196 | 197 | @staticmethod 198 | def wait_for_service_start(service, num_tasks, max_wait=30): 199 | for _ in range(max_wait * 2): 200 | if len(service.tasks()) >= num_tasks: 201 | tasks_to_run = service.tasks(filters={'desired-state': 'running'}) 202 | 203 | if len(tasks_to_run) > 0 and all(task['Status']['State'] == 'running' for task in tasks_to_run): 204 | break 205 | 206 | time.sleep(0.5) 207 | 208 | def get_service_logs(self, service, stdout=True, stderr=False): 209 | logs = list() 210 | 211 | for container in self.remote_client.containers.list(all=True, filters={'name': service.name}): 212 | logs.extend(''.join(char for char in container.logs(stdout=stdout, stderr=stderr)).splitlines()) 213 | 214 | return filter(len, map(lambda x: x.strip(), logs)) 215 | -------------------------------------------------------------------------------- /tests/it_execute.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | from integrationtest_helper import IntegrationTestBase 4 | 5 | 6 | class ExecuteIntegrationTest(IntegrationTestBase): 7 | 8 | def test_execute_hostname(self): 9 | config = """ 10 | server: 11 | host: 0.0.0.0 12 | port: 9001 13 | 14 | endpoints: 15 | - /command: 16 | method: 'POST' 17 | 18 | actions: 19 | - execute: 20 | command: > 21 | {{ request.json.command }} 22 | output: 'host={{ result }}' 23 | shell: false 24 | """ 25 | 26 | self.prepare_file('test-01.yml', config) 27 | 28 | container = self.start_app_container('test-01.yml') 29 | 30 | response = self.request('/command', command='hostname') 31 | 32 | self.assertEqual(response.status_code, 200) 33 | 34 | output = container.logs(stdout=True, stderr=False) 35 | 36 | self.assertIn('host=%s' % container.attrs['Config']['Hostname'], output.strip()) 37 | 38 | def test_execute_base64(self): 39 | config = """ 40 | server: 41 | host: 0.0.0.0 42 | port: 9001 43 | 44 | endpoints: 45 | - /b64: 46 | method: 'POST' 47 | 48 | actions: 49 | - execute: 50 | command: echo -n "{{ request.json.content }}" | base64 - 51 | output: encoded={{ result }} 52 | shell: true 53 | """ 54 | 55 | self.prepare_file('test-02.yml', config) 56 | 57 | container = self.start_app_container('test-02.yml') 58 | 59 | response = self.request('/b64', content='testing') 60 | 61 | self.assertEqual(response.status_code, 200) 62 | 63 | response = self.request('/b64', content='sample') 64 | 65 | self.assertEqual(response.status_code, 200) 66 | 67 | output = container.logs(stdout=True, stderr=False) 68 | 69 | self.assertIn('encoded=%s' % base64.b64encode('testing'), output) 70 | self.assertIn('encoded=%s' % base64.b64encode('sample'), output) 71 | 72 | -------------------------------------------------------------------------------- /tests/it_import.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from integrationtest_helper import IntegrationTestBase 4 | 5 | 6 | class ImportIntegrationTest(IntegrationTestBase): 7 | 8 | def test_import(self): 9 | current_dir = os.path.dirname(__file__) 10 | 11 | with open(os.path.join(current_dir, 'imports/test1/action.py')) as action1: 12 | self.prepare_file('extra_action_1.py', action1.read()) 13 | with open(os.path.join(current_dir, 'imports/test2/action.py')) as action2: 14 | self.prepare_file('extra_action_2.py', action2.read()) 15 | 16 | config = """ 17 | server: 18 | host: 0.0.0.0 19 | port: 9001 20 | imports: 21 | - /var/tmp/extra_action_1.py 22 | - /var/tmp/extra_action_2.py 23 | 24 | endpoints: 25 | - /imports: 26 | method: 'POST' 27 | 28 | actions: 29 | - test1: 30 | action: test-1 31 | - test2: 32 | action: test-2 33 | """ 34 | 35 | self.prepare_file('test-61.yml', config) 36 | 37 | container = self.start_app_container('test-61.yml') 38 | 39 | response = self.request('/imports', test='test') 40 | 41 | self.assertEqual(response.status_code, 200) 42 | 43 | output = container.logs(stdout=True, stderr=False) 44 | 45 | self.assertIn('action=test-1', output.strip()) 46 | self.assertIn('action=test-2', output.strip()) 47 | 48 | -------------------------------------------------------------------------------- /tests/it_log.py: -------------------------------------------------------------------------------- 1 | from integrationtest_helper import IntegrationTestBase 2 | 3 | 4 | class LogIntegrationTest(IntegrationTestBase): 5 | 6 | def test_multi_line_log(self): 7 | config = """ 8 | server: 9 | host: 0.0.0.0 10 | port: 9001 11 | 12 | endpoints: 13 | - /multi: 14 | method: 'POST' 15 | 16 | actions: 17 | - log: 18 | message: | 19 | This is a multi line log, 20 | for the request: {{ request.path }} 21 | """ 22 | 23 | self.prepare_file('test-11.yml', config) 24 | 25 | container = self.start_app_container('test-11.yml') 26 | 27 | response = self.request('/multi', data='none') 28 | 29 | self.assertEqual(response.status_code, 200) 30 | 31 | output = container.logs(stdout=True, stderr=False, tail=20) 32 | 33 | self.assertIn('This is a multi line log,', output) 34 | self.assertIn('for the request: /multi', output) 35 | 36 | def test_multiple_actions(self): 37 | config = """ 38 | server: 39 | host: 0.0.0.0 40 | port: 9001 41 | 42 | endpoints: 43 | - /log/messages: 44 | method: 'POST' 45 | 46 | actions: 47 | - log: 48 | - log: 49 | message: Plain text 50 | - log: 51 | message: 'With request data: {{ request.json.content }}' 52 | """ 53 | 54 | self.prepare_file('test-12.yml', config) 55 | 56 | container = self.start_app_container('test-12.yml') 57 | 58 | response = self.request('/log/messages', content='sample') 59 | 60 | self.assertEqual(response.status_code, 200) 61 | 62 | output = container.logs(stdout=True, stderr=False, tail=20) 63 | 64 | self.assertIn('Processing /log/messages ...', output) 65 | self.assertIn('Plain text', output) 66 | self.assertIn('With request data: sample', output) 67 | 68 | -------------------------------------------------------------------------------- /tests/it_metrics.py: -------------------------------------------------------------------------------- 1 | from integrationtest_helper import IntegrationTestBase 2 | 3 | 4 | class MetricsIntegrationTest(IntegrationTestBase): 5 | 6 | def test_metrics(self): 7 | config = """ 8 | server: 9 | host: 0.0.0.0 10 | port: 9001 11 | 12 | endpoints: 13 | - /test/metrics: 14 | method: 'POST' 15 | 16 | actions: 17 | - log: 18 | - log: 19 | message: Plain text 20 | - log: 21 | message: 'With request data: {{ request.json.content }}' 22 | """ 23 | 24 | self.prepare_file('test-51.yml', config) 25 | 26 | self.start_app_container('test-51.yml') 27 | 28 | response = self.request('/test/metrics', content='sample') 29 | 30 | self.assertEqual(response.status_code, 200) 31 | 32 | response = self.metrics() 33 | 34 | self.assertEqual(response.status_code, 200) 35 | 36 | metrics = response.text 37 | 38 | self.assertIn('python_info{', metrics) 39 | self.assertIn('process_start_time_seconds ', metrics) 40 | 41 | self.assertIn('flask_http_request_total{' 42 | 'method="POST",status="200"} 1.0', metrics) 43 | self.assertIn('flask_http_request_duration_seconds_count{' 44 | 'method="POST",path="/test/metrics",status="200"} 1.0', metrics) 45 | 46 | self.assertIn('webhook_proxy_actions_count{' 47 | 'action_index="0",action_type="log",' 48 | 'http_method="POST",http_route="/test/metrics"} 1.0', metrics) 49 | self.assertIn('webhook_proxy_actions_count{' 50 | 'action_index="1",action_type="log",' 51 | 'http_method="POST",http_route="/test/metrics"} 1.0', metrics) 52 | self.assertIn('webhook_proxy_actions_count{' 53 | 'action_index="2",action_type="log",' 54 | 'http_method="POST",http_route="/test/metrics"} 1.0', metrics) 55 | -------------------------------------------------------------------------------- /tests/test_action.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import unittest 5 | 6 | import docker_helper 7 | 8 | from actions import action, Action 9 | from server import ConfigurationException 10 | from unittest_helper import ActionTestBase, capture_stream 11 | 12 | 13 | class ActionTest(ActionTestBase): 14 | def test_simple_log(self): 15 | actions = [{'log': {'message': 'Hello there!'}}] 16 | output = self._invoke(actions) 17 | 18 | self.assertEqual(output, 'Hello there!') 19 | 20 | def test_log_with_variable(self): 21 | actions = [{'log': {'message': 'HTTP {{ request.method }} {{ request.path }}'}}] 22 | 23 | output = self._invoke(actions) 24 | 25 | self.assertEqual(output, 'HTTP POST /testing') 26 | 27 | def test_evaluate(self): 28 | actions = [{'execute': { 29 | 'command': 'echo -n "Hello"', 30 | 'output': '{% set _ = context.set("message", result) %}' 31 | }}, { 32 | 'eval': { 33 | 'block': 'Hello=={{ context.message }}' 34 | } 35 | }] 36 | 37 | output = self._invoke(actions) 38 | 39 | self.assertEqual(output, 'Hello==Hello') 40 | 41 | def test_custom_action(self): 42 | @action('for-test') 43 | class TestAction(Action): 44 | def __init__(self, **kwargs): 45 | self.kwargs = kwargs 46 | 47 | def _run(self): 48 | print(*('%s=%s' % (key, value) for key, value in self.kwargs.items())) 49 | 50 | actions = [{'for-test': {'string': 'Hello', 'number': 12, 'bool': True}}] 51 | 52 | output = self._invoke(actions) 53 | 54 | self.assertIn('string=Hello', output.split()) 55 | self.assertIn('number=12', output.split()) 56 | self.assertIn('bool=True', output.split()) 57 | 58 | def test_raising_error(self): 59 | actions = [{'log': {'message': 60 | '{% if request.json.fail %}\n' 61 | ' {{ error("Failing with : %s"|format(request.json.fail)) }}\n' 62 | '{% else %}\n' 63 | ' All good\n' 64 | '{% endif %}'}}] 65 | 66 | output = self._invoke(actions) 67 | 68 | self.assertIn('All good', output) 69 | 70 | with capture_stream('stderr', echo=True) as output: 71 | self._invoke(actions, expected_status_code=500, body={'fail': 'test-failure'}) 72 | 73 | output = output.dumps() 74 | 75 | self.assertIn('Failing with : test-failure', output) 76 | 77 | @unittest.skipUnless(os.path.exists('/proc/1/cgroup'), 78 | 'Test is not running in a container') 79 | def test_log_with_docker_helper(self): 80 | actions = [{'log': { 81 | 'message': 'Running in {{ own_container_id }} ' 82 | 'with environment: {{ read_config("TESTING_ENV") }}' 83 | }}] 84 | 85 | os.environ['TESTING_ENV'] = 'webhook-proxy-testing' 86 | 87 | output = self._invoke(actions) 88 | 89 | expected = 'Running in %s with environment: webhook-proxy-testing' % docker_helper.get_current_container_id() 90 | 91 | self.assertEqual(output, expected) 92 | 93 | def test_invalid_action(self): 94 | actions = [{'invalid': {'Should': 'not work'}}] 95 | 96 | self.assertRaises(ConfigurationException, self._invoke, actions) 97 | 98 | def test_wrong_configuration(self): 99 | actions = [{'log': {'unknown_argument': 1}}] 100 | 101 | self.assertRaises(ConfigurationException, self._invoke, actions) 102 | -------------------------------------------------------------------------------- /tests/test_docker_action.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import docker 4 | from docker.errors import NotFound 5 | 6 | from server import ConfigurationException 7 | from unittest_helper import ActionTestBase 8 | 9 | 10 | class DockerActionTest(ActionTestBase): 11 | client = docker.from_env() 12 | 13 | @classmethod 14 | def setUpClass(cls): 15 | cls.client.containers.run(image='alpine', command='echo "Init"', remove=True) 16 | 17 | @classmethod 18 | def tearDownClass(cls): 19 | cls.client.api.close() 20 | 21 | def test_list(self): 22 | container = self.client.containers.run(image='alpine', 23 | command='sleep 10', 24 | detach=True) 25 | 26 | try: 27 | output = self._invoke({'docker': {'$containers': {'$list': None}, 28 | 'output': '{% for container in result %}' 29 | ' container={{ container.name }}' 30 | '{% endfor %}'}}) 31 | 32 | self.assertIn('container=%s' % container.name, output) 33 | 34 | finally: 35 | container.remove(force=True) 36 | 37 | def test_execute(self): 38 | name = 'testing_%s' % int(random.randint(1000, 9999)) 39 | 40 | self.assertRaises(NotFound, self.client.containers.get, name) 41 | 42 | output = self._invoke({'docker': {'$containers': {'$run': { 43 | 'image': '{{ request.json.incoming.image }}', 44 | 'command': 'sh -c \'{{ request.json.incoming.command }}\'', 45 | 'remove': True}}}}, 46 | body={'incoming': {'image': 'alpine', 'command': 'echo "Hello from Docker"'}}) 47 | 48 | self.assertEqual(output, 'Hello from Docker') 49 | 50 | def test_embedded_arguments(self): 51 | name = 'testing_%s' % int(random.randint(1000, 9999)) 52 | 53 | container = self.client.containers.run(image='alpine', command='sleep 10', 54 | name=name, detach=True) 55 | 56 | try: 57 | output = self._invoke({'docker': {'$containers': {'$list': {'filters': { 58 | 'name': '{{ request.json.incoming.pattern }}', 59 | 'status': '{{ request.json.incoming.status.value }}'}}}, 60 | 'output': '{% for container in result %}-{{ container.name }}-{% endfor %}'}}, 61 | body={'incoming': {'pattern': 'testing_', 'status': {'value': 'running'}}}) 62 | 63 | self.assertIn('-%s-' % container.name, output) 64 | 65 | finally: 66 | container.remove(force=True) 67 | 68 | def test_arguments_with_variables(self): 69 | output = self._invoke([ 70 | {'eval': {'block': '{% set _ = context.set("user", "nobody") %}'}}, 71 | {'docker': {'$containers': {'$run': { 72 | 'image': 'alpine', 'command': 'sh -c "env && echo \"user=$(whoami)\""', 73 | 'user': '{{ context.user }}', 74 | 'remove': True, 'environment': [ 75 | 'UPPER={{ "upper"|upper }}', 76 | 'LOWER={{ "LOWER"|lower }}' 77 | ]}}, 78 | 'output': '{{ result }}'}} 79 | ], 80 | body={'incoming': {'pattern': 'testing_', 'status': {'value': 'running'}}}) 81 | 82 | self.assertIn('UPPER=UPPER', output) 83 | self.assertIn('LOWER=lower', output) 84 | self.assertIn('user=nobody', output) 85 | 86 | def test_images(self): 87 | output = self._invoke({'docker': {'$images': {'$list': { 88 | 'filters': {'reference': 'alp*'}}}}}) 89 | 90 | self.assertRegexpMatches('alpine\s+latest', output) 91 | 92 | def test_no_invocation(self): 93 | self.assertRaises(ConfigurationException, self._invoke, {'docker': {'output': 'Uh-oh'}}) 94 | 95 | def test_invalid_invocation(self): 96 | self.assertRaises(ConfigurationException, self._invoke, 97 | {'docker': {'$containers': {'$takeOverTheWorld': {'when': 'now'}}}}) 98 | 99 | def test_invalid_arguments(self): 100 | self._invoke( 101 | {'docker': {'$containers': {'$list': {'filters': {'name': 'test', 'unknown': 1}}}}}, 102 | expected_status_code=500) 103 | -------------------------------------------------------------------------------- /tests/test_docker_compose_action.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | 4 | from unittest_helper import ActionTestBase 5 | 6 | 7 | class DockerComposeActionTest(ActionTestBase): 8 | def test_executions(self): 9 | directory = '/tmp/compose_test_%s' % random.randint(1000, 9999) 10 | os.makedirs(directory) 11 | 12 | with open('%s/docker-compose.yml' % directory, 'w') as composefile: 13 | composefile.write("version: '2' \n" 14 | "services: \n" 15 | " cmps_one: \n" 16 | " image: alpine \n" 17 | " command: sleep 10 \n" 18 | " stop_signal: KILL \n" 19 | " cmps_two: \n" 20 | " image: alpine \n" 21 | " command: sleep 10 \n" 22 | " stop_signal: KILL \n") 23 | 24 | try: 25 | output = self._invoke([ 26 | {'docker-compose': { 27 | 'project_name': 'testing', 28 | 'directory': directory, 29 | '$up': { 30 | 'detached': True 31 | }, 32 | 'output': 'Compose containers:\n' 33 | '{% for container in result %}' 34 | '-C- {{ container.name }}\n' 35 | '{% endfor %}' 36 | }}, 37 | {'docker-compose': { 38 | 'project_name': 'testing', 39 | 'directory': directory, 40 | '$down': { 41 | 'remove_image_type': False, 42 | 'include_volumes': True 43 | } 44 | }} 45 | ]) 46 | 47 | self.assertIn('-C- testing_cmps_one_1', output) 48 | self.assertIn('-C- testing_cmps_two_1', output) 49 | 50 | finally: 51 | os.remove('%s/docker-compose.yml' % directory) 52 | os.rmdir(directory) 53 | -------------------------------------------------------------------------------- /tests/test_docker_swarm_action.py: -------------------------------------------------------------------------------- 1 | from unittest_helper import ActionTestBase 2 | from actions.action_docker_swarm import DockerSwarmAction 3 | 4 | 5 | class DockerSwarmActionTest(ActionTestBase): 6 | def setUp(self): 7 | self.mock_client = MockClient() 8 | DockerSwarmAction.client = self.mock_client 9 | 10 | def test_restart(self): 11 | self._invoke({'docker-swarm': {'$restart': {'service_id': 'mock-service'}}}) 12 | 13 | self.verify('force_update', 1) 14 | 15 | self.mock_client.service_attributes = { 16 | 'Spec': {'TaskTemplate': {'ForceUpdate': 12}} 17 | } 18 | 19 | self._invoke({'docker-swarm': {'$restart': {'service_id': 'fake'}}}) 20 | 21 | self.verify('force_update', 13) 22 | 23 | def test_scale(self): 24 | self._invoke({'docker-swarm': {'$scale': {'service_id': 'mocked', 'replicas': 12}}}) 25 | 26 | self.verify('mode', {'replicated': {'Replicas': 12}}) 27 | 28 | def test_update(self): 29 | self._invoke({'docker-swarm': {'$update': { 30 | 'service_id': 'updating', 31 | 'image': 'test-image:1.0.y' 32 | }}}) 33 | 34 | self.verify('image', 'test-image:1.0.y') 35 | 36 | self._invoke({'docker-swarm': {'$update': { 37 | 'service_id': 'updating', 38 | 'container_labels': [{'test.label': 'test', 'mock.label': 'mock'}] 39 | }}}) 40 | 41 | self.verify('container_labels', 42 | [{'test.label': 'test', 'mock.label': 'mock'}]) 43 | 44 | self._invoke({'docker-swarm': {'$update': { 45 | 'service_id': 'updating', 46 | 'labels': [{'service.label': 'testing'}] 47 | }}}) 48 | 49 | self.verify('labels', [{'service.label': 'testing'}]) 50 | 51 | self._invoke({'docker-swarm': {'$update': { 52 | 'service_id': 'updating', 53 | 'resources': {'mem_limit': 512} 54 | }}}) 55 | 56 | self.verify('resources', {'mem_limit': 512}) 57 | 58 | def verify(self, key, value): 59 | def assertPropertyEquals(data, prop): 60 | self.assertIsNotNone(data) 61 | 62 | if '.' in prop: 63 | current, remainder = prop.split('.', 1) 64 | assertPropertyEquals(data.get(current), remainder) 65 | else: 66 | self.assertEqual(data.get(prop), value, 67 | msg='%s != %s for %s' % (data.get(prop), value, key)) 68 | 69 | assertPropertyEquals(self.mock_client.last_update, key) 70 | 71 | 72 | class MockClient(object): 73 | def __init__(self): 74 | self.last_update = dict() 75 | self.service_attributes = None 76 | 77 | @property 78 | def services(self): 79 | return self 80 | 81 | def get(self, *args, **kwargs): 82 | details = Mock(attrs={ 83 | 'ID': 'testId', 84 | 'Version': {'Index': 12}, 85 | 'Spec': { 86 | 'Name': args[0], 87 | 'Mode': {'Replicated': {'Replicas': 1}}, 88 | 'TaskTemplate': { 89 | 'ContainerSpec': { 90 | 'Image': 'alpine:mock' 91 | }, 92 | 'ForceUpdate': 0 93 | } 94 | } 95 | }, 96 | reload=lambda: True, 97 | update=self.update_service, 98 | decode=lambda: details) 99 | 100 | if self.service_attributes: 101 | self._merge_attributes(details.attrs, self.service_attributes) 102 | 103 | return details 104 | 105 | def _merge_attributes(self, details, overwrite): 106 | for key, value in overwrite.items(): 107 | if key not in details: 108 | details[key] = value 109 | elif isinstance(value, dict): 110 | self._merge_attributes(details[key], overwrite[key]) 111 | else: 112 | details[key] = value 113 | 114 | def update_service(self, **kwargs): 115 | self.last_update = kwargs 116 | return True 117 | 118 | 119 | class Mock(dict): 120 | def __getattr__(self, name): 121 | return self.get(name) 122 | 123 | def update(self, *args, **kwargs): 124 | return self['update'](*args, **kwargs) 125 | -------------------------------------------------------------------------------- /tests/test_execute_action.py: -------------------------------------------------------------------------------- 1 | from unittest_helper import ActionTestBase 2 | 3 | 4 | class ExecuteActionTest(ActionTestBase): 5 | def test_echo(self): 6 | output = self._invoke({'execute': {'command': 'echo "Hello there!"'}}) 7 | 8 | self.assertEqual(output, 'Hello there!') 9 | 10 | def test_format_output(self): 11 | output = self._invoke({'execute': {'command': 'echo "one"; echo "two"', 12 | 'output': '{% for line in result.splitlines() %}' 13 | 'line={{ line }}' 14 | '{% endfor %}'}}) 15 | 16 | self.assertIn('line=one', output) 17 | self.assertIn('line=two', output) 18 | 19 | def test_alternative_shell(self): 20 | output = self._invoke({'execute': {'command': 'echo "Hello from Bash"', 21 | 'shell': 'bash'}}) 22 | 23 | self.assertEqual(output, 'Hello from Bash') 24 | 25 | def test_no_shell(self): 26 | output = self._invoke({'execute': {'command': ['ls', '-l', '/'], 'shell': False}}) 27 | 28 | self.assertIn(' bin', output) 29 | self.assertIn(' usr', output) 30 | self.assertIn(' var', output) 31 | 32 | def test_custom_shell_command(self): 33 | output = self._invoke({'execute': {'command': ['ls', '-l', '/'], 'shell': ['bash', '-c']}}) 34 | 35 | self.assertIn(' bin', output) 36 | self.assertIn(' usr', output) 37 | self.assertIn(' var', output) 38 | -------------------------------------------------------------------------------- /tests/test_github_verify_action.py: -------------------------------------------------------------------------------- 1 | from unittest_helper import ActionTestBase 2 | 3 | import os 4 | 5 | 6 | class ExecuteActionTest(ActionTestBase): 7 | def setUp(self): 8 | self.original_headers = self._headers 9 | 10 | def tearDown(self): 11 | self._headers = self.original_headers 12 | 13 | def test_successful_github_webhook(self): 14 | self._headers['X-Hub-Signature'] = 'sha1=28fdad22dac5d0a631b5ead69dbb05e43b685d6e' 15 | 16 | with open(os.path.join(os.path.dirname(__file__), 'github/webhook.json')) as webhook: 17 | output = self._invoke({'github-verify': { 18 | 'secret': 'TopSecret' 19 | }}, final_body=webhook.read()) 20 | 21 | self.assertEqual(output, 'GitHub webhook successfully validated') 22 | 23 | def test_invalid_github_hash_algorithm(self): 24 | self._headers['X-Hub-Signature'] = 'md5=a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0' 25 | 26 | with open(os.path.join(os.path.dirname(__file__), 'github/webhook.json')) as webhook: 27 | self._invoke({'github-verify': { 28 | 'secret': 'TopSecret' 29 | }}, final_body=webhook.read(), expected_status_code=500) 30 | 31 | def test_invalid_github_signature(self): 32 | self._headers['X-Hub-Signature'] = 'sha1=a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0' 33 | 34 | with open(os.path.join(os.path.dirname(__file__), 'github/webhook.json')) as webhook: 35 | self._invoke({'github-verify': { 36 | 'secret': 'TopSecret' 37 | }}, final_body=webhook.read(), expected_status_code=500) 38 | 39 | def test_invalid_github_secret(self): 40 | self._headers['X-Hub-Signature'] = 'sha1=28fdad22dac5d0a631b5ead69dbb05e43b685d6e' 41 | 42 | with open(os.path.join(os.path.dirname(__file__), 'github/webhook.json')) as webhook: 43 | self._invoke({'github-verify': { 44 | 'secret': 'ThisIsNotRight' 45 | }}, final_body=webhook.read(), expected_status_code=500) 46 | 47 | def test_missing_github_header(self): 48 | with open(os.path.join(os.path.dirname(__file__), 'github/webhook.json')) as webhook: 49 | self._invoke({'github-verify': { 50 | 'secret': 'NoHeader' 51 | }}, final_body=webhook.read(), expected_status_code=500) 52 | -------------------------------------------------------------------------------- /tests/test_http_action.py: -------------------------------------------------------------------------------- 1 | import json 2 | import threading 3 | 4 | import six 5 | 6 | if six.PY2: 7 | from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler 8 | else: 9 | from http.server import HTTPServer, BaseHTTPRequestHandler 10 | 11 | from unittest_helper import ActionTestBase, capture_stream 12 | 13 | 14 | class HttpActionTest(ActionTestBase): 15 | def setUp(self): 16 | self.http_server = None 17 | 18 | def tearDown(self): 19 | if self.http_server: 20 | self.http_server.shutdown() 21 | 22 | def _start_server(self): 23 | class Handler(BaseHTTPRequestHandler): 24 | def do_GET(self): 25 | self._do_ANY() 26 | 27 | def do_POST(self): 28 | self._do_ANY() 29 | 30 | def do_PUT(self): 31 | self._do_ANY() 32 | 33 | def _do_ANY(self): 34 | content_length = int(self.headers.get('Content-Length', '0')) 35 | if content_length and \ 36 | self.headers.get('Content-Type', 'application/json') == 'application/json': 37 | 38 | payload = self.rfile.read(content_length) 39 | 40 | if not isinstance(payload, str) and hasattr(payload, 'decode'): 41 | payload = payload.decode() 42 | 43 | posted = json.loads(payload) 44 | 45 | else: 46 | posted = dict() 47 | 48 | if self.headers.get('X-Fail'): 49 | self.send_error(int(self.headers.get('X-Fail'))) 50 | 51 | else: 52 | self.send_response(200) 53 | 54 | self.end_headers() 55 | 56 | self.wfile.write(six.b('Test finished\n')) 57 | 58 | self.wfile.write(six.b('uri=%s\n' % self.path)) 59 | self.wfile.write(six.b('method=%s\n' % self.command)) 60 | 61 | for key, value in self.headers.items(): 62 | self.wfile.write(six.b('H %s=%s\n' % (key.lower(), value))) 63 | 64 | if isinstance(posted, dict): 65 | for key, value in posted.items(): 66 | if isinstance(value, dict): 67 | self.wfile.write(six.b('B %s=%s\n' % (key, json.dumps(value)))) 68 | 69 | else: 70 | self.wfile.write(six.b('B %s=%s\n' % (key, value))) 71 | 72 | elif isinstance(posted, list): 73 | for value in posted: 74 | if isinstance(value, dict): 75 | self.wfile.write(six.b('AR %s\n' % json.dumps(value))) 76 | 77 | else: 78 | self.wfile.write(six.b('AR %s\n' % value)) 79 | 80 | self.http_server = HTTPServer(('127.0.0.1', 0), Handler) 81 | 82 | def run_server(): 83 | self.http_server.serve_forever() 84 | 85 | threading.Thread(target=run_server).start() 86 | 87 | def _invoke_http(self, **kwargs): 88 | self._start_server() 89 | 90 | self.assertIsNotNone(self.http_server) 91 | 92 | port = self.http_server.server_port 93 | self.assertIsNotNone(port) 94 | self.assertGreater(port, 0) 95 | 96 | expected_status_code = kwargs.pop('expected_status_code', 200) 97 | 98 | args = kwargs.copy() 99 | 100 | if 'target' not in args: 101 | args['target'] = 'http://127.0.0.1:%s' % port 102 | 103 | elif args['target'].startswith('/'): 104 | args['target'] = 'http://127.0.0.1:%s%s' % (port, args['target']) 105 | 106 | return self._invoke({'http': args}, expected_status_code) 107 | 108 | def test_simple_http(self): 109 | output = self._invoke_http( 110 | headers={'X-Test': 'Hello'}, 111 | body=json.dumps({'key': 'value'})) 112 | 113 | self.assertIn('method=POST', output) 114 | self.assertIn('H x-test=Hello', output) 115 | self.assertIn('B key=value', output) 116 | 117 | def test_put_method(self): 118 | output = self._invoke_http( 119 | method='PUT', 120 | target='/put-test', 121 | headers={'X-Method': 'PUT'}, 122 | body=json.dumps({'method': 'PUT'})) 123 | 124 | self.assertIn('method=PUT', output) 125 | self.assertIn('uri=/put-test', output) 126 | self.assertIn('H x-method=PUT', output) 127 | self.assertIn('B method=PUT', output) 128 | 129 | def test_get_method(self): 130 | output = self._invoke_http( 131 | method='GET', 132 | target='/some/remote/endpoint', 133 | headers={'Accept': 'text/plain', 'X-Test': 'test_get_method'}) 134 | 135 | self.assertIn('method=GET', output) 136 | self.assertIn('uri=/some/remote/endpoint', output) 137 | self.assertIn('H accept=text/plain', output) 138 | self.assertIn('H x-test=test_get_method', output) 139 | 140 | def test_target_replacement(self): 141 | output = self._invoke_http( 142 | target='/some/{{ "remote" }}/endpoint') 143 | 144 | self.assertIn('uri=/some/remote/endpoint', output) 145 | 146 | def test_header_replacement(self): 147 | output = self._invoke_http( 148 | headers={'X-Original-Path': '{{ request.path }}'}) 149 | 150 | self.assertIn('H x-original-path=/testing', output) 151 | 152 | def test_body_replacement(self): 153 | output = self._invoke_http( 154 | body=json.dumps({'original': {'request': {'path': '{{ request.path }}'}}})) 155 | 156 | self.assertIn('B original={"request": {"path": "/testing"}}', output) 157 | 158 | def test_json_output_with_replacement(self): 159 | output = self._invoke_http( 160 | json=True, 161 | body={'original': {'request': {'path': '{{ request.path }}'}}}) 162 | 163 | self.assertIn('B original={"request": {"path": "/testing"}}', output) 164 | 165 | def test_json_output_with_array(self): 166 | output = self._invoke_http( 167 | json=True, 168 | body=[ 169 | {'path': '{{ request.path }}'}, 170 | {'method': '{{ request.method }}'}, 171 | {'options': ['a', 'list']} 172 | ] 173 | ) 174 | 175 | self.assertIn('AR {"path": "/testing"}', output) 176 | self.assertIn('AR {"method": "POST"}', output) 177 | self.assertIn('AR {"options": ["a", "list"]}', output) 178 | 179 | def test_output_formatting(self): 180 | output = self._invoke_http( 181 | output='HTTP::{{ response.status_code }}') 182 | 183 | self.assertEqual(output, 'HTTP::200') 184 | 185 | def test_content_length_header(self): 186 | message = 'Hello there!' 187 | 188 | output = self._invoke_http( 189 | headers={'Content-Type': 'text/plain'}, 190 | body=message) 191 | 192 | self.assertIn('H content-length=%s' % len(message), output) 193 | 194 | def test_404_response(self): 195 | output = self._invoke_http(headers={'X-Fail': '404'}) 196 | 197 | self.assertIn('HTTP 404', output) 198 | 199 | def test_503_response(self): 200 | output = self._invoke_http(headers={'X-Fail': '503'}) 201 | 202 | self.assertIn('HTTP 503', output) 203 | 204 | def test_fail_on_error(self): 205 | with capture_stream('stderr') as stderr: 206 | output = self._invoke_http( 207 | headers={'X-Fail': '500'}, 208 | fail_on_error=True, 209 | expected_status_code=500 210 | ) 211 | 212 | error = stderr.dumps() 213 | 214 | self.assertIn('ActionInvocationException: HTTP call failed (HTTP 500)', error) 215 | 216 | def test_does_not_fail_on_success(self): 217 | output = self._invoke_http( 218 | headers={'X-Fail': '204'}, 219 | output='HTTP::{{ response.status_code }}', 220 | fail_on_error=True 221 | ) 222 | 223 | self.assertIn('HTTP::204', output) 224 | -------------------------------------------------------------------------------- /tests/test_import.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from util import ConfigurationException 4 | 5 | from unittest_helper import ActionTestBase 6 | 7 | 8 | class ImportTest(ActionTestBase): 9 | base_dir = os.path.join(os.path.dirname(__file__), 'imports') 10 | 11 | def test_import_external_action(self): 12 | actions_module_path = os.path.join(self.base_dir, 'actions_to_load.py') 13 | 14 | output = self._invoke([ 15 | { 16 | 'sample': {'x': 1} 17 | }, 18 | { 19 | 'json': {'bool': True, 'num': 2, 'str': 'abc'} 20 | } 21 | ], imports=[actions_module_path]) 22 | 23 | self.assertIn('x=1', output) 24 | self.assertIn('"bool": true', output) 25 | self.assertIn('"num": 2', output) 26 | self.assertIn('"str": "abc"', output) 27 | 28 | def test_imports_with_same_filename(self): 29 | imports = [ 30 | os.path.join(self.base_dir, 'test1/action.py'), 31 | os.path.join(self.base_dir, 'test2/action.py') 32 | ] 33 | 34 | output = self._invoke([ 35 | {'test1': {'action': 'test-1'}}, 36 | {'test2': {'action': 'test-2'}} 37 | ], imports=imports) 38 | 39 | self.assertIn('action=test-1', output) 40 | self.assertIn('action=test-2', output) 41 | 42 | def test_raises_configuration_exception_on_failure(self): 43 | self.assertRaises(ConfigurationException, self._invoke, list(), imports=['not-found']) 44 | self.assertRaises(ConfigurationException, self._invoke, list(), 45 | imports=[os.path.join(self.base_dir, 'invalid.py')]) 46 | -------------------------------------------------------------------------------- /tests/test_metrics_action.py: -------------------------------------------------------------------------------- 1 | from unittest_helper import ActionTestBase, unregister_metrics 2 | 3 | from server import Server 4 | from util import ConfigurationException 5 | 6 | 7 | class MetricsActionTest(ActionTestBase): 8 | def test_histogram(self): 9 | output = self._invoke({'metrics': {'histogram': dict( 10 | name='test_histogram', help='Test Histogram' 11 | )}}) 12 | 13 | self.assertEqual(output, 'Tracking metrics: test_histogram') 14 | self.assertIn('test_histogram_count 1.0', self.metrics()) 15 | self.assertIn('test_histogram_sum 0.', self.metrics()) 16 | self.assertIn('test_histogram_bucket{le=', self.metrics()) 17 | 18 | unregister_metrics() 19 | 20 | output = self._invoke({'metrics': {'histogram': dict( 21 | name='test_histogram_with_labels', help='Test Histogram', 22 | labels={'path': '{{ request.path }}'} 23 | )}}) 24 | 25 | self.assertEqual(output, 'Tracking metrics: test_histogram_with_labels') 26 | self.assertIn('test_histogram_with_labels_count{path="/testing"} 1.0', self.metrics()) 27 | self.assertIn('test_histogram_with_labels_sum{path="/testing"} 0.', self.metrics()) 28 | self.assertIn('test_histogram_with_labels_bucket{le=', self.metrics()) 29 | 30 | def test_summary(self): 31 | output = self._invoke({'metrics': {'summary': dict( 32 | name='test_summary', help='Test Summary' 33 | )}}) 34 | 35 | self.assertEqual(output, 'Tracking metrics: test_summary') 36 | self.assertIn('test_summary_count 1.0', self.metrics()) 37 | self.assertIn('test_summary_sum 0.', self.metrics()) 38 | 39 | unregister_metrics() 40 | 41 | output = self._invoke({'metrics': {'summary': dict( 42 | name='test_summary_with_labels', help='Test Summary', 43 | labels={'path': '{{ request.path }}'} 44 | )}}) 45 | 46 | self.assertEqual(output, 'Tracking metrics: test_summary_with_labels') 47 | self.assertIn('test_summary_with_labels_count{path="/testing"} 1.0', self.metrics()) 48 | self.assertIn('test_summary_with_labels_sum{path="/testing"} 0.', self.metrics()) 49 | 50 | def test_gauge(self): 51 | output = self._invoke({'metrics': {'gauge': dict( 52 | name='test_gauge', help='Test Gauge' 53 | )}}) 54 | 55 | self.assertEqual(output, 'Tracking metrics: test_gauge') 56 | self.assertIn('test_gauge 0.0', self.metrics()) 57 | 58 | unregister_metrics() 59 | 60 | output = self._invoke({'metrics': {'gauge': dict( 61 | name='test_gauge_with_labels', help='Test Gauge', 62 | labels={'target': '{{ request.json.target }}'} 63 | )}}, body=dict(target='sample')) 64 | 65 | self.assertEqual(output, 'Tracking metrics: test_gauge_with_labels') 66 | self.assertIn('test_gauge_with_labels{target="sample"} 0.0', self.metrics()) 67 | 68 | def test_counter(self): 69 | output = self._invoke({'metrics': {'counter': dict( 70 | name='test_counter', help='Test Counter' 71 | )}}) 72 | 73 | self.assertEqual(output, 'Tracking metrics: test_counter') 74 | self.assertIn('test_counter_total 1.0', self.metrics()) 75 | 76 | unregister_metrics() 77 | 78 | output = self._invoke({'metrics': {'counter': dict( 79 | name='test_counter_with_labels', help='Test Counter', 80 | labels={'code': '{{ response.status_code }}'} 81 | )}}) 82 | 83 | self.assertEqual(output, 'Tracking metrics: test_counter_with_labels') 84 | self.assertIn('test_counter_with_labels_total{code="200"} 1.0', self.metrics()) 85 | 86 | def test_multiple_endpoints(self): 87 | unregister_metrics() 88 | 89 | server = Server([ 90 | { 91 | '/one': { 92 | 'actions': [{ 93 | 'metrics': { 94 | 'counter': {'name': 'metric_one'} 95 | } 96 | }] 97 | }, 98 | '/two': { 99 | 'actions': [ 100 | { 101 | 'metrics': { 102 | 'counter': {'name': 'metric_two'} 103 | } 104 | }, 105 | { 106 | 'metrics': { 107 | 'counter': {'name': 'metric_xyz'} 108 | } 109 | }] 110 | } 111 | } 112 | ]) 113 | 114 | server.app.testing = True 115 | client = server.app.test_client() 116 | 117 | self._server = server 118 | 119 | client.post('/one', headers={'Content-Type': 'application/json'}, 120 | data='{"test":"1"}', content_type='application/json') 121 | 122 | self.assertIn('metric_one_total 1.0', self.metrics()) 123 | self.assertIn('metric_two_total 0.0', self.metrics()) 124 | self.assertIn('metric_xyz_total 0.0', self.metrics()) 125 | 126 | client.post('/one', headers={'Content-Type': 'application/json'}, 127 | data='{"test":"2"}', content_type='application/json') 128 | 129 | self.assertIn('metric_one_total 2.0', self.metrics()) 130 | self.assertIn('metric_two_total 0.0', self.metrics()) 131 | self.assertIn('metric_xyz_total 0.0', self.metrics()) 132 | 133 | client.post('/two', headers={'Content-Type': 'application/json'}, 134 | data='{"test":"3"}', content_type='application/json') 135 | 136 | self.assertIn('metric_one_total 2.0', self.metrics()) 137 | self.assertIn('metric_two_total 1.0', self.metrics()) 138 | self.assertIn('metric_xyz_total 1.0', self.metrics()) 139 | 140 | def test_invalid_metric(self): 141 | self.assertRaises(ConfigurationException, self._invoke, {'metrics': {'unknown': {}}}) 142 | 143 | def metrics(self): 144 | client = self._server.app.test_client() 145 | 146 | response = client.get('/metrics') 147 | 148 | self.assertEqual(response.status_code, 200) 149 | 150 | return str(response.data) 151 | -------------------------------------------------------------------------------- /tests/test_replay_helper.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json as jsonlib 3 | import time 4 | 5 | import actions.replay_helper as helper 6 | 7 | from actions import action, Action 8 | 9 | from unittest_helper import ActionTestBase, capture_stream 10 | 11 | 12 | class ReplayHelperTest(ActionTestBase): 13 | path = '.unittest-test.db' 14 | 15 | def setUp(self): 16 | os.environ['REPLAY_DATABASE'] = self.path 17 | 18 | if os.path.exists(self.path): 19 | os.remove(self.path) 20 | 21 | def tearDown(self): 22 | if os.path.exists(self.path): 23 | os.remove(self.path) 24 | 25 | del os.environ['REPLAY_DATABASE'] 26 | 27 | def test_with_read_only_db(self): 28 | with helper.read_only_db(self.path) as db: 29 | result = db.execute('SELECT 1').fetchone() 30 | self.assertEqual(1, result[0]) 31 | 32 | def test_with_read_write_db(self): 33 | with helper.read_write_db(self.path) as db: 34 | db.execute('CREATE TABLE test (value integer)') 35 | db.execute('INSERT INTO test VALUES (?)', (1,)) 36 | db.commit() 37 | 38 | result = db.execute('SELECT * FROM test').fetchone() 39 | self.assertEqual(1, result[0]) 40 | 41 | with helper.read_only_db(self.path) as db: 42 | result = db.execute('SELECT * FROM test').fetchone() 43 | self.assertEqual(1, result[0]) 44 | 45 | def test_replay(self): 46 | import requests 47 | 48 | helper.initialize() 49 | 50 | # give it some time to start up 51 | time.sleep(1) 52 | 53 | # use the Flask test client instead of requests 54 | original_requests_request = requests.request 55 | 56 | def test_request(method, url, headers, json): 57 | self.assertTrue(url.endswith('/testing'), msg='Unexpected URL: %s' % url) 58 | self.assertIn('testing', json) 59 | self.assertEqual(json['testing'], True) 60 | 61 | return self._client.open(url, method=method, headers=headers, data=jsonlib.dumps(json)) 62 | 63 | requests.request = test_request 64 | 65 | try: 66 | invocations = [] 67 | 68 | @action('remember') 69 | class RememberAction(Action): 70 | def _run(self): 71 | invocations.append(1) 72 | 73 | self._invoke([ 74 | { 75 | 'log': { 76 | 'message': 'Invoked the testing endpoint' 77 | }, 78 | 'remember': {}, 79 | 'eval': { 80 | 'block': '{{ replay(0.33) }}' 81 | } 82 | } 83 | ]) 84 | 85 | self.assertEqual(1, sum(invocations)) 86 | 87 | with capture_stream(echo=False): 88 | time.sleep(2) 89 | 90 | self.assertGreater(sum(invocations), 1) 91 | 92 | finally: 93 | requests.request = original_requests_request 94 | helper.shutdown() 95 | -------------------------------------------------------------------------------- /tests/test_server.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import json 4 | import time 5 | import unittest 6 | 7 | from server import Server, ConfigurationException 8 | from unittest_helper import unregister_metrics, capture_stream 9 | 10 | 11 | class ServerTest(unittest.TestCase): 12 | def setUp(self): 13 | unregister_metrics() 14 | 15 | self.server = Server([ 16 | { 17 | '/testing': { 18 | 'headers': { 19 | 'X-Sample': '^ab[0-9]+$' 20 | }, 21 | 'body': { 22 | 'key': 'value', 23 | 'item': { 24 | 'prop': '^[0-9]*$' 25 | } 26 | } 27 | } 28 | } 29 | ]) 30 | 31 | self.server.app.testing = True 32 | self.client = self.server.app.test_client() 33 | 34 | def tearDown(self): 35 | unregister_metrics() 36 | 37 | def test_valid_request(self): 38 | headers = { 39 | 'X-Sample': 'ab001' 40 | } 41 | 42 | body = { 43 | 'key': 'value', 44 | 'item': { 45 | 'prop': '123' 46 | } 47 | } 48 | 49 | self._check(200, headers, body) 50 | 51 | def test_valid_request_without_optional(self): 52 | headers = { 53 | 'X-Sample': 'ab001' 54 | } 55 | 56 | body = { 57 | 'key': 'value' 58 | } 59 | 60 | self._check(200, headers, body) 61 | 62 | def test_valid_request_with_list(self): 63 | headers = { 64 | 'X-Sample': 'ab001' 65 | } 66 | 67 | body = { 68 | 'key': 'value', 69 | 'item': [ 70 | { 71 | 'prop': '001' 72 | }, 73 | { 74 | 'prop': '002' 75 | }, 76 | { 77 | 'prop': '999' 78 | } 79 | ] 80 | } 81 | 82 | self._check(200, headers, body) 83 | 84 | def test_valid_request_with_templates(self): 85 | def _delete_env_prop(): 86 | del os.environ['PROP_FROM_ENV'] 87 | 88 | os.environ['PROP_FROM_ENV'] = '123' 89 | self.addCleanup(_delete_env_prop) 90 | 91 | headers = { 92 | 'X-Sample': '{{ "AB001"|lower }}' 93 | } 94 | 95 | body = { 96 | 'key': 'value', 97 | 'item': { 98 | 'prop': '{{ read_config("PROP_FROM_ENV") }}' 99 | } 100 | } 101 | 102 | self._check(200, headers, body) 103 | 104 | def test_valid_request_with_view_args(self): 105 | unregister_metrics() 106 | 107 | self.server = Server([ 108 | { 109 | '/testing///': { 110 | 'actions': [ 111 | { 112 | 'log': { 113 | 'message': 'Parameters: ' 114 | 'a={{ request.view_args["a"] }} ' 115 | 'b={{ request.view_args["b"] }} ' 116 | 'c={{ request.view_args["c"] }} ' 117 | 'x={{ request.view_args["x"] }} <' 118 | } 119 | } 120 | ] 121 | } 122 | } 123 | ]) 124 | 125 | self.server.app.testing = True 126 | self.client = self.server.app.test_client() 127 | 128 | with capture_stream() as sout: 129 | response = self.client.post('/testing/1/abc/def/ghi', data='{}', content_type='application/json') 130 | self.assertEqual(200, response.status_code) 131 | 132 | content = sout.dumps() 133 | self.assertIn('a=1', content) 134 | self.assertIn('b=abc', content) 135 | self.assertIn('c=def/ghi', content) 136 | self.assertIn('x= <', content) 137 | 138 | def test_invalid_headers(self): 139 | headers = { 140 | 'X-Sample': 'invalid' 141 | } 142 | 143 | body = { 144 | 'key': 'value', 145 | 'item': { 146 | 'prop': '123' 147 | } 148 | } 149 | 150 | self._check(409, headers, body) 151 | 152 | def test_invalid_body(self): 153 | headers = { 154 | 'X-Sample': 'ab001' 155 | } 156 | 157 | body = { 158 | 'key': 'value', 159 | 'item': 'prop' 160 | } 161 | 162 | self._check(409, headers, body) 163 | 164 | def test_invalid_body_second_level(self): 165 | headers = { 166 | 'X-Sample': 'ab001' 167 | } 168 | 169 | body = { 170 | 'key': 'value', 171 | 'item': { 172 | 'prop': 'invalid' 173 | } 174 | } 175 | 176 | self._check(409, headers, body) 177 | 178 | def test_invalid_body_with_list(self): 179 | headers = { 180 | 'X-Sample': 'ab001' 181 | } 182 | 183 | body = { 184 | 'key': 'value', 185 | 'item': [ 186 | { 187 | 'prop': '001' 188 | }, 189 | { 190 | 'prop': '002' 191 | }, 192 | { 193 | 'prop': 'notanumber' 194 | } 195 | ] 196 | } 197 | 198 | self._check(409, headers, body) 199 | 200 | def _check(self, expected_status_code, headers, body): 201 | response = self.client.post('/testing', 202 | headers=headers, data=json.dumps(body), 203 | content_type='application/json') 204 | 205 | self.assertEqual(expected_status_code, response.status_code) 206 | 207 | def test_non_json_request(self): 208 | response = self.client.post('/testing', data='plain text', content_type='text/plain') 209 | 210 | self.assertEqual(400, response.status_code) 211 | 212 | def test_non_supported_method(self): 213 | headers = { 214 | 'X-Sample': 'ab001' 215 | } 216 | 217 | body = { 218 | 'key': 'value', 219 | 'item': { 220 | 'prop': '123' 221 | } 222 | } 223 | 224 | response = self.client.put('/testing', 225 | headers=headers, data=json.dumps(body), 226 | content_type='application/json') 227 | 228 | self.assertEqual(405, response.status_code) 229 | 230 | def test_missing_endpoint_configuration_throws_exception(self): 231 | self.assertRaises(ConfigurationException, Server, None) 232 | self.assertRaises(ConfigurationException, Server, list()) 233 | 234 | def test_missing_endpoint_route_throws_exception(self): 235 | unregister_metrics() 236 | self.assertRaises(ConfigurationException, Server, [{None: {'method': 'GET'}}]) 237 | 238 | unregister_metrics() 239 | self.assertRaises(ConfigurationException, Server, [{'': {'method': 'GET'}}]) 240 | 241 | def test_empty_endpoint_settings_accept_empty_body(self): 242 | unregister_metrics() 243 | 244 | server = Server([{'/empty': None}]) 245 | 246 | server.app.testing = True 247 | client = server.app.test_client() 248 | 249 | response = client.post('/empty') 250 | 251 | self.assertEqual(200, response.status_code) 252 | 253 | def test_get_request(self): 254 | unregister_metrics() 255 | 256 | server = Server([{'/get': {'method': 'GET', 'headers': {'X-Method': '(GET|HEAD)'}}}]) 257 | 258 | server.app.testing = True 259 | client = server.app.test_client() 260 | 261 | response = client.get('/get', headers={'X-Method': 'GET'}) 262 | 263 | self.assertEqual(200, response.status_code) 264 | 265 | response = client.get('/get', headers={'X-Method': 'Invalid'}) 266 | 267 | self.assertEqual(409, response.status_code) 268 | 269 | def test_async_request(self): 270 | unregister_metrics() 271 | 272 | self.server = Server([ 273 | { 274 | '/testing': { 275 | 'body': { 276 | 'key': 'value' 277 | }, 278 | 'async': True, 279 | 'actions': [ 280 | { 281 | 'sleep': { 282 | 'seconds': 0.2 283 | } 284 | }, 285 | { 286 | 'log': { 287 | 'message': 'Serving {{ request.path }} with key={{ request.json.key }}' 288 | } 289 | } 290 | ] 291 | } 292 | } 293 | ]) 294 | 295 | self.server.app.testing = True 296 | self.client = self.server.app.test_client() 297 | 298 | _stdout = sys.stdout 299 | _output = [] 300 | 301 | class CapturingStdout(object): 302 | def write(self, content): 303 | _output.append(content) 304 | 305 | sys.stdout = CapturingStdout() 306 | 307 | try: 308 | self._check(200, headers=None, body={'key': 'value'}) 309 | 310 | self.assertNotIn('Serving /testing with key=value', ''.join(_output)) 311 | 312 | time.sleep(0.5) 313 | 314 | self.assertIn('Serving /testing with key=value', ''.join(_output)) 315 | 316 | finally: 317 | sys.stdout = _stdout 318 | 319 | def test_validation_with_templates(self): 320 | unregister_metrics() 321 | 322 | server = Server([ 323 | { 324 | '/vars': { 325 | 'headers': { 326 | 'X-Test': '{{ ["templated", "header"]|join("-") }}' 327 | }, 328 | 'body': { 329 | 'key': '{{ ["templated", "body"]|join("-") }}' 330 | } 331 | } 332 | } 333 | ]) 334 | 335 | server.app.testing = True 336 | client = server.app.test_client() 337 | 338 | headers = { 339 | 'X-Test': 'templated-header' 340 | } 341 | 342 | body = { 343 | 'key': 'templated-body' 344 | } 345 | 346 | response = client.post('/vars', 347 | headers=headers, data=json.dumps(body), 348 | content_type='application/json') 349 | 350 | self.assertEqual(200, response.status_code) 351 | 352 | def test_metrics(self): 353 | unregister_metrics() 354 | 355 | self.server = Server([ 356 | { 357 | '/test/post': { 358 | 'async': True, 359 | 'actions': [ 360 | {'sleep': {'seconds': 0.01}}, 361 | {'log': {}} 362 | ] 363 | }, 364 | '/test/put': { 365 | 'method': 'PUT', 366 | 'actions': [ 367 | {'log': {}}, 368 | {'execute': {'command': 'echo "Executing command"'}} 369 | ] 370 | } 371 | } 372 | ]) 373 | 374 | self.server.app.testing = True 375 | self.client = self.server.app.test_client() 376 | 377 | for _ in range(2): 378 | response = self.client.post('/test/post', 379 | data=json.dumps({'unused': 1}), 380 | content_type='application/json') 381 | 382 | self.assertEqual(response.status_code, 200) 383 | 384 | for _ in range(3): 385 | response = self.client.put('/test/put', 386 | data=json.dumps({'unused': 1}), 387 | content_type='application/json') 388 | 389 | self.assertEqual(response.status_code, 200) 390 | 391 | time.sleep(0.1) 392 | 393 | response = self.client.get('/metrics') 394 | 395 | self.assertEqual(response.status_code, 200) 396 | 397 | metrics = response.data.decode('utf-8') 398 | 399 | self.assertIn('python_info{', metrics) 400 | self.assertIn('process_start_time_seconds ', metrics) 401 | 402 | self.assertIn('flask_http_request_total{' 403 | 'method="POST",status="200"} 2.0', metrics) 404 | self.assertIn('flask_http_request_total{' 405 | 'method="PUT",status="200"} 3.0', metrics) 406 | 407 | self.assertIn('flask_http_request_duration_seconds_bucket{' 408 | 'le="5.0",method="POST",path="/test/post",status="200"} 2.0', metrics) 409 | self.assertIn('flask_http_request_duration_seconds_count{' 410 | 'method="POST",path="/test/post",status="200"} 2.0', metrics) 411 | self.assertIn('flask_http_request_duration_seconds_sum{' 412 | 'method="POST",path="/test/post",status="200"}', metrics) 413 | 414 | self.assertIn('flask_http_request_duration_seconds_bucket{' 415 | 'le="0.5",method="PUT",path="/test/put",status="200"} 3.0', metrics) 416 | self.assertIn('flask_http_request_duration_seconds_count{' 417 | 'method="PUT",path="/test/put",status="200"} 3.0', metrics) 418 | self.assertIn('flask_http_request_duration_seconds_sum{' 419 | 'method="PUT",path="/test/put",status="200"}', metrics) 420 | 421 | self.assertIn('webhook_proxy_actions_count{' 422 | 'action_index="0",action_type="sleep",' 423 | 'http_method="POST",http_route="/test/post"} 2.0', metrics) 424 | self.assertIn('webhook_proxy_actions_sum{' 425 | 'action_index="0",action_type="sleep",' 426 | 'http_method="POST",http_route="/test/post"}', metrics) 427 | self.assertIn('webhook_proxy_actions_count{' 428 | 'action_index="1",action_type="log",' 429 | 'http_method="POST",http_route="/test/post"} 2.0', metrics) 430 | self.assertIn('webhook_proxy_actions_sum{' 431 | 'action_index="1",action_type="log",' 432 | 'http_method="POST",http_route="/test/post"}', metrics) 433 | 434 | self.assertIn('webhook_proxy_actions_count{' 435 | 'action_index="0",action_type="log",' 436 | 'http_method="PUT",http_route="/test/put"} 3.0', metrics) 437 | self.assertIn('webhook_proxy_actions_sum{' 438 | 'action_index="0",action_type="log",' 439 | 'http_method="PUT",http_route="/test/put"}', metrics) 440 | self.assertIn('webhook_proxy_actions_count{' 441 | 'action_index="1",action_type="execute",' 442 | 'http_method="PUT",http_route="/test/put"} 3.0', metrics) 443 | self.assertIn('webhook_proxy_actions_sum{' 444 | 'action_index="1",action_type="execute",' 445 | 'http_method="PUT",http_route="/test/put"}', metrics) 446 | -------------------------------------------------------------------------------- /tests/unittest_helper.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | import unittest 4 | 5 | from server import Server 6 | from prometheus_client import REGISTRY 7 | 8 | 9 | def capture_stream(stream='stdout', echo=False): 10 | _original_stream = getattr(sys, stream) 11 | 12 | class CapturedStream(object): 13 | def __init__(self): 14 | self.lines = list() 15 | 16 | def write(self, line): 17 | self.lines.append(line.strip()) 18 | 19 | if echo: 20 | _original_stream.write(line) 21 | 22 | def dumps(self): 23 | return '\n'.join(str(line.strip()) for line in self.lines if line) 24 | 25 | class CapturedContext(object): 26 | def __enter__(self): 27 | capture = CapturedStream() 28 | 29 | setattr(sys, stream, capture) 30 | 31 | return capture 32 | 33 | def __exit__(self, *args, **kwargs): 34 | setattr(sys, stream, _original_stream) 35 | 36 | return CapturedContext() 37 | 38 | 39 | def unregister_metrics(): 40 | for collector, names in tuple(REGISTRY._collector_to_names.items()): 41 | if any(name.startswith('flask_') or 42 | name.startswith('webhook_proxy_') 43 | for name in names): 44 | 45 | REGISTRY.unregister(collector) 46 | 47 | 48 | class ActionTestBase(unittest.TestCase): 49 | _server = None 50 | _headers = {'Content-Type': 'application/json'} 51 | _body = {'testing': True} 52 | 53 | def tearDown(self): 54 | unregister_metrics() 55 | 56 | def _invoke(self, actions, expected_status_code=200, body=None, **kwargs): 57 | if not isinstance(actions, list): 58 | actions = [actions] 59 | 60 | if not body: 61 | body = self._body 62 | 63 | final_body = kwargs.pop('final_body', json.dumps(body)) 64 | 65 | unregister_metrics() 66 | 67 | server = Server([{'/testing': {'actions': actions}}], **kwargs) 68 | 69 | server.app.testing = True 70 | client = server.app.test_client() 71 | 72 | self._server = server 73 | self._client = client 74 | 75 | with capture_stream() as sout: 76 | response = client.post('/testing', 77 | headers=self._headers, data=final_body, 78 | content_type='application/json') 79 | 80 | self.assertEqual(expected_status_code, response.status_code) 81 | 82 | return sout.dumps() 83 | --------------------------------------------------------------------------------