├── assets ├── out ├── in ├── check └── resource.py ├── requirements.txt ├── requirements_dev.txt ├── .travis.yml ├── setup.cfg ├── Dockerfile ├── test ├── test.sh ├── test_hipchat.py ├── helpers.py └── test_invocation.py ├── Dockerfile.tdd ├── Makefile ├── LICENSE.md └── README.md /assets/out: -------------------------------------------------------------------------------- 1 | resource.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | pylama 2 | pytest 3 | isort 4 | responses 5 | pytest-httpbin 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | services: 4 | - docker 5 | 6 | script: 7 | - make test 8 | -------------------------------------------------------------------------------- /assets/in: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # empty response to have non-failing implicit get after put 4 | echo '{"version": {}}' 5 | -------------------------------------------------------------------------------- /assets/check: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # empty version to keep Concourse happy 4 | # https://concourse-ci.org/implementing-resources.html#resource-check 5 | echo '[]' 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pycodestyle] 2 | max_line_length=120 3 | 4 | [pylama] 5 | linters = pyflakes,mccabe,pycodestyle,pep257,isort 6 | ignore = D104,D202,D203,D204,D100,E302,E303,E128 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | 3 | # install requirements 4 | ADD requirements*.txt setup.cfg ./ 5 | RUN pip install --no-cache-dir -r requirements.txt 6 | 7 | # install asserts 8 | ADD assets/ /opt/resource/ 9 | ADD test/ /opt/resource-tests/ 10 | 11 | RUN /opt/resource-tests/test.sh 12 | -------------------------------------------------------------------------------- /test/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # fail if one command fails 4 | set -e 5 | 6 | # install requirements 7 | pip install --no-cache-dir -r requirements_dev.txt 8 | 9 | # test 10 | pylama /opt/resource /opt/resource-tests/ 11 | py.test -l --tb=short -r fE /opt/resource-tests 12 | 13 | # cleanup 14 | rm -fr /tmp/* 15 | pip uninstall -y -r requirements_dev.txt 16 | -------------------------------------------------------------------------------- /Dockerfile.tdd: -------------------------------------------------------------------------------- 1 | FROM python:3 2 | 3 | # install requirements 4 | ADD requirements*.txt setup.cfg ./ 5 | RUN pip install --no-cache-dir -r requirements.txt 6 | RUN pip install --no-cache-dir -r requirements_dev.txt 7 | 8 | ADD assets/ /opt/resource/ 9 | ADD test/ /opt/resource-tests/ 10 | 11 | ENTRYPOINT RESOURCE_DEBUG=1 py.test -l --tb=short -r fE /opt/resource-tests 12 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | user=aequitas 2 | name=http-resource 3 | 4 | docker=docker 5 | tag = $(user)/$(name) 6 | dockerfile = Dockerfile 7 | 8 | .PHONY: test 9 | 10 | push: build 11 | $(docker) push $(user)/$(name) 12 | 13 | build: 14 | $(docker) build -t $(tag) -f $(dockerfile) . 15 | 16 | test: tag=$(user)/$(name)-test 17 | test: dockerfile=Dockerfile.tdd 18 | test: build 19 | $(docker) run $(args) \ 20 | -e HIPCHAT_TOKEN=${HIPCHAT_TOKEN} \ 21 | $(user)/$(name)-test 22 | -------------------------------------------------------------------------------- /test/test_hipchat.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | from helpers import cmd 5 | 6 | HIPCHAT_TOKEN = os.environ.get('HIPCHAT_TOKEN') 7 | 8 | @pytest.mark.skipif(not HIPCHAT_TOKEN, reason='hipchat token not provided') 9 | def test_hipchat_notify(): 10 | """Test posting notification to Hipchat.""" 11 | source = { 12 | 'uri': 'https://www.hipchat.com/v2/room/2442416/notification', 13 | 'headers': { 14 | 'Authorization': 'Bearer ' + HIPCHAT_TOKEN 15 | }, 16 | 'method': 'POST', 17 | } 18 | 19 | params = { 20 | 'json': { 21 | 'color': 'green', 22 | 'message': 'Build {BUILD_PIPELINE_NAME}/{BUILD_JOB_NAME}, nr: {BUILD_NAME} was a success!', 23 | } 24 | } 25 | 26 | cmd('out', source, params=params) 27 | -------------------------------------------------------------------------------- /test/helpers.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import subprocess 4 | import sys 5 | 6 | ENVIRONMENT = { 7 | 'BUILD_NAME': '1', 8 | 'BUILD_JOB_NAME': 'test-job', 9 | 'BUILD_PIPELINE_NAME': 'test-pipeline', 10 | 'BUILD_ID': '123', 11 | 'TEST': 'true', 12 | } 13 | 14 | def cmd(cmd_name, source, args: list = [], version={}, params={}): 15 | """Wrap command interaction for easier use with python objects.""" 16 | 17 | in_json = json.dumps({ 18 | "source": source, 19 | "version": version, 20 | "params": params, 21 | }) 22 | command = ['/opt/resource/' + cmd_name] + args 23 | environment = dict(os.environ, **ENVIRONMENT) 24 | output = subprocess.check_output(command, env=environment, 25 | stderr=sys.stderr, input=bytes(in_json, 'utf-8')) 26 | 27 | return json.loads(output.decode()) 28 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Johan Bloemberg 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/test_invocation.py: -------------------------------------------------------------------------------- 1 | import json 2 | import subprocess 3 | 4 | import pytest 5 | from helpers import cmd 6 | 7 | 8 | def test_out(httpbin): 9 | """Test out action with minimal input.""" 10 | 11 | data = { 12 | 'source': { 13 | 'uri': httpbin + '/status/200', 14 | }, 15 | 'version': {} 16 | } 17 | subprocess.check_output('/opt/resource/out', input=json.dumps(data).encode()) 18 | 19 | 20 | def test_out_failure(httpbin): 21 | """Test action failing if not OK http response.""" 22 | 23 | data = { 24 | 'source': { 25 | 'uri': httpbin + '/status/404', 26 | }, 27 | 'version': {} 28 | } 29 | with pytest.raises(subprocess.CalledProcessError): 30 | subprocess.check_output('/opt/resource/out', input=json.dumps(data).encode()) 31 | 32 | 33 | def test_auth(httpbin): 34 | """Test basic authentication.""" 35 | 36 | data = { 37 | 'source': { 38 | 'uri': 'http://user:password@{0.host}:{0.port}/basic-auth/user/password'.format(httpbin), 39 | }, 40 | } 41 | subprocess.check_output('/opt/resource/out', input=json.dumps(data).encode()) 42 | 43 | 44 | def test_json(httpbin): 45 | """Json should be passed as JSON content.""" 46 | 47 | source = { 48 | 'uri': httpbin + '/post', 49 | 'method': 'POST', 50 | 'json': { 51 | 'test': 123, 52 | }, 53 | 'version': {} 54 | } 55 | 56 | output = cmd('out', source) 57 | 58 | assert output['json']['test'] == 123 59 | assert output['version'] == {} 60 | 61 | 62 | def test_interpolation(httpbin): 63 | """Values should be interpolated recursively.""" 64 | 65 | source = { 66 | 'uri': httpbin + '/post', 67 | 'method': 'POST', 68 | 'json': { 69 | 'object': { 70 | 'test': '{BUILD_NAME}' 71 | }, 72 | 'array': [ 73 | '{BUILD_NAME}' 74 | ] 75 | } 76 | } 77 | 78 | output = cmd('out', source) 79 | 80 | assert output['json']['object']['test'] == '1' 81 | assert output['json']['array'][0] == '1' 82 | assert output['version'] == {} 83 | 84 | 85 | def test_empty_check(httpbin): 86 | """Check must return an empty response but not nothing.""" 87 | 88 | source = { 89 | 'uri': httpbin + '/post', 90 | 'method': 'POST', 91 | } 92 | 93 | check = cmd('check', source) 94 | 95 | assert check == [] 96 | 97 | 98 | def test_data_urlencode(httpbin): 99 | """Test passing URL encoded data.""" 100 | 101 | source = { 102 | 'uri': httpbin + '/post', 103 | 'method': 'POST', 104 | 'form_data': { 105 | 'field': { 106 | 'test': 123, 107 | }, 108 | } 109 | } 110 | 111 | output = cmd('out', source) 112 | 113 | assert output['form'] == {'field': '{"test": 123}'} 114 | assert output['version'] == {} 115 | 116 | 117 | def test_data_ensure_ascii(httpbin): 118 | """Test form_data json ensure_ascii.""" 119 | 120 | source = { 121 | 'uri': httpbin + '/post', 122 | 'method': 'POST', 123 | 'form_data': { 124 | 'field': { 125 | 'test': '日本語', 126 | }, 127 | }, 128 | } 129 | 130 | output = cmd('out', source) 131 | 132 | assert output['form'] == {'field': '{"test": "日本語"}'} 133 | 134 | 135 | def test_not_parsed_data(httpbin): 136 | """Test form_data in a standard format.""" 137 | 138 | source = { 139 | 'uri': httpbin + '/post', 140 | 'method': 'POST', 141 | 'parse_form_data': False, 142 | 'form_data': { 143 | 'firstname': 'John', 144 | 'lastname': 'Doe' 145 | } 146 | } 147 | 148 | output = cmd('out', source) 149 | 150 | assert output['form'] == {"firstname": "John", "lastname": "Doe"} 151 | assert output['version'] == {} 152 | -------------------------------------------------------------------------------- /assets/resource.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import logging as log 5 | import os 6 | import sys 7 | import tempfile 8 | 9 | import requests 10 | 11 | 12 | class HTTPResource: 13 | """HTTP resource implementation.""" 14 | 15 | def cmd(self, arg, data): 16 | """Make the requests.""" 17 | 18 | method = data.get('method', 'GET') 19 | uri = data['uri'] 20 | headers = data.get('headers', {}) 21 | json_data = data.get('json', None) 22 | ssl_verify = data.get('ssl_verify', True) 23 | ok_responses = data.get('ok_responses', [200, 201, 202, 204]) 24 | form_data = data.get('form_data') 25 | parse_form_data = data.get('parse_form_data', True) 26 | 27 | if isinstance(ssl_verify, bool): 28 | verify = ssl_verify 29 | elif isinstance(ssl_verify, str): 30 | verify = str(tempfile.NamedTemporaryFile(delete=False, prefix='ssl-').write(verify)) 31 | 32 | request_data = None 33 | if form_data and parse_form_data: 34 | request_data = {k: json.dumps(v, ensure_ascii=False) for k, v in form_data.items()} 35 | elif form_data and not parse_form_data: 36 | request_data = form_data 37 | 38 | response = requests.request(method, uri, json=json_data, 39 | data=request_data, headers=headers, verify=verify) 40 | 41 | log.info('http response code: %s', response.status_code) 42 | log.info('http response text: %s', response.text) 43 | 44 | if response.status_code not in ok_responses: 45 | raise Exception('Unexpected response {}'.format(response.status_code)) 46 | 47 | return (response.status_code, response.text) 48 | 49 | def run(self, command_name: str, json_data: str, command_argument: str): 50 | """Parse input/arguments, perform requested command return output.""" 51 | 52 | with tempfile.NamedTemporaryFile(delete=False, prefix=command_name + '-') as f: 53 | f.write(bytes(json_data, 'utf-8')) 54 | 55 | data = json.loads(json_data) 56 | 57 | # allow debug logging to console for tests 58 | if os.environ.get('RESOURCE_DEBUG', False) or data.get('source', {}).get('debug', False): 59 | log.basicConfig(level=log.DEBUG) 60 | else: 61 | logfile = tempfile.NamedTemporaryFile(delete=False, prefix='log') 62 | log.basicConfig(level=log.DEBUG, filename=logfile.name) 63 | stderr = log.StreamHandler() 64 | stderr.setLevel(log.INFO) 65 | log.getLogger().addHandler(stderr) 66 | 67 | log.debug('command: %s', command_name) 68 | log.debug('input: %s', data) 69 | log.debug('args: %s', command_argument) 70 | log.debug('environment: %s', os.environ) 71 | 72 | # initialize values with Concourse environment variables 73 | values = {k: v for k, v in os.environ.items() if k.startswith('BUILD_') or k == 'ATC_EXTERNAL_URL'} 74 | 75 | # combine source and params 76 | params = data.get('source', {}) 77 | params.update(data.get('params', {})) 78 | 79 | # allow also to interpolate params 80 | values.update(params) 81 | 82 | # apply templating of environment variables onto parameters 83 | rendered_params = self._interpolate(params, values) 84 | 85 | status_code, text = self.cmd(command_argument, rendered_params) 86 | 87 | # return empty version object 88 | response = {"version": {}} 89 | 90 | if os.environ.get('TEST', False): 91 | response.update(json.loads(text)) 92 | 93 | return json.dumps(response) 94 | 95 | def _interpolate(self, data, values): 96 | """Recursively apply values using format on all string key and values in data.""" 97 | 98 | if isinstance(data, str): 99 | return data.format(**values) 100 | elif isinstance(data, list): 101 | return [self._interpolate(x, values) for x in data] 102 | elif isinstance(data, dict): 103 | return {self._interpolate(k, values): self._interpolate(v, values) 104 | for k, v in data.items()} 105 | else: 106 | return data 107 | 108 | 109 | print(HTTPResource().run(os.path.basename(__file__), sys.stdin.read(), sys.argv[1:])) 110 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HTTP API Resource 2 | 3 | Concourse resource to allow interaction with (simple) HTTP (REST/JSON) API's. This resource is useful for API's which have simple one request interactions and will not be a one-size-fits-all solution. If your API is more complex [writing your own resource](http://concourse.ci/implementing-resources.html). 4 | 5 | https://hub.docker.com/r/aequitas/http-api-resource/ 6 | 7 | ## Recent changes 8 | - 03-2021 Add support for form data (@ThibaultDelaune-pro) 9 | - 09-2018 Fix error on check (reporter: @OvermindDL1) 10 | - 02-2018 Fix build detail load (@scottasmith, reporter: @pranaysharmadelhi) 11 | - 02-2018 add support for UTF-8 in JSON (@masaki-takano) 12 | - 01-2018 fix concourse 3.1.1 compatibility (@hfinucane) 13 | - 01-2018 added license 14 | - 01-2017 recursive interpolate over lists as well 15 | 16 | ## Source Configuration 17 | 18 | Most of the `source` options can also be used in `params` for the specifc actions. This allows to use a different URL path. For example when a POST and GET use different endpoints. 19 | 20 | Options set in `params` take precedence over options in `source`. 21 | 22 | * `uri`: *Required.* The URI to use for the requests. 23 | Example: `https://www.hipchat.com/v2/room/1234321/notification` 24 | 25 | * `method`: *Optional* Method to use, eg: GET, POST, PATCH (default `GET`). 26 | 27 | * `headers`: *Optional* Object containing headers to pass to the request. 28 | Example: 29 | 30 | headers: 31 | X-Some-Header: some header content 32 | 33 | * `json`: *Optional* JSON to send along with the request, set `application/json` header. 34 | 35 | * `debug`: *Optional* Set debug logging of scripts, takes boolean (default `false`). 36 | 37 | * `ssl_verify`: *Optional* Boolean or SSL CA content (default `true`). 38 | 39 | * `form_data`: *Optional* Dictionary with form field/value pairs to send as data. Values are converted to JSON and URL-encoded. 40 | 41 | * `parse_form_data`: *Optional* Boolean to specify if form_data should be converted to JSON or not (default `true`). If false, set `"Content-Type":"application/x-www-form-urlencoded"` header. 42 | 43 | ## Behavior 44 | 45 | Currently the only useful action the resource supports is `out`. The actions `in` and `check` will be added later. 46 | 47 | ### Interpolation 48 | 49 | All options support interpolation of variables by using [Python string formatting](https://docs.python.org/3.5/library/stdtypes.html#str.format). 50 | 51 | In short it means variables can be used by using single curly brackets (instead of double for Concourse interpolation). Eg: `Build nr. {BUILD_NAME} passed.` 52 | 53 | Build metadata (BUILD_NAME, BUILD_JOB_NAME, BUILD_PIPELINE_NAME and BUILD_ID) are available as well as the merged `source`/`params` objects. Interpolation will happen after merging the two objects. 54 | 55 | Be aware that options containing interpolation variables need to be enclosed in double quotes `"`. 56 | 57 | See Hipchat below for usage example. 58 | 59 | ## Examples 60 | 61 | ### Post notification on HipChat 62 | 63 | This example show use of variable interpolation with build metadata and the params dict. 64 | 65 | Also shows how the usage of a authentication header using Concourse variables. 66 | 67 | 68 | ```yaml 69 | resource_types: 70 | - name: http-api 71 | type: docker-image 72 | source: 73 | repository: aequitas/http-api-resource 74 | tag: latest 75 | 76 | resources: 77 | - name: hipchat 78 | type: http-api 79 | source: 80 | uri: https://www.hipchat.com/v2/room/team_room/notification 81 | method: POST 82 | headers: 83 | Authorization: "Bearer {hipchat_token}" 84 | json: 85 | color: "{color}" 86 | message: "Build {BUILD_PIPELINE_NAME}{BUILD_JOB_NAME}, nr: {BUILD_NAME} {message}!" 87 | hipchat_token: {{HIPCHAT_TOKEN}} 88 | 89 | jobs: 90 | - name: Test and notify 91 | plan: 92 | - task: build 93 | file: ci/build.yaml 94 | on_success: 95 | put: hipchat 96 | params: 97 | color: green 98 | message: was a success 99 | on_failure: 100 | put: hipchat 101 | params: 102 | color: red 103 | message: failed horribly 104 | 105 | ``` 106 | 107 | ### Trigger build in Jenkins 108 | 109 | Trigger the job `job_name` with parameter `package` set to `test`. 110 | 111 | More info: https://wiki.jenkins-ci.org/display/JENKINS/Remote+access+API#RemoteaccessAPI-Submittingjobs 112 | 113 | ```yaml 114 | resources: 115 | - name: jenkins-trigger-job 116 | type: http-api 117 | source: 118 | uri: http://user:token@jenkins.example.com/job/job_name/build 119 | method: POST 120 | form_data: 121 | json: 122 | parameter: 123 | - name: package 124 | value: test 125 | 126 | jobs: 127 | - name: Test and notify 128 | plan: 129 | - task: build 130 | file: ci/build.yaml 131 | 132 | - put: jenkins-trigger-job 133 | 134 | ``` 135 | 136 | 137 | ### Call a Rest API with data formated in a standard form 138 | 139 | Register apps on a Spring Cloud Data Flow instance via a `POST` formated with `application/x-www-form-urlencoded` encoding type. 140 | 141 | More info: https://docs.spring.io/spring-cloud-dataflow/docs/current/reference/htmlsingle/#resources-app-registry-bulk 142 | 143 | ```yaml 144 | resources: 145 | - name: dataflow-rest-api 146 | type: http-api 147 | source: 148 | uri: http://my-dataflow-platform.org 149 | 150 | jobs: 151 | - name: load-apps 152 | public: true 153 | plan: 154 | - put: dataflow-rest-api 155 | params: 156 | uri: http://my-dataflow-platform.org/apps 157 | method: POST 158 | debug: true 159 | parse_form_data: false 160 | form_data: 161 | uri: http://artefact.my-repo.com/artifactory/Maven/appsList/0.0.1/appsList-0.0.1.properties 162 | ``` --------------------------------------------------------------------------------