├── .github └── workflows │ ├── documentation.yaml │ ├── pre-commit.yaml │ └── unit-tests.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .pre-commit-hooks └── double_string_fixer.py ├── .readthedocs.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.md ├── README.md ├── SECURITY.md ├── concoursetools ├── __init__.py ├── __main__.py ├── additional.py ├── cli │ ├── __init__.py │ ├── commands.py │ ├── docstring.py │ └── parser.py ├── colour.py ├── dockertools.py ├── importing.py ├── metadata.py ├── mocking.py ├── parsing.py ├── py.typed ├── resource.py ├── testing.py ├── typing.py └── version.py ├── cspell.config.yaml ├── docs ├── requirements.txt └── source │ ├── _static │ ├── favicon.png │ ├── logo-dark.png │ ├── logo.png │ ├── step_metadata.png │ └── style.css │ ├── additional.rst │ ├── api_reference.rst │ ├── build_metadata.rst │ ├── changelog.rst │ ├── changelog │ ├── 0.8.0.rst │ └── development.rst │ ├── cli.rst │ ├── cli_reference.rst │ ├── conf.py │ ├── debugging.rst │ ├── deployment.rst │ ├── dockertools.rst │ ├── examples │ ├── branches.rst │ ├── build_status.rst │ ├── pipeline.rst │ ├── s3.rst │ ├── secrets.rst │ └── xkcd.rst │ ├── extensions │ ├── __init__.py │ ├── cli.py │ ├── concourse.py │ ├── linecount.py │ ├── wikipedia.py │ └── xkcd.py │ ├── importing.rst │ ├── index.rst │ ├── internals.rst │ ├── main_scripts.rst │ ├── mocking.rst │ ├── parsing.rst │ ├── quickstart.rst │ ├── resource.rst │ ├── testing.rst │ ├── typing.rst │ ├── version.rst │ └── wrappers.rst ├── examples ├── __init__.py ├── build_status.py ├── github_branches.py ├── pipeline.py ├── s3.py ├── secrets.py ├── xkcd.py └── xkcd.xml ├── pyproject.toml ├── requirements-tests.txt ├── setup.py ├── sonar-project.properties └── tests ├── __init__.py ├── resource.py ├── test_additional.py ├── test_cli_commands.py ├── test_cli_parsing.py ├── test_dockertools.py ├── test_examples.py ├── test_importing.py ├── test_metadata.py ├── test_parsing.py ├── test_resource.py ├── test_testing.py └── test_version.py /.github/workflows/documentation.yaml: -------------------------------------------------------------------------------- 1 | name: Build Documentation 2 | on: 3 | pull_request: 4 | branches: [main] 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - name: Set up Python 3.12 11 | uses: actions/setup-python@v3 12 | with: 13 | python-version: "3.12" 14 | - name: Install dependencies 15 | run: | 16 | python -m pip install --upgrade pip 17 | python -m pip install -r docs/requirements.txt --no-deps 18 | - name: Build documentation 19 | run: | 20 | python -m sphinx -b html -aE -n -W --keep-going docs/source docs/build 21 | - name: Check external links 22 | run: | 23 | python -m sphinx -b linkcheck docs/source docs/build 24 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yaml: -------------------------------------------------------------------------------- 1 | name: Run Pre-commit 2 | on: 3 | pull_request: 4 | branches: [main] 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - name: Set up Python 3.12 11 | uses: actions/setup-python@v3 12 | with: 13 | python-version: "3.12" 14 | - name: Install pre-commit 15 | run: | 16 | python -m pip install --upgrade pip 17 | python -m pip install pre-commit 18 | - name: Run pre-commit checks 19 | run: | 20 | python -m pre_commit run --all-files --color always 21 | -------------------------------------------------------------------------------- /.github/workflows/unit-tests.yaml: -------------------------------------------------------------------------------- 1 | name: Unit Tests 2 | on: 3 | pull_request: 4 | branches: [main] 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | fail-fast: false 10 | matrix: 11 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14.0-beta.2"] 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Set up Python ${{ matrix.python-version }} 15 | uses: actions/setup-python@v3 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | - name: Pull docker image 19 | run: | 20 | docker pull concourse/mock-resource:0.13.0 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | python -m pip install -r requirements-tests.txt --no-deps 25 | - name: Run tests 26 | run: | 27 | python -W error -m unittest --verbose 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.egg-info 3 | .venv*/ 4 | .vscode 5 | docs/build 6 | .DS_Store 7 | .mypy_cache 8 | .coverage 9 | coverage.xml 10 | .scannerwork 11 | .env 12 | dist/ 13 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-yaml 8 | - id: check-added-large-files 9 | args: [--enforce-all, --maxkb=100] 10 | - repo: https://github.com/google/yamlfmt 11 | rev: v0.17.0 12 | hooks: 13 | - id: yamlfmt 14 | - repo: https://github.com/hhatto/autopep8 15 | rev: v2.3.2 16 | hooks: 17 | - id: autopep8 18 | exclude: docs/source/conf.py 19 | - repo: https://github.com/pycqa/isort 20 | rev: 6.0.1 21 | hooks: 22 | - id: isort 23 | - repo: https://github.com/jumanjihouse/pre-commit-hooks 24 | rev: 3.0.0 25 | hooks: 26 | - id: require-ascii 27 | - id: script-must-have-extension 28 | - id: forbid-binary 29 | exclude: docs/source/_static/.*.png 30 | - repo: https://github.com/Lucas-C/pre-commit-hooks 31 | rev: v1.5.5 32 | hooks: 33 | - id: forbid-crlf 34 | - id: forbid-tabs 35 | - repo: https://github.com/streetsidesoftware/cspell-cli 36 | rev: v9.0.1 37 | hooks: 38 | - id: cspell 39 | exclude: \.gitignore|.*\.properties 40 | - repo: https://github.com/pycqa/flake8 41 | rev: 7.2.0 42 | hooks: 43 | - id: flake8 44 | args: [--max-line-length=150] 45 | exclude: docs/source/conf.py 46 | - repo: https://github.com/pycqa/pydocstyle 47 | rev: 6.3.0 48 | hooks: 49 | - id: pydocstyle 50 | additional_dependencies: ["tomli"] 51 | exclude: examples|tests|.pre-commit-hooks|conf.py 52 | - repo: https://github.com/regebro/pyroma 53 | rev: "4.2" 54 | hooks: 55 | - id: pyroma 56 | args: ["-d", "--min=10", "."] 57 | - repo: https://github.com/pre-commit/mirrors-mypy 58 | rev: v1.16.0 59 | hooks: 60 | - id: mypy 61 | additional_dependencies: ["types-docutils", "types-python-dateutil", "types-requests"] 62 | - repo: "https://github.com/python-jsonschema/check-jsonschema" 63 | rev: "0.33.0" 64 | hooks: 65 | - id: check-github-workflows 66 | - repo: https://github.com/johannsdg/pre-commit-license-headers 67 | rev: d53087d331942f263cb553dc67b0e51ffa3a3481 68 | hooks: 69 | - id: check-license-headers 70 | types: [python] 71 | args: 72 | - --template 73 | - (C) Crown Copyright [OWNER] 74 | - --owner=GCHQ 75 | - repo: local 76 | hooks: 77 | - id: double-string-fixer 78 | name: string fixer 79 | description: Replace single-quoted strings with double-quoted strings 80 | types: [python] 81 | language: python 82 | entry: python3 .pre-commit-hooks/double_string_fixer.py 83 | -------------------------------------------------------------------------------- /.pre-commit-hooks/double_string_fixer.py: -------------------------------------------------------------------------------- 1 | """ 2 | A copy of https://github.com/pre-commit/pre-commit-hooks/tree/main#double-quote-string-fixer, 3 | except it replaces single-quoted strings with double-quoted strings. 4 | 5 | Copyright (c) 2014 pre-commit dev team: Anthony Sottile, Ken Struys 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in 15 | all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | THE SOFTWARE. 24 | """ 25 | from __future__ import annotations 26 | 27 | import argparse 28 | import io 29 | import re 30 | import sys 31 | import tokenize 32 | from typing import Sequence 33 | 34 | if sys.version_info >= (3, 12): # pragma: >=3.12 cover 35 | FSTRING_START = tokenize.FSTRING_START 36 | FSTRING_END = tokenize.FSTRING_END 37 | else: # pragma: <3.12 cover 38 | FSTRING_START = FSTRING_END = -1 39 | 40 | START_QUOTE_RE = re.compile("^[a-zA-Z]*'") 41 | 42 | 43 | def handle_match(token_text: str) -> str: 44 | if '"""' in token_text or "'''" in token_text: 45 | return token_text 46 | 47 | match = START_QUOTE_RE.match(token_text) 48 | if match is not None: 49 | meat = token_text[match.end():-1] 50 | if '"' in meat or "'" in meat: 51 | return token_text 52 | else: 53 | return match.group().replace("'", '"') + meat + '"' 54 | else: 55 | return token_text 56 | 57 | 58 | def get_line_offsets_by_line_no(src: str) -> list[int]: 59 | # Padded so we can index with line number 60 | offsets = [-1, 0] 61 | for line in src.splitlines(True): 62 | offsets.append(offsets[-1] + len(line)) 63 | return offsets 64 | 65 | 66 | def fix_strings(filename: str) -> int: 67 | with open(filename, encoding="UTF-8", newline="") as f: 68 | contents = f.read() 69 | line_offsets = get_line_offsets_by_line_no(contents) 70 | 71 | # Basically a mutable string 72 | splitcontents = list(contents) 73 | 74 | fstring_depth = 0 75 | 76 | # Iterate in reverse so the offsets are always correct 77 | tokens_l = list(tokenize.generate_tokens(io.StringIO(contents).readline)) 78 | tokens = reversed(tokens_l) 79 | for token_type, token_text, (srow, scol), (erow, ecol), _ in tokens: 80 | if token_type == FSTRING_START: # pragma: >=3.12 cover 81 | fstring_depth += 1 82 | elif token_type == FSTRING_END: # pragma: >=3.12 cover 83 | fstring_depth -= 1 84 | elif fstring_depth == 0 and token_type == tokenize.STRING: 85 | new_text = handle_match(token_text) 86 | splitcontents[ 87 | line_offsets[srow] + scol: 88 | line_offsets[erow] + ecol 89 | ] = new_text 90 | 91 | new_contents = "".join(splitcontents) 92 | if contents != new_contents: 93 | with open(filename, "w", encoding="UTF-8", newline="") as f: 94 | f.write(new_contents) 95 | return 1 96 | else: 97 | return 0 98 | 99 | 100 | def main(argv: Sequence[str] | None = None) -> int: 101 | parser = argparse.ArgumentParser() 102 | parser.add_argument("filenames", nargs="*", help="Filenames to fix") 103 | args = parser.parse_args(argv) 104 | 105 | retv = 0 106 | 107 | for filename in args.filenames: 108 | return_value = fix_strings(filename) 109 | if return_value != 0: 110 | print(f"Fixing strings in {filename}") 111 | retv |= return_value 112 | 113 | return retv 114 | 115 | 116 | if __name__ == "__main__": 117 | raise SystemExit(main()) 118 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | version: 2 4 | build: 5 | os: ubuntu-22.04 6 | tools: 7 | python: "3.12" 8 | sphinx: 9 | configuration: docs/source/conf.py 10 | builder: "html" 11 | fail_on_warning: true 12 | python: 13 | install: 14 | - requirements: docs/requirements.txt 15 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Code of Conduct 2 | 3 | 4 | ## Our Pledge 5 | 6 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make 7 | participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, 8 | disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, 9 | socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. 10 | 11 | 12 | ## Our Standards 13 | 14 | * Being open. Members of the community are open to collaboration. 15 | * Focusing on what is best for the community. We're respectful of the processes set forth in the community, and we work 16 | within them. 17 | * Acknowledging time and effort. We're respectful of the volunteer efforts that permeate the community. We're thoughtful 18 | when addressing the efforts of others, keeping in mind that often times the labour was completed simply for the good 19 | of the community. 20 | * Being respectful of differing viewpoints and experiences. We're receptive to constructive comments and criticism, as 21 | the experiences and skill sets of other members contribute to the whole of our efforts. 22 | * Being considerate towards other community members. We're attentive in our communications and we're tactful when 23 | approaching differing views. 24 | * Using welcoming and inclusive language. We're accepting of all who wish to take part in our activities, fostering an 25 | environment where anyone can participate and everyone can make a difference. 26 | * Take responsibility for our words and our actions. We can all make mistakes; when we do, we take responsibility for 27 | them. If someone has been harmed or offended, we listen carefully and respectfully, and work to right the wrong. 28 | * Step down considerately. When somebody leaves or disengages from the project, we ask that they do so in a way that 29 | minimises disruption to the project. 30 | * Ask for help when unsure. Nobody is expected to be perfect in this community. Asking questions early avoids many 31 | problems later, so questions are encouraged, though they may be directed to the appropriate forum. Those who are asked 32 | should be responsive and helpful. 33 | * Value decisiveness, clarity and consensus. Disagreements, social and technical, are normal, but we do not allow them 34 | to persist and fester leaving others uncertain of the agreed direction. We expect participants in the project to 35 | resolve disagreements constructively. When they cannot, we escalate the matter to structures with designated leaders 36 | to arbitrate and provide clarity and direction. 37 | 38 | 39 | ## Our Responsibilities 40 | 41 | Project maintainers are responsible for clarifying the standards of acceptable behaviour and are expected to take 42 | appropriate and fair corrective action in response to any instances of unacceptable behaviour. Project maintainers have 43 | the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other 44 | contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for 45 | other behaviour that they deem inappropriate, threatening, offensive, or harmful. 46 | 47 | 48 | ## Enforcement 49 | 50 | Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported to the community leaders 51 | responsible for enforcement at [oss@gchq.gov.uk](mailto:oss@gchq.gov.uk). All complaints will be reviewed and 52 | investigated promptly and fairly, and will result in a response that is deemed necessary and appropriate to the 53 | circumstances. The community leaders responsible for enforcement are obligated to maintain confidentiality with regard 54 | to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 55 | 56 | 57 | ## Attribution 58 | 59 | This Code of Conduct has been adapted with modifications from the Contributor Covenant (version 1.4), the Python Code of 60 | conduct and the Ubuntu Code of Conduct (version 2.0). 61 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Concourse Tools 2 | 3 | If you find any issues/bugs in the software, please [file an issue](https://github.com/gchq/ConcourseTools/issues). 4 | Please provide full details of your issue, and ideally code to reproduce it. 5 | 6 | 7 | ## Pull Requests 8 | 9 | Prior to us accepting any work, you must sign the [GCHQ CLA Agreement](https://cla-assistant.io/gchq/ConcourseTools). 10 | We follow a branching strategy for handling contributions: 11 | 12 | 1. Fork the Project 13 | 2. Create your Feature Branch (`git checkout -b feature/new_thing`) 14 | 3. Commit your Changes (`git commit -m 'Add a new thing'`) 15 | 4. Push to the Branch (`git push origin feature/new_thing`) 16 | 5. Open a Pull Request 17 | 18 | ### Pre-commit 19 | 20 | Please make use of the [pre-commit checks](https://pre-commit.com/), which should be installed before any new code is 21 | committed: 22 | 23 | ```shell 24 | $ python3 -m pip install pre-commit 25 | $ pre-commit install 26 | ``` 27 | 28 | To run the checks at any time: 29 | 30 | ```shell 31 | $ pre-commit run --all-files 32 | ``` 33 | 34 | Please do not add any new pre-commit checks yourself and instead raise a ticket. 35 | 36 | ### Tests 37 | 38 | Before opening a pull request, please ensure that the tests pass. To do this, run the following: 39 | 40 | ```shell 41 | $ python3 -m unittest discover . 42 | ``` 43 | 44 | Tests are written with `unittest` and - with the exception of the [example tests](tests/test_examples.py) - do not 45 | require any additional dependencies. To run the example tests you should install the additional dependencies: 46 | 47 | ```shell 48 | $ python3 -m pip install -r requirements-tests.txt --no-deps 49 | ``` 50 | 51 | ### Mypy 52 | 53 | Please also run a type check with [Mypy](https://github.com/python/mypy): 54 | 55 | ```shell 56 | $ python3 -m pip install mypy 57 | $ python3 -m mypy concoursetools 58 | ``` 59 | 60 | ### Documentation 61 | 62 | The documentation for Concourse Tools use [Sphinx](https://www.sphinx-doc.org/en/master/index.html). Please ensure that 63 | new features are appropriately documented before opening your pull request. To build the documentation locally, first 64 | install the dependencies: 65 | 66 | ```shell 67 | $ python3 -m pip install -r docs/requirements.txt 68 | ``` 69 | 70 | Next, run the following: 71 | 72 | ```shell 73 | $ python3 -m sphinx -b html -aE docs/source docs/build 74 | ``` 75 | 76 | Please also make use of the [linkcheck builder](https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-the-linkcheck-builder): 77 | 78 | ```shell 79 | $ python3 -m sphinx -b linkcheck docs/source docs/build # check that all links resolve 80 | ``` 81 | 82 | 83 | ## Coding Standards and Conventions 84 | 85 | Concourse Tools is a fully-typed library, so please ensure all functions, methods and classes are fully typed. 86 | 87 | Concourse Tools uses [Sphinx-style docstrings](https://sphinx-rtd-tutorial.readthedocs.io/en/latest/docstrings.html). 88 | 89 | This project aims to depend only on the standard library, so contributions which add additional dependencies outside of 90 | the standard library are likely to be rejected unless absolutely necessary. 91 | 92 | 93 | ### Style Guide 94 | 95 | Concourse Tools does not have an explicit style guide outside of the pre-commit checks for enforcing PEP8 and double-quote strings. 96 | However, please ensure your code is as readable and clear as possible. Reviewers will highlight any code changes they feel 97 | is inconsistent or difficult to parse. 98 | 99 | Please refrain from running Black over the code, as it can cause readability issues for nested statements. 100 | 101 | Finally, please do not refactor "other people's" code when submitting a pull request. 102 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | ## (C) Crown Copyright GCHQ 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) 6 | 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ConcourseTools logo 5 | 6 | 7 | ![version](https://img.shields.io/badge/version-0.8.0-informational) 8 | ![pre-release](https://img.shields.io/badge/pre--release-beta-red) 9 | ![python](https://img.shields.io/badge/python-%3E%3D3.9-informational) 10 | ![coverage](https://img.shields.io/badge/coverage-96%25-brightgreen) 11 | ![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=orange) 12 | 13 | A Python package for easily implementing Concourse [resource types](https://concourse-ci.org/implementing-resource-types.html). 14 | 15 | 16 | ## About 17 | 18 | [Concourse CI](https://concourse-ci.org/) is an "open-source continuous thing-doer" designed to enable general 19 | automation with intuitive and re-usable components. Resources represent all external inputs and outputs to and from the 20 | pipeline, and many of these have been implemented in open source. In order to best leverage the Python ecosystem of 21 | open-source packages, Concourse Tools abstracts away the implementation details of Concourse resource types to allow 22 | users to focus on writing the code they want to run. 23 | 24 | 25 | ## Installation 26 | 27 | Install from [GitHub](https://github.com/gchq/ConcourseTools/), or from [PyPI](https://pypi.org/project/concoursetools/): 28 | 29 | ```shell 30 | $ pip install concoursetools 31 | ``` 32 | 33 | ## Usage 34 | 35 | Start by familiarising yourself with the Concourse resource "rules" in the [documentation](https://concourse-ci.org/implementing-resource-types.html). To recreate that example, start by creating a new `concourse.py` file in your repository. The first step is to create a `Version` subclass: 36 | 37 | ```python 38 | from dataclasses import dataclass 39 | from concoursetools import TypedVersion 40 | 41 | 42 | @dataclass() 43 | class GitVersion(TypedVersion): 44 | ref: str 45 | ``` 46 | 47 | Next, create a subclass of `ConcourseResource`: 48 | 49 | ```python 50 | from concoursetools import ConcourseResource 51 | 52 | 53 | class GitResource(ConcourseResource[GitVersion]): 54 | 55 | def __init__(self, uri: str, branch: str, private_key: str) -> None: 56 | super().__init__(GitVersion) 57 | self.uri = uri 58 | self.branch = branch 59 | self.private_key = private_key 60 | ``` 61 | 62 | Here, the parameters in the `__init__` method will be taken from the `source` configuration for the resource. 63 | Now, implement the three methods required to define the behaviour of the resource: 64 | 65 | 66 | ```python 67 | from pathlib import Path 68 | from typing import Any 69 | from concoursetools import BuildMetadata 70 | 71 | 72 | class GitResource(ConcourseResource[GitVersion]): 73 | ... 74 | 75 | def fetch_new_versions(self, previous_version: GitVersion | None) -> list[GitVersion]: 76 | ... 77 | 78 | def download_version(self, version: GitVersion, destination_dir: pathlib.Path, 79 | build_metadata: BuildMetadata, **kwargs: Any) -> tuple[GitVersion, dict[str, str]]: 80 | ... 81 | 82 | def publish_new_version(self, sources_dir: pathlib.Path, build_metadata: BuildMetadata, 83 | **kwargs: Any) -> tuple[GitVersion, dict[str, str]]: 84 | ... 85 | ``` 86 | 87 | The keyword arguments in `download_version` and `publish_new_version` correspond to `params` in the `get` step, 88 | and `get_params` in the `put` step respectively. 89 | 90 | Once you are happy with the resource, generate the `Dockerfile` using the Concourse Tools CLI: 91 | 92 | ```shell 93 | $ python3 -m concoursetools dockerfile . 94 | ``` 95 | 96 | Finally, upload the Docker image to a registry, and use it in your pipelines! 97 | 98 | For more information - and for more in-depth examples - see the [documentation](https://concoursetools.readthedocs.io/en/stable/). 99 | 100 | 101 | ## Bugs and Contributions 102 | 103 | Concourse Tools is in beta, and still under somewhat-active development. Contributions, fixes, suggestions and bug 104 | reports are all welcome: Please familiarise yourself with our 105 | [contribution guidelines](https://github.com/gchq/ConcourseTools/blob/main/CONTRIBUTING.md). 106 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | 4 | ## Supported Versions 5 | 6 | Concourse Tools is supported on a best endeavours basis. Patches will be applied to the latest version rather than 7 | retroactively to older versions. To ensure you are using the most secure version of Concoursetools, please make sure you 8 | have installed the [latest version](https://pypi.org/project/concoursetools/). 9 | 10 | 11 | ## Reporting a Vulnerability 12 | 13 | Disclosures of vulnerabilities in Concourse Tools are always welcome. Whilst we aim to write clean and secure code free 14 | from bugs, we recognise that this is an open source project, relying on other of open source libraries that are modified 15 | and updated on a regular basis. We hope that the community will continue to support us as we endeavour to maintain and 16 | develop this tool together. 17 | 18 | If you believe that you have identified a potential vulnerability in the code base, please report this promptly to 19 | [oss@gchq.gov.uk](mailto:oss@gchq.gov.uk). Please describe the problem in as much detail as possible, ideally with 20 | examples. Each report will be dealt with on a case-by-case basis. You will receive regular communication on the 21 | resolution and progress of your report. 22 | -------------------------------------------------------------------------------- /concoursetools/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | A Python package for easily implementing Concourse resource types. 4 | """ 5 | from concoursetools.metadata import BuildMetadata 6 | from concoursetools.resource import ConcourseResource 7 | from concoursetools.version import TypedVersion, Version 8 | 9 | __all__ = ("BuildMetadata", "ConcourseResource", "Version", "TypedVersion") 10 | __author__ = "GCHQ" 11 | __version__ = "0.8.0" 12 | -------------------------------------------------------------------------------- /concoursetools/__main__.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | The main entrypoint for the Concourse Tools CLI: 4 | 5 | $ python3 -m concoursetools --help 6 | """ 7 | import sys 8 | 9 | from concoursetools.cli import cli 10 | 11 | 12 | def main() -> int: 13 | """Run the main function.""" 14 | cli.invoke(sys.argv[1:]) 15 | return 0 16 | 17 | 18 | if __name__ == "__main__": 19 | main() 20 | -------------------------------------------------------------------------------- /concoursetools/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Contains the Concourse Tools CLI. 4 | """ 5 | from __future__ import annotations 6 | 7 | from concoursetools.cli.commands import cli 8 | 9 | __all__ = ("cli",) 10 | -------------------------------------------------------------------------------- /concoursetools/cli/commands.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Commands for the Concourse Tools CLI. 4 | """ 5 | from __future__ import annotations 6 | 7 | from pathlib import Path 8 | import sys 9 | import textwrap 10 | 11 | from concoursetools import dockertools 12 | from concoursetools.cli.parser import CLI 13 | from concoursetools.colour import Colour, colour_print 14 | from concoursetools.importing import import_single_class_from_module 15 | from concoursetools.resource import ConcourseResource 16 | 17 | cli = CLI() 18 | 19 | DEFAULT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" 20 | 21 | 22 | @cli.register(allow_short={"executable", "class_name", "resource_file"}) 23 | def assets(path: str, /, *, executable: str | None = None, resource_file: str = "concourse.py", 24 | class_name: str | None = None) -> None: 25 | """ 26 | Create the assets script directory. 27 | 28 | :param path: The location at which to the script files will be written. 29 | Pass '.' to write scripts to the current directory. 30 | :param executable: The python executable to place at the top of the file. Defaults to '/usr/bin/env python3'. 31 | :param resource_file: The path to the module containing the resource class. Defaults to 'concourse.py'. 32 | :param class_name: The name of the resource class in the module, if there are multiple. 33 | """ 34 | resource_class = import_single_class_from_module(Path(resource_file), parent_class=ConcourseResource, # type: ignore[type-abstract] 35 | class_name=class_name) 36 | assets_folder = Path(path) 37 | assets_folder.mkdir(parents=True, exist_ok=True) 38 | 39 | file_name_to_method_name: dict[dockertools.ScriptName, dockertools.MethodName] = { 40 | "check": "check_main", 41 | "in": "in_main", 42 | "out": "out_main", 43 | } 44 | for file_name, method_name in file_name_to_method_name.items(): 45 | file_path = assets_folder / file_name 46 | dockertools.create_script_file(file_path, resource_class, method_name, 47 | executable or dockertools.DEFAULT_EXECUTABLE) 48 | 49 | 50 | @cli.register(allow_short={"executable", "class_name", "resource_file"}) 51 | def dockerfile(path: str, /, *, executable: str | None = None, image: str = "python", tag: str | None = None, 52 | suffix: str | None = None, resource_file: str = "concourse.py", class_name: str | None = None, 53 | pip_args: str | None = None, include_rsa: bool = False, include_netrc: bool = False, 54 | encoding: str | None = None, no_venv: bool = False, dev: bool = False) -> None: 55 | """ 56 | Create the Dockerfile. 57 | 58 | :param path: The location to which to write the Dockerfile. 59 | Pass '.' to write it to the current directory. 60 | :param executable: The python executable to place at the top of the file. Defaults to '/usr/bin/env python3'. 61 | :param image: Specify the image used in the FROM instruction. 62 | :param tag: The tag to combine with the image. Defaults to the major/minor version of the current Python environment. 63 | :param suffix: An optional suffix to combine with the tag to create the full tag. 64 | :param resource_file: The path to the module containing the resource class. Defaults to 'concourse.py'. 65 | :param class_name: The name of the resource class in the module, if there are multiple. 66 | :param pip_args: An optional string to be appended to all calls to pip, e.g. '--timeout 100'. 67 | :param include_rsa: Enable the Dockerfile to (securely) use your RSA private key during building. 68 | :param include_netrc: Enable the Dockerfile to (securely) use your netrc file during building. 69 | :param encoding: The encoding of the created file. If not passed, Concourse Tools will use the user's default encoding. 70 | :param no_venv: Pass to explicitly not use a virtual environment within the image. This is not recommended and exists 71 | to ensure legacy behaviour. 72 | :param dev: Pass to copy a local version of Concourse Tools to the image, instead of installing from PyPI. 73 | The onus is on the user to ensure that the "concoursetools" exists in the working directory at 74 | Docker build time. 75 | """ 76 | directory_path = Path(path) 77 | if directory_path.is_dir(): 78 | file_path = directory_path / "Dockerfile" 79 | else: 80 | file_path = directory_path 81 | 82 | assets_to_potentially_include = { 83 | "-c": class_name, 84 | "-e": executable, 85 | } 86 | assets_options = {key: value for key, value in assets_to_potentially_include.items() if value is not None} 87 | 88 | cli_split_command = ["python3", "-m", "concoursetools", "assets", ".", "-r", resource_file] 89 | for key, value in assets_options.items(): 90 | if value is not None: 91 | cli_split_command.extend([key, value]) 92 | 93 | cli_command = " ".join(cli_split_command) 94 | if tag is None: 95 | if suffix is None: 96 | tag = DEFAULT_PYTHON_VERSION 97 | else: 98 | tag = f"{DEFAULT_PYTHON_VERSION}-{suffix}" 99 | 100 | final_dockerfile = dockertools.Dockerfile() 101 | 102 | final_dockerfile.new_instruction_group( 103 | dockertools.FromInstruction(image=image, tag=tag), 104 | ) 105 | 106 | if no_venv: 107 | if dev: 108 | raise ValueError("Can only specify --no-venv in production mode") 109 | else: 110 | final_dockerfile.new_instruction_group( 111 | dockertools.RunInstruction(["python3 -m venv /opt/venv"]), 112 | dockertools.Comment("Activate venv"), 113 | dockertools.EnvInstruction({"PATH": "/opt/venv/bin:$PATH"}), 114 | ) 115 | 116 | final_dockerfile.new_instruction_group( 117 | dockertools.CopyInstruction("requirements.txt"), 118 | ) 119 | 120 | mounts: list[dockertools.Mount] = [] 121 | 122 | if include_rsa: 123 | mounts.extend([ 124 | dockertools.SecretMount( 125 | secret_id="private_key", 126 | target="/root/.ssh/id_rsa", 127 | mode=0o600, 128 | required=True, 129 | ), 130 | dockertools.SecretMount( 131 | secret_id="known_hosts", 132 | target="/root/.ssh/known_hosts", 133 | mode=0o644, 134 | ), 135 | ]) 136 | 137 | if include_netrc: 138 | mounts.extend([ 139 | dockertools.SecretMount( 140 | secret_id="netrc", 141 | target="/root/.netrc", 142 | mode=0o600, 143 | required=True, 144 | ), 145 | ]) 146 | 147 | if pip_args is None: 148 | pip_string_suffix = "" 149 | else: 150 | pip_string_suffix = f" {pip_args}" 151 | 152 | if dev: 153 | final_dockerfile.new_instruction_group( 154 | dockertools.CopyInstruction("concoursetools"), 155 | ) 156 | final_dockerfile.new_instruction_group( 157 | dockertools.MultiLineRunInstruction([ 158 | "python3 -m pip install --upgrade pip" + pip_string_suffix, 159 | "pip install ./concoursetools", 160 | "pip install -r requirements.txt --no-deps" + pip_string_suffix, 161 | ], mounts=mounts), 162 | ) 163 | else: 164 | final_dockerfile.new_instruction_group( 165 | dockertools.MultiLineRunInstruction([ 166 | "python3 -m pip install --upgrade pip" + pip_string_suffix, 167 | "pip install -r requirements.txt --no-deps" + pip_string_suffix, 168 | ], mounts=mounts), 169 | ) 170 | 171 | final_dockerfile.new_instruction_group( 172 | dockertools.WorkDirInstruction("/opt/resource/"), 173 | dockertools.CopyInstruction(resource_file, f"./{resource_file}"), 174 | dockertools.RunInstruction([cli_command]), 175 | ) 176 | 177 | final_dockerfile.new_instruction_group( 178 | dockertools.EntryPointInstruction(["python3"]), 179 | ) 180 | 181 | final_dockerfile.write_to_file(file_path, encoding=encoding) 182 | 183 | 184 | @cli.register(allow_short={"executable", "class_name", "resource_file"}) 185 | def legacy(path: str, /, *, executable: str | None = None, resource_file: str = "concourse.py", 186 | class_name: str | None = None, docker: bool = False, include_rsa: bool = False) -> None: 187 | """ 188 | Invoke the legacy CLI. 189 | 190 | :param path: The location at which to place the scripts. 191 | :param executable: The python executable to place at the top of the file. Defaults to '/usr/bin/env python3'. 192 | :param resource_file: The path to the module containing the resource class. Defaults to 'concourse.py'. 193 | :param class_name: The name of the resource class in the module, if there are multiple. 194 | :param docker: Pass to create a skeleton Dockerfile at the path instead. 195 | :param include_rsa: Enable the Dockerfile to (securely) use your RSA private key during building. 196 | """ 197 | colour_print(textwrap.dedent(""" 198 | The legacy CLI has been deprecated. 199 | Please refer to the documentation or help pages for the up to date CLI. 200 | This CLI will be removed in version 0.10.0, or in version 1.0.0, whichever is sooner. 201 | """), colour=Colour.RED) 202 | if docker: 203 | return dockerfile(path, suffix="alpine", executable=executable, resource_file=resource_file, class_name=class_name, 204 | include_rsa=include_rsa, no_venv=True) 205 | assets(path, executable=executable, resource_file=resource_file, class_name=class_name) 206 | -------------------------------------------------------------------------------- /concoursetools/cli/docstring.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Concourse Tools uses a custom CLI tool for easier management of command line functions. 4 | """ 5 | from __future__ import annotations 6 | 7 | from collections.abc import Callable, Generator 8 | from dataclasses import dataclass 9 | import inspect 10 | import re 11 | from typing import TypeVar 12 | 13 | RE_PARAM = re.compile(r":param (.*?):") 14 | RE_WHITESPACE = re.compile(r"\s+") 15 | 16 | CLIFunction = Callable[..., None] 17 | CLIFunctionT = TypeVar("CLIFunctionT", bound=CLIFunction) 18 | T = TypeVar("T") 19 | 20 | 21 | @dataclass(frozen=True) 22 | class Docstring: 23 | """ 24 | Represents a function docstring. 25 | 26 | :param first_line: The first line of the docstring (separated by double whitespace). 27 | :param description: The remaining description before any parameters. 28 | :param parameters: A mapping of parameter name to description, with newlines replaced with whitespace. 29 | """ 30 | first_line: str 31 | description: str 32 | parameters: dict[str, str] 33 | 34 | @classmethod 35 | def from_object(cls, obj: object) -> "Docstring": 36 | """ 37 | Parse an object with a docstring. 38 | 39 | :param obj: An object which may have a docstring, such as a function or module. 40 | """ 41 | raw_docstring = inspect.getdoc(obj) or "" 42 | return cls.from_string(raw_docstring) 43 | 44 | @classmethod 45 | def from_string(cls, raw_docstring: str) -> "Docstring": 46 | """ 47 | Parse a docstring. 48 | 49 | :param raw_docstring: The raw docstring of an object. 50 | """ 51 | try: 52 | first_line, remaining_lines = raw_docstring.split("\n\n", maxsplit=1) 53 | except ValueError: 54 | if RE_PARAM.match(raw_docstring.strip()): # all we have are params with no description 55 | first_line = "" 56 | remaining_lines = raw_docstring.strip() 57 | else: 58 | first_line = raw_docstring.strip() 59 | return cls(first_line, "", {}) 60 | 61 | description, *remaining_params = RE_PARAM.split(remaining_lines.lstrip()) 62 | parameters = {param: " ".join(info.split()).strip() for param, info in _pair_up(remaining_params)} 63 | return cls(first_line, description.strip(), parameters) 64 | 65 | 66 | def _pair_up(data: list[str]) -> Generator[tuple[str, str], None, None]: 67 | """ 68 | Pair up a list of items. 69 | 70 | :param data: A list to be paired. 71 | :raises ValueError: If there are an odd number of values in the list. 72 | 73 | :Example: 74 | >>> list(_pair_up([1, 2, 3, 4, 5, 6])) 75 | [(1, 2), (3, 4), (5, 6)] 76 | """ 77 | for i in range(0, len(data), 2): 78 | try: 79 | yield data[i], data[i + 1] 80 | except IndexError: 81 | raise ValueError(f"Needed an even number of values, got {len(data)}") 82 | -------------------------------------------------------------------------------- /concoursetools/colour.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | The Concourse web UI will interpret :wikipedia:`ANSI colour codes `, 4 | and so a handful of rudimentary functions for formatting with colour are include in 5 | :mod:`concoursetools.colour`. 6 | 7 | .. tip:: 8 | There are plenty of more mature libraries for printing coloured output, 9 | such as `termcolor `_ and 10 | `Rich `_. 11 | Concourse Tools specifically has no external dependencies, and so these 12 | must be actively installed and managed by a user. 13 | """ 14 | from __future__ import annotations 15 | 16 | from collections.abc import Generator 17 | from contextlib import contextmanager 18 | from typing import Any 19 | 20 | END_COLOUR = "\033[0m" 21 | BOLD = "\033[1m" 22 | UNDERLINE = "\033[4m" 23 | 24 | 25 | class _NoPrint(str): 26 | 27 | def __str__(self) -> str: 28 | raise TypeError("Can't print!") 29 | 30 | def __repr__(self) -> str: 31 | return f"{type(self).__name__}()" 32 | 33 | 34 | MISSING_COLOUR = _NoPrint() 35 | 36 | 37 | def colourise(string: str, colour: str) -> str: 38 | """ 39 | Convert a string into a string which will be coloured on print. 40 | 41 | This enables coloured output within an f-string or similar. It is not recommended for 42 | colouring complete strings, as it is far less efficient than the other functions. 43 | 44 | :param string: The string to be colourised. 45 | :param colour: The :wikipedia:`ANSI colour escape code ` 46 | for the required colour. 47 | 48 | :Example: 49 | 50 | >>> print(f"Hello {colourise('world', colour=Colour.RED)}") 51 | """ 52 | return f"{colour}{string}{END_COLOUR}" 53 | 54 | 55 | def colour_print(*values: object, colour: str = MISSING_COLOUR, bold: bool = False, underline: bool = False, 56 | **print_kwargs: Any) -> None: 57 | """ 58 | Print something in colour. 59 | 60 | This function behaves exactly like :func:`print`, just with more functionality: 61 | 62 | :param colour: The :wikipedia:`ANSI colour escape code ` for the required colour. 63 | :param bold: Print the text in **bold**. 64 | :param underline: Print the text with an underline. 65 | 66 | :Example: 67 | 68 | >>> colour_print(1, 2, 3, sep="-", colour=Colour.GREEN) 69 | """ 70 | try: 71 | with print_in_colour(colour, bold=bold, underline=underline): 72 | print(*values, **print_kwargs) 73 | except TypeError as error: 74 | if colour is MISSING_COLOUR: 75 | raise ValueError("You forgot to pass the colour as a keyword argument") from error 76 | raise 77 | 78 | 79 | @contextmanager 80 | def print_in_colour(colour: str, bold: bool = False, underline: bool = False) -> Generator[None, None, None]: 81 | """ 82 | Print anything in colour within a :ref:`context manager `. 83 | 84 | This is especially useful for colourising output from other external functions 85 | which you cannot control. 86 | 87 | :param colour: The :wikipedia:`ANSI colour escape code ` for the required colour. 88 | :param bold: Print the text in **bold**. 89 | :param underline: Print the text with an underline. 90 | 91 | :Example: 92 | 93 | >>> with print_in_colour(Colour.BLUE, bold=True): 94 | ... print("Hello!") 95 | """ 96 | try: 97 | print(colour, end="") 98 | if bold: 99 | print(BOLD, end="") 100 | if underline: 101 | print(UNDERLINE, end="") 102 | yield 103 | finally: 104 | print(END_COLOUR, end="") 105 | 106 | 107 | class Colour: 108 | """ 109 | A few common ANSI colours. 110 | """ 111 | BLUE = "\033[94m" 112 | CYAN = "\033[96m" 113 | GREEN = "\033[92m" 114 | PURPLE = "\033[95m" 115 | RED = "\033[91m" 116 | YELLOW = "\033[93m" 117 | -------------------------------------------------------------------------------- /concoursetools/importing.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Functions for dynamically importing from Python modules. 4 | """ 5 | from __future__ import annotations 6 | 7 | from collections.abc import Generator, Sequence 8 | from contextlib import contextmanager 9 | import importlib.util 10 | import inspect 11 | from pathlib import Path 12 | import sys 13 | from types import ModuleType 14 | from typing import TypeVar 15 | 16 | T = TypeVar("T") 17 | 18 | 19 | def import_single_class_from_module(file_path: Path, parent_class: type[T], class_name: str | None = None) -> type[T]: 20 | """ 21 | Import the resource class from the module. 22 | 23 | Similar to :func:`import_classes_from_module`, but ensures only one class is returned. 24 | 25 | :param file_path: The location of the module as a file path. 26 | :param class_name: The name of the class to extract. Required if multiple are returned. 27 | :param parent_class: All subclasses of this class defined within the module 28 | (not imported from elsewhere) will be extracted. 29 | :returns: The extracted class. 30 | :raises RuntimeError: If too many or too few classes are available in the module, unless the class name is specified. 31 | """ 32 | possible_resource_classes = import_classes_from_module(file_path, parent_class=parent_class) 33 | 34 | if class_name is None: 35 | if len(possible_resource_classes) == 1: 36 | _, resource_class = possible_resource_classes.popitem() 37 | else: 38 | if len(possible_resource_classes) == 0: 39 | raise RuntimeError(f"No subclasses of {parent_class.__name__!r} found in {file_path}") 40 | raise RuntimeError(f"Multiple subclasses of {parent_class.__name__!r} found in {file_path}:" 41 | f" {set(possible_resource_classes)}") 42 | else: 43 | resource_class = possible_resource_classes[class_name] 44 | 45 | return resource_class 46 | 47 | 48 | def import_classes_from_module(file_path: Path, parent_class: type[T]) -> dict[str, type[T]]: 49 | """ 50 | Import all available resource classes from the module. 51 | 52 | :param file_path: The location of the module as a file path. 53 | :param parent_class: All subclasses of this class defined within the module 54 | (not imported from elsewhere) will be extracted. 55 | :returns: A mapping of class name to class. 56 | """ 57 | import_path = file_path_to_import_path(file_path) 58 | module = import_py_file(import_path, file_path) 59 | 60 | possible_resource_classes = {} 61 | for _, cls in inspect.getmembers(module, predicate=inspect.isclass): 62 | try: 63 | class_is_subclass_of_parent = issubclass(cls, parent_class) 64 | except TypeError: 65 | class_is_subclass_of_parent = False 66 | 67 | class_is_defined_in_this_module = (cls.__module__ == import_path) 68 | class_is_not_private = (not cls.__name__.startswith("_")) 69 | 70 | if class_is_subclass_of_parent and class_is_defined_in_this_module and class_is_not_private: 71 | possible_resource_classes[cls.__name__] = cls 72 | 73 | return possible_resource_classes 74 | 75 | 76 | def file_path_to_import_path(file_path: Path) -> str: 77 | """ 78 | Convert a file path to an import path. 79 | 80 | :param file_path: The path to a Python file. 81 | :raises ValueError: If the path doesn't end in a '.py' extension. 82 | 83 | :Example: 84 | >>> file_path_to_import_path(Path("module.py")) 85 | 'module' 86 | >>> file_path_to_import_path(Path("path/to/module.py")) 87 | 'path.to.module' 88 | """ 89 | *path_components, file_name = file_path.parts 90 | module_name, extension = file_name.split(".") 91 | if extension != "py": 92 | raise ValueError(f"{file_path!r} does not appear to be a valid Python module") 93 | 94 | path_components.append(module_name) 95 | import_path = ".".join(path_components) 96 | return import_path 97 | 98 | 99 | def import_py_file(import_path: str, file_path: Path) -> ModuleType: 100 | """ 101 | Import a .py file as a module. 102 | 103 | This is done using a :ref:`standard Python recipe ` via :mod:`importlib.util`. 104 | 105 | :param import_path: The import path added to :data:`sys.modules`. 106 | :param file_path: The path to the .py module. 107 | :returns: The imported module. 108 | :raises FileNotFoundError: If the path does not exist. 109 | """ 110 | try: 111 | spec = importlib.util.spec_from_file_location(import_path, file_path) 112 | if spec is None: 113 | raise RuntimeError("Imported module spec is unexpectedly 'None'") 114 | if spec.loader is None: 115 | raise RuntimeError("Imported module spec loader is unexpectedly 'None'") 116 | 117 | module = importlib.util.module_from_spec(spec) 118 | with edit_sys_path(prepend=[file_path.parent]): 119 | sys.modules[import_path] = module 120 | spec.loader.exec_module(module) 121 | except ModuleNotFoundError as error: 122 | if not file_path.exists(): 123 | raise FileNotFoundError(file_path) from error 124 | raise 125 | 126 | return module 127 | 128 | 129 | @contextmanager 130 | def edit_sys_path(prepend: Sequence[Path] = (), append: Sequence[Path] = ()) -> Generator[None, None, None]: 131 | """ 132 | Temporarily add to :data:`sys.path` within a context manager. 133 | 134 | :param prepend: A sequence of paths to add to :data:`sys.path` *before* the current entries. 135 | :param append: A sequence of paths to add to :data:`sys.path` *after* the current entries. 136 | :seealso: This is used to enable local imports for the :func:`import_py_file`. 137 | """ 138 | original_sys_path = sys.path.copy() # otherwise we just reference the original 139 | try: 140 | sys.path = [str(path) for path in prepend] + sys.path + [str(path) for path in append] 141 | yield 142 | finally: 143 | sys.path = original_sys_path 144 | -------------------------------------------------------------------------------- /concoursetools/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gchq/ConcourseTools/bb28027115429e0379c3dfa038754408eaa08c2b/concoursetools/py.typed -------------------------------------------------------------------------------- /concoursetools/typing.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Concourse Tools contains a small number of additional types for easier resource development. 4 | 5 | .. warning:: 6 | Although Concourse Tools is a typed library, the :class:`concoursetools.resource.ConcourseResource` 7 | class breaks the :wikipedia:`Liskov substitution principle`. If you wish to utilise type hinting 8 | in your development process, then please switch off the check for this type of method overloading. 9 | """ 10 | from __future__ import annotations 11 | 12 | from collections.abc import Callable 13 | from typing import Any, Protocol, TypedDict, TypeVar 14 | 15 | T = TypeVar("T") 16 | 17 | 18 | ResourceConfig = dict[str, Any] 19 | """ 20 | Represents arbitrary configuration passed to a Concourse resource. 21 | See the :concourse:`config-basics.schema.config` schema for more information. 22 | """ 23 | 24 | Params = dict[str, Any] 25 | """ 26 | Represents arbitrary parameters passed to a Concourse resource. 27 | See the :concourse:`config-basics.schema.config` schema for more information. 28 | """ 29 | 30 | Metadata = dict[str, str] 31 | """ 32 | Represents :ref:`Step Metadata` as used by Concourse Tools. 33 | Restrictions on key and value types are determined by :data:`MetadataPair`. 34 | """ 35 | 36 | 37 | class MetadataPair(TypedDict): 38 | """ 39 | Represents :ref:`Step Metadata` as used internally by Concourse. 40 | 41 | Restrictions on key and value types are determined by the 42 | `Go structure itself `_. 43 | """ 44 | name: str 45 | value: str 46 | 47 | 48 | VersionConfig = dict[str, str] 49 | """ 50 | Represents a version of a Concourse resource. 51 | See the :concourse:`config-basics.schema.version` schema for more information. 52 | """ 53 | 54 | VersionT = TypeVar("VersionT", bound="VersionProtocol") 55 | """Represents a generic :class:`~concoursetools.version.Version` subclass.""" 56 | 57 | TypedVersionT = TypeVar("TypedVersionT", bound="TypedVersionProtocol") 58 | """Represents a generic :class:`~concoursetools.version.TypedVersion` subclass.""" 59 | 60 | SortableVersionT = TypeVar("SortableVersionT", bound="SortableVersionProtocol") 61 | """Represents a generic :class:`~concoursetools.version.Version` subclass which is also :ref:`sortable `.""" 62 | 63 | SortableVersionCovariantT = TypeVar("SortableVersionCovariantT", bound="SortableVersionProtocol", covariant=True) 64 | 65 | 66 | class VersionProtocol(Protocol): 67 | """Corresponds to a generic :class:`~concoursetools.version.Version` subclass.""" 68 | def __repr__(self) -> str: 69 | ... 70 | 71 | def __eq__(self, other: object) -> bool: 72 | ... 73 | 74 | def __hash__(self) -> int: 75 | ... 76 | 77 | def to_flat_dict(self) -> VersionConfig: 78 | ... 79 | 80 | @classmethod 81 | def from_flat_dict(cls: type[VersionT], version_dict: VersionConfig) -> VersionT: 82 | ... 83 | 84 | 85 | class SortableVersionProtocol(Protocol): 86 | """Corresponds to a generic :class:`~concoursetools.version.Version` subclass which is also :ref:`sortable `.""" 87 | def __repr__(self) -> str: 88 | ... 89 | 90 | def __eq__(self, other: object) -> bool: 91 | ... 92 | 93 | def __hash__(self) -> int: 94 | ... 95 | 96 | def to_flat_dict(self) -> VersionConfig: 97 | ... 98 | 99 | @classmethod 100 | def from_flat_dict(cls: type[VersionT], version_dict: VersionConfig) -> VersionT: 101 | ... 102 | 103 | def __lt__(self, other: object) -> bool: 104 | ... 105 | 106 | def __le__(self, other: object) -> bool: 107 | ... 108 | 109 | 110 | class TypedVersionProtocol(Protocol): 111 | """Corresponds to a generic :class:`~concoursetools.version.TypedVersion` subclass.""" 112 | def __repr__(self) -> str: 113 | ... 114 | 115 | def __eq__(self, other: object) -> bool: 116 | ... 117 | 118 | def __hash__(self) -> int: 119 | ... 120 | 121 | def to_flat_dict(self) -> VersionConfig: 122 | ... 123 | 124 | @classmethod 125 | def from_flat_dict(cls: type[VersionT], version_dict: VersionConfig) -> VersionT: 126 | ... 127 | 128 | @classmethod 129 | def _flatten_object(cls, obj: Any) -> str: 130 | ... 131 | 132 | @classmethod 133 | def _un_flatten_object(cls, type_: type[TypedVersionT], flat_obj: str) -> TypedVersionT: 134 | ... 135 | 136 | @classmethod 137 | def _get_attribute_type(cls, attribute_name: str) -> type[Any]: 138 | ... 139 | 140 | @classmethod 141 | def flatten(cls, func: Callable[[T], str]) -> Callable[[T], str]: 142 | ... 143 | 144 | @classmethod 145 | def un_flatten(cls, func: Callable[[type[T], str], T]) -> Callable[[type[T], str], T]: 146 | ... 147 | 148 | @staticmethod 149 | def _flatten_default(obj: object) -> str: 150 | ... 151 | 152 | @staticmethod 153 | def _un_flatten_default(type_: type[T], flat_obj: str) -> T: 154 | ... 155 | 156 | 157 | class MultiVersionProtocol(Protocol[SortableVersionCovariantT]): 158 | """Corresponds to a generic :class:`~concoursetools.additional.MultiVersion` subclass.""" 159 | def __init__(self, versions: set[SortableVersionCovariantT]): 160 | ... 161 | 162 | def __repr__(self) -> str: 163 | ... 164 | 165 | def __eq__(self, other: object) -> bool: 166 | ... 167 | 168 | def __hash__(self) -> int: 169 | ... 170 | 171 | @property 172 | def key(self) -> str: 173 | ... 174 | 175 | @property 176 | def sub_version_class(self) -> type[SortableVersionCovariantT]: 177 | ... 178 | 179 | @property 180 | def sub_version_data(self) -> list[VersionConfig]: 181 | ... 182 | 183 | def to_flat_dict(self) -> VersionConfig: 184 | ... 185 | 186 | @classmethod 187 | def from_flat_dict(cls: type[VersionT], version_dict: VersionConfig) -> VersionT: 188 | ... 189 | -------------------------------------------------------------------------------- /cspell.config.yaml: -------------------------------------------------------------------------------- 1 | version: "0.2" 2 | language: en-GB 3 | useGitignore: true 4 | ignorePaths: 5 | - tests 6 | - requirements*.txt 7 | - .pre-commit-config.yaml 8 | - .pre-commit-hooks/* 9 | dictionaries: 10 | - python 11 | words: 12 | - autoclass 13 | - autodata 14 | - autofunction 15 | - automethod 16 | - automodule 17 | - autosectionlabel 18 | - AWSCURRENT 19 | - boto 20 | - botocore 21 | - bysource 22 | - classmethod 23 | - Colors 24 | - colspec 25 | - concoursetools 26 | - dataclass 27 | - dataclasses 28 | - datetime 29 | - dateutil 30 | - Dockerfiles 31 | - dockertools 32 | - docstrings 33 | - elif 34 | - furo 35 | - GCHQ 36 | - gchqdev 37 | - genindex 38 | - HEALTHCHECK 39 | - INPROGRESS 40 | - intersphinx 41 | - kwargs 42 | - linecount 43 | - linenos 44 | - linkcheck 45 | - Liskov 46 | - literalinclude 47 | - maxdepth 48 | - maxsplit 49 | - modindex 50 | - mypy 51 | - ncsc 52 | - ONBUILD 53 | - outdir 54 | - pathlib 55 | - pyobject 56 | - Quickstart 57 | - redef 58 | - refid 59 | - refuri 60 | - rstrip 61 | - rwxr 62 | - seealso 63 | - srcset 64 | - STOPSIGNAL 65 | - stringifying 66 | - termcolor 67 | - tgroup 68 | - thead 69 | - tmpfs 70 | - toctree 71 | - typehints 72 | - undoc 73 | - unitless 74 | - urllib 75 | - vars 76 | - versionadded 77 | - versionchanged 78 | - viewcode 79 | - WORKDIR 80 | - xkcd 81 | ignoreWords: 82 | - cbef 83 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | alabaster==1.0.0 2 | babel==2.17.0 3 | beautifulsoup4==4.13.3 4 | certifi==2025.1.31 5 | charset-normalizer==3.4.1 6 | docutils==0.21.2 7 | furo==2024.8.6 8 | idna==3.10 9 | imagesize==1.4.1 10 | Jinja2==3.1.5 11 | MarkupSafe==3.0.2 12 | packaging==24.2 13 | Pygments==2.19.1 14 | requests==2.32.3 15 | roman-numerals-py==3.0.0 16 | snowballstemmer==2.2.0 17 | soupsieve==2.6 18 | Sphinx==8.2.0 19 | sphinx-autodoc-typehints==3.1.0 20 | sphinx-basic-ng==1.0.0b2 21 | sphinxcontrib-applehelp==2.0.0 22 | sphinxcontrib-devhelp==2.0.0 23 | sphinxcontrib-htmlhelp==2.1.0 24 | sphinxcontrib-jsmath==1.0.1 25 | sphinxcontrib-qthelp==2.0.0 26 | sphinxcontrib-serializinghtml==2.0.0 27 | typing_extensions==4.12.2 28 | urllib3==2.3.0 29 | -------------------------------------------------------------------------------- /docs/source/_static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gchq/ConcourseTools/bb28027115429e0379c3dfa038754408eaa08c2b/docs/source/_static/favicon.png -------------------------------------------------------------------------------- /docs/source/_static/logo-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gchq/ConcourseTools/bb28027115429e0379c3dfa038754408eaa08c2b/docs/source/_static/logo-dark.png -------------------------------------------------------------------------------- /docs/source/_static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gchq/ConcourseTools/bb28027115429e0379c3dfa038754408eaa08c2b/docs/source/_static/logo.png -------------------------------------------------------------------------------- /docs/source/_static/step_metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gchq/ConcourseTools/bb28027115429e0379c3dfa038754408eaa08c2b/docs/source/_static/step_metadata.png -------------------------------------------------------------------------------- /docs/source/_static/style.css: -------------------------------------------------------------------------------- 1 | .toc-drawer { 2 | width: fit-content; 3 | } 4 | -------------------------------------------------------------------------------- /docs/source/additional.rst: -------------------------------------------------------------------------------- 1 | Additional Patterns 2 | =================== 3 | 4 | .. automodule:: concoursetools.additional 5 | 6 | .. autoclass:: concoursetools.additional.OutOnlyConcourseResource 7 | :members: publish_new_version 8 | 9 | .. autoclass:: concoursetools.additional.InOnlyConcourseResource 10 | :members: download_data 11 | 12 | .. autoclass:: concoursetools.additional.TriggerOnChangeConcourseResource 13 | :members: fetch_latest_version 14 | 15 | .. autoclass:: concoursetools.additional.MultiVersionConcourseResource 16 | :members: fetch_latest_sub_versions, download_version 17 | :show-inheritance: 18 | 19 | .. autoclass:: concoursetools.additional.SelfOrganisingConcourseResource 20 | :members: fetch_all_versions 21 | 22 | 23 | Combining Resource Types 24 | ------------------------ 25 | 26 | Occasionally you may wish to implement multiple resource types with the same set of dependencies. Although it is often cleaner to treat these separately and :ref:`build separate Docker images `, there are times where it is easier to build a single image containing all of your resource types, and to select one of them via the resource config. To do this, you can use the :func:`~concoursetools.additional.combine_resource_types` function: 27 | 28 | .. autofunction:: concoursetools.additional.combine_resource_types 29 | -------------------------------------------------------------------------------- /docs/source/api_reference.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | A Concourse resource is a combination of a :class:`~concoursetools.version.Version` class and a :class:`~concoursetools.resource.ConcourseResource` class. 5 | 6 | .. toctree:: 7 | version 8 | resource 9 | build_metadata 10 | additional 11 | -------------------------------------------------------------------------------- /docs/source/build_metadata.rst: -------------------------------------------------------------------------------- 1 | Build Metadata 2 | ============== 3 | 4 | .. automodule:: concoursetools.metadata 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/source/changelog.rst: -------------------------------------------------------------------------------- 1 | What's New 2 | ========== 3 | 4 | .. toctree:: 5 | :glob: 6 | 7 | changelog/development 8 | changelog/* 9 | -------------------------------------------------------------------------------- /docs/source/changelog/development.rst: -------------------------------------------------------------------------------- 1 | Development 2 | =========== 3 | 4 | 5 | * Added Python 3.14 support. 6 | -------------------------------------------------------------------------------- /docs/source/cli.rst: -------------------------------------------------------------------------------- 1 | Command Line 2 | ============ 3 | 4 | .. automodule:: concoursetools.cli.parser 5 | :members: 6 | :exclude-members: Parameter, PositionalArgument, Option, FlagOption 7 | 8 | 9 | .. autoclass:: concoursetools.cli.docstring.Docstring 10 | :members: 11 | 12 | Parameters 13 | ---------- 14 | 15 | .. autoclass:: concoursetools.cli.parser.PositionalArgument 16 | 17 | .. autoclass:: concoursetools.cli.parser.Option 18 | 19 | .. autoclass:: concoursetools.cli.parser.FlagOption 20 | 21 | .. autoclass:: concoursetools.cli.parser.Parameter 22 | :members: 23 | -------------------------------------------------------------------------------- /docs/source/cli_reference.rst: -------------------------------------------------------------------------------- 1 | CLI Reference 2 | ============= 3 | 4 | Concourse Tools contains a CLI for a number of small tasks to help write and deploy resource types. 5 | 6 | 7 | .. cli:: concoursetools.cli.cli 8 | :align: left 9 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Configuration file for the Sphinx documentation builder. 4 | 5 | This file only contains a selection of the most common options. 6 | For a full list see the documentation: https://www.sphinx-doc.org/en/master/usage/configuration.html. 7 | """ 8 | # -- Path setup -------------------------------------------------------------- 9 | 10 | from collections.abc import Callable 11 | from pathlib import Path 12 | import sys 13 | from typing import Any 14 | 15 | import sphinx.config 16 | from sphinx_autodoc_typehints import format_annotation 17 | 18 | CONF_FILE_PATH = Path(__file__).absolute() 19 | SOURCE_FOLDER_PATH = CONF_FILE_PATH.parent 20 | DOCS_FOLDER_PATH = SOURCE_FOLDER_PATH.parent 21 | REPO_FOLDER_PATH = DOCS_FOLDER_PATH.parent 22 | 23 | sys.path.extend([str(DOCS_FOLDER_PATH), str(SOURCE_FOLDER_PATH), str(REPO_FOLDER_PATH)]) 24 | 25 | 26 | import concoursetools 27 | import concoursetools.additional 28 | import concoursetools.cli.parser 29 | import concoursetools.importing 30 | import concoursetools.resource 31 | import concoursetools.typing 32 | import concoursetools.version 33 | 34 | # -- Project information ----------------------------------------------------- 35 | 36 | project = "Concourse Tools" 37 | copyright = "UK Crown" 38 | version = "v" + concoursetools.__version__ 39 | 40 | 41 | # -- General configuration --------------------------------------------------- 42 | 43 | extensions = [ 44 | "sphinx.ext.autodoc", 45 | "sphinx.ext.autosectionlabel", 46 | "sphinx.ext.intersphinx", 47 | "sphinx_autodoc_typehints", 48 | "sphinx.ext.viewcode", 49 | "extensions.cli", 50 | "extensions.concourse", 51 | "extensions.linecount", 52 | "extensions.wikipedia", 53 | "extensions.xkcd", 54 | ] 55 | 56 | toc_object_entries_show_parents = "hide" # don't show prefix in secondary TOC 57 | 58 | always_document_param_types = True 59 | autodoc_member_order = "bysource" 60 | 61 | autodoc_custom_types = { 62 | concoursetools.cli.parser.CLIFunctionT: format_annotation(Callable[..., None], sphinx.config.Config()), 63 | concoursetools.importing.T: ":class:`object`", 64 | concoursetools.typing.VersionT: ":class:`~concoursetools.version.Version`", 65 | concoursetools.typing.SortableVersionT: ":class:`~concoursetools.version.Version`", 66 | } 67 | 68 | suppress_warnings = ["config.cache"] # https://github.com/sphinx-doc/sphinx/issues/12300#issuecomment-2062238457 69 | 70 | 71 | def typehints_formatter(annotation: Any, config: sphinx.config.Config) -> str | None: 72 | """Properly replace custom type aliases.""" 73 | return autodoc_custom_types.get(annotation) 74 | 75 | 76 | nitpicky = True 77 | nitpick_ignore = [ 78 | ("py:class", "concoursetools.additional._PseudoConcourseResource"), 79 | ("py:class", "concoursetools.typing.SortableVersionT"), 80 | ] 81 | 82 | linkcheck_report_timeouts_as_broken = False # silences a warning: https://github.com/sphinx-doc/sphinx/issues/11868 83 | linkcheck_anchors_ignore_for_url = [ 84 | "https://github.com/.*", 85 | ] 86 | linkcheck_ignore = [ 87 | "https://superuser.com/.*", 88 | ] 89 | 90 | intersphinx_mapping = { 91 | "python3": ("https://docs.python.org/3", None), 92 | "requests": ("https://requests.readthedocs.io/en/latest/", None), 93 | } 94 | 95 | html_theme = "furo" 96 | html_favicon = "_static/favicon.png" 97 | 98 | html_static_path = ["_static"] 99 | 100 | html_css_files = [ 101 | "style.css", 102 | "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/fontawesome.min.css", 103 | "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/solid.min.css", 104 | "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/brands.min.css", 105 | ] 106 | 107 | html_theme_options = { 108 | "light_logo": "logo.png", 109 | "dark_logo": "logo-dark.png", 110 | "sidebar_hide_name": True, 111 | "source_repository": "https://github.com/gchq/ConcourseTools/", 112 | "source_branch": "main", 113 | "source_directory": "docs/source/", 114 | "footer_icons": [ 115 | { 116 | "name": "GitHub", 117 | "url": "https://github.com/gchq/ConcourseTools/", 118 | "class": "fa-brands fa-github fa-2x", 119 | }, 120 | { 121 | "name": "PyPI", 122 | "url": "https://pypi.org/project/concoursetools/", 123 | "class": "fa-brands fa-python fa-2x", 124 | } 125 | ], 126 | } 127 | -------------------------------------------------------------------------------- /docs/source/debugging.rst: -------------------------------------------------------------------------------- 1 | Printing/Debugging 2 | ================== 3 | 4 | A Concourse resource is supposed to print output (such as versions and metadata) to ``stdout``, and all debugging messages to ``stderr``. The :class:`~concoursetools.resource.ConcourseResource` class automatically redirects ``stdout`` to ``stderr`` when running its methods, meaning that all printed output from you (and from your dependencies) automatically ends up in ``stderr``. 5 | 6 | Colour 7 | ------ 8 | 9 | .. automodule:: concoursetools.colour 10 | :members: 11 | :exclude-members: Colour 12 | 13 | Available Colours 14 | _________________ 15 | 16 | All ``colour`` arguments are :wikipedia:`ANSI colour escape codes `, but a number of common codes are available as attributes of the :class:`~concoursetools.colour.Colour` class: 17 | 18 | .. autoclass:: concoursetools.colour.Colour 19 | :members: 20 | :undoc-members: 21 | -------------------------------------------------------------------------------- /docs/source/deployment.rst: -------------------------------------------------------------------------------- 1 | Deploying the Resource Type 2 | =========================== 3 | 4 | To properly "deploy" the resource type, your repo should look something like this: 5 | 6 | .. code:: none 7 | 8 | . 9 | |-- Dockerfile 10 | |-- README.md 11 | |-- assets 12 | | |-- check 13 | | |-- in 14 | | |-- out 15 | |-- requirements.txt 16 | |-- concourse.py 17 | |-- tests.py 18 | 19 | 20 | Resource Module 21 | --------------- 22 | 23 | The ``concourse.py`` module contains your :class:`~concoursetools.version.Version` and 24 | :class:`~concoursetools.resource.ConcourseResource` subclasses. It can be called whatever you like, but ``concourse.py`` 25 | is the most clear, and is the default for the CLI. 26 | 27 | .. note:: 28 | 29 | It is possible to split your code into multiple modules, or even packages. This is sometimes useful if you have 30 | written a lot of code, or if it naturally splits. However, make sure that this code is also put in the correct place 31 | in your :ref:`Docker image `. 32 | 33 | 34 | Requirements 35 | ------------ 36 | 37 | The ``requirements.txt`` file should contain a complete list of requirements within your virtual environment (version 38 | pinned) sufficient to reproduce with 39 | 40 | .. code:: shell 41 | 42 | $ pip install -r requirements.txt --no-deps 43 | 44 | If this is not possible, then rewrite your requirements file. If you have any dependencies which are not public, then 45 | you will have to make some adjustments to your :ref:`Dockerfile `. 46 | 47 | 48 | Assets 49 | ------ 50 | 51 | The contents of each of the files in ``assets`` follow the same pattern. For example, here is the contents of 52 | ``assets/check``: 53 | 54 | .. code:: python3 55 | 56 | #!/usr/bin/env python3 57 | """ 58 | Check for new versions of the resource. 59 | """ 60 | from concourse import MyResource 61 | 62 | 63 | if __name__ == "__main__": 64 | MyResource.check_main() 65 | 66 | It is these files which get called by Concourse as part of the resource. Note that none of the asset files have any 67 | extensions, but specific their executable at the top of the file. ``MyResource`` should correspond to your 68 | :class:`~concoursetools.resource.ConcourseResource` subclass, and ``resource`` to the :ref:`module in which you have 69 | placed it `. Replace :meth:`~concoursetools.resource.ConcourseResource.check_main` with 70 | :meth:`~concoursetools.resource.ConcourseResource.in_main` and 71 | :meth:`~concoursetools.resource.ConcourseResource.out_main` for ``assets/in`` and ``assets/out`` respectively. 72 | 73 | .. important:: 74 | 75 | Every file in assets need to be executable. This can be done with 76 | 77 | .. code:: shell 78 | 79 | $ chmod +x assets/* 80 | 81 | .. tip:: 82 | 83 | Because this pattern is suitable for almost every resource type, you can automate the creation of the ``assets`` 84 | folder with the :ref:`cli.assets` CLI command. 85 | 86 | 87 | Dockerfile Structure 88 | -------------------- 89 | 90 | The Dockerfile should look something like: 91 | 92 | .. code-block:: Dockerfile 93 | :linenos: 94 | 95 | FROM python:3.12 96 | 97 | RUN python3 -m venv /opt/venv 98 | # Activate venv 99 | ENV PATH="/opt/venv/bin:$PATH" 100 | 101 | COPY requirements.txt requirements.txt 102 | 103 | RUN \ 104 | python3 -m pip install --upgrade pip && \ 105 | pip install -r requirements.txt --no-deps 106 | 107 | WORKDIR /opt/resource/ 108 | COPY concourse.py ./concourse.py 109 | RUN python3 -m concoursetools . -r concourse.py 110 | 111 | ENTRYPOINT ["python3"] 112 | 113 | .. tip:: 114 | 115 | You can automate the creation of this file with the :ref:`cli.dockerfile` CLI command. 116 | 117 | 118 | Base Image 119 | __________ 120 | 121 | .. code-block:: Dockerfile 122 | :linenos: 123 | 124 | FROM python:3.12 125 | 126 | You should adjust the base image according to your requirements. Concourse Tools will default to ``python:``, 127 | where ```` corresponds to the current major/minor version you are running. However, you may wish to specify 128 | a different base image, such as ``python:3.*-slim`` or ``python:3.*-alpine``. 129 | 130 | 131 | Virtual Environment 132 | ___________________ 133 | 134 | .. code-block:: Dockerfile 135 | :linenos: 136 | :lineno-start: 3 137 | 138 | RUN python3 -m venv /opt/venv 139 | # Activate venv 140 | ENV PATH="/opt/venv/bin:$PATH" 141 | 142 | There is much debate as to whether or not it is worth creating a virtual environment within a Docker container. 143 | Concourse Tools chooses to create one by default in order to maximise the similarity between code running on the image 144 | and code running locally. 145 | 146 | 147 | Installing Requirements 148 | _______________________ 149 | 150 | .. code-block:: Dockerfile 151 | :linenos: 152 | :lineno-start: 7 153 | 154 | COPY requirements.txt requirements.txt 155 | 156 | RUN \ 157 | python3 -m pip install --upgrade pip && \ 158 | pip install -r requirements.txt --no-deps 159 | 160 | By default, Concourse Tools will copy over the ``requirements.txt`` file to use for the resource dependencies. 161 | The installation process is a single command involving two parts: 162 | 163 | 1. Updating ``pip`` to ensure the latest available version at build time, 164 | 2. Installing the static requirements file *without implicit dependencies*. 165 | 166 | If these were two separate commands then Docker would cache the first one and ``pip`` would never be upgraded. 167 | 168 | .. note:: 169 | By passing ``--no-deps`` we ensure that the ``requirements.txt`` file is fully complete, and we are not missing 170 | any implicit dependencies. 171 | 172 | 173 | Including Certs in your Docker Build 174 | ____________________________________ 175 | 176 | .. versionchanged:: 0.8 177 | In previous versions the advice was to use multi-stage builds for this. Although that practice is equally 178 | secure, it makes sense to use `Docker secrets `_ instead. 179 | 180 | If any of your requirements are private then you will need to make your private keys available to the image during the 181 | build process, **without** storing them within the image itself. This can be done by making the following change to the 182 | ``RUN`` command from previous section: 183 | 184 | .. code-block:: Dockerfile 185 | :linenos: 186 | :lineno-start: 7 187 | 188 | COPY requirements.txt requirements.txt 189 | 190 | RUN \ 191 | --mount=type=secret,id=private_key,target=/root/.ssh/id_rsa,mode=0600,required=true \ 192 | --mount=type=secret,id=known_hosts,target=/root/.ssh/known_hosts,mode=0644 \ 193 | python3 -m pip install --upgrade pip && \ 194 | pip install -r requirements.txt --no-deps 195 | 196 | The secrets must then be passed at build time: 197 | 198 | .. code:: shell 199 | 200 | $ docker build \ 201 | --secret id=private_key,src=~/.ssh/id_rsa \ 202 | --secret id=known_hosts,src=~/.ssh/known_hosts \ 203 | . 204 | 205 | The files are then mounted securely using the `correct permissions 206 | `_. 207 | 208 | 209 | Creating Asset Files 210 | ____________________ 211 | 212 | .. code-block:: Dockerfile 213 | :linenos: 214 | :lineno-start: 13 215 | 216 | WORKDIR /opt/resource/ 217 | COPY concourse.py ./concourse.py 218 | RUN python3 -m concoursetools . -r concourse.py 219 | 220 | Concourse requires that your asset files are placed in ``/opt/resource`` on the container, which is done here using 221 | the Concourse Tools CLI to reduce the required code. 222 | 223 | .. warning:: 224 | If you **cannot** use the CLI to create your :ref:`assets ` folder, then you should manually copy your asset 225 | files across to ``/opt/resource``. 226 | 227 | If your resource requires additional modules to work, then you need to ensure they are also copied across **before** 228 | the CLI is invoked: 229 | 230 | .. code-block:: Dockerfile 231 | :linenos: 232 | :lineno-start: 13 233 | 234 | WORKDIR /opt/resource/ 235 | COPY concourse.py ./concourse.py 236 | COPY extra.py ./extra.py 237 | RUN python3 -m concoursetools . -r concourse.py 238 | 239 | 240 | Entry Point 241 | ___________ 242 | 243 | .. code-block:: Dockerfile 244 | :linenos: 245 | :lineno-start: 17 246 | 247 | ENTRYPOINT ["python3"] 248 | 249 | Finally, we specify an ``ENTRYPOINT`` for the container. This has little bearing on the resource itself as Concourse 250 | will specify the scripts it wishes to invoke via the shebang in the scripts. It isn't even used when hijacking the 251 | container (the default is bash). However, it is best practice to set something, and this should make it easiest to 252 | debug locally. 253 | -------------------------------------------------------------------------------- /docs/source/dockertools.rst: -------------------------------------------------------------------------------- 1 | Dockertools 2 | =========== 3 | 4 | .. automodule:: concoursetools.dockertools 5 | :members: create_script_file 6 | 7 | 8 | Dockerfile 9 | ---------- 10 | For easier dynamic creation of Dockerfiles, the module contains a number of 11 | :class:`~concoursetools.dockertools.Instruction` classes: 12 | 13 | .. autoclass:: concoursetools.dockertools.Instruction 14 | :members: 15 | 16 | 17 | A :class:`~concoursetools.dockertools.Comment` class is also included: 18 | 19 | .. autoclass:: concoursetools.dockertools.Comment 20 | :members: 21 | 22 | 23 | All instructions and comments are added to a A :class:`~concoursetools.dockertools.Dockerfile` instance 24 | in *instruction groups*: 25 | 26 | .. autoclass:: concoursetools.dockertools.Dockerfile 27 | :members: 28 | 29 | 30 | Dockerfile Instructions 31 | ----------------------- 32 | 33 | All currently relevant instructions have been implemented: 34 | 35 | .. list-table:: 36 | :header-rows: 1 37 | :align: left 38 | 39 | * - Instruction 40 | - Description 41 | - Classes 42 | * - ``ADD`` 43 | - Add local or remote files and directories. 44 | - 45 | * - ``ARG`` 46 | - Use build-time variables. 47 | - 48 | * - ``CMD`` 49 | - Specify default commands. 50 | - 51 | * - ``COPY`` 52 | - Copy files and directories. 53 | - :class:`~concoursetools.dockertools.CopyInstruction` 54 | * - ``ENTRYPOINT`` 55 | - Specify default executable. 56 | - :class:`~concoursetools.dockertools.EntryPointInstruction` 57 | * - ``ENV`` 58 | - Set environment variables. 59 | - :class:`~concoursetools.dockertools.EnvInstruction` 60 | * - ``EXPOSE`` 61 | - Describe which ports your application is listening on. 62 | - 63 | * - ``FROM`` 64 | - Check a container's health on startup. 65 | - :class:`~concoursetools.dockertools.FromInstruction` 66 | * - ``HEALTHCHECK`` 67 | - Check a container's health on startup. 68 | - 69 | * - ``LABEL`` 70 | - Add metadata to an image. 71 | - 72 | * - ``MAINTAINER`` 73 | - Specify the author of an image. 74 | - 75 | * - ``ONBUILD`` 76 | - Specify instructions for when the image is used in a build. 77 | - 78 | * - ``RUN`` 79 | - Execute build commands. 80 | - :class:`~concoursetools.dockertools.RunInstruction`, :class:`~concoursetools.dockertools.MultiLineRunInstruction` 81 | * - ``SHELL`` 82 | - Set the default shell of an image. 83 | - 84 | * - ``STOPSIGNAL`` 85 | - Specify the system call signal for exiting a container. 86 | - 87 | * - ``USER`` 88 | - Set user and group ID. 89 | - 90 | * - ``VOLUME`` 91 | - Create volume mounts. 92 | - 93 | * - ``WORKDIR`` 94 | - Change working directory. 95 | - :class:`~concoursetools.dockertools.WorkDirInstruction` 96 | 97 | 98 | .. autoclass:: concoursetools.dockertools.CopyInstruction 99 | .. autoclass:: concoursetools.dockertools.EntryPointInstruction 100 | .. autoclass:: concoursetools.dockertools.EnvInstruction 101 | .. autoclass:: concoursetools.dockertools.FromInstruction 102 | .. autoclass:: concoursetools.dockertools.RunInstruction 103 | .. autoclass:: concoursetools.dockertools.MultiLineRunInstruction 104 | .. autoclass:: concoursetools.dockertools.WorkDirInstruction 105 | 106 | 107 | Dockerfile Mounts 108 | ----------------- 109 | 110 | The module also implements some mounts for the ``RUN`` step to facilitate secrets: 111 | 112 | .. autoclass:: concoursetools.dockertools.Mount 113 | :members: 114 | 115 | All currently relevant mount types have been implemented: 116 | 117 | .. list-table:: 118 | :header-rows: 1 119 | :align: left 120 | 121 | * - Mount 122 | - Description 123 | - Classes 124 | * - ``bind`` 125 | - Bind-mount context directories (read-only). 126 | - 127 | * - ``cache`` 128 | - Mount a temporary directory to cache directories for compilers and package managers. 129 | - 130 | * - ``tmpfs`` 131 | - Mount a ``tmpfs`` in the build container. 132 | - 133 | * - ``secret`` 134 | - Allow the build container to access secure files such as private keys without baking them into the image or build cache. 135 | - :class:`~concoursetools.dockertools.SecretMount` 136 | * - ``ssh`` 137 | - Allow the build container to access SSH keys via SSH agents, with support for passphrases. 138 | - 139 | 140 | .. autoclass:: concoursetools.dockertools.SecretMount 141 | -------------------------------------------------------------------------------- /docs/source/examples/branches.rst: -------------------------------------------------------------------------------- 1 | GitHub Branches 2 | =============== 3 | 4 | .. caution:: 5 | This example is for reference only. It is not extensively tested, and it not intended to be a fully-fledged Concourse resource for production pipelines. Copy and paste at your own risk. 6 | 7 | On occasion, you may wish your resource to emit versions when something has changed, and not have a history of this to rely on. For example, imagine that we want to run a pipeline on a number of different branches of a GitHub repository. Our resource could emit a number of versions like this: 8 | 9 | .. code:: json 10 | 11 | { 12 | "branch": "feature/new-module", 13 | "commit": "abcdef..." 14 | } 15 | 16 | But what happens when a check yields two versions from different branches? We could figure out the "latest" version by sorting by commit date, but if we are tracking multiple branches then we don't want to ignore a commit on one just because a more recent commit has been made on another. Previous in Concourse this was "fixed" by passing ``version: every`` to the resource, but this means that if multiple commits are pushed to a branch at once, then each commit will emit a build, which may not be what we want. 17 | 18 | What we really want is to iterate over all branches in our repository, and spin up a branch-specific pipeline for each one using the :concourse:`set-pipeline-step`. This is easy to do, but we need to make sure that new pipelines are added when new branches appear, and that old pipelines are deleted when their branches are removed. We specifically require a resource which will trigger a pipeline whenever something has changed. The generic resource for this pattern is the :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`, but because the "state" is made up of several "sub versions", it makes sense to instead utilise the :class:`~concoursetools.additional.MultiVersionConcourseResource`, which takes care of much of the boiler plate for us, and also allow us to automatically download these subversions as JSON so that we may iterate over them. 19 | 20 | Branch Version 21 | -------------- 22 | 23 | For this example, each "version" will contain an encoded JSON string with all of the subversions. We only care about whether or not that version (and hence the state) has changed, and not whether these versions were linear. To start, we define the subversion schema: 24 | 25 | .. literalinclude:: ../../../examples/github_branches.py 26 | :pyobject: BranchVersion 27 | 28 | We inherit from :class:`~concoursetools.version.TypedVersion` to allow us to use the :func:`~dataclasses.dataclass` decorator and save us some lines of code. We have to specify ``unsafe_hash=True`` in order to ensure that the version instance will be *hashable*, otherwise we won't be able to use it with our resource. We also need to specify ``order=True`` to allow our subversions to be sorted, as this is required to make sure the same set of subversions yield identical versions. We could have made use of the :class:`~concoursetools.version.SortableVersionMixin`, but given that the :func:`~dataclasses.dataclass` decorator does this for us it isn't necessary here. 29 | 30 | Branch Resource 31 | --------------- 32 | 33 | We start by inheriting from :class:`~concoursetools.additional.MultiVersionConcourseResource`. A standard :class:`~concoursetools.resource.ConcourseResource` will take a version class, but this time we need to pass a *subversion class* instead, as well as a key: 34 | 35 | .. literalinclude:: ../../../examples/github_branches.py 36 | :pyobject: Resource.__init__ 37 | :end-at: super().__init__ 38 | :dedent: 39 | 40 | The resource will then construct a :class:`~concoursetools.version.Version` class which wraps the subversion class, and stores the list of subversions as a JSON-encoded string. The key, ``"branches"``, is used as the key in this version. The final version will look like this: 41 | 42 | .. code:: python3 43 | 44 | {"branches": "[{\"name\": \"issue/95\"}, {\"name\": \"master\"}, {\"name\": \"release/6.7.x\"}, {\"name\": \"version\"}, {\"name\": \"version-lts\"}]"} 45 | 46 | We then store a compiled regex and the API route within the class. There is no need to store each parameter separately, as we can just construct the API route in one line: 47 | 48 | .. literalinclude:: ../../../examples/github_branches.py 49 | :pyobject: Resource.__init__ 50 | :dedent: 51 | 52 | The source of this resource would then look something like this: 53 | 54 | .. code:: yaml 55 | 56 | source: 57 | owner: concourse 58 | repo: github-release-resource 59 | regex: release/.* 60 | 61 | We only need to implement the :meth:`~concoursetools.additional.MultiVersionConcourseResource.fetch_latest_sub_versions` method; the results of this will be collected and converted into a final version by the parent class: 62 | 63 | .. literalinclude:: ../../../examples/github_branches.py 64 | :pyobject: Resource.fetch_latest_sub_versions 65 | :dedent: 66 | 67 | The logic required here is incredibly minimal, and we only need to return each available subversion. The returned set will then be sorted by the resource when converting it to its final version, which is why we needed the subversion to be sortable and hashable. 68 | 69 | When the branches change (either with new ones added or existing ones removed) a new version will be emitted and the pipeline will be triggered. Users can then iterate over them in their pipeline using a combination of the :concourse:`across-step` and the :concourse:`set-pipeline-step`: 70 | 71 | .. code:: yaml 72 | 73 | - get: repo-branches 74 | - load_var: branches 75 | file: branches/branches.json 76 | - across: 77 | - var: branch-info 78 | values: ((.:branches)) 79 | set-pipeline: branch-pipeline 80 | file: ... 81 | vars: 82 | branch: ((.:branch-info.name)) 83 | 84 | GitHub Branches Conclusion 85 | -------------------------- 86 | 87 | The final module only requires :linecount:`../../../examples/github_branches.py` lines (including docstrings) and looks like this: 88 | 89 | .. literalinclude:: ../../../examples/github_branches.py 90 | :linenos: 91 | -------------------------------------------------------------------------------- /docs/source/examples/s3.rst: -------------------------------------------------------------------------------- 1 | S3 Presigned URL 2 | ================ 3 | 4 | .. caution:: 5 | This example is for reference only. It is not extensively tested, and it not intended to be a fully-fledged Concourse resource for production pipelines. Copy and paste at your own risk. 6 | 7 | This example will showcase the :class:`~concoursetools.additional.InOnlyConcourseResource`, and how to build a resource to fetch arbitrary data from an external resource **that is not stored externally**. In this particular example we will consider a resource to generate a new `presigned URL `_ for an object in an S3 bucket. 8 | 9 | Traditionally, when Concourse users which to "run an external function" from a pipeline, they create an :class:`~concoursetools.additional.OutOnlyConcourseResource` to do it. Classic examples include setting a build status (as shown in the :ref:`Bitbucket Build Status` example), or to `send a message on Slack `_. Usually, the new version will only contain a small amount of placeholder information, such as the new build status. However, a :concourse:`put-step` is designed to "fetch" the information for further use in the pipeline, and this is only really possible when the new version represents a "state" which is stored with the external service. For example, the `Git resource `_ will push a new commit, and then emit the version corresponding to that commit so that - when the resource runs its implicit :concourse:`get-step` - the information is not overwritten. 10 | 11 | However, in the case of presigned URLs, AWS does **not** store these anywhere such that they are accessible from the server. When they are created, the user does not get given a UUID which allows them to "look up" the URL at a later date. Therefore, even if the :concourse:`put-step` of the resource created the URL and downloaded it to its resource directory, the :concourse:`get-step` would overwrite it with an empty folder. The two main solutions to this are: 12 | 13 | 1. Pass the URL in the newly created version. 14 | 2. Have the :concourse:`put-step` write the URL to a *different* resource directory. 15 | 16 | Option 2 requires another resource to be already available, as it isn't possible to request additional output directories like a :concourse:`task-step`. This is do-able but fragile, and requires care to make sure that no important files in the other resource are overwritten. Option 1 seems cleaner, but the URL might be sensitive, and storing it as plaintext within the version is definitely not ideal. We could consider encrypting it and having the user pass some sort of key, but this is complicating matters greatly. There is a better way. 17 | 18 | The :class:`~concoursetools.additional.InOnlyConcourseResource` is designed to run these "functions" in the :concourse:`get-step`, and to be *triggered* by a :concourse:`put-step`, like so: 19 | 20 | .. code:: yaml 21 | 22 | - put: s3-presigned-url 23 | get_params: 24 | file_path: my-file 25 | 26 | URL Version 27 | ----------- 28 | 29 | Because the version itself isn't important, the :class:`~concoursetools.additional.InOnlyConcourseResource` actually uses a prebuilt version containing nothing but a timestamp of creation time: 30 | 31 | .. literalinclude:: ../../../concoursetools/additional.py 32 | :pyobject: DatetimeVersion 33 | 34 | URL Resource 35 | ------------ 36 | 37 | We start by inheriting from :class:`~concoursetools.additional.InOnlyConcourseResource`. Again, we don't need to pass a version: 38 | 39 | .. literalinclude:: ../../../examples/s3.py 40 | :pyobject: S3SignedURLConcourseResource.__init__ 41 | :dedent: 42 | 43 | All of the resource functionality comes from overloading the :meth:`~concoursetools.additional.InOnlyConcourseResource.download_data` method: 44 | 45 | .. literalinclude:: ../../../examples/s3.py 46 | :pyobject: S3SignedURLConcourseResource.download_data 47 | :dedent: 48 | 49 | This method takes a required ``file_path`` argument to indicate the object for which the URL should be generated. The ``expires_in`` parameter takes a mapping of arguments for :class:`datetime.timedelta` to allow users to specify expiration time more explicitly. Finally, passing a ``file_name`` instructs the URL to name the downloaded file something specific, rather than the original name from within S3. The URL itself is generated using the `generate_presigned_url `_ function. Finally, we don't return a version, so we only need to concern ourselves with the :ref:`Step Metadata`. 50 | 51 | This resource can then be invoked like so: 52 | 53 | 54 | .. code:: yaml 55 | 56 | - put: s3-presigned-url 57 | get_params: 58 | file_path: folder/file.txt 59 | file_name: file.txt 60 | expires_in: 61 | hours: 24 62 | 63 | Once the implicit :concourse:`get-step` is completed, the URL can then be :concourse:`loaded easily ` and used in prior steps: 64 | 65 | .. code:: yaml 66 | 67 | - load_var: s3-url 68 | file: s3-presigned-url/url 69 | 70 | URL Conclusion 71 | -------------- 72 | 73 | The final resource only requires :linecount:`../../../examples/s3.py` lines of code, and looks like this: 74 | 75 | .. literalinclude:: ../../../examples/s3.py 76 | :linenos: 77 | -------------------------------------------------------------------------------- /docs/source/examples/secrets.rst: -------------------------------------------------------------------------------- 1 | AWS Secrets 2 | =========== 3 | 4 | .. caution:: 5 | This example is for reference only. It is not extensively tested, and it not intended to be a fully-fledged Concourse resource for production pipelines. Copy and paste at your own risk. 6 | 7 | This example will showcase the :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`, and how to build a resource to emit versions whenever something has changed, rather than when a new linear version becomes available. In this example, the resource will watch an secret in `AWS SecretsManager `_, and yield a new version whenever the value of that secret has changed. It will also allow the user to optionally download the secret value, and also to update the secret via a string or a file. The functionality will depend heavily on `boto3 `_ in order to reduce the amount of code needed to function. 8 | 9 | 10 | Secrets Version 11 | --------------- 12 | 13 | Every secret in SecretsManager has a number of `versions `_ representing changes to the secret value. This is all the version needs to contain: 14 | 15 | .. literalinclude:: ../../../examples/secrets.py 16 | :pyobject: SecretVersion 17 | 18 | Here, we only inherit from :class:`~concoursetools.version.TypedVersion` to save some lines of code. 19 | 20 | 21 | Secrets Resource 22 | ---------------- 23 | 24 | We start by inheriting from :class:`~concoursetools.additional.TriggerOnChangeConcourseResource` and passing in the version class. The resource should take an ARN string of the secret (AWS recommends this over the secret name). We also define the AWS SecretsManager client in the ``__init__`` method for ease. To get the AWS region, we need only parse the ARN string to avoid asking for duplicate values from the user: 25 | 26 | .. literalinclude:: ../../../examples/secrets.py 27 | :pyobject: Resource.__init__ 28 | :dedent: 29 | 30 | With this resource type, we don't overload from :meth:`~concoursetools.resource.ConcourseResource.fetch_new_versions`, as it would force us to define the behaviour of the trigger. Instead, we let the resource implement this for us, and overload :meth:`~concoursetools.additional.TriggerOnChangeConcourseResource.fetch_latest_version`. If the version we fetch is the same, then no new versions are emitted. If the version is different, then that new version is sent which which will trigger the pipeline: 31 | 32 | .. literalinclude:: ../../../examples/secrets.py 33 | :pyobject: Resource.fetch_latest_version 34 | :dedent: 35 | 36 | We use `list_secret_version_ids `_ with ``IncludeDeprecated=False`` to ensure that we only get versions which are either current or pending. We then iterate over the versions and find the one marked with ``AWSCURRENT``. If we don't find it then we raise an error to alert the user. 37 | 38 | Next, we overload :meth:`~concoursetools.resource.ConcourseResource.download_version` to allow us to download the metadata (and optionally the value) of the new secret: 39 | 40 | .. literalinclude:: ../../../examples/secrets.py 41 | :pyobject: Resource.download_version 42 | :dedent: 43 | 44 | The behaviour of the resource is as follows: 45 | 46 | 1. The metadata of the secret is fetched from AWS using `describe_secret `_. The response metadata is removed, but could potentially be output as :ref:`Step Metadata`. 47 | 2. The metadata is saved to a file. By default this is named ``metadata.json``, but the user can customise this with the parameters of the :concourse:`get-step`. 48 | 3. If the user has requested the secret value also (which is **not** the default behaviour), then this is fetched using `get_secret_value `_ to be saved to a file, which defaults to ``value`` but is again customisable by the user. 49 | 4. If the response contains a string, then this is written directly to the file using :meth:`~pathlib.Path.write_text`, but if it contains a binary then it is instead written using :meth:`~pathlib.Path.write_bytes`. 50 | 51 | The :mod:`json` module is imported as ``json_package`` to avoid a name collision with a future argument. The metadata of the secret contains some :class:`~datetime.datetime` objects, and so a custom :class:`~json.JSONEncoder` is required: 52 | 53 | .. literalinclude:: ../../../examples/secrets.py 54 | :pyobject: DatetimeSafeJSONEncoder 55 | 56 | Finally, we overload :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` to allow the user to update the secret. We *could* make this rotate the secret, but for the purposes of this example we will allow the user to specify a new value exactly: 57 | 58 | .. literalinclude:: ../../../examples/secrets.py 59 | :pyobject: Resource.publish_new_version 60 | :dedent: 61 | 62 | The behaviour is as follows: 63 | 64 | 1. If the user has specified the secret value as JSON, then encode that as a string as pass it forward. 65 | 2. If the secret is specified as a string, then attempt to set the secret value with `put_secret_value `_. 66 | 3. If the secret is specified as a file path, then use :meth:`~pathlib.Path.read_bytes` to get the contents of the file (more resilient than reading the text) and set the ``SecretBinary`` instead. 67 | 4. If none of these have been set, then raise an error. 68 | 5. Pull out the new ID from the response to establish the new version to return, and also pass the ``VersionStages`` as metadata to be output to the console. 69 | 70 | 71 | AWS Secrets Conclusion 72 | ---------------------- 73 | 74 | We have added a lot of functionality for this resource in only :linecount:`../../../examples/secrets.py` lines of code. The final module looks like this: 75 | 76 | .. literalinclude:: ../../../examples/secrets.py 77 | :linenos: 78 | -------------------------------------------------------------------------------- /docs/source/examples/xkcd.rst: -------------------------------------------------------------------------------- 1 | xkcd Comics 2 | =========== 3 | 4 | .. caution:: 5 | This example is for reference only. It is not extensively tested, and it not intended to be a fully-fledged Concourse resource for production pipelines. Copy and paste at your own risk. 6 | 7 | This example will showcase the :class:`~concoursetools.additional.SelfOrganisingConcourseResource`, and how to use it to save lines of code and avoid implementing logic to determine which versions are newer than the last. In this particular example we will build a resource type to trigger on new :xkcd:`xkcd comics <>`. 8 | 9 | .. xkcd:: 1319 10 | 11 | 12 | xkcd Version 13 | ------------ 14 | 15 | Each xkcd comic is associated with an integer, and so these form a natural choice for the version. We'll inherit from :class:`~concoursetools.version.TypedVersion` to minimise the code we need, and make sure our versions are hashable with ``unsafe_hash=True`` as is required by the resource. Finally, we'll also implement the :func:`~operator.__lt__` method to make sure our comparisons use the comic ID as an orderable integer. 16 | 17 | .. literalinclude:: ../../../examples/xkcd.py 18 | :pyobject: ComicVersion 19 | 20 | 21 | xkcd Resource 22 | ------------- 23 | 24 | We start by inheriting from the :class:`~concoursetools.additional.SelfOrganisingConcourseResource`. We pass in our version, and also include an optional configuration parameter for the xkcd link. Although this is unlikely to matter to 99% of users, it is good practice to allow this ot be configured in case a user is hosting their own version of the comics (or if the URL changes), and it will avoid them needing to reimplement your work. 25 | 26 | .. literalinclude:: ../../../examples/xkcd.py 27 | :pyobject: XKCDResource.__init__ 28 | :dedent: 29 | 30 | Next we need to overload :meth:`~concoursetools.additional.SelfOrganisingConcourseResource.fetch_all_versions`. Given that our versions are comparable, the resource will only return versions considered "greater than" the previous versions (if it exists, normal rules apply if it hasn't been passed), and will even order them for us, removing our need to worry about that ourselves. 31 | 32 | Although we could technically scrape the xkcd website to check for new versions, it is much more polite to make use of the available :wikipedia:`RSS` feed or, in this case, the :wikipedia:`Atom ` feed, which can be found :xkcd:`here `. Although there are existing libraries designed to parse Atom feeds, it doesn't take too much effort to pull out the comic IDs, which is all we really need for this to work: 33 | 34 | .. literalinclude:: ../../../examples/xkcd.py 35 | :pyobject: yield_comic_links 36 | 37 | .. literalinclude:: ../../../examples/xkcd.py 38 | :pyobject: yield_comic_ids 39 | 40 | When given the contents of the feed as a string, the ``yield_comic_ids`` will give us all of the integers corresponding to the newer comics. All we need to do is fetch the data from the source, and return our list of versions: 41 | 42 | .. literalinclude:: ../../../examples/xkcd.py 43 | :pyobject: XKCDResource.fetch_all_versions 44 | :dedent: 45 | 46 | The next step is to implement actually "getting" the version. Technically it is possible that between checking the versions and fetching one, it is no longer available in the feed. However, :wikipedia:`Randall ` has been kind enough to implement a :xkcd:`JSON API `, and so we can just use that. 47 | 48 | .. note:: 49 | Since this API yields a link to the current comic, it would be possible to implement this example as a :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`, but we've :ref:`already got an example for that one `. 50 | 51 | We'll fetch this information and pull out some of the useful information such as the title and upload date. We'll also generate the URL, and set this aside as :ref:`Step Metadata`: 52 | 53 | .. literalinclude:: ../../../examples/xkcd.py 54 | :pyobject: XKCDResource.download_version 55 | :dedent: 56 | :end-before: info_path = 57 | 58 | Next we write the metadata to a file, in case the user wishes to use it themselves: 59 | 60 | .. literalinclude:: ../../../examples/xkcd.py 61 | :pyobject: XKCDResource.download_version 62 | :dedent: 63 | :start-at: info_path = 64 | :end-before: if image 65 | 66 | We optionally (but defaulting to :data:`True`) download the image: 67 | 68 | .. literalinclude:: ../../../examples/xkcd.py 69 | :pyobject: XKCDResource.download_version 70 | :dedent: 71 | :start-at: if image 72 | :end-before: if link 73 | 74 | And finally we do something similar with the comic link, and the alt text: 75 | 76 | .. literalinclude:: ../../../examples/xkcd.py 77 | :pyobject: XKCDResource.download_version 78 | :dedent: 79 | :start-at: if link 80 | 81 | We don't intend for the resource to publish new comics (unless :wikipedia:`Randall ` displays some interest), but the :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` method is an :func:`~abc.abstractmethod`, and so we need to explicitly overload it to ensure that the resource can be used at all. We do this by explicitly raising a :class:`NotImplementedError`: 82 | 83 | .. literalinclude:: ../../../examples/xkcd.py 84 | :pyobject: XKCDResource.publish_new_version 85 | :dedent: 86 | 87 | xkcd Conclusion 88 | --------------- 89 | 90 | The final resource only requires :linecount:`../../../examples/xkcd.py` lines of code, and looks like this: 91 | 92 | .. literalinclude:: ../../../examples/xkcd.py 93 | :linenos: 94 | -------------------------------------------------------------------------------- /docs/source/extensions/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Small extensions to make documenting easier for Concourse Tools. 4 | """ 5 | -------------------------------------------------------------------------------- /docs/source/extensions/cli.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Sphinx extension for documenting the custom CLI. 4 | """ 5 | from __future__ import annotations 6 | 7 | from typing import Literal 8 | 9 | from docutils import nodes 10 | from docutils.parsers.rst import directives 11 | from sphinx.application import Sphinx 12 | from sphinx.ext.autodoc import import_object 13 | from sphinx.util.docutils import SphinxDirective 14 | 15 | from concoursetools.cli.parser import CLI 16 | 17 | 18 | def _align_directive(argument: str) -> str: 19 | return directives.choice(argument, ("left", "center", "right")) 20 | 21 | 22 | class CLIDirective(SphinxDirective): 23 | """ 24 | Directive for listing CLI commands in a table. 25 | """ 26 | required_arguments = 1 # The import path of the CLI 27 | option_spec = { 28 | "align": _align_directive, 29 | } 30 | 31 | def run(self) -> list[nodes.Node]: 32 | """ 33 | Process the content of the shield directive. 34 | """ 35 | align: Literal["left", "center", "right"] | None = self.options.get("align") 36 | 37 | headers = [nodes.entry("", nodes.paragraph("", header)) for header in ["Command", "Description"]] 38 | 39 | import_string, = self.arguments 40 | cli = self.import_cli(import_string) 41 | 42 | rows: list[list[nodes.entry]] = [] 43 | 44 | for command_name, command in cli.commands.items(): 45 | rows.append([ 46 | nodes.entry("", nodes.paragraph("", "", nodes.reference("", "", nodes.literal("", command_name), refid=f"cli.{command_name}"))), 47 | nodes.entry("", nodes.paragraph("", command.description or "")), 48 | ]) 49 | 50 | table = self.create_table(headers, rows, align=align) 51 | 52 | nodes_to_return: list[nodes.Node] = [table] 53 | 54 | for command_name, command in cli.commands.items(): 55 | command_section = nodes.section(ids=[f"cli.{command_name}"]) 56 | title = nodes.title(f"cli.{command_name}", "", nodes.literal("", command_name)) 57 | command_section.append(title) 58 | 59 | if command.description is not None: 60 | command_section.append(nodes.paragraph("", command.description)) 61 | 62 | usage_block = nodes.literal_block("", f"$ {command.usage_string()}") 63 | command_section.append(usage_block) 64 | 65 | for positional in command.positional_arguments: 66 | alias_paragraph = nodes.paragraph("", "", nodes.literal("", positional.name)) 67 | description_paragraph = nodes.paragraph("", positional.description or "") 68 | description_paragraph.set_class("cli-option-description") 69 | command_section.extend([alias_paragraph, description_paragraph]) 70 | 71 | for option in command.options: 72 | alias_nodes: list[nodes.Node] = [] 73 | for alias in option.aliases: 74 | alias_nodes.append(nodes.literal("", alias)) 75 | alias_nodes.append(nodes.Text(", ")) 76 | alias_paragraph = nodes.paragraph("", "", *alias_nodes[:-1]) 77 | description_paragraph = nodes.paragraph("", option.description or "") 78 | description_paragraph.set_class("cli-option-description") 79 | command_section.extend([alias_paragraph, description_paragraph]) 80 | 81 | nodes_to_return.append(command_section) 82 | 83 | return nodes_to_return 84 | 85 | def create_table(self, headers: list[nodes.entry], rows: list[list[nodes.entry]], 86 | align: Literal["left", "center", "right"] | None = None) -> nodes.table: 87 | table = nodes.table() 88 | if align is not None: 89 | table["align"] = align 90 | 91 | table_group = nodes.tgroup(cols=len(headers)) 92 | table_group.extend([nodes.colspec()] * len(headers)) 93 | 94 | table.append(table_group) 95 | 96 | header = nodes.thead() 97 | header_row = nodes.row() 98 | header_row.extend(headers) 99 | header.append(header_row) 100 | table_group.append(header) 101 | 102 | body = nodes.tbody() 103 | for row in rows: 104 | body_row = nodes.row() 105 | body_row.extend(row) 106 | body.append(body_row) 107 | 108 | table_group.append(body) 109 | return table 110 | 111 | def import_cli(self, import_string: str) -> CLI: 112 | *module_components, import_object_name = import_string.split(".") 113 | import_path = ".".join(module_components) 114 | 115 | import_result = import_object(import_path, [import_object_name]) 116 | cli: CLI = import_result[-1] 117 | return cli 118 | 119 | 120 | def setup(app: Sphinx) -> dict[str, object]: 121 | """ 122 | Attach the extension to the application. 123 | 124 | :param app: The Sphinx application. 125 | """ 126 | app.add_directive("cli", CLIDirective) 127 | return {"parallel_read_safe": True} 128 | -------------------------------------------------------------------------------- /docs/source/extensions/concourse.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Minor Sphinx extension for creating Concourse documentation links, based on https://sphinx-toolbox.readthedocs.io/en/stable/extensions/wikipedia.html. 4 | 5 | :concourse:`jobs` will create a link to /jobs.html with the text "jobs". 6 | :concourse:`jobs.thing.thing2` will create a link to /jobs.html#thing1.thing2 with the text "thing2". 7 | :concourse:`jobs-a-b-c` will create a link to /jobs-a-b-c.html with the text "jobs a b c". 8 | :concourse:`job ` will create a link to /jobs.html with the text "job". 9 | :concourse:`job ` will create a link to /jobs.html#thing1.thing2 with the text "job". 10 | 11 | Set ``concourse_base_url`` in ``conf.py`` to change the URL used. It must contain "{target}" to be populated. 12 | """ 13 | from __future__ import annotations 14 | 15 | from urllib.parse import quote 16 | 17 | from docutils import nodes 18 | from docutils.nodes import system_message 19 | from docutils.parsers.rst.states import Inliner 20 | from sphinx.application import Sphinx 21 | from sphinx.util.nodes import split_explicit_title 22 | 23 | __all__ = ("make_concourse_link", "setup") 24 | 25 | DEFAULT_BASE_URL = "https://concourse-ci.org/{target}" 26 | 27 | 28 | def make_concourse_link(name: str, rawtext: str, text: str, lineno: int, inliner: Inliner, options: dict[str, object] = {}, 29 | content: list[str] = []) -> tuple[list[nodes.reference], list[system_message]]: 30 | """ 31 | Add a link to the given article on Concourse CI. 32 | 33 | :param name: The local name of the interpreted role, the role name actually used in the document. 34 | :param rawtext: A string containing the entire interpreted text input, including the role and markup. 35 | :param text: The interpreted text content. 36 | :param lineno: The line number where the interpreted text begins. 37 | :param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.source_role`. 38 | It contains the several attributes useful for error reporting and document tree access. 39 | :param options: A dictionary of directive options for customization (from the ``role`` directive), 40 | to be interpreted by the function. Used for additional attributes for the generated elements 41 | and other functionality. 42 | :param content: A list of strings, the directive content for customization (from the ``role`` directive). 43 | To be interpreted by the function. 44 | 45 | :return: A list containing the created node, and a list containing any messages generated during the function. 46 | """ 47 | text = nodes.unescape(text) 48 | _, title, target = split_explicit_title(text) 49 | 50 | title = title.split(".")[-1].replace("-", " ") 51 | 52 | page, *anchors = quote(target.replace(" ", "_"), safe="").split(".", maxsplit=1) 53 | if anchors: 54 | anchor = ".".join(anchors) 55 | new_target = f"{page}.html#{anchor}" 56 | else: 57 | new_target = f"{page}.html" 58 | 59 | base_url: str = inliner.document.settings.env.config.concourse_base_url 60 | ref = base_url.format(target=new_target) 61 | 62 | node = nodes.reference(rawtext, title, refuri=str(ref), **options) 63 | return [node], [] 64 | 65 | 66 | def setup(app: Sphinx) -> dict[str, object]: 67 | """ 68 | Attach the extension to the application. 69 | 70 | :param app: The Sphinx application. 71 | """ 72 | app.add_role("concourse", make_concourse_link) 73 | app.add_config_value("concourse_base_url", DEFAULT_BASE_URL, "env", [str]) 74 | 75 | return {"parallel_read_safe": True} 76 | -------------------------------------------------------------------------------- /docs/source/extensions/linecount.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Minor Sphinx extension for counting the number of lines in a file. 4 | 5 | :linecount:`../relative/path/to/file.txt` will be replaced with the number of lines in `file.txt`. 6 | 7 | Paths should be relative to the directory of the current file, not the source directory. 8 | This is the same path passed to literalinclude. 9 | """ 10 | from __future__ import annotations 11 | 12 | from pathlib import Path 13 | 14 | from docutils import nodes 15 | from docutils.nodes import system_message 16 | from docutils.parsers.rst.states import Inliner 17 | from sphinx.application import Sphinx 18 | 19 | __all__ = ("count_lines", "setup") 20 | 21 | 22 | def count_lines(name: str, rawtext: str, text: str, lineno: int, inliner: Inliner, options: dict[str, object] = {}, 23 | content: list[str] = []) -> tuple[list[nodes.Node], list[system_message]]: 24 | """ 25 | Add a text node containing the number of lines. 26 | 27 | :param name: The local name of the interpreted role, the role name actually used in the document. 28 | :param rawtext: A string containing the entire interpreted text input, including the role and markup. 29 | :param text: The interpreted text content. 30 | :param lineno: The line number where the interpreted text begins. 31 | :param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.source_role`. 32 | It contains the several attributes useful for error reporting and document tree access. 33 | :param options: A dictionary of directive options for customization (from the ``role`` directive), 34 | to be interpreted by the function. Used for additional attributes for the generated elements 35 | and other functionality. 36 | :param content: A list of strings, the directive content for customization (from the ``role`` directive). 37 | To be interpreted by the function. 38 | 39 | :return: A list containing the created node, and a list containing any messages generated during the function. 40 | """ 41 | path = Path(nodes.unescape(text)) 42 | page_path = Path(Path(inliner.document.settings._source)) 43 | resolved_path = (page_path.parent / path).resolve() 44 | with open(resolved_path) as rf: 45 | line_count = sum(1 for _ in rf) 46 | 47 | node = nodes.Text(str(line_count)) 48 | return [node], [] 49 | 50 | 51 | def setup(app: Sphinx) -> dict[str, object]: 52 | """ 53 | Attach the extension to the application. 54 | 55 | :param app: The Sphinx application. 56 | """ 57 | app.add_role("linecount", count_lines) 58 | 59 | return {"parallel_read_safe": True} 60 | -------------------------------------------------------------------------------- /docs/source/extensions/wikipedia.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Minor Sphinx extension for creating Wikipedia links, based on https://sphinx-toolbox.readthedocs.io/en/stable/extensions/wikipedia.html. 4 | 5 | :wikipedia:`Cats` will create a link to /wiki/Cats with the text "Cats". 6 | :wikipedia:`cat ` will create a link to /wiki/Cats with the text "cat". 7 | 8 | Set ``wikipedia_base_url`` in ``conf.py`` to change the URL used. It must contain "{target}" to be populated. 9 | It can also contain "{lang}" to allow the language code (e.g. "de") to be injected with the ``wikipedia_lang`` variable. 10 | """ 11 | from __future__ import annotations 12 | 13 | import re 14 | from urllib.parse import quote 15 | 16 | from docutils import nodes 17 | from docutils.nodes import system_message 18 | from docutils.parsers.rst.states import Inliner 19 | from sphinx.application import Sphinx 20 | from sphinx.util.nodes import split_explicit_title 21 | 22 | __all__ = ("make_wikipedia_link", "setup") 23 | 24 | DEFAULT_BASE_URL = "https://{lang}.wikipedia.org/wiki/{target}" 25 | RE_WIKI_LANG = re.compile(":(.*?):(.*)") 26 | 27 | 28 | def make_wikipedia_link(name: str, rawtext: str, text: str, lineno: int, inliner: Inliner, options: dict[str, object] = {}, 29 | content: list[str] = []) -> tuple[list[nodes.reference], list[system_message]]: 30 | """ 31 | Add a link to the given article on :wikipedia:`Wikipedia`. 32 | 33 | :param name: The local name of the interpreted role, the role name actually used in the document. 34 | :param rawtext: A string containing the entire interpreted text input, including the role and markup. 35 | :param text: The interpreted text content. 36 | :param lineno: The line number where the interpreted text begins. 37 | :param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called ``source_role``. 38 | It contains the several attributes useful for error reporting and document tree access. 39 | :param options: A dictionary of directive options for customization (from the ``role`` directive), 40 | to be interpreted by the function. Used for additional attributes for the generated elements 41 | and other functionality. 42 | :param content: A list of strings, the directive content for customization (from the ``role`` directive). 43 | To be interpreted by the function. 44 | 45 | :return: A list containing the created node, and a list containing any messages generated during the function. 46 | """ 47 | text = nodes.unescape(text) 48 | has_explicit, title, target = split_explicit_title(text) 49 | 50 | if (match := RE_WIKI_LANG.match(target)): 51 | lang, target = match.groups() 52 | if not has_explicit: 53 | title = target 54 | else: # default language 55 | lang = inliner.document.settings.env.config.wikipedia_lang 56 | 57 | base_url: str = inliner.document.settings.env.config.wikipedia_base_url 58 | ref = base_url.format(lang=lang, target=quote(target.replace(" ", "_"), safe="#")) 59 | 60 | node = nodes.reference(rawtext, title, refuri=str(ref), **options) 61 | return [node], [] 62 | 63 | 64 | def setup(app: Sphinx) -> dict[str, object]: 65 | """ 66 | Attach the extension to the application. 67 | 68 | :param app: The Sphinx application. 69 | """ 70 | app.add_role("wikipedia", make_wikipedia_link) 71 | app.add_config_value("wikipedia_base_url", DEFAULT_BASE_URL, "env", [str]) 72 | app.add_config_value("wikipedia_lang", "en", "env", [str]) 73 | 74 | return {"parallel_read_safe": True} 75 | -------------------------------------------------------------------------------- /docs/source/extensions/xkcd.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Minor Sphinx extension for embedding xkcd comics, based on https://sphinx-toolbox.readthedocs.io/en/stable/extensions/wikipedia.html. 4 | 5 | You can embed a particular comic with the following directive: 6 | 7 | .. xkcd:: 8 | 9 | In addition: 10 | 11 | :wikipedia:`path/to/page.html` will create a link to that page on the xkcd website. 12 | 13 | Set ``xkcd_endpoint`` in ``conf.py`` to change the URL used. 14 | """ 15 | from __future__ import annotations 16 | 17 | from typing import Any 18 | 19 | from docutils import nodes 20 | from docutils.parsers.rst import directives 21 | from docutils.parsers.rst.roles import set_classes 22 | from docutils.parsers.rst.states import Inliner 23 | import requests 24 | from sphinx.application import Sphinx 25 | from sphinx.environment import BuildEnvironment 26 | from sphinx.util import logging 27 | from sphinx.util.docutils import SphinxDirective 28 | from sphinx.util.nodes import split_explicit_title 29 | 30 | DEFAULT_XKCD = "https://xkcd.com" 31 | 32 | 33 | class XkcdDirective(SphinxDirective): 34 | """ 35 | Directive for xkcd comics. 36 | """ 37 | required_arguments = 1 # The comic number 38 | option_spec = { 39 | "height": directives.length_or_unitless, 40 | "width": directives.length_or_percentage_or_unitless, 41 | "scale": directives.percentage, 42 | "class": directives.class_option, 43 | "caption": directives.unchanged, 44 | } 45 | 46 | @property 47 | def config(self) -> BuildEnvironment: 48 | """Return the config environment.""" 49 | return self.state.document.settings.env.config 50 | 51 | def run(self) -> list[nodes.Node]: 52 | """ 53 | Process the content of the shield directive. 54 | """ 55 | comic_number, = self.arguments 56 | comic_info = self.get_comic_info(int(comic_number), self.config["xkcd_endpoint"]) 57 | 58 | caption = self.options.pop("caption", "Relevant xkcd") 59 | 60 | set_classes(self.options) 61 | image_node = nodes.image(self.block_text, uri=directives.uri(comic_info["img"]), 62 | alt=comic_info["alt"], **self.options) 63 | self.add_name(image_node) 64 | 65 | reference_node = nodes.reference("", "", refuri=comic_info["link"]) 66 | reference_node += image_node 67 | 68 | caption_node = nodes.caption("", caption) 69 | 70 | figure_node = nodes.figure() 71 | figure_node += reference_node 72 | figure_node += caption_node 73 | return [figure_node] 74 | 75 | def get_comic_info(self, comic_number: int, endpoint: str = DEFAULT_XKCD) -> dict[str, Any]: 76 | comic_link = f"{endpoint}/{comic_number}/" 77 | try: 78 | response = requests.get(f"{endpoint}/{comic_number}/info.0.json") 79 | response.raise_for_status() 80 | except requests.ConnectionError: 81 | logger = logging.getLogger(__name__) 82 | logger.warning("Could not connect to xkcd endpoint") 83 | response_json: dict[str, object] = { 84 | "img": comic_link, 85 | "alt": comic_link, 86 | } 87 | except requests.HTTPError: 88 | latest_response = requests.get(f"{endpoint}/info.0.json") 89 | latest_json: dict[str, Any] = latest_response.json() 90 | most_recent_comic: int = latest_json["num"] 91 | if most_recent_comic < comic_number: 92 | raise ValueError(f"You asked for xkcd #{comic_number}, but the most recent available comic is #{most_recent_comic}") 93 | else: 94 | raise 95 | else: 96 | response_json = response.json() 97 | response_json["link"] = comic_link 98 | return response_json 99 | 100 | 101 | def make_xkcd_link(name: str, rawtext: str, text: str, lineno: int, inliner: Inliner, 102 | options: dict[str, object] = {}, content: list[str] = []) -> tuple[list[nodes.reference], list[nodes.system_message]]: 103 | """ 104 | Add a link to a page on the xkcd website. 105 | 106 | :param name: The local name of the interpreted role, the role name actually used in the document. 107 | :param rawtext: A string containing the entire interpreted text input, including the role and markup. 108 | :param text: The interpreted text content. 109 | :param lineno: The line number where the interpreted text begins. 110 | :param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.source_role`. 111 | It contains the several attributes useful for error reporting and document tree access. 112 | :param options: A dictionary of directive options for customization (from the ``role`` directive), 113 | to be interpreted by the function. Used for additional attributes for the generated elements 114 | and other functionality. 115 | :param content: A list of strings, the directive content for customization (from the ``role`` directive). 116 | To be interpreted by the function. 117 | 118 | :return: A list containing the created node, and a list containing any messages generated during the function. 119 | """ 120 | text = nodes.unescape(text) 121 | _, title, target = split_explicit_title(text) 122 | 123 | endpoint: str = inliner.document.settings.env.config.xkcd_endpoint 124 | ref = f"{endpoint}/{target}" 125 | 126 | node = nodes.reference(rawtext, title, refuri=str(ref), **options) 127 | return [node], [] 128 | 129 | 130 | def setup(app: Sphinx) -> dict[str, object]: 131 | """ 132 | Attach the extension to the application. 133 | 134 | :param app: The Sphinx application. 135 | """ 136 | app.add_directive("xkcd", XkcdDirective) 137 | app.add_role("xkcd", make_xkcd_link) 138 | app.add_config_value("xkcd_endpoint", DEFAULT_XKCD, "env", [str]) 139 | return {"parallel_read_safe": True} 140 | -------------------------------------------------------------------------------- /docs/source/importing.rst: -------------------------------------------------------------------------------- 1 | Dynamic Importing 2 | ================= 3 | 4 | .. automodule:: concoursetools.importing 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Concourse Tools (|version|) 2 | =========================== 3 | 4 | .. admonition:: Description 5 | 6 | A Python package for easily implementing Concourse :concourse:`resource types `, created by GCHQ. 7 | 8 | 9 | 10 | Bugs and Contributions 11 | ---------------------- 12 | Contributions, fixes, suggestions and bug reports are all welcome: please see the `guidance `_. 13 | 14 | 15 | Examples 16 | -------- 17 | 18 | Examples are available which showcase the variety of resources made possible with Concourse Tools: 19 | 20 | .. list-table:: 21 | :header-rows: 1 22 | :align: left 23 | 24 | * - Resource 25 | - Examples 26 | * - :class:`~concoursetools.resource.ConcourseResource` 27 | - :ref:`AWS SageMaker Pipeline` 28 | * - :class:`~concoursetools.additional.OutOnlyConcourseResource` 29 | - :ref:`Bitbucket Build Status` 30 | * - :class:`~concoursetools.additional.SelfOrganisingConcourseResource` 31 | - :ref:`XKCD Comics` 32 | * - :class:`~concoursetools.additional.TriggerOnChangeConcourseResource` 33 | - :ref:`AWS Secrets` 34 | * - :class:`~concoursetools.additional.InOnlyConcourseResource` 35 | - :ref:`S3 Presigned URL` 36 | * - :class:`~concoursetools.additional.MultiVersionConcourseResource` 37 | - :ref:`GitHub Branches` 38 | 39 | Contents 40 | -------- 41 | 42 | .. toctree:: 43 | :maxdepth: 2 44 | 45 | quickstart 46 | api_reference 47 | cli_reference 48 | debugging 49 | testing 50 | deployment 51 | internals 52 | changelog 53 | 54 | .. toctree:: 55 | :caption: Examples 56 | :hidden: 57 | 58 | examples/pipeline 59 | examples/build_status 60 | examples/xkcd 61 | examples/secrets 62 | examples/s3 63 | examples/branches 64 | 65 | Indices and Tables 66 | ------------------ 67 | 68 | * :ref:`genindex` 69 | * :ref:`modindex` 70 | * :ref:`search` 71 | -------------------------------------------------------------------------------- /docs/source/internals.rst: -------------------------------------------------------------------------------- 1 | Internals 2 | ========= 3 | 4 | Concourse Tools contains a number of internal utility functions which may be useful for developers. 5 | 6 | .. toctree:: 7 | parsing 8 | cli 9 | dockertools 10 | importing 11 | main_scripts 12 | typing 13 | -------------------------------------------------------------------------------- /docs/source/main_scripts.rst: -------------------------------------------------------------------------------- 1 | Main Scripts 2 | ============ 3 | When scripts are eventually generated for your resource class, each step corresponds to a function call which encapsulates all logic. All of the following functions accept no arguments, instead extracting parameters directly from the command line, ``stdin``, and environment variables. They will print to ``stdout`` and ``stderr``, and do not return anything. 4 | 5 | Although each method can *technically* be overloaded, this is **not** recommended as all of the resource logic is contained in the standard methods, and this will just add complications. 6 | 7 | .. automethod:: concoursetools.resource.ConcourseResource.check_main 8 | 9 | .. automethod:: concoursetools.resource.ConcourseResource.in_main 10 | 11 | .. automethod:: concoursetools.resource.ConcourseResource.out_main 12 | -------------------------------------------------------------------------------- /docs/source/mocking.rst: -------------------------------------------------------------------------------- 1 | Mocking 2 | ======= 3 | 4 | .. automodule:: concoursetools.mocking 5 | 6 | 7 | Environment Variables 8 | --------------------- 9 | 10 | .. autofunction:: concoursetools.mocking.mock_environ 11 | 12 | .. autofunction:: concoursetools.mocking.create_env_vars 13 | 14 | .. autoclass:: concoursetools.mocking.TestBuildMetadata 15 | 16 | Input / Output 17 | -------------- 18 | 19 | .. autofunction:: concoursetools.mocking.mock_argv 20 | 21 | .. autofunction:: concoursetools.mocking.mock_stdin 22 | 23 | .. autoclass:: concoursetools.mocking.StringIOWrapper 24 | :members: 25 | 26 | 27 | Directory State 28 | --------------- 29 | Often you need to mock certain files when testing your resource, which are usually accessible in the resource folders. Rather than set this up manually, you can pass a directory state to :class:`~concoursetools.mocking.TemporaryDirectoryState` to make this easier. 30 | 31 | .. autoclass:: concoursetools.mocking.TemporaryDirectoryState 32 | :members: 33 | -------------------------------------------------------------------------------- /docs/source/parsing.rst: -------------------------------------------------------------------------------- 1 | Parsing 2 | ======= 3 | 4 | .. automodule:: concoursetools.parsing 5 | 6 | 7 | JSON Parsing 8 | ------------ 9 | 10 | The following functions are responsible for parsing Concourse JSON payloads and returning Python objects. 11 | 12 | .. autofunction:: concoursetools.parsing.parse_check_payload 13 | 14 | .. autofunction:: concoursetools.parsing.parse_in_payload 15 | 16 | .. autofunction:: concoursetools.parsing.parse_out_payload 17 | 18 | .. autofunction:: concoursetools.parsing.parse_metadata 19 | 20 | 21 | JSON Formatting 22 | --------------- 23 | 24 | The following functions are responsible for formatting Concourse JSON payloads from Python objects. 25 | 26 | .. autofunction:: concoursetools.parsing.format_check_output 27 | 28 | .. autofunction:: concoursetools.parsing.format_in_out_output 29 | 30 | .. autofunction:: concoursetools.parsing.format_metadata 31 | -------------------------------------------------------------------------------- /docs/source/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | 4 | Creating a Concourse resource type with Concourse Tools couldn't be simpler. 5 | To begin, install **Concourse Tools** from source or from PyPI (see the footer for links). 6 | 7 | .. code:: shell 8 | 9 | $ pip install concoursetools 10 | 11 | Start by familiarising yourself with the Concourse resource "rules" in the 12 | :concourse:`documentation `. To recreate that example, start by creating a new 13 | ``concourse.py`` file in your repository. The first step is to create a :class:`~concoursetools.version.Version` subclass: 14 | 15 | 16 | .. code:: python 17 | 18 | from dataclasses import dataclass 19 | from concoursetools import TypedVersion 20 | 21 | 22 | @dataclass() 23 | class GitVersion(TypedVersion): 24 | ref: str 25 | 26 | 27 | Next, create a subclass of :class:`~concoursetools.resource.ConcourseResource`: 28 | 29 | .. code:: python 30 | 31 | from concoursetools import ConcourseResource 32 | 33 | 34 | class GitResource(ConcourseResource[GitVersion]): 35 | 36 | def __init__(self, uri: str, branch: str, private_key: str) -> None: 37 | super().__init__(GitVersion) 38 | self.uri = uri 39 | self.branch = branch 40 | self.private_key = private_key 41 | 42 | 43 | Here, the parameters in the ``__init__`` method will be taken from the ``source`` configuration for the resource. 44 | Now, implement the three methods required to define the behaviour of the resource: 45 | 46 | 47 | .. code:: python 48 | 49 | from pathlib import Path 50 | from typing import Any 51 | from concoursetools import BuildMetadata 52 | 53 | 54 | class GitResource(ConcourseResource[GitVersion]): 55 | ... 56 | 57 | def fetch_new_versions(self, previous_version: GitVersion | None) -> list[GitVersion]: 58 | ... 59 | 60 | def download_version(self, version: GitVersion, destination_dir: pathlib.Path, 61 | build_metadata: BuildMetadata, **kwargs: Any) -> tuple[GitVersion, dict[str, str]]: 62 | ... 63 | 64 | def publish_new_version(self, sources_dir: pathlib.Path, build_metadata: BuildMetadata, 65 | **kwargs: Any) -> tuple[GitVersion, dict[str, str]]: 66 | ... 67 | 68 | 69 | The keyword arguments in :meth:`~concoursetools.resource.ConcourseResource.download_version` 70 | and :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` correspond to ``params`` in the ``get`` step, 71 | and ``get_params`` in the ``put`` step respectively. 72 | 73 | Once you are happy with the resource, freeze your requirements into a ``requirements.txt`` file, 74 | then generate the ``Dockerfile`` using the Concourse Tools CLI: 75 | 76 | .. code:: shell 77 | 78 | $ python3 -m concoursetools dockerfile . 79 | 80 | 81 | Finally, upload the Docker image to a registry, and use it in your pipelines! 82 | 83 | 84 | .. tip:: 85 | Check out the :ref:`Examples` section for different ways to leverage Concourse Tools for your use case. 86 | -------------------------------------------------------------------------------- /docs/source/resource.rst: -------------------------------------------------------------------------------- 1 | Resource 2 | ======== 3 | 4 | .. automodule:: concoursetools.resource 5 | 6 | 7 | Concourse Resource Base Class 8 | ----------------------------- 9 | 10 | .. autoclass:: concoursetools.resource.ConcourseResource 11 | :members: 12 | :exclude-members: check_main, in_main, out_main 13 | 14 | 15 | Step Metadata 16 | ------------- 17 | 18 | The :meth:`~concoursetools.resource.ConcourseResource.download_version` and :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` methods return "step metadata", which is displayed in the Concourse web UI like so: 19 | 20 | .. figure:: _static/step_metadata.png 21 | :scale: 75% 22 | 23 | Concourse metadata example from the :concourse:`git-trigger-example.git-trigger-example`. The red outline shows the metadata. 24 | 25 | This metadata dictionary should contain string-only key/value pairs. This is enforced by Concourse Tools, which casts both to strings before emitting any JSON. This is not easily overloaded, as the metadata is only ever intended for visual consumption. 26 | 27 | .. warning:: 28 | 29 | It is okay to leave the metadata dictionary blank, but you **must** return an empty :class:`dict` if so. 30 | 31 | .. tip:: 32 | 33 | If you want to return metadata which is *not* a string, then consider "stringifying" it in a readable way: 34 | 35 | .. code:: python3 36 | 37 | response_codes = [200, 200, 200, 500] 38 | metadata = { 39 | "response_codes": ", ".join(response_codes) 40 | } 41 | 42 | .. warning:: 43 | 44 | This metadata is only shown after a step runs successfully, and is omitted if it failed. 45 | -------------------------------------------------------------------------------- /docs/source/testing.rst: -------------------------------------------------------------------------------- 1 | Testing the Resource 2 | ===================== 3 | 4 | .. automodule:: concoursetools.testing 5 | 6 | 7 | Using Test Wrappers 8 | ------------------- 9 | 10 | Multiple test wrappers are available: 11 | 12 | .. list-table:: 13 | :header-rows: 1 14 | :align: left 15 | 16 | * - Test Class 17 | - Input / Output 18 | - Executed As 19 | * - :class:`~concoursetools.testing.SimpleTestResourceWrapper` 20 | - Python 21 | - Resource class 22 | * - :class:`~concoursetools.testing.JSONTestResourceWrapper` 23 | - JSON 24 | - Main scripts 25 | * - :class:`~concoursetools.testing.ConversionTestResourceWrapper` 26 | - Python 27 | - Main scripts 28 | * - :class:`~concoursetools.testing.FileTestResourceWrapper` 29 | - JSON 30 | - External scripts 31 | * - :class:`~concoursetools.testing.FileConversionTestResourceWrapper` 32 | - Python 33 | - External scripts 34 | * - :class:`~concoursetools.testing.DockerTestResourceWrapper` 35 | - JSON 36 | - Docker container 37 | * - :class:`~concoursetools.testing.DockerConversionTestResourceWrapper` 38 | - Python 39 | - Docker container 40 | 41 | Base Wrapper 42 | ------------ 43 | 44 | All of the above inherit from :class:`~concoursetools.testing.TestResourceWrapper`: 45 | 46 | .. autoclass:: concoursetools.testing.TestResourceWrapper 47 | :members: 48 | 49 | .. toctree:: 50 | mocking 51 | wrappers 52 | -------------------------------------------------------------------------------- /docs/source/typing.rst: -------------------------------------------------------------------------------- 1 | Typing 2 | ====== 3 | 4 | .. automodule:: concoursetools.typing 5 | 6 | Concourse Types 7 | --------------- 8 | 9 | The following types map onto Concourse :concourse:`config-basics.basic-schemas` used throughout the library. 10 | 11 | .. autodata:: concoursetools.typing.ResourceConfig 12 | 13 | .. autodata:: concoursetools.typing.Params 14 | 15 | .. autodata:: concoursetools.typing.Metadata 16 | 17 | .. autodata:: concoursetools.typing.MetadataPair 18 | 19 | .. autodata:: concoursetools.typing.VersionConfig 20 | 21 | 22 | Type Vars 23 | --------- 24 | 25 | .. autodata:: concoursetools.typing.VersionT 26 | 27 | .. autodata:: concoursetools.typing.TypedVersionT 28 | 29 | .. autodata:: concoursetools.typing.SortableVersionT 30 | 31 | Protocols 32 | --------- 33 | 34 | .. autoclass:: concoursetools.typing.VersionProtocol 35 | 36 | .. autoclass:: concoursetools.typing.TypedVersionProtocol 37 | 38 | .. autoclass:: concoursetools.typing.SortableVersionProtocol 39 | -------------------------------------------------------------------------------- /docs/source/version.rst: -------------------------------------------------------------------------------- 1 | Version 2 | ======= 3 | 4 | .. automodule:: concoursetools.version 5 | 6 | Version Parent Class 7 | -------------------- 8 | 9 | .. autoclass:: concoursetools.version.Version 10 | :members: 11 | 12 | 13 | Typed Version Class 14 | ------------------- 15 | 16 | .. autoclass:: concoursetools.version.TypedVersion 17 | :members: flatten, un_flatten 18 | 19 | Supported Types 20 | _______________ 21 | 22 | Concourse Tools has out-of-the-box support for a few common types: 23 | 24 | * :class:`~datetime.datetime`: The datetime object is mapped to an integer timestamp. 25 | * :class:`bool`: The boolean is mapped to the strings ``"True"`` or ``"False"``. 26 | * :class:`~enum.Enum`: Enums are mapped to their *names*, so ``MyEnum.ONE`` maps to ``ONE``. 27 | * :class:`~pathlib.Path`: Paths are mapped to their string representations. 28 | 29 | 30 | Version Comparisons 31 | ------------------- 32 | 33 | It is often beneficial to directly compare version within your code. 34 | 35 | Hashing 36 | _______ 37 | 38 | By default, every version is hashable, and this :func:`hash` is determined by the version class and the output of :meth:`~concoursetools.version.Version.to_flat_dict`. Key/value pairs are sorted and then hashed as a :class:`tuple`, which is combined with the hash of the class. 39 | 40 | Equality 41 | ________ 42 | 43 | By default, two versions are equal if they have the same :func:`hash`. However, you may wish to overload this. For example, consider the following version class: 44 | 45 | .. code:: python3 46 | 47 | class GitBranch(Version): 48 | 49 | def __init__(self, branch_name: str): 50 | self.branch_name = branch_name 51 | 52 | Under the default behaviour, ``GitBranch("main")`` and ``GitBranch("origin/main")`` will be not be considered equal, but this might not be ideal. The fastest way to fix this is to overload :meth:`~object.__eq__` like so: 53 | 54 | .. code:: python3 55 | 56 | class GitBranch(Version): 57 | 58 | def __init__(self, branch_name: str): 59 | self.branch_name = branch_name 60 | 61 | def __eq__(self, other): 62 | return self.branch_name.split("/")[-1] == other.branch_name.split("/")[-1] 63 | 64 | 65 | Ordering 66 | ________ 67 | 68 | Sometimes you need to order versions to simplify your scripts (returning the latest version, etc.), but by default versions are **not** comparable in this way. This can be fixed by also inheriting from :class:`SortableVersionMixin`, which expects you to implement :meth:`~object.__lt__`. We want ``version_a <= version_b`` if and only if ``version_a`` is **no older** than ``version_b``. 69 | 70 | .. autoclass:: concoursetools.version.SortableVersionMixin 71 | :members: 72 | 73 | 74 | Multi Versions 75 | -------------- 76 | 77 | .. autoclass:: concoursetools.additional.MultiVersion 78 | :members: 79 | -------------------------------------------------------------------------------- /docs/source/wrappers.rst: -------------------------------------------------------------------------------- 1 | Test Wrappers 2 | ============= 3 | 4 | .. autoclass:: concoursetools.testing.SimpleTestResourceWrapper 5 | :members: 6 | 7 | .. autoclass:: concoursetools.testing.JSONTestResourceWrapper 8 | :members: 9 | 10 | .. autoclass:: concoursetools.testing.ConversionTestResourceWrapper 11 | :members: 12 | 13 | .. autoclass:: concoursetools.testing.FileTestResourceWrapper 14 | :members: 15 | 16 | .. autoclass:: concoursetools.testing.FileConversionTestResourceWrapper 17 | :members: 18 | 19 | .. autoclass:: concoursetools.testing.DockerTestResourceWrapper 20 | :members: 21 | 22 | .. autoclass:: concoursetools.testing.DockerConversionTestResourceWrapper 23 | :members: 24 | 25 | 26 | Running External Commands 27 | ------------------------- 28 | 29 | .. autofunction:: concoursetools.testing.run_command 30 | 31 | .. autofunction:: concoursetools.testing.run_script 32 | 33 | .. autofunction:: concoursetools.testing.run_docker_container 34 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | -------------------------------------------------------------------------------- /examples/build_status.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | from __future__ import annotations 3 | 4 | from dataclasses import dataclass 5 | from enum import Enum, auto 6 | from pathlib import Path 7 | import subprocess 8 | from typing import Any 9 | 10 | import requests 11 | from requests.auth import AuthBase, HTTPBasicAuth 12 | from urllib3 import disable_warnings as disable_ssl_warnings 13 | 14 | from concoursetools import BuildMetadata 15 | from concoursetools.additional import OutOnlyConcourseResource 16 | from concoursetools.colour import Colour, colour_print 17 | from concoursetools.typing import Metadata 18 | from concoursetools.version import TypedVersion 19 | 20 | 21 | class Driver(Enum): 22 | SERVER = "Bitbucket Server" 23 | CLOUD = "Bitbucket Cloud" 24 | 25 | 26 | class BuildStatus(Enum): 27 | SUCCESSFUL = auto() 28 | INPROGRESS = auto() 29 | FAILED = auto() 30 | 31 | 32 | class BitbucketOAuth(AuthBase): 33 | """ 34 | Adds the correct auth token for OAuth access to bitbucket.com. 35 | """ 36 | def __init__(self, access_token: str): 37 | self.access_token = access_token 38 | 39 | def __call__(self, request: requests.Request) -> requests.Request: 40 | request.headers["Authorization"] = f"Bearer {self.access_token}" 41 | return request 42 | 43 | @classmethod 44 | def from_client_credentials(cls, client_id: str, client_secret: str) -> "BitbucketOAuth": 45 | token_auth = HTTPBasicAuth(client_id, client_secret) 46 | 47 | url = "https://bitbucket.org/site/oauth2/access_token" 48 | data = {"grant_type": "client_credentials"} 49 | 50 | token_response = requests.post(url, auth=token_auth, data=data) 51 | token_response.raise_for_status() 52 | access_token = token_response.json()["access_token"] 53 | return cls(access_token) 54 | 55 | 56 | @dataclass 57 | class Version(TypedVersion): 58 | build_status: BuildStatus 59 | 60 | 61 | class Resource(OutOnlyConcourseResource[Version]): 62 | 63 | def __init__(self, repository: str | None = None, endpoint: str | None = None, 64 | username: str | None = None, password: str | None = None, 65 | client_id: str | None = None, client_secret: str | None = None, 66 | verify_ssl: bool = True, driver: str = "Bitbucket Server", 67 | debug: bool = False) -> None: 68 | super().__init__(Version) 69 | try: 70 | self.driver = Driver(driver) 71 | except ValueError: 72 | possible_values = {enum.value for enum in Driver._member_map_.values()} 73 | raise ValueError(f"Driver must be one of the following: " 74 | f"{possible_values}, not {driver!r}") 75 | 76 | self.auth = create_auth(username, password, client_id, client_secret) 77 | 78 | self.repository = repository 79 | self.endpoint = endpoint 80 | 81 | self.verify_ssl = verify_ssl 82 | self._debug = debug 83 | 84 | if self.driver is Driver.SERVER: 85 | if endpoint is None: 86 | raise ValueError("Must set endpoint when using Bitbucket Server.") 87 | else: 88 | endpoint = endpoint.rstrip("/") 89 | 90 | if self.driver is Driver.CLOUD: 91 | if repository is None: 92 | raise ValueError("Must set repository when using Bitbucket Cloud.") 93 | 94 | def publish_new_version(self, sources_dir: Path, build_metadata: BuildMetadata, 95 | repository: str, build_status: str, key: str | None = None, 96 | name: str | None = None, build_url: str | None = None, 97 | description: str | None = None, 98 | commit_hash: str | None = None) -> tuple[Version, Metadata]: 99 | self.debug("--DEBUG MODE--") 100 | 101 | try: 102 | status = BuildStatus[build_status] 103 | except KeyError: 104 | possible_values = set(BuildStatus._member_names_) 105 | raise ValueError(f"Build status must be one of the following: " 106 | f"{possible_values}, not {build_status!r}") 107 | 108 | if commit_hash is None: 109 | if repository is None: 110 | raise ValueError("Missing repository parameter.") 111 | repo_path = sources_dir / repository 112 | mercurial_path = repo_path / ".hg" 113 | git_path = repo_path / ".git" 114 | 115 | if mercurial_path.exists(): 116 | command = ["hg", "R", str(repo_path), "log", 117 | "--rev", ".", "--template", r"{node}"] 118 | elif git_path.exists(): 119 | command = ["git", "-C", str(repo_path), "rev-parse", "HEAD"] 120 | else: 121 | raise RuntimeError("Cannot detect a repository.") 122 | 123 | commit_hash = subprocess.check_output(command).strip().decode() 124 | 125 | self.debug(f"Commit: {commit_hash}") 126 | 127 | build_url = build_url or build_metadata.build_url() 128 | 129 | key = key or build_metadata.BUILD_JOB_NAME or f"one-off-build-{build_metadata.BUILD_ID}" 130 | 131 | self.debug(f"Build URL: {build_url}") 132 | 133 | description = description or f"Concourse CI build, hijack as #{build_metadata.BUILD_ID}" 134 | 135 | if name is None: 136 | if build_metadata.is_one_off_build: 137 | name = f"One-off build #{build_metadata.BUILD_ID}" 138 | else: 139 | name = f"{build_metadata.BUILD_JOB_NAME} #{build_metadata.BUILD_NAME}" 140 | 141 | description = build_metadata.format_string(description) 142 | 143 | if self.driver is Driver.SERVER: 144 | post_url = f"{self.endpoint}/rest/build-status/1.0/commits/{commit_hash}" 145 | if self.verify_ssl is False: 146 | disable_ssl_warnings() 147 | self.debug("SSL warnings disabled\n") 148 | 149 | else: 150 | post_url = f"https://api.bitbucket.org/2.0/repositories/{self.repository}/commit/{commit_hash}/statuses/build" 151 | 152 | data = { 153 | "state": status.name, 154 | "key": key, 155 | "name": name, 156 | "url": build_url, 157 | "description": description, 158 | } 159 | 160 | self.debug(f"Set build status: {data}") 161 | 162 | response = requests.post(post_url, json=data, auth=self.auth, verify=self.verify_ssl) 163 | 164 | self.debug(f"Request result: {response.json()}") 165 | 166 | version = Version(status) 167 | metadata = { 168 | "HTTP Status Code": str(response.status_code), 169 | } 170 | return version, metadata 171 | 172 | def debug(self, *args: object, colour: str = Colour.CYAN, **kwargs: Any) -> None: 173 | if self._debug: 174 | colour_print(*args, colour=colour, **kwargs) 175 | 176 | 177 | def create_auth(username: str | None = None, 178 | password: str | None = None, 179 | client_id: str | None = None, 180 | client_secret: str | None = None) -> AuthBase: 181 | if username is not None and password is not None: 182 | auth: AuthBase = HTTPBasicAuth(username, password) 183 | elif client_id is not None and client_secret is not None: 184 | auth = BitbucketOAuth.from_client_credentials(client_id, client_secret) 185 | else: 186 | raise ValueError("Must set username/password or OAuth credentials") 187 | return auth 188 | -------------------------------------------------------------------------------- /examples/github_branches.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | from __future__ import annotations 3 | 4 | from dataclasses import dataclass 5 | import re 6 | 7 | import requests 8 | 9 | from concoursetools.additional import MultiVersionConcourseResource 10 | from concoursetools.version import TypedVersion 11 | 12 | 13 | @dataclass(unsafe_hash=True, order=True) 14 | class BranchVersion(TypedVersion): 15 | name: str 16 | 17 | 18 | class Resource(MultiVersionConcourseResource[BranchVersion]): # type: ignore[type-var] 19 | def __init__(self, owner: str, repo: str, regex: str = ".*", 20 | endpoint: str = "https://api.github.com") -> None: 21 | """ 22 | Initialise self. 23 | 24 | :param owner: The owner of the repository. 25 | :param repo: The name of the repository. 26 | :param regex: An optional regex for filtering the branches. Only branches matching this 27 | regex will be considered. Defaults to a regex which matches ALL branches. 28 | :param endpoint: The GitHub API endpoint. Defaults to the public version of GitHub. 29 | """ 30 | super().__init__("branches", BranchVersion) 31 | self.api_route = f"{endpoint}/repos/{owner}/{repo}/branches" 32 | self.regex = re.compile(regex) 33 | 34 | def fetch_latest_sub_versions(self) -> set[BranchVersion]: 35 | headers = {"Accept": "application/vnd.github+json"} 36 | response = requests.get(self.api_route, headers=headers) 37 | branches_info = response.json() 38 | 39 | try: 40 | branch_names = {branch_info["name"] for branch_info in branches_info} 41 | except TypeError as error: # GitHub error: {"message": "..."} 42 | message = branches_info["message"] 43 | raise RuntimeError(message) from error 44 | 45 | return {BranchVersion(branch_name) for branch_name in branch_names 46 | if self.regex.fullmatch(branch_name)} 47 | -------------------------------------------------------------------------------- /examples/pipeline.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | from __future__ import annotations 3 | 4 | from collections.abc import Generator 5 | from dataclasses import dataclass 6 | from datetime import datetime 7 | import json 8 | from pathlib import Path 9 | 10 | import boto3 11 | 12 | from concoursetools import ConcourseResource 13 | from concoursetools.metadata import BuildMetadata 14 | from concoursetools.version import TypedVersion 15 | 16 | 17 | class DatetimeSafeJSONEncoder(json.JSONEncoder): 18 | 19 | def default(self, o: object) -> object: 20 | if isinstance(o, datetime): 21 | return o.isoformat() 22 | return super().default(o) 23 | 24 | 25 | @dataclass(unsafe_hash=True) 26 | class ExecutionVersion(TypedVersion): 27 | execution_arn: str 28 | 29 | 30 | class PipelineResource(ConcourseResource[ExecutionVersion]): 31 | 32 | def __init__(self, pipeline: str, statuses: list[str] = ["Succeeded", "Stopped", "Failed"]) -> None: 33 | super().__init__(ExecutionVersion) 34 | # arn:aws:sagemaker:::pipeline: 35 | _, _, _, region, _, _, pipeline_name = pipeline.split(":") 36 | self._client = boto3.client("sagemaker", region_name=region) 37 | self.pipeline_name = pipeline_name 38 | self.statuses = statuses 39 | 40 | def fetch_new_versions(self, previous_version: ExecutionVersion | None = None) -> list[ExecutionVersion]: 41 | potential_versions = iter(self._yield_potential_execution_versions()) 42 | if previous_version is None: 43 | try: 44 | first_version = next(potential_versions) 45 | except StopIteration: 46 | new_versions = [] 47 | else: 48 | new_versions = [first_version] 49 | else: 50 | new_versions = [] 51 | for potential_version in potential_versions: 52 | new_versions.append(potential_version) 53 | if potential_version == previous_version: 54 | break 55 | else: 56 | new_versions = [new_versions[0]] 57 | 58 | new_versions.reverse() 59 | return new_versions 60 | 61 | def download_version(self, version: ExecutionVersion, destination_dir: Path, 62 | build_metadata: BuildMetadata, download_pipeline: bool = True, 63 | metadata_file: str = "metadata.json", 64 | pipeline_file: str = "pipeline.json") -> tuple[ExecutionVersion, dict[str, str]]: 65 | response = self._client.describe_pipeline_execution(PipelineExecutionArn=version.execution_arn) 66 | response.pop("ResponseMetadata") 67 | 68 | metadata_path = destination_dir / metadata_file 69 | metadata_path.write_text(json.dumps(response, cls=DatetimeSafeJSONEncoder)) 70 | 71 | if download_pipeline: 72 | pipeline_response = self._client.describe_pipeline_definition_for_execution(PipelineExecutionArn=version.execution_arn) 73 | pipeline_path = destination_dir / pipeline_file 74 | pipeline_path.write_text(pipeline_response["PipelineDefinition"]) 75 | 76 | metadata = { 77 | "Display Name": response.get("PipelineExecutionDisplayName"), 78 | "Status": response["PipelineExecutionStatus"], 79 | "Created By": response["CreatedBy"]["UserProfileName"], 80 | "Description": response.get("PipelineExecutionDescription"), 81 | } 82 | 83 | if metadata["Status"] == "Failed": 84 | metadata["Failure Reason"] = response["FailureReason"] 85 | 86 | metadata = {key: value for key, value in metadata.items() if value is not None} 87 | 88 | return version, metadata 89 | 90 | def publish_new_version(self, sources_dir: Path, build_metadata: BuildMetadata, 91 | display_name: str | None = None, description: str | None = None, 92 | parameters: dict[str, str] = {}) -> tuple[ExecutionVersion, dict[str, str]]: 93 | default_description = (f"Execution from build #{build_metadata.BUILD_ID} " 94 | f"of pipeline {build_metadata.BUILD_PIPELINE_NAME}") 95 | kwargs: dict[str, object] = { 96 | "PipelineName": self.pipeline_name, 97 | "PipelineExecutionDescription": description or default_description, 98 | } 99 | 100 | if display_name: 101 | kwargs["PipelineExecutionDisplayName"] = display_name 102 | 103 | if parameters: 104 | kwargs["PipelineParameters"] = [{"Name": name, "Value": value} 105 | for name, value in parameters.items()] 106 | metadata = {f"Parameter: {parameter}": value 107 | for parameter, value in parameters.items()} 108 | else: 109 | metadata = {} 110 | 111 | response = self._client.start_pipeline_execution(**kwargs) 112 | execution_arn = response["PipelineExecutionArn"] 113 | new_version = ExecutionVersion(execution_arn) 114 | return new_version, metadata 115 | 116 | def _yield_potential_execution_versions(self) -> Generator[ExecutionVersion, None, None]: 117 | kwargs = { 118 | "PipelineName": self.pipeline_name, 119 | "SortOrder": "Descending", 120 | } 121 | 122 | first_response = self._client.list_pipeline_executions(**kwargs) 123 | 124 | response = first_response 125 | while True: 126 | for summary in response["PipelineExecutionSummaries"]: 127 | if summary["PipelineExecutionStatus"] in self.statuses: 128 | yield ExecutionVersion(summary["PipelineExecutionArn"]) 129 | 130 | try: 131 | next_token = response["NextToken"] 132 | except KeyError: 133 | break 134 | 135 | response = self._client.list_pipeline_executions(**kwargs, NextToken=next_token) 136 | -------------------------------------------------------------------------------- /examples/s3.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | from __future__ import annotations 3 | 4 | from datetime import timedelta 5 | from pathlib import Path 6 | 7 | import boto3 8 | from botocore.client import Config 9 | 10 | from concoursetools import BuildMetadata 11 | from concoursetools.additional import InOnlyConcourseResource 12 | 13 | 14 | class S3SignedURLConcourseResource(InOnlyConcourseResource): 15 | """ 16 | A Concourse resource type for generating pre-signed URLs for items in S3 buckets. 17 | """ 18 | def __init__(self, bucket_name: str, region_name: str) -> None: 19 | """ 20 | Initialise self. 21 | 22 | :param bucket_name: The name of your bucket. 23 | :param region_name: The name of the region in which your bucket resides. 24 | """ 25 | super().__init__() 26 | self.bucket_name = bucket_name 27 | self.client = boto3.client("s3", region_name=region_name, 28 | config=Config(signature_version="s3v4")) 29 | 30 | def download_data(self, destination_dir: Path, build_metadata: BuildMetadata, 31 | file_path: str, expires_in: dict[str, float], 32 | file_name: str | None = None, 33 | url_file: str = "url") -> dict[str, str]: 34 | params = { 35 | "Bucket": self.bucket_name, 36 | "Key": file_path, 37 | } 38 | if file_name is not None: 39 | # https://stackoverflow.com/a/2612795 40 | content_disposition = f"attachment; filename=\"{file_name}\"" 41 | params["ResponseContentDisposition"] = content_disposition 42 | 43 | expiry_seconds = int(timedelta(**expires_in).total_seconds()) 44 | url = self.client.generate_presigned_url(ClientMethod="get_object", 45 | Params=params, 46 | ExpiresIn=expiry_seconds) 47 | 48 | url_file_path = destination_dir / url_file 49 | url_file_path.write_text(url) 50 | 51 | return {} 52 | -------------------------------------------------------------------------------- /examples/secrets.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | from __future__ import annotations 3 | 4 | from dataclasses import dataclass 5 | from datetime import datetime 6 | import json as json_package 7 | from pathlib import Path 8 | from typing import Any 9 | 10 | import boto3 11 | from botocore.exceptions import ClientError 12 | 13 | from concoursetools import BuildMetadata 14 | from concoursetools.additional import TriggerOnChangeConcourseResource 15 | from concoursetools.version import TypedVersion 16 | 17 | 18 | class DatetimeSafeJSONEncoder(json_package.JSONEncoder): 19 | 20 | def default(self, o: object) -> object: 21 | if isinstance(o, datetime): 22 | return o.isoformat() 23 | return super().default(o) 24 | 25 | 26 | @dataclass(unsafe_hash=True) 27 | class SecretVersion(TypedVersion): 28 | version_id: str 29 | 30 | 31 | class Resource(TriggerOnChangeConcourseResource[SecretVersion]): 32 | """ 33 | :param secret: The full Amazon Resource Name (ARN) of the secret. 34 | """ 35 | def __init__(self, secret: str) -> None: 36 | super().__init__(SecretVersion) 37 | self.secret = secret 38 | 39 | # arn:aws:secretsmanager:::secret: 40 | _, _, _, region, _, _, _ = secret.split(":") 41 | self._client = boto3.client("secretsmanager", region_name=region) 42 | 43 | def fetch_latest_version(self) -> SecretVersion: 44 | try: 45 | response = self._client.list_secret_version_ids(SecretId=self.secret, 46 | IncludeDeprecated=False) 47 | except ClientError: 48 | raise ValueError(f"Cannot find secret: {self.secret!r}") 49 | 50 | versions = response["Versions"] 51 | for version in versions: 52 | version_stages = version["VersionStages"] 53 | if "AWSCURRENT" in version_stages: 54 | version_id = version["VersionId"] 55 | return SecretVersion(version_id) 56 | raise RuntimeError("No current version of the secret could be found.") 57 | 58 | def download_version(self, version: SecretVersion, destination_dir: Path, 59 | build_metadata: BuildMetadata, value: bool = False, 60 | metadata_file: str = "metadata.json", 61 | value_file: str = "value") -> tuple[SecretVersion, dict[str, str]]: 62 | meta_response: dict[str, Any] = self._client.describe_secret(SecretId=self.secret) 63 | meta_response.pop("ResponseMetadata") 64 | 65 | metadata_path = destination_dir / metadata_file 66 | metadata_path.write_text(json_package.dumps(meta_response, 67 | cls=DatetimeSafeJSONEncoder)) 68 | 69 | if value: 70 | value_response = self._client.get_secret_value(SecretId=self.secret) 71 | value_path = destination_dir / value_file 72 | 73 | try: 74 | secret_value = value_response["SecretString"] 75 | except KeyError: 76 | secret_value_as_bytes: bytes = value_response["SecretBinary"] 77 | value_path.write_bytes(secret_value_as_bytes) 78 | else: 79 | value_path.write_text(secret_value) 80 | 81 | return version, {} 82 | 83 | def publish_new_version(self, sources_dir: Path, 84 | build_metadata: BuildMetadata, string: str | None = None, 85 | file: str | None = None, 86 | json: dict[str, str] | None = None) -> tuple[SecretVersion, dict[str, str]]: 87 | if json is not None: 88 | string = json_package.dumps(json) 89 | 90 | if string is not None: 91 | response = self._client.put_secret_value(SecretId=self.secret, 92 | SecretString=string) 93 | elif file is not None: 94 | file_path = sources_dir / file 95 | file_contents = file_path.read_bytes() 96 | response = self._client.put_secret_value(SecretId=self.secret, 97 | SecretBinary=file_contents) 98 | else: 99 | raise ValueError("Missing new value for the secret.") 100 | 101 | version_id = response["VersionId"] 102 | metadata = { 103 | "Version Staging Labels": ", ".join(response["VersionStages"]) 104 | } 105 | return SecretVersion(version_id), metadata 106 | -------------------------------------------------------------------------------- /examples/xkcd.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | from __future__ import annotations 3 | 4 | from collections.abc import Generator 5 | from dataclasses import dataclass 6 | from datetime import datetime 7 | import json 8 | from pathlib import Path 9 | import urllib.parse 10 | import xml.etree.ElementTree as ET 11 | 12 | import requests 13 | 14 | from concoursetools.additional import SelfOrganisingConcourseResource 15 | from concoursetools.metadata import BuildMetadata 16 | from concoursetools.version import SortableVersionMixin, TypedVersion 17 | 18 | 19 | @dataclass(unsafe_hash=True) 20 | class ComicVersion(TypedVersion, SortableVersionMixin): 21 | comic_id: int 22 | 23 | def __lt__(self, other: object) -> bool: 24 | if not isinstance(other, type(self)): 25 | return NotImplemented 26 | return self.comic_id < other.comic_id 27 | 28 | 29 | class XKCDResource(SelfOrganisingConcourseResource[ComicVersion]): 30 | 31 | def __init__(self, url: str = "https://xkcd.com"): 32 | super().__init__(ComicVersion) 33 | self.url = url 34 | 35 | def fetch_all_versions(self) -> set[ComicVersion]: 36 | atom_url = f"{self.url}/atom.xml" 37 | response = requests.get(atom_url) 38 | feed_data = response.text 39 | return {ComicVersion(comic_id) for comic_id in yield_comic_ids(feed_data)} 40 | 41 | def download_version(self, version: ComicVersion, destination_dir: Path, 42 | build_metadata: BuildMetadata, image: bool = True, 43 | link: bool = True, alt: bool = True) -> tuple[ComicVersion, dict[str, str]]: 44 | comic_info_url = f"{self.url}/{version.comic_id}/info.0.json" 45 | response = requests.get(comic_info_url) 46 | info = response.json() 47 | 48 | title = info["title"] 49 | url = f"{self.url}/{version.comic_id}/" 50 | 51 | upload_date = datetime(year=int(info["year"]), month=int(info["month"]), 52 | day=int(info["day"])) 53 | metadata = { 54 | "Title": title, 55 | "Uploaded": upload_date.strftime(r"%d/%m/%Y"), 56 | "URL": f"{self.url}/{version.comic_id}/", 57 | } 58 | 59 | info_path = destination_dir / "info.json" 60 | info_path.write_text(json.dumps(info)) 61 | 62 | if image: 63 | image_path = destination_dir / "image.png" 64 | image_request = requests.get(info["img"], stream=True) 65 | with open(image_path, "wb") as wf: 66 | for chunk in image_request: 67 | wf.write(chunk) 68 | 69 | if link: 70 | link_path = destination_dir / "link.txt" 71 | link_path.write_text(url) 72 | 73 | if alt: 74 | alt_path = destination_dir / "alt.txt" 75 | alt_path.write_text(info["alt"]) 76 | 77 | return version, metadata 78 | 79 | def publish_new_version(self, sources_dir: Path, build_metadata: BuildMetadata) -> tuple[ComicVersion, dict[str, str]]: 80 | raise NotImplementedError 81 | 82 | 83 | def yield_comic_ids(xml_data: str) -> Generator[int, None, None]: 84 | for comic_url in yield_comic_links(xml_data): 85 | parsed_url = urllib.parse.urlparse(comic_url) 86 | comic_id = parsed_url.path.strip("/") 87 | yield int(comic_id) 88 | 89 | 90 | def yield_comic_links(xml_data: str) -> Generator[str, None, None]: 91 | root = ET.fromstring(xml_data) 92 | for entry in root: 93 | if entry.tag.endswith("entry"): 94 | for child in entry: 95 | if child.tag.endswith("link"): 96 | items = dict(child.items()) 97 | yield items["href"] 98 | -------------------------------------------------------------------------------- /examples/xkcd.xml: -------------------------------------------------------------------------------- 1 | 2 | xkcd.com 3 | 4 | https://xkcd.com/ 5 | 2023-08-07T00:00:00Z 6 | 7 | Solar Panel Placement 8 | 9 | 2023-08-07T00:00:00Z 10 | https://xkcd.com/2812/ 11 | 12 | Getting the utility people to run transmission lines to Earth is expensive, but it will pay for itself in no time. 13 | 14 | 15 | 16 | Free Fallin' 17 | 18 | 2023-08-04T00:00:00Z 19 | https://xkcd.com/2811/ 20 | 21 | Their crash investigation team had some particularly harsh words for Dave Matthews. 22 | 23 | 24 | 25 | How to Coil a Cable 26 | 27 | 2023-08-02T00:00:00Z 28 | https://xkcd.com/2810/ 29 | 30 | The ideal mix for maximum competitive cable-coiling energy is one A/V tech, one rock climber, one sailor, and one topologist. 31 | 32 | 33 | 34 | Moon 35 | 36 | 2023-07-31T00:00:00Z 37 | https://xkcd.com/2809/ 38 | 39 | I mean, it's pretty, but it doesn't really affect us beyond that. Except that half the nights aren't really dark, and once or twice a day it makes the oceans flood the coasts. 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "concoursetools" 7 | dynamic = ["version"] 8 | description = "Easily create Concourse resource types in Python." 9 | readme = "README.md" 10 | requires-python = ">=3.9" 11 | keywords = [ 12 | "concourse", 13 | "ci", 14 | "cd", 15 | ] 16 | classifiers = [ 17 | "Development Status :: 4 - Beta", 18 | "Intended Audience :: Developers", 19 | "License :: OSI Approved :: Apache Software License", 20 | "Natural Language :: English", 21 | "Programming Language :: Python :: 3.9", 22 | "Programming Language :: Python :: 3.10", 23 | "Programming Language :: Python :: 3.11", 24 | "Programming Language :: Python :: 3.12", 25 | "Programming Language :: Python :: 3.13", 26 | "Programming Language :: Python :: 3.14", 27 | "Topic :: Software Development", 28 | "Typing :: Typed", 29 | ] 30 | authors = [ 31 | {name = "GCHQ", email = "oss@gchq.gov.uk"}, 32 | ] 33 | license = {text = "Apache License 2.0"} 34 | 35 | [project.urls] 36 | "Homepage" = "https://github.com/gchq/ConcourseTools/" 37 | "Documentation" = "https://concoursetools.readthedocs.io/en/stable/" 38 | "Repository" = "https://github.com/gchq/ConcourseTools/" 39 | "Bug Tracker" = "https://github.com/gchq/ConcourseTools/issues" 40 | "Changelog" = "https://concoursetools.readthedocs.io/en/latest/whats_new.html" 41 | 42 | [tool.setuptools] 43 | packages = [ 44 | "concoursetools", 45 | "concoursetools.cli", 46 | ] 47 | 48 | [tool.setuptools.dynamic] 49 | version = {attr = "concoursetools.__version__"} 50 | 51 | 52 | [tool.autopep8] 53 | max_line_length = 150 54 | ignore = ["E301", "E501"] 55 | in-place = true 56 | recursive = true 57 | aggressive = 3 58 | 59 | 60 | [tool.coverage.run] 61 | command_line = "-m unittest discover" 62 | source = ["concoursetools"] 63 | omit = ["concoursetools/colour.py", "concoursetools/typing.py", "*/__init__.py", "*/__main__.py"] 64 | 65 | [tool.coverage.report] 66 | sort = "Cover" 67 | exclude_lines = [ 68 | "@abstractmethod", 69 | "def __repr__", 70 | "pass", 71 | "raise$", 72 | "raise RuntimeError", 73 | ] 74 | 75 | 76 | [tool.isort] 77 | line_length = 150 78 | force_sort_within_sections = true 79 | 80 | 81 | [tool.mypy] 82 | python_version = "3.12" 83 | check_untyped_defs = true 84 | disallow_untyped_defs = true 85 | warn_redundant_casts = true 86 | warn_unused_ignores = true 87 | warn_return_any = true 88 | disallow_any_generics = true 89 | disable_error_code = ["override"] 90 | 91 | 92 | [tool.pylint] 93 | max-line-length = 150 94 | disable = ["too-many-arguments", "too-few-public-methods"] 95 | good-names = ["wf"] 96 | ignore-paths = ["docs/source/conf.py", "tests"] 97 | 98 | 99 | [tool.pydocstyle] 100 | ignore = ["D102", "D105", "D107", "D200", "D203", "D204", "D205", "D212", "D400", "D415"] 101 | -------------------------------------------------------------------------------- /requirements-tests.txt: -------------------------------------------------------------------------------- 1 | boto3==1.36.25 2 | botocore==1.36.25 3 | certifi==2025.1.31 4 | cffi==1.17.1 5 | charset-normalizer==3.4.1 6 | cryptography==44.0.1 7 | idna==3.10 8 | Jinja2==3.1.5 9 | jmespath==1.0.1 10 | MarkupSafe==3.0.2 11 | moto==5.0.12 12 | pycparser==2.22 13 | python-dateutil==2.9.0.post0 14 | PyYAML==6.0.2 15 | requests==2.32.3 16 | responses==0.25.6 17 | s3transfer==0.11.2 18 | six==1.17.0 19 | urllib3==2.3.0 20 | Werkzeug==3.1.3 21 | xmltodict==0.14.2 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Setup for the concoursetools Python package. 4 | """ 5 | from setuptools import setup 6 | 7 | if __name__ == "__main__": 8 | setup() 9 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | sonar.projectDescription="Static analysis for the Concourse Tools Python library." 2 | 3 | sonar.sources=concoursetools 4 | sonar.tests=tests 5 | 6 | sonar.python.version=3.9,3.10,3.11,3.12,3.13,3.14 7 | 8 | sonar.coverage.exclusions=docs/**/*,tests/**/*,coverage.xml,concoursetools/colour.py,concoursetools/typing.py,**/__init__.py,**/__main__.py 9 | sonar.python.coverage.reportPaths=coverage.xml 10 | sonar.cobertura.reportPath=coverage.xml 11 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | -------------------------------------------------------------------------------- /tests/resource.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Contains a test resource. 4 | """ 5 | from __future__ import annotations 6 | 7 | from copy import copy 8 | from dataclasses import dataclass 9 | from pathlib import Path 10 | 11 | import concoursetools 12 | from concoursetools.metadata import BuildMetadata 13 | from concoursetools.typing import Metadata 14 | 15 | 16 | class TestVersion(concoursetools.Version): 17 | 18 | def __init__(self, ref: str): 19 | self.ref = ref 20 | 21 | 22 | class TestResource(concoursetools.ConcourseResource[TestVersion]): 23 | 24 | def __init__(self, uri: str, branch: str = "main", private_key: str | None = None): 25 | super().__init__(TestVersion) 26 | self.uri = uri 27 | self.branch = branch 28 | self.private_key = private_key 29 | 30 | def fetch_new_versions(self, previous_version: TestVersion | None = None) -> list[TestVersion]: 31 | if previous_version: 32 | print("Previous version found.") 33 | return [TestVersion("7154fe")] 34 | else: 35 | return [TestVersion(ref) for ref in ("61cbef", "d74e01", "7154fe")] 36 | 37 | def download_version(self, version: TestVersion, destination_dir: Path, build_metadata: concoursetools.BuildMetadata, 38 | file_name: str = "README.txt") -> tuple[TestVersion, Metadata]: 39 | print("Downloading.") 40 | readme_path = destination_dir / file_name 41 | readme_path.write_text(f"Downloaded README for ref {version.ref}.\n") 42 | metadata = { 43 | "team_name": build_metadata.BUILD_TEAM_NAME, 44 | } 45 | return version, metadata 46 | 47 | def publish_new_version(self, sources_dir: Path, build_metadata: concoursetools.BuildMetadata, repo: str, 48 | ref_file: str = "ref.txt") -> tuple[TestVersion, Metadata]: 49 | ref_path = sources_dir / repo / ref_file 50 | ref = ref_path.read_text() 51 | print("Uploading.") 52 | return TestVersion(ref), {} 53 | 54 | 55 | @dataclass 56 | class ConcourseMockVersion(concoursetools.TypedVersion): 57 | version: int 58 | privileged: bool 59 | 60 | 61 | ConcourseMockVersion._flatten_functions = copy(ConcourseMockVersion._flatten_functions) 62 | ConcourseMockVersion._un_flatten_functions = copy(ConcourseMockVersion._un_flatten_functions) 63 | 64 | 65 | @ConcourseMockVersion.flatten 66 | def _(obj: bool) -> str: 67 | return str(obj).lower() 68 | 69 | 70 | @ConcourseMockVersion.un_flatten 71 | def _(_type: type[bool], obj: str) -> bool: 72 | return obj == "true" 73 | 74 | 75 | class ConcourseMockResource(concoursetools.ConcourseResource[ConcourseMockVersion]): 76 | 77 | def __init__(self, **kwargs: object) -> None: 78 | super().__init__(ConcourseMockVersion) 79 | 80 | def fetch_new_versions(self, previous_version: ConcourseMockVersion | None = None) -> list[ConcourseMockVersion]: 81 | raise NotImplementedError 82 | 83 | def download_version(self, version: ConcourseMockVersion, destination_dir: Path, 84 | build_metadata: BuildMetadata) -> tuple[ConcourseMockVersion, Metadata]: 85 | raise NotImplementedError 86 | 87 | def publish_new_version(self, sources_dir: Path, build_metadata: BuildMetadata) -> tuple[ConcourseMockVersion, Metadata]: 88 | raise NotImplementedError 89 | -------------------------------------------------------------------------------- /tests/test_cli_commands.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | from contextlib import redirect_stdout 3 | from io import StringIO 4 | import os 5 | from pathlib import Path 6 | import shutil 7 | import sys 8 | from tempfile import TemporaryDirectory 9 | import textwrap 10 | import unittest 11 | import unittest.mock 12 | 13 | from concoursetools.cli import cli 14 | from concoursetools.colour import Colour, colourise 15 | 16 | 17 | class AssetTests(unittest.TestCase): 18 | """ 19 | Tests for creation of the asset files. 20 | """ 21 | def setUp(self) -> None: 22 | """Code to run before each test.""" 23 | self._temp_dir = TemporaryDirectory() 24 | self.temp_dir = Path(self._temp_dir.name) 25 | self._original_dir = Path.cwd() 26 | 27 | path_to_this_file = Path(__file__) 28 | path_to_test_resource_module = path_to_this_file.parent / "resource.py" 29 | shutil.copyfile(path_to_test_resource_module, self.temp_dir / "concourse.py") 30 | os.chdir(self.temp_dir) 31 | 32 | def tearDown(self) -> None: 33 | """Code to run after each test.""" 34 | os.chdir(self._original_dir) 35 | self._temp_dir.cleanup() 36 | 37 | def test_asset_scripts(self) -> None: 38 | asset_dir = self.temp_dir / "assets" 39 | self.assertFalse(asset_dir.exists()) 40 | 41 | new_stdout = StringIO() 42 | with redirect_stdout(new_stdout): 43 | cli.invoke(["assets", "assets", "-c", "TestResource"]) 44 | 45 | self.assertEqual(new_stdout.getvalue(), "") 46 | 47 | self.assertTrue(asset_dir.exists()) 48 | self.assertSetEqual({path.name for path in asset_dir.iterdir()}, {"check", "in", "out"}) 49 | 50 | 51 | class DockerfileTests(unittest.TestCase): 52 | """ 53 | Tests for creating the Dockerfile. 54 | """ 55 | def setUp(self) -> None: 56 | """Code to run before each test.""" 57 | self._temp_dir = TemporaryDirectory() 58 | self.temp_dir = Path(self._temp_dir.name) 59 | self._original_dir = Path.cwd() 60 | 61 | path_to_this_file = Path(__file__) 62 | path_to_test_resource_module = path_to_this_file.parent / "resource.py" 63 | shutil.copyfile(path_to_test_resource_module, self.temp_dir / "concourse.py") 64 | self.dockerfile_path = self.temp_dir / "Dockerfile" 65 | self.assertFalse(self.dockerfile_path.exists()) 66 | 67 | self.current_python_string = f"{sys.version_info.major}.{sys.version_info.minor}" 68 | 69 | os.chdir(self.temp_dir) 70 | 71 | def tearDown(self) -> None: 72 | """Code to run after each test.""" 73 | os.chdir(self._original_dir) 74 | self._temp_dir.cleanup() 75 | 76 | def test_docker(self) -> None: 77 | new_stdout = StringIO() 78 | with redirect_stdout(new_stdout): 79 | cli.invoke(["dockerfile", "."]) 80 | 81 | self.assertEqual(new_stdout.getvalue(), "") 82 | 83 | dockerfile_contents = self.dockerfile_path.read_text() 84 | expected_contents = textwrap.dedent(f""" 85 | FROM python:{self.current_python_string} 86 | 87 | RUN python3 -m venv /opt/venv 88 | # Activate venv 89 | ENV PATH="/opt/venv/bin:$PATH" 90 | 91 | COPY requirements.txt requirements.txt 92 | 93 | RUN \\ 94 | python3 -m pip install --upgrade pip && \\ 95 | pip install -r requirements.txt --no-deps 96 | 97 | WORKDIR /opt/resource/ 98 | COPY concourse.py ./concourse.py 99 | RUN python3 -m concoursetools assets . -r concourse.py 100 | 101 | ENTRYPOINT ["python3"] 102 | """).lstrip() 103 | self.assertEqual(dockerfile_contents, expected_contents) 104 | 105 | 106 | class LegacyTests(unittest.TestCase): 107 | """ 108 | Tests for the legacy CLI. 109 | """ 110 | def setUp(self) -> None: 111 | """Code to run before each test.""" 112 | self._temp_dir = TemporaryDirectory() 113 | self.temp_dir = Path(self._temp_dir.name) 114 | self._original_dir = Path.cwd() 115 | 116 | path_to_this_file = Path(__file__) 117 | path_to_test_resource_module = path_to_this_file.parent / "resource.py" 118 | shutil.copyfile(path_to_test_resource_module, self.temp_dir / "concourse.py") 119 | self.dockerfile_path = self.temp_dir / "Dockerfile" 120 | self.assertFalse(self.dockerfile_path.exists()) 121 | 122 | self.current_python_string = f"{sys.version_info.major}.{sys.version_info.minor}" 123 | 124 | os.chdir(self.temp_dir) 125 | 126 | def tearDown(self) -> None: 127 | """Code to run after each test.""" 128 | os.chdir(self._original_dir) 129 | self._temp_dir.cleanup() 130 | 131 | def test_docker(self) -> None: 132 | new_stdout = StringIO() 133 | with redirect_stdout(new_stdout): 134 | cli.invoke(["legacy", ".", "--docker"]) 135 | 136 | self.assertEqual(new_stdout.getvalue(), colourise(textwrap.dedent(""" 137 | The legacy CLI has been deprecated. 138 | Please refer to the documentation or help pages for the up to date CLI. 139 | This CLI will be removed in version 0.10.0, or in version 1.0.0, whichever is sooner. 140 | 141 | """), colour=Colour.RED)) 142 | 143 | dockerfile_contents = self.dockerfile_path.read_text() 144 | expected_contents = textwrap.dedent(f""" 145 | FROM python:{self.current_python_string}-alpine 146 | 147 | COPY requirements.txt requirements.txt 148 | 149 | RUN \\ 150 | python3 -m pip install --upgrade pip && \\ 151 | pip install -r requirements.txt --no-deps 152 | 153 | WORKDIR /opt/resource/ 154 | COPY concourse.py ./concourse.py 155 | RUN python3 -m concoursetools assets . -r concourse.py 156 | 157 | ENTRYPOINT ["python3"] 158 | """).lstrip() 159 | self.assertEqual(dockerfile_contents, expected_contents) 160 | 161 | def test_asset_scripts(self) -> None: 162 | asset_dir = self.temp_dir / "assets" 163 | self.assertFalse(asset_dir.exists()) 164 | 165 | new_stdout = StringIO() 166 | with redirect_stdout(new_stdout): 167 | cli.invoke(["legacy", "assets", "-c", "TestResource"]) 168 | 169 | self.assertEqual(new_stdout.getvalue(), colourise(textwrap.dedent(""" 170 | The legacy CLI has been deprecated. 171 | Please refer to the documentation or help pages for the up to date CLI. 172 | This CLI will be removed in version 0.10.0, or in version 1.0.0, whichever is sooner. 173 | 174 | """), colour=Colour.RED)) 175 | 176 | self.assertTrue(asset_dir.exists()) 177 | self.assertSetEqual({path.name for path in asset_dir.iterdir()}, {"check", "in", "out"}) 178 | -------------------------------------------------------------------------------- /tests/test_dockertools.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Tests for the dockertools module. 4 | """ 5 | from pathlib import Path 6 | import shutil 7 | import sys 8 | from tempfile import TemporaryDirectory 9 | import textwrap 10 | from unittest import TestCase 11 | 12 | from concoursetools.cli import commands as cli_commands 13 | 14 | 15 | class DockerTests(TestCase): 16 | """ 17 | Tests for the creation of the Dockerfile. 18 | """ 19 | def setUp(self) -> None: 20 | """Code to run before each test.""" 21 | self._temp_dir = TemporaryDirectory() 22 | self.temp_dir = Path(self._temp_dir.name) 23 | 24 | path_to_this_file = Path(__file__) 25 | path_to_test_resource_module = path_to_this_file.parent / "resource.py" 26 | shutil.copyfile(path_to_test_resource_module, self.temp_dir / "concourse.py") 27 | self.dockerfile_path = self.temp_dir / "Dockerfile" 28 | self.assertFalse(self.dockerfile_path.exists()) 29 | 30 | self.current_python_string = f"{sys.version_info.major}.{sys.version_info.minor}" 31 | 32 | def tearDown(self) -> None: 33 | """Code to run after each test.""" 34 | self._temp_dir.cleanup() 35 | 36 | def test_basic_config(self) -> None: 37 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py") 38 | dockerfile_contents = self.dockerfile_path.read_text() 39 | expected_contents = textwrap.dedent(f""" 40 | FROM python:{self.current_python_string} 41 | 42 | RUN python3 -m venv /opt/venv 43 | # Activate venv 44 | ENV PATH="/opt/venv/bin:$PATH" 45 | 46 | COPY requirements.txt requirements.txt 47 | 48 | RUN \\ 49 | python3 -m pip install --upgrade pip && \\ 50 | pip install -r requirements.txt --no-deps 51 | 52 | WORKDIR /opt/resource/ 53 | COPY concourse.py ./concourse.py 54 | RUN python3 -m concoursetools assets . -r concourse.py 55 | 56 | ENTRYPOINT ["python3"] 57 | """).lstrip() 58 | self.assertEqual(dockerfile_contents, expected_contents) 59 | 60 | def test_basic_config_custom_image_and_tag(self) -> None: 61 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", image="node", tag="lts-slim") 62 | dockerfile_contents = self.dockerfile_path.read_text() 63 | expected_contents = textwrap.dedent(""" 64 | FROM node:lts-slim 65 | 66 | RUN python3 -m venv /opt/venv 67 | # Activate venv 68 | ENV PATH="/opt/venv/bin:$PATH" 69 | 70 | COPY requirements.txt requirements.txt 71 | 72 | RUN \\ 73 | python3 -m pip install --upgrade pip && \\ 74 | pip install -r requirements.txt --no-deps 75 | 76 | WORKDIR /opt/resource/ 77 | COPY concourse.py ./concourse.py 78 | RUN python3 -m concoursetools assets . -r concourse.py 79 | 80 | ENTRYPOINT ["python3"] 81 | """).lstrip() 82 | self.assertEqual(dockerfile_contents, expected_contents) 83 | 84 | def test_basic_config_pip_args(self) -> None: 85 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", pip_args="--trusted-host pypi.org") 86 | dockerfile_contents = self.dockerfile_path.read_text() 87 | expected_contents = textwrap.dedent(f""" 88 | FROM python:{self.current_python_string} 89 | 90 | RUN python3 -m venv /opt/venv 91 | # Activate venv 92 | ENV PATH="/opt/venv/bin:$PATH" 93 | 94 | COPY requirements.txt requirements.txt 95 | 96 | RUN \\ 97 | python3 -m pip install --upgrade pip --trusted-host pypi.org && \\ 98 | pip install -r requirements.txt --no-deps --trusted-host pypi.org 99 | 100 | WORKDIR /opt/resource/ 101 | COPY concourse.py ./concourse.py 102 | RUN python3 -m concoursetools assets . -r concourse.py 103 | 104 | ENTRYPOINT ["python3"] 105 | """).lstrip() 106 | self.assertEqual(dockerfile_contents, expected_contents) 107 | 108 | def test_basic_config_no_venv(self) -> None: 109 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", no_venv=True) 110 | dockerfile_contents = self.dockerfile_path.read_text() 111 | expected_contents = textwrap.dedent(f""" 112 | FROM python:{self.current_python_string} 113 | 114 | COPY requirements.txt requirements.txt 115 | 116 | RUN \\ 117 | python3 -m pip install --upgrade pip && \\ 118 | pip install -r requirements.txt --no-deps 119 | 120 | WORKDIR /opt/resource/ 121 | COPY concourse.py ./concourse.py 122 | RUN python3 -m concoursetools assets . -r concourse.py 123 | 124 | ENTRYPOINT ["python3"] 125 | """).lstrip() 126 | self.assertEqual(dockerfile_contents, expected_contents) 127 | 128 | def test_basic_config_with_suffix(self) -> None: 129 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", suffix="slim") 130 | dockerfile_contents = self.dockerfile_path.read_text() 131 | expected_contents = textwrap.dedent(f""" 132 | FROM python:{self.current_python_string}-slim 133 | 134 | RUN python3 -m venv /opt/venv 135 | # Activate venv 136 | ENV PATH="/opt/venv/bin:$PATH" 137 | 138 | COPY requirements.txt requirements.txt 139 | 140 | RUN \\ 141 | python3 -m pip install --upgrade pip && \\ 142 | pip install -r requirements.txt --no-deps 143 | 144 | WORKDIR /opt/resource/ 145 | COPY concourse.py ./concourse.py 146 | RUN python3 -m concoursetools assets . -r concourse.py 147 | 148 | ENTRYPOINT ["python3"] 149 | """).lstrip() 150 | self.assertEqual(dockerfile_contents, expected_contents) 151 | 152 | def test_basic_config_with_different_name(self) -> None: 153 | cli_commands.dockerfile(str(self.temp_dir), resource_file="resource.py") 154 | dockerfile_contents = self.dockerfile_path.read_text() 155 | expected_contents = textwrap.dedent(f""" 156 | FROM python:{self.current_python_string} 157 | 158 | RUN python3 -m venv /opt/venv 159 | # Activate venv 160 | ENV PATH="/opt/venv/bin:$PATH" 161 | 162 | COPY requirements.txt requirements.txt 163 | 164 | RUN \\ 165 | python3 -m pip install --upgrade pip && \\ 166 | pip install -r requirements.txt --no-deps 167 | 168 | WORKDIR /opt/resource/ 169 | COPY resource.py ./resource.py 170 | RUN python3 -m concoursetools assets . -r resource.py 171 | 172 | ENTRYPOINT ["python3"] 173 | """).lstrip() 174 | self.assertEqual(dockerfile_contents, expected_contents) 175 | 176 | def test_basic_config_with_class_name_and_executable(self) -> None: 177 | cli_commands.dockerfile(str(self.temp_dir), class_name="MyResource", executable="/usr/bin/python3") 178 | dockerfile_contents = self.dockerfile_path.read_text() 179 | expected_contents = textwrap.dedent(f""" 180 | FROM python:{self.current_python_string} 181 | 182 | RUN python3 -m venv /opt/venv 183 | # Activate venv 184 | ENV PATH="/opt/venv/bin:$PATH" 185 | 186 | COPY requirements.txt requirements.txt 187 | 188 | RUN \\ 189 | python3 -m pip install --upgrade pip && \\ 190 | pip install -r requirements.txt --no-deps 191 | 192 | WORKDIR /opt/resource/ 193 | COPY concourse.py ./concourse.py 194 | RUN python3 -m concoursetools assets . -r concourse.py -c MyResource -e /usr/bin/python3 195 | 196 | ENTRYPOINT ["python3"] 197 | """).lstrip() 198 | self.assertEqual(dockerfile_contents, expected_contents) 199 | 200 | def test_netrc_config(self) -> None: 201 | cli_commands.dockerfile(str(self.temp_dir), include_netrc=True) 202 | dockerfile_contents = self.dockerfile_path.read_text() 203 | expected_contents = textwrap.dedent(f""" 204 | FROM python:{self.current_python_string} 205 | 206 | RUN python3 -m venv /opt/venv 207 | # Activate venv 208 | ENV PATH="/opt/venv/bin:$PATH" 209 | 210 | COPY requirements.txt requirements.txt 211 | 212 | RUN \\ 213 | --mount=type=secret,id=netrc,target=/root/.netrc,mode=0600,required=true \\ 214 | python3 -m pip install --upgrade pip && \\ 215 | pip install -r requirements.txt --no-deps 216 | 217 | WORKDIR /opt/resource/ 218 | COPY concourse.py ./concourse.py 219 | RUN python3 -m concoursetools assets . -r concourse.py 220 | 221 | ENTRYPOINT ["python3"] 222 | """).lstrip() 223 | self.assertEqual(dockerfile_contents, expected_contents) 224 | 225 | def test_rsa_config(self) -> None: 226 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", include_rsa=True) 227 | dockerfile_contents = self.dockerfile_path.read_text() 228 | expected_contents = textwrap.dedent(f""" 229 | FROM python:{self.current_python_string} 230 | 231 | RUN python3 -m venv /opt/venv 232 | # Activate venv 233 | ENV PATH="/opt/venv/bin:$PATH" 234 | 235 | COPY requirements.txt requirements.txt 236 | 237 | RUN \\ 238 | --mount=type=secret,id=private_key,target=/root/.ssh/id_rsa,mode=0600,required=true \\ 239 | --mount=type=secret,id=known_hosts,target=/root/.ssh/known_hosts,mode=0644 \\ 240 | python3 -m pip install --upgrade pip && \\ 241 | pip install -r requirements.txt --no-deps 242 | 243 | WORKDIR /opt/resource/ 244 | COPY concourse.py ./concourse.py 245 | RUN python3 -m concoursetools assets . -r concourse.py 246 | 247 | ENTRYPOINT ["python3"] 248 | """).lstrip() 249 | self.assertEqual(dockerfile_contents, expected_contents) 250 | 251 | def test_dev_config(self) -> None: 252 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", dev=True) 253 | dockerfile_contents = self.dockerfile_path.read_text() 254 | expected_contents = textwrap.dedent(f""" 255 | FROM python:{self.current_python_string} 256 | 257 | RUN python3 -m venv /opt/venv 258 | # Activate venv 259 | ENV PATH="/opt/venv/bin:$PATH" 260 | 261 | COPY requirements.txt requirements.txt 262 | 263 | COPY concoursetools concoursetools 264 | 265 | RUN \\ 266 | python3 -m pip install --upgrade pip && \\ 267 | pip install ./concoursetools && \\ 268 | pip install -r requirements.txt --no-deps 269 | 270 | WORKDIR /opt/resource/ 271 | COPY concourse.py ./concourse.py 272 | RUN python3 -m concoursetools assets . -r concourse.py 273 | 274 | ENTRYPOINT ["python3"] 275 | """).lstrip() 276 | self.assertEqual(dockerfile_contents, expected_contents) 277 | -------------------------------------------------------------------------------- /tests/test_importing.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | """ 3 | Tests for the dockertools module. 4 | """ 5 | from collections.abc import Generator 6 | from contextlib import contextmanager 7 | import inspect 8 | import os 9 | from pathlib import Path 10 | import secrets 11 | import sys 12 | from tempfile import TemporaryDirectory 13 | import textwrap 14 | from unittest import TestCase 15 | 16 | from concoursetools import ConcourseResource, additional 17 | from concoursetools.importing import (edit_sys_path, file_path_to_import_path, import_classes_from_module, import_py_file, 18 | import_single_class_from_module) 19 | from tests import resource as test_resource 20 | 21 | 22 | class BasicTests(TestCase): 23 | """ 24 | Tests for the utility functions. 25 | """ 26 | def test_import_path_creation(self) -> None: 27 | file_path = Path("path/to/python.py") 28 | import_path = file_path_to_import_path(file_path) 29 | self.assertEqual(import_path, "path.to.python") 30 | 31 | def test_import_path_creation_wrong_extension(self) -> None: 32 | file_path = Path("path/to/file.txt") 33 | with self.assertRaises(ValueError): 34 | file_path_to_import_path(file_path) 35 | 36 | def test_importing_python_file_by_local_path_same_location(self) -> None: 37 | file_contents = textwrap.dedent(""" 38 | def f(x: int, y: int) -> int: 39 | return x + y 40 | """).lstrip() 41 | 42 | import_path, file_name = _random_python_file() 43 | self.assertNotIn(import_path, sys.modules) 44 | 45 | with TemporaryDirectory() as temp_dir: 46 | py_file = Path(temp_dir) / file_name 47 | py_file.write_text(file_contents) 48 | with _chdir(Path(temp_dir)): 49 | module = import_py_file(import_path, Path(file_name)) 50 | 51 | self.assertEqual(module.f(3, 5), 8) 52 | 53 | def test_importing_python_file_by_full_path_same_location(self) -> None: 54 | file_contents = textwrap.dedent(""" 55 | def f(x: int, y: int) -> int: 56 | return x + y 57 | """).lstrip() 58 | 59 | import_path, file_name = _random_python_file() 60 | self.assertNotIn(import_path, sys.modules) 61 | 62 | with TemporaryDirectory() as temp_dir: 63 | py_file = Path(temp_dir) / file_name 64 | py_file.write_text(file_contents) 65 | with _chdir(Path(temp_dir)): 66 | module = import_py_file(import_path, py_file) 67 | 68 | self.assertEqual(module.f(3, 5), 8) 69 | 70 | def test_importing_python_file_by_full_path_other_location(self) -> None: 71 | file_contents = textwrap.dedent(""" 72 | def f(x: int, y: int) -> int: 73 | return x + y 74 | """).lstrip() 75 | 76 | import_path, file_name = _random_python_file() 77 | self.assertNotIn(import_path, sys.modules) 78 | 79 | with TemporaryDirectory() as temp_dir: 80 | py_file = Path(temp_dir) / file_name 81 | py_file.write_text(file_contents) 82 | 83 | with TemporaryDirectory() as temp_dir_2: 84 | with _chdir(Path(temp_dir_2)): 85 | self.assertNotIn(file_name, os.listdir(".")) 86 | module = import_py_file(import_path, py_file) 87 | 88 | self.assertEqual(module.f(3, 5), 8) 89 | 90 | def test_importing_python_file_pair_by_full_path_other_location(self) -> None: 91 | file_contents = textwrap.dedent(""" 92 | def f(x: int, y: int) -> int: 93 | return x + y 94 | """).lstrip() 95 | 96 | import_path, file_name = _random_python_file() 97 | 98 | second_file_contents = textwrap.dedent(f""" 99 | from {import_path} import f 100 | 101 | def g(x: int, y: int) -> int: 102 | return f(x, 3) + f(y, 4) 103 | """).lstrip() 104 | 105 | second_import_path, second_file_name = _random_python_file() 106 | 107 | self.assertNotIn(import_path, sys.modules) 108 | self.assertNotIn(second_import_path, sys.modules) 109 | 110 | with TemporaryDirectory() as temp_dir: 111 | py_file = Path(temp_dir) / file_name 112 | py_file.write_text(file_contents) 113 | 114 | py_file_2 = Path(temp_dir) / second_file_name 115 | py_file_2.write_text(second_file_contents) 116 | 117 | with TemporaryDirectory() as temp_dir_2: 118 | with _chdir(Path(temp_dir_2)): 119 | self.assertNotIn(file_name, os.listdir(".")) 120 | self.assertNotIn(second_file_name, os.listdir(".")) 121 | module = import_py_file(second_import_path, py_file_2) 122 | 123 | self.assertEqual(module.g(3, 5), 15) 124 | 125 | def test_changing_directory(self) -> None: 126 | current_dir = Path.cwd() 127 | with TemporaryDirectory() as temp_dir: 128 | with _chdir(Path(temp_dir)): 129 | self.assertEqual(Path.cwd(), Path(temp_dir).resolve()) 130 | self.assertEqual(Path.cwd(), current_dir.resolve()) 131 | 132 | def test_changing_directory_nested(self) -> None: 133 | current_dir = Path.cwd() 134 | with TemporaryDirectory() as temp_dir_1: 135 | with TemporaryDirectory() as temp_dir_2: 136 | with _chdir(Path(temp_dir_1)): 137 | self.assertEqual(Path.cwd(), Path(temp_dir_1).resolve()) 138 | with _chdir(Path(temp_dir_2)): 139 | self.assertEqual(Path.cwd(), Path(temp_dir_2).resolve()) 140 | self.assertEqual(Path.cwd(), Path(temp_dir_1).resolve()) 141 | self.assertEqual(Path.cwd(), current_dir.resolve()) 142 | 143 | def test_edit_sys_path(self) -> None: 144 | original_sys_path = sys.path.copy() 145 | with TemporaryDirectory() as temp_dir_1: 146 | with TemporaryDirectory() as temp_dir_2: 147 | self.assertNotIn(temp_dir_1, sys.path) 148 | self.assertNotIn(temp_dir_2, sys.path) 149 | 150 | with edit_sys_path(prepend=[Path(temp_dir_1)], append=[Path(temp_dir_2)]): 151 | self.assertEqual(sys.path[0], temp_dir_1) 152 | self.assertEqual(sys.path[-1], temp_dir_2) 153 | self.assertListEqual(sys.path[1:-1], original_sys_path) 154 | 155 | self.assertNotIn(temp_dir_1, sys.path) 156 | self.assertNotIn(temp_dir_2, sys.path) 157 | 158 | def test_importing_classes(self) -> None: 159 | file_path = Path(additional.__file__).relative_to(Path.cwd()) 160 | resource_classes = import_classes_from_module(file_path, parent_class=ConcourseResource) # type: ignore[type-abstract] 161 | expected = { 162 | "InOnlyConcourseResource": additional.InOnlyConcourseResource, 163 | "OutOnlyConcourseResource": additional.OutOnlyConcourseResource, 164 | "MultiVersionConcourseResource": additional.MultiVersionConcourseResource, 165 | "SelfOrganisingConcourseResource": additional.SelfOrganisingConcourseResource, 166 | "TriggerOnChangeConcourseResource": additional.TriggerOnChangeConcourseResource, 167 | } 168 | self.assertEqual(expected.keys(), resource_classes.keys()) 169 | for key, class_1 in resource_classes.items(): 170 | class_2 = expected[key] 171 | self.assertClassEqual(class_1, class_2) 172 | 173 | def test_importing_class_no_name(self) -> None: 174 | file_path = Path(test_resource.__file__).relative_to(Path.cwd()) 175 | with self.assertRaises(RuntimeError): 176 | import_single_class_from_module(file_path, parent_class=ConcourseResource) # type: ignore[type-abstract] 177 | 178 | def test_importing_class_with_name(self) -> None: 179 | file_path = Path(test_resource.__file__).relative_to(Path.cwd()) 180 | resource_class = import_single_class_from_module(file_path, parent_class=ConcourseResource, # type: ignore[type-abstract] 181 | class_name=test_resource.TestResource.__name__) 182 | self.assertClassEqual(resource_class, test_resource.TestResource) 183 | 184 | def test_importing_class_multiple_options(self) -> None: 185 | file_path = Path(additional.__file__).relative_to(Path.cwd()) 186 | with self.assertRaises(RuntimeError): 187 | import_single_class_from_module(file_path, parent_class=ConcourseResource) # type: ignore[type-abstract] 188 | 189 | def test_importing_class_multiple_options_specify_name(self) -> None: 190 | file_path = Path(additional.__file__).relative_to(Path.cwd()) 191 | parent_class = additional.InOnlyConcourseResource 192 | resource_class = import_single_class_from_module(file_path, parent_class=ConcourseResource, # type: ignore[type-abstract] 193 | class_name=parent_class.__name__) 194 | self.assertClassEqual(resource_class, parent_class) 195 | 196 | def assertClassEqual(self, class_1: type[object], class_2: type[object]) -> None: 197 | self.assertEqual(inspect.getsourcefile(class_1), inspect.getsourcefile(class_2)) 198 | self.assertEqual(inspect.getsource(class_1), inspect.getsource(class_2)) 199 | 200 | 201 | @contextmanager 202 | def _chdir(new_dir: Path) -> Generator[None, None, None]: 203 | original_dir = Path.cwd() 204 | try: 205 | os.chdir(new_dir) 206 | yield 207 | finally: 208 | os.chdir(original_dir) 209 | 210 | 211 | def _random_python_file(num_bytes: int = 4, prefix: str = "test_") -> tuple[str, str]: 212 | import_path = f"{prefix}{secrets.token_hex(num_bytes)}" 213 | file_name = f"{import_path}.py" 214 | return import_path, file_name 215 | -------------------------------------------------------------------------------- /tests/test_metadata.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | import json 3 | from unittest import TestCase 4 | 5 | from concoursetools import BuildMetadata 6 | from concoursetools.metadata import _flatten_dict 7 | from concoursetools.mocking import TestBuildMetadata 8 | from concoursetools.testing import create_env_vars, mock_environ 9 | 10 | 11 | class MetadataTests(TestCase): 12 | """ 13 | Tests for the BuildMetadata class. 14 | """ 15 | def test_normal_build(self) -> None: 16 | metadata = BuildMetadata( 17 | BUILD_ID="12345678", 18 | BUILD_TEAM_NAME="my-team", 19 | ATC_EXTERNAL_URL="https://ci.myconcourse.com", 20 | BUILD_JOB_NAME="my-job", 21 | BUILD_NAME="42", 22 | BUILD_PIPELINE_NAME="my-pipeline", 23 | ) 24 | 25 | self.assertFalse(metadata.is_one_off_build) 26 | self.assertFalse(metadata.is_instanced_pipeline) 27 | self.assertDictEqual(metadata.instance_vars(), {}) 28 | self.assertEqual(metadata.build_url(), "https://ci.myconcourse.com/teams/my-team/pipelines/my-pipeline/jobs/my-job/builds/42") 29 | 30 | def test_normal_build_from_env(self) -> None: 31 | env = create_env_vars() 32 | with mock_environ(env): 33 | metadata = BuildMetadata.from_env() 34 | 35 | self.assertFalse(metadata.is_one_off_build) 36 | self.assertFalse(metadata.is_instanced_pipeline) 37 | self.assertDictEqual(metadata.instance_vars(), {}) 38 | self.assertEqual(metadata.build_url(), "https://ci.myconcourse.com/teams/my-team/pipelines/my-pipeline/jobs/my-job/builds/42") 39 | 40 | def test_instanced_pipeline_build(self) -> None: 41 | metadata = BuildMetadata( 42 | BUILD_ID="12345678", 43 | BUILD_TEAM_NAME="my-team", 44 | ATC_EXTERNAL_URL="https://ci.myconcourse.com", 45 | BUILD_JOB_NAME="my-job", 46 | BUILD_NAME="42", 47 | BUILD_PIPELINE_NAME="my-pipeline", 48 | BUILD_PIPELINE_INSTANCE_VARS="{\"key1\":\"value1\",\"key2\":\"value2\"}", 49 | ) 50 | 51 | self.assertFalse(metadata.is_one_off_build) 52 | self.assertTrue(metadata.is_instanced_pipeline) 53 | self.assertDictEqual(metadata.instance_vars(), {"key1": "value1", "key2": "value2"}) 54 | url = "https://ci.myconcourse.com/teams/my-team/pipelines/my-pipeline/jobs/my-job/builds/42?vars.key1=%22value1%22&vars.key2=%22value2%22" 55 | self.assertEqual(metadata.build_url(), url) 56 | 57 | def test_instanced_pipeline_build_from_env(self) -> None: 58 | instance_vars = {"key1": "value1", "key2": "value2"} 59 | env = create_env_vars(instance_vars=instance_vars) 60 | with mock_environ(env): 61 | metadata = BuildMetadata.from_env() 62 | 63 | self.assertFalse(metadata.is_one_off_build) 64 | self.assertTrue(metadata.is_instanced_pipeline) 65 | self.assertDictEqual(metadata.instance_vars(), instance_vars) 66 | url = r"https://ci.myconcourse.com/teams/my-team/pipelines/my-pipeline/jobs/my-job/builds/42?vars.key1=%22value1%22&vars.key2=%22value2%22" 67 | self.assertEqual(metadata.build_url(), url) 68 | 69 | def test_nested_instanced_pipeline_build(self) -> None: 70 | instance_vars = { 71 | "branch": "feature-v8", 72 | "version": { 73 | "from": "3.0.0", 74 | "main": 2, 75 | "to": "2.0.0", 76 | }, 77 | } 78 | 79 | metadata = BuildMetadata( 80 | BUILD_ID="12345678", 81 | BUILD_TEAM_NAME="my-team", 82 | ATC_EXTERNAL_URL="https://ci.myconcourse.com", 83 | BUILD_JOB_NAME="my-job", 84 | BUILD_NAME="42", 85 | BUILD_PIPELINE_NAME="my-pipeline", 86 | BUILD_PIPELINE_INSTANCE_VARS=json.dumps(instance_vars), 87 | ) 88 | 89 | self.assertFalse(metadata.is_one_off_build) 90 | self.assertTrue(metadata.is_instanced_pipeline) 91 | self.assertDictEqual(metadata.instance_vars(), instance_vars) 92 | url = (r"https://ci.myconcourse.com/teams/my-team/pipelines/my-pipeline/jobs/my-job/builds/42" 93 | r"?vars.branch=%22feature-v8%22&vars.version.from=%223.0.0%22&vars.version.main=2&vars.version.to=%222.0.0%22") 94 | self.assertEqual(metadata.build_url(), url) 95 | 96 | def test_one_off_build(self) -> None: 97 | metadata = BuildMetadata( 98 | BUILD_ID="12345678", 99 | BUILD_TEAM_NAME="my-team", 100 | ATC_EXTERNAL_URL="https://ci.myconcourse.com", 101 | BUILD_NAME="42", 102 | ) 103 | 104 | self.assertTrue(metadata.is_one_off_build) 105 | self.assertFalse(metadata.is_instanced_pipeline) 106 | self.assertDictEqual(metadata.instance_vars(), {}) 107 | self.assertEqual(metadata.build_url(), "https://ci.myconcourse.com/builds/12345678") 108 | 109 | def test_one_off_build_from_env(self) -> None: 110 | env = create_env_vars(one_off_build=True) 111 | with mock_environ(env): 112 | metadata = BuildMetadata.from_env() 113 | 114 | self.assertTrue(metadata.is_one_off_build) 115 | self.assertFalse(metadata.is_instanced_pipeline) 116 | self.assertDictEqual(metadata.instance_vars(), {}) 117 | self.assertEqual(metadata.build_url(), "https://ci.myconcourse.com/builds/12345678") 118 | 119 | def test_flattening_nested_dict(self) -> None: 120 | nested_dict = { 121 | "branch": "feature-v8", 122 | "version": { 123 | "from": "3.0.0", 124 | "main": 2, 125 | "to": "2.0.0" 126 | }, 127 | } 128 | flattened_dict = { 129 | "branch": "feature-v8", 130 | "version.from": "3.0.0", 131 | "version.main": 2, 132 | "version.to": "2.0.0", 133 | } 134 | self.assertDictEqual(_flatten_dict(nested_dict), flattened_dict) 135 | 136 | def test_flattening_double_nested_dict(self) -> None: 137 | nested_dict = { 138 | "branch": "feature-v8", 139 | "version": { 140 | "main": 2, 141 | "parents": { 142 | "from": "3.0.0", 143 | "to": "2.0.0", 144 | }, 145 | }, 146 | } 147 | flattened_dict = { 148 | "branch": "feature-v8", 149 | "version.main": 2, 150 | "version.parents.from": "3.0.0", 151 | "version.parents.to": "2.0.0", 152 | } 153 | self.assertDictEqual(_flatten_dict(nested_dict), flattened_dict) 154 | 155 | 156 | class MetadataFormattingTests(TestCase): 157 | """ 158 | Tests for the BuildMetadata.format_string method. 159 | """ 160 | def setUp(self) -> None: 161 | """Code to run before each test.""" 162 | self.metadata = TestBuildMetadata() 163 | 164 | def test_no_interpolation(self) -> None: 165 | new_string = self.metadata.format_string("This is a normal string.") 166 | self.assertEqual(new_string, "This is a normal string.") 167 | 168 | def test_simple_interpolation(self) -> None: 169 | new_string = self.metadata.format_string("The build id is $BUILD_ID and the job name is $BUILD_JOB_NAME.") 170 | self.assertEqual(new_string, "The build id is 12345678 and the job name is my-job.") 171 | 172 | def test_interpolation_with_one_off(self) -> None: 173 | metadata = TestBuildMetadata(one_off_build=True) 174 | new_string = metadata.format_string("The build id is $BUILD_ID and the job name is $BUILD_JOB_NAME.") 175 | self.assertEqual(new_string, "The build id is 12345678 and the job name is .") 176 | 177 | def test_interpolation_incorrect_value(self) -> None: 178 | with self.assertRaises(KeyError): 179 | self.metadata.format_string("The build id is $OTHER.") 180 | 181 | def test_interpolation_incorrect_value_ignore_missing(self) -> None: 182 | new_string = self.metadata.format_string("The build id is $OTHER.", ignore_missing=True) 183 | self.assertEqual(new_string, "The build id is $OTHER.") 184 | 185 | def test_interpolation_with_additional(self) -> None: 186 | new_string = self.metadata.format_string("The build id is $OTHER.", additional_values={"OTHER": "value"}, 187 | ignore_missing=True) 188 | self.assertEqual(new_string, "The build id is value.") 189 | -------------------------------------------------------------------------------- /tests/test_testing.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | from pathlib import Path 3 | from tempfile import TemporaryDirectory 4 | from typing import ClassVar 5 | from unittest import TestCase 6 | 7 | from concoursetools.testing import TemporaryDirectoryState 8 | 9 | 10 | class FolderDictReadTests(TestCase): 11 | temp_dir: ClassVar[TemporaryDirectory[str]] 12 | root: ClassVar[Path] 13 | 14 | @classmethod 15 | def setUpClass(cls) -> None: 16 | cls.temp_dir = TemporaryDirectory() 17 | cls.root = Path(cls.temp_dir.name) 18 | 19 | folder_1 = cls.root / "folder_1" 20 | folder_2 = cls.root / "folder_2" 21 | folder_3 = folder_2 / "folder_3" 22 | 23 | folder_1.mkdir() 24 | folder_2.mkdir() 25 | folder_3.mkdir() 26 | 27 | file_1 = cls.root / "file_1" 28 | file_2 = folder_2 / "file_2" 29 | file_3 = folder_3 / "file_3" 30 | 31 | file_1.write_text("Testing 1\n") 32 | file_2.write_text("Testing 2\n") 33 | file_3.write_text("Testing 3\n") 34 | 35 | @classmethod 36 | def tearDownClass(cls) -> None: 37 | cls.temp_dir.cleanup() 38 | 39 | def test_folder_dict_depth_1(self) -> None: 40 | folder_dict = TemporaryDirectoryState()._get_folder_as_dict(self.root, max_depth=1) 41 | expected = { 42 | "folder_1": ..., 43 | "folder_2": ..., 44 | "file_1": "Testing 1\n", 45 | } 46 | self.assertDictEqual(folder_dict, expected) 47 | 48 | def test_folder_dict_depth_2(self) -> None: 49 | folder_dict = TemporaryDirectoryState()._get_folder_as_dict(self.root, max_depth=2) 50 | expected = { 51 | "folder_1": {}, 52 | "folder_2": { 53 | "folder_3": ..., 54 | "file_2": "Testing 2\n", 55 | }, 56 | "file_1": "Testing 1\n", 57 | } 58 | self.assertDictEqual(folder_dict, expected) 59 | 60 | def test_folder_dict_depth_3(self) -> None: 61 | folder_dict = TemporaryDirectoryState()._get_folder_as_dict(self.root, max_depth=3) 62 | expected = { 63 | "folder_1": {}, 64 | "folder_2": { 65 | "folder_3": { 66 | "file_3": "Testing 3\n", 67 | }, 68 | "file_2": "Testing 2\n", 69 | }, 70 | "file_1": "Testing 1\n", 71 | } 72 | self.assertDictEqual(folder_dict, expected) 73 | 74 | 75 | class FolderDictWriteTests(TestCase): 76 | temp_dir: ClassVar[TemporaryDirectory[str]] 77 | root: ClassVar[Path] 78 | 79 | @classmethod 80 | def setUpClass(cls) -> None: 81 | cls.temp_dir = TemporaryDirectory() 82 | cls.root = Path(cls.temp_dir.name) 83 | 84 | @classmethod 85 | def tearDownClass(cls) -> None: 86 | cls.temp_dir.cleanup() 87 | 88 | def test_folder_dict_depth_3(self) -> None: 89 | original = { 90 | "folder_1": {}, 91 | "folder_2": { 92 | "folder_3": { 93 | "file_3": "Testing 3\n", 94 | }, 95 | "file_2": "Testing 2\n", 96 | }, 97 | "file_1": "Testing 1\n", 98 | } 99 | TemporaryDirectoryState()._set_folder_from_dict(self.root, original) 100 | final_dict = TemporaryDirectoryState()._get_folder_as_dict(self.root, max_depth=3) 101 | self.assertDictEqual(final_dict, original) 102 | -------------------------------------------------------------------------------- /tests/test_version.py: -------------------------------------------------------------------------------- 1 | # (C) Crown Copyright GCHQ 2 | from dataclasses import dataclass 3 | from datetime import datetime 4 | from enum import Enum 5 | from pathlib import Path 6 | from unittest import TestCase 7 | 8 | from concoursetools import Version 9 | from concoursetools.typing import VersionConfig 10 | from concoursetools.version import SortableVersionMixin, TypedVersion 11 | 12 | 13 | class BasicVersion(Version): 14 | 15 | def __init__(self, file_path: str) -> None: 16 | self.file_path = file_path 17 | 18 | 19 | class CreationTests(TestCase): 20 | """ 21 | Tests for the creation of an instance. 22 | """ 23 | def test_base_version(self) -> None: 24 | with self.assertRaises(TypeError): 25 | Version() # type: ignore[abstract] 26 | 27 | 28 | class ComplexVersion(BasicVersion, SortableVersionMixin): 29 | 30 | def __eq__(self, other: object) -> bool: 31 | if not isinstance(other, type(self)): 32 | return NotImplemented 33 | return self.file_name == other.file_name 34 | 35 | def __lt__(self, other: object) -> bool: 36 | if not isinstance(other, type(self)): 37 | return NotImplemented 38 | return self.file_path < other.file_path 39 | 40 | @property 41 | def file_name(self) -> str: 42 | return Path(self.file_path).name 43 | 44 | 45 | class ComparisonTests(TestCase): 46 | 47 | def test_repr(self) -> None: 48 | version_1 = BasicVersion("file.txt") 49 | self.assertEqual(repr(version_1), "BasicVersion(file_path='file.txt')") 50 | 51 | def test_sortable_mixin_with_version_with_abstract(self) -> None: 52 | class MyVersion(Version, SortableVersionMixin): 53 | 54 | def __init__(self, file_path: str) -> None: 55 | self.file_path = file_path 56 | 57 | def __lt__(self, other: object) -> bool: 58 | if not isinstance(other, type(self)): 59 | return NotImplemented 60 | return self.file_path < other.file_path 61 | 62 | self.assertLess(MyVersion("aaa"), MyVersion("bbb")) 63 | 64 | def test_sortable_mixin_with_version_without_abstract(self) -> None: 65 | class MyVersion(Version, SortableVersionMixin): 66 | 67 | def __init__(self, file_path: str) -> None: 68 | self.file_path = file_path 69 | 70 | with self.assertRaises(TypeError): 71 | MyVersion("aaa") # type: ignore[abstract] 72 | 73 | def test_sortable_mixin_with_typed_version_with_abstract(self) -> None: 74 | @dataclass 75 | class MyTypedVersion(TypedVersion, SortableVersionMixin): 76 | file_path: str 77 | 78 | def __lt__(self, other: object) -> bool: 79 | if not isinstance(other, type(self)): 80 | return NotImplemented 81 | return self.file_path < other.file_path 82 | 83 | self.assertLess(MyTypedVersion("aaa"), MyTypedVersion("bbb")) 84 | 85 | def test_sortable_mixin_with_typed_version_without_abstract(self) -> None: 86 | @dataclass 87 | class MyTypedVersion(TypedVersion, SortableVersionMixin): 88 | file_path: str 89 | 90 | with self.assertRaises(TypeError): 91 | MyTypedVersion("aaa") # type: ignore[abstract] 92 | 93 | def test_default_equality(self) -> None: 94 | version_1 = BasicVersion("file.txt") 95 | version_1_again = BasicVersion("file.txt") 96 | version_2 = BasicVersion("folder/file.txt") 97 | version_3 = BasicVersion("image.png") 98 | 99 | self.assertEqual(version_1, version_1_again) 100 | self.assertNotEqual(version_1, version_2) 101 | self.assertNotEqual(version_1, version_3) 102 | self.assertNotEqual(version_2, version_3) 103 | 104 | def test_complex_equality(self) -> None: 105 | version_1 = ComplexVersion("file.txt") 106 | version_1_again = ComplexVersion("file.txt") 107 | version_2 = ComplexVersion("folder/file.txt") 108 | version_3 = ComplexVersion("image.png") 109 | 110 | self.assertEqual(version_1, version_1_again) 111 | self.assertEqual(version_1, version_2) 112 | self.assertNotEqual(version_1, version_3) 113 | self.assertNotEqual(version_2, version_3) 114 | 115 | def test_default_sorting(self) -> None: 116 | version_1 = BasicVersion("file.txt") 117 | version_2 = BasicVersion("folder/file.txt") 118 | version_3 = BasicVersion("image.png") 119 | 120 | with self.assertRaises(TypeError): 121 | sorted([version_1, version_2, version_3]) # type: ignore[type-var] 122 | 123 | def test_complex_sorting(self) -> None: 124 | version_1 = ComplexVersion("file.txt") 125 | version_2 = ComplexVersion("folder/file.txt") 126 | version_3 = ComplexVersion("image.png") 127 | 128 | self.assertListEqual(sorted([version_3, version_1, version_2]), [version_1, version_2, version_3]) 129 | 130 | def test_complex_comparisons(self) -> None: 131 | version_1 = ComplexVersion("file.txt") 132 | version_2 = ComplexVersion("folder/file.txt") 133 | self.assertLess(version_1, version_2) 134 | self.assertGreater(version_2, version_1) 135 | 136 | self.assertLessEqual(version_1, version_2) 137 | self.assertGreaterEqual(version_2, version_1) 138 | self.assertLessEqual(version_1, version_1) 139 | 140 | 141 | class CommitVersion(Version): 142 | 143 | def __init__(self, commit_hash: str, is_merge: bool) -> None: 144 | self.commit_hash = commit_hash 145 | self.is_merge = is_merge 146 | 147 | 148 | @dataclass 149 | class TypedCommitVersion(TypedVersion): 150 | commit_hash: str 151 | date: datetime 152 | is_merge: bool 153 | 154 | 155 | class CommitVersionImproved(CommitVersion): 156 | 157 | @classmethod 158 | def from_flat_dict(cls, version_dict: VersionConfig) -> "CommitVersionImproved": 159 | is_merge = (version_dict["is_merge"] == "True") 160 | return cls(version_dict["commit_hash"], is_merge) 161 | 162 | 163 | class DictTests(TestCase): 164 | """ 165 | Tests for the conversion between version and dict. 166 | """ 167 | def test_non_strings(self) -> None: 168 | version = CommitVersion("abcdef", True) 169 | flat_dict = version.to_flat_dict() 170 | self.assertDictEqual(flat_dict, { 171 | "commit_hash": "abcdef", 172 | "is_merge": "True" 173 | }) 174 | new_version = CommitVersion.from_flat_dict(flat_dict) 175 | self.assertEqual(new_version.commit_hash, "abcdef") 176 | self.assertEqual(new_version.is_merge, "True") 177 | 178 | better_new_version = CommitVersionImproved.from_flat_dict(flat_dict) 179 | self.assertEqual(better_new_version.commit_hash, "abcdef") 180 | self.assertEqual(better_new_version.is_merge, True) 181 | 182 | def test_private_attribute(self) -> None: 183 | 184 | class CommitVersionPrivate(CommitVersion): 185 | 186 | def __init__(self, commit_hash: str, is_merge: bool): 187 | super().__init__(commit_hash, is_merge) 188 | self._force_push = True 189 | 190 | version = CommitVersionPrivate("abcdef", True) 191 | flat_dict = version.to_flat_dict() 192 | self.assertDictEqual(flat_dict, { 193 | "commit_hash": "abcdef", 194 | "is_merge": "True" 195 | }) 196 | 197 | 198 | class MyEnum(Enum): 199 | ONE = 1 200 | TWO = 2 201 | 202 | 203 | class TypedTests(TestCase): 204 | 205 | def test_flattened_and_unflattened_types(self) -> None: 206 | expected = { 207 | "string": "string", 208 | "42": 42, 209 | "True": True, 210 | "False": False, 211 | "1577881800": datetime(2020, 1, 1, 12, 30), 212 | "ONE": MyEnum.ONE, 213 | "/path/to/somewhere": Path("/path/to/somewhere"), 214 | } 215 | for flattened_obj, obj in expected.items(): 216 | with self.subTest(obj=obj): 217 | self.assertEqual(TypedVersion._flatten_object(obj), flattened_obj) 218 | un_flattened_obj = TypedVersion._un_flatten_object(type(obj), flattened_obj) 219 | self.assertEqual(type(un_flattened_obj), type(obj)) 220 | self.assertEqual(un_flattened_obj, obj) 221 | 222 | def test_flattened_and_unflattened_version(self) -> None: 223 | version = TypedCommitVersion("abcdef", datetime(2020, 1, 1, 12, 30), False) 224 | flattened = version.to_flat_dict() 225 | self.assertDictEqual(flattened, { 226 | "commit_hash": "abcdef", 227 | "date": "1577881800", 228 | "is_merge": "False", 229 | }) 230 | self.assertEqual(TypedCommitVersion.from_flat_dict(flattened), version) 231 | 232 | def test_implementing_empty_version(self) -> None: 233 | with self.assertRaises(TypeError): 234 | @dataclass 235 | class _(TypedVersion): 236 | pass 237 | 238 | def test_missing_dataclass(self) -> None: 239 | with self.assertRaises(TypeError): 240 | class _(TypedVersion): 241 | pass 242 | --------------------------------------------------------------------------------