├── .github
└── workflows
│ ├── documentation.yaml
│ ├── pre-commit.yaml
│ └── unit-tests.yaml
├── .gitignore
├── .pre-commit-config.yaml
├── .pre-commit-hooks
└── double_string_fixer.py
├── .readthedocs.yaml
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE.md
├── README.md
├── SECURITY.md
├── concoursetools
├── __init__.py
├── __main__.py
├── additional.py
├── cli
│ ├── __init__.py
│ ├── commands.py
│ ├── docstring.py
│ └── parser.py
├── colour.py
├── dockertools.py
├── importing.py
├── metadata.py
├── mocking.py
├── parsing.py
├── py.typed
├── resource.py
├── testing.py
├── typing.py
└── version.py
├── cspell.config.yaml
├── docs
├── requirements.txt
└── source
│ ├── _static
│ ├── favicon.png
│ ├── logo-dark.png
│ ├── logo.png
│ ├── step_metadata.png
│ └── style.css
│ ├── additional.rst
│ ├── api_reference.rst
│ ├── build_metadata.rst
│ ├── changelog.rst
│ ├── changelog
│ ├── 0.8.0.rst
│ └── development.rst
│ ├── cli.rst
│ ├── cli_reference.rst
│ ├── conf.py
│ ├── debugging.rst
│ ├── deployment.rst
│ ├── dockertools.rst
│ ├── examples
│ ├── branches.rst
│ ├── build_status.rst
│ ├── pipeline.rst
│ ├── s3.rst
│ ├── secrets.rst
│ └── xkcd.rst
│ ├── extensions
│ ├── __init__.py
│ ├── cli.py
│ ├── concourse.py
│ ├── linecount.py
│ ├── wikipedia.py
│ └── xkcd.py
│ ├── importing.rst
│ ├── index.rst
│ ├── internals.rst
│ ├── main_scripts.rst
│ ├── mocking.rst
│ ├── parsing.rst
│ ├── quickstart.rst
│ ├── resource.rst
│ ├── testing.rst
│ ├── typing.rst
│ ├── version.rst
│ └── wrappers.rst
├── examples
├── __init__.py
├── build_status.py
├── github_branches.py
├── pipeline.py
├── s3.py
├── secrets.py
├── xkcd.py
└── xkcd.xml
├── pyproject.toml
├── requirements-tests.txt
├── setup.py
├── sonar-project.properties
└── tests
├── __init__.py
├── resource.py
├── test_additional.py
├── test_cli_commands.py
├── test_cli_parsing.py
├── test_dockertools.py
├── test_examples.py
├── test_importing.py
├── test_metadata.py
├── test_parsing.py
├── test_resource.py
├── test_testing.py
└── test_version.py
/.github/workflows/documentation.yaml:
--------------------------------------------------------------------------------
1 | name: Build Documentation
2 | on:
3 | pull_request:
4 | branches: [main]
5 | jobs:
6 | build:
7 | runs-on: ubuntu-latest
8 | steps:
9 | - uses: actions/checkout@v3
10 | - name: Set up Python 3.12
11 | uses: actions/setup-python@v3
12 | with:
13 | python-version: "3.12"
14 | - name: Install dependencies
15 | run: |
16 | python -m pip install --upgrade pip
17 | python -m pip install -r docs/requirements.txt --no-deps
18 | - name: Build documentation
19 | run: |
20 | python -m sphinx -b html -aE -n -W --keep-going docs/source docs/build
21 | - name: Check external links
22 | run: |
23 | python -m sphinx -b linkcheck docs/source docs/build
24 |
--------------------------------------------------------------------------------
/.github/workflows/pre-commit.yaml:
--------------------------------------------------------------------------------
1 | name: Run Pre-commit
2 | on:
3 | pull_request:
4 | branches: [main]
5 | jobs:
6 | build:
7 | runs-on: ubuntu-latest
8 | steps:
9 | - uses: actions/checkout@v3
10 | - name: Set up Python 3.12
11 | uses: actions/setup-python@v3
12 | with:
13 | python-version: "3.12"
14 | - name: Install pre-commit
15 | run: |
16 | python -m pip install --upgrade pip
17 | python -m pip install pre-commit
18 | - name: Run pre-commit checks
19 | run: |
20 | python -m pre_commit run --all-files --color always
21 |
--------------------------------------------------------------------------------
/.github/workflows/unit-tests.yaml:
--------------------------------------------------------------------------------
1 | name: Unit Tests
2 | on:
3 | pull_request:
4 | branches: [main]
5 | jobs:
6 | build:
7 | runs-on: ubuntu-latest
8 | strategy:
9 | fail-fast: false
10 | matrix:
11 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14.0-beta.2"]
12 | steps:
13 | - uses: actions/checkout@v3
14 | - name: Set up Python ${{ matrix.python-version }}
15 | uses: actions/setup-python@v3
16 | with:
17 | python-version: ${{ matrix.python-version }}
18 | - name: Pull docker image
19 | run: |
20 | docker pull concourse/mock-resource:0.13.0
21 | - name: Install dependencies
22 | run: |
23 | python -m pip install --upgrade pip
24 | python -m pip install -r requirements-tests.txt --no-deps
25 | - name: Run tests
26 | run: |
27 | python -W error -m unittest --verbose
28 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__
2 | *.egg-info
3 | .venv*/
4 | .vscode
5 | docs/build
6 | .DS_Store
7 | .mypy_cache
8 | .coverage
9 | coverage.xml
10 | .scannerwork
11 | .env
12 | dist/
13 |
--------------------------------------------------------------------------------
/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
1 | repos:
2 | - repo: https://github.com/pre-commit/pre-commit-hooks
3 | rev: v5.0.0
4 | hooks:
5 | - id: trailing-whitespace
6 | - id: end-of-file-fixer
7 | - id: check-yaml
8 | - id: check-added-large-files
9 | args: [--enforce-all, --maxkb=100]
10 | - repo: https://github.com/google/yamlfmt
11 | rev: v0.17.0
12 | hooks:
13 | - id: yamlfmt
14 | - repo: https://github.com/hhatto/autopep8
15 | rev: v2.3.2
16 | hooks:
17 | - id: autopep8
18 | exclude: docs/source/conf.py
19 | - repo: https://github.com/pycqa/isort
20 | rev: 6.0.1
21 | hooks:
22 | - id: isort
23 | - repo: https://github.com/jumanjihouse/pre-commit-hooks
24 | rev: 3.0.0
25 | hooks:
26 | - id: require-ascii
27 | - id: script-must-have-extension
28 | - id: forbid-binary
29 | exclude: docs/source/_static/.*.png
30 | - repo: https://github.com/Lucas-C/pre-commit-hooks
31 | rev: v1.5.5
32 | hooks:
33 | - id: forbid-crlf
34 | - id: forbid-tabs
35 | - repo: https://github.com/streetsidesoftware/cspell-cli
36 | rev: v9.0.1
37 | hooks:
38 | - id: cspell
39 | exclude: \.gitignore|.*\.properties
40 | - repo: https://github.com/pycqa/flake8
41 | rev: 7.2.0
42 | hooks:
43 | - id: flake8
44 | args: [--max-line-length=150]
45 | exclude: docs/source/conf.py
46 | - repo: https://github.com/pycqa/pydocstyle
47 | rev: 6.3.0
48 | hooks:
49 | - id: pydocstyle
50 | additional_dependencies: ["tomli"]
51 | exclude: examples|tests|.pre-commit-hooks|conf.py
52 | - repo: https://github.com/regebro/pyroma
53 | rev: "4.2"
54 | hooks:
55 | - id: pyroma
56 | args: ["-d", "--min=10", "."]
57 | - repo: https://github.com/pre-commit/mirrors-mypy
58 | rev: v1.16.0
59 | hooks:
60 | - id: mypy
61 | additional_dependencies: ["types-docutils", "types-python-dateutil", "types-requests"]
62 | - repo: "https://github.com/python-jsonschema/check-jsonschema"
63 | rev: "0.33.0"
64 | hooks:
65 | - id: check-github-workflows
66 | - repo: https://github.com/johannsdg/pre-commit-license-headers
67 | rev: d53087d331942f263cb553dc67b0e51ffa3a3481
68 | hooks:
69 | - id: check-license-headers
70 | types: [python]
71 | args:
72 | - --template
73 | - (C) Crown Copyright [OWNER]
74 | - --owner=GCHQ
75 | - repo: local
76 | hooks:
77 | - id: double-string-fixer
78 | name: string fixer
79 | description: Replace single-quoted strings with double-quoted strings
80 | types: [python]
81 | language: python
82 | entry: python3 .pre-commit-hooks/double_string_fixer.py
83 |
--------------------------------------------------------------------------------
/.pre-commit-hooks/double_string_fixer.py:
--------------------------------------------------------------------------------
1 | """
2 | A copy of https://github.com/pre-commit/pre-commit-hooks/tree/main#double-quote-string-fixer,
3 | except it replaces single-quoted strings with double-quoted strings.
4 |
5 | Copyright (c) 2014 pre-commit dev team: Anthony Sottile, Ken Struys
6 |
7 | Permission is hereby granted, free of charge, to any person obtaining a copy
8 | of this software and associated documentation files (the "Software"), to deal
9 | in the Software without restriction, including without limitation the rights
10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 | copies of the Software, and to permit persons to whom the Software is
12 | furnished to do so, subject to the following conditions:
13 |
14 | The above copyright notice and this permission notice shall be included in
15 | all copies or substantial portions of the Software.
16 |
17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23 | THE SOFTWARE.
24 | """
25 | from __future__ import annotations
26 |
27 | import argparse
28 | import io
29 | import re
30 | import sys
31 | import tokenize
32 | from typing import Sequence
33 |
34 | if sys.version_info >= (3, 12): # pragma: >=3.12 cover
35 | FSTRING_START = tokenize.FSTRING_START
36 | FSTRING_END = tokenize.FSTRING_END
37 | else: # pragma: <3.12 cover
38 | FSTRING_START = FSTRING_END = -1
39 |
40 | START_QUOTE_RE = re.compile("^[a-zA-Z]*'")
41 |
42 |
43 | def handle_match(token_text: str) -> str:
44 | if '"""' in token_text or "'''" in token_text:
45 | return token_text
46 |
47 | match = START_QUOTE_RE.match(token_text)
48 | if match is not None:
49 | meat = token_text[match.end():-1]
50 | if '"' in meat or "'" in meat:
51 | return token_text
52 | else:
53 | return match.group().replace("'", '"') + meat + '"'
54 | else:
55 | return token_text
56 |
57 |
58 | def get_line_offsets_by_line_no(src: str) -> list[int]:
59 | # Padded so we can index with line number
60 | offsets = [-1, 0]
61 | for line in src.splitlines(True):
62 | offsets.append(offsets[-1] + len(line))
63 | return offsets
64 |
65 |
66 | def fix_strings(filename: str) -> int:
67 | with open(filename, encoding="UTF-8", newline="") as f:
68 | contents = f.read()
69 | line_offsets = get_line_offsets_by_line_no(contents)
70 |
71 | # Basically a mutable string
72 | splitcontents = list(contents)
73 |
74 | fstring_depth = 0
75 |
76 | # Iterate in reverse so the offsets are always correct
77 | tokens_l = list(tokenize.generate_tokens(io.StringIO(contents).readline))
78 | tokens = reversed(tokens_l)
79 | for token_type, token_text, (srow, scol), (erow, ecol), _ in tokens:
80 | if token_type == FSTRING_START: # pragma: >=3.12 cover
81 | fstring_depth += 1
82 | elif token_type == FSTRING_END: # pragma: >=3.12 cover
83 | fstring_depth -= 1
84 | elif fstring_depth == 0 and token_type == tokenize.STRING:
85 | new_text = handle_match(token_text)
86 | splitcontents[
87 | line_offsets[srow] + scol:
88 | line_offsets[erow] + ecol
89 | ] = new_text
90 |
91 | new_contents = "".join(splitcontents)
92 | if contents != new_contents:
93 | with open(filename, "w", encoding="UTF-8", newline="") as f:
94 | f.write(new_contents)
95 | return 1
96 | else:
97 | return 0
98 |
99 |
100 | def main(argv: Sequence[str] | None = None) -> int:
101 | parser = argparse.ArgumentParser()
102 | parser.add_argument("filenames", nargs="*", help="Filenames to fix")
103 | args = parser.parse_args(argv)
104 |
105 | retv = 0
106 |
107 | for filename in args.filenames:
108 | return_value = fix_strings(filename)
109 | if return_value != 0:
110 | print(f"Fixing strings in {filename}")
111 | retv |= return_value
112 |
113 | return retv
114 |
115 |
116 | if __name__ == "__main__":
117 | raise SystemExit(main())
118 |
--------------------------------------------------------------------------------
/.readthedocs.yaml:
--------------------------------------------------------------------------------
1 | # Read the Docs configuration file for Sphinx projects
2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
3 | version: 2
4 | build:
5 | os: ubuntu-22.04
6 | tools:
7 | python: "3.12"
8 | sphinx:
9 | configuration: docs/source/conf.py
10 | builder: "html"
11 | fail_on_warning: true
12 | python:
13 | install:
14 | - requirements: docs/requirements.txt
15 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Community Code of Conduct
2 |
3 |
4 | ## Our Pledge
5 |
6 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make
7 | participation in our project and our community a harassment-free experience for everyone, regardless of age, body size,
8 | disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education,
9 | socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
10 |
11 |
12 | ## Our Standards
13 |
14 | * Being open. Members of the community are open to collaboration.
15 | * Focusing on what is best for the community. We're respectful of the processes set forth in the community, and we work
16 | within them.
17 | * Acknowledging time and effort. We're respectful of the volunteer efforts that permeate the community. We're thoughtful
18 | when addressing the efforts of others, keeping in mind that often times the labour was completed simply for the good
19 | of the community.
20 | * Being respectful of differing viewpoints and experiences. We're receptive to constructive comments and criticism, as
21 | the experiences and skill sets of other members contribute to the whole of our efforts.
22 | * Being considerate towards other community members. We're attentive in our communications and we're tactful when
23 | approaching differing views.
24 | * Using welcoming and inclusive language. We're accepting of all who wish to take part in our activities, fostering an
25 | environment where anyone can participate and everyone can make a difference.
26 | * Take responsibility for our words and our actions. We can all make mistakes; when we do, we take responsibility for
27 | them. If someone has been harmed or offended, we listen carefully and respectfully, and work to right the wrong.
28 | * Step down considerately. When somebody leaves or disengages from the project, we ask that they do so in a way that
29 | minimises disruption to the project.
30 | * Ask for help when unsure. Nobody is expected to be perfect in this community. Asking questions early avoids many
31 | problems later, so questions are encouraged, though they may be directed to the appropriate forum. Those who are asked
32 | should be responsive and helpful.
33 | * Value decisiveness, clarity and consensus. Disagreements, social and technical, are normal, but we do not allow them
34 | to persist and fester leaving others uncertain of the agreed direction. We expect participants in the project to
35 | resolve disagreements constructively. When they cannot, we escalate the matter to structures with designated leaders
36 | to arbitrate and provide clarity and direction.
37 |
38 |
39 | ## Our Responsibilities
40 |
41 | Project maintainers are responsible for clarifying the standards of acceptable behaviour and are expected to take
42 | appropriate and fair corrective action in response to any instances of unacceptable behaviour. Project maintainers have
43 | the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other
44 | contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for
45 | other behaviour that they deem inappropriate, threatening, offensive, or harmful.
46 |
47 |
48 | ## Enforcement
49 |
50 | Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported to the community leaders
51 | responsible for enforcement at [oss@gchq.gov.uk](mailto:oss@gchq.gov.uk). All complaints will be reviewed and
52 | investigated promptly and fairly, and will result in a response that is deemed necessary and appropriate to the
53 | circumstances. The community leaders responsible for enforcement are obligated to maintain confidentiality with regard
54 | to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
55 |
56 |
57 | ## Attribution
58 |
59 | This Code of Conduct has been adapted with modifications from the Contributor Covenant (version 1.4), the Python Code of
60 | conduct and the Ubuntu Code of Conduct (version 2.0).
61 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing to Concourse Tools
2 |
3 | If you find any issues/bugs in the software, please [file an issue](https://github.com/gchq/ConcourseTools/issues).
4 | Please provide full details of your issue, and ideally code to reproduce it.
5 |
6 |
7 | ## Pull Requests
8 |
9 | Prior to us accepting any work, you must sign the [GCHQ CLA Agreement](https://cla-assistant.io/gchq/ConcourseTools).
10 | We follow a branching strategy for handling contributions:
11 |
12 | 1. Fork the Project
13 | 2. Create your Feature Branch (`git checkout -b feature/new_thing`)
14 | 3. Commit your Changes (`git commit -m 'Add a new thing'`)
15 | 4. Push to the Branch (`git push origin feature/new_thing`)
16 | 5. Open a Pull Request
17 |
18 | ### Pre-commit
19 |
20 | Please make use of the [pre-commit checks](https://pre-commit.com/), which should be installed before any new code is
21 | committed:
22 |
23 | ```shell
24 | $ python3 -m pip install pre-commit
25 | $ pre-commit install
26 | ```
27 |
28 | To run the checks at any time:
29 |
30 | ```shell
31 | $ pre-commit run --all-files
32 | ```
33 |
34 | Please do not add any new pre-commit checks yourself and instead raise a ticket.
35 |
36 | ### Tests
37 |
38 | Before opening a pull request, please ensure that the tests pass. To do this, run the following:
39 |
40 | ```shell
41 | $ python3 -m unittest discover .
42 | ```
43 |
44 | Tests are written with `unittest` and - with the exception of the [example tests](tests/test_examples.py) - do not
45 | require any additional dependencies. To run the example tests you should install the additional dependencies:
46 |
47 | ```shell
48 | $ python3 -m pip install -r requirements-tests.txt --no-deps
49 | ```
50 |
51 | ### Mypy
52 |
53 | Please also run a type check with [Mypy](https://github.com/python/mypy):
54 |
55 | ```shell
56 | $ python3 -m pip install mypy
57 | $ python3 -m mypy concoursetools
58 | ```
59 |
60 | ### Documentation
61 |
62 | The documentation for Concourse Tools use [Sphinx](https://www.sphinx-doc.org/en/master/index.html). Please ensure that
63 | new features are appropriately documented before opening your pull request. To build the documentation locally, first
64 | install the dependencies:
65 |
66 | ```shell
67 | $ python3 -m pip install -r docs/requirements.txt
68 | ```
69 |
70 | Next, run the following:
71 |
72 | ```shell
73 | $ python3 -m sphinx -b html -aE docs/source docs/build
74 | ```
75 |
76 | Please also make use of the [linkcheck builder](https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-the-linkcheck-builder):
77 |
78 | ```shell
79 | $ python3 -m sphinx -b linkcheck docs/source docs/build # check that all links resolve
80 | ```
81 |
82 |
83 | ## Coding Standards and Conventions
84 |
85 | Concourse Tools is a fully-typed library, so please ensure all functions, methods and classes are fully typed.
86 |
87 | Concourse Tools uses [Sphinx-style docstrings](https://sphinx-rtd-tutorial.readthedocs.io/en/latest/docstrings.html).
88 |
89 | This project aims to depend only on the standard library, so contributions which add additional dependencies outside of
90 | the standard library are likely to be rejected unless absolutely necessary.
91 |
92 |
93 | ### Style Guide
94 |
95 | Concourse Tools does not have an explicit style guide outside of the pre-commit checks for enforcing PEP8 and double-quote strings.
96 | However, please ensure your code is as readable and clear as possible. Reviewers will highlight any code changes they feel
97 | is inconsistent or difficult to parse.
98 |
99 | Please refrain from running Black over the code, as it can cause readability issues for nested statements.
100 |
101 | Finally, please do not refactor "other people's" code when submitting a pull request.
102 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | ## (C) Crown Copyright GCHQ
2 |
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
6 |
7 | Unless required by applicable law or agreed to in writing, software
8 | distributed under the License is distributed on an "AS IS" BASIS,
9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | See the License for the specific language governing permissions and
11 | limitations under the License.
12 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | 
8 | 
9 | 
10 | 
11 | 
12 |
13 | A Python package for easily implementing Concourse [resource types](https://concourse-ci.org/implementing-resource-types.html).
14 |
15 |
16 | ## About
17 |
18 | [Concourse CI](https://concourse-ci.org/) is an "open-source continuous thing-doer" designed to enable general
19 | automation with intuitive and re-usable components. Resources represent all external inputs and outputs to and from the
20 | pipeline, and many of these have been implemented in open source. In order to best leverage the Python ecosystem of
21 | open-source packages, Concourse Tools abstracts away the implementation details of Concourse resource types to allow
22 | users to focus on writing the code they want to run.
23 |
24 |
25 | ## Installation
26 |
27 | Install from [GitHub](https://github.com/gchq/ConcourseTools/), or from [PyPI](https://pypi.org/project/concoursetools/):
28 |
29 | ```shell
30 | $ pip install concoursetools
31 | ```
32 |
33 | ## Usage
34 |
35 | Start by familiarising yourself with the Concourse resource "rules" in the [documentation](https://concourse-ci.org/implementing-resource-types.html). To recreate that example, start by creating a new `concourse.py` file in your repository. The first step is to create a `Version` subclass:
36 |
37 | ```python
38 | from dataclasses import dataclass
39 | from concoursetools import TypedVersion
40 |
41 |
42 | @dataclass()
43 | class GitVersion(TypedVersion):
44 | ref: str
45 | ```
46 |
47 | Next, create a subclass of `ConcourseResource`:
48 |
49 | ```python
50 | from concoursetools import ConcourseResource
51 |
52 |
53 | class GitResource(ConcourseResource[GitVersion]):
54 |
55 | def __init__(self, uri: str, branch: str, private_key: str) -> None:
56 | super().__init__(GitVersion)
57 | self.uri = uri
58 | self.branch = branch
59 | self.private_key = private_key
60 | ```
61 |
62 | Here, the parameters in the `__init__` method will be taken from the `source` configuration for the resource.
63 | Now, implement the three methods required to define the behaviour of the resource:
64 |
65 |
66 | ```python
67 | from pathlib import Path
68 | from typing import Any
69 | from concoursetools import BuildMetadata
70 |
71 |
72 | class GitResource(ConcourseResource[GitVersion]):
73 | ...
74 |
75 | def fetch_new_versions(self, previous_version: GitVersion | None) -> list[GitVersion]:
76 | ...
77 |
78 | def download_version(self, version: GitVersion, destination_dir: pathlib.Path,
79 | build_metadata: BuildMetadata, **kwargs: Any) -> tuple[GitVersion, dict[str, str]]:
80 | ...
81 |
82 | def publish_new_version(self, sources_dir: pathlib.Path, build_metadata: BuildMetadata,
83 | **kwargs: Any) -> tuple[GitVersion, dict[str, str]]:
84 | ...
85 | ```
86 |
87 | The keyword arguments in `download_version` and `publish_new_version` correspond to `params` in the `get` step,
88 | and `get_params` in the `put` step respectively.
89 |
90 | Once you are happy with the resource, generate the `Dockerfile` using the Concourse Tools CLI:
91 |
92 | ```shell
93 | $ python3 -m concoursetools dockerfile .
94 | ```
95 |
96 | Finally, upload the Docker image to a registry, and use it in your pipelines!
97 |
98 | For more information - and for more in-depth examples - see the [documentation](https://concoursetools.readthedocs.io/en/stable/).
99 |
100 |
101 | ## Bugs and Contributions
102 |
103 | Concourse Tools is in beta, and still under somewhat-active development. Contributions, fixes, suggestions and bug
104 | reports are all welcome: Please familiarise yourself with our
105 | [contribution guidelines](https://github.com/gchq/ConcourseTools/blob/main/CONTRIBUTING.md).
106 |
--------------------------------------------------------------------------------
/SECURITY.md:
--------------------------------------------------------------------------------
1 | # Security Policy
2 |
3 |
4 | ## Supported Versions
5 |
6 | Concourse Tools is supported on a best endeavours basis. Patches will be applied to the latest version rather than
7 | retroactively to older versions. To ensure you are using the most secure version of Concoursetools, please make sure you
8 | have installed the [latest version](https://pypi.org/project/concoursetools/).
9 |
10 |
11 | ## Reporting a Vulnerability
12 |
13 | Disclosures of vulnerabilities in Concourse Tools are always welcome. Whilst we aim to write clean and secure code free
14 | from bugs, we recognise that this is an open source project, relying on other of open source libraries that are modified
15 | and updated on a regular basis. We hope that the community will continue to support us as we endeavour to maintain and
16 | develop this tool together.
17 |
18 | If you believe that you have identified a potential vulnerability in the code base, please report this promptly to
19 | [oss@gchq.gov.uk](mailto:oss@gchq.gov.uk). Please describe the problem in as much detail as possible, ideally with
20 | examples. Each report will be dealt with on a case-by-case basis. You will receive regular communication on the
21 | resolution and progress of your report.
22 |
--------------------------------------------------------------------------------
/concoursetools/__init__.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | A Python package for easily implementing Concourse resource types.
4 | """
5 | from concoursetools.metadata import BuildMetadata
6 | from concoursetools.resource import ConcourseResource
7 | from concoursetools.version import TypedVersion, Version
8 |
9 | __all__ = ("BuildMetadata", "ConcourseResource", "Version", "TypedVersion")
10 | __author__ = "GCHQ"
11 | __version__ = "0.8.0"
12 |
--------------------------------------------------------------------------------
/concoursetools/__main__.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | The main entrypoint for the Concourse Tools CLI:
4 |
5 | $ python3 -m concoursetools --help
6 | """
7 | import sys
8 |
9 | from concoursetools.cli import cli
10 |
11 |
12 | def main() -> int:
13 | """Run the main function."""
14 | cli.invoke(sys.argv[1:])
15 | return 0
16 |
17 |
18 | if __name__ == "__main__":
19 | main()
20 |
--------------------------------------------------------------------------------
/concoursetools/cli/__init__.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Contains the Concourse Tools CLI.
4 | """
5 | from __future__ import annotations
6 |
7 | from concoursetools.cli.commands import cli
8 |
9 | __all__ = ("cli",)
10 |
--------------------------------------------------------------------------------
/concoursetools/cli/commands.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Commands for the Concourse Tools CLI.
4 | """
5 | from __future__ import annotations
6 |
7 | from pathlib import Path
8 | import sys
9 | import textwrap
10 |
11 | from concoursetools import dockertools
12 | from concoursetools.cli.parser import CLI
13 | from concoursetools.colour import Colour, colour_print
14 | from concoursetools.importing import import_single_class_from_module
15 | from concoursetools.resource import ConcourseResource
16 |
17 | cli = CLI()
18 |
19 | DEFAULT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
20 |
21 |
22 | @cli.register(allow_short={"executable", "class_name", "resource_file"})
23 | def assets(path: str, /, *, executable: str | None = None, resource_file: str = "concourse.py",
24 | class_name: str | None = None) -> None:
25 | """
26 | Create the assets script directory.
27 |
28 | :param path: The location at which to the script files will be written.
29 | Pass '.' to write scripts to the current directory.
30 | :param executable: The python executable to place at the top of the file. Defaults to '/usr/bin/env python3'.
31 | :param resource_file: The path to the module containing the resource class. Defaults to 'concourse.py'.
32 | :param class_name: The name of the resource class in the module, if there are multiple.
33 | """
34 | resource_class = import_single_class_from_module(Path(resource_file), parent_class=ConcourseResource, # type: ignore[type-abstract]
35 | class_name=class_name)
36 | assets_folder = Path(path)
37 | assets_folder.mkdir(parents=True, exist_ok=True)
38 |
39 | file_name_to_method_name: dict[dockertools.ScriptName, dockertools.MethodName] = {
40 | "check": "check_main",
41 | "in": "in_main",
42 | "out": "out_main",
43 | }
44 | for file_name, method_name in file_name_to_method_name.items():
45 | file_path = assets_folder / file_name
46 | dockertools.create_script_file(file_path, resource_class, method_name,
47 | executable or dockertools.DEFAULT_EXECUTABLE)
48 |
49 |
50 | @cli.register(allow_short={"executable", "class_name", "resource_file"})
51 | def dockerfile(path: str, /, *, executable: str | None = None, image: str = "python", tag: str | None = None,
52 | suffix: str | None = None, resource_file: str = "concourse.py", class_name: str | None = None,
53 | pip_args: str | None = None, include_rsa: bool = False, include_netrc: bool = False,
54 | encoding: str | None = None, no_venv: bool = False, dev: bool = False) -> None:
55 | """
56 | Create the Dockerfile.
57 |
58 | :param path: The location to which to write the Dockerfile.
59 | Pass '.' to write it to the current directory.
60 | :param executable: The python executable to place at the top of the file. Defaults to '/usr/bin/env python3'.
61 | :param image: Specify the image used in the FROM instruction.
62 | :param tag: The tag to combine with the image. Defaults to the major/minor version of the current Python environment.
63 | :param suffix: An optional suffix to combine with the tag to create the full tag.
64 | :param resource_file: The path to the module containing the resource class. Defaults to 'concourse.py'.
65 | :param class_name: The name of the resource class in the module, if there are multiple.
66 | :param pip_args: An optional string to be appended to all calls to pip, e.g. '--timeout 100'.
67 | :param include_rsa: Enable the Dockerfile to (securely) use your RSA private key during building.
68 | :param include_netrc: Enable the Dockerfile to (securely) use your netrc file during building.
69 | :param encoding: The encoding of the created file. If not passed, Concourse Tools will use the user's default encoding.
70 | :param no_venv: Pass to explicitly not use a virtual environment within the image. This is not recommended and exists
71 | to ensure legacy behaviour.
72 | :param dev: Pass to copy a local version of Concourse Tools to the image, instead of installing from PyPI.
73 | The onus is on the user to ensure that the "concoursetools" exists in the working directory at
74 | Docker build time.
75 | """
76 | directory_path = Path(path)
77 | if directory_path.is_dir():
78 | file_path = directory_path / "Dockerfile"
79 | else:
80 | file_path = directory_path
81 |
82 | assets_to_potentially_include = {
83 | "-c": class_name,
84 | "-e": executable,
85 | }
86 | assets_options = {key: value for key, value in assets_to_potentially_include.items() if value is not None}
87 |
88 | cli_split_command = ["python3", "-m", "concoursetools", "assets", ".", "-r", resource_file]
89 | for key, value in assets_options.items():
90 | if value is not None:
91 | cli_split_command.extend([key, value])
92 |
93 | cli_command = " ".join(cli_split_command)
94 | if tag is None:
95 | if suffix is None:
96 | tag = DEFAULT_PYTHON_VERSION
97 | else:
98 | tag = f"{DEFAULT_PYTHON_VERSION}-{suffix}"
99 |
100 | final_dockerfile = dockertools.Dockerfile()
101 |
102 | final_dockerfile.new_instruction_group(
103 | dockertools.FromInstruction(image=image, tag=tag),
104 | )
105 |
106 | if no_venv:
107 | if dev:
108 | raise ValueError("Can only specify --no-venv in production mode")
109 | else:
110 | final_dockerfile.new_instruction_group(
111 | dockertools.RunInstruction(["python3 -m venv /opt/venv"]),
112 | dockertools.Comment("Activate venv"),
113 | dockertools.EnvInstruction({"PATH": "/opt/venv/bin:$PATH"}),
114 | )
115 |
116 | final_dockerfile.new_instruction_group(
117 | dockertools.CopyInstruction("requirements.txt"),
118 | )
119 |
120 | mounts: list[dockertools.Mount] = []
121 |
122 | if include_rsa:
123 | mounts.extend([
124 | dockertools.SecretMount(
125 | secret_id="private_key",
126 | target="/root/.ssh/id_rsa",
127 | mode=0o600,
128 | required=True,
129 | ),
130 | dockertools.SecretMount(
131 | secret_id="known_hosts",
132 | target="/root/.ssh/known_hosts",
133 | mode=0o644,
134 | ),
135 | ])
136 |
137 | if include_netrc:
138 | mounts.extend([
139 | dockertools.SecretMount(
140 | secret_id="netrc",
141 | target="/root/.netrc",
142 | mode=0o600,
143 | required=True,
144 | ),
145 | ])
146 |
147 | if pip_args is None:
148 | pip_string_suffix = ""
149 | else:
150 | pip_string_suffix = f" {pip_args}"
151 |
152 | if dev:
153 | final_dockerfile.new_instruction_group(
154 | dockertools.CopyInstruction("concoursetools"),
155 | )
156 | final_dockerfile.new_instruction_group(
157 | dockertools.MultiLineRunInstruction([
158 | "python3 -m pip install --upgrade pip" + pip_string_suffix,
159 | "pip install ./concoursetools",
160 | "pip install -r requirements.txt --no-deps" + pip_string_suffix,
161 | ], mounts=mounts),
162 | )
163 | else:
164 | final_dockerfile.new_instruction_group(
165 | dockertools.MultiLineRunInstruction([
166 | "python3 -m pip install --upgrade pip" + pip_string_suffix,
167 | "pip install -r requirements.txt --no-deps" + pip_string_suffix,
168 | ], mounts=mounts),
169 | )
170 |
171 | final_dockerfile.new_instruction_group(
172 | dockertools.WorkDirInstruction("/opt/resource/"),
173 | dockertools.CopyInstruction(resource_file, f"./{resource_file}"),
174 | dockertools.RunInstruction([cli_command]),
175 | )
176 |
177 | final_dockerfile.new_instruction_group(
178 | dockertools.EntryPointInstruction(["python3"]),
179 | )
180 |
181 | final_dockerfile.write_to_file(file_path, encoding=encoding)
182 |
183 |
184 | @cli.register(allow_short={"executable", "class_name", "resource_file"})
185 | def legacy(path: str, /, *, executable: str | None = None, resource_file: str = "concourse.py",
186 | class_name: str | None = None, docker: bool = False, include_rsa: bool = False) -> None:
187 | """
188 | Invoke the legacy CLI.
189 |
190 | :param path: The location at which to place the scripts.
191 | :param executable: The python executable to place at the top of the file. Defaults to '/usr/bin/env python3'.
192 | :param resource_file: The path to the module containing the resource class. Defaults to 'concourse.py'.
193 | :param class_name: The name of the resource class in the module, if there are multiple.
194 | :param docker: Pass to create a skeleton Dockerfile at the path instead.
195 | :param include_rsa: Enable the Dockerfile to (securely) use your RSA private key during building.
196 | """
197 | colour_print(textwrap.dedent("""
198 | The legacy CLI has been deprecated.
199 | Please refer to the documentation or help pages for the up to date CLI.
200 | This CLI will be removed in version 0.10.0, or in version 1.0.0, whichever is sooner.
201 | """), colour=Colour.RED)
202 | if docker:
203 | return dockerfile(path, suffix="alpine", executable=executable, resource_file=resource_file, class_name=class_name,
204 | include_rsa=include_rsa, no_venv=True)
205 | assets(path, executable=executable, resource_file=resource_file, class_name=class_name)
206 |
--------------------------------------------------------------------------------
/concoursetools/cli/docstring.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Concourse Tools uses a custom CLI tool for easier management of command line functions.
4 | """
5 | from __future__ import annotations
6 |
7 | from collections.abc import Callable, Generator
8 | from dataclasses import dataclass
9 | import inspect
10 | import re
11 | from typing import TypeVar
12 |
13 | RE_PARAM = re.compile(r":param (.*?):")
14 | RE_WHITESPACE = re.compile(r"\s+")
15 |
16 | CLIFunction = Callable[..., None]
17 | CLIFunctionT = TypeVar("CLIFunctionT", bound=CLIFunction)
18 | T = TypeVar("T")
19 |
20 |
21 | @dataclass(frozen=True)
22 | class Docstring:
23 | """
24 | Represents a function docstring.
25 |
26 | :param first_line: The first line of the docstring (separated by double whitespace).
27 | :param description: The remaining description before any parameters.
28 | :param parameters: A mapping of parameter name to description, with newlines replaced with whitespace.
29 | """
30 | first_line: str
31 | description: str
32 | parameters: dict[str, str]
33 |
34 | @classmethod
35 | def from_object(cls, obj: object) -> "Docstring":
36 | """
37 | Parse an object with a docstring.
38 |
39 | :param obj: An object which may have a docstring, such as a function or module.
40 | """
41 | raw_docstring = inspect.getdoc(obj) or ""
42 | return cls.from_string(raw_docstring)
43 |
44 | @classmethod
45 | def from_string(cls, raw_docstring: str) -> "Docstring":
46 | """
47 | Parse a docstring.
48 |
49 | :param raw_docstring: The raw docstring of an object.
50 | """
51 | try:
52 | first_line, remaining_lines = raw_docstring.split("\n\n", maxsplit=1)
53 | except ValueError:
54 | if RE_PARAM.match(raw_docstring.strip()): # all we have are params with no description
55 | first_line = ""
56 | remaining_lines = raw_docstring.strip()
57 | else:
58 | first_line = raw_docstring.strip()
59 | return cls(first_line, "", {})
60 |
61 | description, *remaining_params = RE_PARAM.split(remaining_lines.lstrip())
62 | parameters = {param: " ".join(info.split()).strip() for param, info in _pair_up(remaining_params)}
63 | return cls(first_line, description.strip(), parameters)
64 |
65 |
66 | def _pair_up(data: list[str]) -> Generator[tuple[str, str], None, None]:
67 | """
68 | Pair up a list of items.
69 |
70 | :param data: A list to be paired.
71 | :raises ValueError: If there are an odd number of values in the list.
72 |
73 | :Example:
74 | >>> list(_pair_up([1, 2, 3, 4, 5, 6]))
75 | [(1, 2), (3, 4), (5, 6)]
76 | """
77 | for i in range(0, len(data), 2):
78 | try:
79 | yield data[i], data[i + 1]
80 | except IndexError:
81 | raise ValueError(f"Needed an even number of values, got {len(data)}")
82 |
--------------------------------------------------------------------------------
/concoursetools/colour.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | The Concourse web UI will interpret :wikipedia:`ANSI colour codes `,
4 | and so a handful of rudimentary functions for formatting with colour are include in
5 | :mod:`concoursetools.colour`.
6 |
7 | .. tip::
8 | There are plenty of more mature libraries for printing coloured output,
9 | such as `termcolor `_ and
10 | `Rich `_.
11 | Concourse Tools specifically has no external dependencies, and so these
12 | must be actively installed and managed by a user.
13 | """
14 | from __future__ import annotations
15 |
16 | from collections.abc import Generator
17 | from contextlib import contextmanager
18 | from typing import Any
19 |
20 | END_COLOUR = "\033[0m"
21 | BOLD = "\033[1m"
22 | UNDERLINE = "\033[4m"
23 |
24 |
25 | class _NoPrint(str):
26 |
27 | def __str__(self) -> str:
28 | raise TypeError("Can't print!")
29 |
30 | def __repr__(self) -> str:
31 | return f"{type(self).__name__}()"
32 |
33 |
34 | MISSING_COLOUR = _NoPrint()
35 |
36 |
37 | def colourise(string: str, colour: str) -> str:
38 | """
39 | Convert a string into a string which will be coloured on print.
40 |
41 | This enables coloured output within an f-string or similar. It is not recommended for
42 | colouring complete strings, as it is far less efficient than the other functions.
43 |
44 | :param string: The string to be colourised.
45 | :param colour: The :wikipedia:`ANSI colour escape code `
46 | for the required colour.
47 |
48 | :Example:
49 |
50 | >>> print(f"Hello {colourise('world', colour=Colour.RED)}")
51 | """
52 | return f"{colour}{string}{END_COLOUR}"
53 |
54 |
55 | def colour_print(*values: object, colour: str = MISSING_COLOUR, bold: bool = False, underline: bool = False,
56 | **print_kwargs: Any) -> None:
57 | """
58 | Print something in colour.
59 |
60 | This function behaves exactly like :func:`print`, just with more functionality:
61 |
62 | :param colour: The :wikipedia:`ANSI colour escape code ` for the required colour.
63 | :param bold: Print the text in **bold**.
64 | :param underline: Print the text with an underline.
65 |
66 | :Example:
67 |
68 | >>> colour_print(1, 2, 3, sep="-", colour=Colour.GREEN)
69 | """
70 | try:
71 | with print_in_colour(colour, bold=bold, underline=underline):
72 | print(*values, **print_kwargs)
73 | except TypeError as error:
74 | if colour is MISSING_COLOUR:
75 | raise ValueError("You forgot to pass the colour as a keyword argument") from error
76 | raise
77 |
78 |
79 | @contextmanager
80 | def print_in_colour(colour: str, bold: bool = False, underline: bool = False) -> Generator[None, None, None]:
81 | """
82 | Print anything in colour within a :ref:`context manager `.
83 |
84 | This is especially useful for colourising output from other external functions
85 | which you cannot control.
86 |
87 | :param colour: The :wikipedia:`ANSI colour escape code ` for the required colour.
88 | :param bold: Print the text in **bold**.
89 | :param underline: Print the text with an underline.
90 |
91 | :Example:
92 |
93 | >>> with print_in_colour(Colour.BLUE, bold=True):
94 | ... print("Hello!")
95 | """
96 | try:
97 | print(colour, end="")
98 | if bold:
99 | print(BOLD, end="")
100 | if underline:
101 | print(UNDERLINE, end="")
102 | yield
103 | finally:
104 | print(END_COLOUR, end="")
105 |
106 |
107 | class Colour:
108 | """
109 | A few common ANSI colours.
110 | """
111 | BLUE = "\033[94m"
112 | CYAN = "\033[96m"
113 | GREEN = "\033[92m"
114 | PURPLE = "\033[95m"
115 | RED = "\033[91m"
116 | YELLOW = "\033[93m"
117 |
--------------------------------------------------------------------------------
/concoursetools/importing.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Functions for dynamically importing from Python modules.
4 | """
5 | from __future__ import annotations
6 |
7 | from collections.abc import Generator, Sequence
8 | from contextlib import contextmanager
9 | import importlib.util
10 | import inspect
11 | from pathlib import Path
12 | import sys
13 | from types import ModuleType
14 | from typing import TypeVar
15 |
16 | T = TypeVar("T")
17 |
18 |
19 | def import_single_class_from_module(file_path: Path, parent_class: type[T], class_name: str | None = None) -> type[T]:
20 | """
21 | Import the resource class from the module.
22 |
23 | Similar to :func:`import_classes_from_module`, but ensures only one class is returned.
24 |
25 | :param file_path: The location of the module as a file path.
26 | :param class_name: The name of the class to extract. Required if multiple are returned.
27 | :param parent_class: All subclasses of this class defined within the module
28 | (not imported from elsewhere) will be extracted.
29 | :returns: The extracted class.
30 | :raises RuntimeError: If too many or too few classes are available in the module, unless the class name is specified.
31 | """
32 | possible_resource_classes = import_classes_from_module(file_path, parent_class=parent_class)
33 |
34 | if class_name is None:
35 | if len(possible_resource_classes) == 1:
36 | _, resource_class = possible_resource_classes.popitem()
37 | else:
38 | if len(possible_resource_classes) == 0:
39 | raise RuntimeError(f"No subclasses of {parent_class.__name__!r} found in {file_path}")
40 | raise RuntimeError(f"Multiple subclasses of {parent_class.__name__!r} found in {file_path}:"
41 | f" {set(possible_resource_classes)}")
42 | else:
43 | resource_class = possible_resource_classes[class_name]
44 |
45 | return resource_class
46 |
47 |
48 | def import_classes_from_module(file_path: Path, parent_class: type[T]) -> dict[str, type[T]]:
49 | """
50 | Import all available resource classes from the module.
51 |
52 | :param file_path: The location of the module as a file path.
53 | :param parent_class: All subclasses of this class defined within the module
54 | (not imported from elsewhere) will be extracted.
55 | :returns: A mapping of class name to class.
56 | """
57 | import_path = file_path_to_import_path(file_path)
58 | module = import_py_file(import_path, file_path)
59 |
60 | possible_resource_classes = {}
61 | for _, cls in inspect.getmembers(module, predicate=inspect.isclass):
62 | try:
63 | class_is_subclass_of_parent = issubclass(cls, parent_class)
64 | except TypeError:
65 | class_is_subclass_of_parent = False
66 |
67 | class_is_defined_in_this_module = (cls.__module__ == import_path)
68 | class_is_not_private = (not cls.__name__.startswith("_"))
69 |
70 | if class_is_subclass_of_parent and class_is_defined_in_this_module and class_is_not_private:
71 | possible_resource_classes[cls.__name__] = cls
72 |
73 | return possible_resource_classes
74 |
75 |
76 | def file_path_to_import_path(file_path: Path) -> str:
77 | """
78 | Convert a file path to an import path.
79 |
80 | :param file_path: The path to a Python file.
81 | :raises ValueError: If the path doesn't end in a '.py' extension.
82 |
83 | :Example:
84 | >>> file_path_to_import_path(Path("module.py"))
85 | 'module'
86 | >>> file_path_to_import_path(Path("path/to/module.py"))
87 | 'path.to.module'
88 | """
89 | *path_components, file_name = file_path.parts
90 | module_name, extension = file_name.split(".")
91 | if extension != "py":
92 | raise ValueError(f"{file_path!r} does not appear to be a valid Python module")
93 |
94 | path_components.append(module_name)
95 | import_path = ".".join(path_components)
96 | return import_path
97 |
98 |
99 | def import_py_file(import_path: str, file_path: Path) -> ModuleType:
100 | """
101 | Import a .py file as a module.
102 |
103 | This is done using a :ref:`standard Python recipe ` via :mod:`importlib.util`.
104 |
105 | :param import_path: The import path added to :data:`sys.modules`.
106 | :param file_path: The path to the .py module.
107 | :returns: The imported module.
108 | :raises FileNotFoundError: If the path does not exist.
109 | """
110 | try:
111 | spec = importlib.util.spec_from_file_location(import_path, file_path)
112 | if spec is None:
113 | raise RuntimeError("Imported module spec is unexpectedly 'None'")
114 | if spec.loader is None:
115 | raise RuntimeError("Imported module spec loader is unexpectedly 'None'")
116 |
117 | module = importlib.util.module_from_spec(spec)
118 | with edit_sys_path(prepend=[file_path.parent]):
119 | sys.modules[import_path] = module
120 | spec.loader.exec_module(module)
121 | except ModuleNotFoundError as error:
122 | if not file_path.exists():
123 | raise FileNotFoundError(file_path) from error
124 | raise
125 |
126 | return module
127 |
128 |
129 | @contextmanager
130 | def edit_sys_path(prepend: Sequence[Path] = (), append: Sequence[Path] = ()) -> Generator[None, None, None]:
131 | """
132 | Temporarily add to :data:`sys.path` within a context manager.
133 |
134 | :param prepend: A sequence of paths to add to :data:`sys.path` *before* the current entries.
135 | :param append: A sequence of paths to add to :data:`sys.path` *after* the current entries.
136 | :seealso: This is used to enable local imports for the :func:`import_py_file`.
137 | """
138 | original_sys_path = sys.path.copy() # otherwise we just reference the original
139 | try:
140 | sys.path = [str(path) for path in prepend] + sys.path + [str(path) for path in append]
141 | yield
142 | finally:
143 | sys.path = original_sys_path
144 |
--------------------------------------------------------------------------------
/concoursetools/py.typed:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gchq/ConcourseTools/bb28027115429e0379c3dfa038754408eaa08c2b/concoursetools/py.typed
--------------------------------------------------------------------------------
/concoursetools/typing.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Concourse Tools contains a small number of additional types for easier resource development.
4 |
5 | .. warning::
6 | Although Concourse Tools is a typed library, the :class:`concoursetools.resource.ConcourseResource`
7 | class breaks the :wikipedia:`Liskov substitution principle`. If you wish to utilise type hinting
8 | in your development process, then please switch off the check for this type of method overloading.
9 | """
10 | from __future__ import annotations
11 |
12 | from collections.abc import Callable
13 | from typing import Any, Protocol, TypedDict, TypeVar
14 |
15 | T = TypeVar("T")
16 |
17 |
18 | ResourceConfig = dict[str, Any]
19 | """
20 | Represents arbitrary configuration passed to a Concourse resource.
21 | See the :concourse:`config-basics.schema.config` schema for more information.
22 | """
23 |
24 | Params = dict[str, Any]
25 | """
26 | Represents arbitrary parameters passed to a Concourse resource.
27 | See the :concourse:`config-basics.schema.config` schema for more information.
28 | """
29 |
30 | Metadata = dict[str, str]
31 | """
32 | Represents :ref:`Step Metadata` as used by Concourse Tools.
33 | Restrictions on key and value types are determined by :data:`MetadataPair`.
34 | """
35 |
36 |
37 | class MetadataPair(TypedDict):
38 | """
39 | Represents :ref:`Step Metadata` as used internally by Concourse.
40 |
41 | Restrictions on key and value types are determined by the
42 | `Go structure itself `_.
43 | """
44 | name: str
45 | value: str
46 |
47 |
48 | VersionConfig = dict[str, str]
49 | """
50 | Represents a version of a Concourse resource.
51 | See the :concourse:`config-basics.schema.version` schema for more information.
52 | """
53 |
54 | VersionT = TypeVar("VersionT", bound="VersionProtocol")
55 | """Represents a generic :class:`~concoursetools.version.Version` subclass."""
56 |
57 | TypedVersionT = TypeVar("TypedVersionT", bound="TypedVersionProtocol")
58 | """Represents a generic :class:`~concoursetools.version.TypedVersion` subclass."""
59 |
60 | SortableVersionT = TypeVar("SortableVersionT", bound="SortableVersionProtocol")
61 | """Represents a generic :class:`~concoursetools.version.Version` subclass which is also :ref:`sortable `."""
62 |
63 | SortableVersionCovariantT = TypeVar("SortableVersionCovariantT", bound="SortableVersionProtocol", covariant=True)
64 |
65 |
66 | class VersionProtocol(Protocol):
67 | """Corresponds to a generic :class:`~concoursetools.version.Version` subclass."""
68 | def __repr__(self) -> str:
69 | ...
70 |
71 | def __eq__(self, other: object) -> bool:
72 | ...
73 |
74 | def __hash__(self) -> int:
75 | ...
76 |
77 | def to_flat_dict(self) -> VersionConfig:
78 | ...
79 |
80 | @classmethod
81 | def from_flat_dict(cls: type[VersionT], version_dict: VersionConfig) -> VersionT:
82 | ...
83 |
84 |
85 | class SortableVersionProtocol(Protocol):
86 | """Corresponds to a generic :class:`~concoursetools.version.Version` subclass which is also :ref:`sortable `."""
87 | def __repr__(self) -> str:
88 | ...
89 |
90 | def __eq__(self, other: object) -> bool:
91 | ...
92 |
93 | def __hash__(self) -> int:
94 | ...
95 |
96 | def to_flat_dict(self) -> VersionConfig:
97 | ...
98 |
99 | @classmethod
100 | def from_flat_dict(cls: type[VersionT], version_dict: VersionConfig) -> VersionT:
101 | ...
102 |
103 | def __lt__(self, other: object) -> bool:
104 | ...
105 |
106 | def __le__(self, other: object) -> bool:
107 | ...
108 |
109 |
110 | class TypedVersionProtocol(Protocol):
111 | """Corresponds to a generic :class:`~concoursetools.version.TypedVersion` subclass."""
112 | def __repr__(self) -> str:
113 | ...
114 |
115 | def __eq__(self, other: object) -> bool:
116 | ...
117 |
118 | def __hash__(self) -> int:
119 | ...
120 |
121 | def to_flat_dict(self) -> VersionConfig:
122 | ...
123 |
124 | @classmethod
125 | def from_flat_dict(cls: type[VersionT], version_dict: VersionConfig) -> VersionT:
126 | ...
127 |
128 | @classmethod
129 | def _flatten_object(cls, obj: Any) -> str:
130 | ...
131 |
132 | @classmethod
133 | def _un_flatten_object(cls, type_: type[TypedVersionT], flat_obj: str) -> TypedVersionT:
134 | ...
135 |
136 | @classmethod
137 | def _get_attribute_type(cls, attribute_name: str) -> type[Any]:
138 | ...
139 |
140 | @classmethod
141 | def flatten(cls, func: Callable[[T], str]) -> Callable[[T], str]:
142 | ...
143 |
144 | @classmethod
145 | def un_flatten(cls, func: Callable[[type[T], str], T]) -> Callable[[type[T], str], T]:
146 | ...
147 |
148 | @staticmethod
149 | def _flatten_default(obj: object) -> str:
150 | ...
151 |
152 | @staticmethod
153 | def _un_flatten_default(type_: type[T], flat_obj: str) -> T:
154 | ...
155 |
156 |
157 | class MultiVersionProtocol(Protocol[SortableVersionCovariantT]):
158 | """Corresponds to a generic :class:`~concoursetools.additional.MultiVersion` subclass."""
159 | def __init__(self, versions: set[SortableVersionCovariantT]):
160 | ...
161 |
162 | def __repr__(self) -> str:
163 | ...
164 |
165 | def __eq__(self, other: object) -> bool:
166 | ...
167 |
168 | def __hash__(self) -> int:
169 | ...
170 |
171 | @property
172 | def key(self) -> str:
173 | ...
174 |
175 | @property
176 | def sub_version_class(self) -> type[SortableVersionCovariantT]:
177 | ...
178 |
179 | @property
180 | def sub_version_data(self) -> list[VersionConfig]:
181 | ...
182 |
183 | def to_flat_dict(self) -> VersionConfig:
184 | ...
185 |
186 | @classmethod
187 | def from_flat_dict(cls: type[VersionT], version_dict: VersionConfig) -> VersionT:
188 | ...
189 |
--------------------------------------------------------------------------------
/cspell.config.yaml:
--------------------------------------------------------------------------------
1 | version: "0.2"
2 | language: en-GB
3 | useGitignore: true
4 | ignorePaths:
5 | - tests
6 | - requirements*.txt
7 | - .pre-commit-config.yaml
8 | - .pre-commit-hooks/*
9 | dictionaries:
10 | - python
11 | words:
12 | - autoclass
13 | - autodata
14 | - autofunction
15 | - automethod
16 | - automodule
17 | - autosectionlabel
18 | - AWSCURRENT
19 | - boto
20 | - botocore
21 | - bysource
22 | - classmethod
23 | - Colors
24 | - colspec
25 | - concoursetools
26 | - dataclass
27 | - dataclasses
28 | - datetime
29 | - dateutil
30 | - Dockerfiles
31 | - dockertools
32 | - docstrings
33 | - elif
34 | - furo
35 | - GCHQ
36 | - gchqdev
37 | - genindex
38 | - HEALTHCHECK
39 | - INPROGRESS
40 | - intersphinx
41 | - kwargs
42 | - linecount
43 | - linenos
44 | - linkcheck
45 | - Liskov
46 | - literalinclude
47 | - maxdepth
48 | - maxsplit
49 | - modindex
50 | - mypy
51 | - ncsc
52 | - ONBUILD
53 | - outdir
54 | - pathlib
55 | - pyobject
56 | - Quickstart
57 | - redef
58 | - refid
59 | - refuri
60 | - rstrip
61 | - rwxr
62 | - seealso
63 | - srcset
64 | - STOPSIGNAL
65 | - stringifying
66 | - termcolor
67 | - tgroup
68 | - thead
69 | - tmpfs
70 | - toctree
71 | - typehints
72 | - undoc
73 | - unitless
74 | - urllib
75 | - vars
76 | - versionadded
77 | - versionchanged
78 | - viewcode
79 | - WORKDIR
80 | - xkcd
81 | ignoreWords:
82 | - cbef
83 |
--------------------------------------------------------------------------------
/docs/requirements.txt:
--------------------------------------------------------------------------------
1 | alabaster==1.0.0
2 | babel==2.17.0
3 | beautifulsoup4==4.13.3
4 | certifi==2025.1.31
5 | charset-normalizer==3.4.1
6 | docutils==0.21.2
7 | furo==2024.8.6
8 | idna==3.10
9 | imagesize==1.4.1
10 | Jinja2==3.1.5
11 | MarkupSafe==3.0.2
12 | packaging==24.2
13 | Pygments==2.19.1
14 | requests==2.32.3
15 | roman-numerals-py==3.0.0
16 | snowballstemmer==2.2.0
17 | soupsieve==2.6
18 | Sphinx==8.2.0
19 | sphinx-autodoc-typehints==3.1.0
20 | sphinx-basic-ng==1.0.0b2
21 | sphinxcontrib-applehelp==2.0.0
22 | sphinxcontrib-devhelp==2.0.0
23 | sphinxcontrib-htmlhelp==2.1.0
24 | sphinxcontrib-jsmath==1.0.1
25 | sphinxcontrib-qthelp==2.0.0
26 | sphinxcontrib-serializinghtml==2.0.0
27 | typing_extensions==4.12.2
28 | urllib3==2.3.0
29 |
--------------------------------------------------------------------------------
/docs/source/_static/favicon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gchq/ConcourseTools/bb28027115429e0379c3dfa038754408eaa08c2b/docs/source/_static/favicon.png
--------------------------------------------------------------------------------
/docs/source/_static/logo-dark.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gchq/ConcourseTools/bb28027115429e0379c3dfa038754408eaa08c2b/docs/source/_static/logo-dark.png
--------------------------------------------------------------------------------
/docs/source/_static/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gchq/ConcourseTools/bb28027115429e0379c3dfa038754408eaa08c2b/docs/source/_static/logo.png
--------------------------------------------------------------------------------
/docs/source/_static/step_metadata.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gchq/ConcourseTools/bb28027115429e0379c3dfa038754408eaa08c2b/docs/source/_static/step_metadata.png
--------------------------------------------------------------------------------
/docs/source/_static/style.css:
--------------------------------------------------------------------------------
1 | .toc-drawer {
2 | width: fit-content;
3 | }
4 |
--------------------------------------------------------------------------------
/docs/source/additional.rst:
--------------------------------------------------------------------------------
1 | Additional Patterns
2 | ===================
3 |
4 | .. automodule:: concoursetools.additional
5 |
6 | .. autoclass:: concoursetools.additional.OutOnlyConcourseResource
7 | :members: publish_new_version
8 |
9 | .. autoclass:: concoursetools.additional.InOnlyConcourseResource
10 | :members: download_data
11 |
12 | .. autoclass:: concoursetools.additional.TriggerOnChangeConcourseResource
13 | :members: fetch_latest_version
14 |
15 | .. autoclass:: concoursetools.additional.MultiVersionConcourseResource
16 | :members: fetch_latest_sub_versions, download_version
17 | :show-inheritance:
18 |
19 | .. autoclass:: concoursetools.additional.SelfOrganisingConcourseResource
20 | :members: fetch_all_versions
21 |
22 |
23 | Combining Resource Types
24 | ------------------------
25 |
26 | Occasionally you may wish to implement multiple resource types with the same set of dependencies. Although it is often cleaner to treat these separately and :ref:`build separate Docker images `, there are times where it is easier to build a single image containing all of your resource types, and to select one of them via the resource config. To do this, you can use the :func:`~concoursetools.additional.combine_resource_types` function:
27 |
28 | .. autofunction:: concoursetools.additional.combine_resource_types
29 |
--------------------------------------------------------------------------------
/docs/source/api_reference.rst:
--------------------------------------------------------------------------------
1 | API Reference
2 | =============
3 |
4 | A Concourse resource is a combination of a :class:`~concoursetools.version.Version` class and a :class:`~concoursetools.resource.ConcourseResource` class.
5 |
6 | .. toctree::
7 | version
8 | resource
9 | build_metadata
10 | additional
11 |
--------------------------------------------------------------------------------
/docs/source/build_metadata.rst:
--------------------------------------------------------------------------------
1 | Build Metadata
2 | ==============
3 |
4 | .. automodule:: concoursetools.metadata
5 | :members:
6 |
--------------------------------------------------------------------------------
/docs/source/changelog.rst:
--------------------------------------------------------------------------------
1 | What's New
2 | ==========
3 |
4 | .. toctree::
5 | :glob:
6 |
7 | changelog/development
8 | changelog/*
9 |
--------------------------------------------------------------------------------
/docs/source/changelog/development.rst:
--------------------------------------------------------------------------------
1 | Development
2 | ===========
3 |
4 |
5 | * Added Python 3.14 support.
6 |
--------------------------------------------------------------------------------
/docs/source/cli.rst:
--------------------------------------------------------------------------------
1 | Command Line
2 | ============
3 |
4 | .. automodule:: concoursetools.cli.parser
5 | :members:
6 | :exclude-members: Parameter, PositionalArgument, Option, FlagOption
7 |
8 |
9 | .. autoclass:: concoursetools.cli.docstring.Docstring
10 | :members:
11 |
12 | Parameters
13 | ----------
14 |
15 | .. autoclass:: concoursetools.cli.parser.PositionalArgument
16 |
17 | .. autoclass:: concoursetools.cli.parser.Option
18 |
19 | .. autoclass:: concoursetools.cli.parser.FlagOption
20 |
21 | .. autoclass:: concoursetools.cli.parser.Parameter
22 | :members:
23 |
--------------------------------------------------------------------------------
/docs/source/cli_reference.rst:
--------------------------------------------------------------------------------
1 | CLI Reference
2 | =============
3 |
4 | Concourse Tools contains a CLI for a number of small tasks to help write and deploy resource types.
5 |
6 |
7 | .. cli:: concoursetools.cli.cli
8 | :align: left
9 |
--------------------------------------------------------------------------------
/docs/source/conf.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Configuration file for the Sphinx documentation builder.
4 |
5 | This file only contains a selection of the most common options.
6 | For a full list see the documentation: https://www.sphinx-doc.org/en/master/usage/configuration.html.
7 | """
8 | # -- Path setup --------------------------------------------------------------
9 |
10 | from collections.abc import Callable
11 | from pathlib import Path
12 | import sys
13 | from typing import Any
14 |
15 | import sphinx.config
16 | from sphinx_autodoc_typehints import format_annotation
17 |
18 | CONF_FILE_PATH = Path(__file__).absolute()
19 | SOURCE_FOLDER_PATH = CONF_FILE_PATH.parent
20 | DOCS_FOLDER_PATH = SOURCE_FOLDER_PATH.parent
21 | REPO_FOLDER_PATH = DOCS_FOLDER_PATH.parent
22 |
23 | sys.path.extend([str(DOCS_FOLDER_PATH), str(SOURCE_FOLDER_PATH), str(REPO_FOLDER_PATH)])
24 |
25 |
26 | import concoursetools
27 | import concoursetools.additional
28 | import concoursetools.cli.parser
29 | import concoursetools.importing
30 | import concoursetools.resource
31 | import concoursetools.typing
32 | import concoursetools.version
33 |
34 | # -- Project information -----------------------------------------------------
35 |
36 | project = "Concourse Tools"
37 | copyright = "UK Crown"
38 | version = "v" + concoursetools.__version__
39 |
40 |
41 | # -- General configuration ---------------------------------------------------
42 |
43 | extensions = [
44 | "sphinx.ext.autodoc",
45 | "sphinx.ext.autosectionlabel",
46 | "sphinx.ext.intersphinx",
47 | "sphinx_autodoc_typehints",
48 | "sphinx.ext.viewcode",
49 | "extensions.cli",
50 | "extensions.concourse",
51 | "extensions.linecount",
52 | "extensions.wikipedia",
53 | "extensions.xkcd",
54 | ]
55 |
56 | toc_object_entries_show_parents = "hide" # don't show prefix in secondary TOC
57 |
58 | always_document_param_types = True
59 | autodoc_member_order = "bysource"
60 |
61 | autodoc_custom_types = {
62 | concoursetools.cli.parser.CLIFunctionT: format_annotation(Callable[..., None], sphinx.config.Config()),
63 | concoursetools.importing.T: ":class:`object`",
64 | concoursetools.typing.VersionT: ":class:`~concoursetools.version.Version`",
65 | concoursetools.typing.SortableVersionT: ":class:`~concoursetools.version.Version`",
66 | }
67 |
68 | suppress_warnings = ["config.cache"] # https://github.com/sphinx-doc/sphinx/issues/12300#issuecomment-2062238457
69 |
70 |
71 | def typehints_formatter(annotation: Any, config: sphinx.config.Config) -> str | None:
72 | """Properly replace custom type aliases."""
73 | return autodoc_custom_types.get(annotation)
74 |
75 |
76 | nitpicky = True
77 | nitpick_ignore = [
78 | ("py:class", "concoursetools.additional._PseudoConcourseResource"),
79 | ("py:class", "concoursetools.typing.SortableVersionT"),
80 | ]
81 |
82 | linkcheck_report_timeouts_as_broken = False # silences a warning: https://github.com/sphinx-doc/sphinx/issues/11868
83 | linkcheck_anchors_ignore_for_url = [
84 | "https://github.com/.*",
85 | ]
86 | linkcheck_ignore = [
87 | "https://superuser.com/.*",
88 | ]
89 |
90 | intersphinx_mapping = {
91 | "python3": ("https://docs.python.org/3", None),
92 | "requests": ("https://requests.readthedocs.io/en/latest/", None),
93 | }
94 |
95 | html_theme = "furo"
96 | html_favicon = "_static/favicon.png"
97 |
98 | html_static_path = ["_static"]
99 |
100 | html_css_files = [
101 | "style.css",
102 | "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/fontawesome.min.css",
103 | "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/solid.min.css",
104 | "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/brands.min.css",
105 | ]
106 |
107 | html_theme_options = {
108 | "light_logo": "logo.png",
109 | "dark_logo": "logo-dark.png",
110 | "sidebar_hide_name": True,
111 | "source_repository": "https://github.com/gchq/ConcourseTools/",
112 | "source_branch": "main",
113 | "source_directory": "docs/source/",
114 | "footer_icons": [
115 | {
116 | "name": "GitHub",
117 | "url": "https://github.com/gchq/ConcourseTools/",
118 | "class": "fa-brands fa-github fa-2x",
119 | },
120 | {
121 | "name": "PyPI",
122 | "url": "https://pypi.org/project/concoursetools/",
123 | "class": "fa-brands fa-python fa-2x",
124 | }
125 | ],
126 | }
127 |
--------------------------------------------------------------------------------
/docs/source/debugging.rst:
--------------------------------------------------------------------------------
1 | Printing/Debugging
2 | ==================
3 |
4 | A Concourse resource is supposed to print output (such as versions and metadata) to ``stdout``, and all debugging messages to ``stderr``. The :class:`~concoursetools.resource.ConcourseResource` class automatically redirects ``stdout`` to ``stderr`` when running its methods, meaning that all printed output from you (and from your dependencies) automatically ends up in ``stderr``.
5 |
6 | Colour
7 | ------
8 |
9 | .. automodule:: concoursetools.colour
10 | :members:
11 | :exclude-members: Colour
12 |
13 | Available Colours
14 | _________________
15 |
16 | All ``colour`` arguments are :wikipedia:`ANSI colour escape codes `, but a number of common codes are available as attributes of the :class:`~concoursetools.colour.Colour` class:
17 |
18 | .. autoclass:: concoursetools.colour.Colour
19 | :members:
20 | :undoc-members:
21 |
--------------------------------------------------------------------------------
/docs/source/deployment.rst:
--------------------------------------------------------------------------------
1 | Deploying the Resource Type
2 | ===========================
3 |
4 | To properly "deploy" the resource type, your repo should look something like this:
5 |
6 | .. code:: none
7 |
8 | .
9 | |-- Dockerfile
10 | |-- README.md
11 | |-- assets
12 | | |-- check
13 | | |-- in
14 | | |-- out
15 | |-- requirements.txt
16 | |-- concourse.py
17 | |-- tests.py
18 |
19 |
20 | Resource Module
21 | ---------------
22 |
23 | The ``concourse.py`` module contains your :class:`~concoursetools.version.Version` and
24 | :class:`~concoursetools.resource.ConcourseResource` subclasses. It can be called whatever you like, but ``concourse.py``
25 | is the most clear, and is the default for the CLI.
26 |
27 | .. note::
28 |
29 | It is possible to split your code into multiple modules, or even packages. This is sometimes useful if you have
30 | written a lot of code, or if it naturally splits. However, make sure that this code is also put in the correct place
31 | in your :ref:`Docker image `.
32 |
33 |
34 | Requirements
35 | ------------
36 |
37 | The ``requirements.txt`` file should contain a complete list of requirements within your virtual environment (version
38 | pinned) sufficient to reproduce with
39 |
40 | .. code:: shell
41 |
42 | $ pip install -r requirements.txt --no-deps
43 |
44 | If this is not possible, then rewrite your requirements file. If you have any dependencies which are not public, then
45 | you will have to make some adjustments to your :ref:`Dockerfile `.
46 |
47 |
48 | Assets
49 | ------
50 |
51 | The contents of each of the files in ``assets`` follow the same pattern. For example, here is the contents of
52 | ``assets/check``:
53 |
54 | .. code:: python3
55 |
56 | #!/usr/bin/env python3
57 | """
58 | Check for new versions of the resource.
59 | """
60 | from concourse import MyResource
61 |
62 |
63 | if __name__ == "__main__":
64 | MyResource.check_main()
65 |
66 | It is these files which get called by Concourse as part of the resource. Note that none of the asset files have any
67 | extensions, but specific their executable at the top of the file. ``MyResource`` should correspond to your
68 | :class:`~concoursetools.resource.ConcourseResource` subclass, and ``resource`` to the :ref:`module in which you have
69 | placed it `. Replace :meth:`~concoursetools.resource.ConcourseResource.check_main` with
70 | :meth:`~concoursetools.resource.ConcourseResource.in_main` and
71 | :meth:`~concoursetools.resource.ConcourseResource.out_main` for ``assets/in`` and ``assets/out`` respectively.
72 |
73 | .. important::
74 |
75 | Every file in assets need to be executable. This can be done with
76 |
77 | .. code:: shell
78 |
79 | $ chmod +x assets/*
80 |
81 | .. tip::
82 |
83 | Because this pattern is suitable for almost every resource type, you can automate the creation of the ``assets``
84 | folder with the :ref:`cli.assets` CLI command.
85 |
86 |
87 | Dockerfile Structure
88 | --------------------
89 |
90 | The Dockerfile should look something like:
91 |
92 | .. code-block:: Dockerfile
93 | :linenos:
94 |
95 | FROM python:3.12
96 |
97 | RUN python3 -m venv /opt/venv
98 | # Activate venv
99 | ENV PATH="/opt/venv/bin:$PATH"
100 |
101 | COPY requirements.txt requirements.txt
102 |
103 | RUN \
104 | python3 -m pip install --upgrade pip && \
105 | pip install -r requirements.txt --no-deps
106 |
107 | WORKDIR /opt/resource/
108 | COPY concourse.py ./concourse.py
109 | RUN python3 -m concoursetools . -r concourse.py
110 |
111 | ENTRYPOINT ["python3"]
112 |
113 | .. tip::
114 |
115 | You can automate the creation of this file with the :ref:`cli.dockerfile` CLI command.
116 |
117 |
118 | Base Image
119 | __________
120 |
121 | .. code-block:: Dockerfile
122 | :linenos:
123 |
124 | FROM python:3.12
125 |
126 | You should adjust the base image according to your requirements. Concourse Tools will default to ``python:``,
127 | where ```` corresponds to the current major/minor version you are running. However, you may wish to specify
128 | a different base image, such as ``python:3.*-slim`` or ``python:3.*-alpine``.
129 |
130 |
131 | Virtual Environment
132 | ___________________
133 |
134 | .. code-block:: Dockerfile
135 | :linenos:
136 | :lineno-start: 3
137 |
138 | RUN python3 -m venv /opt/venv
139 | # Activate venv
140 | ENV PATH="/opt/venv/bin:$PATH"
141 |
142 | There is much debate as to whether or not it is worth creating a virtual environment within a Docker container.
143 | Concourse Tools chooses to create one by default in order to maximise the similarity between code running on the image
144 | and code running locally.
145 |
146 |
147 | Installing Requirements
148 | _______________________
149 |
150 | .. code-block:: Dockerfile
151 | :linenos:
152 | :lineno-start: 7
153 |
154 | COPY requirements.txt requirements.txt
155 |
156 | RUN \
157 | python3 -m pip install --upgrade pip && \
158 | pip install -r requirements.txt --no-deps
159 |
160 | By default, Concourse Tools will copy over the ``requirements.txt`` file to use for the resource dependencies.
161 | The installation process is a single command involving two parts:
162 |
163 | 1. Updating ``pip`` to ensure the latest available version at build time,
164 | 2. Installing the static requirements file *without implicit dependencies*.
165 |
166 | If these were two separate commands then Docker would cache the first one and ``pip`` would never be upgraded.
167 |
168 | .. note::
169 | By passing ``--no-deps`` we ensure that the ``requirements.txt`` file is fully complete, and we are not missing
170 | any implicit dependencies.
171 |
172 |
173 | Including Certs in your Docker Build
174 | ____________________________________
175 |
176 | .. versionchanged:: 0.8
177 | In previous versions the advice was to use multi-stage builds for this. Although that practice is equally
178 | secure, it makes sense to use `Docker secrets `_ instead.
179 |
180 | If any of your requirements are private then you will need to make your private keys available to the image during the
181 | build process, **without** storing them within the image itself. This can be done by making the following change to the
182 | ``RUN`` command from previous section:
183 |
184 | .. code-block:: Dockerfile
185 | :linenos:
186 | :lineno-start: 7
187 |
188 | COPY requirements.txt requirements.txt
189 |
190 | RUN \
191 | --mount=type=secret,id=private_key,target=/root/.ssh/id_rsa,mode=0600,required=true \
192 | --mount=type=secret,id=known_hosts,target=/root/.ssh/known_hosts,mode=0644 \
193 | python3 -m pip install --upgrade pip && \
194 | pip install -r requirements.txt --no-deps
195 |
196 | The secrets must then be passed at build time:
197 |
198 | .. code:: shell
199 |
200 | $ docker build \
201 | --secret id=private_key,src=~/.ssh/id_rsa \
202 | --secret id=known_hosts,src=~/.ssh/known_hosts \
203 | .
204 |
205 | The files are then mounted securely using the `correct permissions
206 | `_.
207 |
208 |
209 | Creating Asset Files
210 | ____________________
211 |
212 | .. code-block:: Dockerfile
213 | :linenos:
214 | :lineno-start: 13
215 |
216 | WORKDIR /opt/resource/
217 | COPY concourse.py ./concourse.py
218 | RUN python3 -m concoursetools . -r concourse.py
219 |
220 | Concourse requires that your asset files are placed in ``/opt/resource`` on the container, which is done here using
221 | the Concourse Tools CLI to reduce the required code.
222 |
223 | .. warning::
224 | If you **cannot** use the CLI to create your :ref:`assets ` folder, then you should manually copy your asset
225 | files across to ``/opt/resource``.
226 |
227 | If your resource requires additional modules to work, then you need to ensure they are also copied across **before**
228 | the CLI is invoked:
229 |
230 | .. code-block:: Dockerfile
231 | :linenos:
232 | :lineno-start: 13
233 |
234 | WORKDIR /opt/resource/
235 | COPY concourse.py ./concourse.py
236 | COPY extra.py ./extra.py
237 | RUN python3 -m concoursetools . -r concourse.py
238 |
239 |
240 | Entry Point
241 | ___________
242 |
243 | .. code-block:: Dockerfile
244 | :linenos:
245 | :lineno-start: 17
246 |
247 | ENTRYPOINT ["python3"]
248 |
249 | Finally, we specify an ``ENTRYPOINT`` for the container. This has little bearing on the resource itself as Concourse
250 | will specify the scripts it wishes to invoke via the shebang in the scripts. It isn't even used when hijacking the
251 | container (the default is bash). However, it is best practice to set something, and this should make it easiest to
252 | debug locally.
253 |
--------------------------------------------------------------------------------
/docs/source/dockertools.rst:
--------------------------------------------------------------------------------
1 | Dockertools
2 | ===========
3 |
4 | .. automodule:: concoursetools.dockertools
5 | :members: create_script_file
6 |
7 |
8 | Dockerfile
9 | ----------
10 | For easier dynamic creation of Dockerfiles, the module contains a number of
11 | :class:`~concoursetools.dockertools.Instruction` classes:
12 |
13 | .. autoclass:: concoursetools.dockertools.Instruction
14 | :members:
15 |
16 |
17 | A :class:`~concoursetools.dockertools.Comment` class is also included:
18 |
19 | .. autoclass:: concoursetools.dockertools.Comment
20 | :members:
21 |
22 |
23 | All instructions and comments are added to a A :class:`~concoursetools.dockertools.Dockerfile` instance
24 | in *instruction groups*:
25 |
26 | .. autoclass:: concoursetools.dockertools.Dockerfile
27 | :members:
28 |
29 |
30 | Dockerfile Instructions
31 | -----------------------
32 |
33 | All currently relevant instructions have been implemented:
34 |
35 | .. list-table::
36 | :header-rows: 1
37 | :align: left
38 |
39 | * - Instruction
40 | - Description
41 | - Classes
42 | * - ``ADD``
43 | - Add local or remote files and directories.
44 | -
45 | * - ``ARG``
46 | - Use build-time variables.
47 | -
48 | * - ``CMD``
49 | - Specify default commands.
50 | -
51 | * - ``COPY``
52 | - Copy files and directories.
53 | - :class:`~concoursetools.dockertools.CopyInstruction`
54 | * - ``ENTRYPOINT``
55 | - Specify default executable.
56 | - :class:`~concoursetools.dockertools.EntryPointInstruction`
57 | * - ``ENV``
58 | - Set environment variables.
59 | - :class:`~concoursetools.dockertools.EnvInstruction`
60 | * - ``EXPOSE``
61 | - Describe which ports your application is listening on.
62 | -
63 | * - ``FROM``
64 | - Check a container's health on startup.
65 | - :class:`~concoursetools.dockertools.FromInstruction`
66 | * - ``HEALTHCHECK``
67 | - Check a container's health on startup.
68 | -
69 | * - ``LABEL``
70 | - Add metadata to an image.
71 | -
72 | * - ``MAINTAINER``
73 | - Specify the author of an image.
74 | -
75 | * - ``ONBUILD``
76 | - Specify instructions for when the image is used in a build.
77 | -
78 | * - ``RUN``
79 | - Execute build commands.
80 | - :class:`~concoursetools.dockertools.RunInstruction`, :class:`~concoursetools.dockertools.MultiLineRunInstruction`
81 | * - ``SHELL``
82 | - Set the default shell of an image.
83 | -
84 | * - ``STOPSIGNAL``
85 | - Specify the system call signal for exiting a container.
86 | -
87 | * - ``USER``
88 | - Set user and group ID.
89 | -
90 | * - ``VOLUME``
91 | - Create volume mounts.
92 | -
93 | * - ``WORKDIR``
94 | - Change working directory.
95 | - :class:`~concoursetools.dockertools.WorkDirInstruction`
96 |
97 |
98 | .. autoclass:: concoursetools.dockertools.CopyInstruction
99 | .. autoclass:: concoursetools.dockertools.EntryPointInstruction
100 | .. autoclass:: concoursetools.dockertools.EnvInstruction
101 | .. autoclass:: concoursetools.dockertools.FromInstruction
102 | .. autoclass:: concoursetools.dockertools.RunInstruction
103 | .. autoclass:: concoursetools.dockertools.MultiLineRunInstruction
104 | .. autoclass:: concoursetools.dockertools.WorkDirInstruction
105 |
106 |
107 | Dockerfile Mounts
108 | -----------------
109 |
110 | The module also implements some mounts for the ``RUN`` step to facilitate secrets:
111 |
112 | .. autoclass:: concoursetools.dockertools.Mount
113 | :members:
114 |
115 | All currently relevant mount types have been implemented:
116 |
117 | .. list-table::
118 | :header-rows: 1
119 | :align: left
120 |
121 | * - Mount
122 | - Description
123 | - Classes
124 | * - ``bind``
125 | - Bind-mount context directories (read-only).
126 | -
127 | * - ``cache``
128 | - Mount a temporary directory to cache directories for compilers and package managers.
129 | -
130 | * - ``tmpfs``
131 | - Mount a ``tmpfs`` in the build container.
132 | -
133 | * - ``secret``
134 | - Allow the build container to access secure files such as private keys without baking them into the image or build cache.
135 | - :class:`~concoursetools.dockertools.SecretMount`
136 | * - ``ssh``
137 | - Allow the build container to access SSH keys via SSH agents, with support for passphrases.
138 | -
139 |
140 | .. autoclass:: concoursetools.dockertools.SecretMount
141 |
--------------------------------------------------------------------------------
/docs/source/examples/branches.rst:
--------------------------------------------------------------------------------
1 | GitHub Branches
2 | ===============
3 |
4 | .. caution::
5 | This example is for reference only. It is not extensively tested, and it not intended to be a fully-fledged Concourse resource for production pipelines. Copy and paste at your own risk.
6 |
7 | On occasion, you may wish your resource to emit versions when something has changed, and not have a history of this to rely on. For example, imagine that we want to run a pipeline on a number of different branches of a GitHub repository. Our resource could emit a number of versions like this:
8 |
9 | .. code:: json
10 |
11 | {
12 | "branch": "feature/new-module",
13 | "commit": "abcdef..."
14 | }
15 |
16 | But what happens when a check yields two versions from different branches? We could figure out the "latest" version by sorting by commit date, but if we are tracking multiple branches then we don't want to ignore a commit on one just because a more recent commit has been made on another. Previous in Concourse this was "fixed" by passing ``version: every`` to the resource, but this means that if multiple commits are pushed to a branch at once, then each commit will emit a build, which may not be what we want.
17 |
18 | What we really want is to iterate over all branches in our repository, and spin up a branch-specific pipeline for each one using the :concourse:`set-pipeline-step`. This is easy to do, but we need to make sure that new pipelines are added when new branches appear, and that old pipelines are deleted when their branches are removed. We specifically require a resource which will trigger a pipeline whenever something has changed. The generic resource for this pattern is the :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`, but because the "state" is made up of several "sub versions", it makes sense to instead utilise the :class:`~concoursetools.additional.MultiVersionConcourseResource`, which takes care of much of the boiler plate for us, and also allow us to automatically download these subversions as JSON so that we may iterate over them.
19 |
20 | Branch Version
21 | --------------
22 |
23 | For this example, each "version" will contain an encoded JSON string with all of the subversions. We only care about whether or not that version (and hence the state) has changed, and not whether these versions were linear. To start, we define the subversion schema:
24 |
25 | .. literalinclude:: ../../../examples/github_branches.py
26 | :pyobject: BranchVersion
27 |
28 | We inherit from :class:`~concoursetools.version.TypedVersion` to allow us to use the :func:`~dataclasses.dataclass` decorator and save us some lines of code. We have to specify ``unsafe_hash=True`` in order to ensure that the version instance will be *hashable*, otherwise we won't be able to use it with our resource. We also need to specify ``order=True`` to allow our subversions to be sorted, as this is required to make sure the same set of subversions yield identical versions. We could have made use of the :class:`~concoursetools.version.SortableVersionMixin`, but given that the :func:`~dataclasses.dataclass` decorator does this for us it isn't necessary here.
29 |
30 | Branch Resource
31 | ---------------
32 |
33 | We start by inheriting from :class:`~concoursetools.additional.MultiVersionConcourseResource`. A standard :class:`~concoursetools.resource.ConcourseResource` will take a version class, but this time we need to pass a *subversion class* instead, as well as a key:
34 |
35 | .. literalinclude:: ../../../examples/github_branches.py
36 | :pyobject: Resource.__init__
37 | :end-at: super().__init__
38 | :dedent:
39 |
40 | The resource will then construct a :class:`~concoursetools.version.Version` class which wraps the subversion class, and stores the list of subversions as a JSON-encoded string. The key, ``"branches"``, is used as the key in this version. The final version will look like this:
41 |
42 | .. code:: python3
43 |
44 | {"branches": "[{\"name\": \"issue/95\"}, {\"name\": \"master\"}, {\"name\": \"release/6.7.x\"}, {\"name\": \"version\"}, {\"name\": \"version-lts\"}]"}
45 |
46 | We then store a compiled regex and the API route within the class. There is no need to store each parameter separately, as we can just construct the API route in one line:
47 |
48 | .. literalinclude:: ../../../examples/github_branches.py
49 | :pyobject: Resource.__init__
50 | :dedent:
51 |
52 | The source of this resource would then look something like this:
53 |
54 | .. code:: yaml
55 |
56 | source:
57 | owner: concourse
58 | repo: github-release-resource
59 | regex: release/.*
60 |
61 | We only need to implement the :meth:`~concoursetools.additional.MultiVersionConcourseResource.fetch_latest_sub_versions` method; the results of this will be collected and converted into a final version by the parent class:
62 |
63 | .. literalinclude:: ../../../examples/github_branches.py
64 | :pyobject: Resource.fetch_latest_sub_versions
65 | :dedent:
66 |
67 | The logic required here is incredibly minimal, and we only need to return each available subversion. The returned set will then be sorted by the resource when converting it to its final version, which is why we needed the subversion to be sortable and hashable.
68 |
69 | When the branches change (either with new ones added or existing ones removed) a new version will be emitted and the pipeline will be triggered. Users can then iterate over them in their pipeline using a combination of the :concourse:`across-step` and the :concourse:`set-pipeline-step`:
70 |
71 | .. code:: yaml
72 |
73 | - get: repo-branches
74 | - load_var: branches
75 | file: branches/branches.json
76 | - across:
77 | - var: branch-info
78 | values: ((.:branches))
79 | set-pipeline: branch-pipeline
80 | file: ...
81 | vars:
82 | branch: ((.:branch-info.name))
83 |
84 | GitHub Branches Conclusion
85 | --------------------------
86 |
87 | The final module only requires :linecount:`../../../examples/github_branches.py` lines (including docstrings) and looks like this:
88 |
89 | .. literalinclude:: ../../../examples/github_branches.py
90 | :linenos:
91 |
--------------------------------------------------------------------------------
/docs/source/examples/s3.rst:
--------------------------------------------------------------------------------
1 | S3 Presigned URL
2 | ================
3 |
4 | .. caution::
5 | This example is for reference only. It is not extensively tested, and it not intended to be a fully-fledged Concourse resource for production pipelines. Copy and paste at your own risk.
6 |
7 | This example will showcase the :class:`~concoursetools.additional.InOnlyConcourseResource`, and how to build a resource to fetch arbitrary data from an external resource **that is not stored externally**. In this particular example we will consider a resource to generate a new `presigned URL `_ for an object in an S3 bucket.
8 |
9 | Traditionally, when Concourse users which to "run an external function" from a pipeline, they create an :class:`~concoursetools.additional.OutOnlyConcourseResource` to do it. Classic examples include setting a build status (as shown in the :ref:`Bitbucket Build Status` example), or to `send a message on Slack `_. Usually, the new version will only contain a small amount of placeholder information, such as the new build status. However, a :concourse:`put-step` is designed to "fetch" the information for further use in the pipeline, and this is only really possible when the new version represents a "state" which is stored with the external service. For example, the `Git resource `_ will push a new commit, and then emit the version corresponding to that commit so that - when the resource runs its implicit :concourse:`get-step` - the information is not overwritten.
10 |
11 | However, in the case of presigned URLs, AWS does **not** store these anywhere such that they are accessible from the server. When they are created, the user does not get given a UUID which allows them to "look up" the URL at a later date. Therefore, even if the :concourse:`put-step` of the resource created the URL and downloaded it to its resource directory, the :concourse:`get-step` would overwrite it with an empty folder. The two main solutions to this are:
12 |
13 | 1. Pass the URL in the newly created version.
14 | 2. Have the :concourse:`put-step` write the URL to a *different* resource directory.
15 |
16 | Option 2 requires another resource to be already available, as it isn't possible to request additional output directories like a :concourse:`task-step`. This is do-able but fragile, and requires care to make sure that no important files in the other resource are overwritten. Option 1 seems cleaner, but the URL might be sensitive, and storing it as plaintext within the version is definitely not ideal. We could consider encrypting it and having the user pass some sort of key, but this is complicating matters greatly. There is a better way.
17 |
18 | The :class:`~concoursetools.additional.InOnlyConcourseResource` is designed to run these "functions" in the :concourse:`get-step`, and to be *triggered* by a :concourse:`put-step`, like so:
19 |
20 | .. code:: yaml
21 |
22 | - put: s3-presigned-url
23 | get_params:
24 | file_path: my-file
25 |
26 | URL Version
27 | -----------
28 |
29 | Because the version itself isn't important, the :class:`~concoursetools.additional.InOnlyConcourseResource` actually uses a prebuilt version containing nothing but a timestamp of creation time:
30 |
31 | .. literalinclude:: ../../../concoursetools/additional.py
32 | :pyobject: DatetimeVersion
33 |
34 | URL Resource
35 | ------------
36 |
37 | We start by inheriting from :class:`~concoursetools.additional.InOnlyConcourseResource`. Again, we don't need to pass a version:
38 |
39 | .. literalinclude:: ../../../examples/s3.py
40 | :pyobject: S3SignedURLConcourseResource.__init__
41 | :dedent:
42 |
43 | All of the resource functionality comes from overloading the :meth:`~concoursetools.additional.InOnlyConcourseResource.download_data` method:
44 |
45 | .. literalinclude:: ../../../examples/s3.py
46 | :pyobject: S3SignedURLConcourseResource.download_data
47 | :dedent:
48 |
49 | This method takes a required ``file_path`` argument to indicate the object for which the URL should be generated. The ``expires_in`` parameter takes a mapping of arguments for :class:`datetime.timedelta` to allow users to specify expiration time more explicitly. Finally, passing a ``file_name`` instructs the URL to name the downloaded file something specific, rather than the original name from within S3. The URL itself is generated using the `generate_presigned_url `_ function. Finally, we don't return a version, so we only need to concern ourselves with the :ref:`Step Metadata`.
50 |
51 | This resource can then be invoked like so:
52 |
53 |
54 | .. code:: yaml
55 |
56 | - put: s3-presigned-url
57 | get_params:
58 | file_path: folder/file.txt
59 | file_name: file.txt
60 | expires_in:
61 | hours: 24
62 |
63 | Once the implicit :concourse:`get-step` is completed, the URL can then be :concourse:`loaded easily ` and used in prior steps:
64 |
65 | .. code:: yaml
66 |
67 | - load_var: s3-url
68 | file: s3-presigned-url/url
69 |
70 | URL Conclusion
71 | --------------
72 |
73 | The final resource only requires :linecount:`../../../examples/s3.py` lines of code, and looks like this:
74 |
75 | .. literalinclude:: ../../../examples/s3.py
76 | :linenos:
77 |
--------------------------------------------------------------------------------
/docs/source/examples/secrets.rst:
--------------------------------------------------------------------------------
1 | AWS Secrets
2 | ===========
3 |
4 | .. caution::
5 | This example is for reference only. It is not extensively tested, and it not intended to be a fully-fledged Concourse resource for production pipelines. Copy and paste at your own risk.
6 |
7 | This example will showcase the :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`, and how to build a resource to emit versions whenever something has changed, rather than when a new linear version becomes available. In this example, the resource will watch an secret in `AWS SecretsManager `_, and yield a new version whenever the value of that secret has changed. It will also allow the user to optionally download the secret value, and also to update the secret via a string or a file. The functionality will depend heavily on `boto3 `_ in order to reduce the amount of code needed to function.
8 |
9 |
10 | Secrets Version
11 | ---------------
12 |
13 | Every secret in SecretsManager has a number of `versions `_ representing changes to the secret value. This is all the version needs to contain:
14 |
15 | .. literalinclude:: ../../../examples/secrets.py
16 | :pyobject: SecretVersion
17 |
18 | Here, we only inherit from :class:`~concoursetools.version.TypedVersion` to save some lines of code.
19 |
20 |
21 | Secrets Resource
22 | ----------------
23 |
24 | We start by inheriting from :class:`~concoursetools.additional.TriggerOnChangeConcourseResource` and passing in the version class. The resource should take an ARN string of the secret (AWS recommends this over the secret name). We also define the AWS SecretsManager client in the ``__init__`` method for ease. To get the AWS region, we need only parse the ARN string to avoid asking for duplicate values from the user:
25 |
26 | .. literalinclude:: ../../../examples/secrets.py
27 | :pyobject: Resource.__init__
28 | :dedent:
29 |
30 | With this resource type, we don't overload from :meth:`~concoursetools.resource.ConcourseResource.fetch_new_versions`, as it would force us to define the behaviour of the trigger. Instead, we let the resource implement this for us, and overload :meth:`~concoursetools.additional.TriggerOnChangeConcourseResource.fetch_latest_version`. If the version we fetch is the same, then no new versions are emitted. If the version is different, then that new version is sent which which will trigger the pipeline:
31 |
32 | .. literalinclude:: ../../../examples/secrets.py
33 | :pyobject: Resource.fetch_latest_version
34 | :dedent:
35 |
36 | We use `list_secret_version_ids `_ with ``IncludeDeprecated=False`` to ensure that we only get versions which are either current or pending. We then iterate over the versions and find the one marked with ``AWSCURRENT``. If we don't find it then we raise an error to alert the user.
37 |
38 | Next, we overload :meth:`~concoursetools.resource.ConcourseResource.download_version` to allow us to download the metadata (and optionally the value) of the new secret:
39 |
40 | .. literalinclude:: ../../../examples/secrets.py
41 | :pyobject: Resource.download_version
42 | :dedent:
43 |
44 | The behaviour of the resource is as follows:
45 |
46 | 1. The metadata of the secret is fetched from AWS using `describe_secret `_. The response metadata is removed, but could potentially be output as :ref:`Step Metadata`.
47 | 2. The metadata is saved to a file. By default this is named ``metadata.json``, but the user can customise this with the parameters of the :concourse:`get-step`.
48 | 3. If the user has requested the secret value also (which is **not** the default behaviour), then this is fetched using `get_secret_value `_ to be saved to a file, which defaults to ``value`` but is again customisable by the user.
49 | 4. If the response contains a string, then this is written directly to the file using :meth:`~pathlib.Path.write_text`, but if it contains a binary then it is instead written using :meth:`~pathlib.Path.write_bytes`.
50 |
51 | The :mod:`json` module is imported as ``json_package`` to avoid a name collision with a future argument. The metadata of the secret contains some :class:`~datetime.datetime` objects, and so a custom :class:`~json.JSONEncoder` is required:
52 |
53 | .. literalinclude:: ../../../examples/secrets.py
54 | :pyobject: DatetimeSafeJSONEncoder
55 |
56 | Finally, we overload :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` to allow the user to update the secret. We *could* make this rotate the secret, but for the purposes of this example we will allow the user to specify a new value exactly:
57 |
58 | .. literalinclude:: ../../../examples/secrets.py
59 | :pyobject: Resource.publish_new_version
60 | :dedent:
61 |
62 | The behaviour is as follows:
63 |
64 | 1. If the user has specified the secret value as JSON, then encode that as a string as pass it forward.
65 | 2. If the secret is specified as a string, then attempt to set the secret value with `put_secret_value `_.
66 | 3. If the secret is specified as a file path, then use :meth:`~pathlib.Path.read_bytes` to get the contents of the file (more resilient than reading the text) and set the ``SecretBinary`` instead.
67 | 4. If none of these have been set, then raise an error.
68 | 5. Pull out the new ID from the response to establish the new version to return, and also pass the ``VersionStages`` as metadata to be output to the console.
69 |
70 |
71 | AWS Secrets Conclusion
72 | ----------------------
73 |
74 | We have added a lot of functionality for this resource in only :linecount:`../../../examples/secrets.py` lines of code. The final module looks like this:
75 |
76 | .. literalinclude:: ../../../examples/secrets.py
77 | :linenos:
78 |
--------------------------------------------------------------------------------
/docs/source/examples/xkcd.rst:
--------------------------------------------------------------------------------
1 | xkcd Comics
2 | ===========
3 |
4 | .. caution::
5 | This example is for reference only. It is not extensively tested, and it not intended to be a fully-fledged Concourse resource for production pipelines. Copy and paste at your own risk.
6 |
7 | This example will showcase the :class:`~concoursetools.additional.SelfOrganisingConcourseResource`, and how to use it to save lines of code and avoid implementing logic to determine which versions are newer than the last. In this particular example we will build a resource type to trigger on new :xkcd:`xkcd comics <>`.
8 |
9 | .. xkcd:: 1319
10 |
11 |
12 | xkcd Version
13 | ------------
14 |
15 | Each xkcd comic is associated with an integer, and so these form a natural choice for the version. We'll inherit from :class:`~concoursetools.version.TypedVersion` to minimise the code we need, and make sure our versions are hashable with ``unsafe_hash=True`` as is required by the resource. Finally, we'll also implement the :func:`~operator.__lt__` method to make sure our comparisons use the comic ID as an orderable integer.
16 |
17 | .. literalinclude:: ../../../examples/xkcd.py
18 | :pyobject: ComicVersion
19 |
20 |
21 | xkcd Resource
22 | -------------
23 |
24 | We start by inheriting from the :class:`~concoursetools.additional.SelfOrganisingConcourseResource`. We pass in our version, and also include an optional configuration parameter for the xkcd link. Although this is unlikely to matter to 99% of users, it is good practice to allow this ot be configured in case a user is hosting their own version of the comics (or if the URL changes), and it will avoid them needing to reimplement your work.
25 |
26 | .. literalinclude:: ../../../examples/xkcd.py
27 | :pyobject: XKCDResource.__init__
28 | :dedent:
29 |
30 | Next we need to overload :meth:`~concoursetools.additional.SelfOrganisingConcourseResource.fetch_all_versions`. Given that our versions are comparable, the resource will only return versions considered "greater than" the previous versions (if it exists, normal rules apply if it hasn't been passed), and will even order them for us, removing our need to worry about that ourselves.
31 |
32 | Although we could technically scrape the xkcd website to check for new versions, it is much more polite to make use of the available :wikipedia:`RSS` feed or, in this case, the :wikipedia:`Atom ` feed, which can be found :xkcd:`here `. Although there are existing libraries designed to parse Atom feeds, it doesn't take too much effort to pull out the comic IDs, which is all we really need for this to work:
33 |
34 | .. literalinclude:: ../../../examples/xkcd.py
35 | :pyobject: yield_comic_links
36 |
37 | .. literalinclude:: ../../../examples/xkcd.py
38 | :pyobject: yield_comic_ids
39 |
40 | When given the contents of the feed as a string, the ``yield_comic_ids`` will give us all of the integers corresponding to the newer comics. All we need to do is fetch the data from the source, and return our list of versions:
41 |
42 | .. literalinclude:: ../../../examples/xkcd.py
43 | :pyobject: XKCDResource.fetch_all_versions
44 | :dedent:
45 |
46 | The next step is to implement actually "getting" the version. Technically it is possible that between checking the versions and fetching one, it is no longer available in the feed. However, :wikipedia:`Randall ` has been kind enough to implement a :xkcd:`JSON API `, and so we can just use that.
47 |
48 | .. note::
49 | Since this API yields a link to the current comic, it would be possible to implement this example as a :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`, but we've :ref:`already got an example for that one `.
50 |
51 | We'll fetch this information and pull out some of the useful information such as the title and upload date. We'll also generate the URL, and set this aside as :ref:`Step Metadata`:
52 |
53 | .. literalinclude:: ../../../examples/xkcd.py
54 | :pyobject: XKCDResource.download_version
55 | :dedent:
56 | :end-before: info_path =
57 |
58 | Next we write the metadata to a file, in case the user wishes to use it themselves:
59 |
60 | .. literalinclude:: ../../../examples/xkcd.py
61 | :pyobject: XKCDResource.download_version
62 | :dedent:
63 | :start-at: info_path =
64 | :end-before: if image
65 |
66 | We optionally (but defaulting to :data:`True`) download the image:
67 |
68 | .. literalinclude:: ../../../examples/xkcd.py
69 | :pyobject: XKCDResource.download_version
70 | :dedent:
71 | :start-at: if image
72 | :end-before: if link
73 |
74 | And finally we do something similar with the comic link, and the alt text:
75 |
76 | .. literalinclude:: ../../../examples/xkcd.py
77 | :pyobject: XKCDResource.download_version
78 | :dedent:
79 | :start-at: if link
80 |
81 | We don't intend for the resource to publish new comics (unless :wikipedia:`Randall ` displays some interest), but the :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` method is an :func:`~abc.abstractmethod`, and so we need to explicitly overload it to ensure that the resource can be used at all. We do this by explicitly raising a :class:`NotImplementedError`:
82 |
83 | .. literalinclude:: ../../../examples/xkcd.py
84 | :pyobject: XKCDResource.publish_new_version
85 | :dedent:
86 |
87 | xkcd Conclusion
88 | ---------------
89 |
90 | The final resource only requires :linecount:`../../../examples/xkcd.py` lines of code, and looks like this:
91 |
92 | .. literalinclude:: ../../../examples/xkcd.py
93 | :linenos:
94 |
--------------------------------------------------------------------------------
/docs/source/extensions/__init__.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Small extensions to make documenting easier for Concourse Tools.
4 | """
5 |
--------------------------------------------------------------------------------
/docs/source/extensions/cli.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Sphinx extension for documenting the custom CLI.
4 | """
5 | from __future__ import annotations
6 |
7 | from typing import Literal
8 |
9 | from docutils import nodes
10 | from docutils.parsers.rst import directives
11 | from sphinx.application import Sphinx
12 | from sphinx.ext.autodoc import import_object
13 | from sphinx.util.docutils import SphinxDirective
14 |
15 | from concoursetools.cli.parser import CLI
16 |
17 |
18 | def _align_directive(argument: str) -> str:
19 | return directives.choice(argument, ("left", "center", "right"))
20 |
21 |
22 | class CLIDirective(SphinxDirective):
23 | """
24 | Directive for listing CLI commands in a table.
25 | """
26 | required_arguments = 1 # The import path of the CLI
27 | option_spec = {
28 | "align": _align_directive,
29 | }
30 |
31 | def run(self) -> list[nodes.Node]:
32 | """
33 | Process the content of the shield directive.
34 | """
35 | align: Literal["left", "center", "right"] | None = self.options.get("align")
36 |
37 | headers = [nodes.entry("", nodes.paragraph("", header)) for header in ["Command", "Description"]]
38 |
39 | import_string, = self.arguments
40 | cli = self.import_cli(import_string)
41 |
42 | rows: list[list[nodes.entry]] = []
43 |
44 | for command_name, command in cli.commands.items():
45 | rows.append([
46 | nodes.entry("", nodes.paragraph("", "", nodes.reference("", "", nodes.literal("", command_name), refid=f"cli.{command_name}"))),
47 | nodes.entry("", nodes.paragraph("", command.description or "")),
48 | ])
49 |
50 | table = self.create_table(headers, rows, align=align)
51 |
52 | nodes_to_return: list[nodes.Node] = [table]
53 |
54 | for command_name, command in cli.commands.items():
55 | command_section = nodes.section(ids=[f"cli.{command_name}"])
56 | title = nodes.title(f"cli.{command_name}", "", nodes.literal("", command_name))
57 | command_section.append(title)
58 |
59 | if command.description is not None:
60 | command_section.append(nodes.paragraph("", command.description))
61 |
62 | usage_block = nodes.literal_block("", f"$ {command.usage_string()}")
63 | command_section.append(usage_block)
64 |
65 | for positional in command.positional_arguments:
66 | alias_paragraph = nodes.paragraph("", "", nodes.literal("", positional.name))
67 | description_paragraph = nodes.paragraph("", positional.description or "")
68 | description_paragraph.set_class("cli-option-description")
69 | command_section.extend([alias_paragraph, description_paragraph])
70 |
71 | for option in command.options:
72 | alias_nodes: list[nodes.Node] = []
73 | for alias in option.aliases:
74 | alias_nodes.append(nodes.literal("", alias))
75 | alias_nodes.append(nodes.Text(", "))
76 | alias_paragraph = nodes.paragraph("", "", *alias_nodes[:-1])
77 | description_paragraph = nodes.paragraph("", option.description or "")
78 | description_paragraph.set_class("cli-option-description")
79 | command_section.extend([alias_paragraph, description_paragraph])
80 |
81 | nodes_to_return.append(command_section)
82 |
83 | return nodes_to_return
84 |
85 | def create_table(self, headers: list[nodes.entry], rows: list[list[nodes.entry]],
86 | align: Literal["left", "center", "right"] | None = None) -> nodes.table:
87 | table = nodes.table()
88 | if align is not None:
89 | table["align"] = align
90 |
91 | table_group = nodes.tgroup(cols=len(headers))
92 | table_group.extend([nodes.colspec()] * len(headers))
93 |
94 | table.append(table_group)
95 |
96 | header = nodes.thead()
97 | header_row = nodes.row()
98 | header_row.extend(headers)
99 | header.append(header_row)
100 | table_group.append(header)
101 |
102 | body = nodes.tbody()
103 | for row in rows:
104 | body_row = nodes.row()
105 | body_row.extend(row)
106 | body.append(body_row)
107 |
108 | table_group.append(body)
109 | return table
110 |
111 | def import_cli(self, import_string: str) -> CLI:
112 | *module_components, import_object_name = import_string.split(".")
113 | import_path = ".".join(module_components)
114 |
115 | import_result = import_object(import_path, [import_object_name])
116 | cli: CLI = import_result[-1]
117 | return cli
118 |
119 |
120 | def setup(app: Sphinx) -> dict[str, object]:
121 | """
122 | Attach the extension to the application.
123 |
124 | :param app: The Sphinx application.
125 | """
126 | app.add_directive("cli", CLIDirective)
127 | return {"parallel_read_safe": True}
128 |
--------------------------------------------------------------------------------
/docs/source/extensions/concourse.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Minor Sphinx extension for creating Concourse documentation links, based on https://sphinx-toolbox.readthedocs.io/en/stable/extensions/wikipedia.html.
4 |
5 | :concourse:`jobs` will create a link to /jobs.html with the text "jobs".
6 | :concourse:`jobs.thing.thing2` will create a link to /jobs.html#thing1.thing2 with the text "thing2".
7 | :concourse:`jobs-a-b-c` will create a link to /jobs-a-b-c.html with the text "jobs a b c".
8 | :concourse:`job ` will create a link to /jobs.html with the text "job".
9 | :concourse:`job ` will create a link to /jobs.html#thing1.thing2 with the text "job".
10 |
11 | Set ``concourse_base_url`` in ``conf.py`` to change the URL used. It must contain "{target}" to be populated.
12 | """
13 | from __future__ import annotations
14 |
15 | from urllib.parse import quote
16 |
17 | from docutils import nodes
18 | from docutils.nodes import system_message
19 | from docutils.parsers.rst.states import Inliner
20 | from sphinx.application import Sphinx
21 | from sphinx.util.nodes import split_explicit_title
22 |
23 | __all__ = ("make_concourse_link", "setup")
24 |
25 | DEFAULT_BASE_URL = "https://concourse-ci.org/{target}"
26 |
27 |
28 | def make_concourse_link(name: str, rawtext: str, text: str, lineno: int, inliner: Inliner, options: dict[str, object] = {},
29 | content: list[str] = []) -> tuple[list[nodes.reference], list[system_message]]:
30 | """
31 | Add a link to the given article on Concourse CI.
32 |
33 | :param name: The local name of the interpreted role, the role name actually used in the document.
34 | :param rawtext: A string containing the entire interpreted text input, including the role and markup.
35 | :param text: The interpreted text content.
36 | :param lineno: The line number where the interpreted text begins.
37 | :param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.source_role`.
38 | It contains the several attributes useful for error reporting and document tree access.
39 | :param options: A dictionary of directive options for customization (from the ``role`` directive),
40 | to be interpreted by the function. Used for additional attributes for the generated elements
41 | and other functionality.
42 | :param content: A list of strings, the directive content for customization (from the ``role`` directive).
43 | To be interpreted by the function.
44 |
45 | :return: A list containing the created node, and a list containing any messages generated during the function.
46 | """
47 | text = nodes.unescape(text)
48 | _, title, target = split_explicit_title(text)
49 |
50 | title = title.split(".")[-1].replace("-", " ")
51 |
52 | page, *anchors = quote(target.replace(" ", "_"), safe="").split(".", maxsplit=1)
53 | if anchors:
54 | anchor = ".".join(anchors)
55 | new_target = f"{page}.html#{anchor}"
56 | else:
57 | new_target = f"{page}.html"
58 |
59 | base_url: str = inliner.document.settings.env.config.concourse_base_url
60 | ref = base_url.format(target=new_target)
61 |
62 | node = nodes.reference(rawtext, title, refuri=str(ref), **options)
63 | return [node], []
64 |
65 |
66 | def setup(app: Sphinx) -> dict[str, object]:
67 | """
68 | Attach the extension to the application.
69 |
70 | :param app: The Sphinx application.
71 | """
72 | app.add_role("concourse", make_concourse_link)
73 | app.add_config_value("concourse_base_url", DEFAULT_BASE_URL, "env", [str])
74 |
75 | return {"parallel_read_safe": True}
76 |
--------------------------------------------------------------------------------
/docs/source/extensions/linecount.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Minor Sphinx extension for counting the number of lines in a file.
4 |
5 | :linecount:`../relative/path/to/file.txt` will be replaced with the number of lines in `file.txt`.
6 |
7 | Paths should be relative to the directory of the current file, not the source directory.
8 | This is the same path passed to literalinclude.
9 | """
10 | from __future__ import annotations
11 |
12 | from pathlib import Path
13 |
14 | from docutils import nodes
15 | from docutils.nodes import system_message
16 | from docutils.parsers.rst.states import Inliner
17 | from sphinx.application import Sphinx
18 |
19 | __all__ = ("count_lines", "setup")
20 |
21 |
22 | def count_lines(name: str, rawtext: str, text: str, lineno: int, inliner: Inliner, options: dict[str, object] = {},
23 | content: list[str] = []) -> tuple[list[nodes.Node], list[system_message]]:
24 | """
25 | Add a text node containing the number of lines.
26 |
27 | :param name: The local name of the interpreted role, the role name actually used in the document.
28 | :param rawtext: A string containing the entire interpreted text input, including the role and markup.
29 | :param text: The interpreted text content.
30 | :param lineno: The line number where the interpreted text begins.
31 | :param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.source_role`.
32 | It contains the several attributes useful for error reporting and document tree access.
33 | :param options: A dictionary of directive options for customization (from the ``role`` directive),
34 | to be interpreted by the function. Used for additional attributes for the generated elements
35 | and other functionality.
36 | :param content: A list of strings, the directive content for customization (from the ``role`` directive).
37 | To be interpreted by the function.
38 |
39 | :return: A list containing the created node, and a list containing any messages generated during the function.
40 | """
41 | path = Path(nodes.unescape(text))
42 | page_path = Path(Path(inliner.document.settings._source))
43 | resolved_path = (page_path.parent / path).resolve()
44 | with open(resolved_path) as rf:
45 | line_count = sum(1 for _ in rf)
46 |
47 | node = nodes.Text(str(line_count))
48 | return [node], []
49 |
50 |
51 | def setup(app: Sphinx) -> dict[str, object]:
52 | """
53 | Attach the extension to the application.
54 |
55 | :param app: The Sphinx application.
56 | """
57 | app.add_role("linecount", count_lines)
58 |
59 | return {"parallel_read_safe": True}
60 |
--------------------------------------------------------------------------------
/docs/source/extensions/wikipedia.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Minor Sphinx extension for creating Wikipedia links, based on https://sphinx-toolbox.readthedocs.io/en/stable/extensions/wikipedia.html.
4 |
5 | :wikipedia:`Cats` will create a link to /wiki/Cats with the text "Cats".
6 | :wikipedia:`cat ` will create a link to /wiki/Cats with the text "cat".
7 |
8 | Set ``wikipedia_base_url`` in ``conf.py`` to change the URL used. It must contain "{target}" to be populated.
9 | It can also contain "{lang}" to allow the language code (e.g. "de") to be injected with the ``wikipedia_lang`` variable.
10 | """
11 | from __future__ import annotations
12 |
13 | import re
14 | from urllib.parse import quote
15 |
16 | from docutils import nodes
17 | from docutils.nodes import system_message
18 | from docutils.parsers.rst.states import Inliner
19 | from sphinx.application import Sphinx
20 | from sphinx.util.nodes import split_explicit_title
21 |
22 | __all__ = ("make_wikipedia_link", "setup")
23 |
24 | DEFAULT_BASE_URL = "https://{lang}.wikipedia.org/wiki/{target}"
25 | RE_WIKI_LANG = re.compile(":(.*?):(.*)")
26 |
27 |
28 | def make_wikipedia_link(name: str, rawtext: str, text: str, lineno: int, inliner: Inliner, options: dict[str, object] = {},
29 | content: list[str] = []) -> tuple[list[nodes.reference], list[system_message]]:
30 | """
31 | Add a link to the given article on :wikipedia:`Wikipedia`.
32 |
33 | :param name: The local name of the interpreted role, the role name actually used in the document.
34 | :param rawtext: A string containing the entire interpreted text input, including the role and markup.
35 | :param text: The interpreted text content.
36 | :param lineno: The line number where the interpreted text begins.
37 | :param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called ``source_role``.
38 | It contains the several attributes useful for error reporting and document tree access.
39 | :param options: A dictionary of directive options for customization (from the ``role`` directive),
40 | to be interpreted by the function. Used for additional attributes for the generated elements
41 | and other functionality.
42 | :param content: A list of strings, the directive content for customization (from the ``role`` directive).
43 | To be interpreted by the function.
44 |
45 | :return: A list containing the created node, and a list containing any messages generated during the function.
46 | """
47 | text = nodes.unescape(text)
48 | has_explicit, title, target = split_explicit_title(text)
49 |
50 | if (match := RE_WIKI_LANG.match(target)):
51 | lang, target = match.groups()
52 | if not has_explicit:
53 | title = target
54 | else: # default language
55 | lang = inliner.document.settings.env.config.wikipedia_lang
56 |
57 | base_url: str = inliner.document.settings.env.config.wikipedia_base_url
58 | ref = base_url.format(lang=lang, target=quote(target.replace(" ", "_"), safe="#"))
59 |
60 | node = nodes.reference(rawtext, title, refuri=str(ref), **options)
61 | return [node], []
62 |
63 |
64 | def setup(app: Sphinx) -> dict[str, object]:
65 | """
66 | Attach the extension to the application.
67 |
68 | :param app: The Sphinx application.
69 | """
70 | app.add_role("wikipedia", make_wikipedia_link)
71 | app.add_config_value("wikipedia_base_url", DEFAULT_BASE_URL, "env", [str])
72 | app.add_config_value("wikipedia_lang", "en", "env", [str])
73 |
74 | return {"parallel_read_safe": True}
75 |
--------------------------------------------------------------------------------
/docs/source/extensions/xkcd.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Minor Sphinx extension for embedding xkcd comics, based on https://sphinx-toolbox.readthedocs.io/en/stable/extensions/wikipedia.html.
4 |
5 | You can embed a particular comic with the following directive:
6 |
7 | .. xkcd::
8 |
9 | In addition:
10 |
11 | :wikipedia:`path/to/page.html` will create a link to that page on the xkcd website.
12 |
13 | Set ``xkcd_endpoint`` in ``conf.py`` to change the URL used.
14 | """
15 | from __future__ import annotations
16 |
17 | from typing import Any
18 |
19 | from docutils import nodes
20 | from docutils.parsers.rst import directives
21 | from docutils.parsers.rst.roles import set_classes
22 | from docutils.parsers.rst.states import Inliner
23 | import requests
24 | from sphinx.application import Sphinx
25 | from sphinx.environment import BuildEnvironment
26 | from sphinx.util import logging
27 | from sphinx.util.docutils import SphinxDirective
28 | from sphinx.util.nodes import split_explicit_title
29 |
30 | DEFAULT_XKCD = "https://xkcd.com"
31 |
32 |
33 | class XkcdDirective(SphinxDirective):
34 | """
35 | Directive for xkcd comics.
36 | """
37 | required_arguments = 1 # The comic number
38 | option_spec = {
39 | "height": directives.length_or_unitless,
40 | "width": directives.length_or_percentage_or_unitless,
41 | "scale": directives.percentage,
42 | "class": directives.class_option,
43 | "caption": directives.unchanged,
44 | }
45 |
46 | @property
47 | def config(self) -> BuildEnvironment:
48 | """Return the config environment."""
49 | return self.state.document.settings.env.config
50 |
51 | def run(self) -> list[nodes.Node]:
52 | """
53 | Process the content of the shield directive.
54 | """
55 | comic_number, = self.arguments
56 | comic_info = self.get_comic_info(int(comic_number), self.config["xkcd_endpoint"])
57 |
58 | caption = self.options.pop("caption", "Relevant xkcd")
59 |
60 | set_classes(self.options)
61 | image_node = nodes.image(self.block_text, uri=directives.uri(comic_info["img"]),
62 | alt=comic_info["alt"], **self.options)
63 | self.add_name(image_node)
64 |
65 | reference_node = nodes.reference("", "", refuri=comic_info["link"])
66 | reference_node += image_node
67 |
68 | caption_node = nodes.caption("", caption)
69 |
70 | figure_node = nodes.figure()
71 | figure_node += reference_node
72 | figure_node += caption_node
73 | return [figure_node]
74 |
75 | def get_comic_info(self, comic_number: int, endpoint: str = DEFAULT_XKCD) -> dict[str, Any]:
76 | comic_link = f"{endpoint}/{comic_number}/"
77 | try:
78 | response = requests.get(f"{endpoint}/{comic_number}/info.0.json")
79 | response.raise_for_status()
80 | except requests.ConnectionError:
81 | logger = logging.getLogger(__name__)
82 | logger.warning("Could not connect to xkcd endpoint")
83 | response_json: dict[str, object] = {
84 | "img": comic_link,
85 | "alt": comic_link,
86 | }
87 | except requests.HTTPError:
88 | latest_response = requests.get(f"{endpoint}/info.0.json")
89 | latest_json: dict[str, Any] = latest_response.json()
90 | most_recent_comic: int = latest_json["num"]
91 | if most_recent_comic < comic_number:
92 | raise ValueError(f"You asked for xkcd #{comic_number}, but the most recent available comic is #{most_recent_comic}")
93 | else:
94 | raise
95 | else:
96 | response_json = response.json()
97 | response_json["link"] = comic_link
98 | return response_json
99 |
100 |
101 | def make_xkcd_link(name: str, rawtext: str, text: str, lineno: int, inliner: Inliner,
102 | options: dict[str, object] = {}, content: list[str] = []) -> tuple[list[nodes.reference], list[nodes.system_message]]:
103 | """
104 | Add a link to a page on the xkcd website.
105 |
106 | :param name: The local name of the interpreted role, the role name actually used in the document.
107 | :param rawtext: A string containing the entire interpreted text input, including the role and markup.
108 | :param text: The interpreted text content.
109 | :param lineno: The line number where the interpreted text begins.
110 | :param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.source_role`.
111 | It contains the several attributes useful for error reporting and document tree access.
112 | :param options: A dictionary of directive options for customization (from the ``role`` directive),
113 | to be interpreted by the function. Used for additional attributes for the generated elements
114 | and other functionality.
115 | :param content: A list of strings, the directive content for customization (from the ``role`` directive).
116 | To be interpreted by the function.
117 |
118 | :return: A list containing the created node, and a list containing any messages generated during the function.
119 | """
120 | text = nodes.unescape(text)
121 | _, title, target = split_explicit_title(text)
122 |
123 | endpoint: str = inliner.document.settings.env.config.xkcd_endpoint
124 | ref = f"{endpoint}/{target}"
125 |
126 | node = nodes.reference(rawtext, title, refuri=str(ref), **options)
127 | return [node], []
128 |
129 |
130 | def setup(app: Sphinx) -> dict[str, object]:
131 | """
132 | Attach the extension to the application.
133 |
134 | :param app: The Sphinx application.
135 | """
136 | app.add_directive("xkcd", XkcdDirective)
137 | app.add_role("xkcd", make_xkcd_link)
138 | app.add_config_value("xkcd_endpoint", DEFAULT_XKCD, "env", [str])
139 | return {"parallel_read_safe": True}
140 |
--------------------------------------------------------------------------------
/docs/source/importing.rst:
--------------------------------------------------------------------------------
1 | Dynamic Importing
2 | =================
3 |
4 | .. automodule:: concoursetools.importing
5 | :members:
6 |
--------------------------------------------------------------------------------
/docs/source/index.rst:
--------------------------------------------------------------------------------
1 | Concourse Tools (|version|)
2 | ===========================
3 |
4 | .. admonition:: Description
5 |
6 | A Python package for easily implementing Concourse :concourse:`resource types `, created by GCHQ.
7 |
8 |
9 |
10 | Bugs and Contributions
11 | ----------------------
12 | Contributions, fixes, suggestions and bug reports are all welcome: please see the `guidance `_.
13 |
14 |
15 | Examples
16 | --------
17 |
18 | Examples are available which showcase the variety of resources made possible with Concourse Tools:
19 |
20 | .. list-table::
21 | :header-rows: 1
22 | :align: left
23 |
24 | * - Resource
25 | - Examples
26 | * - :class:`~concoursetools.resource.ConcourseResource`
27 | - :ref:`AWS SageMaker Pipeline`
28 | * - :class:`~concoursetools.additional.OutOnlyConcourseResource`
29 | - :ref:`Bitbucket Build Status`
30 | * - :class:`~concoursetools.additional.SelfOrganisingConcourseResource`
31 | - :ref:`XKCD Comics`
32 | * - :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`
33 | - :ref:`AWS Secrets`
34 | * - :class:`~concoursetools.additional.InOnlyConcourseResource`
35 | - :ref:`S3 Presigned URL`
36 | * - :class:`~concoursetools.additional.MultiVersionConcourseResource`
37 | - :ref:`GitHub Branches`
38 |
39 | Contents
40 | --------
41 |
42 | .. toctree::
43 | :maxdepth: 2
44 |
45 | quickstart
46 | api_reference
47 | cli_reference
48 | debugging
49 | testing
50 | deployment
51 | internals
52 | changelog
53 |
54 | .. toctree::
55 | :caption: Examples
56 | :hidden:
57 |
58 | examples/pipeline
59 | examples/build_status
60 | examples/xkcd
61 | examples/secrets
62 | examples/s3
63 | examples/branches
64 |
65 | Indices and Tables
66 | ------------------
67 |
68 | * :ref:`genindex`
69 | * :ref:`modindex`
70 | * :ref:`search`
71 |
--------------------------------------------------------------------------------
/docs/source/internals.rst:
--------------------------------------------------------------------------------
1 | Internals
2 | =========
3 |
4 | Concourse Tools contains a number of internal utility functions which may be useful for developers.
5 |
6 | .. toctree::
7 | parsing
8 | cli
9 | dockertools
10 | importing
11 | main_scripts
12 | typing
13 |
--------------------------------------------------------------------------------
/docs/source/main_scripts.rst:
--------------------------------------------------------------------------------
1 | Main Scripts
2 | ============
3 | When scripts are eventually generated for your resource class, each step corresponds to a function call which encapsulates all logic. All of the following functions accept no arguments, instead extracting parameters directly from the command line, ``stdin``, and environment variables. They will print to ``stdout`` and ``stderr``, and do not return anything.
4 |
5 | Although each method can *technically* be overloaded, this is **not** recommended as all of the resource logic is contained in the standard methods, and this will just add complications.
6 |
7 | .. automethod:: concoursetools.resource.ConcourseResource.check_main
8 |
9 | .. automethod:: concoursetools.resource.ConcourseResource.in_main
10 |
11 | .. automethod:: concoursetools.resource.ConcourseResource.out_main
12 |
--------------------------------------------------------------------------------
/docs/source/mocking.rst:
--------------------------------------------------------------------------------
1 | Mocking
2 | =======
3 |
4 | .. automodule:: concoursetools.mocking
5 |
6 |
7 | Environment Variables
8 | ---------------------
9 |
10 | .. autofunction:: concoursetools.mocking.mock_environ
11 |
12 | .. autofunction:: concoursetools.mocking.create_env_vars
13 |
14 | .. autoclass:: concoursetools.mocking.TestBuildMetadata
15 |
16 | Input / Output
17 | --------------
18 |
19 | .. autofunction:: concoursetools.mocking.mock_argv
20 |
21 | .. autofunction:: concoursetools.mocking.mock_stdin
22 |
23 | .. autoclass:: concoursetools.mocking.StringIOWrapper
24 | :members:
25 |
26 |
27 | Directory State
28 | ---------------
29 | Often you need to mock certain files when testing your resource, which are usually accessible in the resource folders. Rather than set this up manually, you can pass a directory state to :class:`~concoursetools.mocking.TemporaryDirectoryState` to make this easier.
30 |
31 | .. autoclass:: concoursetools.mocking.TemporaryDirectoryState
32 | :members:
33 |
--------------------------------------------------------------------------------
/docs/source/parsing.rst:
--------------------------------------------------------------------------------
1 | Parsing
2 | =======
3 |
4 | .. automodule:: concoursetools.parsing
5 |
6 |
7 | JSON Parsing
8 | ------------
9 |
10 | The following functions are responsible for parsing Concourse JSON payloads and returning Python objects.
11 |
12 | .. autofunction:: concoursetools.parsing.parse_check_payload
13 |
14 | .. autofunction:: concoursetools.parsing.parse_in_payload
15 |
16 | .. autofunction:: concoursetools.parsing.parse_out_payload
17 |
18 | .. autofunction:: concoursetools.parsing.parse_metadata
19 |
20 |
21 | JSON Formatting
22 | ---------------
23 |
24 | The following functions are responsible for formatting Concourse JSON payloads from Python objects.
25 |
26 | .. autofunction:: concoursetools.parsing.format_check_output
27 |
28 | .. autofunction:: concoursetools.parsing.format_in_out_output
29 |
30 | .. autofunction:: concoursetools.parsing.format_metadata
31 |
--------------------------------------------------------------------------------
/docs/source/quickstart.rst:
--------------------------------------------------------------------------------
1 | Quickstart
2 | ==========
3 |
4 | Creating a Concourse resource type with Concourse Tools couldn't be simpler.
5 | To begin, install **Concourse Tools** from source or from PyPI (see the footer for links).
6 |
7 | .. code:: shell
8 |
9 | $ pip install concoursetools
10 |
11 | Start by familiarising yourself with the Concourse resource "rules" in the
12 | :concourse:`documentation `. To recreate that example, start by creating a new
13 | ``concourse.py`` file in your repository. The first step is to create a :class:`~concoursetools.version.Version` subclass:
14 |
15 |
16 | .. code:: python
17 |
18 | from dataclasses import dataclass
19 | from concoursetools import TypedVersion
20 |
21 |
22 | @dataclass()
23 | class GitVersion(TypedVersion):
24 | ref: str
25 |
26 |
27 | Next, create a subclass of :class:`~concoursetools.resource.ConcourseResource`:
28 |
29 | .. code:: python
30 |
31 | from concoursetools import ConcourseResource
32 |
33 |
34 | class GitResource(ConcourseResource[GitVersion]):
35 |
36 | def __init__(self, uri: str, branch: str, private_key: str) -> None:
37 | super().__init__(GitVersion)
38 | self.uri = uri
39 | self.branch = branch
40 | self.private_key = private_key
41 |
42 |
43 | Here, the parameters in the ``__init__`` method will be taken from the ``source`` configuration for the resource.
44 | Now, implement the three methods required to define the behaviour of the resource:
45 |
46 |
47 | .. code:: python
48 |
49 | from pathlib import Path
50 | from typing import Any
51 | from concoursetools import BuildMetadata
52 |
53 |
54 | class GitResource(ConcourseResource[GitVersion]):
55 | ...
56 |
57 | def fetch_new_versions(self, previous_version: GitVersion | None) -> list[GitVersion]:
58 | ...
59 |
60 | def download_version(self, version: GitVersion, destination_dir: pathlib.Path,
61 | build_metadata: BuildMetadata, **kwargs: Any) -> tuple[GitVersion, dict[str, str]]:
62 | ...
63 |
64 | def publish_new_version(self, sources_dir: pathlib.Path, build_metadata: BuildMetadata,
65 | **kwargs: Any) -> tuple[GitVersion, dict[str, str]]:
66 | ...
67 |
68 |
69 | The keyword arguments in :meth:`~concoursetools.resource.ConcourseResource.download_version`
70 | and :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` correspond to ``params`` in the ``get`` step,
71 | and ``get_params`` in the ``put`` step respectively.
72 |
73 | Once you are happy with the resource, freeze your requirements into a ``requirements.txt`` file,
74 | then generate the ``Dockerfile`` using the Concourse Tools CLI:
75 |
76 | .. code:: shell
77 |
78 | $ python3 -m concoursetools dockerfile .
79 |
80 |
81 | Finally, upload the Docker image to a registry, and use it in your pipelines!
82 |
83 |
84 | .. tip::
85 | Check out the :ref:`Examples` section for different ways to leverage Concourse Tools for your use case.
86 |
--------------------------------------------------------------------------------
/docs/source/resource.rst:
--------------------------------------------------------------------------------
1 | Resource
2 | ========
3 |
4 | .. automodule:: concoursetools.resource
5 |
6 |
7 | Concourse Resource Base Class
8 | -----------------------------
9 |
10 | .. autoclass:: concoursetools.resource.ConcourseResource
11 | :members:
12 | :exclude-members: check_main, in_main, out_main
13 |
14 |
15 | Step Metadata
16 | -------------
17 |
18 | The :meth:`~concoursetools.resource.ConcourseResource.download_version` and :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` methods return "step metadata", which is displayed in the Concourse web UI like so:
19 |
20 | .. figure:: _static/step_metadata.png
21 | :scale: 75%
22 |
23 | Concourse metadata example from the :concourse:`git-trigger-example.git-trigger-example`. The red outline shows the metadata.
24 |
25 | This metadata dictionary should contain string-only key/value pairs. This is enforced by Concourse Tools, which casts both to strings before emitting any JSON. This is not easily overloaded, as the metadata is only ever intended for visual consumption.
26 |
27 | .. warning::
28 |
29 | It is okay to leave the metadata dictionary blank, but you **must** return an empty :class:`dict` if so.
30 |
31 | .. tip::
32 |
33 | If you want to return metadata which is *not* a string, then consider "stringifying" it in a readable way:
34 |
35 | .. code:: python3
36 |
37 | response_codes = [200, 200, 200, 500]
38 | metadata = {
39 | "response_codes": ", ".join(response_codes)
40 | }
41 |
42 | .. warning::
43 |
44 | This metadata is only shown after a step runs successfully, and is omitted if it failed.
45 |
--------------------------------------------------------------------------------
/docs/source/testing.rst:
--------------------------------------------------------------------------------
1 | Testing the Resource
2 | =====================
3 |
4 | .. automodule:: concoursetools.testing
5 |
6 |
7 | Using Test Wrappers
8 | -------------------
9 |
10 | Multiple test wrappers are available:
11 |
12 | .. list-table::
13 | :header-rows: 1
14 | :align: left
15 |
16 | * - Test Class
17 | - Input / Output
18 | - Executed As
19 | * - :class:`~concoursetools.testing.SimpleTestResourceWrapper`
20 | - Python
21 | - Resource class
22 | * - :class:`~concoursetools.testing.JSONTestResourceWrapper`
23 | - JSON
24 | - Main scripts
25 | * - :class:`~concoursetools.testing.ConversionTestResourceWrapper`
26 | - Python
27 | - Main scripts
28 | * - :class:`~concoursetools.testing.FileTestResourceWrapper`
29 | - JSON
30 | - External scripts
31 | * - :class:`~concoursetools.testing.FileConversionTestResourceWrapper`
32 | - Python
33 | - External scripts
34 | * - :class:`~concoursetools.testing.DockerTestResourceWrapper`
35 | - JSON
36 | - Docker container
37 | * - :class:`~concoursetools.testing.DockerConversionTestResourceWrapper`
38 | - Python
39 | - Docker container
40 |
41 | Base Wrapper
42 | ------------
43 |
44 | All of the above inherit from :class:`~concoursetools.testing.TestResourceWrapper`:
45 |
46 | .. autoclass:: concoursetools.testing.TestResourceWrapper
47 | :members:
48 |
49 | .. toctree::
50 | mocking
51 | wrappers
52 |
--------------------------------------------------------------------------------
/docs/source/typing.rst:
--------------------------------------------------------------------------------
1 | Typing
2 | ======
3 |
4 | .. automodule:: concoursetools.typing
5 |
6 | Concourse Types
7 | ---------------
8 |
9 | The following types map onto Concourse :concourse:`config-basics.basic-schemas` used throughout the library.
10 |
11 | .. autodata:: concoursetools.typing.ResourceConfig
12 |
13 | .. autodata:: concoursetools.typing.Params
14 |
15 | .. autodata:: concoursetools.typing.Metadata
16 |
17 | .. autodata:: concoursetools.typing.MetadataPair
18 |
19 | .. autodata:: concoursetools.typing.VersionConfig
20 |
21 |
22 | Type Vars
23 | ---------
24 |
25 | .. autodata:: concoursetools.typing.VersionT
26 |
27 | .. autodata:: concoursetools.typing.TypedVersionT
28 |
29 | .. autodata:: concoursetools.typing.SortableVersionT
30 |
31 | Protocols
32 | ---------
33 |
34 | .. autoclass:: concoursetools.typing.VersionProtocol
35 |
36 | .. autoclass:: concoursetools.typing.TypedVersionProtocol
37 |
38 | .. autoclass:: concoursetools.typing.SortableVersionProtocol
39 |
--------------------------------------------------------------------------------
/docs/source/version.rst:
--------------------------------------------------------------------------------
1 | Version
2 | =======
3 |
4 | .. automodule:: concoursetools.version
5 |
6 | Version Parent Class
7 | --------------------
8 |
9 | .. autoclass:: concoursetools.version.Version
10 | :members:
11 |
12 |
13 | Typed Version Class
14 | -------------------
15 |
16 | .. autoclass:: concoursetools.version.TypedVersion
17 | :members: flatten, un_flatten
18 |
19 | Supported Types
20 | _______________
21 |
22 | Concourse Tools has out-of-the-box support for a few common types:
23 |
24 | * :class:`~datetime.datetime`: The datetime object is mapped to an integer timestamp.
25 | * :class:`bool`: The boolean is mapped to the strings ``"True"`` or ``"False"``.
26 | * :class:`~enum.Enum`: Enums are mapped to their *names*, so ``MyEnum.ONE`` maps to ``ONE``.
27 | * :class:`~pathlib.Path`: Paths are mapped to their string representations.
28 |
29 |
30 | Version Comparisons
31 | -------------------
32 |
33 | It is often beneficial to directly compare version within your code.
34 |
35 | Hashing
36 | _______
37 |
38 | By default, every version is hashable, and this :func:`hash` is determined by the version class and the output of :meth:`~concoursetools.version.Version.to_flat_dict`. Key/value pairs are sorted and then hashed as a :class:`tuple`, which is combined with the hash of the class.
39 |
40 | Equality
41 | ________
42 |
43 | By default, two versions are equal if they have the same :func:`hash`. However, you may wish to overload this. For example, consider the following version class:
44 |
45 | .. code:: python3
46 |
47 | class GitBranch(Version):
48 |
49 | def __init__(self, branch_name: str):
50 | self.branch_name = branch_name
51 |
52 | Under the default behaviour, ``GitBranch("main")`` and ``GitBranch("origin/main")`` will be not be considered equal, but this might not be ideal. The fastest way to fix this is to overload :meth:`~object.__eq__` like so:
53 |
54 | .. code:: python3
55 |
56 | class GitBranch(Version):
57 |
58 | def __init__(self, branch_name: str):
59 | self.branch_name = branch_name
60 |
61 | def __eq__(self, other):
62 | return self.branch_name.split("/")[-1] == other.branch_name.split("/")[-1]
63 |
64 |
65 | Ordering
66 | ________
67 |
68 | Sometimes you need to order versions to simplify your scripts (returning the latest version, etc.), but by default versions are **not** comparable in this way. This can be fixed by also inheriting from :class:`SortableVersionMixin`, which expects you to implement :meth:`~object.__lt__`. We want ``version_a <= version_b`` if and only if ``version_a`` is **no older** than ``version_b``.
69 |
70 | .. autoclass:: concoursetools.version.SortableVersionMixin
71 | :members:
72 |
73 |
74 | Multi Versions
75 | --------------
76 |
77 | .. autoclass:: concoursetools.additional.MultiVersion
78 | :members:
79 |
--------------------------------------------------------------------------------
/docs/source/wrappers.rst:
--------------------------------------------------------------------------------
1 | Test Wrappers
2 | =============
3 |
4 | .. autoclass:: concoursetools.testing.SimpleTestResourceWrapper
5 | :members:
6 |
7 | .. autoclass:: concoursetools.testing.JSONTestResourceWrapper
8 | :members:
9 |
10 | .. autoclass:: concoursetools.testing.ConversionTestResourceWrapper
11 | :members:
12 |
13 | .. autoclass:: concoursetools.testing.FileTestResourceWrapper
14 | :members:
15 |
16 | .. autoclass:: concoursetools.testing.FileConversionTestResourceWrapper
17 | :members:
18 |
19 | .. autoclass:: concoursetools.testing.DockerTestResourceWrapper
20 | :members:
21 |
22 | .. autoclass:: concoursetools.testing.DockerConversionTestResourceWrapper
23 | :members:
24 |
25 |
26 | Running External Commands
27 | -------------------------
28 |
29 | .. autofunction:: concoursetools.testing.run_command
30 |
31 | .. autofunction:: concoursetools.testing.run_script
32 |
33 | .. autofunction:: concoursetools.testing.run_docker_container
34 |
--------------------------------------------------------------------------------
/examples/__init__.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 |
--------------------------------------------------------------------------------
/examples/build_status.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | from __future__ import annotations
3 |
4 | from dataclasses import dataclass
5 | from enum import Enum, auto
6 | from pathlib import Path
7 | import subprocess
8 | from typing import Any
9 |
10 | import requests
11 | from requests.auth import AuthBase, HTTPBasicAuth
12 | from urllib3 import disable_warnings as disable_ssl_warnings
13 |
14 | from concoursetools import BuildMetadata
15 | from concoursetools.additional import OutOnlyConcourseResource
16 | from concoursetools.colour import Colour, colour_print
17 | from concoursetools.typing import Metadata
18 | from concoursetools.version import TypedVersion
19 |
20 |
21 | class Driver(Enum):
22 | SERVER = "Bitbucket Server"
23 | CLOUD = "Bitbucket Cloud"
24 |
25 |
26 | class BuildStatus(Enum):
27 | SUCCESSFUL = auto()
28 | INPROGRESS = auto()
29 | FAILED = auto()
30 |
31 |
32 | class BitbucketOAuth(AuthBase):
33 | """
34 | Adds the correct auth token for OAuth access to bitbucket.com.
35 | """
36 | def __init__(self, access_token: str):
37 | self.access_token = access_token
38 |
39 | def __call__(self, request: requests.Request) -> requests.Request:
40 | request.headers["Authorization"] = f"Bearer {self.access_token}"
41 | return request
42 |
43 | @classmethod
44 | def from_client_credentials(cls, client_id: str, client_secret: str) -> "BitbucketOAuth":
45 | token_auth = HTTPBasicAuth(client_id, client_secret)
46 |
47 | url = "https://bitbucket.org/site/oauth2/access_token"
48 | data = {"grant_type": "client_credentials"}
49 |
50 | token_response = requests.post(url, auth=token_auth, data=data)
51 | token_response.raise_for_status()
52 | access_token = token_response.json()["access_token"]
53 | return cls(access_token)
54 |
55 |
56 | @dataclass
57 | class Version(TypedVersion):
58 | build_status: BuildStatus
59 |
60 |
61 | class Resource(OutOnlyConcourseResource[Version]):
62 |
63 | def __init__(self, repository: str | None = None, endpoint: str | None = None,
64 | username: str | None = None, password: str | None = None,
65 | client_id: str | None = None, client_secret: str | None = None,
66 | verify_ssl: bool = True, driver: str = "Bitbucket Server",
67 | debug: bool = False) -> None:
68 | super().__init__(Version)
69 | try:
70 | self.driver = Driver(driver)
71 | except ValueError:
72 | possible_values = {enum.value for enum in Driver._member_map_.values()}
73 | raise ValueError(f"Driver must be one of the following: "
74 | f"{possible_values}, not {driver!r}")
75 |
76 | self.auth = create_auth(username, password, client_id, client_secret)
77 |
78 | self.repository = repository
79 | self.endpoint = endpoint
80 |
81 | self.verify_ssl = verify_ssl
82 | self._debug = debug
83 |
84 | if self.driver is Driver.SERVER:
85 | if endpoint is None:
86 | raise ValueError("Must set endpoint when using Bitbucket Server.")
87 | else:
88 | endpoint = endpoint.rstrip("/")
89 |
90 | if self.driver is Driver.CLOUD:
91 | if repository is None:
92 | raise ValueError("Must set repository when using Bitbucket Cloud.")
93 |
94 | def publish_new_version(self, sources_dir: Path, build_metadata: BuildMetadata,
95 | repository: str, build_status: str, key: str | None = None,
96 | name: str | None = None, build_url: str | None = None,
97 | description: str | None = None,
98 | commit_hash: str | None = None) -> tuple[Version, Metadata]:
99 | self.debug("--DEBUG MODE--")
100 |
101 | try:
102 | status = BuildStatus[build_status]
103 | except KeyError:
104 | possible_values = set(BuildStatus._member_names_)
105 | raise ValueError(f"Build status must be one of the following: "
106 | f"{possible_values}, not {build_status!r}")
107 |
108 | if commit_hash is None:
109 | if repository is None:
110 | raise ValueError("Missing repository parameter.")
111 | repo_path = sources_dir / repository
112 | mercurial_path = repo_path / ".hg"
113 | git_path = repo_path / ".git"
114 |
115 | if mercurial_path.exists():
116 | command = ["hg", "R", str(repo_path), "log",
117 | "--rev", ".", "--template", r"{node}"]
118 | elif git_path.exists():
119 | command = ["git", "-C", str(repo_path), "rev-parse", "HEAD"]
120 | else:
121 | raise RuntimeError("Cannot detect a repository.")
122 |
123 | commit_hash = subprocess.check_output(command).strip().decode()
124 |
125 | self.debug(f"Commit: {commit_hash}")
126 |
127 | build_url = build_url or build_metadata.build_url()
128 |
129 | key = key or build_metadata.BUILD_JOB_NAME or f"one-off-build-{build_metadata.BUILD_ID}"
130 |
131 | self.debug(f"Build URL: {build_url}")
132 |
133 | description = description or f"Concourse CI build, hijack as #{build_metadata.BUILD_ID}"
134 |
135 | if name is None:
136 | if build_metadata.is_one_off_build:
137 | name = f"One-off build #{build_metadata.BUILD_ID}"
138 | else:
139 | name = f"{build_metadata.BUILD_JOB_NAME} #{build_metadata.BUILD_NAME}"
140 |
141 | description = build_metadata.format_string(description)
142 |
143 | if self.driver is Driver.SERVER:
144 | post_url = f"{self.endpoint}/rest/build-status/1.0/commits/{commit_hash}"
145 | if self.verify_ssl is False:
146 | disable_ssl_warnings()
147 | self.debug("SSL warnings disabled\n")
148 |
149 | else:
150 | post_url = f"https://api.bitbucket.org/2.0/repositories/{self.repository}/commit/{commit_hash}/statuses/build"
151 |
152 | data = {
153 | "state": status.name,
154 | "key": key,
155 | "name": name,
156 | "url": build_url,
157 | "description": description,
158 | }
159 |
160 | self.debug(f"Set build status: {data}")
161 |
162 | response = requests.post(post_url, json=data, auth=self.auth, verify=self.verify_ssl)
163 |
164 | self.debug(f"Request result: {response.json()}")
165 |
166 | version = Version(status)
167 | metadata = {
168 | "HTTP Status Code": str(response.status_code),
169 | }
170 | return version, metadata
171 |
172 | def debug(self, *args: object, colour: str = Colour.CYAN, **kwargs: Any) -> None:
173 | if self._debug:
174 | colour_print(*args, colour=colour, **kwargs)
175 |
176 |
177 | def create_auth(username: str | None = None,
178 | password: str | None = None,
179 | client_id: str | None = None,
180 | client_secret: str | None = None) -> AuthBase:
181 | if username is not None and password is not None:
182 | auth: AuthBase = HTTPBasicAuth(username, password)
183 | elif client_id is not None and client_secret is not None:
184 | auth = BitbucketOAuth.from_client_credentials(client_id, client_secret)
185 | else:
186 | raise ValueError("Must set username/password or OAuth credentials")
187 | return auth
188 |
--------------------------------------------------------------------------------
/examples/github_branches.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | from __future__ import annotations
3 |
4 | from dataclasses import dataclass
5 | import re
6 |
7 | import requests
8 |
9 | from concoursetools.additional import MultiVersionConcourseResource
10 | from concoursetools.version import TypedVersion
11 |
12 |
13 | @dataclass(unsafe_hash=True, order=True)
14 | class BranchVersion(TypedVersion):
15 | name: str
16 |
17 |
18 | class Resource(MultiVersionConcourseResource[BranchVersion]): # type: ignore[type-var]
19 | def __init__(self, owner: str, repo: str, regex: str = ".*",
20 | endpoint: str = "https://api.github.com") -> None:
21 | """
22 | Initialise self.
23 |
24 | :param owner: The owner of the repository.
25 | :param repo: The name of the repository.
26 | :param regex: An optional regex for filtering the branches. Only branches matching this
27 | regex will be considered. Defaults to a regex which matches ALL branches.
28 | :param endpoint: The GitHub API endpoint. Defaults to the public version of GitHub.
29 | """
30 | super().__init__("branches", BranchVersion)
31 | self.api_route = f"{endpoint}/repos/{owner}/{repo}/branches"
32 | self.regex = re.compile(regex)
33 |
34 | def fetch_latest_sub_versions(self) -> set[BranchVersion]:
35 | headers = {"Accept": "application/vnd.github+json"}
36 | response = requests.get(self.api_route, headers=headers)
37 | branches_info = response.json()
38 |
39 | try:
40 | branch_names = {branch_info["name"] for branch_info in branches_info}
41 | except TypeError as error: # GitHub error: {"message": "..."}
42 | message = branches_info["message"]
43 | raise RuntimeError(message) from error
44 |
45 | return {BranchVersion(branch_name) for branch_name in branch_names
46 | if self.regex.fullmatch(branch_name)}
47 |
--------------------------------------------------------------------------------
/examples/pipeline.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | from __future__ import annotations
3 |
4 | from collections.abc import Generator
5 | from dataclasses import dataclass
6 | from datetime import datetime
7 | import json
8 | from pathlib import Path
9 |
10 | import boto3
11 |
12 | from concoursetools import ConcourseResource
13 | from concoursetools.metadata import BuildMetadata
14 | from concoursetools.version import TypedVersion
15 |
16 |
17 | class DatetimeSafeJSONEncoder(json.JSONEncoder):
18 |
19 | def default(self, o: object) -> object:
20 | if isinstance(o, datetime):
21 | return o.isoformat()
22 | return super().default(o)
23 |
24 |
25 | @dataclass(unsafe_hash=True)
26 | class ExecutionVersion(TypedVersion):
27 | execution_arn: str
28 |
29 |
30 | class PipelineResource(ConcourseResource[ExecutionVersion]):
31 |
32 | def __init__(self, pipeline: str, statuses: list[str] = ["Succeeded", "Stopped", "Failed"]) -> None:
33 | super().__init__(ExecutionVersion)
34 | # arn:aws:sagemaker:::pipeline:
35 | _, _, _, region, _, _, pipeline_name = pipeline.split(":")
36 | self._client = boto3.client("sagemaker", region_name=region)
37 | self.pipeline_name = pipeline_name
38 | self.statuses = statuses
39 |
40 | def fetch_new_versions(self, previous_version: ExecutionVersion | None = None) -> list[ExecutionVersion]:
41 | potential_versions = iter(self._yield_potential_execution_versions())
42 | if previous_version is None:
43 | try:
44 | first_version = next(potential_versions)
45 | except StopIteration:
46 | new_versions = []
47 | else:
48 | new_versions = [first_version]
49 | else:
50 | new_versions = []
51 | for potential_version in potential_versions:
52 | new_versions.append(potential_version)
53 | if potential_version == previous_version:
54 | break
55 | else:
56 | new_versions = [new_versions[0]]
57 |
58 | new_versions.reverse()
59 | return new_versions
60 |
61 | def download_version(self, version: ExecutionVersion, destination_dir: Path,
62 | build_metadata: BuildMetadata, download_pipeline: bool = True,
63 | metadata_file: str = "metadata.json",
64 | pipeline_file: str = "pipeline.json") -> tuple[ExecutionVersion, dict[str, str]]:
65 | response = self._client.describe_pipeline_execution(PipelineExecutionArn=version.execution_arn)
66 | response.pop("ResponseMetadata")
67 |
68 | metadata_path = destination_dir / metadata_file
69 | metadata_path.write_text(json.dumps(response, cls=DatetimeSafeJSONEncoder))
70 |
71 | if download_pipeline:
72 | pipeline_response = self._client.describe_pipeline_definition_for_execution(PipelineExecutionArn=version.execution_arn)
73 | pipeline_path = destination_dir / pipeline_file
74 | pipeline_path.write_text(pipeline_response["PipelineDefinition"])
75 |
76 | metadata = {
77 | "Display Name": response.get("PipelineExecutionDisplayName"),
78 | "Status": response["PipelineExecutionStatus"],
79 | "Created By": response["CreatedBy"]["UserProfileName"],
80 | "Description": response.get("PipelineExecutionDescription"),
81 | }
82 |
83 | if metadata["Status"] == "Failed":
84 | metadata["Failure Reason"] = response["FailureReason"]
85 |
86 | metadata = {key: value for key, value in metadata.items() if value is not None}
87 |
88 | return version, metadata
89 |
90 | def publish_new_version(self, sources_dir: Path, build_metadata: BuildMetadata,
91 | display_name: str | None = None, description: str | None = None,
92 | parameters: dict[str, str] = {}) -> tuple[ExecutionVersion, dict[str, str]]:
93 | default_description = (f"Execution from build #{build_metadata.BUILD_ID} "
94 | f"of pipeline {build_metadata.BUILD_PIPELINE_NAME}")
95 | kwargs: dict[str, object] = {
96 | "PipelineName": self.pipeline_name,
97 | "PipelineExecutionDescription": description or default_description,
98 | }
99 |
100 | if display_name:
101 | kwargs["PipelineExecutionDisplayName"] = display_name
102 |
103 | if parameters:
104 | kwargs["PipelineParameters"] = [{"Name": name, "Value": value}
105 | for name, value in parameters.items()]
106 | metadata = {f"Parameter: {parameter}": value
107 | for parameter, value in parameters.items()}
108 | else:
109 | metadata = {}
110 |
111 | response = self._client.start_pipeline_execution(**kwargs)
112 | execution_arn = response["PipelineExecutionArn"]
113 | new_version = ExecutionVersion(execution_arn)
114 | return new_version, metadata
115 |
116 | def _yield_potential_execution_versions(self) -> Generator[ExecutionVersion, None, None]:
117 | kwargs = {
118 | "PipelineName": self.pipeline_name,
119 | "SortOrder": "Descending",
120 | }
121 |
122 | first_response = self._client.list_pipeline_executions(**kwargs)
123 |
124 | response = first_response
125 | while True:
126 | for summary in response["PipelineExecutionSummaries"]:
127 | if summary["PipelineExecutionStatus"] in self.statuses:
128 | yield ExecutionVersion(summary["PipelineExecutionArn"])
129 |
130 | try:
131 | next_token = response["NextToken"]
132 | except KeyError:
133 | break
134 |
135 | response = self._client.list_pipeline_executions(**kwargs, NextToken=next_token)
136 |
--------------------------------------------------------------------------------
/examples/s3.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | from __future__ import annotations
3 |
4 | from datetime import timedelta
5 | from pathlib import Path
6 |
7 | import boto3
8 | from botocore.client import Config
9 |
10 | from concoursetools import BuildMetadata
11 | from concoursetools.additional import InOnlyConcourseResource
12 |
13 |
14 | class S3SignedURLConcourseResource(InOnlyConcourseResource):
15 | """
16 | A Concourse resource type for generating pre-signed URLs for items in S3 buckets.
17 | """
18 | def __init__(self, bucket_name: str, region_name: str) -> None:
19 | """
20 | Initialise self.
21 |
22 | :param bucket_name: The name of your bucket.
23 | :param region_name: The name of the region in which your bucket resides.
24 | """
25 | super().__init__()
26 | self.bucket_name = bucket_name
27 | self.client = boto3.client("s3", region_name=region_name,
28 | config=Config(signature_version="s3v4"))
29 |
30 | def download_data(self, destination_dir: Path, build_metadata: BuildMetadata,
31 | file_path: str, expires_in: dict[str, float],
32 | file_name: str | None = None,
33 | url_file: str = "url") -> dict[str, str]:
34 | params = {
35 | "Bucket": self.bucket_name,
36 | "Key": file_path,
37 | }
38 | if file_name is not None:
39 | # https://stackoverflow.com/a/2612795
40 | content_disposition = f"attachment; filename=\"{file_name}\""
41 | params["ResponseContentDisposition"] = content_disposition
42 |
43 | expiry_seconds = int(timedelta(**expires_in).total_seconds())
44 | url = self.client.generate_presigned_url(ClientMethod="get_object",
45 | Params=params,
46 | ExpiresIn=expiry_seconds)
47 |
48 | url_file_path = destination_dir / url_file
49 | url_file_path.write_text(url)
50 |
51 | return {}
52 |
--------------------------------------------------------------------------------
/examples/secrets.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | from __future__ import annotations
3 |
4 | from dataclasses import dataclass
5 | from datetime import datetime
6 | import json as json_package
7 | from pathlib import Path
8 | from typing import Any
9 |
10 | import boto3
11 | from botocore.exceptions import ClientError
12 |
13 | from concoursetools import BuildMetadata
14 | from concoursetools.additional import TriggerOnChangeConcourseResource
15 | from concoursetools.version import TypedVersion
16 |
17 |
18 | class DatetimeSafeJSONEncoder(json_package.JSONEncoder):
19 |
20 | def default(self, o: object) -> object:
21 | if isinstance(o, datetime):
22 | return o.isoformat()
23 | return super().default(o)
24 |
25 |
26 | @dataclass(unsafe_hash=True)
27 | class SecretVersion(TypedVersion):
28 | version_id: str
29 |
30 |
31 | class Resource(TriggerOnChangeConcourseResource[SecretVersion]):
32 | """
33 | :param secret: The full Amazon Resource Name (ARN) of the secret.
34 | """
35 | def __init__(self, secret: str) -> None:
36 | super().__init__(SecretVersion)
37 | self.secret = secret
38 |
39 | # arn:aws:secretsmanager:::secret:
40 | _, _, _, region, _, _, _ = secret.split(":")
41 | self._client = boto3.client("secretsmanager", region_name=region)
42 |
43 | def fetch_latest_version(self) -> SecretVersion:
44 | try:
45 | response = self._client.list_secret_version_ids(SecretId=self.secret,
46 | IncludeDeprecated=False)
47 | except ClientError:
48 | raise ValueError(f"Cannot find secret: {self.secret!r}")
49 |
50 | versions = response["Versions"]
51 | for version in versions:
52 | version_stages = version["VersionStages"]
53 | if "AWSCURRENT" in version_stages:
54 | version_id = version["VersionId"]
55 | return SecretVersion(version_id)
56 | raise RuntimeError("No current version of the secret could be found.")
57 |
58 | def download_version(self, version: SecretVersion, destination_dir: Path,
59 | build_metadata: BuildMetadata, value: bool = False,
60 | metadata_file: str = "metadata.json",
61 | value_file: str = "value") -> tuple[SecretVersion, dict[str, str]]:
62 | meta_response: dict[str, Any] = self._client.describe_secret(SecretId=self.secret)
63 | meta_response.pop("ResponseMetadata")
64 |
65 | metadata_path = destination_dir / metadata_file
66 | metadata_path.write_text(json_package.dumps(meta_response,
67 | cls=DatetimeSafeJSONEncoder))
68 |
69 | if value:
70 | value_response = self._client.get_secret_value(SecretId=self.secret)
71 | value_path = destination_dir / value_file
72 |
73 | try:
74 | secret_value = value_response["SecretString"]
75 | except KeyError:
76 | secret_value_as_bytes: bytes = value_response["SecretBinary"]
77 | value_path.write_bytes(secret_value_as_bytes)
78 | else:
79 | value_path.write_text(secret_value)
80 |
81 | return version, {}
82 |
83 | def publish_new_version(self, sources_dir: Path,
84 | build_metadata: BuildMetadata, string: str | None = None,
85 | file: str | None = None,
86 | json: dict[str, str] | None = None) -> tuple[SecretVersion, dict[str, str]]:
87 | if json is not None:
88 | string = json_package.dumps(json)
89 |
90 | if string is not None:
91 | response = self._client.put_secret_value(SecretId=self.secret,
92 | SecretString=string)
93 | elif file is not None:
94 | file_path = sources_dir / file
95 | file_contents = file_path.read_bytes()
96 | response = self._client.put_secret_value(SecretId=self.secret,
97 | SecretBinary=file_contents)
98 | else:
99 | raise ValueError("Missing new value for the secret.")
100 |
101 | version_id = response["VersionId"]
102 | metadata = {
103 | "Version Staging Labels": ", ".join(response["VersionStages"])
104 | }
105 | return SecretVersion(version_id), metadata
106 |
--------------------------------------------------------------------------------
/examples/xkcd.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | from __future__ import annotations
3 |
4 | from collections.abc import Generator
5 | from dataclasses import dataclass
6 | from datetime import datetime
7 | import json
8 | from pathlib import Path
9 | import urllib.parse
10 | import xml.etree.ElementTree as ET
11 |
12 | import requests
13 |
14 | from concoursetools.additional import SelfOrganisingConcourseResource
15 | from concoursetools.metadata import BuildMetadata
16 | from concoursetools.version import SortableVersionMixin, TypedVersion
17 |
18 |
19 | @dataclass(unsafe_hash=True)
20 | class ComicVersion(TypedVersion, SortableVersionMixin):
21 | comic_id: int
22 |
23 | def __lt__(self, other: object) -> bool:
24 | if not isinstance(other, type(self)):
25 | return NotImplemented
26 | return self.comic_id < other.comic_id
27 |
28 |
29 | class XKCDResource(SelfOrganisingConcourseResource[ComicVersion]):
30 |
31 | def __init__(self, url: str = "https://xkcd.com"):
32 | super().__init__(ComicVersion)
33 | self.url = url
34 |
35 | def fetch_all_versions(self) -> set[ComicVersion]:
36 | atom_url = f"{self.url}/atom.xml"
37 | response = requests.get(atom_url)
38 | feed_data = response.text
39 | return {ComicVersion(comic_id) for comic_id in yield_comic_ids(feed_data)}
40 |
41 | def download_version(self, version: ComicVersion, destination_dir: Path,
42 | build_metadata: BuildMetadata, image: bool = True,
43 | link: bool = True, alt: bool = True) -> tuple[ComicVersion, dict[str, str]]:
44 | comic_info_url = f"{self.url}/{version.comic_id}/info.0.json"
45 | response = requests.get(comic_info_url)
46 | info = response.json()
47 |
48 | title = info["title"]
49 | url = f"{self.url}/{version.comic_id}/"
50 |
51 | upload_date = datetime(year=int(info["year"]), month=int(info["month"]),
52 | day=int(info["day"]))
53 | metadata = {
54 | "Title": title,
55 | "Uploaded": upload_date.strftime(r"%d/%m/%Y"),
56 | "URL": f"{self.url}/{version.comic_id}/",
57 | }
58 |
59 | info_path = destination_dir / "info.json"
60 | info_path.write_text(json.dumps(info))
61 |
62 | if image:
63 | image_path = destination_dir / "image.png"
64 | image_request = requests.get(info["img"], stream=True)
65 | with open(image_path, "wb") as wf:
66 | for chunk in image_request:
67 | wf.write(chunk)
68 |
69 | if link:
70 | link_path = destination_dir / "link.txt"
71 | link_path.write_text(url)
72 |
73 | if alt:
74 | alt_path = destination_dir / "alt.txt"
75 | alt_path.write_text(info["alt"])
76 |
77 | return version, metadata
78 |
79 | def publish_new_version(self, sources_dir: Path, build_metadata: BuildMetadata) -> tuple[ComicVersion, dict[str, str]]:
80 | raise NotImplementedError
81 |
82 |
83 | def yield_comic_ids(xml_data: str) -> Generator[int, None, None]:
84 | for comic_url in yield_comic_links(xml_data):
85 | parsed_url = urllib.parse.urlparse(comic_url)
86 | comic_id = parsed_url.path.strip("/")
87 | yield int(comic_id)
88 |
89 |
90 | def yield_comic_links(xml_data: str) -> Generator[str, None, None]:
91 | root = ET.fromstring(xml_data)
92 | for entry in root:
93 | if entry.tag.endswith("entry"):
94 | for child in entry:
95 | if child.tag.endswith("link"):
96 | items = dict(child.items())
97 | yield items["href"]
98 |
--------------------------------------------------------------------------------
/examples/xkcd.xml:
--------------------------------------------------------------------------------
1 |
2 | xkcd.com
3 |
4 | https://xkcd.com/
5 | 2023-08-07T00:00:00Z
6 |
7 | Solar Panel Placement
8 |
9 | 2023-08-07T00:00:00Z
10 | https://xkcd.com/2812/
11 |
12 |
13 |
14 |
15 |
16 | Free Fallin'
17 |
18 | 2023-08-04T00:00:00Z
19 | https://xkcd.com/2811/
20 |
21 |
22 |
23 |
24 |
25 | How to Coil a Cable
26 |
27 | 2023-08-02T00:00:00Z
28 | https://xkcd.com/2810/
29 |
30 |
31 |
32 |
33 |
34 | Moon
35 |
36 | 2023-07-31T00:00:00Z
37 | https://xkcd.com/2809/
38 |
39 |
40 |
41 |
42 |
43 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [build-system]
2 | requires = ["setuptools"]
3 | build-backend = "setuptools.build_meta"
4 |
5 | [project]
6 | name = "concoursetools"
7 | dynamic = ["version"]
8 | description = "Easily create Concourse resource types in Python."
9 | readme = "README.md"
10 | requires-python = ">=3.9"
11 | keywords = [
12 | "concourse",
13 | "ci",
14 | "cd",
15 | ]
16 | classifiers = [
17 | "Development Status :: 4 - Beta",
18 | "Intended Audience :: Developers",
19 | "License :: OSI Approved :: Apache Software License",
20 | "Natural Language :: English",
21 | "Programming Language :: Python :: 3.9",
22 | "Programming Language :: Python :: 3.10",
23 | "Programming Language :: Python :: 3.11",
24 | "Programming Language :: Python :: 3.12",
25 | "Programming Language :: Python :: 3.13",
26 | "Programming Language :: Python :: 3.14",
27 | "Topic :: Software Development",
28 | "Typing :: Typed",
29 | ]
30 | authors = [
31 | {name = "GCHQ", email = "oss@gchq.gov.uk"},
32 | ]
33 | license = {text = "Apache License 2.0"}
34 |
35 | [project.urls]
36 | "Homepage" = "https://github.com/gchq/ConcourseTools/"
37 | "Documentation" = "https://concoursetools.readthedocs.io/en/stable/"
38 | "Repository" = "https://github.com/gchq/ConcourseTools/"
39 | "Bug Tracker" = "https://github.com/gchq/ConcourseTools/issues"
40 | "Changelog" = "https://concoursetools.readthedocs.io/en/latest/whats_new.html"
41 |
42 | [tool.setuptools]
43 | packages = [
44 | "concoursetools",
45 | "concoursetools.cli",
46 | ]
47 |
48 | [tool.setuptools.dynamic]
49 | version = {attr = "concoursetools.__version__"}
50 |
51 |
52 | [tool.autopep8]
53 | max_line_length = 150
54 | ignore = ["E301", "E501"]
55 | in-place = true
56 | recursive = true
57 | aggressive = 3
58 |
59 |
60 | [tool.coverage.run]
61 | command_line = "-m unittest discover"
62 | source = ["concoursetools"]
63 | omit = ["concoursetools/colour.py", "concoursetools/typing.py", "*/__init__.py", "*/__main__.py"]
64 |
65 | [tool.coverage.report]
66 | sort = "Cover"
67 | exclude_lines = [
68 | "@abstractmethod",
69 | "def __repr__",
70 | "pass",
71 | "raise$",
72 | "raise RuntimeError",
73 | ]
74 |
75 |
76 | [tool.isort]
77 | line_length = 150
78 | force_sort_within_sections = true
79 |
80 |
81 | [tool.mypy]
82 | python_version = "3.12"
83 | check_untyped_defs = true
84 | disallow_untyped_defs = true
85 | warn_redundant_casts = true
86 | warn_unused_ignores = true
87 | warn_return_any = true
88 | disallow_any_generics = true
89 | disable_error_code = ["override"]
90 |
91 |
92 | [tool.pylint]
93 | max-line-length = 150
94 | disable = ["too-many-arguments", "too-few-public-methods"]
95 | good-names = ["wf"]
96 | ignore-paths = ["docs/source/conf.py", "tests"]
97 |
98 |
99 | [tool.pydocstyle]
100 | ignore = ["D102", "D105", "D107", "D200", "D203", "D204", "D205", "D212", "D400", "D415"]
101 |
--------------------------------------------------------------------------------
/requirements-tests.txt:
--------------------------------------------------------------------------------
1 | boto3==1.36.25
2 | botocore==1.36.25
3 | certifi==2025.1.31
4 | cffi==1.17.1
5 | charset-normalizer==3.4.1
6 | cryptography==44.0.1
7 | idna==3.10
8 | Jinja2==3.1.5
9 | jmespath==1.0.1
10 | MarkupSafe==3.0.2
11 | moto==5.0.12
12 | pycparser==2.22
13 | python-dateutil==2.9.0.post0
14 | PyYAML==6.0.2
15 | requests==2.32.3
16 | responses==0.25.6
17 | s3transfer==0.11.2
18 | six==1.17.0
19 | urllib3==2.3.0
20 | Werkzeug==3.1.3
21 | xmltodict==0.14.2
22 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Setup for the concoursetools Python package.
4 | """
5 | from setuptools import setup
6 |
7 | if __name__ == "__main__":
8 | setup()
9 |
--------------------------------------------------------------------------------
/sonar-project.properties:
--------------------------------------------------------------------------------
1 | sonar.projectDescription="Static analysis for the Concourse Tools Python library."
2 |
3 | sonar.sources=concoursetools
4 | sonar.tests=tests
5 |
6 | sonar.python.version=3.9,3.10,3.11,3.12,3.13,3.14
7 |
8 | sonar.coverage.exclusions=docs/**/*,tests/**/*,coverage.xml,concoursetools/colour.py,concoursetools/typing.py,**/__init__.py,**/__main__.py
9 | sonar.python.coverage.reportPaths=coverage.xml
10 | sonar.cobertura.reportPath=coverage.xml
11 |
--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 |
--------------------------------------------------------------------------------
/tests/resource.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Contains a test resource.
4 | """
5 | from __future__ import annotations
6 |
7 | from copy import copy
8 | from dataclasses import dataclass
9 | from pathlib import Path
10 |
11 | import concoursetools
12 | from concoursetools.metadata import BuildMetadata
13 | from concoursetools.typing import Metadata
14 |
15 |
16 | class TestVersion(concoursetools.Version):
17 |
18 | def __init__(self, ref: str):
19 | self.ref = ref
20 |
21 |
22 | class TestResource(concoursetools.ConcourseResource[TestVersion]):
23 |
24 | def __init__(self, uri: str, branch: str = "main", private_key: str | None = None):
25 | super().__init__(TestVersion)
26 | self.uri = uri
27 | self.branch = branch
28 | self.private_key = private_key
29 |
30 | def fetch_new_versions(self, previous_version: TestVersion | None = None) -> list[TestVersion]:
31 | if previous_version:
32 | print("Previous version found.")
33 | return [TestVersion("7154fe")]
34 | else:
35 | return [TestVersion(ref) for ref in ("61cbef", "d74e01", "7154fe")]
36 |
37 | def download_version(self, version: TestVersion, destination_dir: Path, build_metadata: concoursetools.BuildMetadata,
38 | file_name: str = "README.txt") -> tuple[TestVersion, Metadata]:
39 | print("Downloading.")
40 | readme_path = destination_dir / file_name
41 | readme_path.write_text(f"Downloaded README for ref {version.ref}.\n")
42 | metadata = {
43 | "team_name": build_metadata.BUILD_TEAM_NAME,
44 | }
45 | return version, metadata
46 |
47 | def publish_new_version(self, sources_dir: Path, build_metadata: concoursetools.BuildMetadata, repo: str,
48 | ref_file: str = "ref.txt") -> tuple[TestVersion, Metadata]:
49 | ref_path = sources_dir / repo / ref_file
50 | ref = ref_path.read_text()
51 | print("Uploading.")
52 | return TestVersion(ref), {}
53 |
54 |
55 | @dataclass
56 | class ConcourseMockVersion(concoursetools.TypedVersion):
57 | version: int
58 | privileged: bool
59 |
60 |
61 | ConcourseMockVersion._flatten_functions = copy(ConcourseMockVersion._flatten_functions)
62 | ConcourseMockVersion._un_flatten_functions = copy(ConcourseMockVersion._un_flatten_functions)
63 |
64 |
65 | @ConcourseMockVersion.flatten
66 | def _(obj: bool) -> str:
67 | return str(obj).lower()
68 |
69 |
70 | @ConcourseMockVersion.un_flatten
71 | def _(_type: type[bool], obj: str) -> bool:
72 | return obj == "true"
73 |
74 |
75 | class ConcourseMockResource(concoursetools.ConcourseResource[ConcourseMockVersion]):
76 |
77 | def __init__(self, **kwargs: object) -> None:
78 | super().__init__(ConcourseMockVersion)
79 |
80 | def fetch_new_versions(self, previous_version: ConcourseMockVersion | None = None) -> list[ConcourseMockVersion]:
81 | raise NotImplementedError
82 |
83 | def download_version(self, version: ConcourseMockVersion, destination_dir: Path,
84 | build_metadata: BuildMetadata) -> tuple[ConcourseMockVersion, Metadata]:
85 | raise NotImplementedError
86 |
87 | def publish_new_version(self, sources_dir: Path, build_metadata: BuildMetadata) -> tuple[ConcourseMockVersion, Metadata]:
88 | raise NotImplementedError
89 |
--------------------------------------------------------------------------------
/tests/test_cli_commands.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | from contextlib import redirect_stdout
3 | from io import StringIO
4 | import os
5 | from pathlib import Path
6 | import shutil
7 | import sys
8 | from tempfile import TemporaryDirectory
9 | import textwrap
10 | import unittest
11 | import unittest.mock
12 |
13 | from concoursetools.cli import cli
14 | from concoursetools.colour import Colour, colourise
15 |
16 |
17 | class AssetTests(unittest.TestCase):
18 | """
19 | Tests for creation of the asset files.
20 | """
21 | def setUp(self) -> None:
22 | """Code to run before each test."""
23 | self._temp_dir = TemporaryDirectory()
24 | self.temp_dir = Path(self._temp_dir.name)
25 | self._original_dir = Path.cwd()
26 |
27 | path_to_this_file = Path(__file__)
28 | path_to_test_resource_module = path_to_this_file.parent / "resource.py"
29 | shutil.copyfile(path_to_test_resource_module, self.temp_dir / "concourse.py")
30 | os.chdir(self.temp_dir)
31 |
32 | def tearDown(self) -> None:
33 | """Code to run after each test."""
34 | os.chdir(self._original_dir)
35 | self._temp_dir.cleanup()
36 |
37 | def test_asset_scripts(self) -> None:
38 | asset_dir = self.temp_dir / "assets"
39 | self.assertFalse(asset_dir.exists())
40 |
41 | new_stdout = StringIO()
42 | with redirect_stdout(new_stdout):
43 | cli.invoke(["assets", "assets", "-c", "TestResource"])
44 |
45 | self.assertEqual(new_stdout.getvalue(), "")
46 |
47 | self.assertTrue(asset_dir.exists())
48 | self.assertSetEqual({path.name for path in asset_dir.iterdir()}, {"check", "in", "out"})
49 |
50 |
51 | class DockerfileTests(unittest.TestCase):
52 | """
53 | Tests for creating the Dockerfile.
54 | """
55 | def setUp(self) -> None:
56 | """Code to run before each test."""
57 | self._temp_dir = TemporaryDirectory()
58 | self.temp_dir = Path(self._temp_dir.name)
59 | self._original_dir = Path.cwd()
60 |
61 | path_to_this_file = Path(__file__)
62 | path_to_test_resource_module = path_to_this_file.parent / "resource.py"
63 | shutil.copyfile(path_to_test_resource_module, self.temp_dir / "concourse.py")
64 | self.dockerfile_path = self.temp_dir / "Dockerfile"
65 | self.assertFalse(self.dockerfile_path.exists())
66 |
67 | self.current_python_string = f"{sys.version_info.major}.{sys.version_info.minor}"
68 |
69 | os.chdir(self.temp_dir)
70 |
71 | def tearDown(self) -> None:
72 | """Code to run after each test."""
73 | os.chdir(self._original_dir)
74 | self._temp_dir.cleanup()
75 |
76 | def test_docker(self) -> None:
77 | new_stdout = StringIO()
78 | with redirect_stdout(new_stdout):
79 | cli.invoke(["dockerfile", "."])
80 |
81 | self.assertEqual(new_stdout.getvalue(), "")
82 |
83 | dockerfile_contents = self.dockerfile_path.read_text()
84 | expected_contents = textwrap.dedent(f"""
85 | FROM python:{self.current_python_string}
86 |
87 | RUN python3 -m venv /opt/venv
88 | # Activate venv
89 | ENV PATH="/opt/venv/bin:$PATH"
90 |
91 | COPY requirements.txt requirements.txt
92 |
93 | RUN \\
94 | python3 -m pip install --upgrade pip && \\
95 | pip install -r requirements.txt --no-deps
96 |
97 | WORKDIR /opt/resource/
98 | COPY concourse.py ./concourse.py
99 | RUN python3 -m concoursetools assets . -r concourse.py
100 |
101 | ENTRYPOINT ["python3"]
102 | """).lstrip()
103 | self.assertEqual(dockerfile_contents, expected_contents)
104 |
105 |
106 | class LegacyTests(unittest.TestCase):
107 | """
108 | Tests for the legacy CLI.
109 | """
110 | def setUp(self) -> None:
111 | """Code to run before each test."""
112 | self._temp_dir = TemporaryDirectory()
113 | self.temp_dir = Path(self._temp_dir.name)
114 | self._original_dir = Path.cwd()
115 |
116 | path_to_this_file = Path(__file__)
117 | path_to_test_resource_module = path_to_this_file.parent / "resource.py"
118 | shutil.copyfile(path_to_test_resource_module, self.temp_dir / "concourse.py")
119 | self.dockerfile_path = self.temp_dir / "Dockerfile"
120 | self.assertFalse(self.dockerfile_path.exists())
121 |
122 | self.current_python_string = f"{sys.version_info.major}.{sys.version_info.minor}"
123 |
124 | os.chdir(self.temp_dir)
125 |
126 | def tearDown(self) -> None:
127 | """Code to run after each test."""
128 | os.chdir(self._original_dir)
129 | self._temp_dir.cleanup()
130 |
131 | def test_docker(self) -> None:
132 | new_stdout = StringIO()
133 | with redirect_stdout(new_stdout):
134 | cli.invoke(["legacy", ".", "--docker"])
135 |
136 | self.assertEqual(new_stdout.getvalue(), colourise(textwrap.dedent("""
137 | The legacy CLI has been deprecated.
138 | Please refer to the documentation or help pages for the up to date CLI.
139 | This CLI will be removed in version 0.10.0, or in version 1.0.0, whichever is sooner.
140 |
141 | """), colour=Colour.RED))
142 |
143 | dockerfile_contents = self.dockerfile_path.read_text()
144 | expected_contents = textwrap.dedent(f"""
145 | FROM python:{self.current_python_string}-alpine
146 |
147 | COPY requirements.txt requirements.txt
148 |
149 | RUN \\
150 | python3 -m pip install --upgrade pip && \\
151 | pip install -r requirements.txt --no-deps
152 |
153 | WORKDIR /opt/resource/
154 | COPY concourse.py ./concourse.py
155 | RUN python3 -m concoursetools assets . -r concourse.py
156 |
157 | ENTRYPOINT ["python3"]
158 | """).lstrip()
159 | self.assertEqual(dockerfile_contents, expected_contents)
160 |
161 | def test_asset_scripts(self) -> None:
162 | asset_dir = self.temp_dir / "assets"
163 | self.assertFalse(asset_dir.exists())
164 |
165 | new_stdout = StringIO()
166 | with redirect_stdout(new_stdout):
167 | cli.invoke(["legacy", "assets", "-c", "TestResource"])
168 |
169 | self.assertEqual(new_stdout.getvalue(), colourise(textwrap.dedent("""
170 | The legacy CLI has been deprecated.
171 | Please refer to the documentation or help pages for the up to date CLI.
172 | This CLI will be removed in version 0.10.0, or in version 1.0.0, whichever is sooner.
173 |
174 | """), colour=Colour.RED))
175 |
176 | self.assertTrue(asset_dir.exists())
177 | self.assertSetEqual({path.name for path in asset_dir.iterdir()}, {"check", "in", "out"})
178 |
--------------------------------------------------------------------------------
/tests/test_dockertools.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Tests for the dockertools module.
4 | """
5 | from pathlib import Path
6 | import shutil
7 | import sys
8 | from tempfile import TemporaryDirectory
9 | import textwrap
10 | from unittest import TestCase
11 |
12 | from concoursetools.cli import commands as cli_commands
13 |
14 |
15 | class DockerTests(TestCase):
16 | """
17 | Tests for the creation of the Dockerfile.
18 | """
19 | def setUp(self) -> None:
20 | """Code to run before each test."""
21 | self._temp_dir = TemporaryDirectory()
22 | self.temp_dir = Path(self._temp_dir.name)
23 |
24 | path_to_this_file = Path(__file__)
25 | path_to_test_resource_module = path_to_this_file.parent / "resource.py"
26 | shutil.copyfile(path_to_test_resource_module, self.temp_dir / "concourse.py")
27 | self.dockerfile_path = self.temp_dir / "Dockerfile"
28 | self.assertFalse(self.dockerfile_path.exists())
29 |
30 | self.current_python_string = f"{sys.version_info.major}.{sys.version_info.minor}"
31 |
32 | def tearDown(self) -> None:
33 | """Code to run after each test."""
34 | self._temp_dir.cleanup()
35 |
36 | def test_basic_config(self) -> None:
37 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py")
38 | dockerfile_contents = self.dockerfile_path.read_text()
39 | expected_contents = textwrap.dedent(f"""
40 | FROM python:{self.current_python_string}
41 |
42 | RUN python3 -m venv /opt/venv
43 | # Activate venv
44 | ENV PATH="/opt/venv/bin:$PATH"
45 |
46 | COPY requirements.txt requirements.txt
47 |
48 | RUN \\
49 | python3 -m pip install --upgrade pip && \\
50 | pip install -r requirements.txt --no-deps
51 |
52 | WORKDIR /opt/resource/
53 | COPY concourse.py ./concourse.py
54 | RUN python3 -m concoursetools assets . -r concourse.py
55 |
56 | ENTRYPOINT ["python3"]
57 | """).lstrip()
58 | self.assertEqual(dockerfile_contents, expected_contents)
59 |
60 | def test_basic_config_custom_image_and_tag(self) -> None:
61 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", image="node", tag="lts-slim")
62 | dockerfile_contents = self.dockerfile_path.read_text()
63 | expected_contents = textwrap.dedent("""
64 | FROM node:lts-slim
65 |
66 | RUN python3 -m venv /opt/venv
67 | # Activate venv
68 | ENV PATH="/opt/venv/bin:$PATH"
69 |
70 | COPY requirements.txt requirements.txt
71 |
72 | RUN \\
73 | python3 -m pip install --upgrade pip && \\
74 | pip install -r requirements.txt --no-deps
75 |
76 | WORKDIR /opt/resource/
77 | COPY concourse.py ./concourse.py
78 | RUN python3 -m concoursetools assets . -r concourse.py
79 |
80 | ENTRYPOINT ["python3"]
81 | """).lstrip()
82 | self.assertEqual(dockerfile_contents, expected_contents)
83 |
84 | def test_basic_config_pip_args(self) -> None:
85 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", pip_args="--trusted-host pypi.org")
86 | dockerfile_contents = self.dockerfile_path.read_text()
87 | expected_contents = textwrap.dedent(f"""
88 | FROM python:{self.current_python_string}
89 |
90 | RUN python3 -m venv /opt/venv
91 | # Activate venv
92 | ENV PATH="/opt/venv/bin:$PATH"
93 |
94 | COPY requirements.txt requirements.txt
95 |
96 | RUN \\
97 | python3 -m pip install --upgrade pip --trusted-host pypi.org && \\
98 | pip install -r requirements.txt --no-deps --trusted-host pypi.org
99 |
100 | WORKDIR /opt/resource/
101 | COPY concourse.py ./concourse.py
102 | RUN python3 -m concoursetools assets . -r concourse.py
103 |
104 | ENTRYPOINT ["python3"]
105 | """).lstrip()
106 | self.assertEqual(dockerfile_contents, expected_contents)
107 |
108 | def test_basic_config_no_venv(self) -> None:
109 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", no_venv=True)
110 | dockerfile_contents = self.dockerfile_path.read_text()
111 | expected_contents = textwrap.dedent(f"""
112 | FROM python:{self.current_python_string}
113 |
114 | COPY requirements.txt requirements.txt
115 |
116 | RUN \\
117 | python3 -m pip install --upgrade pip && \\
118 | pip install -r requirements.txt --no-deps
119 |
120 | WORKDIR /opt/resource/
121 | COPY concourse.py ./concourse.py
122 | RUN python3 -m concoursetools assets . -r concourse.py
123 |
124 | ENTRYPOINT ["python3"]
125 | """).lstrip()
126 | self.assertEqual(dockerfile_contents, expected_contents)
127 |
128 | def test_basic_config_with_suffix(self) -> None:
129 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", suffix="slim")
130 | dockerfile_contents = self.dockerfile_path.read_text()
131 | expected_contents = textwrap.dedent(f"""
132 | FROM python:{self.current_python_string}-slim
133 |
134 | RUN python3 -m venv /opt/venv
135 | # Activate venv
136 | ENV PATH="/opt/venv/bin:$PATH"
137 |
138 | COPY requirements.txt requirements.txt
139 |
140 | RUN \\
141 | python3 -m pip install --upgrade pip && \\
142 | pip install -r requirements.txt --no-deps
143 |
144 | WORKDIR /opt/resource/
145 | COPY concourse.py ./concourse.py
146 | RUN python3 -m concoursetools assets . -r concourse.py
147 |
148 | ENTRYPOINT ["python3"]
149 | """).lstrip()
150 | self.assertEqual(dockerfile_contents, expected_contents)
151 |
152 | def test_basic_config_with_different_name(self) -> None:
153 | cli_commands.dockerfile(str(self.temp_dir), resource_file="resource.py")
154 | dockerfile_contents = self.dockerfile_path.read_text()
155 | expected_contents = textwrap.dedent(f"""
156 | FROM python:{self.current_python_string}
157 |
158 | RUN python3 -m venv /opt/venv
159 | # Activate venv
160 | ENV PATH="/opt/venv/bin:$PATH"
161 |
162 | COPY requirements.txt requirements.txt
163 |
164 | RUN \\
165 | python3 -m pip install --upgrade pip && \\
166 | pip install -r requirements.txt --no-deps
167 |
168 | WORKDIR /opt/resource/
169 | COPY resource.py ./resource.py
170 | RUN python3 -m concoursetools assets . -r resource.py
171 |
172 | ENTRYPOINT ["python3"]
173 | """).lstrip()
174 | self.assertEqual(dockerfile_contents, expected_contents)
175 |
176 | def test_basic_config_with_class_name_and_executable(self) -> None:
177 | cli_commands.dockerfile(str(self.temp_dir), class_name="MyResource", executable="/usr/bin/python3")
178 | dockerfile_contents = self.dockerfile_path.read_text()
179 | expected_contents = textwrap.dedent(f"""
180 | FROM python:{self.current_python_string}
181 |
182 | RUN python3 -m venv /opt/venv
183 | # Activate venv
184 | ENV PATH="/opt/venv/bin:$PATH"
185 |
186 | COPY requirements.txt requirements.txt
187 |
188 | RUN \\
189 | python3 -m pip install --upgrade pip && \\
190 | pip install -r requirements.txt --no-deps
191 |
192 | WORKDIR /opt/resource/
193 | COPY concourse.py ./concourse.py
194 | RUN python3 -m concoursetools assets . -r concourse.py -c MyResource -e /usr/bin/python3
195 |
196 | ENTRYPOINT ["python3"]
197 | """).lstrip()
198 | self.assertEqual(dockerfile_contents, expected_contents)
199 |
200 | def test_netrc_config(self) -> None:
201 | cli_commands.dockerfile(str(self.temp_dir), include_netrc=True)
202 | dockerfile_contents = self.dockerfile_path.read_text()
203 | expected_contents = textwrap.dedent(f"""
204 | FROM python:{self.current_python_string}
205 |
206 | RUN python3 -m venv /opt/venv
207 | # Activate venv
208 | ENV PATH="/opt/venv/bin:$PATH"
209 |
210 | COPY requirements.txt requirements.txt
211 |
212 | RUN \\
213 | --mount=type=secret,id=netrc,target=/root/.netrc,mode=0600,required=true \\
214 | python3 -m pip install --upgrade pip && \\
215 | pip install -r requirements.txt --no-deps
216 |
217 | WORKDIR /opt/resource/
218 | COPY concourse.py ./concourse.py
219 | RUN python3 -m concoursetools assets . -r concourse.py
220 |
221 | ENTRYPOINT ["python3"]
222 | """).lstrip()
223 | self.assertEqual(dockerfile_contents, expected_contents)
224 |
225 | def test_rsa_config(self) -> None:
226 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", include_rsa=True)
227 | dockerfile_contents = self.dockerfile_path.read_text()
228 | expected_contents = textwrap.dedent(f"""
229 | FROM python:{self.current_python_string}
230 |
231 | RUN python3 -m venv /opt/venv
232 | # Activate venv
233 | ENV PATH="/opt/venv/bin:$PATH"
234 |
235 | COPY requirements.txt requirements.txt
236 |
237 | RUN \\
238 | --mount=type=secret,id=private_key,target=/root/.ssh/id_rsa,mode=0600,required=true \\
239 | --mount=type=secret,id=known_hosts,target=/root/.ssh/known_hosts,mode=0644 \\
240 | python3 -m pip install --upgrade pip && \\
241 | pip install -r requirements.txt --no-deps
242 |
243 | WORKDIR /opt/resource/
244 | COPY concourse.py ./concourse.py
245 | RUN python3 -m concoursetools assets . -r concourse.py
246 |
247 | ENTRYPOINT ["python3"]
248 | """).lstrip()
249 | self.assertEqual(dockerfile_contents, expected_contents)
250 |
251 | def test_dev_config(self) -> None:
252 | cli_commands.dockerfile(str(self.temp_dir), resource_file="concourse.py", dev=True)
253 | dockerfile_contents = self.dockerfile_path.read_text()
254 | expected_contents = textwrap.dedent(f"""
255 | FROM python:{self.current_python_string}
256 |
257 | RUN python3 -m venv /opt/venv
258 | # Activate venv
259 | ENV PATH="/opt/venv/bin:$PATH"
260 |
261 | COPY requirements.txt requirements.txt
262 |
263 | COPY concoursetools concoursetools
264 |
265 | RUN \\
266 | python3 -m pip install --upgrade pip && \\
267 | pip install ./concoursetools && \\
268 | pip install -r requirements.txt --no-deps
269 |
270 | WORKDIR /opt/resource/
271 | COPY concourse.py ./concourse.py
272 | RUN python3 -m concoursetools assets . -r concourse.py
273 |
274 | ENTRYPOINT ["python3"]
275 | """).lstrip()
276 | self.assertEqual(dockerfile_contents, expected_contents)
277 |
--------------------------------------------------------------------------------
/tests/test_importing.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | """
3 | Tests for the dockertools module.
4 | """
5 | from collections.abc import Generator
6 | from contextlib import contextmanager
7 | import inspect
8 | import os
9 | from pathlib import Path
10 | import secrets
11 | import sys
12 | from tempfile import TemporaryDirectory
13 | import textwrap
14 | from unittest import TestCase
15 |
16 | from concoursetools import ConcourseResource, additional
17 | from concoursetools.importing import (edit_sys_path, file_path_to_import_path, import_classes_from_module, import_py_file,
18 | import_single_class_from_module)
19 | from tests import resource as test_resource
20 |
21 |
22 | class BasicTests(TestCase):
23 | """
24 | Tests for the utility functions.
25 | """
26 | def test_import_path_creation(self) -> None:
27 | file_path = Path("path/to/python.py")
28 | import_path = file_path_to_import_path(file_path)
29 | self.assertEqual(import_path, "path.to.python")
30 |
31 | def test_import_path_creation_wrong_extension(self) -> None:
32 | file_path = Path("path/to/file.txt")
33 | with self.assertRaises(ValueError):
34 | file_path_to_import_path(file_path)
35 |
36 | def test_importing_python_file_by_local_path_same_location(self) -> None:
37 | file_contents = textwrap.dedent("""
38 | def f(x: int, y: int) -> int:
39 | return x + y
40 | """).lstrip()
41 |
42 | import_path, file_name = _random_python_file()
43 | self.assertNotIn(import_path, sys.modules)
44 |
45 | with TemporaryDirectory() as temp_dir:
46 | py_file = Path(temp_dir) / file_name
47 | py_file.write_text(file_contents)
48 | with _chdir(Path(temp_dir)):
49 | module = import_py_file(import_path, Path(file_name))
50 |
51 | self.assertEqual(module.f(3, 5), 8)
52 |
53 | def test_importing_python_file_by_full_path_same_location(self) -> None:
54 | file_contents = textwrap.dedent("""
55 | def f(x: int, y: int) -> int:
56 | return x + y
57 | """).lstrip()
58 |
59 | import_path, file_name = _random_python_file()
60 | self.assertNotIn(import_path, sys.modules)
61 |
62 | with TemporaryDirectory() as temp_dir:
63 | py_file = Path(temp_dir) / file_name
64 | py_file.write_text(file_contents)
65 | with _chdir(Path(temp_dir)):
66 | module = import_py_file(import_path, py_file)
67 |
68 | self.assertEqual(module.f(3, 5), 8)
69 |
70 | def test_importing_python_file_by_full_path_other_location(self) -> None:
71 | file_contents = textwrap.dedent("""
72 | def f(x: int, y: int) -> int:
73 | return x + y
74 | """).lstrip()
75 |
76 | import_path, file_name = _random_python_file()
77 | self.assertNotIn(import_path, sys.modules)
78 |
79 | with TemporaryDirectory() as temp_dir:
80 | py_file = Path(temp_dir) / file_name
81 | py_file.write_text(file_contents)
82 |
83 | with TemporaryDirectory() as temp_dir_2:
84 | with _chdir(Path(temp_dir_2)):
85 | self.assertNotIn(file_name, os.listdir("."))
86 | module = import_py_file(import_path, py_file)
87 |
88 | self.assertEqual(module.f(3, 5), 8)
89 |
90 | def test_importing_python_file_pair_by_full_path_other_location(self) -> None:
91 | file_contents = textwrap.dedent("""
92 | def f(x: int, y: int) -> int:
93 | return x + y
94 | """).lstrip()
95 |
96 | import_path, file_name = _random_python_file()
97 |
98 | second_file_contents = textwrap.dedent(f"""
99 | from {import_path} import f
100 |
101 | def g(x: int, y: int) -> int:
102 | return f(x, 3) + f(y, 4)
103 | """).lstrip()
104 |
105 | second_import_path, second_file_name = _random_python_file()
106 |
107 | self.assertNotIn(import_path, sys.modules)
108 | self.assertNotIn(second_import_path, sys.modules)
109 |
110 | with TemporaryDirectory() as temp_dir:
111 | py_file = Path(temp_dir) / file_name
112 | py_file.write_text(file_contents)
113 |
114 | py_file_2 = Path(temp_dir) / second_file_name
115 | py_file_2.write_text(second_file_contents)
116 |
117 | with TemporaryDirectory() as temp_dir_2:
118 | with _chdir(Path(temp_dir_2)):
119 | self.assertNotIn(file_name, os.listdir("."))
120 | self.assertNotIn(second_file_name, os.listdir("."))
121 | module = import_py_file(second_import_path, py_file_2)
122 |
123 | self.assertEqual(module.g(3, 5), 15)
124 |
125 | def test_changing_directory(self) -> None:
126 | current_dir = Path.cwd()
127 | with TemporaryDirectory() as temp_dir:
128 | with _chdir(Path(temp_dir)):
129 | self.assertEqual(Path.cwd(), Path(temp_dir).resolve())
130 | self.assertEqual(Path.cwd(), current_dir.resolve())
131 |
132 | def test_changing_directory_nested(self) -> None:
133 | current_dir = Path.cwd()
134 | with TemporaryDirectory() as temp_dir_1:
135 | with TemporaryDirectory() as temp_dir_2:
136 | with _chdir(Path(temp_dir_1)):
137 | self.assertEqual(Path.cwd(), Path(temp_dir_1).resolve())
138 | with _chdir(Path(temp_dir_2)):
139 | self.assertEqual(Path.cwd(), Path(temp_dir_2).resolve())
140 | self.assertEqual(Path.cwd(), Path(temp_dir_1).resolve())
141 | self.assertEqual(Path.cwd(), current_dir.resolve())
142 |
143 | def test_edit_sys_path(self) -> None:
144 | original_sys_path = sys.path.copy()
145 | with TemporaryDirectory() as temp_dir_1:
146 | with TemporaryDirectory() as temp_dir_2:
147 | self.assertNotIn(temp_dir_1, sys.path)
148 | self.assertNotIn(temp_dir_2, sys.path)
149 |
150 | with edit_sys_path(prepend=[Path(temp_dir_1)], append=[Path(temp_dir_2)]):
151 | self.assertEqual(sys.path[0], temp_dir_1)
152 | self.assertEqual(sys.path[-1], temp_dir_2)
153 | self.assertListEqual(sys.path[1:-1], original_sys_path)
154 |
155 | self.assertNotIn(temp_dir_1, sys.path)
156 | self.assertNotIn(temp_dir_2, sys.path)
157 |
158 | def test_importing_classes(self) -> None:
159 | file_path = Path(additional.__file__).relative_to(Path.cwd())
160 | resource_classes = import_classes_from_module(file_path, parent_class=ConcourseResource) # type: ignore[type-abstract]
161 | expected = {
162 | "InOnlyConcourseResource": additional.InOnlyConcourseResource,
163 | "OutOnlyConcourseResource": additional.OutOnlyConcourseResource,
164 | "MultiVersionConcourseResource": additional.MultiVersionConcourseResource,
165 | "SelfOrganisingConcourseResource": additional.SelfOrganisingConcourseResource,
166 | "TriggerOnChangeConcourseResource": additional.TriggerOnChangeConcourseResource,
167 | }
168 | self.assertEqual(expected.keys(), resource_classes.keys())
169 | for key, class_1 in resource_classes.items():
170 | class_2 = expected[key]
171 | self.assertClassEqual(class_1, class_2)
172 |
173 | def test_importing_class_no_name(self) -> None:
174 | file_path = Path(test_resource.__file__).relative_to(Path.cwd())
175 | with self.assertRaises(RuntimeError):
176 | import_single_class_from_module(file_path, parent_class=ConcourseResource) # type: ignore[type-abstract]
177 |
178 | def test_importing_class_with_name(self) -> None:
179 | file_path = Path(test_resource.__file__).relative_to(Path.cwd())
180 | resource_class = import_single_class_from_module(file_path, parent_class=ConcourseResource, # type: ignore[type-abstract]
181 | class_name=test_resource.TestResource.__name__)
182 | self.assertClassEqual(resource_class, test_resource.TestResource)
183 |
184 | def test_importing_class_multiple_options(self) -> None:
185 | file_path = Path(additional.__file__).relative_to(Path.cwd())
186 | with self.assertRaises(RuntimeError):
187 | import_single_class_from_module(file_path, parent_class=ConcourseResource) # type: ignore[type-abstract]
188 |
189 | def test_importing_class_multiple_options_specify_name(self) -> None:
190 | file_path = Path(additional.__file__).relative_to(Path.cwd())
191 | parent_class = additional.InOnlyConcourseResource
192 | resource_class = import_single_class_from_module(file_path, parent_class=ConcourseResource, # type: ignore[type-abstract]
193 | class_name=parent_class.__name__)
194 | self.assertClassEqual(resource_class, parent_class)
195 |
196 | def assertClassEqual(self, class_1: type[object], class_2: type[object]) -> None:
197 | self.assertEqual(inspect.getsourcefile(class_1), inspect.getsourcefile(class_2))
198 | self.assertEqual(inspect.getsource(class_1), inspect.getsource(class_2))
199 |
200 |
201 | @contextmanager
202 | def _chdir(new_dir: Path) -> Generator[None, None, None]:
203 | original_dir = Path.cwd()
204 | try:
205 | os.chdir(new_dir)
206 | yield
207 | finally:
208 | os.chdir(original_dir)
209 |
210 |
211 | def _random_python_file(num_bytes: int = 4, prefix: str = "test_") -> tuple[str, str]:
212 | import_path = f"{prefix}{secrets.token_hex(num_bytes)}"
213 | file_name = f"{import_path}.py"
214 | return import_path, file_name
215 |
--------------------------------------------------------------------------------
/tests/test_metadata.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | import json
3 | from unittest import TestCase
4 |
5 | from concoursetools import BuildMetadata
6 | from concoursetools.metadata import _flatten_dict
7 | from concoursetools.mocking import TestBuildMetadata
8 | from concoursetools.testing import create_env_vars, mock_environ
9 |
10 |
11 | class MetadataTests(TestCase):
12 | """
13 | Tests for the BuildMetadata class.
14 | """
15 | def test_normal_build(self) -> None:
16 | metadata = BuildMetadata(
17 | BUILD_ID="12345678",
18 | BUILD_TEAM_NAME="my-team",
19 | ATC_EXTERNAL_URL="https://ci.myconcourse.com",
20 | BUILD_JOB_NAME="my-job",
21 | BUILD_NAME="42",
22 | BUILD_PIPELINE_NAME="my-pipeline",
23 | )
24 |
25 | self.assertFalse(metadata.is_one_off_build)
26 | self.assertFalse(metadata.is_instanced_pipeline)
27 | self.assertDictEqual(metadata.instance_vars(), {})
28 | self.assertEqual(metadata.build_url(), "https://ci.myconcourse.com/teams/my-team/pipelines/my-pipeline/jobs/my-job/builds/42")
29 |
30 | def test_normal_build_from_env(self) -> None:
31 | env = create_env_vars()
32 | with mock_environ(env):
33 | metadata = BuildMetadata.from_env()
34 |
35 | self.assertFalse(metadata.is_one_off_build)
36 | self.assertFalse(metadata.is_instanced_pipeline)
37 | self.assertDictEqual(metadata.instance_vars(), {})
38 | self.assertEqual(metadata.build_url(), "https://ci.myconcourse.com/teams/my-team/pipelines/my-pipeline/jobs/my-job/builds/42")
39 |
40 | def test_instanced_pipeline_build(self) -> None:
41 | metadata = BuildMetadata(
42 | BUILD_ID="12345678",
43 | BUILD_TEAM_NAME="my-team",
44 | ATC_EXTERNAL_URL="https://ci.myconcourse.com",
45 | BUILD_JOB_NAME="my-job",
46 | BUILD_NAME="42",
47 | BUILD_PIPELINE_NAME="my-pipeline",
48 | BUILD_PIPELINE_INSTANCE_VARS="{\"key1\":\"value1\",\"key2\":\"value2\"}",
49 | )
50 |
51 | self.assertFalse(metadata.is_one_off_build)
52 | self.assertTrue(metadata.is_instanced_pipeline)
53 | self.assertDictEqual(metadata.instance_vars(), {"key1": "value1", "key2": "value2"})
54 | url = "https://ci.myconcourse.com/teams/my-team/pipelines/my-pipeline/jobs/my-job/builds/42?vars.key1=%22value1%22&vars.key2=%22value2%22"
55 | self.assertEqual(metadata.build_url(), url)
56 |
57 | def test_instanced_pipeline_build_from_env(self) -> None:
58 | instance_vars = {"key1": "value1", "key2": "value2"}
59 | env = create_env_vars(instance_vars=instance_vars)
60 | with mock_environ(env):
61 | metadata = BuildMetadata.from_env()
62 |
63 | self.assertFalse(metadata.is_one_off_build)
64 | self.assertTrue(metadata.is_instanced_pipeline)
65 | self.assertDictEqual(metadata.instance_vars(), instance_vars)
66 | url = r"https://ci.myconcourse.com/teams/my-team/pipelines/my-pipeline/jobs/my-job/builds/42?vars.key1=%22value1%22&vars.key2=%22value2%22"
67 | self.assertEqual(metadata.build_url(), url)
68 |
69 | def test_nested_instanced_pipeline_build(self) -> None:
70 | instance_vars = {
71 | "branch": "feature-v8",
72 | "version": {
73 | "from": "3.0.0",
74 | "main": 2,
75 | "to": "2.0.0",
76 | },
77 | }
78 |
79 | metadata = BuildMetadata(
80 | BUILD_ID="12345678",
81 | BUILD_TEAM_NAME="my-team",
82 | ATC_EXTERNAL_URL="https://ci.myconcourse.com",
83 | BUILD_JOB_NAME="my-job",
84 | BUILD_NAME="42",
85 | BUILD_PIPELINE_NAME="my-pipeline",
86 | BUILD_PIPELINE_INSTANCE_VARS=json.dumps(instance_vars),
87 | )
88 |
89 | self.assertFalse(metadata.is_one_off_build)
90 | self.assertTrue(metadata.is_instanced_pipeline)
91 | self.assertDictEqual(metadata.instance_vars(), instance_vars)
92 | url = (r"https://ci.myconcourse.com/teams/my-team/pipelines/my-pipeline/jobs/my-job/builds/42"
93 | r"?vars.branch=%22feature-v8%22&vars.version.from=%223.0.0%22&vars.version.main=2&vars.version.to=%222.0.0%22")
94 | self.assertEqual(metadata.build_url(), url)
95 |
96 | def test_one_off_build(self) -> None:
97 | metadata = BuildMetadata(
98 | BUILD_ID="12345678",
99 | BUILD_TEAM_NAME="my-team",
100 | ATC_EXTERNAL_URL="https://ci.myconcourse.com",
101 | BUILD_NAME="42",
102 | )
103 |
104 | self.assertTrue(metadata.is_one_off_build)
105 | self.assertFalse(metadata.is_instanced_pipeline)
106 | self.assertDictEqual(metadata.instance_vars(), {})
107 | self.assertEqual(metadata.build_url(), "https://ci.myconcourse.com/builds/12345678")
108 |
109 | def test_one_off_build_from_env(self) -> None:
110 | env = create_env_vars(one_off_build=True)
111 | with mock_environ(env):
112 | metadata = BuildMetadata.from_env()
113 |
114 | self.assertTrue(metadata.is_one_off_build)
115 | self.assertFalse(metadata.is_instanced_pipeline)
116 | self.assertDictEqual(metadata.instance_vars(), {})
117 | self.assertEqual(metadata.build_url(), "https://ci.myconcourse.com/builds/12345678")
118 |
119 | def test_flattening_nested_dict(self) -> None:
120 | nested_dict = {
121 | "branch": "feature-v8",
122 | "version": {
123 | "from": "3.0.0",
124 | "main": 2,
125 | "to": "2.0.0"
126 | },
127 | }
128 | flattened_dict = {
129 | "branch": "feature-v8",
130 | "version.from": "3.0.0",
131 | "version.main": 2,
132 | "version.to": "2.0.0",
133 | }
134 | self.assertDictEqual(_flatten_dict(nested_dict), flattened_dict)
135 |
136 | def test_flattening_double_nested_dict(self) -> None:
137 | nested_dict = {
138 | "branch": "feature-v8",
139 | "version": {
140 | "main": 2,
141 | "parents": {
142 | "from": "3.0.0",
143 | "to": "2.0.0",
144 | },
145 | },
146 | }
147 | flattened_dict = {
148 | "branch": "feature-v8",
149 | "version.main": 2,
150 | "version.parents.from": "3.0.0",
151 | "version.parents.to": "2.0.0",
152 | }
153 | self.assertDictEqual(_flatten_dict(nested_dict), flattened_dict)
154 |
155 |
156 | class MetadataFormattingTests(TestCase):
157 | """
158 | Tests for the BuildMetadata.format_string method.
159 | """
160 | def setUp(self) -> None:
161 | """Code to run before each test."""
162 | self.metadata = TestBuildMetadata()
163 |
164 | def test_no_interpolation(self) -> None:
165 | new_string = self.metadata.format_string("This is a normal string.")
166 | self.assertEqual(new_string, "This is a normal string.")
167 |
168 | def test_simple_interpolation(self) -> None:
169 | new_string = self.metadata.format_string("The build id is $BUILD_ID and the job name is $BUILD_JOB_NAME.")
170 | self.assertEqual(new_string, "The build id is 12345678 and the job name is my-job.")
171 |
172 | def test_interpolation_with_one_off(self) -> None:
173 | metadata = TestBuildMetadata(one_off_build=True)
174 | new_string = metadata.format_string("The build id is $BUILD_ID and the job name is $BUILD_JOB_NAME.")
175 | self.assertEqual(new_string, "The build id is 12345678 and the job name is .")
176 |
177 | def test_interpolation_incorrect_value(self) -> None:
178 | with self.assertRaises(KeyError):
179 | self.metadata.format_string("The build id is $OTHER.")
180 |
181 | def test_interpolation_incorrect_value_ignore_missing(self) -> None:
182 | new_string = self.metadata.format_string("The build id is $OTHER.", ignore_missing=True)
183 | self.assertEqual(new_string, "The build id is $OTHER.")
184 |
185 | def test_interpolation_with_additional(self) -> None:
186 | new_string = self.metadata.format_string("The build id is $OTHER.", additional_values={"OTHER": "value"},
187 | ignore_missing=True)
188 | self.assertEqual(new_string, "The build id is value.")
189 |
--------------------------------------------------------------------------------
/tests/test_testing.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | from pathlib import Path
3 | from tempfile import TemporaryDirectory
4 | from typing import ClassVar
5 | from unittest import TestCase
6 |
7 | from concoursetools.testing import TemporaryDirectoryState
8 |
9 |
10 | class FolderDictReadTests(TestCase):
11 | temp_dir: ClassVar[TemporaryDirectory[str]]
12 | root: ClassVar[Path]
13 |
14 | @classmethod
15 | def setUpClass(cls) -> None:
16 | cls.temp_dir = TemporaryDirectory()
17 | cls.root = Path(cls.temp_dir.name)
18 |
19 | folder_1 = cls.root / "folder_1"
20 | folder_2 = cls.root / "folder_2"
21 | folder_3 = folder_2 / "folder_3"
22 |
23 | folder_1.mkdir()
24 | folder_2.mkdir()
25 | folder_3.mkdir()
26 |
27 | file_1 = cls.root / "file_1"
28 | file_2 = folder_2 / "file_2"
29 | file_3 = folder_3 / "file_3"
30 |
31 | file_1.write_text("Testing 1\n")
32 | file_2.write_text("Testing 2\n")
33 | file_3.write_text("Testing 3\n")
34 |
35 | @classmethod
36 | def tearDownClass(cls) -> None:
37 | cls.temp_dir.cleanup()
38 |
39 | def test_folder_dict_depth_1(self) -> None:
40 | folder_dict = TemporaryDirectoryState()._get_folder_as_dict(self.root, max_depth=1)
41 | expected = {
42 | "folder_1": ...,
43 | "folder_2": ...,
44 | "file_1": "Testing 1\n",
45 | }
46 | self.assertDictEqual(folder_dict, expected)
47 |
48 | def test_folder_dict_depth_2(self) -> None:
49 | folder_dict = TemporaryDirectoryState()._get_folder_as_dict(self.root, max_depth=2)
50 | expected = {
51 | "folder_1": {},
52 | "folder_2": {
53 | "folder_3": ...,
54 | "file_2": "Testing 2\n",
55 | },
56 | "file_1": "Testing 1\n",
57 | }
58 | self.assertDictEqual(folder_dict, expected)
59 |
60 | def test_folder_dict_depth_3(self) -> None:
61 | folder_dict = TemporaryDirectoryState()._get_folder_as_dict(self.root, max_depth=3)
62 | expected = {
63 | "folder_1": {},
64 | "folder_2": {
65 | "folder_3": {
66 | "file_3": "Testing 3\n",
67 | },
68 | "file_2": "Testing 2\n",
69 | },
70 | "file_1": "Testing 1\n",
71 | }
72 | self.assertDictEqual(folder_dict, expected)
73 |
74 |
75 | class FolderDictWriteTests(TestCase):
76 | temp_dir: ClassVar[TemporaryDirectory[str]]
77 | root: ClassVar[Path]
78 |
79 | @classmethod
80 | def setUpClass(cls) -> None:
81 | cls.temp_dir = TemporaryDirectory()
82 | cls.root = Path(cls.temp_dir.name)
83 |
84 | @classmethod
85 | def tearDownClass(cls) -> None:
86 | cls.temp_dir.cleanup()
87 |
88 | def test_folder_dict_depth_3(self) -> None:
89 | original = {
90 | "folder_1": {},
91 | "folder_2": {
92 | "folder_3": {
93 | "file_3": "Testing 3\n",
94 | },
95 | "file_2": "Testing 2\n",
96 | },
97 | "file_1": "Testing 1\n",
98 | }
99 | TemporaryDirectoryState()._set_folder_from_dict(self.root, original)
100 | final_dict = TemporaryDirectoryState()._get_folder_as_dict(self.root, max_depth=3)
101 | self.assertDictEqual(final_dict, original)
102 |
--------------------------------------------------------------------------------
/tests/test_version.py:
--------------------------------------------------------------------------------
1 | # (C) Crown Copyright GCHQ
2 | from dataclasses import dataclass
3 | from datetime import datetime
4 | from enum import Enum
5 | from pathlib import Path
6 | from unittest import TestCase
7 |
8 | from concoursetools import Version
9 | from concoursetools.typing import VersionConfig
10 | from concoursetools.version import SortableVersionMixin, TypedVersion
11 |
12 |
13 | class BasicVersion(Version):
14 |
15 | def __init__(self, file_path: str) -> None:
16 | self.file_path = file_path
17 |
18 |
19 | class CreationTests(TestCase):
20 | """
21 | Tests for the creation of an instance.
22 | """
23 | def test_base_version(self) -> None:
24 | with self.assertRaises(TypeError):
25 | Version() # type: ignore[abstract]
26 |
27 |
28 | class ComplexVersion(BasicVersion, SortableVersionMixin):
29 |
30 | def __eq__(self, other: object) -> bool:
31 | if not isinstance(other, type(self)):
32 | return NotImplemented
33 | return self.file_name == other.file_name
34 |
35 | def __lt__(self, other: object) -> bool:
36 | if not isinstance(other, type(self)):
37 | return NotImplemented
38 | return self.file_path < other.file_path
39 |
40 | @property
41 | def file_name(self) -> str:
42 | return Path(self.file_path).name
43 |
44 |
45 | class ComparisonTests(TestCase):
46 |
47 | def test_repr(self) -> None:
48 | version_1 = BasicVersion("file.txt")
49 | self.assertEqual(repr(version_1), "BasicVersion(file_path='file.txt')")
50 |
51 | def test_sortable_mixin_with_version_with_abstract(self) -> None:
52 | class MyVersion(Version, SortableVersionMixin):
53 |
54 | def __init__(self, file_path: str) -> None:
55 | self.file_path = file_path
56 |
57 | def __lt__(self, other: object) -> bool:
58 | if not isinstance(other, type(self)):
59 | return NotImplemented
60 | return self.file_path < other.file_path
61 |
62 | self.assertLess(MyVersion("aaa"), MyVersion("bbb"))
63 |
64 | def test_sortable_mixin_with_version_without_abstract(self) -> None:
65 | class MyVersion(Version, SortableVersionMixin):
66 |
67 | def __init__(self, file_path: str) -> None:
68 | self.file_path = file_path
69 |
70 | with self.assertRaises(TypeError):
71 | MyVersion("aaa") # type: ignore[abstract]
72 |
73 | def test_sortable_mixin_with_typed_version_with_abstract(self) -> None:
74 | @dataclass
75 | class MyTypedVersion(TypedVersion, SortableVersionMixin):
76 | file_path: str
77 |
78 | def __lt__(self, other: object) -> bool:
79 | if not isinstance(other, type(self)):
80 | return NotImplemented
81 | return self.file_path < other.file_path
82 |
83 | self.assertLess(MyTypedVersion("aaa"), MyTypedVersion("bbb"))
84 |
85 | def test_sortable_mixin_with_typed_version_without_abstract(self) -> None:
86 | @dataclass
87 | class MyTypedVersion(TypedVersion, SortableVersionMixin):
88 | file_path: str
89 |
90 | with self.assertRaises(TypeError):
91 | MyTypedVersion("aaa") # type: ignore[abstract]
92 |
93 | def test_default_equality(self) -> None:
94 | version_1 = BasicVersion("file.txt")
95 | version_1_again = BasicVersion("file.txt")
96 | version_2 = BasicVersion("folder/file.txt")
97 | version_3 = BasicVersion("image.png")
98 |
99 | self.assertEqual(version_1, version_1_again)
100 | self.assertNotEqual(version_1, version_2)
101 | self.assertNotEqual(version_1, version_3)
102 | self.assertNotEqual(version_2, version_3)
103 |
104 | def test_complex_equality(self) -> None:
105 | version_1 = ComplexVersion("file.txt")
106 | version_1_again = ComplexVersion("file.txt")
107 | version_2 = ComplexVersion("folder/file.txt")
108 | version_3 = ComplexVersion("image.png")
109 |
110 | self.assertEqual(version_1, version_1_again)
111 | self.assertEqual(version_1, version_2)
112 | self.assertNotEqual(version_1, version_3)
113 | self.assertNotEqual(version_2, version_3)
114 |
115 | def test_default_sorting(self) -> None:
116 | version_1 = BasicVersion("file.txt")
117 | version_2 = BasicVersion("folder/file.txt")
118 | version_3 = BasicVersion("image.png")
119 |
120 | with self.assertRaises(TypeError):
121 | sorted([version_1, version_2, version_3]) # type: ignore[type-var]
122 |
123 | def test_complex_sorting(self) -> None:
124 | version_1 = ComplexVersion("file.txt")
125 | version_2 = ComplexVersion("folder/file.txt")
126 | version_3 = ComplexVersion("image.png")
127 |
128 | self.assertListEqual(sorted([version_3, version_1, version_2]), [version_1, version_2, version_3])
129 |
130 | def test_complex_comparisons(self) -> None:
131 | version_1 = ComplexVersion("file.txt")
132 | version_2 = ComplexVersion("folder/file.txt")
133 | self.assertLess(version_1, version_2)
134 | self.assertGreater(version_2, version_1)
135 |
136 | self.assertLessEqual(version_1, version_2)
137 | self.assertGreaterEqual(version_2, version_1)
138 | self.assertLessEqual(version_1, version_1)
139 |
140 |
141 | class CommitVersion(Version):
142 |
143 | def __init__(self, commit_hash: str, is_merge: bool) -> None:
144 | self.commit_hash = commit_hash
145 | self.is_merge = is_merge
146 |
147 |
148 | @dataclass
149 | class TypedCommitVersion(TypedVersion):
150 | commit_hash: str
151 | date: datetime
152 | is_merge: bool
153 |
154 |
155 | class CommitVersionImproved(CommitVersion):
156 |
157 | @classmethod
158 | def from_flat_dict(cls, version_dict: VersionConfig) -> "CommitVersionImproved":
159 | is_merge = (version_dict["is_merge"] == "True")
160 | return cls(version_dict["commit_hash"], is_merge)
161 |
162 |
163 | class DictTests(TestCase):
164 | """
165 | Tests for the conversion between version and dict.
166 | """
167 | def test_non_strings(self) -> None:
168 | version = CommitVersion("abcdef", True)
169 | flat_dict = version.to_flat_dict()
170 | self.assertDictEqual(flat_dict, {
171 | "commit_hash": "abcdef",
172 | "is_merge": "True"
173 | })
174 | new_version = CommitVersion.from_flat_dict(flat_dict)
175 | self.assertEqual(new_version.commit_hash, "abcdef")
176 | self.assertEqual(new_version.is_merge, "True")
177 |
178 | better_new_version = CommitVersionImproved.from_flat_dict(flat_dict)
179 | self.assertEqual(better_new_version.commit_hash, "abcdef")
180 | self.assertEqual(better_new_version.is_merge, True)
181 |
182 | def test_private_attribute(self) -> None:
183 |
184 | class CommitVersionPrivate(CommitVersion):
185 |
186 | def __init__(self, commit_hash: str, is_merge: bool):
187 | super().__init__(commit_hash, is_merge)
188 | self._force_push = True
189 |
190 | version = CommitVersionPrivate("abcdef", True)
191 | flat_dict = version.to_flat_dict()
192 | self.assertDictEqual(flat_dict, {
193 | "commit_hash": "abcdef",
194 | "is_merge": "True"
195 | })
196 |
197 |
198 | class MyEnum(Enum):
199 | ONE = 1
200 | TWO = 2
201 |
202 |
203 | class TypedTests(TestCase):
204 |
205 | def test_flattened_and_unflattened_types(self) -> None:
206 | expected = {
207 | "string": "string",
208 | "42": 42,
209 | "True": True,
210 | "False": False,
211 | "1577881800": datetime(2020, 1, 1, 12, 30),
212 | "ONE": MyEnum.ONE,
213 | "/path/to/somewhere": Path("/path/to/somewhere"),
214 | }
215 | for flattened_obj, obj in expected.items():
216 | with self.subTest(obj=obj):
217 | self.assertEqual(TypedVersion._flatten_object(obj), flattened_obj)
218 | un_flattened_obj = TypedVersion._un_flatten_object(type(obj), flattened_obj)
219 | self.assertEqual(type(un_flattened_obj), type(obj))
220 | self.assertEqual(un_flattened_obj, obj)
221 |
222 | def test_flattened_and_unflattened_version(self) -> None:
223 | version = TypedCommitVersion("abcdef", datetime(2020, 1, 1, 12, 30), False)
224 | flattened = version.to_flat_dict()
225 | self.assertDictEqual(flattened, {
226 | "commit_hash": "abcdef",
227 | "date": "1577881800",
228 | "is_merge": "False",
229 | })
230 | self.assertEqual(TypedCommitVersion.from_flat_dict(flattened), version)
231 |
232 | def test_implementing_empty_version(self) -> None:
233 | with self.assertRaises(TypeError):
234 | @dataclass
235 | class _(TypedVersion):
236 | pass
237 |
238 | def test_missing_dataclass(self) -> None:
239 | with self.assertRaises(TypeError):
240 | class _(TypedVersion):
241 | pass
242 |
--------------------------------------------------------------------------------