├── src └── feditest │ ├── protocols │ ├── __init__.py │ ├── fediverse │ │ ├── utils.py │ │ └── diag.py │ ├── web │ │ └── __init__.py │ ├── activitypub │ │ ├── utils.py │ │ ├── __init__.py │ │ └── diag.py │ ├── sandbox │ │ └── __init__.py │ └── webfinger │ │ ├── abstract.py │ │ └── __init__.py │ ├── testruntranscriptserializer │ ├── templates │ │ └── testplantranscript_default │ │ │ ├── partials │ │ │ ├── shared │ │ │ │ ├── head.jinja2 │ │ │ │ ├── footer.jinja2 │ │ │ │ ├── mobile.jinja2 │ │ │ │ └── summary.jinja2 │ │ │ ├── matrix │ │ │ │ ├── testresult.jinja2 │ │ │ │ ├── metadata.jinja2 │ │ │ │ └── matrix.jinja2 │ │ │ └── shared_session │ │ │ │ ├── testresult.jinja2 │ │ │ │ ├── metadata.jinja2 │ │ │ │ └── results.jinja2 │ │ │ ├── session_single.jinja2 │ │ │ ├── matrix.jinja2 │ │ │ └── session_with_matrix.jinja2 │ ├── json.py │ ├── summary.py │ ├── __init__.py │ ├── tap.py │ └── html.py │ ├── cli │ ├── commands │ │ ├── version.py │ │ ├── list_nodedrivers.py │ │ ├── list_tests.py │ │ ├── create_constellation.py │ │ ├── create_session_template.py │ │ ├── create_testplan.py │ │ ├── convert_transcript.py │ │ ├── info.py │ │ └── run.py │ ├── __init__.py │ └── utils.py │ ├── nodedrivers │ ├── saas │ │ └── __init__.py │ ├── manual │ │ └── __init__.py │ ├── imp │ │ └── __init__.py │ ├── sandbox │ │ └── __init__.py │ └── wordpress │ │ └── ubos.py │ ├── disabled.py │ ├── tests.py │ ├── reporting.py │ └── testruncontroller.py ├── .gitignore ├── .gitattributes ├── tests.smoke ├── imp.node.json ├── mastodon.ubos.node.json ├── wordpress.ubos.node.json ├── mastodon_api.session.json ├── webfinger.session.json ├── mastodon_api_mastodon_api.session.json ├── tests │ ├── webfinger.py │ ├── node_with_mastodon_api.py │ └── nodes_with_mastodon_api_communicate.py └── README.md ├── README-PyPI.md ├── REUSE.toml ├── tests.unit ├── dummy.py ├── conftest.py ├── test_10_register_nodedrivers.py ├── test_10_create_test_plan_constallation.py ├── test_10_register_tests.py ├── test_20_create_test_plan_session_template.py ├── test_10_registry.py ├── test_50_run_passes.py ├── test_50_run_skip.py ├── test_40_fallback_fediverse_accounts_without_roles_from_testplan.py ├── test_30_create_testplan.py ├── test_40_fallback_fediverse_accounts_with_roles_from_testplan.py ├── test_40_report_node_driver_errors.py ├── test_50_run_not_implemented.py ├── test_50_run_exceptions.py ├── test_40_ubos_mastodon_accounts_from_testplan.py ├── test_50_transcript.py ├── test_40_static_account_manager_fallback_fediverse_accounts.py ├── test_50_run_testplan_sandbox.py ├── test_60_use_manually_entered_application_name.py └── test_50_run_multistep_assertion_raises.py ├── hatch_build.py ├── README.md ├── .github └── workflows │ └── pre-commit.yml ├── LICENSE ├── .pre-commit-config.yaml ├── pyproject.toml ├── RELEASE-HOWTO.md └── Makefile /src/feditest/protocols/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | venv*/ 3 | __pycache__/ 4 | .vscode/ 5 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.ubos-backup filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /tests.smoke/imp.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "nodedriver" : "ImpInProcessNodeDriver" 3 | } 4 | -------------------------------------------------------------------------------- /tests.smoke/mastodon.ubos.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "nodedriver": "MastodonUbosNodeDriver" 3 | } -------------------------------------------------------------------------------- /src/feditest/protocols/fediverse/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Fediverse testing utils 3 | """ 4 | 5 | pass 6 | -------------------------------------------------------------------------------- /tests.smoke/wordpress.ubos.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "nodedriver": "WordPressPlusPluginsUbosNodeDriver" 3 | } 4 | -------------------------------------------------------------------------------- /README-PyPI.md: -------------------------------------------------------------------------------- 1 | # FediTest 2 | 3 | Test framework to test distributed, heterogeneous systems... 4 | 5 | ...with complex protocols 6 | 7 | ...such as the Fediverse. 8 | 9 | --- 10 | 11 | More info and documentation: [feditest.org](https://feditest.org/). 12 | -------------------------------------------------------------------------------- /REUSE.toml: -------------------------------------------------------------------------------- 1 | # Attempting to follow https://reuse.software/spec-3.3/#reusetoml 2 | # If this is wrong or misleading, please point it out 3 | 4 | version = 1 5 | 6 | [[annotations]] 7 | path = "**" 8 | SPDX-FileCopyrightText = "2023 and later, Johannes Ernst https://j12t.org/ and other FediTest contributors" 9 | SPDX-License-Identifier = "MIT" 10 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/head.jinja2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/footer.jinja2: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests.smoke/mastodon_api.session.json: -------------------------------------------------------------------------------- 1 | { 2 | "constellation": { 3 | "roles": { 4 | "server": null 5 | }, 6 | "name": null 7 | }, 8 | "tests": [ 9 | { 10 | "name": "node_with_mastodon_api::CreateNoteTest", 11 | "rolemapping": null, 12 | "skip": null 13 | } 14 | ], 15 | "name": null 16 | } 17 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/mobile.jinja2: -------------------------------------------------------------------------------- 1 |
2 |

This FediTest report requires a desktop or laptop-class monitor.

3 |

It cannot be viewed on a mobile device with a small screen.

4 |

Sorry about that. But then, you weren't going to use a phone to fix reported bugs, either, right?

5 |
-------------------------------------------------------------------------------- /tests.smoke/webfinger.session.json: -------------------------------------------------------------------------------- 1 | { 2 | "constellation": { 3 | "roles": { 4 | "client": null, 5 | "server": null 6 | }, 7 | "name": null 8 | }, 9 | "tests": [ 10 | { 11 | "name": "webfinger::fetch", 12 | "rolemapping": null, 13 | "skip": null 14 | } 15 | ], 16 | "name": null 17 | } 18 | -------------------------------------------------------------------------------- /src/feditest/protocols/fediverse/diag.py: -------------------------------------------------------------------------------- 1 | from feditest.protocols.activitypub.diag import ActivityPubDiagNode 2 | from feditest.protocols.fediverse import FediverseNode 3 | from feditest.protocols.webfinger.diag import WebFingerDiagClient, WebFingerDiagServer 4 | 5 | 6 | class FediverseDiagNode(WebFingerDiagClient, WebFingerDiagServer,ActivityPubDiagNode,FediverseNode): 7 | """ 8 | FIXME 9 | """ 10 | -------------------------------------------------------------------------------- /src/feditest/protocols/web/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Types for Nodes that act as HTTP clients and servers. 3 | """ 4 | 5 | from feditest.nodedrivers import Node 6 | 7 | 8 | class WebClient(Node): 9 | """ 10 | Abstract class used for Nodes that speak HTTP as client. 11 | """ 12 | pass 13 | 14 | 15 | class WebServer(Node): 16 | """ 17 | Abstract class used for Nodes that speak HTTP as server. 18 | """ 19 | pass 20 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/testresult.jinja2: -------------------------------------------------------------------------------- 1 | {%- if result %} 2 | 3 |
{{ result.short_title() | e }}
4 | 5 | {%- else %} 6 | 7 |
Passed
8 | 9 | {%- endif %} 10 | -------------------------------------------------------------------------------- /tests.smoke/mastodon_api_mastodon_api.session.json: -------------------------------------------------------------------------------- 1 | { 2 | "constellation": { 3 | "roles": { 4 | "leader_node": null, 5 | "follower_node": null 6 | }, 7 | "name": null 8 | }, 9 | "tests": [ 10 | { 11 | "name": "nodes_with_mastodon_api_communicate::FollowTest", 12 | "rolemapping": null, 13 | "skip": null 14 | } 15 | ], 16 | "name": null 17 | } 18 | -------------------------------------------------------------------------------- /tests.unit/dummy.py: -------------------------------------------------------------------------------- 1 | # 2 | # Dummy classes for testing 3 | # 4 | 5 | from feditest.nodedrivers import AccountManager, Node, NodeConfiguration, NodeDriver 6 | 7 | 8 | class DummyNode(Node): 9 | pass 10 | 11 | 12 | class DummyNodeDriver(NodeDriver): 13 | # Python 3.12 @Override 14 | def _provision_node(self, rolename: str, config: NodeConfiguration, account_manager: AccountManager | None) -> Node: 15 | return DummyNode(rolename, config, account_manager) 16 | 17 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/json.py: -------------------------------------------------------------------------------- 1 | from typing import IO 2 | 3 | from feditest.testruntranscript import TestRunTranscript 4 | from feditest.testruntranscriptserializer import FileOrStdoutTestRunTranscriptSerializer 5 | 6 | 7 | class JsonTestRunTranscriptSerializer(FileOrStdoutTestRunTranscriptSerializer): 8 | """ 9 | An object that knows how to serialize a TestRun into JSON format 10 | """ 11 | def _write(self, transcript: TestRunTranscript, fd: IO[str]): 12 | transcript.write(fd) 13 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/testresult.jinja2: -------------------------------------------------------------------------------- 1 | {%- if result %} 2 |
3 | 4 | 5 | 
{{ result | e }}
6 |
7 | {%- else %} 8 |
9 |
Passed
10 |
11 | {%- endif %} 12 | -------------------------------------------------------------------------------- /tests.unit/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Don't accidentally consider anything below feditest to be a pyunit test. 3 | """ 4 | 5 | import inspect 6 | 7 | import pytest 8 | 9 | 10 | @pytest.hookimpl(wrapper=True) 11 | def pytest_pycollect_makeitem(collector, name, obj): 12 | # Ignore all feditest classes and function using 13 | # pytest naming conventions. 14 | if isinstance(obj, type) or inspect.isfunction(obj) or inspect.ismethod(obj): 15 | m = obj.__module__.split(".") 16 | if len(m) > 0 and m[0] == "feditest": 17 | yield 18 | return None 19 | result = yield 20 | return result 21 | -------------------------------------------------------------------------------- /hatch_build.py: -------------------------------------------------------------------------------- 1 | # 2 | # Set a dynamic version number, see https://hatch.pypa.io/dev/how-to/config/dynamic-metadata/ 3 | # At release time, override with env var: FEDITEST_RELEASE_VERSION=y 4 | # 5 | 6 | from datetime import datetime 7 | import os 8 | 9 | from hatchling.metadata.plugin.interface import MetadataHookInterface 10 | 11 | 12 | class JSONMetaDataHook(MetadataHookInterface): 13 | def update(self, metadata): 14 | if 'FEDITEST_RELEASE_VERSION' in os.environ and os.environ['FEDITEST_RELEASE_VERSION'].lower() == 'y': 15 | metadata['version'] = metadata['base_version'] 16 | else: 17 | metadata['version'] = metadata['base_version'] + '.dev' + datetime.now().strftime("%Y%m%d%H%M%S") 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `feditest`: test federated protocols such as those in the Fediverse 2 | 3 | This repo contains: 4 | 5 | * the FediTest test framework 6 | 7 | which allows you to define and run test plans that involve constellations of servers (like Fediverse instances) whose communication you want to test. 8 | 9 | The actual tests for the Fediverse are in their own [repository](https://github.com/fediverse-devnet/feditest-tests-fediverse). 10 | 11 | For more details, check out [feditest.org](https://feditest.org/) and find us on Matrix in [#fediverse-testing:matrix.org](https://matrix.to/#/%23fediverse-testing:matrix.org). 12 | 13 | Found a bug? You must be kidding; like in all of Arch Linux, there are no bugs in this software. But if we happen to be wrong, submit a bug report with as much detail as possible right here for this project on Github. 14 | -------------------------------------------------------------------------------- /src/feditest/cli/commands/version.py: -------------------------------------------------------------------------------- 1 | """ 2 | Show version 3 | """ 4 | 5 | from argparse import ArgumentParser, Namespace, _SubParsersAction 6 | 7 | from feditest.utils import FEDITEST_VERSION 8 | 9 | def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: 10 | """ 11 | Run this command. 12 | """ 13 | if len(remaining): 14 | parser.print_help() 15 | return 0 16 | 17 | print(FEDITEST_VERSION) 18 | return 0 19 | 20 | 21 | def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> ArgumentParser: 22 | """ 23 | Add command-line options for this sub-command 24 | parent_parser: the parent argparse parser 25 | cmd_name: name of this command 26 | """ 27 | parser = parent_parser.add_parser( cmd_name, help='Show feditest version') 28 | 29 | return parser 30 | -------------------------------------------------------------------------------- /src/feditest/nodedrivers/saas/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | 4 | from feditest.nodedrivers import AccountManager, NodeConfiguration 5 | from feditest.nodedrivers.fallback.fediverse import AbstractFallbackFediverseNodeDriver, FallbackFediverseNode 6 | 7 | 8 | class FediverseSaasNodeDriver(AbstractFallbackFediverseNodeDriver): 9 | """ 10 | A NodeDriver that supports all protocols but doesn't automate anything and assumes the 11 | Node under test exists as a website that we don't have/can provision/unprovision. 12 | """ 13 | # Python 3.12 @override 14 | def _provision_node(self, rolename: str, config: NodeConfiguration, account_manager: AccountManager | None) -> FallbackFediverseNode: 15 | return FallbackFediverseNode(rolename, config, account_manager) 16 | 17 | 18 | # No need to override _unprovision_node() 19 | 20 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/summary.py: -------------------------------------------------------------------------------- 1 | from typing import IO 2 | 3 | from feditest.testruntranscript import TestRunTranscript 4 | from feditest.testruntranscriptserializer import FileOrStdoutTestRunTranscriptSerializer 5 | 6 | 7 | class SummaryTestRunTranscriptSerializer(FileOrStdoutTestRunTranscriptSerializer): 8 | """ 9 | Knows how to serialize a TestRunTranscript into a single-line summary. 10 | """ 11 | def _write(self, transcript: TestRunTranscript, fd: IO[str]): 12 | summary = transcript.build_summary() 13 | 14 | print(f'Test summary: total={ summary.n_total }' 15 | + f', passed={ summary.n_passed }' 16 | + f', failed={ summary.n_failed }' 17 | + f', skipped={ summary.n_skipped }' 18 | + f', errors={ summary.n_errored }.', 19 | file=fd) 20 | 21 | 22 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/metadata.jinja2: -------------------------------------------------------------------------------- 1 |
2 | {%- set started = run_session.started %} 3 | {%- set ended = run_session.ended %} 4 | {%- set duration = ended - started %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | {%- for key in ['username', 'hostname', 'platform'] -%} 16 | {%- if getattr(transcript,key) %} 17 | 18 | 19 | 20 | 21 | {%- endif %} 22 | {%- endfor %} 23 | 24 | 25 | 26 | 27 | 28 |
Started{{ format_timestamp(started) }}
Ended{{ format_timestamp(ended) }} (total: {{ format_duration(duration) }})
{{ key.capitalize() }}{{ getattr(transcript, key) | e }}
FediTest version{{ getattr(transcript, 'feditest_version') }}
29 |
-------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/metadata.jinja2: -------------------------------------------------------------------------------- 1 |
2 | {%- set started = getattr(transcript, 'started') %} 3 | {%- set ended = getattr(transcript, 'ended') %} 4 | {%- set duration = ended - started %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | {%- for key in ['username', 'hostname', 'platform'] -%} 16 | {%- if getattr(transcript,key) %} 17 | 18 | 19 | 20 | 21 | {%- endif %} 22 | {%- endfor %} 23 | 24 | 25 | 26 | 27 | 28 |
Started{{ format_timestamp(started) }}
Ended{{ format_timestamp(ended) }} (total: {{ format_duration(duration) }})
{{ key.capitalize() | e }}{{ getattr(transcript, key) | e }}
FediTest version{{ getattr(transcript, 'feditest_version') | e }}
29 |
-------------------------------------------------------------------------------- /src/feditest/disabled.py: -------------------------------------------------------------------------------- 1 | # This is a little hack, but useful. 2 | # The problem: 3 | # You have a file that contains some @tests and maybe some @steps inside @tests. You want to temporarily disable them. 4 | # You can go through the file, and change all the @test and @step annotations. 5 | # Or, you can change the import statement from something like: 6 | # from feditest import AssertionFailure, InteropLevel, SpecLevel, step, test 7 | # to 8 | # from feditest.disabled import AssertionFailure, InteropLevel, SpecLevel, step, test 9 | # 10 | 11 | from typing import Callable 12 | 13 | from feditest import AssertionFailure, InteropLevel, SpecLevel, all_node_drivers, all_tests, assert_that, nodedriver # noqa: F401 14 | 15 | def test(to_register: Callable[..., None] | type) -> Callable[..., None] | type: 16 | """ 17 | Disabled: do nothing 18 | """ 19 | return to_register 20 | 21 | 22 | def step(to_register: Callable[..., None]) -> Callable[..., None]: 23 | """ 24 | Disabled: do nothing 25 | """ 26 | return to_register -------------------------------------------------------------------------------- /tests.smoke/tests/webfinger.py: -------------------------------------------------------------------------------- 1 | from feditest import InteropLevel, SpecLevel, assert_that, test 2 | from feditest.protocols.webfinger import WebFingerServer 3 | from feditest.protocols.webfinger.diag import WebFingerDiagClient 4 | from feditest.protocols.webfinger.utils import wf_error 5 | from hamcrest import not_none 6 | 7 | 8 | @test 9 | def fetch( 10 | client: WebFingerDiagClient, 11 | server: WebFingerServer 12 | ) -> None: 13 | """ 14 | Perform a normal, simple query on an existing account. 15 | This is not a WebFinger conformance test, it's too lenient for this. 16 | This is a smoke test that tests FediTest can perform these kinds of tests. 17 | """ 18 | test_id = server.obtain_account_identifier() 19 | 20 | webfinger_response = client.diag_perform_webfinger_query(test_id) 21 | 22 | assert_that( 23 | webfinger_response.jrd, 24 | not_none(), 25 | wf_error(webfinger_response), 26 | spec_level=SpecLevel.MUST, 27 | interop_level=InteropLevel.PROBLEM) 28 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | name: Pre-commit hooks 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - develop 7 | 8 | jobs: 9 | pre-commit: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v3 15 | 16 | - name: Set up Python 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: '3.11.9' 20 | 21 | - name: Cache pre-commit environments 22 | id: cache-pre-commit 23 | uses: actions/cache@v3 24 | with: 25 | path: ~/.cache/pre-commit 26 | key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} 27 | restore-keys: | 28 | ${{ runner.os }}-pre-commit- 29 | 30 | - name: Install dependencies 31 | run: | 32 | python -m pip install --upgrade pip 33 | pip install pip-tools 34 | pip-compile -o requirements.txt pyproject.toml 35 | pip install --requirement requirements.txt 36 | 37 | - name: Run pre-commit hooks 38 | run: pre-commit run --all-files 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright © 2023 and later, Johannes Ernst https://j12t.org/ and other FediTest contributors 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | this software and associated documentation files (the “Software”), to deal in 6 | the Software without restriction, including without limitation the rights to 7 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | the Software, and to permit persons to whom the Software is furnished to do so, 9 | subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /src/feditest/cli/commands/list_nodedrivers.py: -------------------------------------------------------------------------------- 1 | """ 2 | List the available drivers for nodes that can be tested 3 | """ 4 | 5 | from argparse import ArgumentParser, Namespace, _SubParsersAction 6 | 7 | import feditest 8 | import feditest.cli 9 | 10 | def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: 11 | """ 12 | Run this command. 13 | """ 14 | if len(remaining): 15 | parser.print_help() 16 | return 0 17 | 18 | if args.nodedriversdir: 19 | feditest.load_node_drivers_from(args.nodedriversdir) 20 | feditest.load_default_node_drivers() 21 | 22 | for name in sorted(feditest.all_node_drivers.keys()): 23 | print(name) 24 | 25 | return 0 26 | 27 | 28 | def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> ArgumentParser: 29 | """ 30 | Add command-line options for this sub-command 31 | parent_parser: the parent argparse parser 32 | cmd_name: name of this command 33 | """ 34 | parser = parent_parser.add_parser(cmd_name, help='List the available drivers for nodes that can be tested') 35 | parser.add_argument('--nodedriversdir', action='append', help='Directory or directories where to find extra drivers for nodes that can be tested') 36 | 37 | return parser 38 | -------------------------------------------------------------------------------- /src/feditest/nodedrivers/manual/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A NodeDriver that supports all protocols but doesn't automate anything. 3 | """ 4 | 5 | from feditest.nodedrivers import AccountManager, Node, NodeConfiguration 6 | from feditest.nodedrivers.fallback.fediverse import AbstractFallbackFediverseNodeDriver, FallbackFediverseNode 7 | from feditest.protocols.fediverse import FediverseNode 8 | from feditest.utils import prompt_user 9 | 10 | 11 | class FediverseManualNodeDriver(AbstractFallbackFediverseNodeDriver): 12 | """ 13 | A NodeDriver that supports all web server-side protocols but doesn't automate anything. 14 | """ 15 | # Python 3.12 @override 16 | def _provision_node(self, rolename: str, config: NodeConfiguration, account_manager: AccountManager | None) -> FediverseNode: 17 | prompt_user( 18 | f'Manually provision the Node for constellation role { rolename }' 19 | + f' at host { config.hostname } with app { config.app } and hit return when done: ') 20 | return FallbackFediverseNode(rolename, config, account_manager) 21 | 22 | 23 | # Python 3.12 @override 24 | def _unprovision_node(self, node: Node) -> None: 25 | prompt_user(f'Manually unprovision the Node for constellation role { node.rolename } and hit return when done: ') 26 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_single.jinja2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include "partials/shared/head.jinja2" %} 5 | {{ transcript.plan.name | e }} | FediTest 6 | 7 | 8 |
9 |

FediTest Report: {{ transcript.plan.name | e }}

10 |

{{ transcript.id }}

11 |
12 | {% include "partials/shared/mobile.jinja2" %} 13 | 20 |
21 |

Test Run Summary

22 | {% include "partials/shared/summary.jinja2" %} 23 |
24 |
25 | {% include "partials/shared_session/results.jinja2" %} 26 |
27 |
28 |

Test Run Metadata

29 | {% include "partials/shared_session/metadata.jinja2" %} 30 |
31 | {% include "partials/shared/footer.jinja2" %} 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/feditest/cli/commands/list_tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | List the available tests 3 | """ 4 | 5 | from argparse import ArgumentParser, Namespace, _SubParsersAction 6 | import re 7 | 8 | import feditest 9 | 10 | def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: 11 | """ 12 | Run this command. 13 | """ 14 | if len(remaining): 15 | parser.print_help() 16 | return 0 17 | 18 | pattern = re.compile(args.filter_regex) if args.filter_regex else None 19 | 20 | feditest.load_default_tests() 21 | feditest.load_tests_from(args.testsdir) 22 | 23 | for name in sorted(feditest.all_tests.keys()): 24 | if pattern is None or pattern.match(name): 25 | print(name) 26 | 27 | return 0 28 | 29 | def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> ArgumentParser: 30 | """ 31 | Add command-line options for this sub-command 32 | parent_parser: the parent argparse parser 33 | cmd_name: name of this command 34 | """ 35 | parser = parent_parser.add_parser(cmd_name, help='List the available tests') 36 | parser.add_argument('--filter-regex', default=None, help='Only list tests whose name matches this regular expression') 37 | parser.add_argument('--testsdir', action='append', help='Directory or directories where to find tests') 38 | 39 | return parser 40 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include "partials/shared/head.jinja2" %} 5 | {{ transcript | e }} ({{ transcript.id }}) | FediTest 6 | 7 | 8 |
9 |

FediTest: {{ transcript | e }}

10 |

{{ transcript.id }}

11 |
12 | {% include "partials/shared/mobile.jinja2" %} 13 | 20 |
21 |

Test Run Summary

22 | {% include "partials/shared/summary.jinja2" %} 23 |
24 |
25 | {% with session_links=true %} 26 |

Test Results

27 | {% include "partials/matrix/matrix.jinja2" %} 28 | {% endwith %} 29 |
30 |
31 |

Test Run Metadata

32 | {% include "partials/matrix/metadata.jinja2" %} 33 |
34 | {% include "partials/shared/footer.jinja2" %} 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_with_matrix.jinja2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include "partials/shared/head.jinja2" %} 5 | {{ run_session | e }} ({{ transcript.id }}) | FediTest 6 | 7 | 8 |
9 |

FediTest: {{ run_session | e }}

10 |

{{ transcript.id }} ⇧

11 |
12 | {% include "partials/shared/mobile.jinja2" %} 13 | 20 |
21 |

Test Run Session Summary

22 | {% include "partials/shared/summary.jinja2" %} 23 |
24 |
25 | {% include "partials/shared_session/results.jinja2" %} 26 |
27 |
28 |

Test Run Session Metadata

29 | {% include "partials/shared_session/metadata.jinja2" %} 30 |
31 | {% include "partials/shared/footer.jinja2" %} 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/feditest/protocols/activitypub/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | ActivityPub testing utils 3 | """ 4 | 5 | from typing import Any 6 | 7 | from hamcrest.core.base_matcher import BaseMatcher 8 | from hamcrest.core.description import Description 9 | 10 | from feditest.nodedrivers import Node 11 | from feditest.utils import boolean_response_parse_validate, prompt_user_parse_validate 12 | 13 | 14 | class MemberOfCollectionMatcher(BaseMatcher[Any]): 15 | """ 16 | Custom matcher: decide whether a URI is a member of Collection identified by another URI 17 | """ 18 | def __init__(self, collection_uri: str, node: Node): 19 | """ 20 | collection_uri: the URI identifying the collection which to examine 21 | """ 22 | self._collection_uri = collection_uri 23 | self._node = node 24 | 25 | 26 | def _matches(self, member_candidate_uri: str) -> bool: 27 | ret = prompt_user_parse_validate( 28 | f'Is "{ member_candidate_uri }" a member of the collection at URI "{ self._collection_uri }"? ', 29 | parse_validate=boolean_response_parse_validate) 30 | return ret 31 | 32 | 33 | def describe_to(self, description: Description) -> None: 34 | description.append_text('Not a member of the set') 35 | 36 | 37 | def is_member_of_collection_at(arg: str, node: Node) -> MemberOfCollectionMatcher : 38 | return MemberOfCollectionMatcher(arg, node) 39 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/__init__.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | import io 3 | import sys 4 | from typing import IO 5 | 6 | from feditest.testruntranscript import TestRunTranscript 7 | 8 | 9 | class TestRunTranscriptSerializer(ABC): 10 | """ 11 | An object that knows how to serialize a TestRunTranscript into some output format. 12 | """ 13 | @abstractmethod 14 | def write(self, transcript: TestRunTranscript, dest: str | None): 15 | ... 16 | 17 | 18 | class FileOrStdoutTestRunTranscriptSerializer(TestRunTranscriptSerializer): 19 | def write(self, transcript: TestRunTranscript, dest: str | None) -> None: 20 | """ 21 | dest: name of the file to write to, or stdout 22 | """ 23 | if dest and isinstance(dest,str): 24 | with open(dest, "w", encoding="utf8") as out: 25 | self._write(transcript, out) 26 | else: 27 | self._write(transcript, sys.stdout) 28 | 29 | 30 | def write_to_string(self, transcript: TestRunTranscript) -> str: 31 | """ 32 | Return the written content as a string; this is for testing. 33 | """ 34 | string_io = io.StringIO() 35 | self._write(transcript, string_io) 36 | return string_io.getvalue() 37 | 38 | 39 | @abstractmethod 40 | def _write(self, transcript: TestRunTranscript, fd: IO[str]) -> None: 41 | ... 42 | 43 | 44 | -------------------------------------------------------------------------------- /tests.unit/test_10_register_nodedrivers.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test that @nodedriver annotations register NodeDrivers correctly. 3 | """ 4 | 5 | import pytest 6 | 7 | import feditest 8 | from feditest import nodedriver 9 | from feditest.nodedrivers import NodeDriver 10 | 11 | 12 | @pytest.fixture(scope="module", autouse=True) 13 | def init(): 14 | """ Keep these isolated to this module """ 15 | feditest.all_node_drivers = {} 16 | feditest._loading_node_drivers = True 17 | 18 | @nodedriver 19 | class NodeDriver1(NodeDriver): 20 | pass 21 | 22 | @nodedriver 23 | class NodeDriver2(NodeDriver): 24 | pass 25 | 26 | @nodedriver 27 | class NodeDriver3(NodeDriver): 28 | pass 29 | 30 | feditest._loading_node_drivers = False 31 | 32 | 33 | def test_node_drivers_registered() -> None: 34 | assert len(feditest.all_node_drivers) == 3 35 | 36 | prefix = 'test_10_register_nodedrivers.init..' 37 | assert prefix + 'NodeDriver1' in feditest.all_node_drivers 38 | assert prefix + 'NodeDriver2' in feditest.all_node_drivers 39 | assert prefix + 'NodeDriver3' in feditest.all_node_drivers 40 | 41 | # Can't directly refer to NodeDriverX for some reason 42 | assert feditest.all_node_drivers.get(prefix + 'NodeDriver1').__name__.endswith('NodeDriver1') 43 | assert feditest.all_node_drivers.get(prefix + 'NodeDriver2').__name__.endswith('NodeDriver2') 44 | assert feditest.all_node_drivers.get(prefix + 'NodeDriver3').__name__.endswith('NodeDriver3') 45 | -------------------------------------------------------------------------------- /tests.unit/test_10_create_test_plan_constallation.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test the equivalent of `feditest create-constellation` 3 | """ 4 | 5 | import pytest 6 | 7 | from feditest.testplan import TestPlanConstellation, TestPlanConstellationNode 8 | 9 | @pytest.fixture(scope="session") 10 | def node1() -> TestPlanConstellationNode: 11 | return TestPlanConstellationNode( 'node1-driver', { 'foo' : 'Foo', 'bar' : 'Bar'}) 12 | 13 | 14 | @pytest.fixture(scope="session") 15 | def node2() -> TestPlanConstellationNode: 16 | return TestPlanConstellationNode( 'node2-driver', { 'baz' : 'Baz'}) 17 | 18 | 19 | def test_unnamed( 20 | node1: TestPlanConstellationNode, 21 | node2: TestPlanConstellationNode 22 | ) -> None: 23 | """ 24 | TestPlanConstellations don't have automatic names. 25 | """ 26 | roles = { 27 | 'role1' : node1, 28 | 'role2' : node2 29 | } 30 | 31 | constellation = TestPlanConstellation(roles) 32 | 33 | assert len(constellation.roles) == 2 34 | assert constellation.name is None 35 | 36 | 37 | def test_named( 38 | node1: TestPlanConstellationNode, 39 | node2: TestPlanConstellationNode 40 | ) -> None: 41 | """ 42 | TestPlanConstellations can be named 43 | """ 44 | 45 | NAME = 'My constellation' 46 | 47 | roles = { 48 | 'role1' : node1, 49 | 'role2' : node2 50 | } 51 | 52 | constellation = TestPlanConstellation(roles) 53 | constellation.name = NAME 54 | 55 | assert len(constellation.roles) == 2 56 | assert constellation.name == NAME 57 | 58 | -------------------------------------------------------------------------------- /src/feditest/cli/commands/create_constellation.py: -------------------------------------------------------------------------------- 1 | """ 2 | Combine node definitions into a constellation. 3 | """ 4 | 5 | from argparse import ArgumentParser, Namespace, _SubParsersAction 6 | 7 | from feditest.cli.utils import create_constellation_from_nodes 8 | from feditest.testplan import TestPlanConstellation, TestPlanConstellationNode 9 | 10 | 11 | def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: 12 | """ 13 | Run this command. 14 | """ 15 | 16 | roles : dict[str,TestPlanConstellationNode | None] = {} 17 | 18 | if remaining: 19 | parser.print_help() 20 | return 0 21 | 22 | constellation = TestPlanConstellation(roles) 23 | 24 | constellation = create_constellation_from_nodes(args) 25 | if args.name: 26 | constellation.name = args.name 27 | 28 | if args.out: 29 | constellation.save(args.out) 30 | else: 31 | constellation.print() 32 | 33 | return 0 34 | 35 | 36 | def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> ArgumentParser: 37 | """ 38 | Add command-line options for this sub-command 39 | parent_parser: the parent argparse parser 40 | cmd_name: name of this command 41 | """ 42 | parser = parent_parser.add_parser(cmd_name, help='Combine node definitions into a constellation') 43 | parser.add_argument('--name', default=None, required=False, help='Name of the generated constellation') 44 | parser.add_argument('--node', action='append', required=True, 45 | help="Use = to specify that the node definition in 'file' is supposed to be used for constellation role 'role'") 46 | parser.add_argument('--out', '-o', default=None, required=False, help='Name of the file for the generated constellation') 47 | 48 | return parser 49 | -------------------------------------------------------------------------------- /src/feditest/protocols/sandbox/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Abstractions for the toy "Sandbox" protocol. 3 | """ 4 | 5 | from datetime import datetime, UTC 6 | from typing import List 7 | 8 | from feditest.nodedrivers import Node, NotImplementedByNodeError 9 | 10 | class SandboxLogEvent: 11 | """ 12 | The structure of the data inserted into the log. 13 | """ 14 | def __init__(self, a: float, b: float, c: float): 15 | self.when = datetime.now(UTC) 16 | self.a = a 17 | self.b = b 18 | self.c = c 19 | 20 | 21 | class SandboxMultServer(Node): 22 | """ 23 | This is a "Server" Node in a to-be-tested toy protocol. It is only useful to illustrate how FediTest works. 24 | """ 25 | def mult(self, a: float, b: float) -> float: 26 | """ 27 | The operation that's being tested 28 | """ 29 | raise NotImplementedByNodeError(self, SandboxMultServer.mult) 30 | 31 | 32 | def start_logging(self): 33 | """ 34 | Activate logging of mult() operations 35 | """ 36 | raise NotImplementedByNodeError(self, SandboxMultServer.start_logging) 37 | 38 | 39 | def get_and_clear_log(self) -> List[SandboxLogEvent]: 40 | """ 41 | Stop logging of mult() operations, return what has been logged so far 42 | and clear the log 43 | """ 44 | raise NotImplementedByNodeError(self, SandboxMultServer.get_and_clear_log) 45 | 46 | 47 | class SandboxMultClient(Node): 48 | """ 49 | This is a "Client" Node in a to-be-tested toy protocol. It is only useful to illustrate how FediTest works. 50 | """ 51 | def cause_mult(self, server: SandboxMultServer, a: float, b: float) -> float: 52 | """ 53 | Enable FediTest to make the client perform the mult() operation on the server. 54 | """ 55 | raise NotImplementedByNodeError(self, SandboxMultClient.cause_mult) 56 | -------------------------------------------------------------------------------- /tests.unit/test_10_register_tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test that @test and @step annotations register their functions/classes/methods correctly. 3 | """ 4 | 5 | import pytest 6 | 7 | import feditest 8 | from feditest import step, test 9 | from feditest.tests import TestFromTestClass, TestFromTestFunction 10 | 11 | 12 | @pytest.fixture(scope="module", autouse=True) 13 | def init(): 14 | """ Keep these isolated to this module """ 15 | feditest.all_tests = {} 16 | feditest._registered_as_test = {} 17 | feditest._registered_as_test_step = {} 18 | feditest._loading_tests = True 19 | 20 | @test 21 | def test1() -> None: 22 | return 23 | 24 | @test 25 | def test2() -> None: 26 | return 27 | 28 | @test 29 | def test3() -> None: 30 | return 31 | 32 | @test 33 | class TestA(): 34 | @step 35 | def testa1(self) -> None: 36 | return 37 | 38 | @step 39 | def testa2(self) -> None: 40 | return 41 | 42 | feditest._loading_tests = False 43 | feditest._load_tests_pass2() 44 | 45 | 46 | def test_tests_registered() -> None: 47 | assert len(feditest.all_tests) == 4 48 | 49 | 50 | def test_functions() -> None: 51 | functions = [ testInstance for testInstance in feditest.all_tests.values() if isinstance(testInstance, TestFromTestFunction) ] 52 | assert len(functions) == 3 53 | functions[0].name.endswith('test1') 54 | functions[1].name.endswith('test2') 55 | functions[2].name.endswith('test3') 56 | 57 | 58 | def test_classes() -> None: 59 | classes = [ testInstance for testInstance in feditest.all_tests.values() if isinstance(testInstance, TestFromTestClass) ] 60 | assert len(classes) == 1 61 | 62 | singleClass = classes[0] 63 | assert singleClass.name.endswith('TestA') 64 | 65 | assert len(singleClass.steps) == 2 66 | singleClass.steps[0].name.startswith('testa') 67 | singleClass.steps[1].name.startswith('testa') 68 | -------------------------------------------------------------------------------- /src/feditest/cli/commands/create_session_template.py: -------------------------------------------------------------------------------- 1 | """ 2 | Create a template for a test session. This is very similar to list-tests, but the output is to be used 3 | as input for generate-testplan. 4 | """ 5 | 6 | from argparse import ArgumentParser, Namespace, _SubParsersAction 7 | 8 | import feditest 9 | from feditest.cli.utils import create_session_template_from_tests 10 | 11 | 12 | def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: 13 | """ 14 | Run this command. 15 | """ 16 | if len(remaining): 17 | parser.print_help() 18 | return 0 19 | 20 | feditest.load_default_tests() 21 | feditest.load_tests_from(args.testsdir) 22 | 23 | session_template = create_session_template_from_tests(args) 24 | 25 | if args.out: 26 | session_template.save(args.out) 27 | else: 28 | session_template.print() 29 | 30 | return 0 31 | 32 | 33 | def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> ArgumentParser: 34 | """ 35 | Add command-line options for this sub-command 36 | parent_parser: the parent argparse parser 37 | cmd_name: name of this command 38 | """ 39 | # general flags and options 40 | parser = parent_parser.add_parser(cmd_name, help='Create a template for a test session') 41 | parser.add_argument('--testsdir', action='append', help='Directory or directories where to find tests') 42 | 43 | # session template options 44 | parser.add_argument('--name', default=None, required=False, help='Name of the created test session template') 45 | parser.add_argument('--filter-regex', default=None, help='Only include tests whose name matches this regular expression') 46 | parser.add_argument('--test', action='append', help='Include this/these named tests(s)') 47 | 48 | # output options 49 | parser.add_argument('--out', '-o', default=None, required=False, help='Name of the file for the created test session template') 50 | 51 | return parser 52 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/matrix.jinja2: -------------------------------------------------------------------------------- 1 | {% macro column_headings(first) %} 2 | 3 | {{ first | e }} 4 | {%- for run_session in transcript.sessions %} 5 | {%- set constellation = run_session.constellation %} 6 | 7 |
8 |

{{ run_session }}

9 |
10 | {%- for role, node in constellation.nodes.items() %} 11 |
{{ role | e }}
12 |
{{ node.appdata['app'] | e }}
13 | {%- endfor %} 14 |
15 |
16 | 17 | {%- endfor %} 18 | 19 | {% endmacro %} 20 | 21 |
22 | 23 | 24 | 25 | {%- for run_session in transcript.sessions %} 26 | 27 | {%- endfor %} 28 | 29 | 30 | {{ column_headings("{0} tests in {1} sessions (alphabetical order)".format(len(transcript.test_meta), len(transcript.sessions))) }} 31 | 32 | 33 | {%- for test_index, ( _, test_meta ) in enumerate(sorted(transcript.test_meta.items())) %} 34 | 35 | 41 | {%- for session_index, run_session in enumerate(transcript.sessions) %} 42 | {%- for result in get_results_for(transcript, run_session, test_meta) %} 43 | {% include "partials/matrix/testresult.jinja2" %} 44 | {%- endfor %} 45 | {%- endfor %} 46 | 47 | {%- endfor %} 48 | 49 | 50 | {{ column_headings("") }} 51 | 52 |
36 | {{ permit_line_breaks_in_identifier(test_meta.name) | safe }} 37 | {%- if test_meta.description %} 38 | {{ test_meta.description | e }} 39 | {%- endif %} 40 |
53 |
54 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_install_hook_types: 2 | - pre-push 3 | 4 | default_language_version: 5 | python: python3.11 6 | 7 | repos: 8 | - repo: https://github.com/pre-commit/pre-commit-hooks 9 | rev: v4.6.0 10 | hooks: 11 | # - id: trailing-whitespace 12 | # args: [--markdown-linebreak-ext=md] 13 | # - id: end-of-file-fixer 14 | - id: mixed-line-ending 15 | - id: check-yaml 16 | - id: sort-simple-yaml 17 | - id: check-added-large-files 18 | 19 | # - repo: https://github.com/psf/black 20 | # rev: 24.8.0 21 | # hooks: 22 | # - id: black 23 | # args: 24 | # - --config=pyproject.toml 25 | 26 | # - repo: https://github.com/pycqa/isort 27 | # rev: 5.13.2 28 | # hooks: 29 | # - id: isort 30 | # name: isort (python) 31 | # args: ["--profile", "black", "--filter-files"] 32 | 33 | - repo: https://github.com/astral-sh/ruff-pre-commit 34 | # Ruff version. 35 | rev: v0.6.0 36 | hooks: 37 | # Run the linter. 38 | - id: ruff 39 | args: ["--fix"] 40 | # Run the formatter. 41 | # - id: ruff-format 42 | 43 | - repo: local 44 | hooks: 45 | - id: pytest 46 | name: pytest 47 | entry: pytest 48 | language: system 49 | #language: script 50 | pass_filenames: false 51 | # alternatively you could `types: [python]` so it only runs when python files change 52 | # though tests might be invalidated if you were to say change a data file 53 | always_run: true 54 | 55 | - repo: https://github.com/pre-commit/mirrors-mypy 56 | rev: v1.11.2 57 | hooks: 58 | - id: mypy 59 | files: src 60 | args: 61 | - "--ignore-missing-imports" # Not managing to get it to work from pre-commit without this flag 62 | - "--namespace-packages" 63 | - "--explicit-package-bases" 64 | - "--install-types" 65 | - "--non-interactive" 66 | 67 | - repo: local 68 | hooks: 69 | - id: nocommit 70 | name: NOCOMMIT check 71 | entry: NOCOMMIT 72 | language: pygrep 73 | exclude: .pre-commit-config.yaml 74 | types: [text] 75 | -------------------------------------------------------------------------------- /tests.unit/test_20_create_test_plan_session_template.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test the equivalent of `feditest create-session-template` 3 | """ 4 | 5 | import pytest 6 | 7 | import feditest 8 | from feditest import test 9 | from feditest.nodedrivers import Node 10 | from feditest.testplan import TestPlanSessionTemplate, TestPlanTestSpec 11 | 12 | 13 | @pytest.fixture(scope="module", autouse=True) 14 | def init(): 15 | """ Keep these isolated to this module """ 16 | feditest.all_tests = {} 17 | feditest._registered_as_test = {} 18 | feditest._registered_as_test_step = {} 19 | feditest._loading_tests = True 20 | 21 | @test 22 | def test1(role_a: Node) -> None: 23 | return 24 | 25 | @test 26 | def test2(role_a: Node, role_c: Node) -> None: 27 | return 28 | 29 | @test 30 | def test3(role_b: Node) -> None: 31 | return 32 | 33 | feditest._loading_tests = False 34 | feditest._load_tests_pass2() 35 | 36 | 37 | SESSION_TEMPLATE_NAME = 'My Session' 38 | 39 | 40 | def _session_template(session_name: str | None) -> TestPlanSessionTemplate: 41 | test_plan_specs : list[TestPlanTestSpec]= [] 42 | for name in sorted(feditest.all_tests.keys()): 43 | test = feditest.all_tests.get(name) 44 | if test is None: # make linter happy 45 | continue 46 | test_plan_spec = TestPlanTestSpec(name) 47 | test_plan_specs.append(test_plan_spec) 48 | 49 | session = TestPlanSessionTemplate(test_plan_specs, session_name) 50 | return session 51 | 52 | 53 | @pytest.fixture() 54 | def unnamed() -> TestPlanSessionTemplate: 55 | return _session_template(None) 56 | 57 | 58 | @pytest.fixture() 59 | def named() -> TestPlanSessionTemplate: 60 | return _session_template(SESSION_TEMPLATE_NAME) 61 | 62 | 63 | def test_session_template_unnamed(unnamed: TestPlanSessionTemplate) -> None: 64 | assert unnamed.name is None 65 | assert str(unnamed) == 'Unnamed' 66 | assert len(unnamed.tests) == 3 67 | 68 | 69 | def test_session_template_named(named: TestPlanSessionTemplate) -> None: 70 | assert named.name == SESSION_TEMPLATE_NAME 71 | assert str(named) == SESSION_TEMPLATE_NAME 72 | assert len(named.tests) == 3 73 | -------------------------------------------------------------------------------- /tests.smoke/tests/node_with_mastodon_api.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for our implementation of the NodeWithMastodonAPI. 3 | This tests produces regular Python assertion errors, not feditest assertion errors, because 4 | problems here are problems in our NodeDriver, not Fediverse interop problems. 5 | """ 6 | 7 | from datetime import datetime 8 | 9 | from feditest import poll_until, step, test 10 | from feditest.nodedrivers.mastodon import NodeWithMastodonAPI 11 | 12 | # @test 13 | # def app_version( 14 | # server: NodeWithMastodonAPI 15 | # ) -> None: 16 | # # FIXME: need to implement property mastodon_api_app_version 17 | # # This should access Mastodon without an authenticated user and we don't currently have code for how to do that 18 | # assert re.fullmatch(r'\d+\.\d+\.\d+', server.mastodon_api_app_version), "Invalid version" 19 | 20 | 21 | @test 22 | class CreateNoteTest: 23 | """ 24 | Tests that we can create a Note through the Mastodon API. 25 | """ 26 | def __init__(self, 27 | server: NodeWithMastodonAPI 28 | ) -> None: 29 | self.server = server 30 | self.actor_acct_uri = None 31 | self.note_uri = None 32 | 33 | 34 | @step 35 | def provision_actor(self): 36 | self.actor_acct_uri = self.server.obtain_actor_acct_uri() 37 | assert self.actor_acct_uri 38 | 39 | 40 | # @step 41 | # def start_reset_all(self): 42 | # self._reset_all() 43 | 44 | 45 | @step 46 | def create_note(self): 47 | self.note_uri = self.server.make_create_note(self.actor_acct_uri, f"testing make_create_note {datetime.now()}") 48 | assert self.note_uri 49 | 50 | 51 | @step 52 | def wait_for_note_in_inbox(self): 53 | poll_until(lambda: self.server.actor_has_received_object(self.actor_acct_uri, self.note_uri)) 54 | 55 | 56 | # @step 57 | # def end_reset_all(self): 58 | # self._reset_all() 59 | 60 | 61 | def _reset_all(self): 62 | """ 63 | Clean up data. This is here so the test is usable with non-brand-new instances. 64 | """ 65 | self.server.delete_all_followers_of(self.actor_acct_uri) 66 | self.server.delete_all_statuses_by(self.actor_acct_uri) 67 | 68 | -------------------------------------------------------------------------------- /tests.smoke/README.md: -------------------------------------------------------------------------------- 1 | # FediTest smoke tests 2 | 3 | ## Files 4 | 5 | The files in this directory: 6 | 7 | ### Session Templates 8 | 9 | `mastodon_api.session.json`: 10 | : Tests our use of the Mastodon API against a single Node that implements `NodeWithMastodonAPI`, such as Mastodon or WordPress with plugins 11 | 12 | `mastodon_api_mastodon_api.session.json`: 13 | : Simple tests that test whether two `NodeWithMastodonAPI`s can communicate. 14 | 15 | ### Constellations 16 | 17 | `mastodon_mastodon.ubos.constellation.json`: 18 | : A Constellation consisting of two instances of Mastodon, running on UBOS 19 | 20 | `mastodon.saas.constellation.json`: 21 | : A (degenerate) Constellation consisting of only one instance of Mastodon that already runs at a public hostname 22 | 23 | `mastodon.ubos.constellation.json`: 24 | : A (degenerate) Constellation consisting of only one instance of Mastodon, running on UBOS 25 | 26 | `wordpress_mastodon.ubos.constellation.json`: 27 | : A Constellation consisting of one instance of Mastodon and one of WordPress with plugins, running on UBOS 28 | 29 | `wordpress.saas.constellation.json`: 30 | : A (degenerate) Constellation consisting of only one instance of WordPress with plugins that already runs at a public hostname 31 | 32 | `wordpress.ubos.constellation.json`: 33 | : A (degenerate) Constellation consisting of only one instance of WordPress with plugins, running on UBOS 34 | 35 | ### Actual Tests 36 | 37 | `tests/node_with_mastodon_api.py`: 38 | : Tests the Mastodon API 39 | 40 | `tests/nodes_with_mastodon_api_communicate.py`: 41 | : Tests that two nodes with the Mastodon API can communicate 42 | 43 | ## How to run 44 | 45 | Combine a session template with a constellation, such as: 46 | 47 | `feditest run --session mastodon_api.session.json --constellation mastodon.ubos.constellation.json` 48 | : Runs the Mastodon API test against a single Mastodon node on UBOS 49 | 50 | `feditest run --session mastodon_api.session.json --constellation wordpress.ubos.constellation.json` 51 | : Runs the Mastodon API test against a single WordPress plus plugins node on UBOS 52 | 53 | etc. 54 | 55 | If you invoke FediTest from any directory other than this one, make sure you specify a `--testsdir ` to this directory's 56 | subdirectory `tests` so FediTest can find the test files. 57 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | 6 | [project] 7 | name = "feditest" 8 | base_version = "0.6" 9 | dynamic = ["version"] 10 | authors = [ 11 | { name="Johannes Ernst", email="git@j12t.org" }, 12 | { name="Steve Bate", email="svc-github@stevebate.net" } 13 | ] 14 | maintainers = [ 15 | { name="Johannes Ernst", email="git@j12t.org" }, 16 | { name="Steve Bate", email="svc-github@stevebate.net" } 17 | ] 18 | dependencies = [ 19 | "cryptography", 20 | "httpx", 21 | "langcodes", 22 | "msgspec", 23 | "multidict", 24 | "jinja2", 25 | "pyhamcrest", 26 | "requests", 27 | "types-requests", 28 | "pre-commit", 29 | 30 | # For testing: not sure how to specify this just for testing 31 | "pytest", 32 | "beautifulsoup4" 33 | ] 34 | 35 | description = "Test framework to test distributed, heterogeneous systems with complex protocols such as the Fediverse" 36 | readme = "README-PyPI.md" 37 | 38 | # We develop on 3.11, so we can support debian 12 (including Raspberry PI OS) systems, 39 | # which have not been upgraded to 3.12 yet. 40 | requires-python = ">=3.11" 41 | # We really want 3.12 so we can use @override 42 | # Do not specify an upper boundary, see https://github.com/fediverse-devnet/feditest/issues/412 43 | 44 | classifiers = [ 45 | "Development Status :: 3 - Alpha", 46 | "Environment :: Console", 47 | "Intended Audience :: Developers", 48 | "License :: OSI Approved :: MIT License", 49 | "Operating System :: MacOS :: MacOS X", 50 | "Operating System :: POSIX :: Linux", 51 | "Programming Language :: Python :: 3", 52 | "Topic :: Software Development :: Testing" 53 | ] 54 | 55 | [project.scripts] 56 | feditest = "feditest.cli:main" 57 | 58 | [project.urls] 59 | Homepage = "https://feditest.org/" 60 | 61 | [tool.hatch.build.targets.sdist] 62 | exclude = [ 63 | "docs/" 64 | ] 65 | 66 | [tool.hatch.metadata.hooks.custom] 67 | # Empty: https://hatch.pypa.io/dev/how-to/config/dynamic-metadata/ 68 | 69 | [tool.pylint."MESSAGES CONTROL"] 70 | max-line-length=120 71 | disable="arguments-renamed, empty-docstring, global-variable-not-assigned, line-too-long, missing-class-docstring, missing-function-docstring, too-few-public-methods, too-many-arguments" 72 | 73 | [tool.pytest.ini_options] 74 | pythonpath = ["src"] 75 | testpaths = [ "tests.unit" ] 76 | 77 | [tool.ruff] 78 | target-version = "py311" 79 | -------------------------------------------------------------------------------- /src/feditest/cli/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Main entry point for CLI invocation 3 | """ 4 | 5 | from argparse import ArgumentError, ArgumentParser, Action 6 | import importlib 7 | import sys 8 | import traceback 9 | from types import ModuleType 10 | 11 | from feditest.reporting import fatal, set_reporting_level, warning 12 | from feditest.utils import find_submodules 13 | import feditest.cli.commands 14 | 15 | def main() -> None: 16 | """ 17 | Main entry point for CLI invocation. 18 | """ 19 | 20 | # Discover and install sub-commands 21 | cmds = find_commands() 22 | 23 | parser = ArgumentParser(description='FediTest: test federated protocols') 24 | parser.add_argument('-v', '--verbose', action='count', default=0, 25 | help='Display extra output. May be repeated for even more output' ) 26 | cmd_parsers : Action = parser.add_subparsers(dest='command', required=True) 27 | cmd_sub_parsers : dict[str,ArgumentParser] = {} 28 | 29 | for cmd_name, cmd in cmds.items(): 30 | cmd_sub_parsers[cmd_name] = cmd.add_sub_parser(cmd_parsers, cmd_name) 31 | 32 | args,remaining = parser.parse_known_args(sys.argv[1:]) 33 | cmd_name = args.command 34 | 35 | set_reporting_level(args.verbose) 36 | 37 | if sys.version_info.major != 3 or sys.version_info.minor != 11: 38 | warning(f"feditest currently requires Python 3.11. You are using { sys.version }" 39 | + " and may get unpredictable results. We'll get to other versions in the future.") 40 | 41 | if cmd_name in cmds: 42 | try : 43 | ret = cmds[cmd_name].run(cmd_sub_parsers[cmd_name], args, remaining) 44 | sys.exit( ret ) 45 | 46 | except ArgumentError as e: 47 | fatal(e.message) 48 | except Exception as e: # pylint: disable=broad-exception-caught 49 | if args.verbose > 1: 50 | traceback.print_exception( e ) 51 | fatal( str(type(e)), '--', e ) 52 | 53 | else: 54 | fatal('Sub-command not found:', cmd_name, '. Add --help for help.' ) 55 | 56 | 57 | def find_commands() -> dict[str,ModuleType]: 58 | """ 59 | Find available commands. 60 | """ 61 | cmd_names = find_submodules(feditest.cli.commands) 62 | 63 | cmds = {} 64 | for cmd_name in cmd_names: 65 | mod = importlib.import_module('feditest.cli.commands.' + cmd_name) 66 | cmds[cmd_name.replace('_', '-')] = mod 67 | 68 | return cmds 69 | 70 | 71 | if __name__ == '__main__': 72 | main() 73 | -------------------------------------------------------------------------------- /src/feditest/cli/commands/create_testplan.py: -------------------------------------------------------------------------------- 1 | """ 2 | Create a test plan. 3 | """ 4 | 5 | from argparse import ArgumentParser, Namespace, _SubParsersAction 6 | import feditest 7 | from feditest.cli.utils import create_plan_from_session_and_constellations 8 | from feditest.reporting import fatal 9 | 10 | def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: 11 | """ 12 | Run this command. 13 | """ 14 | if len(remaining): 15 | parser.print_help() 16 | return 0 17 | 18 | feditest.load_default_tests() 19 | feditest.load_tests_from(args.testsdir) 20 | 21 | test_plan = create_plan_from_session_and_constellations(args) 22 | if test_plan: 23 | if args.out: 24 | test_plan.save(args.out) 25 | else: 26 | test_plan.print() 27 | else: 28 | fatal('Failed to create test plan from the provided arguments') 29 | 30 | return 0 31 | 32 | 33 | def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> ArgumentParser: 34 | """ 35 | Add command-line options for this sub-command 36 | parent_parser: the parent argparse parser 37 | cmd_name: name of this command 38 | """ 39 | # general flags and options 40 | parser = parent_parser.add_parser(cmd_name, help='Create a test plan by running all provided test sessions in all provided constellations') 41 | parser.add_argument('--testsdir', action='append', help='Directory or directories where to find tests') 42 | 43 | # test plan options 44 | parser.add_argument('--name', default=None, required=False, help='Name of the generated test plan') 45 | parser.add_argument('--constellation', action='append', help='File(s) each containing a JSON fragment defining a constellation') 46 | parser.add_argument('--session', '--session-template', required=False, help='File containing a JSON fragment defining a test session') 47 | parser.add_argument('--node', action='append', 48 | help="Use = to specify that the node definition in 'file' is supposed to be used for constellation role 'role'") 49 | parser.add_argument('--filter-regex', default=None, help='Only include tests whose name matches this regular expression') 50 | parser.add_argument('--test', action='append', help='Run this/these named tests(s)') 51 | 52 | # output options 53 | parser.add_argument('--out', '-o', default=None, required=False, help='Name of the file for the generated test plan') 54 | 55 | return parser 56 | -------------------------------------------------------------------------------- /tests.unit/test_10_registry.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test the UBOS host registry / CA 3 | """ 4 | 5 | import tempfile 6 | 7 | from feditest.registry import Registry 8 | 9 | 10 | def test_allocates_domain(): 11 | r = Registry.create() 12 | assert len(r.ca.domain) > 4 13 | 14 | 15 | def test_uses_domain(): 16 | D = 'something.example' 17 | r = Registry.create( D ) 18 | assert r.ca.domain == D 19 | 20 | 21 | def test_root_ca(): 22 | r = Registry.create() 23 | rr = r.obtain_registry_root() 24 | assert 'PRIVATE KEY' in rr.key 25 | assert 'CERTIFICATE' in rr.cert 26 | assert isinstance(rr.key, str) 27 | assert isinstance(rr.cert, str) 28 | 29 | 30 | def test_new_hosts(): 31 | D = 'something.example' 32 | r = Registry.create( D ) 33 | 34 | h1 = r.obtain_new_hostname() 35 | h2 = r.obtain_new_hostname('foo') 36 | h3 = r.obtain_new_hostname() 37 | h4 = r.obtain_new_hostname('bar') 38 | 39 | assert h1 40 | assert h2 41 | assert h3 42 | assert h4 43 | assert h1.startswith('unnamed') 44 | assert h2.startswith('foo') 45 | assert h3.startswith('unnamed') 46 | assert h4.startswith('bar') 47 | assert h1.endswith('.' + D) 48 | assert h2.endswith('.' + D) 49 | assert h3.endswith('.' + D) 50 | assert h4.endswith('.' + D) 51 | 52 | 53 | def test_new_host_and_cert(): 54 | D = 'something.example' 55 | r = Registry.create( D ) 56 | 57 | h1info = r.obtain_new_hostinfo() 58 | 59 | assert h1info.host.startswith('unnamed') 60 | assert h1info.host.endswith('.' + D) 61 | assert 'PRIVATE KEY' in h1info.key 62 | assert 'CERTIFICATE' in h1info.cert 63 | assert isinstance(h1info.key, str) 64 | assert isinstance(h1info.cert, str) 65 | 66 | 67 | def test_save_restore(): 68 | D = 'something.example' 69 | r1 = Registry.create( D ) 70 | 71 | for i in range(5): 72 | r1.obtain_new_hostinfo('') 73 | 74 | file = tempfile.NamedTemporaryFile(delete=True).name 75 | r1.save(file) 76 | r2 = Registry.load(file) 77 | 78 | assert r2.ca.domain == r1.ca.domain 79 | assert r2.ca.key == r1.ca.key 80 | assert r2.ca.cert == r1.ca.cert 81 | 82 | assert len(r2.hosts) == len(r1.hosts) 83 | for host in r2.hosts: 84 | hostinfo1 = r1.hosts[host] 85 | hostinfo2 = r2.hosts[host] 86 | 87 | assert hostinfo1 88 | assert hostinfo2 89 | 90 | assert hostinfo2.host == hostinfo1.host 91 | assert hostinfo2.key == hostinfo1.key 92 | assert hostinfo2.cert == hostinfo1.cert 93 | 94 | -------------------------------------------------------------------------------- /tests.unit/test_50_run_passes.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run a test that passes. 3 | """ 4 | 5 | import pytest 6 | 7 | import feditest 8 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec 9 | from feditest.testrun import TestRun 10 | from feditest.testruncontroller import AutomaticTestRunController 11 | from feditest import test 12 | 13 | 14 | @pytest.fixture(scope="module", autouse=True) 15 | def init_node_drivers(): 16 | """ 17 | Cleanly define the NodeDrivers. 18 | """ 19 | feditest.all_node_drivers = {} 20 | feditest.load_default_node_drivers() 21 | 22 | 23 | @pytest.fixture(scope="module", autouse=True) 24 | def init_tests(): 25 | """ 26 | Cleanly define some tests. 27 | """ 28 | feditest.all_tests = {} 29 | feditest._registered_as_test = {} 30 | feditest._registered_as_test_step = {} 31 | feditest._loading_tests = True 32 | 33 | ## 34 | ## FediTest tests start here 35 | ## 36 | 37 | @test 38 | def passes() -> None: 39 | """ 40 | This test always passes. 41 | """ 42 | return 43 | 44 | ## 45 | ## FediTest tests end here 46 | ## (Don't forget the next two lines) 47 | ## 48 | 49 | feditest._loading_tests = False 50 | feditest._load_tests_pass2() 51 | 52 | 53 | @pytest.fixture(autouse=True) 54 | def test_plan_fixture() -> TestPlan: 55 | """ 56 | The test plan tests all known tests. 57 | """ 58 | constellation = TestPlanConstellation({}, 'No nodes needed') 59 | tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] 60 | session = TestPlanSessionTemplate(tests, "Test a test that passes") 61 | ret = TestPlan(session, [ constellation ]) 62 | ret.properties_validate() 63 | return ret 64 | 65 | 66 | def test_run_testplan(test_plan_fixture: TestPlan): 67 | test_plan_fixture.check_can_be_executed() 68 | 69 | test_run = TestRun(test_plan_fixture) 70 | controller = AutomaticTestRunController(test_run) 71 | test_run.run(controller) 72 | 73 | transcript = test_run.transcribe() 74 | summary = transcript.build_summary() 75 | 76 | assert summary.n_total == 1 77 | assert summary.n_failed == 0 78 | assert summary.n_skipped == 0 79 | assert summary.n_errored == 0 80 | assert summary.n_passed == 1 81 | 82 | assert len(transcript.sessions) == 1 83 | assert len(transcript.sessions[0].run_tests) == 1 84 | assert transcript.sessions[0].run_tests[0].result is None 85 | -------------------------------------------------------------------------------- /src/feditest/protocols/activitypub/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Abstractions for the ActivityPub protocol 3 | """ 4 | 5 | from feditest.nodedrivers import NotImplementedByNodeError 6 | from feditest.protocols.web import WebServer 7 | 8 | 9 | class ActivityPubNode(WebServer): 10 | """ 11 | A Node that can speak ActivityPub. 12 | """ 13 | def obtain_actor_document_uri(self, rolename: str | None = None) -> str: 14 | """ 15 | Smart factory method to return the https URI to an Actor document on this Node that 16 | either exists already or is newly created. Different rolenames produce different 17 | results; the same rolename produces the same result. 18 | rolename: refer to this Actor by this rolename; used to disambiguate multiple 19 | Actors on the same server by how they are used in tests 20 | return: the URI 21 | """ 22 | raise NotImplementedByNodeError(self, ActivityPubNode.obtain_actor_document_uri) 23 | 24 | # You might expect lots of more methods here. Sorry to disappoint. But there's a reason: 25 | # 26 | # Example: you might expect a method that checks that some actor A is following actor B 27 | # (which is hosted on this ActivityPubNode). You might think that could be implemented 28 | # in one of the following ways: 29 | # 30 | # * an API call to the ActivityPubNode or a database query. Yep, it could, but that's 31 | # a lot of work to implement, and many applications don't have such an API. 32 | # 33 | # * find the following collection of actor B, and look into that collection. That would 34 | # require "something" to perform HTTP GET requests oforn B's actor document, and the 35 | # collection URIs. That works, but who is that "something"? It cannot be FediTest, 36 | # otherwise FediTest would become its own Node in the current Constellation, thereby 37 | # changing it quite a bit. This is particularly important when applications require 38 | # authorized fetch to fetch follower collections, and suddenly FediTest needs to 39 | # first exchange public keys etc. and for that it would have to be an HTTP server, 40 | # with DNS and TLS certs and nope, we are not going there. 41 | # 42 | # Instead, we ask an ActivityPubDiagNode in the Constellation to perform the fetch 43 | # of the followers collection on our behalf. It is already part of the Constellation 44 | # and likely has already exchanged keys. 45 | # 46 | # So: you find what you want on the "other" Node which is likeley an ActivityPubDiagNode 47 | # anyway. 48 | -------------------------------------------------------------------------------- /tests.unit/test_50_run_skip.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run a test that wants to be skipped. 3 | """ 4 | 5 | import pytest 6 | 7 | import feditest 8 | from feditest.nodedrivers import SkipTestException 9 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec 10 | from feditest.testrun import TestRun 11 | from feditest.testruncontroller import AutomaticTestRunController 12 | from feditest import test 13 | 14 | 15 | @pytest.fixture(scope="module", autouse=True) 16 | def init_node_drivers(): 17 | """ 18 | Cleanly define the NodeDrivers. 19 | """ 20 | feditest.all_node_drivers = {} 21 | feditest.load_default_node_drivers() 22 | 23 | 24 | @pytest.fixture(scope="module", autouse=True) 25 | def init_tests(): 26 | """ 27 | Cleanly define some tests. 28 | """ 29 | feditest.all_tests = {} 30 | feditest._registered_as_test = {} 31 | feditest._registered_as_test_step = {} 32 | feditest._loading_tests = True 33 | 34 | ## 35 | ## FediTest tests start here 36 | ## 37 | 38 | @test 39 | def skip() -> None: 40 | """ 41 | Always skips itself. 42 | """ 43 | raise SkipTestException('We skipped this.') 44 | 45 | ## 46 | ## FediTest tests end here 47 | ## (Don't forget the next two lines) 48 | ## 49 | 50 | feditest._loading_tests = False 51 | feditest._load_tests_pass2() 52 | 53 | 54 | @pytest.fixture(autouse=True) 55 | def the_test_plan() -> TestPlan: 56 | """ 57 | The test plan tests all known tests. 58 | """ 59 | 60 | constellation = TestPlanConstellation({}, 'No nodes needed') 61 | tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] 62 | session = TestPlanSessionTemplate(tests, "Test a test that wants to be skipped") 63 | ret = TestPlan(session, [ constellation ]) 64 | ret.properties_validate() 65 | return ret 66 | 67 | 68 | def test_run_testplan(the_test_plan: TestPlan): 69 | the_test_plan.check_can_be_executed() 70 | 71 | test_run = TestRun(the_test_plan) 72 | controller = AutomaticTestRunController(test_run) 73 | test_run.run(controller) 74 | 75 | transcript = test_run.transcribe() 76 | summary = transcript.build_summary() 77 | 78 | assert summary.n_total == 1 79 | assert summary.n_failed == 0 80 | assert summary.n_skipped == 1 81 | assert summary.n_errored == 0 82 | assert summary.n_passed == 0 83 | 84 | assert len(transcript.sessions) == 1 85 | assert len(transcript.sessions[0].run_tests) == 1 86 | assert transcript.sessions[0].run_tests[0].result.type == 'SkipTestException' -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/results.jinja2: -------------------------------------------------------------------------------- 1 |

Constellation

2 |
3 | {%- for role_name, node in run_session.constellation.nodes.items() %} 4 |
5 |

{{ role_name | e }}

6 |
{{ local_name_with_tooltip(node.node_driver) }}
7 |
{{ node.appdata['app'] | e }}
8 |
{{ node.appdata['app_version'] or '?' | e }}
9 | {%- if node.parameters %} 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | {%- for key, value in node.parameters.items() %} 18 | 19 | 20 | 21 | 22 | {%- endfor %} 23 | 24 |
Parameters:
{{ key | e }}{{ value | e }}
25 | {%- endif %} 26 |
27 | {%- endfor %} 28 |
29 | 30 |

Test Results

31 |
32 | {%- for test_index, run_test in enumerate(run_session.run_tests) %} 33 | {%- set plan_test_spec = transcript.plan.session_template.tests[run_test.plan_test_index] %} 34 | {%- set test_meta = transcript.test_meta[plan_test_spec.name] %} 35 |
36 |

Test: {{ test_meta.name | e }}

37 | {%- if test_meta.description %} 38 |
{{ test_meta.description | e }}
39 | {%- endif %} 40 |

Started {{ format_timestamp(run_test.started) }}, ended {{ format_timestamp(run_test.ended) }} (duration: {{ format_duration(run_test.ended - run_test.started) }})

41 | {%- with result=run_test.worst_result %} 42 | {%- include "partials/shared_session/testresult.jinja2" %} 43 | {%- endwith %} 44 | {%- for test_step_index, run_step in enumerate(run_test.run_steps or []) %} 45 |
46 | {% set test_step_meta = test_meta.steps[run_step.plan_step_index] %} 47 |
Test step: {{ test_step_meta.name | e }}
48 | {%- if test_step_meta.description %} 49 |
{{ test_step_meta.description | e }}
50 | {%- endif %} 51 |

Started {{ format_timestamp(run_test.started) }}, ended {{ format_timestamp(run_test.ended) }} (duration: {{ format_duration(run_test.ended - run_test.started) }})

52 | {%- with result=run_step.result, idmod='step' %} 53 | {%- include "partials/shared_session/testresult.jinja2" %} 54 | {%- endwith %} 55 |
56 | {%- endfor %} 57 |
58 | {%- endfor %} 59 |
60 | -------------------------------------------------------------------------------- /tests.unit/test_40_fallback_fediverse_accounts_without_roles_from_testplan.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test that Accounts and NonExistingAccounts are parsed correctly when given in a TestPlan that 3 | specifies a FallbackFediverseNode. This is the test in which the Accounts do not have pre-assigned roles. 4 | """ 5 | 6 | from typing import cast 7 | 8 | import pytest 9 | 10 | import feditest 11 | from feditest.nodedrivers.saas import FediverseSaasNodeDriver 12 | from feditest.protocols.fediverse import ( 13 | USERID_ACCOUNT_FIELD, 14 | USERID_NON_EXISTING_ACCOUNT_FIELD, 15 | FediverseAccount, 16 | FediverseNonExistingAccount 17 | ) 18 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate 19 | 20 | 21 | HOSTNAME = 'localhost' 22 | NODE1_ROLE = 'node1-role' 23 | 24 | 25 | @pytest.fixture(scope="module", autouse=True) 26 | def init(): 27 | """ Clean init """ 28 | feditest.all_tests = {} 29 | feditest._registered_as_test = {} 30 | feditest._registered_as_test_step = {} 31 | feditest._loading_tests = True 32 | 33 | feditest._loading_tests = False 34 | feditest._load_tests_pass2() 35 | 36 | 37 | @pytest.fixture(autouse=True) 38 | def test_plan_fixture() -> TestPlan: 39 | node_driver = FediverseSaasNodeDriver() 40 | parameters = { 41 | 'hostname' : 'example.com', # Avoid interactive question 42 | 'app' : 'test-dummy' # Avoid interactive question 43 | } 44 | plan_accounts = [ 45 | { 46 | USERID_ACCOUNT_FIELD.name : 'foo' 47 | } 48 | ] 49 | plan_non_existing_accounts = [ 50 | { 51 | USERID_NON_EXISTING_ACCOUNT_FIELD.name : 'nonfoo' 52 | } 53 | ] 54 | node1 = TestPlanConstellationNode(node_driver, parameters, plan_accounts, plan_non_existing_accounts) 55 | constellation = TestPlanConstellation({ NODE1_ROLE : node1 }) 56 | session_template = TestPlanSessionTemplate([]) 57 | ret = TestPlan(session_template, [ constellation ]) 58 | ret.properties_validate() 59 | return ret 60 | 61 | 62 | def test_populate(test_plan_fixture: TestPlan) -> None: 63 | """ 64 | Tests parsing the TestPlan 65 | """ 66 | node1 = test_plan_fixture.constellations[0].roles[NODE1_ROLE] 67 | node_driver = node1.nodedriver 68 | 69 | node_config, account_manager = node_driver.create_configuration_account_manager(NODE1_ROLE, node1) 70 | node_driver.provision_node('test', node_config, account_manager) 71 | 72 | acc1 = cast(FediverseAccount | None, account_manager.get_account_by_role('role1')) 73 | 74 | assert acc1 is None 75 | 76 | acc1 = account_manager.obtain_account_by_role('role1') 77 | 78 | assert acc1 79 | assert acc1.role is None 80 | assert acc1.actor_acct_uri == 'acct:foo@example.com' 81 | 82 | non_acc1 = cast(FediverseNonExistingAccount | None, account_manager.get_non_existing_account_by_role('nonrole1')) 83 | 84 | assert non_acc1 is None 85 | 86 | non_acc1 = account_manager.obtain_non_existing_account_by_role('nonrole1') 87 | 88 | assert non_acc1 89 | assert non_acc1.role is None 90 | assert non_acc1.actor_acct_uri == 'acct:nonfoo@example.com' 91 | 92 | -------------------------------------------------------------------------------- /tests.unit/test_30_create_testplan.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test the equivalent of `feditest create-testplan` 3 | """ 4 | 5 | import pytest 6 | 7 | import feditest 8 | from feditest import test 9 | from feditest.nodedrivers import Node 10 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec 11 | 12 | 13 | @pytest.fixture(scope="module", autouse=True) 14 | def init(): 15 | """ Keep these isolated to this module """ 16 | feditest.all_tests = {} 17 | feditest._registered_as_test = {} 18 | feditest._registered_as_test_step = {} 19 | feditest._loading_tests = True 20 | 21 | @test 22 | def test1(role_a: Node) -> None: 23 | return 24 | 25 | @test 26 | def test2(role_a: Node) -> None: 27 | return 28 | 29 | @test 30 | def test3(role_b: Node) -> None: 31 | return 32 | 33 | feditest._loading_tests = False 34 | feditest._load_tests_pass2() 35 | 36 | 37 | @pytest.fixture 38 | def test_specs() -> list[TestPlanTestSpec]: 39 | return [ 40 | TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None 41 | ] 42 | 43 | 44 | @pytest.fixture 45 | def unnamed_constellations() -> list[TestPlanConstellation]: 46 | return [ 47 | TestPlanConstellation({'role_a': None, 'role_b': None}, None), 48 | TestPlanConstellation({'role_a': None, 'role_b': None, 'role_c': None}, None) 49 | ] 50 | 51 | 52 | @pytest.fixture 53 | def unnamed_session_template(test_specs: list[TestPlanTestSpec]) -> TestPlanSessionTemplate: 54 | return TestPlanSessionTemplate(test_specs) 55 | 56 | 57 | def construct_testplan(constellations: list[TestPlanConstellation], session_template: TestPlanSessionTemplate, testplan_name: str) -> TestPlan: 58 | """ 59 | Helper to put it together. 60 | """ 61 | test_plan = TestPlan(session_template, constellations, testplan_name) 62 | test_plan.properties_validate() 63 | test_plan.simplify() 64 | 65 | return test_plan 66 | 67 | 68 | def test_structure(unnamed_constellations: list[TestPlanConstellation], unnamed_session_template: TestPlanSessionTemplate) -> None: 69 | """ 70 | Test the structure of the TestPlan, ignore the naming. 71 | """ 72 | test_plan = construct_testplan(unnamed_constellations, unnamed_session_template, None) 73 | assert test_plan.session_template 74 | assert len(test_plan.constellations) == 2 75 | 76 | 77 | def test_all_unnamed(unnamed_constellations: list[TestPlanConstellation], unnamed_session_template: TestPlanSessionTemplate) -> None: 78 | """ 79 | Only test the naming. 80 | """ 81 | test_plan = construct_testplan(unnamed_constellations, unnamed_session_template, None) 82 | assert test_plan.name is None 83 | assert str(test_plan) == "Unnamed" 84 | 85 | 86 | def test_testplan_named(unnamed_constellations: list[TestPlanConstellation], unnamed_session_template: TestPlanSessionTemplate) -> None: 87 | """ 88 | Only test the naming. 89 | """ 90 | TESTPLAN_NAME = 'My test plan' 91 | test_plan = construct_testplan(unnamed_constellations, unnamed_session_template, TESTPLAN_NAME) 92 | assert test_plan.name == TESTPLAN_NAME 93 | assert str(test_plan) == TESTPLAN_NAME 94 | 95 | -------------------------------------------------------------------------------- /tests.smoke/tests/nodes_with_mastodon_api_communicate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests that two nodes that implement the Mastodon API can follow each other. 3 | """ 4 | 5 | from datetime import datetime 6 | import time 7 | 8 | from feditest import poll_until, step, test 9 | from feditest.nodedrivers.mastodon import NodeWithMastodonAPI 10 | 11 | 12 | @test 13 | class FollowTest: 14 | def __init__(self, 15 | leader_node: NodeWithMastodonAPI, 16 | follower_node: NodeWithMastodonAPI 17 | ) -> None: 18 | self.leader_node = leader_node 19 | self.leader_actor_acct_uri = None 20 | 21 | self.follower_node = follower_node 22 | self.follower_actor_acct_uri = None 23 | 24 | self.leader_note_uri = None 25 | 26 | 27 | @step 28 | def provision_actors(self): 29 | self.leader_actor_acct_uri = self.leader_node.obtain_actor_acct_uri() 30 | assert self.leader_actor_acct_uri 31 | self.leader_node.set_auto_accept_follow(self.leader_actor_acct_uri, True) 32 | 33 | self.follower_actor_acct_uri = self.follower_node.obtain_actor_acct_uri() 34 | assert self.follower_actor_acct_uri 35 | 36 | 37 | # @step 38 | # def start_reset_all(self): 39 | # self._reset_all() 40 | 41 | 42 | @step 43 | def follow(self): 44 | self.follower_node.make_follow(self.follower_actor_acct_uri, self.leader_actor_acct_uri) 45 | 46 | 47 | @step 48 | def wait_until_actor_is_followed_by_actor(self): 49 | time.sleep(1) # Sometimes there seems to be a race condition in Mastodon, or something like that. 50 | # If we proceed too quickly, the API returns 422 "User already exists" or such 51 | # in response to a search, which makes no sense. 52 | poll_until(lambda: self.leader_node.actor_is_followed_by_actor(self.leader_actor_acct_uri, self.follower_actor_acct_uri)) 53 | 54 | 55 | @step 56 | def wait_until_actor_is_following_actor(self): 57 | poll_until(lambda: self.follower_node.actor_is_following_actor(self.follower_actor_acct_uri, self.leader_actor_acct_uri)) 58 | 59 | 60 | @step 61 | def leader_creates_note(self): 62 | self.leader_note_uri = self.leader_node.make_create_note(self.leader_actor_acct_uri, f"testing leader_creates_note {datetime.now()}") 63 | assert self.leader_note_uri 64 | 65 | 66 | @step 67 | def wait_until_note_received(self): 68 | poll_until(lambda: self.follower_node.actor_has_received_object(self.follower_actor_acct_uri, self.leader_note_uri)) 69 | 70 | 71 | # @step 72 | # def end_reset_all(self): 73 | # self._reset_all() 74 | 75 | 76 | def _reset_all(self): 77 | """ 78 | Clean up data. This is here so the test is usable with non-brand-new instances. 79 | """ 80 | self.leader_node.delete_all_followers_of(self.leader_actor_acct_uri) 81 | self.leader_node.delete_all_following_of(self.leader_actor_acct_uri) 82 | self.leader_node.delete_all_statuses_by(self.leader_actor_acct_uri) 83 | 84 | self.follower_node.delete_all_followers_of(self.follower_actor_acct_uri) 85 | self.follower_node.delete_all_following_of(self.follower_actor_acct_uri) 86 | self.follower_node.delete_all_statuses_by(self.follower_actor_acct_uri) 87 | 88 | -------------------------------------------------------------------------------- /tests.unit/test_40_fallback_fediverse_accounts_with_roles_from_testplan.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test that Accounts and NonExistingAccounts are parsed correctly when given in a TestPlan that 3 | specifies a FallbackFediverseNode. This is the test in which the Accounts have pre-assigned roles. 4 | """ 5 | 6 | from typing import cast 7 | 8 | import pytest 9 | 10 | import feditest 11 | from feditest.nodedrivers.saas import FediverseSaasNodeDriver 12 | from feditest.protocols.fediverse import ( 13 | ROLE_ACCOUNT_FIELD, 14 | ROLE_NON_EXISTING_ACCOUNT_FIELD, 15 | USERID_ACCOUNT_FIELD, 16 | USERID_NON_EXISTING_ACCOUNT_FIELD, 17 | FediverseAccount, 18 | FediverseNonExistingAccount 19 | ) 20 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate 21 | 22 | 23 | HOSTNAME = 'localhost' 24 | NODE1_ROLE = 'node1-role' 25 | 26 | 27 | @pytest.fixture(scope="module", autouse=True) 28 | def init(): 29 | """ Clean init """ 30 | feditest.all_tests = {} 31 | feditest._registered_as_test = {} 32 | feditest._registered_as_test_step = {} 33 | feditest._loading_tests = True 34 | 35 | feditest._loading_tests = False 36 | feditest._load_tests_pass2() 37 | 38 | 39 | @pytest.fixture(autouse=True) 40 | def test_plan_fixture() -> TestPlan: 41 | node_driver = FediverseSaasNodeDriver() 42 | parameters = { 43 | 'hostname' : 'example.com', # Avoid interactive question 44 | 'app' : 'test-dummy' # Avoid interactive question 45 | } 46 | plan_accounts = [ 47 | { 48 | ROLE_ACCOUNT_FIELD.name : 'role1', 49 | USERID_ACCOUNT_FIELD.name : 'foo' 50 | } 51 | ] 52 | plan_non_existing_accounts = [ 53 | { 54 | ROLE_NON_EXISTING_ACCOUNT_FIELD.name : 'nonrole1', 55 | USERID_NON_EXISTING_ACCOUNT_FIELD.name : 'nonfoo' 56 | } 57 | ] 58 | node1 = TestPlanConstellationNode(node_driver, parameters, plan_accounts, plan_non_existing_accounts) 59 | constellation = TestPlanConstellation({ NODE1_ROLE : node1 }) 60 | session_template = TestPlanSessionTemplate([]) 61 | ret = TestPlan(session_template, [ constellation ]) 62 | ret.properties_validate() 63 | return ret 64 | 65 | 66 | def test_populate(test_plan_fixture: TestPlan) -> None: 67 | """ 68 | Tests parsing the TestPlan 69 | """ 70 | node1 = test_plan_fixture.constellations[0].roles[NODE1_ROLE] 71 | node_driver = node1.nodedriver 72 | 73 | node_config, account_manager = node_driver.create_configuration_account_manager(NODE1_ROLE, node1) 74 | node_driver.provision_node('test', node_config, account_manager) 75 | 76 | acc1 = cast(FediverseAccount | None, account_manager.get_account_by_role('role1')) 77 | 78 | assert acc1 79 | assert acc1.role == 'role1' 80 | assert acc1.actor_acct_uri == 'acct:foo@example.com' 81 | 82 | acc11 = account_manager.obtain_account_by_role('role1') 83 | assert acc11 == acc1 84 | 85 | non_acc1 = cast(FediverseNonExistingAccount | None, account_manager.get_non_existing_account_by_role('nonrole1')) 86 | assert non_acc1 87 | assert non_acc1.role == 'nonrole1' 88 | assert non_acc1.actor_acct_uri == 'acct:nonfoo@example.com' 89 | 90 | non_acc11 = account_manager.obtain_non_existing_account_by_role('nonrole1') 91 | assert non_acc11 == non_acc1 92 | 93 | -------------------------------------------------------------------------------- /RELEASE-HOWTO.md: -------------------------------------------------------------------------------- 1 | # Release How-To 2 | 3 | ## Smoke test and test with sandbox 4 | 5 | 1. On the Mac: 6 | 1. Repo `feditest`: `git checkout develop` 7 | 1. In `pyproject.toml`, change `project` / `version` to the new version `VERSION` (needed so the generated files have the right version in them before check-in) 8 | 1. Clean rebuild: 9 | 1. `rm -rf venv.*` 10 | 1. `make venv` 11 | 1. `make lint`: ruff and mypy show no errors 12 | 1. `make tests`: unit tests show no errors (smoke tests don't run on macOS) 13 | 1. Repo `feditest-tests-sandbox`: `git checkout develop` 14 | 1. Clean re-run and report generation of the sandbox tests: 15 | 1. `make -f Makefile.create clean FEDITEST=../feditest/venv.darwin.default/bin/feditest` 16 | 1. `make -f Makefile.run clean FEDITEST=../feditest/venv.darwin.default/bin/feditest` 17 | 1. `make -f Makefile.create examples FEDITEST=../feditest/venv.darwin.default/bin/feditest` 18 | 1. `make -f Makefile.run sandbox FEDITEST=../feditest/venv.darwin.default/bin/feditest` 19 | 1. `open examples/testresults/*.html` and check for plausibility of reports 20 | 21 | ## Smoke test and test quickstart 22 | 23 | 1. On UBOS: 24 | 1. Repo `feditest`: `git checkout develop` 25 | 1. Clean rebuild: 26 | 1. `rm -rf venv.*` 27 | 1. `make venv` 28 | 1. `make lint`: ruff and mypy show no errors 29 | 1. `make tests`: unit tests and smoke tests show no errors (other than WordPress timeout-related) 30 | 1. Repo `feditest-tests-fediverse`: `git checkout develop` 31 | 1. Clean re-run and report generation of the quickstart tests: 32 | 1. Run quickstart examples from https://feditest.org/quickstart/evaluate/, but instead of `feditest` use `../feditest/venv.linux.default/bin/feditest`. 33 | 1. `xdg-open results/*.html` and check for plausibility of reports 34 | 35 | ## Tag versions 36 | 37 | 1. On the Mac: 38 | 1. Update repo `feditest-tests-fediverse`, branch `develop`: 39 | 1. `git tag -a vVERSION -m vVERSION` 40 | 1. `git push` 41 | 1. `git push --tags` 42 | 1. Update repo `feditest-tests-sandbox`, branch `develop`: 43 | 1. `git tag -a vVERSION -m vVERSION` 44 | 1. `git push` 45 | 1. `git push --tags` 46 | 1. Update repo `feditest`, branch `develop`: 47 | 1. `git tag -a vVERSION -m vVERSION` 48 | 1. `git push` 49 | 1. `git push --tags` 50 | 51 | ## Merge into main 52 | 53 | 1. Repo `feditest-tests-fediverse`: pull request `develop` into `main` 54 | 1. Repo `feditest-tests-sandbox`: pull request `develop` into `main` 55 | 1. Repo `feditest`: pull request `develop` into `main` 56 | 1. Approve all three pull requests 57 | 58 | ## Publish to PyPi 59 | 60 | 1. On the Mac: 61 | 1. Repo `feditest`, branch `main` 62 | 1. `make release` 63 | 1. `venv.release/bin/twine upload dist/*` 64 | 1. `pip3.11 install --upgrade feditest` 65 | 1. `feditest version` now shows `VERSION` 66 | 67 | ## Publish to UBOS repos 68 | 69 | 1. On UBOS: 70 | 1. Build `feditest` for the UBOS package repos so it can be installed with `pacman -S feditest` 71 | 72 | ## Create release notes 73 | 74 | 1. Repo: `feditest.org`: create release notes 75 | 1. `git push` 76 | 77 | ## Announce 78 | 79 | 1. `https://matrix.to/#/#fediverse-testing:matrix.org`: post link to release notes 80 | 1. `@feditest@mastodon.social`: post link to release notes 81 | -------------------------------------------------------------------------------- /tests.unit/test_40_report_node_driver_errors.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test that NodeDriver errors are reported in the test reports 3 | """ 4 | 5 | import feditest 6 | import pytest 7 | from feditest import nodedriver 8 | from feditest.nodedrivers import AccountManager, Node, NodeConfiguration, NodeDriver 9 | from feditest.testplan import ( 10 | TestPlan, 11 | TestPlanConstellation, 12 | TestPlanConstellationNode, 13 | TestPlanSessionTemplate, 14 | TestPlanTestSpec, 15 | ) 16 | from feditest.testrun import TestRun 17 | from feditest.testruncontroller import AutomaticTestRunController 18 | from feditest.testruntranscriptserializer.json import JsonTestRunTranscriptSerializer 19 | from feditest.testruntranscriptserializer.summary import SummaryTestRunTranscriptSerializer 20 | from feditest.testruntranscriptserializer.tap import TapTestRunTranscriptSerializer 21 | 22 | 23 | class NodeDriverTestException(Exception): 24 | pass 25 | 26 | 27 | @pytest.fixture(scope="module", autouse=True) 28 | def init(): 29 | global node_driver_name 30 | 31 | """ Keep these isolated to this module """ 32 | feditest.all_tests = {} 33 | feditest._registered_as_test = {} 34 | feditest._registered_as_test_step = {} 35 | feditest._loading_tests = True 36 | 37 | @feditest.test 38 | def dummy() -> None: 39 | return 40 | 41 | feditest._loading_tests = False 42 | feditest._load_tests_pass2() 43 | 44 | """ The NodeDriver we use for testing """ 45 | feditest.all_node_drivers = {} 46 | feditest._loading_node_drivers = True 47 | 48 | @nodedriver 49 | class Faulty_NodeDriver(NodeDriver): 50 | def _provision_node( 51 | self, 52 | rolename: str, 53 | config: NodeConfiguration, 54 | account_manager: AccountManager | None 55 | ) -> Node: 56 | raise NodeDriverTestException() 57 | 58 | feditest._loading_node_drivers = False 59 | 60 | for t in feditest.all_tests: 61 | print( f'TEST: { t }') 62 | 63 | 64 | def test_faulty_node_driver_reporting() -> None: 65 | plan = TestPlan( 66 | TestPlanSessionTemplate( 67 | [ 68 | TestPlanTestSpec('test_40_report_node_driver_errors::init..dummy') 69 | ] 70 | ), 71 | [ 72 | TestPlanConstellation( { 73 | 'node' : TestPlanConstellationNode( 74 | nodedriver = 'test_40_report_node_driver_errors.init..Faulty_NodeDriver', 75 | parameters = { 'app' : 'Dummy for test_faulty_node_driver_reporting'} 76 | ) 77 | }), 78 | ] 79 | ) 80 | plan.properties_validate() 81 | run = TestRun(plan) 82 | controller = AutomaticTestRunController(run) 83 | 84 | run.run(controller) 85 | 86 | transcript : feditest.testruntranscript.TestRunTranscript = run.transcribe() 87 | # transcript.save('transcript.json') 88 | 89 | summary = SummaryTestRunTranscriptSerializer().write_to_string(transcript) 90 | # print(summary) 91 | assert 'errors=1' in summary 92 | 93 | tap = TapTestRunTranscriptSerializer().write_to_string(transcript) 94 | # print(tap) 95 | assert 'errors: 1' in tap 96 | 97 | j = JsonTestRunTranscriptSerializer().write_to_string(transcript) 98 | # print(j) 99 | assert f'"type": "{NodeDriverTestException.__name__}"' in j 100 | -------------------------------------------------------------------------------- /tests.unit/test_50_run_not_implemented.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run a test that throws a NotImplemented error. 3 | """ 4 | 5 | import pytest 6 | 7 | import feditest 8 | from feditest.nodedrivers import AccountManager, Node, NodeConfiguration, NodeDriver, NotImplementedByNodeError 9 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec 10 | from feditest.testrun import TestRun 11 | from feditest.testruncontroller import AutomaticTestRunController 12 | from feditest import test 13 | 14 | 15 | class DummyNode(Node): 16 | def missing_method(self): 17 | pass 18 | 19 | 20 | class DummyNodeDriver(NodeDriver): 21 | def _provision_node(self, rolename: str, config: NodeConfiguration, account_manager: AccountManager | None) -> Node: 22 | return DummyNode(rolename, config, account_manager) 23 | 24 | 25 | def missing_method(self): 26 | pass 27 | 28 | 29 | @pytest.fixture(scope="module", autouse=True) 30 | def init_node_drivers(): 31 | """ 32 | Cleanly define the NodeDrivers. 33 | """ 34 | feditest.all_node_drivers = {} 35 | feditest.load_default_node_drivers() 36 | 37 | 38 | @pytest.fixture(scope="module", autouse=True) 39 | def init_tests(): 40 | """ 41 | Cleanly define some tests. 42 | """ 43 | feditest.all_tests = {} 44 | feditest._registered_as_test = {} 45 | feditest._registered_as_test_step = {} 46 | feditest._loading_tests = True 47 | 48 | ## 49 | ## FediTest tests start here 50 | ## 51 | 52 | @test 53 | def not_implemented_by_node_error() -> None: 54 | """ 55 | A Node does not implement a method. 56 | """ 57 | driver = DummyNodeDriver() 58 | node = driver.provision_node('testrole', NodeConfiguration(driver, 'dummy')) 59 | 60 | raise NotImplementedByNodeError(node, DummyNode.missing_method) 61 | 62 | ## 63 | ## FediTest tests end here 64 | ## (Don't forget the next two lines) 65 | ## 66 | 67 | feditest._loading_tests = False 68 | feditest._load_tests_pass2() 69 | 70 | 71 | @pytest.fixture(autouse=True) 72 | def test_plan_fixture() -> TestPlan: 73 | """ 74 | The test plan tests all known tests. 75 | """ 76 | 77 | constellation = TestPlanConstellation({}, 'No nodes needed') 78 | tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] 79 | session = TestPlanSessionTemplate(tests, "Test tests that throw NotImplemented errors") 80 | ret = TestPlan(session, [ constellation ]) 81 | ret.properties_validate() 82 | return ret 83 | 84 | 85 | def test_run_testplan(test_plan_fixture: TestPlan): 86 | test_plan_fixture.check_can_be_executed() 87 | 88 | test_run = TestRun(test_plan_fixture) 89 | controller = AutomaticTestRunController(test_run) 90 | test_run.run(controller) 91 | 92 | transcript = test_run.transcribe() 93 | summary = transcript.build_summary() 94 | 95 | assert summary.n_total == 1 96 | assert summary.n_failed == 0 97 | assert summary.n_skipped == 1 # NotImplemented exceptions cause skips 98 | assert summary.n_errored == 0 99 | assert summary.n_passed == 0 100 | 101 | assert len(transcript.sessions) == 1 102 | assert len(transcript.sessions[0].run_tests) == 1 103 | assert transcript.sessions[0].run_tests[0].result.type == 'NotImplementedByNodeError' -------------------------------------------------------------------------------- /src/feditest/cli/commands/convert_transcript.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert a TestRunTranscript to a different format 3 | """ 4 | 5 | from argparse import ArgumentParser, Namespace, _SubParsersAction 6 | 7 | from feditest.reporting import warning 8 | from feditest.testruntranscript import TestRunTranscript 9 | from feditest.testruntranscriptserializer.json import JsonTestRunTranscriptSerializer 10 | from feditest.testruntranscriptserializer.html import HtmlRunTranscriptSerializer 11 | from feditest.testruntranscriptserializer.summary import SummaryTestRunTranscriptSerializer 12 | from feditest.testruntranscriptserializer.tap import TapTestRunTranscriptSerializer 13 | from feditest.utils import FEDITEST_VERSION 14 | 15 | def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: 16 | """ 17 | Run this command. 18 | """ 19 | 20 | transcript = TestRunTranscript.load(args.in_file) 21 | if not transcript.is_compatible_type(): 22 | warning(f'Transcript has unexpected type { transcript.type }: incompatibilities may occur.') 23 | 24 | if not transcript.has_compatible_version(): 25 | warning(f'Transcript was created by FediTest { transcript.feditest_version }, you are running FediTest { FEDITEST_VERSION }: incompatibilities may occur.') 26 | 27 | if isinstance(args.html, str): 28 | HtmlRunTranscriptSerializer(args.template_path).write(transcript, args.html) 29 | elif args.html: 30 | warning('--html requires a filename: skipping') 31 | elif args.template_path: 32 | warning('--template-path only supported with --html. Ignoring.') 33 | 34 | if isinstance(args.tap, str) or args.tap: 35 | TapTestRunTranscriptSerializer().write(transcript, args.tap) 36 | 37 | if isinstance(args.json, str) or args.json: 38 | JsonTestRunTranscriptSerializer().write(transcript, args.json) 39 | 40 | if isinstance(args.summary, str) or args.summary: 41 | SummaryTestRunTranscriptSerializer().write(transcript, args.summary) 42 | 43 | return 0 44 | 45 | 46 | def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> ArgumentParser: 47 | """ 48 | Add command-line options for this sub-command 49 | parent_parser: the parent argparse parser 50 | cmd_name: name of this command 51 | """ 52 | parser = parent_parser.add_parser(cmd_name, help='Convert a transcript of a TestRun to a different format') 53 | parser.add_argument('--in', required=True, dest="in_file", help='JSON file containing the transcript') 54 | parser.add_argument('--tap', nargs="?", const=True, default=False, 55 | help="Write results in TAP format to stdout, or to the provided file (if given).") 56 | html_group = parser.add_argument_group('html', 'HTML options') 57 | html_group.add_argument('--html', 58 | help="Write results in HTML format to the provided file.") 59 | html_group.add_argument('--template-path', required=False, 60 | help="When specifying --html, use this template path override (comma separated directory names)") 61 | parser.add_argument('--json', nargs="?", const=True, default=False, 62 | help="Write results in JSON format to stdout, or to the provided file (if given).") 63 | parser.add_argument('--summary', nargs="?", const=True, default=False, 64 | help="Write summary to stdout, or to the provided file (if given). This is the default if no other output option is given") 65 | 66 | return parser 67 | -------------------------------------------------------------------------------- /src/feditest/protocols/webfinger/abstract.py: -------------------------------------------------------------------------------- 1 | """ 2 | Functionality that may be shared by several WebFinger Node implementations. 3 | """ 4 | 5 | from typing import cast 6 | 7 | from feditest.protocols.web.diag import HttpRequest, HttpRequestResponsePair, WebDiagClient 8 | from feditest.protocols.webfinger import WebFingerServer 9 | from feditest.protocols.webfinger.diag import ClaimedJrd, WebFingerDiagClient 10 | from feditest.protocols.webfinger.utils import construct_webfinger_uri_for, WebFingerQueryDiagResponse 11 | from feditest.utils import ParsedUri 12 | 13 | 14 | class AbstractWebFingerDiagClient(WebFingerDiagClient): 15 | # Python 3.12 @override 16 | def diag_perform_webfinger_query( 17 | self, 18 | resource_uri: str, 19 | rels: list[str] | None = None, 20 | server: WebFingerServer | None = None 21 | ) -> WebFingerQueryDiagResponse: 22 | 23 | query_url = construct_webfinger_uri_for(resource_uri, rels, server.hostname() if server else None ) 24 | parsed_uri = ParsedUri.parse(query_url) 25 | if not parsed_uri: 26 | raise ValueError('Not a valid URI:', query_url) # can't avoid this 27 | 28 | first_request = HttpRequest(parsed_uri) 29 | current_request = first_request 30 | pair : HttpRequestResponsePair | None = None 31 | for redirect_count in range(10, 0, -1): 32 | pair = self.http(current_request) 33 | if pair.response and pair.response.is_redirect(): 34 | if redirect_count <= 0: 35 | return WebFingerQueryDiagResponse(pair, None, [ WebDiagClient.TooManyRedirectsError(current_request) ]) 36 | parsed_location_uri = ParsedUri.parse(pair.response.location()) 37 | if not parsed_location_uri: 38 | return WebFingerQueryDiagResponse(pair, None, [ ValueError('Location header is not a valid URI:', query_url, '(from', resource_uri, ')') ] ) 39 | current_request = HttpRequest(parsed_location_uri) 40 | break 41 | 42 | # I guess we always have a non-null responses here, but mypy complains without the cast 43 | pair = cast(HttpRequestResponsePair, pair) 44 | ret_pair = HttpRequestResponsePair(first_request, current_request, pair.response) 45 | if ret_pair.response is None: 46 | raise RuntimeError('Unexpected None HTTP response') 47 | 48 | excs : list[Exception] = [] 49 | if ret_pair.response.http_status != 200: 50 | excs.append(WebFingerDiagClient.WrongHttpStatusError(ret_pair)) 51 | 52 | content_type = ret_pair.response.content_type() 53 | if (content_type is None or (content_type != "application/jrd+json" 54 | and not content_type.startswith( "application/jrd+json;" )) 55 | ): 56 | excs.append(WebFingerDiagClient.WrongContentTypeError(ret_pair)) 57 | 58 | jrd : ClaimedJrd | None = None 59 | 60 | if ret_pair.response.payload is None: 61 | raise RuntimeError('Unexpected None payload in HTTP response') 62 | 63 | try: 64 | json_string = ret_pair.response.payload.decode(encoding=ret_pair.response.payload_charset() or "utf8") 65 | 66 | jrd = ClaimedJrd(json_string) # May throw JSONDecodeError 67 | jrd.validate() # May throw JrdError 68 | except ExceptionGroup as exc: 69 | excs += exc.exceptions 70 | except Exception as exc: 71 | excs.append(exc) 72 | 73 | return WebFingerQueryDiagResponse(ret_pair, jrd, excs) 74 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/tap.py: -------------------------------------------------------------------------------- 1 | from typing import IO 2 | 3 | from feditest.testruntranscript import TestRunTranscript 4 | from feditest.testruntranscriptserializer import FileOrStdoutTestRunTranscriptSerializer 5 | 6 | 7 | class TapTestRunTranscriptSerializer(FileOrStdoutTestRunTranscriptSerializer): 8 | """ 9 | Knows how to serialize a TestRunTranscript into a report in TAP format. 10 | """ 11 | def _write(self, transcript: TestRunTranscript, fd: IO[str]): 12 | plan = transcript.plan 13 | summary = transcript.build_summary() 14 | 15 | fd.write("TAP version 14\n") 16 | fd.write(f"# test plan: { plan }\n") 17 | for key in ['started', 'ended', 'platform', 'username', 'hostname']: 18 | value = getattr(transcript, key) 19 | if value: 20 | fd.write(f"# {key}: {value}\n") 21 | 22 | test_id = 0 23 | for session_transcript in transcript.sessions: 24 | plan_session_template = plan.session_template 25 | constellation = session_transcript.constellation 26 | 27 | fd.write(f"# session: { session_transcript }\n") 28 | fd.write(f"# constellation: { constellation }\n") 29 | fd.write("# roles:\n") 30 | for role_name, node in constellation.nodes.items(): 31 | if role_name in session_transcript.constellation.nodes: 32 | transcript_role = session_transcript.constellation.nodes[role_name] 33 | fd.write(f"# - name: {role_name}\n") 34 | if node: 35 | fd.write(f"# driver: {node.node_driver}\n") 36 | fd.write(f"# app: {transcript_role.appdata['app']}\n") 37 | fd.write(f"# app_version: {transcript_role.appdata['app_version'] or '?'}\n") 38 | else: 39 | fd.write(f"# - name: {role_name} -- not instantiated\n") 40 | 41 | for test_index, run_test in enumerate(session_transcript.run_tests): 42 | test_id += 1 43 | 44 | plan_test_spec = plan_session_template.tests[run_test.plan_test_index] 45 | test_meta = transcript.test_meta[plan_test_spec.name] 46 | 47 | result = run_test.worst_result 48 | if result: 49 | fd.write(f"not ok {test_id} - {test_meta.name}\n") 50 | fd.write(" ---\n") 51 | fd.write(f" problem: {result.type} ({ result.spec_level }, { result.interop_level })\n") 52 | if result.msg: 53 | fd.write(" message:\n") 54 | fd.write("\n".join( [ f" { p }" for p in result.msg.strip().split("\n") ] ) + "\n") 55 | fd.write(" where:\n") 56 | for loc in result.stacktrace: 57 | fd.write(f" {loc[0]} {loc[1]}\n") 58 | fd.write(" ...\n") 59 | else: 60 | directives = "" # FIXME f" # SKIP {test.skip}" if test.skip else "" 61 | fd.write(f"ok {test_id} - {test_meta.name}{directives}\n") 62 | 63 | fd.write(f"1..{test_id}\n") 64 | fd.write("# test run summary:\n") 65 | fd.write(f"# total: {summary.n_total}\n") 66 | fd.write(f"# passed: {summary.n_passed}\n") 67 | fd.write(f"# failed: {summary.n_failed}\n") 68 | fd.write(f"# skipped: {summary.n_skipped}\n") 69 | fd.write(f"# errors: {summary.n_errored}\n") 70 | -------------------------------------------------------------------------------- /src/feditest/protocols/webfinger/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Abstractions for the WebFinger protocol 3 | """ 4 | 5 | from feditest.nodedrivers import NotImplementedByNodeError 6 | from feditest.protocols.web import WebClient, WebServer 7 | 8 | 9 | class WebFingerClient(WebClient): 10 | """ 11 | A Node that acts as a WebFinger client. 12 | """ 13 | def perform_webfinger_query(self, resource_uri: str) -> None: 14 | """ 15 | Make this Node perform a WebFinger query for the provided resource_uri. 16 | The resource_uri must be a valid, absolute URI, such as 'acct:foo@bar.com` or 17 | 'https://example.com/aabc' (not escaped). 18 | This returns None as it is unreasonable to assume that a non-diag Node can implement 19 | this call otherwise. However, it may throw exceptions. 20 | It is used with a WebFingerDiagServer to determine whether this WebFingerClient performs 21 | valid WebFinger queries. 22 | """ 23 | raise NotImplementedByNodeError(self, WebFingerClient.perform_webfinger_query) 24 | 25 | 26 | class WebFingerServer(WebServer): 27 | """ 28 | A Node that acts as a WebFinger server. 29 | 30 | The implementation code in this class is here entirely for fallback purposes. Given this, 31 | we are not trying to manage the collection behind the smart factory methods. 32 | """ 33 | def obtain_account_identifier(self, rolename: str | None = None) -> str: 34 | """ 35 | Smart factory method to return the identifier to an account on this Node that 36 | a client is supposed to be able to perform WebFinger resolution on. Different 37 | rolenames produce different results; the same rolename produces the same result. 38 | The identifier is of the form ``acct:foo@bar.com``. 39 | rolename: refer to this account by this rolename; used to disambiguate multiple 40 | accounts on the same server by how they are used in tests 41 | return: the identifier 42 | """ 43 | raise NotImplementedByNodeError(self, WebFingerServer.obtain_account_identifier) 44 | 45 | 46 | def obtain_non_existing_account_identifier(self, rolename: str | None = None ) -> str: 47 | """ 48 | Smart factory method to return the identifier of an account that does not exist on this Node, 49 | but that nevertheless follows the rules for identifiers of this Node. Different rolenames 50 | produce different results; the same rolename produces the same result. 51 | The identifier is of the form ``acct:foo@bar.com``. 52 | rolename: refer to this account by this rolename; used to disambiguate multiple 53 | accounts on the same server by how they are used in tests 54 | return: the identifier 55 | """ 56 | raise NotImplementedByNodeError(self, WebFingerServer.obtain_non_existing_account_identifier) 57 | 58 | 59 | def obtain_account_identifier_requiring_percent_encoding(self, rolename: str | None = None) -> str: 60 | """ 61 | Smart factory method to return the identifier of an existing or newly created account on this 62 | Node that contains characters that require percent-encoding when provided as resource in a WebFinger 63 | query. Different rolenames produce different results; the same rolename produces the same result. 64 | 65 | If the Node does not ever issue such identifiers, raise NotImplementedByNodeException 66 | """ 67 | raise NotImplementedByNodeError(self, WebFingerServer.obtain_account_identifier_requiring_percent_encoding) 68 | -------------------------------------------------------------------------------- /src/feditest/nodedrivers/imp/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | An in-process Node implementation for now. 3 | """ 4 | 5 | import httpx 6 | from multidict import MultiDict 7 | 8 | from feditest.nodedrivers import AccountManager, Node, NodeConfiguration, NodeDriver, HOSTNAME_PAR 9 | from feditest.protocols.web.diag import ( 10 | HttpRequest, 11 | HttpRequestResponsePair, 12 | HttpResponse, 13 | WebDiagClient 14 | ) 15 | from feditest.protocols.webfinger.abstract import AbstractWebFingerDiagClient 16 | from feditest.reporting import trace 17 | from feditest.testplan import TestPlanConstellationNode, TestPlanNodeParameter 18 | from feditest.utils import FEDITEST_VERSION 19 | 20 | _HEADERS = { 21 | "User-Agent": f"feditest/{ FEDITEST_VERSION }", 22 | "Origin": "https://test.example" # to trigger CORS headers in response 23 | } 24 | 25 | class Imp(AbstractWebFingerDiagClient): 26 | """ 27 | In-process diagnostic WebFinger client. 28 | """ 29 | # Python 3.12 @override 30 | def http(self, request: HttpRequest, follow_redirects: bool = True, verify=False) -> HttpRequestResponsePair: 31 | trace( f'Performing HTTP { request.method } on { request.parsed_uri.uri }') 32 | 33 | httpx_response = None 34 | # Do not follow redirects automatically, we need to know whether there are any 35 | with httpx.Client(verify=verify, follow_redirects=follow_redirects) as httpx_client: 36 | httpx_request = httpx.Request(request.method, request.parsed_uri.uri, headers=_HEADERS) # FIXME more arguments 37 | httpx_response = httpx_client.send(httpx_request) 38 | 39 | # FIXME: catch Tls exception and raise WebDiagClient.TlsError 40 | 41 | if httpx_response: 42 | response_headers : MultiDict = MultiDict() 43 | for key, value in httpx_response.headers.items(): 44 | response_headers.add(key.lower(), value) 45 | ret = HttpRequestResponsePair(request, request, HttpResponse(httpx_response.status_code, response_headers, httpx_response.read())) 46 | trace( f'HTTP query returns { ret }') 47 | return ret 48 | raise WebDiagClient.HttpUnsuccessfulError(request) 49 | 50 | 51 | # Python 3.12 @override 52 | def add_cert_to_trust_store(self, root_cert: str) -> None: 53 | """ 54 | On the Imp, we don't do this (for now?) 55 | """ 56 | return 57 | 58 | 59 | # Python 3.12 @override 60 | def remove_cert_from_trust_store(self, root_cert: str) -> None: 61 | return 62 | 63 | 64 | class ImpInProcessNodeDriver(NodeDriver): 65 | """ 66 | Knows how to instantiate an Imp. 67 | """ 68 | # Python 3.12 @override 69 | @staticmethod 70 | def test_plan_node_parameters() -> list[TestPlanNodeParameter]: 71 | return [] 72 | 73 | 74 | # Python 3.12 @override 75 | def create_configuration_account_manager(self, rolename: str, test_plan_node: TestPlanConstellationNode) -> tuple[NodeConfiguration, AccountManager | None]: 76 | return ( 77 | NodeConfiguration( 78 | self, 79 | 'Imp', 80 | FEDITEST_VERSION, 81 | test_plan_node.parameter(HOSTNAME_PAR) 82 | ), 83 | None 84 | ) 85 | 86 | 87 | # Python 3.12 @override 88 | def _provision_node(self, rolename: str, config: NodeConfiguration, account_manager: AccountManager | None) -> Imp: 89 | return Imp(rolename, config, account_manager) 90 | 91 | 92 | # Python 3.12 @override 93 | def _unprovision_node(self, node: Node) -> None: 94 | pass 95 | -------------------------------------------------------------------------------- /tests.unit/test_50_run_exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run a tests that have errors (not test failures), i.e. tests that are buggy. 3 | """ 4 | 5 | import pytest 6 | 7 | import feditest 8 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec 9 | from feditest.testrun import TestRun 10 | from feditest.testruncontroller import AutomaticTestRunController 11 | from feditest import test 12 | 13 | 14 | @pytest.fixture(scope="module", autouse=True) 15 | def init_node_drivers(): 16 | """ 17 | Cleanly define the NodeDrivers. 18 | """ 19 | feditest.all_node_drivers = {} 20 | feditest.load_default_node_drivers() 21 | 22 | 23 | @pytest.fixture(scope="module", autouse=True) 24 | def init_tests(): 25 | """ 26 | Cleanly define some tests. 27 | """ 28 | feditest.all_tests = {} 29 | feditest._registered_as_test = {} 30 | feditest._registered_as_test_step = {} 31 | feditest._loading_tests = True 32 | 33 | ## 34 | ## FediTest tests start here 35 | ## 36 | 37 | @test 38 | def assertion_error() -> None: 39 | """ 40 | This test fails a standard Python assertion. 41 | """ 42 | assert False 43 | 44 | 45 | @test 46 | def attribute_error() -> None: 47 | """ 48 | This test always raises an AttributeError. 49 | """ 50 | a = None 51 | return a.b 52 | 53 | 54 | @test 55 | def type_error() -> None: 56 | """ 57 | This test always raises a TypeError. 58 | """ 59 | a = None 60 | return a + 2 61 | 62 | 63 | @test 64 | def value_error() -> None: 65 | """ 66 | This test always raises a ValueError. 67 | """ 68 | raise ValueError('This test raises a ValueError.') 69 | 70 | 71 | ## 72 | ## FediTest tests end here 73 | ## (Don't forget the next two lines) 74 | ### 75 | 76 | feditest._loading_tests = False 77 | feditest._load_tests_pass2() 78 | 79 | 80 | @pytest.fixture(autouse=True) 81 | def test_plan_fixture() -> TestPlan: 82 | """ 83 | The test plan tests all known tests. 84 | """ 85 | constellation = TestPlanConstellation({}, 'No nodes needed') 86 | tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] 87 | session = TestPlanSessionTemplate(tests, "Tests buggy tests") 88 | ret = TestPlan(session, [ constellation ]) 89 | ret.properties_validate() 90 | return ret 91 | 92 | 93 | def test_run_testplan(test_plan_fixture: TestPlan): 94 | test_plan_fixture.check_can_be_executed() 95 | 96 | test_run = TestRun(test_plan_fixture) 97 | controller = AutomaticTestRunController(test_run) 98 | test_run.run(controller) 99 | 100 | transcript = test_run.transcribe() 101 | summary = transcript.build_summary() 102 | 103 | assert summary.n_total == 4 104 | assert summary.n_failed == 0 105 | assert summary.n_skipped == 0 106 | assert summary.n_errored == 4 107 | assert summary.n_passed == 0 108 | 109 | assert len(transcript.sessions) == 1 110 | assert len(transcript.sessions[0].run_tests) == 4 111 | assert transcript.sessions[0].run_tests[0].result.type == 'AssertionError' 112 | assert transcript.sessions[0].run_tests[1].result.type == 'AttributeError' 113 | assert transcript.sessions[0].run_tests[2].result.type == 'TypeError' 114 | assert transcript.sessions[0].run_tests[3].result.type == 'ValueError' 115 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # make venv 3 | # Create a python environment for your platform and install the required dependencies int 4 | # it. It will be in ./venv.linux.default if you are Linux 5 | # 6 | # make build 7 | # run pip install in your venv 8 | # 9 | # make lint 10 | # Run several linters on the code 11 | # 12 | # make test 13 | # Run unit tests 14 | # 15 | # NOTE: This does not add the venv to your $PATH. You have to do that yourself if you want that. 16 | # 17 | 18 | UNAME?=$(shell uname -s | tr [A-Z] [a-z]) 19 | TAG?=default 20 | VENV?=venv.$(UNAME).$(TAG) 21 | PYTHON?=python3.11 22 | FEDITEST?=$(VENV)/bin/feditest -v 23 | DOMAIN?=--domain 1234.lan 24 | 25 | 26 | default : lint 27 | 28 | all : lint tests 29 | 30 | build : venv 31 | $(VENV)/bin/pip install . 32 | 33 | venv : $(VENV) 34 | 35 | $(VENV) : 36 | @which $(PYTHON) || ( echo 'No executable called "python". Append your python to the make command, like "make PYTHON=your-python"' && false ) 37 | $(PYTHON) -mvenv $(VENV) 38 | $(VENV)/bin/pip install ruff mypy pylint 39 | 40 | lint : build 41 | $(VENV)/bin/ruff check 42 | MYPYPATH=src $(VENV)/bin/mypy --namespace-packages --explicit-package-bases --install-types --non-interactive src 43 | @# These options should be the same flags as in .pre-commit-config.yml, except that I can't get it to 44 | @# work there without the "--ignore-missing-imports" flag, while it does work without it here 45 | 46 | @# MYPYPATH is needed because apparently some type checking ignores the directory option given as command-line argument 47 | @# $(VENV)/bin/pylint src 48 | 49 | tests : tests.unit tests.smoke 50 | 51 | tests.unit : 52 | $(VENV)/bin/pytest -v 53 | 54 | tests.smoke : tests.smoke.webfinger tests.smoke.api tests.smoke.fediverse 55 | 56 | tests.smoke.webfinger : 57 | $(FEDITEST) run --testsdir tests.smoke/tests --session tests.smoke/webfinger.session.json --node client=tests.smoke/imp.node.json --node server=tests.smoke/wordpress.ubos.node.json $(DOMAIN) 58 | $(FEDITEST) run --testsdir tests.smoke/tests --session tests.smoke/webfinger.session.json --node client=tests.smoke/imp.node.json --node server=tests.smoke/mastodon.ubos.node.json $(DOMAIN) 59 | 60 | tests.smoke.api : 61 | $(FEDITEST) run --testsdir tests.smoke/tests --session tests.smoke/mastodon_api.session.json --node server=tests.smoke/wordpress.ubos.node.json $(DOMAIN) 62 | $(FEDITEST) run --testsdir tests.smoke/tests --session tests.smoke/mastodon_api.session.json --node server=tests.smoke/mastodon.ubos.node.json $(DOMAIN) 63 | 64 | tests.smoke.fediverse : 65 | $(FEDITEST) run --testsdir tests.smoke/tests --session tests.smoke/mastodon_api_mastodon_api.session.json --node leader_node=tests.smoke/mastodon.ubos.node.json --node follower_node=tests.smoke/mastodon.ubos.node.json $(DOMAIN) 66 | $(FEDITEST) run --testsdir tests.smoke/tests --session tests.smoke/mastodon_api_mastodon_api.session.json --node leader_node=tests.smoke/wordpress.ubos.node.json --node follower_node=tests.smoke/mastodon.ubos.node.json $(DOMAIN) 67 | 68 | release : 69 | @which $(PYTHON) || ( echo 'No executable called "python". Append your python to the make command, like "make PYTHON=your-python"' && false ) 70 | [[ -d venv.release ]] && rm -rf venv.release || true 71 | [[ -d dist ]] && rm -rf dist || true 72 | $(PYTHON) -mvenv venv.release 73 | FEDITEST_RELEASE_VERSION=y venv.release/bin/pip install twine 74 | FEDITEST_RELEASE_VERSION=y venv.release/bin/pip install --upgrade build 75 | FEDITEST_RELEASE_VERSION=y venv.release/bin/python -m build 76 | @echo WARNING: YOU ARE NOT DONE YET 77 | @echo The actual push to pypi.org you need to do manually. Enter: 78 | @echo venv.release/bin/twine upload dist/* 79 | 80 | .PHONY: all default venv build lint tests tests.unit tests.smoke tests.smoke.webfinger tests.smoke.api tests.smoke.fediverse release 81 | -------------------------------------------------------------------------------- /tests.unit/test_40_ubos_mastodon_accounts_from_testplan.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test that Accounts and NonExistingAccounts are parsed correctly when given in a TestPlan that 3 | specifies a MastodonUbosNodeDriver 4 | """ 5 | 6 | from typing import cast 7 | 8 | import pytest 9 | 10 | import feditest 11 | from feditest.nodedrivers.mastodon import ( 12 | MastodonAccount, 13 | MastodonOAuthTokenAccount, 14 | MastodonUserPasswordAccount, 15 | EMAIL_ACCOUNT_FIELD, 16 | PASSWORD_ACCOUNT_FIELD, 17 | OAUTH_TOKEN_ACCOUNT_FIELD, 18 | ROLE_ACCOUNT_FIELD, 19 | ROLE_NON_EXISTING_ACCOUNT_FIELD, 20 | USERID_ACCOUNT_FIELD, 21 | USERID_NON_EXISTING_ACCOUNT_FIELD 22 | ) 23 | from feditest.nodedrivers.mastodon.ubos import MastodonUbosNodeDriver 24 | from feditest.protocols.fediverse import FediverseNonExistingAccount 25 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate 26 | 27 | 28 | HOSTNAME = 'localhost' 29 | NODE1_ROLE = 'node1-role' 30 | 31 | 32 | @pytest.fixture(scope="module", autouse=True) 33 | def init(): 34 | """ Clean init """ 35 | feditest.all_tests = {} 36 | feditest._registered_as_test = {} 37 | feditest._registered_as_test_step = {} 38 | feditest._loading_tests = True 39 | 40 | feditest._loading_tests = False 41 | feditest._load_tests_pass2() 42 | 43 | 44 | @pytest.fixture(autouse=True) 45 | def the_test_plan() -> TestPlan: 46 | node_driver = MastodonUbosNodeDriver() 47 | parameters = None 48 | plan_accounts = [ 49 | { 50 | ROLE_ACCOUNT_FIELD.name : 'role1', 51 | USERID_ACCOUNT_FIELD.name : 'foo', 52 | EMAIL_ACCOUNT_FIELD.name : 'foo@bar.com', 53 | PASSWORD_ACCOUNT_FIELD.name : 'verysecret' 54 | }, 55 | { 56 | ROLE_ACCOUNT_FIELD.name : 'role2', 57 | USERID_ACCOUNT_FIELD.name : 'bar', 58 | OAUTH_TOKEN_ACCOUNT_FIELD.name : 'tokentokentoken' 59 | } 60 | ] 61 | plan_non_existing_accounts = [ 62 | { 63 | ROLE_NON_EXISTING_ACCOUNT_FIELD.name : 'nonrole1', 64 | USERID_NON_EXISTING_ACCOUNT_FIELD.name : 'nouser' 65 | } 66 | ] 67 | node1 = TestPlanConstellationNode(node_driver, parameters, plan_accounts, plan_non_existing_accounts) 68 | constellation = TestPlanConstellation({ NODE1_ROLE : node1 }) 69 | session = TestPlanSessionTemplate([]) 70 | ret = TestPlan(session, [ constellation ]) 71 | ret.properties_validate() 72 | return ret 73 | 74 | 75 | def test_parse(the_test_plan: TestPlan) -> None: 76 | """ 77 | Tests parsing the TestPlan 78 | """ 79 | node1 = the_test_plan.constellations[0].roles[NODE1_ROLE] 80 | node_driver = node1.nodedriver 81 | 82 | node_config, account_manager = node_driver.create_configuration_account_manager(NODE1_ROLE, node1) 83 | 84 | acc1 = cast(MastodonAccount | None, account_manager.get_account_by_role('role1')) 85 | assert acc1 86 | assert acc1.role == 'role1' 87 | assert acc1.userid == 'foo' 88 | assert isinstance(acc1, MastodonUserPasswordAccount) 89 | assert acc1._email == 'foo@bar.com' 90 | assert acc1._password == 'verysecret' 91 | 92 | acc2 = cast(MastodonAccount | None, account_manager.get_account_by_role('role2')) 93 | assert acc2 94 | assert acc2.role == 'role2' 95 | assert acc2.userid == 'bar' 96 | assert isinstance(acc2, MastodonOAuthTokenAccount) 97 | assert acc2._oauth_token == 'tokentokentoken' 98 | 99 | non_acc1 = cast(FediverseNonExistingAccount | None, account_manager.get_non_existing_account_by_role('nonrole1')) 100 | assert non_acc1 101 | assert non_acc1.role == 'nonrole1' 102 | assert non_acc1.userid == 'nouser' 103 | 104 | -------------------------------------------------------------------------------- /src/feditest/tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | Classes that represent tests. 3 | """ 4 | 5 | from abc import ABC, abstractmethod 6 | from collections.abc import Callable 7 | from inspect import getfullargspec 8 | from typing import Any 9 | 10 | 11 | class Test(ABC): 12 | """ 13 | Captures the notion of a Test, such as "see whether a follower is told about a new post". 14 | """ 15 | def __init__(self, name: str, description: str | None, builtin: bool) -> None: 16 | self.name: str = name 17 | self.description: str | None = description 18 | self._builtin: bool = builtin 19 | 20 | 21 | def __str__(self): 22 | return self.name 23 | 24 | 25 | @abstractmethod 26 | def metadata(self) -> dict[str, Any]: 27 | ... 28 | 29 | 30 | @abstractmethod 31 | def needed_local_role_names(self) -> set[str]: 32 | """ 33 | Determines the local names of the constellation roles this test needs. These may be mapped to 34 | constellation roles in the test definition. 35 | """ 36 | ... 37 | 38 | 39 | @property 40 | def builtin(self): 41 | """ 42 | If true, do not add this test to a test session when the session is created by collecting tests. 43 | """ 44 | return self._builtin 45 | 46 | 47 | class TestFromTestFunction(Test): 48 | """ 49 | A test that is defined as a single function. 50 | """ 51 | def __init__(self, name: str, description: str | None, test_function: Callable[..., None], builtin: bool = False) -> None: 52 | super().__init__(name, description, builtin) 53 | 54 | self.test_function = test_function 55 | 56 | 57 | # Python 3.12 @override 58 | def metadata(self) -> dict[str, Any]: 59 | return { 60 | 'Name:' : self.name, 61 | 'Description:' : self.description 62 | } 63 | 64 | 65 | # Python 3.12 @override 66 | def needed_local_role_names(self) -> set[str]: 67 | ret = {} 68 | function_spec = getfullargspec(self.test_function) 69 | for arg in function_spec.args: 70 | ret[arg] = 1 71 | return set(ret) 72 | 73 | 74 | 75 | class TestStepInTestClass: 76 | """ 77 | A step in a TestFromTestClass. TestSteps for the same Test are all declared with @step in the same class, 78 | and will be executed in sequence unless specified otherwise. 79 | """ 80 | def __init__(self, name: str, description: str | None, test: 'TestFromTestClass', test_step_function: Callable[[Any],None]) -> None: 81 | self.name: str = name 82 | self.description: str | None = description 83 | self.test = test 84 | self.test_step_function: Callable[[Any], None] = test_step_function 85 | 86 | 87 | def __str__(self): 88 | return self.name 89 | 90 | 91 | class TestFromTestClass(Test): 92 | def __init__(self, name: str, description: str | None, clazz: type, builtin: bool = False) -> None: 93 | super().__init__(name, description, builtin) 94 | 95 | self.clazz = clazz 96 | self.steps : list[TestStepInTestClass] = [] 97 | 98 | 99 | # Python 3.12 @override 100 | def metadata(self) -> dict[str, Any]: 101 | return { 102 | 'Name:' : self.name, 103 | 'Description:' : self.description, 104 | 'Steps:' : len(self.steps) 105 | } 106 | 107 | 108 | # Python 3.12 @override 109 | def needed_local_role_names(self) -> set[str]: 110 | """ 111 | Determines the names of the constellation roles this test step needs. 112 | It determines that by creating the union of the parameter names of all the TestSteps in the Test 113 | """ 114 | ret = {} 115 | function_spec = getfullargspec(self.clazz.__init__) # type: ignore [misc] 116 | for arg in function_spec.args[1:]: # first is self 117 | ret[arg] = 1 118 | return set(ret) 119 | -------------------------------------------------------------------------------- /src/feditest/cli/commands/info.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provide information on a variety of objects 3 | """ 4 | 5 | from argparse import ArgumentParser, Namespace, _SubParsersAction 6 | from typing import Any 7 | 8 | import feditest 9 | import feditest.cli 10 | from feditest.utils import format_name_value_string 11 | from feditest.reporting import warning 12 | 13 | 14 | def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: 15 | """ 16 | Run this command. 17 | """ 18 | if len(remaining): 19 | parser.print_help() 20 | return 0 21 | 22 | feditest.load_default_tests() 23 | feditest.load_tests_from(args.testsdir) 24 | 25 | feditest.load_default_node_drivers() 26 | if args.nodedriversdir: 27 | feditest.load_node_drivers_from(args.nodedriversdir) 28 | 29 | if args.test: 30 | return run_info_test(args.test) 31 | 32 | if args.nodedriver: 33 | return run_info_node_driver(args.nodedriver) 34 | 35 | parser.print_help() 36 | return 0 37 | 38 | 39 | def run_info_test(name: str) -> int: 40 | """ 41 | Provide information on a test 42 | """ 43 | test = feditest.all_tests.get(name) 44 | if test: 45 | test_metadata = test.metadata() 46 | needed_role_names = test.needed_local_role_names() 47 | if needed_role_names: 48 | test_metadata['Needed roles:'] = sorted(needed_role_names) 49 | 50 | print(format_name_value_string(test_metadata), end='') 51 | return 0 52 | 53 | warning( 'Test not found:', name) 54 | return 1 55 | 56 | 57 | def run_info_node_driver(name: str) -> int: 58 | """ 59 | Provide information on a node driver 60 | """ 61 | node_driver_class = feditest.all_node_drivers.get(name) 62 | if node_driver_class: 63 | node_driver_metadata : dict[str, Any] = { 64 | 'Node driver name:' : node_driver_class.__qualname__, 65 | 'Description:' : node_driver_class.__doc__, 66 | } 67 | pars = node_driver_class.test_plan_node_parameters() 68 | if pars: 69 | node_driver_metadata_pars = {} 70 | for par in pars: 71 | node_driver_metadata_pars[par.name] = par.description 72 | node_driver_metadata['Parameters:'] = node_driver_metadata_pars 73 | 74 | account_fields = node_driver_class.test_plan_node_account_fields() 75 | if account_fields: 76 | node_driver_metadata_fields = {} 77 | for field in account_fields: 78 | node_driver_metadata_fields[field.name] = field.description 79 | node_driver_metadata['Account fields:'] = node_driver_metadata_fields 80 | 81 | non_existing_account_fields = node_driver_class.test_plan_node_non_existing_account_fields() 82 | if non_existing_account_fields: 83 | node_driver_metadata_non_fields = {} 84 | for field in non_existing_account_fields: 85 | node_driver_metadata_non_fields[field.name] = field.description 86 | node_driver_metadata['Non-existing Account fields:'] = node_driver_metadata_non_fields 87 | 88 | print(format_name_value_string(node_driver_metadata), end='') 89 | return 0 90 | 91 | warning( 'Node driver not found:', name) 92 | return 1 93 | 94 | 95 | def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> ArgumentParser: 96 | """ 97 | Add command-line options for this sub-command 98 | parent_parser: the parent argparse parser 99 | cmd_name: name of this command 100 | """ 101 | parser = parent_parser.add_parser( cmd_name, help='Provide information on a variety of objects') 102 | parser.add_argument('--testsdir', action='append', help='Directory or directories where to find tests') 103 | parser.add_argument('--nodedriversdir', action='append', help='Directory or directories where to find extra drivers for nodes that can be tested') 104 | type_group = parser.add_mutually_exclusive_group(required=True) 105 | type_group.add_argument('--test', help='Provide information about a test.') 106 | type_group.add_argument('--nodedriver', help='Provide information about a driver for a node to be tested.') 107 | 108 | return parser 109 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/summary.jinja2: -------------------------------------------------------------------------------- 1 | {% if summary.n_total %} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | {%- if summary.n_passed != 0 %} 13 | 16 | {%- else %} 17 | 20 | {%- endif %} 21 | 22 | 23 | 24 | 93 | 94 | 95 | 96 | {%- if summary.n_skipped != 0 %} 97 | 103 | {%- else %} 104 | 107 | {%- endif %} 108 | 109 | 110 | 111 | {%- if summary.n_errored != 0 %} 112 | 118 | {%- else %} 119 | 122 | {%- endif %} 123 | 124 | 125 | 126 | 127 | 128 | 131 | 132 | 133 |
StatusCount
Passed 14 |
{{ summary.n_passed }} ({{ '%.1f' % ( summary.n_passed * 100.0 / summary.n_total ) }}%)
15 | 		18 |
0
19 |
Failed 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | {%- for interop_level in feditest.InteropLevel %} 44 | 47 | {%- endfor %} 48 | 54 | 55 | {%- for spec_level in [ feditest.SpecLevel.SHOULD, feditest.SpecLevel.IMPLIED, feditest.SpecLevel.UNSPECIFIED ] %} 56 | 57 | 58 | {%- for interop_level in feditest.InteropLevel %} 59 | 64 | {%- endfor %} 65 | 71 | 72 | {%- endfor %} 73 | 74 | 75 | {%- for interop_level in feditest.InteropLevel %} 76 | 82 | {%- endfor %} 83 | 89 | 90 | 91 |
Interoperability
CompromisedDegradedUnaffectedUnknownTotal
ConformanceMust 45 |
{{ summary.count_failures_for(feditest.SpecLevel.MUST, interop_level) }}
46 | 		49 |
50 | {{ summary.count_failures_for(feditest.SpecLevel.MUST, None) }} 51 | ({{ '%.1f' % ( summary.count_failures_for(feditest.SpecLevel.MUST, None) * 100.0 / summary.n_total ) }}%) 52 |
53 |
{{ spec_level.formatted_name | e }} 60 |
61 | {{ summary.count_failures_for(spec_level, interop_level) }} 62 |
63 | 		66 |
67 | {{ summary.count_failures_for(spec_level, None) }} 68 | ({{ '%.1f' % ( summary.count_failures_for(spec_level, None) * 100.0 / summary.n_total ) }}%) 69 |
70 |
Total 77 |
78 | {{ summary.count_failures_for(None, interop_level) }} 79 | ({{ '%.1f' % ( summary.count_failures_for(None, interop_level) * 100.0 / summary.n_total ) }}%) 80 |
81 | 		84 |
85 | {{ summary.n_failed }} 86 | ({{ '%.1f' % ( summary.n_failed * 100.0 / summary.n_total ) }}%) 87 |
88 |
92 |
Skipped 98 |
99 | {{ summary.n_skipped }} 100 | ({{ '%.1f' % ( summary.n_skipped * 100.0 / summary.n_total ) }}%) 101 |
102 | 		105 |
0
106 |
Errors 113 |
114 | {{ summary.n_errored }} 115 | ({{ '%.1f' % ( summary.n_errored * 100.0 / summary.n_total ) }}%) 116 |
117 | 		120 |
0
121 |
Total 129 |
{{ summary.n_total }}
130 |
134 | {%- else %} 135 |

No tests were run.

136 | {%- endif %} -------------------------------------------------------------------------------- /tests.unit/test_50_transcript.py: -------------------------------------------------------------------------------- 1 | # 2 | # Test the transcript has all the expected values 3 | # 4 | 5 | import pytest 6 | 7 | import feditest 8 | from feditest import nodedriver, test 9 | from feditest.utils import FEDITEST_VERSION 10 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate, TestPlanTestSpec 11 | from feditest.testrun import TestRun 12 | from feditest.testruncontroller import AutomaticTestRunController 13 | from feditest.testruntranscript import TestRunResultTranscript 14 | 15 | from dummy import DummyNodeDriver 16 | 17 | APP_NAMES = [ 18 | 'APP_0', 19 | 'APP_1', 20 | 'APP_2', 21 | 'APP_3' 22 | ] 23 | driver_names = [] 24 | 25 | @pytest.fixture(scope="module", autouse=True) 26 | def init_node_drivers(): 27 | global driver_names 28 | 29 | """ Keep these isolated to this module """ 30 | feditest.all_node_drivers = {} 31 | feditest._loading_node_drivers = True 32 | 33 | @nodedriver 34 | class NodeDriver1(DummyNodeDriver): 35 | pass 36 | 37 | @nodedriver 38 | class NodeDriver2(DummyNodeDriver): 39 | pass 40 | 41 | feditest._loading_node_drivers = False 42 | 43 | driver_names = list(feditest.all_node_drivers.keys()) 44 | 45 | 46 | @pytest.fixture(scope="module", autouse=True) 47 | def init_tests(): 48 | """ 49 | Cleanly define some tests. 50 | """ 51 | feditest.all_tests = {} 52 | feditest._registered_as_test = {} 53 | feditest._registered_as_test_step = {} 54 | feditest._loading_tests = True 55 | 56 | ## 57 | ## FediTest tests start here 58 | ## 59 | 60 | @test 61 | def passes() -> None: 62 | """ 63 | This test always passes. 64 | """ 65 | return 66 | 67 | ## 68 | ## FediTest tests end here 69 | ## (Don't forget the next two lines) 70 | ## 71 | 72 | feditest._loading_tests = False 73 | feditest._load_tests_pass2() 74 | 75 | 76 | @pytest.fixture(autouse=True) 77 | def test_plan_fixture() -> TestPlan: 78 | """ 79 | The test plan tests all known tests. 80 | """ 81 | constellations = [ 82 | TestPlanConstellation( 83 | { 84 | 'node1a': TestPlanConstellationNode( 85 | driver_names[0], 86 | { 87 | 'app' : APP_NAMES[0] 88 | } 89 | ), 90 | 'node2a': TestPlanConstellationNode( 91 | driver_names[1], 92 | { 93 | 'app' : APP_NAMES[1] 94 | } 95 | ) 96 | }, 97 | 'constellation-1'), 98 | TestPlanConstellation( 99 | { 100 | 'node1b': TestPlanConstellationNode( 101 | driver_names[0], 102 | { 103 | 'app' : APP_NAMES[2] 104 | } 105 | ), 106 | 'node2b': TestPlanConstellationNode( 107 | driver_names[1], 108 | { 109 | 'app' : APP_NAMES[3] 110 | } 111 | ) 112 | }, 113 | 'constellation-2') 114 | ] 115 | tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] 116 | session = TestPlanSessionTemplate(tests, "Test a test that passes") 117 | ret = TestPlan(session, constellations) 118 | ret.properties_validate() 119 | # ret.print() 120 | return ret 121 | 122 | 123 | @pytest.fixture 124 | def transcript(test_plan_fixture: TestPlan) -> TestRunResultTranscript: 125 | test_plan_fixture.check_can_be_executed() 126 | 127 | test_run = TestRun(test_plan_fixture) 128 | controller = AutomaticTestRunController(test_run) 129 | test_run.run(controller) 130 | 131 | ret = test_run.transcribe() 132 | return ret 133 | 134 | 135 | def test_transcript(transcript: TestRunResultTranscript): 136 | assert transcript.plan 137 | assert transcript.id 138 | assert transcript.started 139 | assert transcript.ended 140 | 141 | assert len(transcript.sessions) == 2 142 | assert len(transcript.test_meta) == 1 143 | assert transcript.result is None 144 | assert transcript.type == 'feditest-testrun-transcript' 145 | assert transcript.feditest_version == FEDITEST_VERSION 146 | 147 | for i in range(0, 1): 148 | assert transcript.sessions[i].started 149 | assert transcript.sessions[i].ended 150 | assert len(transcript.sessions[i].run_tests) == 1 151 | assert transcript.sessions[i].run_tests[0].started 152 | assert transcript.sessions[i].run_tests[0].ended 153 | assert transcript.sessions[i].run_tests[0].result is None 154 | 155 | -------------------------------------------------------------------------------- /tests.unit/test_40_static_account_manager_fallback_fediverse_accounts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test the StaticAccountManager with the FediverseAccounts and FediverseNonExistingAccounts defined 3 | in the fallback Fediverse implementation. 4 | """ 5 | 6 | from typing import cast 7 | 8 | import pytest 9 | 10 | import feditest 11 | from feditest.nodedrivers import StaticAccountManager 12 | from feditest.protocols.fediverse import ( 13 | FediverseAccount, 14 | FediverseNonExistingAccount 15 | ) 16 | 17 | @pytest.fixture(scope="module", autouse=True) 18 | def init(): 19 | """ Clean init """ 20 | feditest.all_tests = {} 21 | feditest._registered_as_test = {} 22 | feditest._registered_as_test_step = {} 23 | feditest._loading_tests = True 24 | 25 | feditest._loading_tests = False 26 | feditest._load_tests_pass2() 27 | 28 | 29 | @pytest.fixture(autouse=True) 30 | def account_manager() -> StaticAccountManager: 31 | 32 | initial_accounts : list[FediverseAccount] = [ 33 | FediverseAccount(None, 'user-0-unallocated'), 34 | FediverseAccount('role1', 'user-1-role1'), 35 | FediverseAccount(None, 'user-2-unallocated'), 36 | FediverseAccount('role3', 'user-3-role3'), 37 | ] 38 | initial_non_existing_accounts : list[FediverseNonExistingAccount] = [ 39 | FediverseNonExistingAccount(None, 'nonuser-0-unallocated'), 40 | FediverseNonExistingAccount('role1', 'nonuser-1-role1'), 41 | FediverseNonExistingAccount(None, 'nonuser-2-unallocated'), 42 | FediverseNonExistingAccount('role3', 'nonuser-3-role3'), 43 | ] 44 | ret = StaticAccountManager(initial_accounts, initial_non_existing_accounts) 45 | return ret 46 | 47 | 48 | def test_initial_accounts(account_manager: StaticAccountManager) -> None: 49 | """ 50 | Test that AccountManager has sorted the provided Accounts into the right buckets. 51 | """ 52 | acc1 = cast(FediverseAccount | None, account_manager.get_account_by_role('role1')) 53 | assert acc1 54 | assert acc1.role == 'role1' 55 | assert acc1.userid == 'user-1-role1' 56 | 57 | acc3 = cast(FediverseAccount | None, account_manager.get_account_by_role('role3')) 58 | assert acc3 59 | assert acc3.role == 'role3' 60 | assert acc3.userid == 'user-3-role3' 61 | 62 | 63 | def test_initial_non_existing_accounts(account_manager: StaticAccountManager) -> None: 64 | """ 65 | Test that AccountManager has sorted the provided NonExistingAccounts into the right buckets. 66 | """ 67 | acc1 = cast(FediverseNonExistingAccount | None, account_manager.get_non_existing_account_by_role('role1')) 68 | assert acc1 69 | assert acc1.role == 'role1' 70 | assert acc1.userid == 'nonuser-1-role1' 71 | 72 | acc3 = cast(FediverseNonExistingAccount | None, account_manager.get_non_existing_account_by_role('role3')) 73 | assert acc3 74 | assert acc3.role == 'role3' 75 | assert acc3.userid == 'nonuser-3-role3' 76 | 77 | 78 | def test_allocates_accounts_correctly(account_manager: StaticAccountManager) -> None: 79 | """ 80 | Test that the right accounts are returned given the assigned and non-assigned roles. 81 | """ 82 | # Do things a little out of order 83 | acc2 = cast(FediverseAccount | None, account_manager.get_account_by_role('role2')) 84 | assert acc2 is None 85 | 86 | acc0 = cast(FediverseAccount | None, account_manager.get_account_by_role('role0')) 87 | assert acc0 is None 88 | 89 | acc0 = cast(FediverseAccount | None, account_manager.obtain_account_by_role('role0')) 90 | assert acc0 91 | assert acc0.role is None 92 | assert acc0.userid == 'user-0-unallocated' 93 | 94 | acc2 = cast(FediverseAccount | None, account_manager.get_account_by_role('role2')) 95 | assert acc2 is None 96 | 97 | acc2 = cast(FediverseAccount | None, account_manager.obtain_account_by_role('role2')) 98 | assert acc2 99 | assert acc2.role is None 100 | assert acc2.userid == 'user-2-unallocated' 101 | 102 | 103 | def test_allocates_non_existing_accountscorrectly(account_manager: StaticAccountManager) -> None: 104 | """ 105 | Test that the right non-existing accounts are returned given the assigned and non-assigned roles. 106 | """ 107 | # Do things a little out of order 108 | acc2 = cast(FediverseNonExistingAccount | None, account_manager.get_non_existing_account_by_role('role2')) 109 | assert acc2 is None 110 | 111 | acc0 = cast(FediverseNonExistingAccount | None, account_manager.get_non_existing_account_by_role('role0')) 112 | assert acc0 is None 113 | 114 | acc0 = cast(FediverseNonExistingAccount | None, account_manager.obtain_non_existing_account_by_role('role0')) 115 | assert acc0 116 | assert acc0.role is None 117 | assert acc0.userid == 'nonuser-0-unallocated' 118 | 119 | acc2 = cast(FediverseNonExistingAccount | None, account_manager.get_non_existing_account_by_role('role2')) 120 | assert acc2 is None 121 | 122 | acc2 = cast(FediverseNonExistingAccount | None, account_manager.obtain_non_existing_account_by_role('role2')) 123 | assert acc2 124 | assert acc2.role is None 125 | assert acc2.userid == 'nonuser-2-unallocated' 126 | -------------------------------------------------------------------------------- /src/feditest/reporting.py: -------------------------------------------------------------------------------- 1 | """ 2 | Reporting functionality 3 | """ 4 | 5 | import logging 6 | import logging.config 7 | import sys 8 | import traceback 9 | 10 | logging.config.dictConfig({ 11 | 'version' : 1, 12 | 'disable_existing_loggers' : False, 13 | 'formatters' : { 14 | 'standard' : { 15 | 'format' : '%(asctime)s [%(levelname)s] %(name)s: %(message)s', 16 | 'datefmt' : '%Y-%m-%dT%H:%M:%SZ' 17 | }, 18 | }, 19 | 'handlers' : { 20 | 'default' : { 21 | 'level' : 'DEBUG', 22 | 'formatter' : 'standard', 23 | 'class' : 'logging.StreamHandler', 24 | 'stream' : 'ext://sys.stderr' 25 | } 26 | }, 27 | 'loggers' : { 28 | '' : { # root logger -- set level to most output that can happen 29 | 'handlers' : [ 'default' ], 30 | 'level' : 'WARNING', 31 | 'propagate' : True 32 | } 33 | } 34 | }) 35 | LOG = logging.getLogger( 'feditest' ) 36 | 37 | def set_reporting_level(n_verbose_flags: int) : 38 | if n_verbose_flags == 1: 39 | LOG.setLevel(logging.INFO) 40 | elif n_verbose_flags >= 2: 41 | LOG.setLevel(logging.DEBUG) 42 | 43 | def trace(*args): 44 | """ 45 | Emit a trace message. 46 | 47 | args: the message or message components 48 | """ 49 | if LOG.isEnabledFor(logging.DEBUG): 50 | LOG.debug(_construct_msg(True, False, args)) 51 | 52 | 53 | def is_trace_active() : 54 | """ 55 | Is trace logging on? 56 | 57 | return: True or False 58 | """ 59 | return LOG.isEnabledFor(logging.DEBUG) 60 | 61 | 62 | def info(*args): 63 | """ 64 | Emit an info message. 65 | 66 | args: msg: the message or message components 67 | """ 68 | if LOG.isEnabledFor(logging.INFO): 69 | LOG.info(_construct_msg(False, False, args)) 70 | 71 | 72 | def is_info_active(): 73 | """ 74 | Is info logging on? 75 | 76 | return: True or False 77 | """ 78 | return LOG.isEnabledFor(logging.INFO) 79 | 80 | 81 | def warning(*args): 82 | """ 83 | Emit a warning message. 84 | 85 | args: the message or message components 86 | """ 87 | 88 | if LOG.isEnabledFor(logging.WARNING): 89 | LOG.warning(_construct_msg(False, LOG.isEnabledFor(logging.DEBUG), args)) 90 | 91 | 92 | def is_warning_active(): 93 | """ 94 | Is warning logging on? 95 | 96 | return: True or False 97 | """ 98 | return LOG.isEnabledFor(logging.WARNING) 99 | 100 | 101 | def error(*args): 102 | """ 103 | Emit an error message. 104 | 105 | args: the message or message components 106 | """ 107 | if LOG.isEnabledFor(logging.ERROR): 108 | LOG.error(_construct_msg(False, LOG.isEnabledFor(logging.DEBUG), args)) 109 | 110 | 111 | def is_error_active(): 112 | """ 113 | Is error logging on? 114 | 115 | return: True or False 116 | """ 117 | return LOG.isEnabledFor(logging.ERROR) 118 | 119 | 120 | def fatal(*args): 121 | """ 122 | Emit a fatal error message and exit with code 1. 123 | 124 | args: the message or message components 125 | """ 126 | 127 | if args: 128 | if LOG.isEnabledFor(logging.CRITICAL): 129 | LOG.critical(_construct_msg(False, LOG.isEnabledFor(logging.DEBUG), args)) 130 | 131 | raise SystemExit(255) # Don't call exit() because that will close stdin 132 | 133 | 134 | def is_fatal_active(): 135 | """ 136 | Is fatal logging on? 137 | 138 | return: True or False 139 | """ 140 | return LOG.isEnabledFor(logging.CRITICAL) 141 | 142 | 143 | def _construct_msg(with_loc, with_tb, *args): 144 | """ 145 | Construct a message from these arguments. 146 | 147 | with_loc: construct message with location info 148 | with_tb: construct message with traceback if an exception is the last argument 149 | args: the message or message components 150 | return: string message 151 | """ 152 | if with_loc: 153 | frame = sys._getframe(2) # pylint: disable=protected-access 154 | loc = frame.f_code.co_filename 155 | loc += '#' 156 | loc += str(frame.f_lineno) 157 | loc += ' ' 158 | loc += frame.f_code.co_name 159 | loc += ':' 160 | ret = loc 161 | else: 162 | ret = '' 163 | 164 | 165 | def m(a): 166 | """ 167 | Formats provided arguments into something suitable for log messages. 168 | 169 | a: the argument 170 | return: string for the log 171 | """ 172 | if a is None: 173 | return '' 174 | if callable(a): 175 | return str(a) 176 | if isinstance(a, OSError): 177 | return type(a).__name__ + ' ' + str(a) 178 | return a 179 | 180 | args2 = map(m, *args) 181 | ret += ' '.join(map(str, args2)) 182 | 183 | if with_tb and len(*args) > 0: 184 | *_, last = iter(*args) 185 | if isinstance(last, Exception): 186 | ret += ''.join(traceback.format_exception(type(last), last, last.__traceback__)) 187 | 188 | return ret 189 | -------------------------------------------------------------------------------- /src/feditest/cli/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions used by the CLI commands. 3 | """ 4 | 5 | from argparse import ArgumentError, Namespace 6 | import re 7 | 8 | from msgspec import ValidationError 9 | 10 | import feditest 11 | from feditest.tests import Test 12 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate, TestPlanTestSpec 13 | 14 | def create_plan_from_testplan(args: Namespace) -> TestPlan: 15 | if args.constellation: 16 | raise ArgumentError(None, '--testplan already defines --constellation. Do not provide both.') 17 | if args.session: 18 | raise ArgumentError(None, '--testplan already defines --session. Do not provide both.') 19 | if args.node: 20 | raise ArgumentError(None, '--testplan already defines --node via the contained constellation. Do not provide both.') 21 | if args.test: 22 | raise ArgumentError(None, '--testplan already defines --test via the contained session. Do not provide both.') 23 | plan = TestPlan.load(args.testplan) 24 | return plan 25 | 26 | 27 | def create_plan_from_session_and_constellations(args: Namespace) -> TestPlan | None: 28 | session = create_session(args) 29 | constellations = create_constellations(args) 30 | 31 | plan = TestPlan(session, constellations, args.name) 32 | plan.properties_validate() 33 | plan.simplify() 34 | return plan 35 | 36 | 37 | def create_session(args: Namespace) -> TestPlanSessionTemplate: 38 | if args.session: 39 | session_template = create_session_from_files(args) 40 | else: 41 | session_template = create_session_template_from_tests(args) 42 | return session_template 43 | 44 | 45 | def create_session_from_files(args: Namespace) -> TestPlanSessionTemplate: 46 | if args.filter_regex: 47 | raise ArgumentError(None, '--session already defines the tests, do not provide --filter-regex') 48 | if args.test: 49 | raise ArgumentError(None, '--session already defines --test. Do not provide both.') 50 | return TestPlanSessionTemplate.load(args.session) 51 | 52 | 53 | def create_session_template_from_tests(args: Namespace) -> TestPlanSessionTemplate: 54 | test_plan_specs : list[TestPlanTestSpec]= [] 55 | tests : list[Test]= [] 56 | 57 | if args.test: 58 | if args.filter_regex: 59 | raise ArgumentError(None, '--filter-regex already defines --test. Do not provide both.') 60 | for name in args.test: 61 | test = feditest.all_tests.get(name) 62 | if test is None: 63 | raise ArgumentError(None, f'Cannot find test: "{ name }".') 64 | tests.append(test) 65 | 66 | elif args.filter_regex: 67 | pattern = re.compile(args.filter_regex) 68 | for name in sorted(feditest.all_tests.keys()): 69 | if pattern.match(name): 70 | test = feditest.all_tests.get(name) 71 | if test is None: # make linter happy 72 | continue 73 | if test.builtin: 74 | continue 75 | tests.append(test) 76 | 77 | else: 78 | for name in sorted(feditest.all_tests.keys()): 79 | test = feditest.all_tests.get(name) 80 | if test is None: # make linter happy 81 | continue 82 | if test.builtin: 83 | continue 84 | tests.append(test) 85 | 86 | for test in tests: 87 | name = test.name 88 | test_plan_spec = TestPlanTestSpec(name) 89 | test_plan_specs.append(test_plan_spec) 90 | 91 | session = TestPlanSessionTemplate(test_plan_specs, args.name) 92 | return session 93 | 94 | 95 | def create_constellations(args: Namespace) -> list[TestPlanConstellation]: 96 | if args.constellation: 97 | constellations = create_constellations_from_files(args) 98 | else: 99 | constellation = create_constellation_from_nodes(args) 100 | constellations = [ constellation ] 101 | return constellations 102 | 103 | 104 | def create_constellations_from_files(args: Namespace) -> list[TestPlanConstellation]: 105 | if args.node: 106 | raise ArgumentError(None, '--constellation already defines --node. Do not provide both.') 107 | 108 | constellations = [] 109 | for constellation_file in args.constellation: 110 | try: 111 | constellations.append(TestPlanConstellation.load(constellation_file)) 112 | except ValidationError as e: 113 | raise ArgumentError(None, f'Constellation file { constellation_file }: { e }') 114 | return constellations 115 | 116 | 117 | def create_constellation_from_nodes(args: Namespace) -> TestPlanConstellation: 118 | # Don't check for empty nodes: we need that for testing feditest 119 | roles : dict[str, TestPlanConstellationNode | None] = {} 120 | if args.node: 121 | for nodepair in args.node: 122 | if '=' not in nodepair: 123 | raise ArgumentError(None, 'Syntax error in argument: need --node =') 124 | rolename, nodefile = nodepair.split('=', 1) 125 | if not rolename: 126 | raise ArgumentError(None, f'Rolename component of --node must not be empty: "{ nodepair }".') 127 | if rolename in roles: 128 | raise ArgumentError(None, f'Role is already taken: "{ rolename }".') 129 | if not nodefile: 130 | raise ArgumentError(None, f'Filename component must not be empty: "{ nodepair }".') 131 | node = TestPlanConstellationNode.load(nodefile) 132 | roles[rolename] = node 133 | 134 | constellation = TestPlanConstellation(roles) 135 | return constellation 136 | -------------------------------------------------------------------------------- /tests.unit/test_50_run_testplan_sandbox.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run some sandbox protocol tests. 3 | """ 4 | 5 | from typing import List 6 | 7 | from hamcrest import equal_to, close_to 8 | import pytest 9 | 10 | import feditest 11 | from feditest import assert_that, step, test, SpecLevel 12 | from feditest.protocols.sandbox import SandboxLogEvent, SandboxMultClient, SandboxMultServer 13 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate, TestPlanTestSpec 14 | from feditest.testrun import TestRun 15 | from feditest.testruncontroller import AutomaticTestRunController 16 | 17 | 18 | @pytest.fixture(scope="module", autouse=True) 19 | def init_node_drivers(): 20 | """ 21 | Cleanly define the NodeDrivers. 22 | """ 23 | feditest.all_node_drivers = {} 24 | feditest.load_default_node_drivers() 25 | 26 | 27 | @pytest.fixture(scope="module", autouse=True) 28 | def init_tests(): 29 | """ 30 | Cleanly define some tests. 31 | These tests are the same as those run in "sandbox-all-clientA-vs-server1.json" from the feditest-tests-sandbox repo. 32 | """ 33 | feditest.all_tests = {} 34 | feditest._registered_as_test = {} 35 | feditest._registered_as_test_step = {} 36 | feditest._loading_tests = True 37 | 38 | ## 39 | ## FediTest tests start here 40 | ## 41 | 42 | @test 43 | class ExampleTest1: 44 | """ 45 | Tests the sandbox toy protocol using a FediTest test class. 46 | """ 47 | def __init__(self, 48 | client: SandboxMultClient, 49 | server: SandboxMultServer 50 | ) -> None: 51 | self.client = client 52 | self.server = server 53 | 54 | # We put some test data into the test class instance to demonstrate how it can be passed 55 | # along several test steps, even if one or more of them fail (with a soft assertion error) 56 | 57 | self.a : float = 2.1 58 | self.b : int = 7 59 | 60 | 61 | @step 62 | def step1(self): 63 | self.server.start_logging() 64 | 65 | self.c : float = self.client.cause_mult(self.server, self.a, self.b) 66 | 67 | assert_that(self.c, close_to(15.0, 0.5)) 68 | 69 | log: List[SandboxLogEvent] = self.server.get_and_clear_log() 70 | 71 | assert_that(len(log), equal_to(1)) 72 | assert_that(log[0].a, equal_to(self.a)) 73 | assert_that(log[0].b, equal_to(self.b)) 74 | assert_that(log[0].c, equal_to(self.c)) 75 | 76 | 77 | @step 78 | def step2(self): 79 | 80 | c_squared = self.client.cause_mult(self.server, self.c, self.c) 81 | 82 | assert_that(c_squared, close_to(self.c * self.c, 0.001), spec_level=SpecLevel.SHOULD) 83 | assert_that(c_squared, close_to(self.c * self.c, 0.5)) 84 | 85 | 86 | @test 87 | def example_test1( 88 | client: SandboxMultClient, 89 | server: SandboxMultServer 90 | ) -> None: 91 | """ 92 | Tests the sandbox toy protocol using a FediTest test function with hard asserts. 93 | """ 94 | a : float = 2 95 | b : int = 7 96 | 97 | server.start_logging() 98 | 99 | c : float = client.cause_mult(server, a, b) 100 | 101 | assert_that(c, equal_to(14.0)) 102 | 103 | log: List[SandboxLogEvent] = server.get_and_clear_log() 104 | 105 | assert_that(len(log), equal_to(1)) 106 | assert_that(log[0].a, equal_to(a)) 107 | assert_that(log[0].b, equal_to(b)) 108 | assert_that(log[0].c, equal_to(c)) 109 | 110 | 111 | @test 112 | def example_test2( 113 | client: SandboxMultClient, 114 | server: SandboxMultServer 115 | ) -> None: 116 | """ 117 | Tests the sandbox toy protocol using a FedTest test function with hard asserts. 118 | """ 119 | a : float = 2.1 120 | b : int = 7 121 | 122 | c : float = client.cause_mult(server, a, b) 123 | 124 | assert_that(c, close_to(14.7, 0.01), spec_level=SpecLevel.SHOULD) 125 | 126 | 127 | @test 128 | def example_test3( 129 | client: SandboxMultClient, 130 | server: SandboxMultServer 131 | ) -> None: 132 | """ 133 | Tests the sandbox toy protocol using a FedTest test function. 134 | """ 135 | a : int = -7 136 | b : int = 8 137 | 138 | c = client.cause_mult(server, a, b) 139 | 140 | assert_that(c, equal_to(a * b)) 141 | 142 | ## 143 | ## FediTest tests end here 144 | ## (Don't forget the next two lines) 145 | ## 146 | 147 | feditest._loading_tests = False 148 | feditest._load_tests_pass2() 149 | 150 | 151 | @pytest.fixture(autouse=True) 152 | def test_plan_fixture() -> TestPlan: 153 | """ 154 | The test plan tests all known tests. 155 | """ 156 | roles = { 157 | 'client' : TestPlanConstellationNode('SandboxMultClientDriver_ImplementationA'), 158 | 'server' : TestPlanConstellationNode('SandboxMultServerDriver_Implementation1'), 159 | } 160 | constellation = TestPlanConstellation(roles, 'clientA vs server1') 161 | tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] 162 | session = TestPlanSessionTemplate(tests, "clientA vs server") 163 | ret = TestPlan(session, [ constellation ], "All sandbox tests running clientA against server1") 164 | ret.properties_validate() 165 | return ret 166 | 167 | 168 | def test_run_testplan(test_plan_fixture: TestPlan): 169 | test_plan_fixture.check_can_be_executed() 170 | 171 | test_run = TestRun(test_plan_fixture) 172 | controller = AutomaticTestRunController(test_run) 173 | test_run.run(controller) 174 | 175 | transcript = test_run.transcribe() 176 | 177 | assert transcript.build_summary().n_failed == 0 178 | 179 | -------------------------------------------------------------------------------- /src/feditest/nodedrivers/sandbox/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | 4 | from typing import List 5 | 6 | from feditest.nodedrivers import AccountManager, NodeConfiguration, NodeDriver, HOSTNAME_PAR 7 | from feditest.protocols.sandbox import SandboxLogEvent, SandboxMultClient, SandboxMultServer 8 | from feditest.testplan import TestPlanConstellationNode, TestPlanNodeParameter 9 | from feditest.utils import FEDITEST_VERSION 10 | 11 | 12 | class SandboxMultClient_ImplementationA(SandboxMultClient): 13 | """ 14 | A client implementation in the Sandbox protocol that can be tested. It's trivially simple. 15 | """ 16 | def cause_mult(self, server: SandboxMultServer, a: float, b: float) -> float: 17 | c = server.mult(a, b) 18 | return c 19 | 20 | 21 | class SandboxMultClientDriver_ImplementationA(NodeDriver): 22 | """ 23 | Driver for the client implementation, so the client can be provisioned and unprovisioned for 24 | test sessions. 25 | """ 26 | # Python 3.12 @override 27 | @staticmethod 28 | def test_plan_node_parameters() -> list[TestPlanNodeParameter]: 29 | return [ HOSTNAME_PAR ] 30 | 31 | 32 | # Python 3.12 @override 33 | def create_configuration_account_manager(self, rolename: str, test_plan_node: TestPlanConstellationNode) -> tuple[NodeConfiguration, AccountManager | None]: 34 | return ( 35 | NodeConfiguration( 36 | self, 37 | 'SandboxMultClient_ImplementationA', 38 | FEDITEST_VERSION, 39 | test_plan_node.parameter(HOSTNAME_PAR) 40 | ), 41 | None 42 | ) 43 | 44 | 45 | # Python 3.12 @override 46 | def _provision_node(self, rolename: str, config: NodeConfiguration, account_manager: AccountManager | None) -> SandboxMultClient_ImplementationA: 47 | return SandboxMultClient_ImplementationA(rolename, config, account_manager) 48 | 49 | 50 | class SandboxMultServer_Implementation1(SandboxMultServer): 51 | """ 52 | First server implementation in the Sandbox protocol with some test instrumentation. 53 | This server implementation simply calculates a*b. 54 | """ 55 | def __init__(self, rolename: str, config: NodeConfiguration): 56 | super().__init__(rolename, config) # Has no AccountManager 57 | self._log : List[SandboxLogEvent] | None = None 58 | 59 | 60 | def mult(self, a: float, b: float) -> float: 61 | # Here's the key 'mult' functionality: 62 | c = a*b 63 | if self._log is not None: 64 | self._log.append(SandboxLogEvent(a, b, c)) 65 | return c 66 | 67 | 68 | def start_logging(self): 69 | self._log = [] 70 | 71 | 72 | def get_and_clear_log(self): 73 | ret = self._log 74 | self._log = None 75 | return ret 76 | 77 | 78 | class SandboxMultServerDriver_Implementation1(NodeDriver): 79 | """ 80 | Driver for the first server implementation, so this server implementation can be provisioned and unprovisioned for 81 | test sessions. 82 | """ 83 | # Python 3.12 @override 84 | @staticmethod 85 | def test_plan_node_parameters() -> list[TestPlanNodeParameter]: 86 | return [ HOSTNAME_PAR ] 87 | 88 | 89 | # Python 3.12 @override 90 | def create_configuration_account_manager(self, rolename: str, test_plan_node: TestPlanConstellationNode) -> tuple[NodeConfiguration, AccountManager | None]: 91 | return ( 92 | NodeConfiguration( 93 | self, 94 | 'SandboxMultServer_Implementation1', 95 | FEDITEST_VERSION, 96 | test_plan_node.parameter(HOSTNAME_PAR) 97 | ), 98 | None 99 | ) 100 | 101 | 102 | # Python 3.12 @override 103 | def _provision_node(self, rolename: str, config: NodeConfiguration, account_manager: AccountManager | None) -> SandboxMultServer_Implementation1: 104 | return SandboxMultServer_Implementation1(rolename, config) 105 | 106 | 107 | class SandboxMultServer_Implementation2Faulty(SandboxMultServer): 108 | """ 109 | Second server implementation in the Sandbox protocol with some test instrumentation. 110 | This server calculates a*b through a for loop using integers rather than floats 111 | """ 112 | def __init__(self, rolename: str, config: NodeConfiguration): 113 | super().__init__(rolename, config) # Has no AccountManager 114 | self._log : List[SandboxLogEvent] | None = None 115 | 116 | 117 | def mult(self, a: float, b: float) -> float: 118 | c = 0.0 119 | # Here's the key 'mult' functionality, but it only works for positive a's that are integers! 120 | a_int = int(a) 121 | for _ in range(0, a_int) : 122 | c += b 123 | if self._log is not None: 124 | self._log.append(SandboxLogEvent(a, b, c)) 125 | return c 126 | 127 | 128 | def start_logging(self): 129 | self._log = [] 130 | 131 | 132 | def get_and_clear_log(self): 133 | ret = self._log 134 | self._log = None 135 | return ret 136 | 137 | 138 | class SandboxMultServerDriver_Implementation2Faulty(NodeDriver): 139 | """ 140 | Driver for the second server implementation, so this server implementation can be provisioned and unprovisioned for 141 | test sessions. 142 | """ 143 | # Python 3.12 @override 144 | @staticmethod 145 | def test_plan_node_parameters() -> list[TestPlanNodeParameter]: 146 | return [ HOSTNAME_PAR ] 147 | 148 | 149 | # Python 3.12 @override 150 | def create_configuration_account_manager(self, rolename: str, test_plan_node: TestPlanConstellationNode) -> tuple[NodeConfiguration, AccountManager | None]: 151 | return ( 152 | NodeConfiguration( 153 | self, 154 | 'SandboxMultServer_Implementation2Faulty', 155 | FEDITEST_VERSION, 156 | test_plan_node.parameter(HOSTNAME_PAR) 157 | ), 158 | None 159 | ) 160 | 161 | 162 | # Python 3.12 @override 163 | def _provision_node(self, rolename: str, config: NodeConfiguration, account_manager: AccountManager | None) -> SandboxMultServer_Implementation2Faulty: 164 | return SandboxMultServer_Implementation2Faulty(rolename, config) 165 | -------------------------------------------------------------------------------- /src/feditest/protocols/activitypub/diag.py: -------------------------------------------------------------------------------- 1 | from typing import Any, cast 2 | 3 | from . import ActivityPubNode 4 | from feditest.protocols.web.diag import WebDiagClient, WebDiagServer 5 | 6 | # Note: 7 | # The data elements held by the classes here are all untyped. That's because we want to be able 8 | # to store data we receive even if it is invalid according to the spec. 9 | # check_integrity() helps figure out whether it is valid or not. 10 | 11 | class AnyObject: 12 | """ 13 | This container is used to hold any instance of any of the ActivityStreams types. 14 | We use a generic container because we also want to be able to hold objects 15 | that are invalid according to the spec. 16 | """ 17 | def __init__(self, uri: str, json: Any): 18 | self._uri = uri 19 | self._json = json 20 | 21 | 22 | def check_is_valid_object(self) -> bool: 23 | """ 24 | Interpret this instance as an ActivityStreams Object, and check whether it is valid. 25 | """ 26 | json = cast(dict, self._json) 27 | if 'type' not in json: 28 | return False 29 | type = json['type'] 30 | if not isinstance(type,str): 31 | return False 32 | return 'Object' == type 33 | 34 | 35 | def as_actor(self) -> 'Actor': 36 | """ 37 | Interpret this instance as an Actor, and return an instance of the Actor class. 38 | """ 39 | # FIXME: check that this is indeed a valid Actor, and throw exception if it is not 40 | return Actor(self) 41 | 42 | 43 | def as_collection(self) -> 'Collection': 44 | """ 45 | Interpret this instance as a Collection, and return an instance of the Collection class. 46 | """ 47 | # FIXME: check that this is indeed a valid Collection, and throw exception if it is not 48 | return Collection(self) 49 | 50 | 51 | def json_field(self, name:str): 52 | """ 53 | Convenience method to access field 'name' in the JSON. 54 | """ 55 | json = cast(dict, self._json) 56 | return json.get(name) 57 | 58 | 59 | class Actor: 60 | """ 61 | A facade in front of AnyObject that interprets AnyObject as an Actor. 62 | """ 63 | def __init__(self, delegate: AnyObject): 64 | self._delegate = delegate 65 | 66 | 67 | def followers_uri(self): 68 | # FIXME can this be in different format, like a list? 69 | return self._delegate.json_field('followers') 70 | 71 | 72 | def following_uri(self): 73 | # FIXME can this be in different format, like a list? 74 | return self._delegate.json_field('following') 75 | 76 | 77 | class Activity: 78 | """ 79 | A facade in front of AnyObject that interprets AnyObject as an Activity. 80 | """ 81 | def __init__(self, delegate: AnyObject): 82 | self._delegate = delegate 83 | 84 | 85 | class Collection: 86 | """ 87 | A facade in front of AnyObject that interprets AnyObject as a Collection. 88 | """ 89 | def __init__(self, delegate: AnyObject): 90 | self._delegate = delegate 91 | 92 | 93 | def is_ordered(self): 94 | return 'OrderedCollection' == self._delegate.json_field('type') 95 | 96 | 97 | # Work in progress 98 | 99 | # def items(self) -> Iterator[AnyObject]: 100 | # items = self._delegate.json_field('orderedItems' if self.is_ordered() else 'items') 101 | # if items is not None: 102 | # for item in items: 103 | # if isinstance(item,str): 104 | # yield AnyObject(item) 105 | # else: 106 | # raise Exception(f'Cannot process yet: {item}') 107 | # elif first := self._delegate.json_field('first') is not None: 108 | # if isinstance(first,str): 109 | # first_collection = AnyObject(first).as_collection() 110 | # yield from first_collection.items() 111 | # else: 112 | # raise Exception(f'Cannot process yet: {first}') 113 | # elif next := self._delegate.json_field('next') is not None: 114 | # if isinstance(next,str): 115 | # next_collection = AnyObject(next).as_collection() 116 | # yield from next_collection.items() 117 | # else: 118 | # raise Exception(f'Cannot process yet: {first}') 119 | 120 | 121 | # def contains(self, matcher: Callable[[AnyObject],bool]) -> bool: 122 | # """ 123 | # Returns true if this Collection contains an item, as determined by the 124 | # matcher object. This method passes the members of this collection to the 125 | # matcher one at a time, and the matcher decides when there is a match. 126 | # """ 127 | # for item in self.items(): 128 | # if matcher(item): 129 | # return True 130 | # return False 131 | 132 | 133 | # def contains_item_with_id(self, id: str) -> bool: 134 | # """ 135 | # Convenience method that looks for items that are simple object identifiers. 136 | # FIXME: this can be much more complicated in ActivityStreams, but this 137 | # implementation is all we need right now. 138 | # """ 139 | # return self.contains(lambda candidate: id == candidate if isinstance(candidate,str) else False) 140 | 141 | 142 | class ActivityPubDiagNode(WebDiagClient, WebDiagServer,ActivityPubNode): 143 | pass 144 | 145 | # Work in progress 146 | 147 | # def fetch_remote_actor_document(remote_actor_acct_uri: str) -> Actor: 148 | # pass 149 | 150 | 151 | # def set_inbox_uri_to(actor_acct_uri: str, inbox_uri: str | None): 152 | # pass 153 | 154 | 155 | # def set_outbox_uri_to(actor_acct_uri: str, outbox_uri: str | None): 156 | # pass 157 | 158 | 159 | # def add_to_followers_collection(actor_acct_uri: str, to_be_added_actor_acct_uri: str): 160 | # pass 161 | 162 | 163 | # def add_to_following_collection(actor_acct_uri: str, to_be_added_actor_acct_uri: str): 164 | # pass 165 | 166 | 167 | # def add_to_outbox(actor_acct_uri: str, to_be_added_activity: Activity): 168 | # pass 169 | 170 | 171 | # def add_to_inbox(actor_acct_uri: str, to_be_added_activity: Activity): 172 | # pass 173 | 174 | 175 | # def read_inbox_of(actor_acct_uri: str, inbox_collection: Collection): 176 | # pass 177 | 178 | 179 | # def read_outbox_of(actor_acct_uri: str, outbox_collection: Collection): 180 | # pass -------------------------------------------------------------------------------- /src/feditest/testruncontroller.py: -------------------------------------------------------------------------------- 1 | """ 2 | Classes that know how to control a TestRun. 3 | """ 4 | 5 | from abc import ABC, abstractmethod 6 | 7 | import feditest.testrun 8 | from feditest.reporting import is_trace_active 9 | 10 | class TestRunControlException(Exception,ABC): 11 | """ 12 | Superclass of all exceptions we use to control test run execution. 13 | """ 14 | pass 15 | 16 | 17 | class AbortTestRunException(TestRunControlException): 18 | """ 19 | Stop executing this test run as quickly as possible. 20 | Don't run any more test steps and tests. Shut down the current constellation and don't run any more sessions. 21 | """ 22 | pass 23 | 24 | 25 | class AbortTestRunSessionException(TestRunControlException): 26 | """ 27 | Stop executing this test run session as quickly as possible, and continue with the next test run session. 28 | """ 29 | pass 30 | 31 | 32 | class AbortTestException(TestRunControlException): 33 | """ 34 | Stop executing this test as quickly as possible, and continue with the next test in the current test run session. 35 | """ 36 | pass 37 | 38 | 39 | class TestRunController(ABC): 40 | def __init__(self, run: 'feditest.testrun.TestRun' ): 41 | self.run = run 42 | 43 | 44 | @abstractmethod 45 | def determine_next_constellation_index(self, last_constellation_index: int = -1) -> int: 46 | """ 47 | last_constellation_index: -1 means: we haven't started yet 48 | """ 49 | ... 50 | 51 | 52 | @abstractmethod 53 | def determine_next_test_index(self, last_test_index: int = -1) -> int: 54 | """ 55 | last_test_index: -1 means: we haven't started yet 56 | """ 57 | ... 58 | 59 | 60 | @abstractmethod 61 | def determine_next_test_step_index(self, last_test_step_index: int = -1) -> int: 62 | """ 63 | last_test_step_index: -1 means: we haven't started yet 64 | """ 65 | ... 66 | 67 | 68 | class AutomaticTestRunController(TestRunController): 69 | def determine_next_constellation_index(self, last_constellation_index: int = -1) -> int: 70 | return last_constellation_index+1 71 | 72 | 73 | def determine_next_test_index(self, last_test_index: int = -1) -> int: 74 | return last_test_index+1 75 | 76 | 77 | def determine_next_test_step_index(self, last_test_step_index: int = -1) -> int: 78 | return last_test_step_index+1 79 | 80 | 81 | class InteractiveTestRunController(TestRunController): 82 | def determine_next_constellation_index(self, last_constellation_index: int = -1) -> int: 83 | """ 84 | A TestRunSession with a certain TestRunConstellation has just completed. Which TestRunConstellation should 85 | we run it with next? 86 | """ 87 | if last_constellation_index >= 0: 88 | prompt = 'Which Constellation to run tests with next? n(ext constellation), r(repeat just completed constellation), (constellation number), q(uit): ' 89 | else: 90 | prompt = 'Which Constellation to run first? n(ext/first constellation), (constellation number), q(uit): ' 91 | while True: 92 | answer = self._prompt_user(prompt) 93 | match answer: 94 | case 'n': 95 | return last_constellation_index+1 96 | case 'r': 97 | return last_constellation_index 98 | case 'q': 99 | raise AbortTestRunException() 100 | try: 101 | parsed = int(answer) 102 | if parsed >= 0: 103 | return parsed 104 | except ValueError: 105 | pass 106 | print('Command not recognized, try again.') 107 | 108 | 109 | def determine_next_test_index(self, last_test_index: int = -1) -> int: 110 | """ 111 | A Test has just completed. Which Test should we run next? 112 | """ 113 | if last_test_index >= 0: 114 | prompt = 'Which Test to run next? n(ext test), r(repeat just completed test), a(bort current session), q(uit): ' 115 | else: 116 | prompt = 'Which Test to run first? n(ext/first test), (test number), a(bort current session), q(uit): ' 117 | while True: 118 | answer = self._prompt_user(prompt) 119 | match answer: 120 | case 'n': 121 | return last_test_index+1 122 | case 'r': 123 | return last_test_index 124 | case 'a': 125 | raise AbortTestRunSessionException() 126 | case 'q': 127 | raise AbortTestRunException() 128 | try: 129 | parsed = int(answer) 130 | if parsed >= 0: 131 | return parsed 132 | except ValueError: 133 | pass 134 | print('Command not recognized, try again.') 135 | 136 | 137 | def determine_next_test_step_index(self, last_test_step_index: int = -1) -> int: 138 | """ 139 | A Test Step as just completed. Which Test Step should we run next? 140 | """ 141 | if last_test_step_index >= 0: 142 | prompt = 'Which Test Step to run next? n(ext test step), r(repeat just completed test test), c(ancel current test), a(bort current session), q(uit): ' 143 | else: 144 | prompt = 'Which Test Step to run first? n(ext/first test step), (test step number), c(ancel current test), a(bort current session), q(uit): ' 145 | while True: 146 | answer = self._prompt_user(prompt) 147 | match answer: 148 | case 'n': 149 | return last_test_step_index+1 150 | case 'r': 151 | return last_test_step_index 152 | case 'c': 153 | raise AbortTestException() 154 | case 'a': 155 | raise AbortTestRunSessionException() 156 | case 'q': 157 | raise AbortTestRunException() 158 | try: 159 | parsed = int(answer) 160 | if parsed >= 0: 161 | return parsed 162 | except ValueError: 163 | pass 164 | print('Command not recognized, try again.') 165 | 166 | 167 | def _prompt_user(self, question: str) -> str: 168 | # In case of debugging, there's a lot of output, and it can be hard to tell where the steps end 169 | if is_trace_active(): 170 | print() 171 | 172 | ret = input(f'Interactive: { question }') 173 | 174 | if is_trace_active(): 175 | print() 176 | 177 | return ret 178 | -------------------------------------------------------------------------------- /src/feditest/cli/commands/run.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run one or more tests 3 | """ 4 | 5 | from argparse import ArgumentParser, Namespace, _SubParsersAction 6 | from typing import cast 7 | 8 | import feditest 9 | from feditest.cli.utils import ( 10 | create_plan_from_session_and_constellations, 11 | create_plan_from_testplan 12 | ) 13 | from feditest.registry import Registry, set_registry_singleton 14 | from feditest.reporting import fatal, warning 15 | from feditest.testplan import TestPlan 16 | from feditest.testrun import TestRun 17 | from feditest.testruncontroller import AutomaticTestRunController, InteractiveTestRunController, TestRunController 18 | from feditest.testruntranscriptserializer.json import JsonTestRunTranscriptSerializer 19 | from feditest.testruntranscriptserializer.html import HtmlRunTranscriptSerializer 20 | from feditest.testruntranscriptserializer.summary import SummaryTestRunTranscriptSerializer 21 | from feditest.testruntranscriptserializer.tap import TapTestRunTranscriptSerializer 22 | from feditest.utils import FEDITEST_VERSION, hostname_validate 23 | 24 | 25 | def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: 26 | """ 27 | Run this command. 28 | """ 29 | if len(remaining): 30 | parser.print_help() 31 | return 0 32 | 33 | feditest.load_default_tests() 34 | feditest.load_tests_from(args.testsdir) 35 | 36 | feditest.load_default_node_drivers() 37 | if args.nodedriversdir: 38 | feditest.load_node_drivers_from(args.nodedriversdir) 39 | 40 | if args.domain: 41 | set_registry_singleton(Registry.create(args.domain)) # overwrite 42 | 43 | # Determine testplan. While we are at it, check consistency of arguments. 44 | plan : TestPlan | None = None 45 | if args.testplan: 46 | plan = create_plan_from_testplan(args) 47 | else: 48 | plan = create_plan_from_session_and_constellations(args) 49 | 50 | if not plan: 51 | fatal('Cannot find or create test plan ') 52 | return 1 # make linter happy 53 | 54 | if not plan.is_compatible_type(): 55 | warning(f'Test plan has unexpected type { plan.type }: incompatibilities may occur.') 56 | if not plan.has_compatible_version(): 57 | warning(f'Test plan was created by FediTest { plan.feditest_version }, you are running FediTest { FEDITEST_VERSION }: incompatibilities may occur.') 58 | plan.check_can_be_executed() 59 | 60 | test_run = TestRun(plan, args.who) 61 | if args.interactive : 62 | warning('--interactive: implementation is incomplete') 63 | controller : TestRunController = InteractiveTestRunController(test_run) 64 | else: 65 | controller = AutomaticTestRunController(test_run) 66 | test_run.run(controller) 67 | 68 | transcript : feditest.testruntranscript.TestRunTranscript = test_run.transcribe() 69 | 70 | if isinstance(args.tap, str) or args.tap: 71 | TapTestRunTranscriptSerializer().write(transcript, cast(str|None, args.tap)) 72 | 73 | if isinstance(args.html, str): 74 | HtmlRunTranscriptSerializer(args.template_path).write(transcript, args.html) 75 | elif args.html: 76 | warning('--html requires a filename: skipping') 77 | elif args.template_path: 78 | warning('--template-path only supported with --html. Ignoring.') 79 | 80 | if isinstance(args.json, str) or args.json: 81 | JsonTestRunTranscriptSerializer().write(transcript, args.json) 82 | 83 | if isinstance(args.summary, str) or args.summary: 84 | SummaryTestRunTranscriptSerializer().write(transcript, args.summary) 85 | 86 | if transcript.build_summary().n_failed > 0: 87 | print('FAILED.') 88 | return 1 89 | return 0 90 | 91 | 92 | def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> ArgumentParser: 93 | """ 94 | Add command-line options for this sub-command 95 | parent_parser: the parent argparse parser 96 | cmd_name: name of this command 97 | """ 98 | # general flags and options 99 | parser = parent_parser.add_parser(cmd_name, help='Run one or more tests' ) 100 | parser.add_argument('--testsdir', action='append', help='Directory or directories where to find tests') 101 | parser.add_argument('--nodedriversdir', action='append', help='Directory or directories where to find extra drivers for nodes that can be tested') 102 | parser.add_argument('--domain', type=hostname_validate, help='Local-only DNS domain for the DNS hostnames that are auto-generated for nodes') 103 | parser.add_argument('-i', '--interactive', action="store_true", help="Run the tests interactively") 104 | parser.add_argument('--who', action='store_true', help="Record who ran the test plan on what host.") 105 | 106 | # test plan options. We do not use argparse groups, as the situation is more complicated than argparse seems to support 107 | parser.add_argument('--name', default=None, required=False, help='Name of the generated test plan') 108 | parser.add_argument('--testplan', help='Name of the file that contains the test plan to run') 109 | parser.add_argument('--constellation', action='append', help='File(s) each containing a JSON fragment defining a constellation') 110 | parser.add_argument('--session', '--session-template', required=False, help='File(s) each containing a JSON fragment defining a test session') 111 | parser.add_argument('--node', action='append', 112 | help="Use = to specify that the node definition in 'file' is supposed to be used for constellation role 'role'") 113 | parser.add_argument('--filter-regex', default=None, help='Only include tests whose name matches this regular expression') 114 | parser.add_argument('--test', action='append', help='Run this/these named tests(s)') 115 | 116 | # output options 117 | parser.add_argument('--tap', nargs="?", const=True, default=False, 118 | help="Write results in TAP format to stdout, or to the provided file (if given).") 119 | html_group = parser.add_argument_group('html', 'HTML options') 120 | html_group.add_argument('--html', 121 | help="Write results in HTML format to the provided file.") 122 | html_group.add_argument('--template-path', required=False, 123 | help="When specifying --html, use this template path override (comma separated directory names)") 124 | parser.add_argument('--json', '--testresult', nargs="?", const=True, default=False, 125 | help="Write results in JSON format to stdout, or to the provided file (if given).") 126 | parser.add_argument('--summary', nargs="?", const=True, default=False, 127 | help="Write summary to stdout, or to the provided file (if given). This is the default if no other output option is given") 128 | 129 | return parser 130 | -------------------------------------------------------------------------------- /tests.unit/test_60_use_manually_entered_application_name.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test that when the user enters an application name at the command-line, it shows up 3 | in the report. 4 | """ 5 | 6 | import os 7 | import os.path 8 | import tempfile 9 | 10 | import pytest 11 | from bs4 import BeautifulSoup 12 | 13 | import feditest 14 | from feditest import test 15 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate, TestPlanTestSpec 16 | from feditest.testrun import TestRun 17 | from feditest.testruncontroller import AutomaticTestRunController 18 | from feditest.testruntranscript import TestRunResultTranscript 19 | from feditest.testruntranscriptserializer.html import HtmlRunTranscriptSerializer 20 | 21 | APP_NAMES = [ 22 | 'SENDER_APP_1', 23 | 'RECEIVER_APP_1', 24 | 'SENDER_APP_2', 25 | 'RECEIVER_APP_2' 26 | ] 27 | 28 | # Use the default NodeDrivers 29 | 30 | @pytest.fixture(scope="module", autouse=True) 31 | def init_node_drivers(): 32 | """ 33 | Cleanly define the NodeDrivers. 34 | """ 35 | feditest.load_default_node_drivers() 36 | 37 | 38 | @pytest.fixture(scope="module", autouse=True) 39 | def init_tests(): 40 | """ 41 | Cleanly define some tests. 42 | """ 43 | feditest.all_tests = {} 44 | feditest._registered_as_test = {} 45 | feditest._registered_as_test_step = {} 46 | feditest._loading_tests = True 47 | 48 | ## 49 | ## FediTest tests start here 50 | ## 51 | 52 | @test 53 | def passes() -> None: 54 | """ 55 | This test always passes. 56 | """ 57 | return 58 | 59 | ## 60 | ## FediTest tests end here 61 | ## (Don't forget the next two lines) 62 | ## 63 | 64 | feditest._loading_tests = False 65 | feditest._load_tests_pass2() 66 | 67 | 68 | @pytest.fixture(scope="module", autouse=True) 69 | def test_plan_fixture() -> TestPlan: 70 | """ 71 | The test plan tests all known tests. 72 | """ 73 | constellations = [ 74 | TestPlanConstellation( 75 | { 76 | 'sender_node': TestPlanConstellationNode( 77 | 'FediverseSaasNodeDriver', 78 | { 79 | 'app' : APP_NAMES[0], 80 | 'hostname' : 'senderA' 81 | }), 82 | 'receiver_node': TestPlanConstellationNode( 83 | 'FediverseSaasNodeDriver', 84 | { 85 | 'app' : APP_NAMES[1], 86 | 'hostname' : 'senderB' 87 | }) 88 | }, 89 | 'constellation-1'), 90 | TestPlanConstellation( 91 | { 92 | 'sender_node': TestPlanConstellationNode( 93 | 'FediverseSaasNodeDriver', 94 | { 95 | 'app' : APP_NAMES[2], 96 | 'hostname' : 'senderA' 97 | }), 98 | 'receiver_node': TestPlanConstellationNode( 99 | 'FediverseSaasNodeDriver', 100 | { 101 | 'app' : APP_NAMES[3], 102 | 'hostname' : 'senderB' 103 | }) 104 | }, 105 | 'constellation-2') 106 | ] 107 | tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] 108 | session = TestPlanSessionTemplate(tests, "Test a test that passes") 109 | ret = TestPlan(session, constellations) 110 | ret.properties_validate() 111 | # ret.print() 112 | return ret 113 | 114 | 115 | @pytest.fixture(scope="module") 116 | def transcript(test_plan_fixture: TestPlan) -> TestRunResultTranscript: 117 | test_plan_fixture.check_can_be_executed() 118 | 119 | test_run = TestRun(test_plan_fixture) 120 | controller = AutomaticTestRunController(test_run) 121 | test_run.run(controller) 122 | 123 | ret = test_run.transcribe() 124 | return ret 125 | 126 | 127 | @pytest.fixture(scope="module") 128 | def html_base_name(transcript: TestRunResultTranscript) -> str: 129 | def gen(tmpdirname: str) -> str: 130 | outbase = f'{ tmpdirname }/htmlout' 131 | HtmlRunTranscriptSerializer().write(transcript, f'{ outbase }.html') 132 | return outbase 133 | 134 | if False: # Change this to False if you want to see what is being generated 135 | with tempfile.TemporaryDirectory() as tmpdirname: 136 | ret = gen(tmpdirname) 137 | yield ret 138 | # The yield will cause the TemporaryDirectory to be deleted at the end of the test 139 | return None 140 | 141 | else: 142 | dir = f'/tmp/{ os.path.basename(__file__)[:-3] }' # without extension 143 | os.makedirs(dir, exist_ok=True) 144 | ret = gen(dir) # without extension 145 | yield ret # apparently yielding from one branch, and not the other does not work 146 | return None 147 | 148 | 149 | @pytest.fixture(scope="module") 150 | def main_html_soup(html_base_name: str) -> BeautifulSoup: 151 | content = None 152 | with open(f'{ html_base_name }.html') as fd: 153 | content = fd.read() 154 | 155 | ret = BeautifulSoup(content, 'html.parser') 156 | # print(ret.prettify()) 157 | return ret 158 | 159 | 160 | @pytest.fixture(scope="module") 161 | def session0_html_soup(html_base_name: str) -> BeautifulSoup: 162 | content = None 163 | with open(f'{ html_base_name }.0.html') as fd: 164 | content = fd.read() 165 | 166 | ret = BeautifulSoup(content, 'html.parser') 167 | # print(ret.prettify()) 168 | return ret 169 | 170 | 171 | ## Main HTML doc 172 | 173 | def test_main_title(main_html_soup: BeautifulSoup): 174 | title = main_html_soup.head.title.string 175 | 176 | # Don't have an empty string prior to the | 177 | assert title.split('|')[0].strip() 178 | 179 | # Don't have 'None" in the title 180 | assert 'None' not in title 181 | 182 | 183 | def test_main_h1(main_html_soup: BeautifulSoup): 184 | # Don't have 'None" in the title 185 | *_, h1 = main_html_soup.body.header.h1.strings 186 | assert 'None' not in h1 187 | 188 | 189 | def test_main_app_properties(main_html_soup: BeautifulSoup): 190 | # Use 'app' properties, not FediverseSaasNodeDriver 191 | for dl in main_html_soup.body.find_all('dl', class_='roles'): 192 | for dd in dl.find_all('dd'): 193 | assert dd.string in APP_NAMES 194 | 195 | # Session HTML doc 196 | 197 | def test_session0_title(session0_html_soup: BeautifulSoup): 198 | title = session0_html_soup.head.title.string 199 | 200 | # Don't have an empty string prior to the | 201 | assert title.split('|')[0].strip() 202 | 203 | # Don't have 'Session 0" in the title 204 | assert 'Session 0' not in title 205 | 206 | 207 | def test_session0_h1(session0_html_soup: BeautifulSoup): 208 | *_, h1 = session0_html_soup.body.header.h1.strings 209 | 210 | # Don't have an empty string prior to the | 211 | assert h1.split('|')[0].strip() 212 | 213 | # Don't have 'Session 0" in the title 214 | assert 'Session 0' not in h1 215 | -------------------------------------------------------------------------------- /src/feditest/testruntranscriptserializer/html.py: -------------------------------------------------------------------------------- 1 | # import contextlib 2 | import html 3 | import os.path 4 | import re 5 | import shutil 6 | from typing import Any, Iterator 7 | 8 | import jinja2 9 | 10 | import feditest 11 | from feditest.reporting import fatal 12 | from feditest.testruntranscript import TestMetaTranscript, TestRunResultTranscript, TestRunSessionTranscript, TestRunTranscript 13 | from feditest.testruntranscriptserializer import TestRunTranscriptSerializer 14 | 15 | 16 | def _get_results_for(run_transcript: TestRunTranscript, session_transcript: TestRunSessionTranscript, test_meta: TestMetaTranscript) -> Iterator[TestRunResultTranscript | None]: 17 | """ 18 | Determine the set of test results running test_meta within session_transcript, and return it as an Iterator. 19 | This is a set, not a single value, because we might run the same test multiple times (perhaps with differing role 20 | assignments) in the same session. The run_transcript is passed in because session_transcript does not have a pointer "up". 21 | """ 22 | plan_session_template = run_transcript.plan.session_template 23 | for test_transcript in session_transcript.run_tests: 24 | plan_testspec = plan_session_template.tests[test_transcript.plan_test_index] 25 | if plan_testspec.name == test_meta.name: 26 | yield test_transcript.worst_result 27 | return None 28 | 29 | 30 | def _derive_full_and_local_filename(base: str, suffix: str) -> tuple[str,str]: 31 | """ 32 | Given a base filename, derive another filename (e.g. generate a .css filename from an .html filename). 33 | Return the full filename with path, and the local filename 34 | """ 35 | dir = os.path.dirname(base) 36 | local = os.path.basename(base) 37 | last_dot = local.rfind('.') 38 | if last_dot > 0: 39 | derived = f'{ local[0:last_dot] }{ suffix }' 40 | else: 41 | derived = f'{ local }.{ suffix }' 42 | return (os.path.join(dir, derived), derived) 43 | 44 | 45 | class HtmlRunTranscriptSerializer(TestRunTranscriptSerializer): 46 | """ 47 | Generates the FediTest reports as HTML. 48 | If the transcript contains one session, it will generate one HTML file to the provided destination. 49 | 50 | If the transcript contains multiple sessions, it will generate one HTML file per session and 51 | an overview test matrix. The test matrix will be at the provided destination, and the session 52 | files will have longer file names starting with the filename of the destination. 53 | 54 | A CSS file will be written to the provided destination with an extra extension. 55 | """ 56 | 57 | def __init__(self, template_path: str | None = None): 58 | if template_path: 59 | self.template_path = [ t.strip() for t in template_path.split(",") ] 60 | else: 61 | self.template_path = [ os.path.join(os.path.dirname(__file__), "templates/testplantranscript_default") ] 62 | 63 | self.jinja2_env = jinja2.Environment( 64 | loader=jinja2.FileSystemLoader(self.template_path), 65 | autoescape=jinja2.select_autoescape() 66 | ) 67 | self.jinja2_env.filters["regex_sub"] = lambda s, pattern, replacement: re.sub( 68 | pattern, replacement, s 69 | ) 70 | 71 | 72 | # Python 3.12 @override 73 | def write(self, transcript: TestRunTranscript, dest: str | None): 74 | if dest is None: 75 | fatal('Cannot write --html to stdout.') 76 | return # make linter happy 77 | if len(transcript.sessions) == 0: 78 | fatal('No session in transcript: cannot transcribe') 79 | 80 | ( cssfile, local_cssfile ) = _derive_full_and_local_filename(dest, '.css') 81 | base_context = dict( 82 | feditest=feditest, 83 | cssfile = local_cssfile, 84 | getattr=getattr, 85 | sorted=sorted, 86 | enumerate=enumerate, 87 | get_results_for=_get_results_for, 88 | remove_white=lambda s: re.sub("[ \t\n\a]", "_", str(s)), 89 | permit_line_breaks_in_identifier=lambda s: re.sub( 90 | r"(\.|::)", r"\1", s 91 | ), 92 | local_name_with_tooltip=lambda n: f'{ n.split(".")[-1] }', 93 | format_timestamp=lambda ts: ts.strftime("%Y:%m:%d-%H:%M:%S.%fZ") if ts else "", 94 | format_duration=lambda s: str(s), # makes it easier to change in the future 95 | len=len 96 | ) 97 | 98 | try: 99 | if len(transcript.sessions) == 1: 100 | self.write_single_session(transcript, base_context, dest) 101 | else: 102 | self.write_matrix_and_sessions(transcript, base_context, dest) 103 | 104 | except jinja2.exceptions.TemplateNotFound as ex: 105 | msg = f"ERROR: template '{ex}' not found.\n" 106 | msg += "Searched in the following directories:" 107 | for entry in self.template_path: 108 | msg += f"\n {entry}" 109 | fatal(msg) 110 | 111 | # One this worked, we can add the CSS file 112 | for path in self.template_path: 113 | css_candidate = os.path.join(path, 'static', 'feditest.css') 114 | if os.path.exists(css_candidate): 115 | shutil.copyfile(css_candidate, cssfile) 116 | break 117 | 118 | 119 | def write_single_session(self, transcript: TestRunTranscript, context: dict[str, Any], dest: str): 120 | run_session = transcript.sessions[0] 121 | context.update( 122 | transcript=transcript, 123 | run_session=run_session, 124 | summary=run_session.build_summary() # if we use 'summary', we can use shared/summary.jinja2 125 | ) 126 | with open(dest, "w") as fp: 127 | session_template = self.jinja2_env.get_template("session_single.jinja2") 128 | fp.write(session_template.render(**context)) 129 | 130 | 131 | def write_matrix_and_sessions(self, transcript: TestRunTranscript, context: dict[str, Any], dest: str): 132 | matrix_context = dict(context) 133 | matrix_context.update( 134 | transcript=transcript, 135 | session_file_path=lambda session: _derive_full_and_local_filename(dest, f'.{ session.run_session_index }.html')[1], 136 | summary=transcript.build_summary() # if we use 'summary', we can use shared/summary.jinja2 137 | ) 138 | with open(dest, "w") as fp: 139 | matrix_template = self.jinja2_env.get_template("matrix.jinja2") 140 | fp.write(matrix_template.render(**matrix_context)) 141 | 142 | for run_session in transcript.sessions: 143 | session_context = dict(context) 144 | session_context.update( 145 | transcript=transcript, 146 | run_session=run_session, 147 | summary=run_session.build_summary(), # if we use 'summary', we can use shared/summary.jinja2 148 | matrix_file_path=os.path.basename(dest) 149 | ) 150 | session_dest = _derive_full_and_local_filename(dest, f'.{ run_session.run_session_index }.html')[0] 151 | with open(session_dest, "w") as fp: 152 | session_template = self.jinja2_env.get_template("session_with_matrix.jinja2") 153 | fp.write(session_template.render(**session_context)) 154 | -------------------------------------------------------------------------------- /tests.unit/test_50_run_multistep_assertion_raises.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run a multi-step test that raises various failures in different steps 3 | """ 4 | 5 | import pytest 6 | 7 | import feditest 8 | from feditest import assert_that, step, test, InteropLevel, SpecLevel 9 | from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec 10 | from feditest.testrun import TestRun 11 | from feditest.testruncontroller import AutomaticTestRunController 12 | 13 | 14 | @pytest.fixture(scope="module", autouse=True) 15 | def init_node_drivers(): 16 | """ 17 | Cleanly define the NodeDrivers. 18 | """ 19 | feditest.all_node_drivers = {} 20 | feditest.load_default_node_drivers() 21 | 22 | 23 | @pytest.fixture(scope="module", autouse=True) 24 | def init_tests(): 25 | """ 26 | Cleanly define some tests. 27 | """ 28 | feditest.all_tests = {} 29 | feditest._registered_as_test = {} 30 | feditest._registered_as_test_step = {} 31 | feditest._loading_tests = True 32 | 33 | ## 34 | ## FediTest tests start here 35 | ## 36 | 37 | @test 38 | class Example: 39 | """ 40 | A multi-step test class that raises various failures in different steps. 41 | """ 42 | def __init__(self): 43 | pass 44 | 45 | 46 | @step 47 | def step01_default(self) -> None: 48 | assert_that(False, 'This was the default!') 49 | 50 | 51 | @step 52 | def step02_must(self) -> None: 53 | assert_that(False, 'This was MUST!', spec_level=SpecLevel.MUST) 54 | 55 | 56 | @step 57 | def step03_should(self) -> None: 58 | assert_that(False, 'This was SHOULD!', spec_level=SpecLevel.SHOULD) 59 | 60 | 61 | @step 62 | def step04_implied(self) -> None: 63 | assert_that(False, 'This was IMPLIED!', spec_level=SpecLevel.IMPLIED) 64 | 65 | 66 | @step 67 | def step05_problem(self) -> None: 68 | assert_that(False, 'This is PROBLEM!', interop_level=InteropLevel.PROBLEM) 69 | 70 | 71 | @step 72 | def step06_degraded(self) -> None: 73 | assert_that(False, 'This is DEGRADED!', interop_level=InteropLevel.DEGRADED) 74 | 75 | 76 | @step 77 | def step07_unaffected(self) -> None: 78 | assert_that(False, 'This is UNAFFECTED!', interop_level=InteropLevel.UNAFFECTED) 79 | 80 | 81 | @step 82 | def step08_unknown(self) -> None: 83 | assert_that(False, 'This is UNKNOWN!', interop_level=InteropLevel.UNKNOWN) 84 | 85 | 86 | @step 87 | def step09_must_problem(self) -> None: 88 | assert_that(False, 'This was MUST, PROBLEM!', spec_level=SpecLevel.MUST, interop_level=InteropLevel.PROBLEM) 89 | 90 | 91 | @step 92 | def step10_must_degraded(self) -> None: 93 | assert_that(False, 'This was MUST, DEGRADED!', spec_level=SpecLevel.MUST, interop_level=InteropLevel.DEGRADED) 94 | 95 | 96 | @step 97 | def step11_must_unaffected(self) -> None: 98 | assert_that(False, 'This was MUST, UNAFFECTED!', spec_level=SpecLevel.MUST, interop_level=InteropLevel.UNAFFECTED) 99 | 100 | 101 | @step 102 | def step12_must_unknown(self) -> None: 103 | assert_that(False, 'This was MUST, UNKNOWN!', spec_level=SpecLevel.MUST, interop_level=InteropLevel.UNKNOWN) 104 | 105 | 106 | @step 107 | def step13_should_problem(self) -> None: 108 | assert_that(False, 'This was SHOULD, PROBLEM!', spec_level=SpecLevel.SHOULD, interop_level=InteropLevel.PROBLEM) 109 | 110 | 111 | @step 112 | def step14_should_degraded(self) -> None: 113 | assert_that(False, 'This was SHOULD, DEGRADED!', spec_level=SpecLevel.SHOULD, interop_level=InteropLevel.DEGRADED) 114 | 115 | 116 | @step 117 | def step15_should_unaffected(self) -> None: 118 | assert_that(False, 'This was SHOULD, UNAFFECTED!', spec_level=SpecLevel.SHOULD, interop_level=InteropLevel.UNAFFECTED) 119 | 120 | 121 | @step 122 | def step16_should_unkown(self) -> None: 123 | assert_that(False, 'This was SHOULD, UNKNOWN!', spec_level=SpecLevel.SHOULD, interop_level=InteropLevel.UNKNOWN) 124 | 125 | 126 | @step 127 | def step17_implied_problem(self) -> None: 128 | assert_that(False, 'This was IMPLIED, PROBLEM!', spec_level=SpecLevel.IMPLIED, interop_level=InteropLevel.PROBLEM) 129 | 130 | 131 | @step 132 | def step18_implied_degraded(self) -> None: 133 | assert_that(False, 'This was IMPLIED, DEGRADED!', spec_level=SpecLevel.IMPLIED, interop_level=InteropLevel.DEGRADED) 134 | 135 | 136 | @step 137 | def step19_implied_unaffected(self) -> None: 138 | assert_that(False, 'This was IMPLIED, UNAFFECTED!', spec_level=SpecLevel.IMPLIED, interop_level=InteropLevel.UNAFFECTED) 139 | 140 | 141 | @step 142 | def step20_implied_unknown(self) -> None: 143 | assert_that(False, 'This was IMPLIED, UNKNOWN!', spec_level=SpecLevel.IMPLIED, interop_level=InteropLevel.UNKNOWN) 144 | 145 | 146 | @step 147 | def step21_unspecified_problem(self) -> None: 148 | assert_that(False, 'This was UNSPECIFIED, PROBLEM!', spec_level=SpecLevel.UNSPECIFIED, interop_level=InteropLevel.PROBLEM) 149 | 150 | 151 | @step 152 | def step22_unspecified_degraded(self) -> None: 153 | assert_that(False, 'This was UNSPECIFIED, DEGRADED!', spec_level=SpecLevel.UNSPECIFIED, interop_level=InteropLevel.DEGRADED) 154 | 155 | 156 | @step 157 | def step23_unspecified_unaffected(self) -> None: 158 | assert_that(False, 'This was UNSPECIFIED, UNAFFECTED!', spec_level=SpecLevel.UNSPECIFIED, interop_level=InteropLevel.UNAFFECTED) 159 | 160 | 161 | @step 162 | def step24_unspecified_unknown(self) -> None: 163 | assert_that(False, 'This was UNSPECIFIED, UNKNOWN!', spec_level=SpecLevel.UNSPECIFIED, interop_level=InteropLevel.UNKNOWN) 164 | 165 | ## 166 | ## FediTest tests end here 167 | ## (Don't forget the next two lines) 168 | ## 169 | 170 | feditest._loading_tests = False 171 | feditest._load_tests_pass2() 172 | 173 | 174 | @pytest.fixture(autouse=True) 175 | def test_plan_fixture() -> TestPlan: 176 | """ 177 | The test plan tests all known tests. 178 | """ 179 | 180 | constellation = TestPlanConstellation({}, 'No nodes needed') 181 | tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] 182 | session = TestPlanSessionTemplate(tests, "Test a test whose steps raises multiple AssertionFailures") 183 | ret = TestPlan(session, [ constellation ]) 184 | ret.properties_validate() 185 | return ret 186 | 187 | 188 | def test_run_testplan(test_plan_fixture: TestPlan): 189 | test_plan_fixture.check_can_be_executed() 190 | 191 | test_run = TestRun(test_plan_fixture) 192 | controller = AutomaticTestRunController(test_run) 193 | test_run.run(controller) 194 | 195 | transcript = test_run.transcribe() 196 | summary = transcript.build_summary() 197 | 198 | assert summary.n_total == 1 199 | assert summary.n_failed == 1 200 | assert summary.n_skipped == 0 201 | assert summary.n_errored == 0 202 | assert summary.n_passed == 0 203 | 204 | assert len(transcript.sessions) == 1 205 | assert len(transcript.sessions[0].run_tests) == 1 206 | assert len(transcript.sessions[0].run_tests[0].run_steps) == 1 # It never getst to the other steps 207 | 208 | assert transcript.sessions[0].run_tests[0].run_steps[0].result.type == 'AssertionFailure' 209 | assert transcript.sessions[0].run_tests[0].run_steps[0].result.spec_level == SpecLevel.MUST.name 210 | assert transcript.sessions[0].run_tests[0].run_steps[0].result.interop_level == InteropLevel.UNKNOWN.name 211 | -------------------------------------------------------------------------------- /src/feditest/nodedrivers/wordpress/ubos.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | 4 | import os 5 | from typing import cast 6 | 7 | from feditest.nodedrivers import ( 8 | Account, 9 | AccountManager, 10 | DefaultAccountManager, 11 | NonExistingAccount, 12 | Node, 13 | NodeConfiguration 14 | ) 15 | from feditest.nodedrivers.mastodon.ubos import MastodonUbosNodeConfiguration 16 | from feditest.nodedrivers.ubos import UbosNodeConfiguration, UbosNodeDriver 17 | from feditest.nodedrivers.wordpress import ( 18 | OAUTH_TOKEN_ACCOUNT_FIELD, 19 | ROLE_ACCOUNT_FIELD, 20 | ROLE_NON_EXISTING_ACCOUNT_FIELD, 21 | USERID_ACCOUNT_FIELD, 22 | USERID_NON_EXISTING_ACCOUNT_FIELD, 23 | WordPressAccount, 24 | WordPressPlusPluginsNode 25 | ) 26 | from feditest.protocols.fediverse import FediverseNonExistingAccount 27 | from feditest.reporting import trace 28 | from feditest.testplan import TestPlanConstellationNode, TestPlanNodeAccountField, TestPlanNodeNonExistingAccountField 29 | 30 | 31 | 32 | class WordPressUbosAccountManager(DefaultAccountManager): 33 | """ 34 | Knows how to provision new accounts in WordPress 35 | """ 36 | # Python 3.12 @override 37 | def set_node(self, node: Node) -> None: 38 | """ 39 | We override this so we can insert the admin account in the list of accounts, now that the Node has been instantiated. 40 | """ 41 | super().set_node(node) 42 | 43 | if not self._accounts_allocated_to_role and not self._accounts_not_allocated_to_role: 44 | config = cast(UbosNodeConfiguration, node.config) 45 | admin_account = WordPressAccount(None, config.admin_userid, None, 1) # We know this is account with internal identifier 1 46 | admin_account.set_node(node) 47 | self._accounts_not_allocated_to_role.append(admin_account) 48 | 49 | 50 | class WordPressPlusPluginsUbosNode(WordPressPlusPluginsNode): 51 | """ 52 | A WordPress+plugins Node running on UBOS. This means we know how to interact with it exactly. 53 | """ 54 | # Python 3.12 @override 55 | def provision_account_for_role(self, role: str | None = None) -> Account | None: 56 | trace('Provisioning new user') 57 | raise NotImplementedError('FIXME') 58 | 59 | 60 | def provision_non_existing_account_for_role(self, role: str | None = None) -> NonExistingAccount | None: 61 | raise NotImplementedError('FIXME') 62 | 63 | 64 | def add_cert_to_trust_store(self, root_cert: str) -> None: 65 | config = cast(UbosNodeConfiguration, self.config) 66 | node_driver = cast(WordPressPlusPluginsUbosNodeDriver, self.node_driver) 67 | 68 | node_driver.add_cert_to_trust_store_via(root_cert, config.rshcmd) 69 | 70 | 71 | def remove_cert_from_trust_store(self, root_cert: str) -> None: 72 | config = cast(UbosNodeConfiguration, self.config) 73 | node_driver = cast(WordPressPlusPluginsUbosNodeDriver, self.node_driver) 74 | 75 | node_driver.remove_cert_from_trust_store_via(root_cert, config.rshcmd) 76 | 77 | 78 | # Python 3.12 @override 79 | def _provision_oauth_token_for(self, account: WordPressAccount, oauth_client_id: str) -> str : 80 | # Code from here: https://wordpress.org/support/topic/programmatically-obtaining-oauth-token-for-testing/ 81 | # $desired_token = '123'; 82 | # $user_id = 1; 83 | # $oauth = new Enable_Mastodon_Apps\Mastodon_OAuth(); 84 | # $oauth->get_token_storage()->setAccessToken( $desired_token, $app->get_client_id(), $user_id, time() + HOUR_IN_SECONDS, $app->get_scopes() ); 85 | 86 | trace(f'Provisioning OAuth token on {self} for user with name="{ account.userid }".') 87 | config = cast(UbosNodeConfiguration, self.config) 88 | node_driver = cast(WordPressPlusPluginsUbosNodeDriver, self.node_driver) 89 | 90 | token = os.urandom(16).hex() 91 | php_script = f""" 92 | get_token_storage()->setAccessToken( "{ token }", "{ oauth_client_id }", { account.internal_userid }, time() + HOUR_IN_SECONDS, 'read write follow push' ); 99 | """ 100 | dir = f'/ubos/http/sites/{ config.siteid }' 101 | cmd = f'cd { dir } && sudo sudo -u http php' # from user ubosdev -> root -> http 102 | 103 | trace( f'PHP script is "{ php_script }"') 104 | result = node_driver._exec_shell(cmd, config.rshcmd, stdin_content=php_script, capture_output=True) 105 | if result.returncode: 106 | raise Exception(self, f'Failed to create OAuth token for user with id="{ account.userid }", cmd="{ cmd }", stdout="{ result.stdout}", stderr="{ result.stderr }"') 107 | return token 108 | 109 | 110 | class WordPressPlusPluginsUbosNodeDriver(UbosNodeDriver): 111 | """ 112 | Knows how to instantiate Mastodon via UBOS. 113 | """ 114 | # Python 3.12 @override 115 | @staticmethod 116 | def test_plan_node_account_fields() -> list[TestPlanNodeAccountField]: 117 | return [ USERID_ACCOUNT_FIELD, OAUTH_TOKEN_ACCOUNT_FIELD, ROLE_ACCOUNT_FIELD ] 118 | 119 | 120 | # Python 3.12 @override 121 | @staticmethod 122 | def test_plan_node_non_existing_account_fields() -> list[TestPlanNodeNonExistingAccountField]: 123 | return [ USERID_NON_EXISTING_ACCOUNT_FIELD, ROLE_NON_EXISTING_ACCOUNT_FIELD ] 124 | 125 | 126 | # Python 3.12 @override 127 | def create_configuration_account_manager(self, rolename: str, test_plan_node: TestPlanConstellationNode) -> tuple[NodeConfiguration, AccountManager | None]: 128 | accounts : list[Account] = [] 129 | if test_plan_node.accounts: 130 | for index, account_info in enumerate(test_plan_node.accounts): 131 | accounts.append(WordPressAccount.create_from_account_info_in_testplan( 132 | account_info, 133 | f'Constellation role "{ rolename }", NodeDriver "{ self }, Account { index }: ')) 134 | 135 | non_existing_accounts : list[NonExistingAccount] = [] 136 | if test_plan_node.non_existing_accounts: 137 | for index, non_existing_account_info in enumerate(test_plan_node.non_existing_accounts): 138 | non_existing_accounts.append(FediverseNonExistingAccount.create_from_non_existing_account_info_in_testplan( 139 | non_existing_account_info, 140 | f'Constellation role "{ rolename }", NodeDriver "{ self }, Non-existing account { index }: ')) 141 | 142 | # Once has the Node has been instantiated (we can't do that here yet): if the user did not specify at least one Account, we add the admin account 143 | 144 | return ( 145 | MastodonUbosNodeConfiguration.create_from_node_in_testplan( 146 | test_plan_node, 147 | self, 148 | appconfigjson = { 149 | "appid" : "wordpress", 150 | "accessoryids" : [ 151 | "wordpress-plugin-activitypub", 152 | "wordpress-plugin-enable-mastodon-apps", 153 | "wordpress-plugin-friends", 154 | "wordpress-plugin-webfinger" 155 | ], 156 | "context" : "", 157 | "customizationpoints" : { 158 | "wordpress" : { 159 | "disablessrfprotection" : { 160 | "value" : True 161 | } 162 | } 163 | } 164 | }, 165 | defaults = { 166 | 'app' : 'WordPress+plugins' 167 | }), 168 | WordPressUbosAccountManager(accounts, non_existing_accounts) 169 | ) 170 | 171 | # Python 3.12 @override 172 | def _instantiate_ubos_node(self, rolename: str, config: UbosNodeConfiguration, account_manager: AccountManager) -> Node: 173 | return WordPressPlusPluginsUbosNode(rolename, config, account_manager) 174 | --------------------------------------------------------------------------------