├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug.yml │ ├── config.yml │ ├── doc.yml │ ├── feature_request.yml │ └── maintenance.yml ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── scripts │ └── get_latest_changelog.py └── workflows │ ├── auto_approve.yml │ ├── code_quality.yml │ ├── codeql.yml │ ├── release_bump.yml │ └── release_publish.yml ├── .gitignore ├── .semantic_release └── CHANGELOG.md.j2 ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DEVELOPMENT.md ├── LICENSE ├── NOTICE ├── README.md ├── VERIFYING_PGP_SIGNATURE.md ├── hatch.toml ├── hatch_version_hook.py ├── pipeline ├── build.sh └── publish.sh ├── pyproject.toml ├── requirements-development.txt ├── requirements-release.txt ├── requirements-testing.txt ├── scripts └── add_copyright_headers.sh ├── src └── openjd │ ├── __main__.py │ └── cli │ ├── __init__.py │ ├── _check │ ├── __init__.py │ └── _check_command.py │ ├── _common │ ├── __init__.py │ ├── _extensions.py │ ├── _job_from_template.py │ └── _validation_utils.py │ ├── _create_argparser.py │ ├── _run │ ├── __init__.py │ ├── _local_session │ │ ├── __init__.py │ │ ├── _actions.py │ │ ├── _logs.py │ │ └── _session_manager.py │ └── _run_command.py │ ├── _schema │ ├── __init__.py │ └── _schema_command.py │ ├── _summary │ ├── __init__.py │ ├── _summary_command.py │ └── _summary_output.py │ └── py.typed └── test └── openjd ├── cli ├── __init__.py ├── conftest.py ├── templates │ ├── basic.yaml │ ├── basic_dependency_job.yaml │ ├── chunked_job.yaml │ ├── env_1.yaml │ ├── env_2.yaml │ ├── env_fails_enter.yaml │ ├── env_fails_exit.yaml │ ├── env_with_param.yaml │ ├── job_sleep_exit_normal.yaml │ ├── job_with_test_steps.yaml │ ├── redacted_env.yaml │ ├── simple_with_j_param.yaml │ └── simple_with_j_param_exit_1.yaml ├── test_check_command.py ├── test_chunked_job.py ├── test_common.py ├── test_local_session.py ├── test_redacted_env.py ├── test_run_command.py ├── test_run_with_env.py ├── test_schema_command.py └── test_summary_command.py └── test_main.py /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @OpenJobDescription/Developers -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug.yml: -------------------------------------------------------------------------------- 1 | name: "\U0001F41B Bug Report" 2 | description: Report a bug 3 | title: "Bug: (short bug description)" 4 | labels: ["bug", "needs triage"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thank you for taking the time to fill out this bug report! 10 | 11 | ⚠️ If the bug that you are reporting is a security-related issue or security vulnerability, 12 | then please do not create a report via this template. Instead please 13 | notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) 14 | or directly via email to [AWS Security](aws-security@amazon.com). 15 | 16 | - type: textarea 17 | id: description 18 | attributes: 19 | label: Describe the bug 20 | description: What is the problem? A clear and concise description of the bug. 21 | validations: 22 | required: true 23 | 24 | - type: textarea 25 | id: expected_behaviour 26 | attributes: 27 | label: Expected Behaviour 28 | description: What did you expect to happen? 29 | validations: 30 | required: true 31 | 32 | - type: textarea 33 | id: current_behaviour 34 | attributes: 35 | label: Current Behaviour 36 | description: What actually happened? Please include as much detail as you can. 37 | validations: 38 | required: true 39 | 40 | - type: textarea 41 | id: reproduction_steps 42 | attributes: 43 | label: Reproduction Steps 44 | description: | 45 | Please provide as much detail as you can to help us understand how we can reproduce the bug. 46 | Step by step instructions and self-contained code snippets are ideal. 47 | validations: 48 | required: true 49 | 50 | - type: textarea 51 | id: environment 52 | attributes: 53 | label: Environment 54 | description: Please provide information on the environment and software versions that you are using to reproduce the bug. 55 | value: | 56 | At minimum: 57 | 1. Operating system (e.g. Windows Server 2022; Amazon Linux 2023; etc.) 58 | 2. Output of `python3 --version` 59 | 3. Version of your openjd-cli package (see: `pip show openjd-cli`): 60 | 61 | Please share other details about your environment that you think might be relevant to reproducing the bug. 62 | validations: 63 | required: true 64 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/doc.yml: -------------------------------------------------------------------------------- 1 | 2 | name: "📕 Documentation Issue" 3 | description: Issue in the documentation 4 | title: "Docs: (short description of the issue)" 5 | labels: ["documentation", "needs triage"] 6 | body: 7 | - type: textarea 8 | id: documentation_issue 9 | attributes: 10 | label: Documentation Issue 11 | description: Describe the issue 12 | validations: 13 | required: true -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.yml: -------------------------------------------------------------------------------- 1 | name: "\U0001F680 Feature Request" 2 | description: Request a new feature 3 | title: "Feature request: (short description of the feature)" 4 | labels: ["enhancement", "needs triage"] 5 | body: 6 | - type: textarea 7 | id: problem 8 | attributes: 9 | label: Describe the problem 10 | description: | 11 | Help us understand the problem that you are trying to solve, and why it is important to you. 12 | Provide as much detail as you are able. 13 | validations: 14 | required: true 15 | 16 | - type: textarea 17 | id: proposed_solution 18 | attributes: 19 | label: Proposed Solution 20 | description: | 21 | Describe your proposed feature that you see solving this problem for you. If you have a 22 | full or partial prototype implementation then please open a draft pull request and link to 23 | it here as well. 24 | validations: 25 | required: true 26 | 27 | - type: textarea 28 | id: use_case 29 | attributes: 30 | label: Example Use Cases 31 | description: | 32 | Provide some sample code snippets or shell scripts that show how **you** would use this feature as 33 | you have proposed it. 34 | validations: 35 | required: true 36 | 37 | 38 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/maintenance.yml: -------------------------------------------------------------------------------- 1 | name: "🛠️ Maintenance" 2 | description: Some type of improvement 3 | title: "Maintenance: (short description of the issue)" 4 | labels: ["maintenance", "needs triage"] 5 | body: 6 | - type: textarea 7 | id: description 8 | attributes: 9 | label: Description 10 | description: Describe the improvement and why it is important to do. 11 | validations: 12 | required: true 13 | - type: textarea 14 | id: solution 15 | attributes: 16 | label: Solution 17 | description: Provide any ideas you have for how the suggestion can be implemented. 18 | validations: 19 | required: true 20 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Fixes: ** 2 | 3 | ### What was the problem/requirement? (What/Why) 4 | 5 | ### What was the solution? (How) 6 | 7 | ### What is the impact of this change? 8 | 9 | ### How was this change tested? 10 | 11 | See [DEVELOPMENT.md](https://github.com/OpenJobDescription/openjd-cli/blob/mainline/DEVELOPMENT.md#testing) for information on running tests. 12 | 13 | - Have you run the tests? 14 | 15 | ### Was this change documented? 16 | 17 | - Are relevant docstrings in the code base updated? 18 | 19 | ### Is this a breaking change? 20 | 21 | A breaking change is one that modifies a public contract in a way that is not backwards compatible. See the 22 | [Public Interfaces](https://github.com/OpenJobDescription/openjd-cli/blob/mainline/DEVELOPMENT.md#the-packages-public-interface) section 23 | of the DEVELOPMENT.md for more information on the public contracts. 24 | 25 | If so, then please describe the changes that users of this package must make to update their scripts, or Python applications. 26 | 27 | ### Does this change impact security? 28 | 29 | - Does the change need to be threat modeled? For example, does it create or modify files/directories that must only be readable by the process owner? 30 | - If so, then please label this pull request with the "security" label. We'll work with you to analyze the threats. 31 | 32 | ---- 33 | 34 | *By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.* -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | day: "monday" 13 | commit-message: 14 | prefix: "chore(deps):" 15 | - package-ecosystem: "github-actions" 16 | directory: "/" # Location of package manifests 17 | schedule: 18 | interval: "weekly" 19 | day: "monday" 20 | commit-message: 21 | prefix: "chore(github):" -------------------------------------------------------------------------------- /.github/scripts/get_latest_changelog.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | """ 3 | This script gets the changelog notes for the latest version of this package. It makes the following assumptions 4 | 1. A file called CHANGELOG.md is in the current directory that has the changelog 5 | 2. The changelog file is formatted in a way such that level 2 headers are: 6 | a. The only indication of the beginning of a version's changelog notes. 7 | b. Always begin with `## ` 8 | 3. The changelog file contains the newest version's changelog notes at the top of the file. 9 | 10 | Example CHANGELOG.md: 11 | ``` 12 | ## 1.0.0 (2024-02-06) 13 | 14 | ### BREAKING CHANGES 15 | * **api**: rename all APIs 16 | 17 | ## 0.1.0 (2024-02-06) 18 | 19 | ### Features 20 | * **api**: add new api 21 | ``` 22 | 23 | Running this script on the above CHANGELOG.md should return the following contents: 24 | ``` 25 | ## 1.0.0 (2024-02-06) 26 | 27 | ### BREAKING CHANGES 28 | * **api**: rename all APIs 29 | 30 | ``` 31 | """ 32 | import re 33 | 34 | h2 = r"^##\s.*$" 35 | with open("CHANGELOG.md") as f: 36 | contents = f.read() 37 | matches = re.findall(h2, contents, re.MULTILINE) 38 | changelog = contents[: contents.find(matches[1]) - 1] if len(matches) > 1 else contents 39 | print(changelog) 40 | -------------------------------------------------------------------------------- /.github/workflows/auto_approve.yml: -------------------------------------------------------------------------------- 1 | name: Dependabot auto-approve 2 | on: pull_request 3 | 4 | permissions: 5 | pull-requests: write 6 | 7 | jobs: 8 | dependabot: 9 | runs-on: ubuntu-latest 10 | if: ${{ github.actor == 'dependabot[bot]' }} 11 | steps: 12 | - name: Dependabot metadata 13 | id: metadata 14 | uses: dependabot/fetch-metadata@v2 15 | with: 16 | github-token: "${{ secrets.GITHUB_TOKEN }}" 17 | - name: Approve a PR 18 | run: gh pr review --approve "$PR_URL" 19 | env: 20 | PR_URL: ${{ github.event.pull_request.html_url }} 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/code_quality.yml: -------------------------------------------------------------------------------- 1 | name: Code Quality 2 | 3 | on: 4 | pull_request: 5 | branches: [ mainline, release ] 6 | workflow_call: 7 | inputs: 8 | branch: 9 | required: false 10 | type: string 11 | 12 | jobs: 13 | Test: 14 | name: Python 15 | strategy: 16 | matrix: 17 | os: [ubuntu-latest, windows-latest, macos-latest] 18 | python-version: ['3.9', '3.10', '3.11', '3.12'] 19 | uses: OpenJobDescription/.github/.github/workflows/reusable_python_build.yml@mainline 20 | with: 21 | os: ${{ matrix.os }} 22 | python-version: ${{ matrix.python-version }} 23 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ "mainline" ] 6 | pull_request: 7 | branches: [ "mainline" ] 8 | schedule: 9 | - cron: '0 8 * * MON' 10 | 11 | jobs: 12 | Analysis: 13 | name: Analysis 14 | uses: OpenJobDescription/.github/.github/workflows/reusable_codeql.yml@mainline 15 | -------------------------------------------------------------------------------- /.github/workflows/release_bump.yml: -------------------------------------------------------------------------------- 1 | name: "Release: Bump" 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | force_version_bump: 7 | required: false 8 | default: "" 9 | type: choice 10 | options: 11 | - "" 12 | - patch 13 | - minor 14 | - major 15 | 16 | concurrency: 17 | group: release 18 | 19 | jobs: 20 | UnitTests: 21 | name: Unit Tests 22 | uses: ./.github/workflows/code_quality.yml 23 | with: 24 | branch: mainline 25 | 26 | Bump: 27 | name: Version Bump 28 | needs: UnitTests 29 | uses: OpenJobDescription/.github/.github/workflows/reusable_bump.yml@mainline 30 | secrets: inherit 31 | with: 32 | force_version_bump: ${{ inputs.force_version_bump }} -------------------------------------------------------------------------------- /.github/workflows/release_publish.yml: -------------------------------------------------------------------------------- 1 | name: "Release: Publish" 2 | run-name: "Release: ${{ github.event.head_commit.message }}" 3 | 4 | on: 5 | push: 6 | branches: 7 | - mainline 8 | paths: 9 | - CHANGELOG.md 10 | 11 | concurrency: 12 | group: release 13 | 14 | jobs: 15 | Publish: 16 | name: Publish Release 17 | permissions: 18 | id-token: write 19 | contents: write 20 | uses: OpenJobDescription/.github/.github/workflows/reusable_publish.yml@mainline 21 | secrets: inherit 22 | # PyPI does not support reusable workflows yet 23 | # # See https://github.com/pypi/warehouse/issues/11096 24 | PublishToPyPI: 25 | needs: Publish 26 | runs-on: ubuntu-latest 27 | environment: release 28 | permissions: 29 | id-token: write 30 | steps: 31 | - name: Checkout 32 | uses: actions/checkout@v4 33 | with: 34 | ref: release 35 | fetch-depth: 0 36 | - name: Set up Python 37 | uses: actions/setup-python@v5 38 | with: 39 | python-version: '3.9' 40 | - name: Install dependencies 41 | run: | 42 | pip install --upgrade hatch 43 | - name: Build 44 | run: hatch -v build 45 | # # See https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-pypi 46 | - name: Publish to PyPI 47 | uses: pypa/gh-action-pypi-publish@release/v1 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *# 3 | *.swp 4 | 5 | *.DS_Store 6 | 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | /.coverage 12 | /.coverage.* 13 | /.cache 14 | /.pytest_cache 15 | /.mypy_cache 16 | /.ruff_cache 17 | /.attach_pid* 18 | /.venv 19 | 20 | /doc/_apidoc/ 21 | /build 22 | /dist 23 | _version.py -------------------------------------------------------------------------------- /.semantic_release/CHANGELOG.md.j2: -------------------------------------------------------------------------------- 1 | {% for version, release in context.history.released.items() %} 2 | ## {{ version.as_tag() }} ({{ release.tagged_date.strftime("%Y-%m-%d") }}) 3 | 4 | {% if "breaking" in release["elements"] %} 5 | ### BREAKING CHANGES 6 | {% for commit in release["elements"]["breaking"] %} 7 | * {% if commit.scope %}**{{ commit.scope }}**: {% endif %}{{ commit.commit.summary[commit.commit.summary.find(": ")+1:].strip() }} ([`{{ commit.short_hash }}`]({{ commit.commit.hexsha | commit_hash_url }})) 8 | {% endfor %} 9 | {% endif %} 10 | 11 | {% if "features" in release["elements"] %} 12 | ### Features 13 | {% for commit in release["elements"]["features"] %} 14 | * {% if commit.scope %}**{{ commit.scope }}**: {% endif %}{{ commit.commit.summary[commit.commit.summary.find(": ")+1:].strip() }} ([`{{ commit.short_hash }}`]({{ commit.commit.hexsha | commit_hash_url }})) 15 | {% endfor %} 16 | {% endif %} 17 | 18 | {% if "bug fixes" in release["elements"] %} 19 | ### Bug Fixes 20 | {% for commit in release["elements"]["bug fixes"] %} 21 | * {% if commit.scope %}**{{ commit.scope }}**: {% endif %}{{ commit.commit.summary[commit.commit.summary.find(":")+1:].strip() }} ([`{{ commit.short_hash }}`]({{ commit.commit.hexsha | commit_hash_url }})) 22 | {% endfor %} 23 | {% endif %} 24 | 25 | {% endfor %} -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.7.0 (2025-03-19) 2 | 3 | ### BREAKING CHANGES 4 | * The functions generate_jobs and job_from_template have changed to accept the environments and return the job parameters 5 | alongside the job. Constructing the LocalSession class now requires job parameters. 6 | 7 | ### Bug Fixes 8 | * Running jobs with env templates does not support parameters ([`65a15b4`](https://github.com/OpenJobDescription/openjd-cli/commit/65a15b414bbd90a77c6b63451f5f052db2b8fcf8)) 9 | 10 | ## 0.6.1 (2025-03-13) 11 | 12 | 13 | 14 | ### Bug Fixes 15 | * `openjd run` command runs sessions twice ([`16ee6ae`](https://github.com/OpenJobDescription/openjd-cli/commit/16ee6ae63900b752cbaec7f1e68e758a64a56c99)) 16 | 17 | ## 0.6.0 (2025-03-07) 18 | 19 | ### BREAKING CHANGES 20 | * The logging output has changed to use relative timestamps by default, and print more messages about the job and steps that are running. 21 | 22 | ### Features 23 | * Support adaptive chunking, general CLI improvement ([`664d008`](https://github.com/OpenJobDescription/openjd-cli/commit/664d0083c0e9d2d973a88e1e630e2af6cef67cc1)) 24 | 25 | ## 0.5.1 (2025-02-26) 26 | 27 | ### Features 28 | 29 | * Update to use Pydantic V2, and support task chunking (#134) ([`ad53f68`](https://github.com/OpenJobDescription/openjd-cli/pull/134/commits/ad53f689117d98273fb034916bcdd250e49ccffd)) 30 | 31 | ## 0.5.0 (2024-11-13) 32 | 33 | 34 | ### Features 35 | * **deps**: Update openjd-model to 0.5.* and openjd-session to 0.9.* (#118) ([`ef97dd8`](https://github.com/OpenJobDescription/openjd-cli/commit/ef97dd81e9dfc2e45a9c2605ae256b679961cd7b)) 36 | 37 | 38 | ## 0.4.4 (2024-07-23) 39 | 40 | 41 | 42 | ### Bug Fixes 43 | * run command exits when an action reaches timeout (#97) ([`d031a91`](https://github.com/OpenJobDescription/openjd-cli/commit/d031a91ee6d1796b33701c466dc52f5615ead3c6)) 44 | * run subcommand now exits all entered environments (#98) ([`f6a54b2`](https://github.com/OpenJobDescription/openjd-cli/commit/f6a54b20d5058b4c144b237c960212d9e9c5d2be)) 45 | 46 | ## 0.4.3 (2024-04-16) 47 | 48 | ### Documenation 49 | * Windows is no longer marked as experimental in documentation. 50 | 51 | 52 | ## 0.4.2 (2024-03-11) 53 | 54 | 55 | ### Features 56 | * update to openjd-sessions 0.7.* (#66) ([`4d44db9`](https://github.com/OpenJobDescription/openjd-cli/commit/4d44db92e1c2e2e6aa1aa326bccc45ea8a5a31d4)) 57 | 58 | 59 | ## 0.4.1 (2024-02-21) 60 | 61 | 62 | ### Features 63 | * add template-debugging options to `openjd run` (#52) ([`80e8cb9`](https://github.com/OpenJobDescription/openjd-cli/commit/80e8cb9f12392dbd2e89c2bb850853640f2dc706)) 64 | 65 | 66 | ## 0.4.0 (2024-02-13) 67 | 68 | ### BREAKING CHANGES 69 | * public release (#41) ([`88dd089`](https://github.com/OpenJobDescription/openjd-cli/commit/88dd089848422b54acf99e1d69fbeacb61691676)) 70 | 71 | 72 | 73 | ## 0.3.0 (2024-02-12) 74 | 75 | ### BREAKING CHANGES 76 | * modifying how tasks are selected in run command (#37) ([`59c41d9`](https://github.com/OpenJobDescription/openjd-cli/commit/59c41d90eda95e666e49c37d1fdfe0d570742b32)) 77 | * Update openjd-cli to pass template_dir/cwd to preprocess_job_parameters (#29) ([`0983e1d`](https://github.com/OpenJobDescription/openjd-cli/commit/0983e1d0e3cece60ef825ec2d1b86dc20f2da22d)) 78 | 79 | ### Features 80 | * Allow job parameters as JSON string (#34) ([`8708b2c`](https://github.com/OpenJobDescription/openjd-cli/commit/8708b2ced5945465fd6706d95eac0bb1ac6317ca)) 81 | * support environment templates (#30) ([`b845d70`](https://github.com/OpenJobDescription/openjd-cli/commit/b845d70944863c11c308b50669cbdf99d037eeb3)) 82 | 83 | ### Bug Fixes 84 | * improve missing job parameter error for summary command (#44) ([`975863a`](https://github.com/OpenJobDescription/openjd-cli/commit/975863a7097d536ca786561e0adec665bb0eec77)) 85 | * Allow job parameter values to be empty strings. (#26) ([`b8959d0`](https://github.com/OpenJobDescription/openjd-cli/commit/b8959d077cc5cd4697f101ba3e45adceddb4ccac)) 86 | 87 | ## 0.2.0 (2023-11-06) 88 | 89 | ### BREAKING CHANGES 90 | * accept path mapping rules as per schema (#16) ([`a9d71fd`](https://github.com/OpenJobDescription/openjd-cli/commit/a9d71fd0ddba50cda9a6edeec93ad1cf3ce67fbd)) 91 | 92 | 93 | 94 | ## 0.1.4 (2023-11-01) 95 | 96 | 97 | 98 | ### Bug Fixes 99 | * add entrypoint to packaging (#14) ([`55cc90a`](https://github.com/OpenJobDescription/openjd-cli/commit/55cc90a58ec85271c4b84d392ddc627225a8bde9)) 100 | 101 | ## 0.1.3 (2023-10-27) 102 | 103 | 104 | 105 | 106 | ## 0.1.1 (2023-09-14) 107 | 108 | 109 | 110 | ### Bug Fixes 111 | * add back cli entrypoint (#8) ([`10188dd`](https://github.com/OpenJobDescription/openjd-cli/commit/10188ddf971dc51b043994858719fe91bbce8a68)) 112 | 113 | ## 0.1.0 (2023-09-12) 114 | 115 | * Initial import from internal repository 116 | 117 | 118 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | Table of contents: 10 | 11 | * [Reporting Bugs/Feature Requests](#reporting-bugsfeature-requests) 12 | * [Development](#development) 13 | * [Finding contributions to work on](#finding-contributions-to-work-on) 14 | * [Talk with us first](#talk-with-us-first) 15 | * [Contributing via Pull Requests](#contributing-via-pull-requests) 16 | * [Conventional Commits](#conventional-commits) 17 | * [Licensing](#licensing) 18 | 19 | ## Reporting Bugs/Feature Requests 20 | 21 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 22 | 23 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 24 | reported the issue. Please try to include as much information as you can. 25 | 26 | ## Development 27 | 28 | We welcome you to contribute features and bug fixes via a [pull request](https://help.github.com/articles/creating-a-pull-request/). 29 | If you are new to contributing to GitHub repositories, then you may find the 30 | [GitHub documentation on collaborating with the fork and pull model](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/getting-started/about-collaborative-development-models#fork-and-pull-model) 31 | informative; this is the model that we follow. 32 | 33 | Please see [DEVELOPMENT.md](./DEVELOPMENT.md) for information about how to navigate this package's 34 | code base and development practices. 35 | 36 | ### Finding contributions to work on 37 | 38 | If you are not sure what you would like to contribute, then looking at the existing issues is a great way to find 39 | something to contribute on. Looking at 40 | [issues that have the "help wanted" or "good first issue" labels](https://github.com/OpenJobDescription/openjd-cli/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22%2C%22help+wanted%22) 41 | are a good place to start, but please dive into any issue that interests you whether it has those labels or not. 42 | 43 | ### Talk with us first 44 | 45 | We ask that you please [open a feature request issue](https://github.com/OpenJobDescription/openjd-cli/issues/new/choose) 46 | (if one does not already exist) and talk with us before posting a pull request that contains a significant amount of work, 47 | or one that proposes a change to a public interface such as to the interface of a publicly exported Python function or to 48 | the command-line interfaces' commands or arguments. We want to make sure that your time and effort is respected by working 49 | with you to design the change before you spend much of your time on it. If you want to create a draft pull request to show what 50 | you are thinking and then talk with us, then that works with us as well. 51 | 52 | We prefer that this package contain primarily features that are useful to many users of it, rather than features that are specific 53 | to niche workflows. If you have a feature in mind, but are not sure whether it is niche or not then please 54 | [open a feature request issue](https://github.com/OpenJobDescription/openjd-cli/issues/new/choose). We will do our best to help 55 | you make that assessment, and posting a public issue will help others find your feature idea and add their support if they 56 | would also find it useful. 57 | 58 | ### Contributing via Pull Requests 59 | 60 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 61 | 62 | 1. You are working against the latest source on the *mainline* branch. 63 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 64 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 65 | 4. Your pull request will be focused on a single change - it is easier for us to understand when a change is focused rather 66 | than changing multiple things at once. 67 | 68 | To send us a pull request, please: 69 | 70 | 1. Fork the repository. 71 | 2. Modify the source and add tests for your change; please focus on the specific change you are contributing. 72 | If you also reformat all the code, it will be hard for us to focus on your change. 73 | Please see [DEVELOPMENT.md](./DEVELOPMENT.md) for tips. 74 | 3. Ensure tests pass. Please see the [Testing](./DEVELOPMENT.md#testing) section for information on tests. 75 | 4. Commit to your fork using clear commit messages. Note that all AWS Deadline Cloud GitHub repositories require the use 76 | of [conventional commit](#conventional-commits) syntax for the title of your commit. 77 | 5. Send us a pull request, answering any default questions in the pull request interface. 78 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 79 | 80 | GitHub provides additional documentation on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 81 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 82 | 83 | ### Conventional commits 84 | 85 | The commits in this repository are all required to use [conventional commit syntax](https://www.conventionalcommits.org/en/v1.0.0/) 86 | in their title to help us identify the kind of change that is being made, automatically generate the changelog, and 87 | automatically identify next release version number. Only the first commit that deviates from mainline in your pull request 88 | must adhere to this requirement. 89 | 90 | We ask that you use these commit types in your commit titles: 91 | 92 | * `feat` - When the pull request adds a new feature or functionality; 93 | * `fix` - When the pull request is implementing a fix to a bug; 94 | * `test` - When the pull request is only implementing an addition or change to tests or the testing infrastructure; 95 | * `docs` - When the pull request is primarily implementing an addition or change to the package's documentation; 96 | * `refactor` - When the pull request is implementing only a refactor of existing code; 97 | * `ci` - When the pull request is implementing a change to the CI infrastructure of the packge; 98 | * `chore` - When the pull request is a generic maintenance task. 99 | 100 | We also require that the type in your conventional commit title end in an exclaimation point (e.g. `feat!` or `fix!`) 101 | if the pull request should be considered to be a breaking change in some way. Please also include a "BREAKING CHANGE" footer 102 | in the description of your commit in this case ([example](https://www.conventionalcommits.org/en/v1.0.0/#commit-message-with-both--and-breaking-change-footer)). 103 | Examples of breaking changes include any that implements a backwards-imcompatible change to a public Python interface, 104 | the command-line interface, or the like. 105 | 106 | If you need change a commit message, then please see the 107 | [GitHub documentation on the topic](https://docs.github.com/en/pull-requests/committing-changes-to-your-project/creating-and-editing-commits/changing-a-commit-message) 108 | to guide you. 109 | 110 | ## Licensing 111 | 112 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 113 | -------------------------------------------------------------------------------- /DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | # Development documentation 2 | 3 | This documentation provides guidance on developer workflows for working with the code in this repository. 4 | 5 | Table of Contents: 6 | * [Development Environment Setup](#development-environment-setup) 7 | * [The Development Loop](#the-development-loop) 8 | * [Testing](#testing) 9 | * [Writing tests](#writing-tests) 10 | * [Running tests](#running-tests) 11 | * [Things to Know](#things-to-know) 12 | * [Package's Public Interface](#the-packages-public-interface) 13 | 14 | ## Development Environment Setup 15 | 16 | To develop the Python code in this repository you will need: 17 | 18 | 1. Python 3.9 or higher. We recommend [mise](https://github.com/jdx/mise) if you would like to run more than one version 19 | of Python on the same system. When running unit tests against all supported Python versions, for instance. 20 | 2. The [hatch](https://github.com/pypa/hatch) package installed (`pip install --upgrade hatch`) into your Python environment. 21 | 22 | You can develop on a Linux, MacOs, or Windows workstation, but you will find that some of the support scripting is specific to 23 | Linux workstations. 24 | 25 | ## The Development Loop 26 | 27 | We have configured [hatch](https://github.com/pypa/hatch) commands to support a standard development loop. You can run the following 28 | from any directory of this repository: 29 | 30 | * `hatch build` - To build the installable Python wheel and sdist packages into the `dist/` directory. 31 | * `hatch run test` - To run the PyTest unit tests found in the `test/` directory. See [Testing](#testing). 32 | * `hatch run all:test` - To run the PyTest unit tests against all available supported versions of Python. 33 | * `hatch run lint` - To check that the package's formatting adheres to our standards. 34 | * `hatch run fmt` - To automatically reformat all code to adhere to our formatting standards. 35 | * `hatch shell` - Enter a shell environment where you can run the `deadline` command-line directly as it is implemented in your 36 | checked-out local git repository. 37 | * `hatch env prune` - Delete all of your isolated workspace [environments](https://hatch.pypa.io/1.12/environment/) 38 | for this package. 39 | 40 | If you are not sure about how to approach development for this package, then we have some suggestions. 41 | 42 | 1. Run python within a `hatch shell` environment for interactive development. Python will import your in-development 43 | codebase when you `import openjd.cli` from this environment. This makes it easy to use interactive python, the python 44 | debugger, and short test scripts to develop and test your changes. 45 | * Note that if you make changes to your source and are running interactive Python then you will need to use 46 | [importlib.reload](https://docs.python.org/3/library/importlib.html#importlib.reload) to reload the the module(s) that 47 | you modified for your modifications to take effect. 48 | 2. Run the test suite frequently (See [Testing](#testing)), and modify/add to it as you are developing your change, rather than 49 | only when your change is complete. The test suite runs very quickly, and this will help surface regressions that your change may 50 | cause before you get too far into your implementation. 51 | 52 | Once you are satisfied with your code, and all relevant tests pass, then run `hatch run fmt` to fix up the formatting of 53 | your code and post your pull request. 54 | 55 | Note: Hatch uses [environments](https://hatch.pypa.io/1.12/environment/) to isolate the Python development workspace 56 | for this package from your system or virtual environment Python. If your build/test run is not making sense, then 57 | sometimes pruning (`hatch env prune`) all of these environments for the package can fix the issue. 58 | 59 | ## Testing 60 | 61 | The objective for the tests of this package are to act as regression tests to help identify unintended changes to 62 | functionality in the package. As such, we strive to have high test coverage of the different behaviours/functionality 63 | that the package contains. Code coverage metrics are not the goal, but rather are a guide to help identify places 64 | where there may be gaps in testing coverage. 65 | 66 | All tests are all located under the `test/` directory of this repository. If you are adding or modifying 67 | functionality, then you will almost always want to be writing one or more tests to demonstrate that your 68 | logic behaves as expected and that future changes do not accidentally break your change. 69 | 70 | ### Writing Tests 71 | 72 | If you want assistance developing tests, then please don't hesitate to open a draft pull request and ask for help. 73 | We'll do our best to help you out and point you in the right direction. We also suggest looking at the existing tests 74 | for the same or similar functions for inspiration (search for calls to the function within the `test/` 75 | subdirectories). You will also find both the official [PyTest documentation](https://docs.pytest.org/en/stable/) 76 | and [unitest.mock documentation](https://docs.python.org/3.8/library/unittest.mock.html) very informative (we do). 77 | 78 | Our tests are implemented using the [PyTest](https://docs.pytest.org/en/stable/) testing framework, 79 | and tests make use of Python's [unittest.mock](https://docs.python.org/3.8/library/unittest.mock.html) 80 | package to avoid runtime dependencies and narrowly focus tests on a specific aspect of the implementation. 81 | 82 | Though our tests make a lot of use of `unittest.mock` now, our goal is to decrease the usage of mocks to a bare minimum 83 | over time. Using a mock inherrently encodes assumptions into the tests about how the mocked functionality functions. So, 84 | if a change is made that violates those assumptions then the test suite will not catch it, and we may end up releasing broken code. 85 | 86 | ### Running Tests 87 | 88 | You can run tests with: 89 | 90 | * `hatch run test` - To run the tests with your default Python runtime. 91 | * `hatch run all:test` - To run the tests with all of the supported Python runtime versions that you have installed. 92 | 93 | Any arguments that you add to these commands are passed through to PyTest. So, if you want to, say, run the 94 | [Python debugger](https://docs.python.org/3/library/pdb.html) to investigate a test failure then you can run: `hatch run test --pdb` 95 | 96 | ### Super verbose test output 97 | 98 | If you find that you need much more information from a failing test (say you're debugging a 99 | deadlocking test) then a way to get verbose output from the test is to enable Pytest 100 | [Live Logging](https://docs.pytest.org/en/latest/how-to/logging.html#live-logs): 101 | 102 | 1. Add a `pytest.ini` to the root directory of the repository that contains (Note: for some reason, 103 | setting `log_cli` and `log_cli_level` in `pyproject.toml` does not work for us, nor does setting the options 104 | on the command-line; if you figure out how to get it to work then please update this section): 105 | ``` 106 | [pytest] 107 | xfail_strict = False 108 | log_cli = true 109 | log_cli_level = 10 110 | ``` 111 | 2. Modify `pyproject.toml` to set the following additional `addopts` in the `tool.pytest.ini_options` section: 112 | ``` 113 | "-vvvvv", 114 | "--numprocesses=1" 115 | ``` 116 | 3. Add logging statements to your tests as desired and run the test(s) that you are debugging. 117 | 118 | ## Things to Know 119 | 120 | ### The Package's Public Interface 121 | 122 | This package is an application wherein we are explicit and intentional with what we expose as public. 123 | The command-line interface is, obviously, public as that is the purpose of the application. There are 124 | no public Python interfaces; this package is not intended to be used as a library. 125 | 126 | The standard convention in Python is to prefix things with an underscore character ('_') to 127 | signify that the thing is private to the implementation, and is not intended to be used by 128 | external consumers of the thing. 129 | 130 | We use this convention in this package in two ways: 131 | 132 | 1. In filenames. 133 | 1. Any file whose name is not prefixed with an underscore **is** a part of the public 134 | interface of this package. The name may not change and public symbols (classes, modules, 135 | functions, etc.) defined in the file may not be moved to other files or renamed without a 136 | major version number change. 137 | 2. Any file whose name is prefixed with an underscore is an internal module of the package 138 | and is not part of the public interface. These files can be renamed, refactored, have symbols 139 | renamed, etc. Any symbol defined in one of these files that is intended to be part of this 140 | package's public interface must be imported into an appropriate `__init__.py` file. 141 | 2. Every symbol that is defined or imported in a public module and is not intended to be part 142 | of the module's public interface is prefixed with an underscore. 143 | 144 | For example, a public module in this package will be defined with the following style: 145 | 146 | ```python 147 | # The os module is not part of this file's external interface 148 | import os as _os 149 | 150 | # PublicClass is part of this file's external interface. 151 | class PublicClass: 152 | def publicmethod(self): 153 | pass 154 | 155 | def _privatemethod(self): 156 | pass 157 | 158 | # _PrivateClass is not part of this file's external interface. 159 | class _PrivateClass: 160 | def publicmethod(self): 161 | pass 162 | 163 | def _privatemethod(self): 164 | pass 165 | ``` 166 | 167 | #### On `import os as _os` 168 | 169 | Every module/symbol that is imported into a Python module becomes a part of that module's interface. 170 | Thus, if we have a module called `foo.py` such as: 171 | 172 | ```python 173 | # foo.py 174 | 175 | import os 176 | ``` 177 | 178 | Then, the `os` module becomes part of the public interface for `foo.py` and a consumer of that module 179 | is free to do: 180 | 181 | ```python 182 | from foo import os 183 | ``` 184 | 185 | We don't want all (generally, we don't want any) of our imports to become part of the public API for 186 | the module, so we import modules/symbols into a public module with the following style: 187 | 188 | ```python 189 | import os as _os 190 | from typing import Dict as _Dict 191 | ``` -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Open Job Description - CLI 2 | 3 | [![pypi](https://img.shields.io/pypi/v/openjd-cli.svg)](https://pypi.python.org/pypi/openjd-cli) 4 | [![python](https://img.shields.io/pypi/pyversions/openjd-cli.svg?style=flat)](https://pypi.python.org/pypi/openjd-cli) 5 | [![license](https://img.shields.io/pypi/l/openjd-cli.svg?style=flat)](https://github.com/OpenJobDescription/openjd-cli/blob/mainline/LICENSE) 6 | 7 | Open Job Description (OpenJD) is a flexible open specification for defining render jobs which are portable 8 | between studios and render solutions. This package provides a command-line interface that can be used 9 | to validate OpenJD templates, run OpenJD jobs locally, and more. 10 | 11 | For more information about Open Job Description and our goals with it, please see the 12 | Open Job Description [Wiki on GitHub](https://github.com/OpenJobDescription/openjd-specifications/wiki). 13 | 14 | ## Compatibility 15 | 16 | This library requires: 17 | 18 | 1. Python 3.9 or higher; 19 | 2. Linux, MacOS, or Windows operating system; 20 | 3. On Linux/MacOS: 21 | * `sudo` 22 | 23 | ## Versioning 24 | 25 | This package's version follows [Semantic Versioning 2.0](https://semver.org/), but is still considered to be in its 26 | initial development, thus backwards incompatible versions are denoted by minor version bumps. To help illustrate how 27 | versions will increment during this initial development stage, they are described below: 28 | 29 | 1. The MAJOR version is currently 0, indicating initial development. 30 | 2. The MINOR version is currently incremented when backwards incompatible changes are introduced to the public API. 31 | 3. The PATCH version is currently incremented when bug fixes or backwards compatible changes are introduced to the public API. 32 | 33 | ## Contributing 34 | 35 | We encourage all contributions to this package. Whether it's a bug report, new feature, correction, or additional 36 | documentation, we greatly value feedback and contributions from our community. 37 | 38 | Please see [CONTRIBUTING.md](./CONTRIBUTING.md) for our contributing guidelines. 39 | 40 | ## Commands 41 | 42 | ### Getting Help 43 | 44 | The main `openjd` command and all subcommands support a `--help` option to display 45 | information on how to use the command. 46 | 47 | ### `check` 48 | 49 | Validates, or reports any syntax errors that appear in the schema of a Job Template file. 50 | 51 | #### Arguments 52 | 53 | |Name|Type|Required|Description|Example| 54 | |---|---|---|---|---| 55 | |`path`|path|yes|A path leading to a Job or Environment template file.|`/path/to/template.json`| 56 | |`--output`|string|no|How to display the results of the command. Allowed values are `human-readable` (default), `json`, and `yaml`.|`--output json`, `--output yaml`| 57 | 58 | #### Example 59 | ```sh 60 | $ openjd check /path/to/job.template.json 61 | 62 | Template at '/path/to/job.template.json' passes validation checks! 63 | ``` 64 | 65 | ### `summary` 66 | 67 | Displays summary information about a sample Job or Step, and the Steps and Tasks therein. The user may provide parameters to 68 | customize the Job, as parameters can have an impact on the amount of Steps and Tasks that a job consists of. 69 | 70 | #### Arguments 71 | 72 | |Name|Type|Required| Description |Example| 73 | |---|---|---|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---| 74 | |`path`|path|yes| A path leading to a Job template file. |`/path/to/job.template.json`| 75 | |`--job-param`, `-p`|string, path|no| The values for the job template's parameters. Can be provided as key-value pairs, inline JSON string, or path(s) to a JSON or YAML document. If provided more than once then the given values are combined in the order that they appear. |`--job-param MyParam=5`, `-p file://parameter_file.json`, `-p '{"MyParam": "5"}'`| 76 | |`--step`|string|no| The name of the Step to summarize. |`--step Step1`| 77 | |`--output`|string|no| How to display the results of the command. Allowed values are `human-readable` (default), `json`, and `yaml`. |`--output json`, `--output yaml`| 78 | 79 | #### Example 80 | ```sh 81 | $ openjd summary /path/to/job.template.json \ 82 | --job-param JobName=SampleJob \ 83 | --job-param '{"FileToRender": "sample.blend"}' \ 84 | --job-param file://some_more_parameters.json 85 | 86 | --- Summary for 'SampleJob' --- 87 | 88 | Parameters: 89 | - JobName (STRING): SampleJob 90 | - FileToRender (PATH): sample.blend 91 | - AnotherParameter (INT): 10 92 | 93 | Total steps: 2 94 | Total tasks: 15 95 | Total environments: 0 96 | 97 | --- Steps in 'SampleJob' --- 98 | 99 | 1. 'Step1' 100 | 1 Task parameter(s) 101 | 10 total Tasks 102 | 103 | 2. 'Step2' 104 | 2 Task parameter(s) 105 | 5 total Tasks 106 | ``` 107 | 108 | ### `run` 109 | 110 | Given a Job Template, Job Parameters, and optional Environment Templates this will run a set of the Tasks 111 | from the constructed Job locally within an OpenJD Sesssion. 112 | 113 | Please see [How Jobs Are Run](https://github.com/OpenJobDescription/openjd-specifications/wiki/How-Jobs-Are-Run) for 114 | details on how Open Job Description's Jobs are run within Sessions. 115 | 116 | #### Arguments 117 | 118 | |Name|Type|Required| Description |Example| 119 | |---|---|---|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---| 120 | |`path`|path|yes| A path leading to a Job template file. |`/path/to/job.template.json`| 121 | |`--step`|string|yes| The name of the Step to run in a local Session. |`--step Step1`| 122 | |`--environment`|paths|no| Path to a file containing Environment Template definitions. Can be provided multiple times. |`--environment /path/to/env.template1.json --environment /path/to/env.template2.yaml`| 123 | |`--job-param`, `-p`|string, path|no| The values for the job template's parameters. Can be provided as key-value pairs, inline JSON string, or as path(s) to a JSON or YAML document. If provided more than once then the given values are combined in the order that they appear. |`--job-param MyParam=5`, `-p file://parameter_file.json`, `-p '{"MyParam": "5"}'`| 124 | |`--task-param`, `-tp`|string|no| Instructs the command to run a single task in a Session with the given value for one of the task parameters. The option must be provided once for each task parameter defined for the Step, with each instance providing the value for a different task parameter. Mutually exclusive with `--tasks` and `--maximum-tasks`. |`-tp MyParam=5 -tp MyOtherParam=Foo`| 125 | |`--tasks`|string, path|no| Instructs the command to run one or more tasks for the Step in a Session. The argument must be either the filename of a JSON or YAML file containing an array of maps from task parameter name to value; or an inlined JSON string of the same. Mutually exclusive with `--task-param/-tp` and `--maximum-tasks`. |`--tasks '[{"MyParam": 5}]'`, `--tasks file://parameter_set_file.json`| 126 | |`--maximum-tasks`|integer|no| A maximum number of Tasks to run from this Step. Unless present, the Session will run all Tasks defined in the Step's parameter space or the Task(s) selected by the `--task-param` or `--tasks` arguments. Mutually exclusive with `--task-param/-tp` and `--tasks`. |`--maximum-tasks 5`| 127 | |`--run-dependencies`|flag|no| If present, runs all of a Step's dependencies in the Session prior to the Step itself. |`--run-dependencies`| 128 | |`--path-mapping-rules`|string, path|no| The path mapping rules to apply to the template. Should be a JSON-formatted list of Open Job Description path mapping rules, provided as a string or a path to a JSON/YAML document prefixed with 'file://'. |`--path-mapping-rules [{"source_os": "Windows", "source_path": "C:\test", "destination_path": "/mnt/test"}]`, `--path-mapping-rules file://rules_file.json`| 129 | |`--preserve`|flag|no| If present, the Session's working directory will not be deleted when the run is completed. |`--preserve`| 130 | |`--verbose`|flag|no| If present, then verbose logging will be enabled in the Session's log. |`--verbose`| 131 | |`--output`|string|no| How to display the results of the command. Allowed values are `human-readable` (default), `json`, and `yaml`. |`--output json`, `--output yaml`| 132 | 133 | #### Example 134 | ```sh 135 | $ openjd run /path/to/job.template.json --step Step1 \ 136 | --job-param PingServer=amazon.com \ 137 | --task-param PingCount=20 \ 138 | --task-param PingDelay=30 139 | 140 | # ... Task logs accompanied by timestamps ... 141 | 142 | --- Results of local session --- 143 | 144 | Session ended successfully 145 | Job: MyJob 146 | Step: Step1 147 | Duration: 1.0 seconds 148 | Chunks run: 1 149 | 150 | ``` 151 | 152 | ### `schema` 153 | Returns the Open Job Description model as a JSON schema document body. 154 | 155 | #### Arguments 156 | 157 | |Name|Type|Required|Description|Example| 158 | |---|---|---|---|---| 159 | |`--version`|string|yes|The specification version to get the JSON schema for.|`--version jobtemplate-2023-09`| 160 | |`--output`|string|no|How to display the results of the command. Allowed values are `human-readable` (default), `json`, and `yaml`.|`--output json`, `--output yaml`| 161 | 162 | #### Example 163 | ```sh 164 | $ openjd schema --version jobtemplate-2023-09 165 | 166 | { 167 | "title": "JobTemplate", 168 | # ... JSON body corresponding to the Open Job Description model schema ... 169 | } 170 | ``` 171 | 172 | ## Downloading 173 | 174 | You can download this package from: 175 | - [PyPI](https://pypi.org/project/openjd-cli/) 176 | - [GitHub releases](https://github.com/OpenJobDescription/openjd-cli/releases) 177 | 178 | ### Verifying GitHub Releases 179 | 180 | See [VERIFYING_PGP_SIGNATURE](VERIFYING_PGP_SIGNATURE.md) for more information. 181 | 182 | ## Security 183 | 184 | We take all security reports seriously. When we receive such reports, we will 185 | investigate and subsequently address any potential vulnerabilities as quickly 186 | as possible. If you discover a potential security issue in this project, please 187 | notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) 188 | or directly via email to [AWS Security](aws-security@amazon.com). Please do not 189 | create a public GitHub issue in this project. 190 | 191 | ## License 192 | 193 | This project is licensed under the Apache-2.0 License. 194 | 195 | -------------------------------------------------------------------------------- /VERIFYING_PGP_SIGNATURE.md: -------------------------------------------------------------------------------- 1 | ### Verifying GitHub Releases 2 | 3 | You can verify the authenticity of the release artifacts using the `gpg` command line tool. 4 | 5 | 1) Download the desired release artifacts from the GitHub releases page. Make sure to download the corresponding PGP signature file (ending with `.sig`) as well. 6 | For example, if you would like to verify your download of the wheel for version `1.2.3`, you should have the following files downloaded: 7 | ``` 8 | openjd_cli-1.2.3-py3-none-any.whl 9 | openjd_cli-1.2.3-py3-none-any.whl.sig 10 | ``` 11 | 12 | 2) Install the `gpg` command line tool. The installation process varies by operating system. Please refer to the GnuPG website for instructions: https://gnupg.org/download/ 13 | 14 | 3) Save the following contents to a file called `openjobdescription-pgp.asc`: 15 | ``` 16 | -----BEGIN PGP PUBLIC KEY BLOCK----- 17 | 18 | mQINBGXGjx0BEACdChrQ/nch2aYGJ4fxHNQwlPE42jeHECqTdlc1V/mug+7qN7Pc 19 | C4NQk4t68Y72WX/NG49gRfpAxPlSeNt18c3vJ9/sWTukmonWYGK0jQGnDWjuVgFT 20 | XtvJAAQBFilQXN8h779Th2lEuD4bQX+mGB7l60Xvh7vIehE3C4Srbp6KJXskPLPo 21 | dz/dx7a+GXRiyYCYbGX4JziXSjQZRc0tIaxLn/GDm7VnXpdHcUk3qJitree61oC8 22 | agtRHCH5s56E8wt8fXzyStElMkFIZsoLDlLp5lFqT81En9ho/+K6RLBkIj0mC8G7 23 | BafpHKlxkrIgNK3pWACL93GE6xihqwkZMCAeqloVvkOTdfAKDHuDSEHwKxHG3cZ1 24 | /e1YhtkPMVF+NMeoQavykUGVUT1bRoVNdk6bYsnbUjUI1A+JNf6MqvdRJyckZqEC 25 | ylkBekBp/SFpFHvQkRCpfVizm2GSrjdZKgXpm1ZlQJyMRVzc/XPbqdSWhz52r3IC 26 | eudwReHDc+6J5rs6tg3NbFfPVfCBMSqHlu1HRewWAllIp1+y6nfL4U3iEsUvZ1Y6 27 | IV3defHIP3kNPU14ZWf3G5rvJDZrIRnjoWhDcaVmivmB/cSdDzphL5FovSI8dsPm 28 | iU/JZGQb3EvZq+nl4pOiK32hETJ/fgCCzgUA3WqGeFNUNSI9KYZgBe6daQARAQAB 29 | tDRPcGVuIEpvYiBEZXNjcmlwdGlvbiA8b3BlbmpvYmRlc2NyaXB0aW9uQGFtYXpv 30 | bi5jb20+iQJXBBMBCABBFiEEvBcWYrv5OB7Tl2sZovDwWbzECYcFAmXGjx0DGy8E 31 | BQkDwmcABQsJCAcCAiICBhUKCQgLAgMWAgECHgcCF4AACgkQovDwWbzECYcSHRAA 32 | itPYx48xnJiT6tfnult9ZGivhcXhrMlvirVYOqEtRrt0l18sjr84K8mV71eqFwMx 33 | GS7e4iQP6guqW9biQfMA5/Id8ZjE7jNbF0LUGsY6Ktj+yOlAbTR+x5qr7Svb7oEs 34 | TMB/l9HBZ1WtIRzcUk9XYqzvYQr5TT997A63F28u32RchJ+5ECAz4g/p91aWxwVo 35 | HIfN10sGzttoukJCzC10CZAVscJB+nnoUbB/o3bPak6GUxBHpMgomb0K5g4Z4fXY 36 | 4AZ9jKFoLgNcExdwteiUdSEnRorZ5Ny8sP84lwJziD3wuamVUsZ1C/KiQJBGTp5e 37 | LUY38J1oIwptw5fqjaAq2GQxEaIknWQ4fr3ZvNYUuGUt5FbHe5U5XF34gC8PK7v7 38 | bT/7sVdZZzKFScDLfH5N36M5FrXfTaXsVbfrRoa2j7U0kndyVEZyJsKVAQ8vgwbJ 39 | w/w2hKkyQLAg3l5yO5CHLGatsfSIzea4WoOAaroxiNtL9gzVXzqpw6qPEsH9hsws 40 | HsPEQWXHmDQvFTNUU14qic1Vc5fyxCBXIAGAPBd20b+219XznJ5uBKUgtvnqcItj 41 | nMYe6Btxh+pjrTA15X/p81z6sB7dkL1hPHfawLhCEzJbIPyyBTQYqY00/ap4Rj7t 42 | kzSiyzBejniFfAZ6eYBWsej7uXUsVndBF1ggZynPTeE= 43 | =iaEm 44 | -----END PGP PUBLIC KEY BLOCK----- 45 | ``` 46 | 47 | 4) Import the OpenPGP key for Open Job Description by running the following command: 48 | 49 | ``` 50 | gpg --import --armor openjobdescription-pgp.asc 51 | ``` 52 | 53 | 5) Determine whether to trust the OpenPGP key. Some factors to consider when deciding whether or not to trust the above key are: 54 | 55 | - The internet connection you’ve used to obtain the GPG key from this website is secure 56 | - The device that you are accessing this website on is secure 57 | 58 | If you have decided to trust the OpenPGP key, then edit the key to trust with `gpg` like the following example: 59 | ``` 60 | $ gpg --edit-key A2F0F059BCC40987 61 | gpg (GnuPG) 2.0.22; Copyright (C) 2013 Free Software Foundation, Inc. 62 | This is free software: you are free to change and redistribute it. 63 | There is NO WARRANTY, to the extent permitted by law. 64 | 65 | 66 | pub 4096R/BCC40987 created: 2024-02-09 expires: 2026-02-08 usage: SCEA 67 | trust: unknown validity: unknown 68 | [ unknown] (1). Open Job Description 69 | 70 | gpg> trust 71 | pub 4096R/BCC40987 created: 2024-02-09 expires: 2026-02-08 usage: SCEA 72 | trust: unknown validity: unknown 73 | [ unknown] (1). Open Job Description 74 | 75 | Please decide how far you trust this user to correctly verify other users' keys 76 | (by looking at passports, checking fingerprints from different sources, etc.) 77 | 78 | 1 = I don't know or won't say 79 | 2 = I do NOT trust 80 | 3 = I trust marginally 81 | 4 = I trust fully 82 | 5 = I trust ultimately 83 | m = back to the main menu 84 | 85 | Your decision? 5 86 | Do you really want to set this key to ultimate trust? (y/N) y 87 | 88 | pub 4096R/BCC40987 created: 2024-02-09 expires: 2026-02-08 usage: SCEA 89 | trust: ultimate validity: unknown 90 | [ unknown] (1). Open Job Description 91 | Please note that the shown key validity is not necessarily correct 92 | unless you restart the program. 93 | 94 | gpg> quit 95 | ``` 96 | 97 | 6) Verify the signature of the Open Job Description release via `gpg --verify`. The command for verifying the example files from step 1 would be: 98 | 99 | ``` 100 | gpg --verify ./openjd_cli-1.2.3-py3-none-any.whl.sig ./openjd_cli-1.2.3-py3-none-any.whl 101 | ``` -------------------------------------------------------------------------------- /hatch.toml: -------------------------------------------------------------------------------- 1 | [envs.default] 2 | pre-install-commands = [ 3 | "pip install -r requirements-testing.txt" 4 | ] 5 | 6 | [envs.default.scripts] 7 | sync = "pip install -r requirements-testing.txt" 8 | test = "pytest --cov-config pyproject.toml {args:test}" 9 | typing = "mypy {args:src test}" 10 | style = [ 11 | "ruff check {args:.}", 12 | "black --check --diff {args:.}", 13 | ] 14 | fmt = [ 15 | "black {args:.}", 16 | "style", 17 | ] 18 | lint = [ 19 | "style", 20 | "typing", 21 | ] 22 | 23 | [[envs.all.matrix]] 24 | python = ["3.9", "3.10", "3.11", "3.12", "3.13"] 25 | 26 | [envs.codebuild.scripts] 27 | build = "hatch build" 28 | 29 | [envs.release] 30 | detached = true 31 | 32 | [envs.release.scripts] 33 | deps = "pip install -r requirements-release.txt" 34 | bump = "semantic-release -v --strict version --no-push --no-commit --no-tag --skip-build {args}" 35 | version = "semantic-release -v --strict version --print {args}" 36 | -------------------------------------------------------------------------------- /hatch_version_hook.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import shutil 4 | import sys 5 | 6 | from dataclasses import dataclass 7 | from hatchling.builders.hooks.plugin.interface import BuildHookInterface 8 | from typing import Any, Optional 9 | 10 | 11 | _logger = logging.Logger(__name__, logging.INFO) 12 | _stdout_handler = logging.StreamHandler(sys.stdout) 13 | _stdout_handler.addFilter(lambda record: record.levelno <= logging.INFO) 14 | _stderr_handler = logging.StreamHandler(sys.stderr) 15 | _stderr_handler.addFilter(lambda record: record.levelno > logging.INFO) 16 | _logger.addHandler(_stdout_handler) 17 | _logger.addHandler(_stderr_handler) 18 | 19 | 20 | @dataclass 21 | class CopyConfig: 22 | sources: list[str] 23 | destinations: list[str] 24 | 25 | 26 | class CustomBuildHookException(Exception): 27 | pass 28 | 29 | 30 | class CustomBuildHook(BuildHookInterface): 31 | """ 32 | A Hatch build hook that is pulled in automatically by Hatch's "custom" hook support 33 | See: https://hatch.pypa.io/1.6/plugins/build-hook/custom/ 34 | This build hook copies files from one location (sources) to another (destinations). 35 | Config options: 36 | - `log_level (str)`: The logging level. Any value accepted by logging.Logger.setLevel is allowed. Default is INFO. 37 | - `copy_map (list[dict])`: A list of mappings of files to copy and the destinations to copy them into. In TOML files, 38 | this is expressed as an array of tables. See https://toml.io/en/v1.0.0#array-of-tables 39 | Example TOML config: 40 | ``` 41 | [tool.hatch.build.hooks.custom] 42 | path = "hatch_hook.py" 43 | log_level = "DEBUG" 44 | [[tool.hatch.build.hooks.custom.copy_map]] 45 | sources = [ 46 | "_version.py", 47 | ] 48 | destinations = [ 49 | "src/openjd", 50 | ] 51 | [[tool.hatch.build.hooks.custom.copy_map]] 52 | sources = [ 53 | "something_the_tests_need.py", 54 | "something_else_the_tests_need.ini", 55 | ] 56 | destinations = [ 57 | "test/openjd", 58 | ] 59 | ``` 60 | """ 61 | 62 | REQUIRED_OPTS = [ 63 | "copy_map", 64 | ] 65 | 66 | def initialize(self, version: str, build_data: dict[str, Any]) -> None: 67 | if not self._prepare(): 68 | return 69 | 70 | for copy_cfg in self.copy_map: 71 | _logger.info(f"Copying {copy_cfg.sources} to {copy_cfg.destinations}") 72 | for destination in copy_cfg.destinations: 73 | for source in copy_cfg.sources: 74 | copy_func = shutil.copy if os.path.isfile(source) else shutil.copytree 75 | copy_func( 76 | os.path.join(self.root, source), 77 | os.path.join(self.root, destination), 78 | ) 79 | _logger.info("Copy complete") 80 | 81 | def clean(self, versions: list[str]) -> None: 82 | if not self._prepare(): 83 | return 84 | 85 | for copy_cfg in self.copy_map: 86 | _logger.info(f"Cleaning {copy_cfg.sources} from {copy_cfg.destinations}") 87 | cleaned_count = 0 88 | for destination in copy_cfg.destinations: 89 | for source in copy_cfg.sources: 90 | source_path = os.path.join(self.root, destination, source) 91 | remove_func = os.remove if os.path.isfile(source_path) else os.rmdir 92 | try: 93 | remove_func(source_path) 94 | except FileNotFoundError: 95 | _logger.debug(f"Skipping {source_path} because it does not exist...") 96 | else: 97 | cleaned_count += 1 98 | _logger.info(f"Cleaned {cleaned_count} items") 99 | 100 | def _prepare(self) -> bool: 101 | missing_required_opts = [ 102 | opt for opt in self.REQUIRED_OPTS if opt not in self.config or not self.config[opt] 103 | ] 104 | if missing_required_opts: 105 | _logger.warn( 106 | f"Required options {missing_required_opts} are missing or empty. " 107 | "Contining without copying sources to destinations...", 108 | file=sys.stderr, 109 | ) 110 | return False 111 | 112 | log_level = self.config.get("log_level") 113 | if log_level: 114 | _logger.setLevel(log_level) 115 | 116 | return True 117 | 118 | @property 119 | def copy_map(self) -> Optional[list[CopyConfig]]: 120 | raw_copy_map: list[dict] = self.config.get("copy_map") 121 | if not raw_copy_map: 122 | return None 123 | 124 | if not ( 125 | isinstance(raw_copy_map, list) 126 | and all(isinstance(copy_cfg, dict) for copy_cfg in raw_copy_map) 127 | ): 128 | raise CustomBuildHookException( 129 | f'"copy_map" config option is a nonvalid type. Expected list[dict], but got {raw_copy_map}' 130 | ) 131 | 132 | def verify_list_of_file_paths(file_paths: Any, config_name: str): 133 | if not (isinstance(file_paths, list) and all(isinstance(fp, str) for fp in file_paths)): 134 | raise CustomBuildHookException( 135 | f'"{config_name}" config option is a nonvalid type. Expected list[str], but got {file_paths}' 136 | ) 137 | 138 | missing_paths = [ 139 | fp for fp in file_paths if not os.path.exists(os.path.join(self.root, fp)) 140 | ] 141 | if len(missing_paths) > 0: 142 | raise CustomBuildHookException( 143 | f'"{config_name}" config option contains some file paths that do not exist: {missing_paths}' 144 | ) 145 | 146 | copy_map: list[CopyConfig] = [] 147 | for copy_cfg in raw_copy_map: 148 | destinations: list[str] = copy_cfg.get("destinations") 149 | verify_list_of_file_paths(destinations, "destinations") 150 | 151 | sources: list[str] = copy_cfg.get("sources") 152 | verify_list_of_file_paths(sources, "source") 153 | 154 | copy_map.append(CopyConfig(sources, destinations)) 155 | 156 | return copy_map 157 | -------------------------------------------------------------------------------- /pipeline/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Set the -e option 3 | set -e 4 | 5 | pip install --upgrade pip 6 | pip install --upgrade hatch 7 | pip install --upgrade twine 8 | hatch run codebuild:lint 9 | hatch run codebuild:test 10 | hatch run codebuild:build -------------------------------------------------------------------------------- /pipeline/publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Set the -e option 3 | set -e 4 | 5 | ./pipeline/build.sh 6 | twine upload --repository codeartifact dist/* --verbose -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling", "hatch-vcs"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "openjd-cli" 7 | authors = [ 8 | {name = "Amazon Web Services"}, 9 | ] 10 | dynamic = ["version"] 11 | readme = "README.md" 12 | license = "Apache-2.0" 13 | requires-python = ">=3.9" 14 | description = "Provides a command-line interface for working with Open Job Description templates." 15 | # https://pypi.org/classifiers/ 16 | classifiers = [ 17 | "Development Status :: 5 - Production/Stable", 18 | "Programming Language :: Python", 19 | "Programming Language :: Python :: 3 :: Only", 20 | "Programming Language :: Python :: 3.9", 21 | "Programming Language :: Python :: 3.10", 22 | "Programming Language :: Python :: 3.11", 23 | "Programming Language :: Python :: 3.12", 24 | "Operating System :: POSIX :: Linux", 25 | "Operating System :: Microsoft :: Windows", 26 | "Operating System :: MacOS", 27 | "License :: OSI Approved :: Apache Software License", 28 | "Intended Audience :: Developers", 29 | "Intended Audience :: End Users/Desktop" 30 | ] 31 | dependencies = [ 32 | "openjd-sessions >= 0.10.3,< 0.11", 33 | "openjd-model >= 0.8,< 0.9" 34 | ] 35 | 36 | [project.urls] 37 | Homepage = "https://github.com/OpenJobDescription/openjd-cli" 38 | Source = "https://github.com/OpenJobDescription/openjd-cli" 39 | 40 | 41 | [project.scripts] 42 | openjd = "openjd.cli:main" 43 | 44 | 45 | [tool.hatch.build] 46 | artifacts = [ 47 | "*_version.py", 48 | ] 49 | 50 | [tool.hatch.version] 51 | source = "vcs" 52 | 53 | [tool.hatch.version.raw-options] 54 | version_scheme = "post-release" 55 | 56 | [tool.hatch.build.hooks.vcs] 57 | version-file = "_version.py" 58 | 59 | [tool.hatch.build.hooks.custom] 60 | path = "hatch_version_hook.py" 61 | 62 | [[tool.hatch.build.hooks.custom.copy_map]] 63 | sources = [ 64 | "_version.py", 65 | ] 66 | destinations = [ 67 | "src/openjd/cli", 68 | ] 69 | 70 | [tool.hatch.build.targets.sdist] 71 | include = [ 72 | "src/openjd", 73 | "hatch_version_hook.py", 74 | ] 75 | 76 | [tool.hatch.build.targets.wheel] 77 | packages = [ 78 | "src/openjd", 79 | ] 80 | only-include = [ 81 | "src/openjd", 82 | ] 83 | 84 | [tool.hatch.build.targets.wheel.force-include] 85 | "src/openjd/__main__.py" = "openjd/__main__.py" 86 | 87 | [tool.mypy] 88 | check_untyped_defs = false 89 | show_error_codes = false 90 | pretty = true 91 | ignore_missing_imports = true 92 | disallow_incomplete_defs = false 93 | disallow_untyped_calls = false 94 | show_error_context = true 95 | strict_equality = false 96 | python_version = "3.9" 97 | warn_redundant_casts = true 98 | warn_unused_configs = true 99 | warn_unused_ignores = false 100 | # Tell mypy that there's a namespace package at src/openjd 101 | namespace_packages = true 102 | explicit_package_bases = true 103 | mypy_path = "src" 104 | 105 | # See: https://docs.pydantic.dev/mypy_plugin/ 106 | # - Helps mypy understand pydantic typing. 107 | plugins = "pydantic.mypy" 108 | 109 | [tool.ruff] 110 | line-length = 100 111 | 112 | [tool.ruff.lint] 113 | ignore = [ 114 | "E501", 115 | # Double Check if this should be fixed 116 | "E731", 117 | ] 118 | 119 | [tool.ruff.lint.pep8-naming] 120 | classmethod-decorators = [ 121 | "classmethod", 122 | # pydantic decorators are classmethod decorators 123 | # suppress N805 errors on classes decorated with them 124 | "pydantic.validator", 125 | "pydantic.root_validator", 126 | ] 127 | 128 | [tool.ruff.lint.isort] 129 | known-first-party = [ 130 | "openjd", 131 | ] 132 | 133 | [tool.black] 134 | line-length = 100 135 | 136 | [tool.pytest.ini_options] 137 | xfail_strict = false 138 | addopts = [ 139 | "-rfEx", 140 | "--durations=5", 141 | "--cov=src/openjd/cli", 142 | "--color=yes", 143 | "--cov-report=html:build/coverage", 144 | "--cov-report=xml:build/coverage/coverage.xml", 145 | "--cov-report=term-missing", 146 | "--numprocesses=auto", 147 | "--timeout=60" 148 | ] 149 | 150 | 151 | [tool.coverage.run] 152 | branch = true 153 | parallel = true 154 | 155 | 156 | [tool.coverage.paths] 157 | source = [ 158 | "src/" 159 | ] 160 | 161 | [tool.coverage.report] 162 | show_missing = true 163 | fail_under = 92 164 | omit= [ 165 | "src/openjd/cli/_version.py" 166 | ] 167 | 168 | [tool.semantic_release] 169 | # Can be removed or set to true once we are v1 170 | major_on_zero = false 171 | tag_format = "{version}" 172 | 173 | [tool.semantic_release.commit_parser_options] 174 | allowed_tags = [ 175 | "build", 176 | "chore", 177 | "ci", 178 | "docs", 179 | "feat", 180 | "fix", 181 | "perf", 182 | "style", 183 | "refactor", 184 | "test", 185 | ] 186 | minor_tags = [] 187 | patch_tags = [ 188 | "chore", 189 | "feat", 190 | "fix", 191 | "refactor", 192 | ] 193 | 194 | [tool.semantic_release.publish] 195 | upload_to_vcs_release = false 196 | 197 | [tool.semantic_release.changelog] 198 | template_dir = ".semantic_release" 199 | 200 | [tool.semantic_release.changelog.environment] 201 | trim_blocks = true 202 | lstrip_blocks = true 203 | 204 | [tool.semantic_release.branches.release] 205 | match = "(mainline|release)" 206 | -------------------------------------------------------------------------------- /requirements-development.txt: -------------------------------------------------------------------------------- 1 | hatch == 1.14.* 2 | hatch-vcs == 0.5.* -------------------------------------------------------------------------------- /requirements-release.txt: -------------------------------------------------------------------------------- 1 | python-semantic-release == 9.21.* 2 | -------------------------------------------------------------------------------- /requirements-testing.txt: -------------------------------------------------------------------------------- 1 | coverage[toml] == 7.* 2 | pytest == 8.3.* 3 | pytest-cov == 6.1.* 4 | pytest-timeout == 2.4.* 5 | pytest-xdist == 3.7.* 6 | types-PyYAML == 6.* 7 | black == 25.* 8 | ruff == 0.11.* 9 | mypy == 1.16.* 10 | -------------------------------------------------------------------------------- /scripts/add_copyright_headers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | if [ $# -eq 0 ]; then 5 | echo "Usage: add-copyright-headers ..." >&2 6 | exit 1 7 | fi 8 | 9 | for file in "$@"; do 10 | if ! head -1 | grep 'Copyright ' "$file" >/dev/null; then 11 | case "$file" in 12 | *.java) 13 | CONTENT=$(cat "$file") 14 | cat > "$file" </dev/null; then 23 | CONTENT=$(tail -n +2 "$file") 24 | cat > "$file" < 27 | $CONTENT 28 | EOF 29 | else 30 | CONTENT=$(cat "$file") 31 | cat > "$file" < 33 | $CONTENT 34 | EOF 35 | fi 36 | ;; 37 | *.py) 38 | CONTENT=$(cat "$file") 39 | cat > "$file" < "$file" < "$file" < "$file" <&2 71 | exit 1 72 | ;; 73 | esac 74 | fi 75 | done -------------------------------------------------------------------------------- /src/openjd/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from .cli._create_argparser import main 4 | 5 | 6 | main() 7 | -------------------------------------------------------------------------------- /src/openjd/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from ._create_argparser import main 4 | 5 | __all__ = ["main"] 6 | -------------------------------------------------------------------------------- /src/openjd/cli/_check/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from .._common import add_common_arguments, CommonArgument, SubparserGroup 4 | from ._check_command import add_check_arguments, do_check 5 | 6 | 7 | def populate_argparser(subcommands: SubparserGroup) -> None: 8 | """Adds the `check` command and all of its arguments to the given parser.""" 9 | check_parser = subcommands.add( 10 | "check", 11 | usage="openjd check JOB_TEMPLATE_PATH [arguments]", 12 | description="Given an Open Job Description template file, parse the file and run validation checks against it to ensure that it is correctly formed.", 13 | ) 14 | 15 | # add all arguments through `add_common_arguments` 16 | add_common_arguments(check_parser, {CommonArgument.PATH}) 17 | add_check_arguments(check_parser) 18 | check_parser.set_defaults(func=do_check) 19 | -------------------------------------------------------------------------------- /src/openjd/cli/_check/_check_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser, Namespace 4 | from openjd.model import ( 5 | DecodeValidationError, 6 | TemplateSpecificationVersion, 7 | decode_job_template, 8 | decode_environment_template, 9 | ) 10 | 11 | from .._common import ( 12 | add_extensions_argument, 13 | read_template, 14 | OpenJDCliResult, 15 | print_cli_result, 16 | process_extensions_argument, 17 | ) 18 | 19 | 20 | def add_check_arguments(run_parser: ArgumentParser): 21 | add_extensions_argument(run_parser) 22 | 23 | 24 | @print_cli_result 25 | def do_check(args: Namespace) -> OpenJDCliResult: 26 | """Open a provided template file and check its schema for errors.""" 27 | 28 | extensions = process_extensions_argument(args.extensions) 29 | 30 | try: 31 | # Raises: RuntimeError 32 | template_object = read_template(args.path) 33 | 34 | # Raises: KeyError 35 | document_version = template_object["specificationVersion"] 36 | 37 | # Raises: ValueError 38 | template_version = TemplateSpecificationVersion(document_version) 39 | 40 | # Raises: DecodeValidationError 41 | if TemplateSpecificationVersion.is_job_template(template_version): 42 | decode_job_template(template=template_object, supported_extensions=extensions) 43 | elif TemplateSpecificationVersion.is_environment_template(template_version): 44 | decode_environment_template(template=template_object) 45 | else: 46 | return OpenJDCliResult( 47 | status="error", 48 | message=f"Unknown template 'specificationVersion' ({document_version}).", 49 | ) 50 | 51 | except KeyError: 52 | return OpenJDCliResult( 53 | status="error", message="ERROR: Missing field 'specificationVersion'" 54 | ) 55 | except RuntimeError as exc: 56 | return OpenJDCliResult(status="error", message=f"ERROR: {str(exc)}") 57 | except DecodeValidationError as exc: 58 | return OpenJDCliResult( 59 | status="error", message=f"ERROR: '{str(args.path)}' failed checks: {str(exc)}" 60 | ) 61 | 62 | return OpenJDCliResult( 63 | status="success", message=f"Template at '{str(args.path)}' passes validation checks." 64 | ) 65 | -------------------------------------------------------------------------------- /src/openjd/cli/_common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser, Namespace, _SubParsersAction 4 | from dataclasses import asdict, dataclass 5 | from enum import Enum 6 | from pathlib import Path 7 | from typing import Callable, Literal 8 | import json 9 | import yaml 10 | import os 11 | 12 | from ._extensions import add_extensions_argument, process_extensions_argument, SUPPORTED_EXTENSIONS 13 | from ._job_from_template import ( 14 | job_from_template, 15 | get_job_params, 16 | get_params_from_file, 17 | ) 18 | from ._validation_utils import ( 19 | get_doc_type, 20 | read_template, 21 | read_job_template, 22 | read_environment_template, 23 | ) 24 | from openjd.model import DecodeValidationError, Job, JobParameterValues, EnvironmentTemplate 25 | 26 | __all__ = [ 27 | "add_extensions_argument", 28 | "get_doc_type", 29 | "get_job_params", 30 | "get_params_from_file", 31 | "process_extensions_argument", 32 | "read_template", 33 | "read_job_template", 34 | "read_environment_template", 35 | "validate_task_parameters", 36 | "SUPPORTED_EXTENSIONS", 37 | ] 38 | 39 | 40 | class CommonArgument(Enum): 41 | """ 42 | Used as literal options for which shared arguments 43 | a certain command uses. 44 | """ 45 | 46 | PATH = "path" 47 | JOB_PARAMS = "job_params" 48 | 49 | 50 | def add_common_arguments( 51 | parser: ArgumentParser, common_arg_options: set[CommonArgument] = set() 52 | ) -> None: 53 | """ 54 | Adds arguments that are used across commands. 55 | Universal arguments for all commands are added first, 56 | followed by arguments that are common among certain 57 | commands, but not universal. 58 | """ 59 | 60 | # Universal arguments for all commands 61 | parser.add_argument( 62 | "--output", 63 | choices=["human-readable", "json", "yaml"], 64 | default="human-readable", 65 | help="How to format the command's output.", 66 | ) 67 | 68 | # Retain order in common arguments by 69 | # checking set membership in a well-known order 70 | if CommonArgument.PATH in common_arg_options: 71 | parser.add_argument( 72 | "path", 73 | type=Path, 74 | action="store", 75 | help="The path to the template file.", 76 | ) 77 | if CommonArgument.JOB_PARAMS in common_arg_options: 78 | parser.add_argument( 79 | "--job-param", 80 | "-p", 81 | dest="job_params", 82 | type=str, 83 | action="append", 84 | metavar=('KEY=VALUE, file://PATH_TO_PARAMS, \'{"KEY": "VALUE", ... }\''), 85 | help=( 86 | "Use these Job parameters with the provided template. Can be provided as key-value pairs, " 87 | "path(s) to a JSON or YAML document prefixed with 'file://', or inline JSON. If this option " 88 | "is provided more than once then the given values are all combined in the order that they appear." 89 | ), 90 | ) 91 | 92 | 93 | class SubparserGroup: 94 | """ 95 | Wraps the `_SubParsersAction` type from the `argparse` library 96 | so each subcommand can be created & populated in their own 97 | respective module. 98 | """ 99 | 100 | group: _SubParsersAction 101 | 102 | def __init__(self, base: ArgumentParser, **kwargs): 103 | self.group = base.add_subparsers(**kwargs) 104 | 105 | def add(self, name: str, description: str, **kwargs) -> ArgumentParser: 106 | """ 107 | Wraps the `add_parser` function so multiple modules can 108 | add subcommands to the same base parser. 109 | 110 | For our purposes, we only expose the `description` keyword, but other 111 | keywords used by `add_parser` can still be passed in and used. 112 | """ 113 | return self.group.add_parser(name, **kwargs) 114 | 115 | 116 | def generate_job( 117 | args: Namespace, 118 | environments: list[EnvironmentTemplate] = [], 119 | *, 120 | supported_extensions: list[str], 121 | ) -> tuple[Job, JobParameterValues]: 122 | try: 123 | # Raises: RuntimeError, DecodeValidationError 124 | template = read_job_template(args.path, supported_extensions=supported_extensions) 125 | 126 | # Raises: RuntimeError 127 | return job_from_template( 128 | template, 129 | environments, 130 | args.job_params if args.job_params else None, 131 | Path(os.path.abspath(args.path.parent)), 132 | Path(os.getcwd()), 133 | ) 134 | except RuntimeError as rte: 135 | raise RuntimeError(f"ERROR generating Job: {str(rte)}") 136 | except DecodeValidationError as dve: 137 | raise RuntimeError(f"ERROR validating template: {str(dve)}") 138 | 139 | 140 | @dataclass 141 | class OpenJDCliResult(BaseException): 142 | """ 143 | Denotes the result of a command, including its status (success/error) 144 | and an accompanying message. 145 | 146 | Commands that require more information in their results will subclass 147 | OpenJDCliResult to add more fields. 148 | """ 149 | 150 | status: Literal["success", "error"] 151 | message: str 152 | 153 | def __str__(self) -> str: 154 | return self.message 155 | 156 | 157 | def _asdict_omit_null(attrs: list) -> dict: 158 | """ 159 | Retrieves a dataclass' attributes in a dictionary, omitting any fields with None or empty values. 160 | """ 161 | 162 | return {attr: value for (attr, value) in attrs if value} 163 | 164 | 165 | def print_cli_result(command: Callable[[Namespace], OpenJDCliResult]) -> Callable: 166 | """ 167 | Takes the result of a command and formats the output according to the user's specification. 168 | Used to decorate the `do_` functions for each command. 169 | """ 170 | 171 | def format_results(args: Namespace) -> OpenJDCliResult: 172 | response = command(args) 173 | 174 | if args.output == "human-readable": 175 | print(str(response)) 176 | else: 177 | if args.output == "json": 178 | print(json.dumps(asdict(response, dict_factory=_asdict_omit_null), indent=4)) 179 | else: 180 | print( 181 | yaml.safe_dump( 182 | asdict(response, dict_factory=_asdict_omit_null), sort_keys=False 183 | ) 184 | ) 185 | 186 | if response.status == "error": 187 | raise SystemExit(1) 188 | 189 | return response 190 | 191 | return format_results 192 | -------------------------------------------------------------------------------- /src/openjd/cli/_common/_extensions.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser 4 | from typing import Optional 5 | 6 | # This is the list of Open Job Description extensions with implemented support 7 | SUPPORTED_EXTENSIONS = ["TASK_CHUNKING", "REDACTED_ENV_VARS"] 8 | 9 | 10 | def add_extensions_argument(run_parser: ArgumentParser): 11 | run_parser.add_argument( 12 | "--extensions", 13 | help=f"A comma-separated list of Open Job Description extension names to enable. Defaults to all that are implemented: {','.join(SUPPORTED_EXTENSIONS)}", 14 | ) 15 | 16 | 17 | def process_extensions_argument(extensions: Optional[str]) -> list[str]: 18 | """Process the comma-separated extensions argument and return a list of supported extensions.""" 19 | 20 | # If the option is not provided, default to all the supported extensions. 21 | if extensions is None: 22 | return SUPPORTED_EXTENSIONS 23 | 24 | extensions_list = [ 25 | extension.strip().upper() for extension in extensions.split(",") if extension.strip() != "" 26 | ] 27 | 28 | unsupported_extensions = set(extensions_list) - set(SUPPORTED_EXTENSIONS) 29 | if unsupported_extensions: 30 | raise ValueError( 31 | f"Unsupported Open Job Description extension(s): {', '.join(sorted(unsupported_extensions))}" 32 | ) 33 | 34 | return extensions_list 35 | -------------------------------------------------------------------------------- /src/openjd/cli/_common/_job_from_template.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | from __future__ import annotations 3 | 4 | import json 5 | from pathlib import Path 6 | import re 7 | from typing import Optional, Union 8 | import yaml 9 | 10 | from ._validation_utils import get_doc_type 11 | from openjd.model import ( 12 | DecodeValidationError, 13 | DocumentType, 14 | EnvironmentTemplate, 15 | Job, 16 | JobParameterValues, 17 | JobTemplate, 18 | create_job, 19 | preprocess_job_parameters, 20 | ) 21 | 22 | 23 | def get_params_from_file(parameter_string: str) -> Union[dict, list]: 24 | """ 25 | Resolves the supplied Job Parameter filepath into a JSON object with its contents. 26 | 27 | Raises: RuntimeError if the file can't be opened 28 | """ 29 | parameter_file = Path(parameter_string.removeprefix("file://")).expanduser() 30 | 31 | if not parameter_file.exists(): 32 | raise RuntimeError(f"Provided parameter file '{str(parameter_file)}' does not exist.") 33 | if not parameter_file.is_file(): 34 | raise RuntimeError(f"Provided parameter file '{str(parameter_file)}' is not a file.") 35 | 36 | # Raises: RuntimeError 37 | doc_type = get_doc_type(parameter_file) 38 | 39 | try: 40 | parameter_string = parameter_file.read_text() 41 | except OSError: 42 | raise RuntimeError(f"Could not open parameter file '{str(parameter_file)}'.") 43 | 44 | try: 45 | if doc_type == DocumentType.YAML: 46 | # Raises: YAMLError 47 | parameters = yaml.safe_load(parameter_string) 48 | else: 49 | # Raises: JSONDecodeError 50 | parameters = json.loads(parameter_string) 51 | except (yaml.YAMLError, json.JSONDecodeError) as exc: 52 | raise RuntimeError( 53 | f"Parameter file '{str(parameter_file)}' is formatted incorrectly: {str(exc)}" 54 | ) 55 | 56 | return parameters 57 | 58 | 59 | def get_job_params(parameter_args: Optional[list[str]]) -> dict: 60 | """ 61 | Resolves Job Parameters from a list of command-line arguments. 62 | Arguments may be a filepath or a string with format 'Key=Value'. 63 | 64 | Raises: RuntimeError if the provided Parameters are formatted incorrectly or can't be opened 65 | """ 66 | parameter_dict: dict = {} 67 | 68 | for arg in parameter_args or []: 69 | arg = arg.strip() 70 | # Case 1: Provided argument is a filepath 71 | if arg.startswith("file://"): 72 | # Raises: RuntimeError 73 | parameters = get_params_from_file(arg) 74 | 75 | if isinstance(parameters, dict): 76 | parameter_dict.update(parameters) 77 | else: 78 | raise RuntimeError(f"Job parameter file '{arg}' should contain a dictionary.") 79 | 80 | # Case 2: Provided as a JSON string 81 | elif re.match("^{(.*)}$", arg): 82 | try: 83 | # Raises: JSONDecodeError 84 | parameters = json.loads(arg) 85 | except (json.JSONDecodeError, TypeError): 86 | raise RuntimeError( 87 | f"Job parameter string ('{arg}') not formatted correctly. It must be key=value pairs, inline JSON, or a path to a JSON or YAML document prefixed with 'file://'." 88 | ) 89 | if not isinstance(parameters, dict): 90 | # This should never happen. Including it out of a sense of paranoia. 91 | raise RuntimeError( 92 | f"Job parameter ('{arg}') must contain a dictionary mapping job parameters to their value." 93 | ) 94 | parameter_dict.update(parameters) 95 | 96 | # Case 3: Provided argument is a Key=Value string 97 | elif regex_match := re.match("^([^=]+)=(.*)$", arg): 98 | parameter_dict.update({regex_match[1]: regex_match[2]}) 99 | 100 | else: 101 | raise RuntimeError( 102 | f"Job parameter string ('{arg}') not formatted correctly. It must be key=value pairs, inline JSON, or a path to a JSON or YAML document prefixed with 'file://'." 103 | ) 104 | 105 | return parameter_dict 106 | 107 | 108 | def job_from_template( 109 | template: JobTemplate, 110 | environments: list[EnvironmentTemplate], 111 | parameter_args: list[str] | None, 112 | job_template_dir: Path, 113 | current_working_dir: Path, 114 | ) -> tuple[Job, JobParameterValues]: 115 | """ 116 | Given a decoded Job Template and a user-input parameter dictionary, 117 | generates a Job object and the parameter values for running the job. 118 | 119 | Raises: RuntimeError if parameters are an unsupported type or don't correspond to the template 120 | """ 121 | parameter_dict = get_job_params(parameter_args) 122 | 123 | try: 124 | parameter_values = preprocess_job_parameters( 125 | job_template=template, 126 | job_parameter_values=parameter_dict, 127 | job_template_dir=job_template_dir, 128 | current_working_dir=current_working_dir, 129 | environment_templates=environments, 130 | ) 131 | except ValueError as ve: 132 | raise RuntimeError(str(ve)) 133 | 134 | try: 135 | job = create_job( 136 | job_template=template, 137 | job_parameter_values=parameter_values, 138 | environment_templates=environments, 139 | ) 140 | return (job, parameter_values) 141 | except DecodeValidationError as dve: 142 | raise RuntimeError(f"Could not generate Job from template and parameters: {str(dve)}") 143 | -------------------------------------------------------------------------------- /src/openjd/cli/_common/_validation_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from typing import Any 4 | from pathlib import Path 5 | 6 | from openjd.model import ( 7 | DecodeValidationError, 8 | DocumentType, 9 | EnvironmentTemplate, 10 | JobTemplate, 11 | document_string_to_object, 12 | decode_environment_template, 13 | decode_job_template, 14 | ) 15 | 16 | 17 | def get_doc_type(filepath: Path) -> DocumentType: 18 | if filepath.suffix.lower() == ".json": 19 | return DocumentType.JSON 20 | elif filepath.suffix.lower() in (".yaml", ".yml"): 21 | return DocumentType.YAML 22 | raise RuntimeError(f"'{str(filepath)}' is not JSON or YAML.") 23 | 24 | 25 | def read_template(template_file: Path) -> dict[str, Any]: 26 | """Open a JSON or YAML-formatted file and attempt to parse it into a JobTemplate object. 27 | Raises a RuntimeError if the file doesn't exist or can't be opened, and raises a 28 | DecodeValidationError if its contents can't be parsed into a valid JobTemplate. 29 | """ 30 | 31 | if not template_file.exists(): 32 | raise RuntimeError(f"'{str(template_file)}' does not exist.") 33 | 34 | if template_file.is_file(): 35 | # Raises: RuntimeError 36 | filetype = get_doc_type(template_file) 37 | else: 38 | raise RuntimeError(f"'{str(template_file)}' is not a file.") 39 | 40 | try: 41 | template_string = template_file.read_text(encoding="utf-8") 42 | except OSError as exc: 43 | raise RuntimeError(f"Could not open file '{str(template_file)}': {str(exc)}") 44 | 45 | try: 46 | # Raises: DecodeValidationError 47 | template_object = document_string_to_object( 48 | document=template_string, document_type=filetype 49 | ) 50 | except DecodeValidationError as exc: 51 | raise RuntimeError(f"'{str(template_file)}' failed checks: {str(exc)}") 52 | 53 | return template_object 54 | 55 | 56 | def read_job_template(template_file: Path, *, supported_extensions: list[str]) -> JobTemplate: 57 | """Open a JSON or YAML-formatted file and attempt to parse it into a JobTemplate object. 58 | Raises a RuntimeError if the file doesn't exist or can't be opened, and raises a 59 | DecodeValidationError if its contents can't be parsed into a valid JobTemplate. 60 | """ 61 | # Raises RuntimeError 62 | template_object = read_template(template_file) 63 | 64 | # Raises: DecodeValidationError 65 | template = decode_job_template( 66 | template=template_object, supported_extensions=supported_extensions 67 | ) 68 | 69 | return template 70 | 71 | 72 | def read_environment_template(template_file: Path) -> EnvironmentTemplate: 73 | """Open a JSON or YAML-formatted file and attempt to parse it into an EnvironmentTemplate object. 74 | Raises a RuntimeError if the file doesn't exist or can't be opened, and raises a 75 | DecodeValidationError if its contents can't be parsed into a valid EnvironmentTemplate. 76 | """ 77 | # Raises RuntimeError 78 | template_object = read_template(template_file) 79 | 80 | # Raises: DecodeValidationError 81 | template = decode_environment_template(template=template_object) 82 | 83 | return template 84 | -------------------------------------------------------------------------------- /src/openjd/cli/_create_argparser.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser 4 | import sys 5 | import traceback 6 | from typing import Optional 7 | 8 | from ._common import SubparserGroup 9 | 10 | from ._check import populate_argparser as populate_check_subparser 11 | from ._summary import populate_argparser as populate_summary_subparser 12 | from ._run import populate_argparser as populate_run_subparser 13 | from ._schema import populate_argparser as populate_schema_subparser 14 | 15 | 16 | # Our CLI subcommand construction requires that all leaf subcommands define a default 17 | # 'func' property which is a Callable[[],None] that implements the subcommand. 18 | # After parsing, we call that `func` argument of the resulting args object. 19 | 20 | 21 | def create_argparser() -> ArgumentParser: 22 | """Generate the root argparser for the CLI""" 23 | parser = ArgumentParser(prog="openjd", usage="openjd [arguments]") 24 | parser.set_defaults(func=lambda _: parser.print_help()) 25 | subcommands = SubparserGroup( 26 | parser, 27 | title="commands", 28 | ) 29 | populate_check_subparser(subcommands) 30 | populate_summary_subparser(subcommands) 31 | populate_run_subparser(subcommands) 32 | populate_schema_subparser(subcommands) 33 | return parser 34 | 35 | 36 | def main(arg_list: Optional[list[str]] = None) -> None: 37 | """Main function for invoking the CLI""" 38 | parser = create_argparser() 39 | 40 | if arg_list is None: 41 | arg_list = sys.argv[1:] 42 | 43 | args = parser.parse_args(arg_list) 44 | try: 45 | # Raises: 46 | # SystemExit - on failure 47 | args.func(args) 48 | except Exception as exc: 49 | print(f"ERROR: {str(exc)}", file=sys.stderr) 50 | traceback.print_exc() 51 | sys.exit(1) 52 | -------------------------------------------------------------------------------- /src/openjd/cli/_run/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from ._run_command import add_run_arguments, do_run 4 | from .._common import add_common_arguments, CommonArgument, SubparserGroup 5 | 6 | 7 | def populate_argparser(subcommands: SubparserGroup) -> None: 8 | """Adds the `run` command and all of its arguments to the given parser.""" 9 | run_parser = subcommands.add( 10 | "run", 11 | description="Takes a Job Template and runs the entire job or a selected Step from the job.", 12 | usage="openjd run JOB_TEMPLATE_PATH [arguments]", 13 | ) 14 | add_common_arguments(run_parser, {CommonArgument.PATH, CommonArgument.JOB_PARAMS}) 15 | add_run_arguments(run_parser) 16 | run_parser.set_defaults(func=do_run) 17 | -------------------------------------------------------------------------------- /src/openjd/cli/_run/_local_session/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /src/openjd/cli/_run/_local_session/_actions.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from enum import Enum 4 | 5 | from openjd.model import Step, TaskParameterSet 6 | from openjd.model.v2023_09 import Environment 7 | from openjd.sessions import Session 8 | 9 | 10 | class EnvironmentType(str, Enum): 11 | """ 12 | The three different types of environment types that can be entered/exited in a session. 13 | """ 14 | 15 | EXTERNAL = "EXTERNAL" 16 | JOB = "JOB" 17 | STEP = "STEP" 18 | ALL = "ALL" 19 | 20 | def matches(self, other: "EnvironmentType") -> bool: 21 | """Environment types match if they are equal, or one of them is ALL.""" 22 | return self == other or self == EnvironmentType.ALL or other == EnvironmentType.ALL 23 | 24 | 25 | class SessionAction: 26 | _session: Session 27 | duration: float 28 | 29 | def __init__(self, session: Session): 30 | self._session = session 31 | 32 | def run(self): 33 | """ 34 | Subclasses of `SessionAction` should have 35 | custom implementations of this depending on their type. 36 | """ 37 | 38 | 39 | class RunTaskAction(SessionAction): 40 | _step: Step 41 | _parameters: TaskParameterSet 42 | 43 | def __init__(self, session: Session, step: Step, parameters: TaskParameterSet): 44 | super(RunTaskAction, self).__init__(session) 45 | self._step = step 46 | self._parameters = parameters 47 | 48 | def run(self): 49 | self._session.run_task( 50 | step_script=self._step.script, task_parameter_values=self._parameters 51 | ) 52 | 53 | def __str__(self): 54 | parameters = {name: parameter.value for name, parameter in self._parameters.items()} 55 | return f"Run Step '{self._step.name}' with Task parameters '{str(parameters)}'" 56 | 57 | 58 | class EnterEnvironmentAction(SessionAction): 59 | _environment: Environment 60 | _id: str 61 | 62 | def __init__(self, session: Session, environment: Environment, env_id: str): 63 | super(EnterEnvironmentAction, self).__init__(session) 64 | self._environment = environment 65 | self._id = env_id 66 | 67 | def run(self): 68 | self._session.enter_environment(environment=self._environment, identifier=self._id) 69 | 70 | def __str__(self): 71 | return f"Enter Environment '{self._environment.name}'" 72 | 73 | 74 | class ExitEnvironmentAction(SessionAction): 75 | _id: str 76 | _keep_session_running: bool 77 | 78 | def __init__(self, session: Session, id: str, keep_session_running: bool): 79 | super(ExitEnvironmentAction, self).__init__(session) 80 | self._id = id 81 | self._keep_session_running = keep_session_running 82 | 83 | def run(self): 84 | self._session.exit_environment( 85 | identifier=self._id, keep_session_running=self._keep_session_running 86 | ) 87 | 88 | def __str__(self): 89 | return f"Exit Environment '{self._id}'" 90 | -------------------------------------------------------------------------------- /src/openjd/cli/_run/_local_session/_logs.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from dataclasses import dataclass 4 | from logging import Handler, LogRecord 5 | from datetime import datetime, timezone 6 | from enum import Enum 7 | 8 | 9 | class LoggingTimestampFormat(str, Enum): 10 | """ 11 | Different formats for the timestamp of each log entry 12 | """ 13 | 14 | RELATIVE = "relative" 15 | LOCAL = "local" 16 | UTC = "utc" 17 | 18 | 19 | @dataclass 20 | class LogEntry: 21 | """ 22 | Log information from a sample Session to return in a CLI result. 23 | """ 24 | 25 | timestamp: str 26 | message: str 27 | 28 | def __str__(self) -> str: 29 | return f"{self.timestamp}\t{self.message}" 30 | 31 | 32 | class LocalSessionLogHandler(Handler): 33 | """ 34 | A custom Handler that formats and records logs in a dataclass. 35 | Used to print logs to `stdout` in real time while also storing 36 | them in memory. 37 | 38 | It prints a timestamp that is relative to the session start. 39 | """ 40 | 41 | messages: list[LogEntry] = [] 42 | _should_print: bool 43 | _session_start_timestamp: datetime 44 | _timestamp_format: LoggingTimestampFormat 45 | 46 | def __init__( 47 | self, 48 | should_print: bool, 49 | session_start_timestamp: datetime, 50 | timestamp_format: LoggingTimestampFormat, 51 | ): 52 | super(LocalSessionLogHandler, self).__init__() 53 | self._should_print = should_print 54 | self._session_start_timestamp = session_start_timestamp 55 | self._timestamp_format = timestamp_format 56 | 57 | def handle(self, record: LogRecord) -> bool: 58 | if self._timestamp_format == LoggingTimestampFormat.RELATIVE: 59 | timestamp = str( 60 | datetime.fromtimestamp(record.created, timezone.utc) - self._session_start_timestamp 61 | ) 62 | elif self._timestamp_format == LoggingTimestampFormat.LOCAL: 63 | timestamp = str( 64 | datetime.fromtimestamp(record.created, timezone.utc).astimezone().isoformat() 65 | ) 66 | else: 67 | timestamp = str(datetime.fromtimestamp(record.created, timezone.utc).isoformat()) 68 | 69 | record.created = datetime.fromtimestamp(record.created, timezone.utc).timestamp() 70 | new_record = LogEntry( 71 | timestamp=timestamp, 72 | message=record.getMessage(), 73 | ) 74 | self.messages.append(new_record) 75 | 76 | if self._should_print: 77 | print(new_record) 78 | 79 | # No filters are applied to the message, so always return True 80 | return True 81 | -------------------------------------------------------------------------------- /src/openjd/cli/_run/_local_session/_session_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from queue import Queue 4 | from threading import Event 5 | import time 6 | from typing import Any, Iterable, Optional, Type 7 | from types import FrameType, TracebackType 8 | from signal import signal, SIGINT, SIGTERM, SIG_DFL 9 | from itertools import islice 10 | from datetime import datetime, timedelta, timezone 11 | 12 | from ._actions import ( 13 | EnterEnvironmentAction, 14 | ExitEnvironmentAction, 15 | RunTaskAction, 16 | SessionAction, 17 | EnvironmentType, 18 | ) 19 | from ._logs import LocalSessionLogHandler, LogEntry, LoggingTimestampFormat 20 | from ..._common import SUPPORTED_EXTENSIONS 21 | 22 | from openjd.model import ( 23 | IntRangeExpr, 24 | Job, 25 | JobParameterValues, 26 | RevisionExtensions, 27 | SpecificationRevision, 28 | Step, 29 | StepParameterSpaceIterator, 30 | TaskParameterSet, 31 | ) 32 | from openjd.sessions import ( 33 | LOG, 34 | ActionState, 35 | ActionStatus, 36 | Session, 37 | SessionState, 38 | PathMappingRule, 39 | ) 40 | 41 | 42 | class LocalSessionFailed(RuntimeError): 43 | """ 44 | Raised when an action in the session fails. 45 | """ 46 | 47 | def __init__(self, failed_action: SessionAction): 48 | self.failed_action = failed_action 49 | super().__init__(f"Action failed: {failed_action}") 50 | 51 | 52 | class LocalSession: 53 | """ 54 | A class to manage a `Session` object from the `sessions` module, 55 | to run tasks of a job in a locally-running Session launched from the CLI. 56 | 57 | An OpenJD session's purpose is to run tasks from a single job. It can run 58 | tasks from different steps, as long as it enters the step environments 59 | before and exits them after. 60 | """ 61 | 62 | session_id: str 63 | failed: bool = False 64 | task_run_count: int = 0 65 | _job: Job 66 | _maximum_tasks: int 67 | _openjd_session: Session 68 | _enter_env_queue: Queue[EnterEnvironmentAction] 69 | _action_queue: Queue[RunTaskAction] 70 | _current_action: Optional[SessionAction] 71 | _failed_action: Optional[SessionAction] 72 | _action_ended: Event 73 | _path_mapping_rules: Optional[list[PathMappingRule]] 74 | _environments: Optional[list[Any]] 75 | _environments_entered: list[tuple[EnvironmentType, str]] 76 | _log_handler: LocalSessionLogHandler 77 | _cleanup_called: bool 78 | 79 | def __init__( 80 | self, 81 | *, 82 | job: Job, 83 | job_parameter_values: JobParameterValues, 84 | session_id: str, 85 | timestamp_format: LoggingTimestampFormat = LoggingTimestampFormat.RELATIVE, 86 | path_mapping_rules: Optional[list[PathMappingRule]] = None, 87 | environments: Optional[list[Any]] = None, 88 | should_print_logs: bool = True, 89 | retain_working_dir: bool = False, 90 | revision_extensions: RevisionExtensions = RevisionExtensions( 91 | spec_rev=SpecificationRevision.v2023_09, supported_extensions=SUPPORTED_EXTENSIONS 92 | ), 93 | ): 94 | self.session_id = session_id 95 | self._action_ended = Event() 96 | self._job = job 97 | self._timestamp_format = timestamp_format 98 | self._path_mapping_rules = path_mapping_rules 99 | self._environments = environments 100 | 101 | # Create an OpenJD Session 102 | self._openjd_session = Session( 103 | session_id=self.session_id, 104 | job_parameter_values=job_parameter_values, 105 | path_mapping_rules=self._path_mapping_rules, 106 | callback=self._action_callback, 107 | retain_working_dir=retain_working_dir, 108 | revision_extensions=revision_extensions, 109 | ) 110 | 111 | self._should_print_logs = should_print_logs 112 | self._cleanup_called = False 113 | self._started = False 114 | 115 | self._current_action = None 116 | self._failed_action = None 117 | self._environments_entered = [] 118 | 119 | # Initialize the action queue 120 | self._enter_env_queue: Queue[EnterEnvironmentAction] = Queue() 121 | self._action_queue: Queue[RunTaskAction] = Queue() 122 | 123 | def _context_manager_cleanup(self): 124 | try: 125 | # Exit all the environments that were entered 126 | self.run_environment_exits(type=EnvironmentType.ALL, keep_session_running=False) 127 | finally: 128 | signal(SIGINT, SIG_DFL) 129 | signal(SIGTERM, SIG_DFL) 130 | self._started = False 131 | 132 | # A blank line to separate the job log output from this status message 133 | LOG.info( 134 | msg="", 135 | extra={"session_id": self.session_id}, 136 | ) 137 | if self.failed: 138 | LOG.info( 139 | msg=f"Open Job Description CLI: ERROR executing action: '{self.failed_action}' (see Task logs for details)", 140 | extra={"session_id": self.session_id}, 141 | ) 142 | else: 143 | LOG.info( 144 | msg="Open Job Description CLI: All actions completed successfully!", 145 | extra={"session_id": self.session_id}, 146 | ) 147 | self.cleanup() 148 | self._started = False 149 | 150 | def __enter__(self) -> "LocalSession": 151 | # Add log handling 152 | session_start_timestamp = datetime.now(timezone.utc) 153 | self._log_handler = LocalSessionLogHandler( 154 | should_print=self._should_print_logs, 155 | session_start_timestamp=session_start_timestamp, 156 | timestamp_format=self._timestamp_format, 157 | ) 158 | LOG.addHandler(self._log_handler) 159 | LOG.info( 160 | msg=f"Open Job Description CLI: Session start {session_start_timestamp.astimezone().isoformat()}", 161 | extra={"session_id": self.session_id}, 162 | ) 163 | LOG.info( 164 | msg=f"Open Job Description CLI: Running job '{self._job.name}'", 165 | extra={"session_id": self.session_id}, 166 | ) 167 | signal(SIGINT, self._sigint_handler) 168 | signal(SIGTERM, self._sigint_handler) 169 | 170 | self._started = True 171 | 172 | # Enter all the external and job environments 173 | try: 174 | self.run_environment_enters(self._environments, EnvironmentType.EXTERNAL) 175 | self.run_environment_enters(self._job.jobEnvironments, EnvironmentType.JOB) 176 | except LocalSessionFailed: 177 | # If __enter__ fails, __exit__ won't be called so need to clean up here 178 | self._context_manager_cleanup() 179 | raise 180 | 181 | return self 182 | 183 | def __exit__( 184 | self, 185 | exc_type: Optional[Type[BaseException]], 186 | exc_value: Optional[BaseException], 187 | traceback: Optional[TracebackType], 188 | ) -> None: 189 | # __enter__ should have been called before __exit__ 190 | if not self._started: 191 | raise RuntimeError("Session was not started via a with statement.") 192 | 193 | self._context_manager_cleanup() 194 | 195 | def _sigint_handler(self, signum: int, frame: Optional[FrameType]) -> None: 196 | """Signal handler that is invoked when the process receives a SIGINT/SIGTERM""" 197 | LOG.info("Interruption signal recieved.") 198 | self.cancel() 199 | 200 | def run_environment_enters(self, environments: Optional[list[Any]], type: EnvironmentType): 201 | """Enter one or more environments in the session.""" 202 | if environments is None: 203 | return 204 | 205 | if self._openjd_session.state != SessionState.READY: 206 | raise RuntimeError( 207 | f"Session must be in READY state, but is in {self._openjd_session.state.name}" 208 | ) 209 | 210 | for env in environments: 211 | env_id = f"{type.name} - {env.name}" 212 | self._action_ended.clear() 213 | self._current_action = EnterEnvironmentAction( 214 | session=self._openjd_session, environment=env, env_id=env_id 215 | ) 216 | self._environments_entered.append((type, env_id)) 217 | self._current_action.run() 218 | self._action_ended.wait() 219 | if self.failed: 220 | self._failed_action = self._current_action 221 | self._current_action = None 222 | raise LocalSessionFailed(self._failed_action) 223 | self._current_action = None 224 | 225 | def run_environment_exits(self, type: EnvironmentType, *, keep_session_running: bool): 226 | """Exit environments that were entered in this session, in reverse order. 227 | Only exits environments matching the provided environment type. 228 | """ 229 | if self._openjd_session.state not in (SessionState.READY, SessionState.READY_ENDING): 230 | raise RuntimeError( 231 | f"Session must be in READY or READY_ENDING state, but is in {self._openjd_session.state.name}" 232 | ) 233 | 234 | failed_action = None 235 | 236 | while self._environments_entered and self._environments_entered[-1][0].matches(type): 237 | env_id = self._environments_entered.pop()[1] 238 | prev_action_failed = self.failed 239 | self._action_ended.clear() 240 | self._current_action = ExitEnvironmentAction( 241 | session=self._openjd_session, id=env_id, keep_session_running=keep_session_running 242 | ) 243 | self._current_action.run() 244 | self._action_ended.wait() 245 | if self.failed and not prev_action_failed: 246 | failed_action = self._failed_action = self._current_action 247 | self._current_action = None 248 | 249 | if failed_action: 250 | raise LocalSessionFailed(failed_action) 251 | 252 | def run_task(self, step: Step, parameter_set: TaskParameterSet) -> None: 253 | """Run a single task of a step in the session.""" 254 | if self._openjd_session.state != SessionState.READY: 255 | raise RuntimeError( 256 | f"Session must be in READY state, but is in {self._openjd_session.state.name}" 257 | ) 258 | 259 | self._action_ended.clear() 260 | self._current_action = RunTaskAction( 261 | session=self._openjd_session, step=step, parameters=parameter_set 262 | ) 263 | self._current_action.run() 264 | self._action_ended.wait() 265 | if self.failed: 266 | self._failed_action = self._current_action 267 | self._current_action = None 268 | raise LocalSessionFailed(self._failed_action) 269 | self._current_action = None 270 | 271 | def _run_tasks_adaptive_chunking( 272 | self, step: Step, task_parameters: StepParameterSpaceIterator, maximum_tasks: Optional[int] 273 | ): 274 | """Runs all the tasks of the task_parameters iterator with adaptive chunking.""" 275 | completed_task_count = 0 276 | completed_task_duration = 0.0 277 | target_runtime_seconds = int( 278 | step.parameterSpace.taskParameterDefinitions[ # type: ignore 279 | task_parameters.chunks_parameter_name # type: ignore 280 | ].chunks.targetRuntimeSeconds 281 | ) # type: ignore 282 | 283 | while True: 284 | # Get the next chunk to run 285 | parameter_set = next(task_parameters, None) 286 | if parameter_set is None: 287 | break 288 | 289 | start_seconds = time.perf_counter() 290 | # This may raise a LocalSessionFailed exception 291 | self.run_task(step, parameter_set) 292 | duration = time.perf_counter() - start_seconds 293 | 294 | # Accumulate the task count and duration from running this chunk 295 | completed_task_count += len( 296 | IntRangeExpr.from_str(parameter_set[task_parameters.chunks_parameter_name].value) # type: ignore 297 | ) 298 | completed_task_duration += duration 299 | 300 | # Estimate a chunk size based on the statistics, and update the iterator. Note that this 301 | # logic is very simple, providing a good starting point that behaves reasonably for other implementations 302 | # to follow. 303 | duration_per_task = completed_task_duration / completed_task_count 304 | adaptive_chunk_size = target_runtime_seconds / duration_per_task 305 | if ( 306 | completed_task_count < 10 307 | and adaptive_chunk_size > task_parameters.chunks_default_task_count # type: ignore 308 | ): 309 | # When we have data about only a few tasks, gradually blend in the new estimate instead of cutting over immediately 310 | adaptive_chunk_size = ( 311 | 0.75 * task_parameters.chunks_default_task_count + 0.25 * adaptive_chunk_size # type: ignore 312 | ) 313 | adaptive_chunk_size = max(int(adaptive_chunk_size), 1) 314 | if adaptive_chunk_size != task_parameters.chunks_default_task_count: 315 | LOG.info( 316 | msg=f"Open Job Description CLI: Ran {completed_task_count} tasks in {timedelta(seconds=completed_task_duration)}, average {timedelta(seconds=completed_task_duration / completed_task_count)}", 317 | extra={"session_id": self.session_id}, 318 | ) 319 | LOG.info( 320 | msg=f"Open Job Description CLI: Adjusting chunk size from {task_parameters.chunks_default_task_count} to {adaptive_chunk_size}", 321 | extra={"session_id": self.session_id}, 322 | ) 323 | task_parameters.chunks_default_task_count = adaptive_chunk_size 324 | 325 | # If a maximum task count was specified, count them down 326 | if maximum_tasks and maximum_tasks > 0: 327 | maximum_tasks -= 1 328 | if maximum_tasks == 0: 329 | break 330 | 331 | def run_step( 332 | self, 333 | step: Step, 334 | task_parameters: Optional[Iterable[TaskParameterSet]] = None, 335 | maximum_tasks: Optional[int] = None, 336 | ) -> None: 337 | """Run a step in the session. Optional parameters control which tasks to run.""" 338 | if self._openjd_session.state != SessionState.READY: 339 | raise RuntimeError( 340 | f"Session must be in READY state, but is in {self._openjd_session.state.name}" 341 | ) 342 | 343 | LOG.info( 344 | msg=f"Open Job Description CLI: Running step '{step.name}'", 345 | extra={"session_id": self.session_id}, 346 | ) 347 | 348 | if task_parameters is None: 349 | task_parameters = StepParameterSpaceIterator(space=step.parameterSpace) 350 | 351 | # Enter all the step environments 352 | self.run_environment_enters(step.stepEnvironments, EnvironmentType.STEP) 353 | 354 | try: 355 | # Run the tasks 356 | if ( 357 | isinstance(task_parameters, StepParameterSpaceIterator) 358 | and task_parameters.chunks_adaptive 359 | ): 360 | self._run_tasks_adaptive_chunking(step, task_parameters, maximum_tasks) 361 | else: 362 | # Run without adaptive chunking 363 | if maximum_tasks and maximum_tasks > 0: 364 | task_parameters = islice(task_parameters, maximum_tasks) 365 | 366 | for parameter_set in task_parameters: 367 | # This may raise a LocalSessionFailed exception 368 | self.run_task(step, parameter_set) 369 | finally: 370 | # Exit all the step environments 371 | self.run_environment_exits(type=EnvironmentType.STEP, keep_session_running=True) 372 | 373 | def cleanup(self) -> None: 374 | if not self._cleanup_called: 375 | LOG.info( 376 | msg="Open Job Description CLI: Local Session ended! Now cleaning up Session resources.", 377 | extra={"session_id": self.session_id}, 378 | ) 379 | self._log_handler.close() 380 | LOG.removeHandler(self._log_handler) 381 | 382 | self._openjd_session.cleanup() 383 | self._cleanup_called = True 384 | 385 | @property 386 | def failed_action(self) -> Optional[SessionAction]: 387 | """The action that failed, if any.""" 388 | return self._failed_action 389 | 390 | def cancel(self): 391 | LOG.info( 392 | msg="Open Job Description CLI: Cancelling the session...", 393 | extra={"session_id": self.session_id}, 394 | ) 395 | 396 | if self._openjd_session.state == SessionState.RUNNING: 397 | # The action will call self._action_callback when it has exited, 398 | # and that will exit the loop in self.run() 399 | self._openjd_session.cancel_action() 400 | 401 | LOG.info( 402 | msg=f"Open Job Description CLI: Session terminated by user while running action: '{str(self._current_action)}'.", 403 | extra={"session_id": self.session_id}, 404 | ) 405 | self.failed = True 406 | 407 | def get_log_messages(self) -> list[LogEntry]: 408 | return self._log_handler.messages 409 | 410 | def _action_callback(self, session_id: str, new_status: ActionStatus) -> None: 411 | if new_status.state == ActionState.SUCCESS: 412 | if isinstance(self._current_action, RunTaskAction): 413 | self.task_run_count += 1 414 | self._action_ended.set() 415 | if new_status.state in (ActionState.FAILED, ActionState.CANCELED, ActionState.TIMEOUT): 416 | self.failed = True 417 | self._action_ended.set() 418 | -------------------------------------------------------------------------------- /src/openjd/cli/_schema/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from .._common import SubparserGroup, add_common_arguments 4 | from ._schema_command import add_schema_arguments, do_get_schema 5 | 6 | 7 | def populate_argparser(subcommands: SubparserGroup) -> None: 8 | """ 9 | Adds the `schema` command to the given parser. 10 | """ 11 | schema_parser = subcommands.add( 12 | "schema", 13 | description="Returns a JSON Schema document for the Job template model.", 14 | ) 15 | add_common_arguments(schema_parser, set()) 16 | add_schema_arguments(schema_parser) 17 | schema_parser.set_defaults(func=do_get_schema) 18 | -------------------------------------------------------------------------------- /src/openjd/cli/_schema/_schema_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser, Namespace 4 | import json 5 | from typing import Union 6 | 7 | from .._common import OpenJDCliResult, print_cli_result 8 | from openjd.model import EnvironmentTemplate, JobTemplate, TemplateSpecificationVersion 9 | 10 | 11 | def add_schema_arguments(schema_parser: ArgumentParser) -> None: 12 | allowed_values = [ 13 | v.value 14 | for v in TemplateSpecificationVersion 15 | if TemplateSpecificationVersion.is_job_template(v) 16 | or TemplateSpecificationVersion.is_environment_template(v) 17 | ] 18 | schema_parser.add_argument( 19 | "--version", 20 | action="store", 21 | type=TemplateSpecificationVersion, 22 | required=True, 23 | help=f"The specification version to return a JSON schema document for. Allowed values: {', '.join(allowed_values)}", 24 | ) 25 | 26 | 27 | def _process_regex(target: dict) -> None: 28 | """ 29 | Translates Python's language-specific regex into a JSON-compatible format. 30 | """ 31 | 32 | if "pattern" in target and isinstance(target["pattern"], str): 33 | target["pattern"] = target["pattern"].replace("(?-m:", "(?:") 34 | target["pattern"] = target["pattern"].replace("\\Z", "$") 35 | 36 | for attr in target.keys(): 37 | if isinstance(target[attr], dict): 38 | _process_regex(target[attr]) 39 | 40 | 41 | @print_cli_result 42 | def do_get_schema(args: Namespace) -> OpenJDCliResult: 43 | """ 44 | Uses Pydantic to convert the Open Job Description Job template model 45 | into a JSON schema document to compare in-development 46 | Job templates against. 47 | """ 48 | 49 | Template: Union[type[JobTemplate], type[EnvironmentTemplate]] 50 | if args.version == TemplateSpecificationVersion.JOBTEMPLATE_v2023_09: 51 | from openjd.model.v2023_09 import JobTemplate as Template 52 | elif args.version == TemplateSpecificationVersion.ENVIRONMENT_v2023_09: 53 | from openjd.model.v2023_09 import EnvironmentTemplate as Template 54 | else: 55 | return OpenJDCliResult( 56 | status="error", message=f"ERROR: Cannot generate schema for version '{args.version}'." 57 | ) 58 | 59 | schema_doc: dict = {} 60 | 61 | try: 62 | schema_doc = Template.model_json_schema() 63 | _process_regex(schema_doc) 64 | except Exception as e: 65 | return OpenJDCliResult(status="error", message=f"ERROR generating schema: {str(e)}") 66 | 67 | return OpenJDCliResult(status="success", message=json.dumps(schema_doc, indent=4)) 68 | -------------------------------------------------------------------------------- /src/openjd/cli/_summary/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from .._common import add_common_arguments, CommonArgument, SubparserGroup 4 | from ._summary_command import add_summary_arguments, do_summary 5 | 6 | 7 | def populate_argparser(subcommands: SubparserGroup) -> None: 8 | """Adds the `summary` command's arguments to the given subcommand parser.""" 9 | summary_parser = subcommands.add( 10 | "summary", 11 | usage="openjd summary JOB_TEMPLATE_PATH [arguments]", 12 | description="Print summary information about a Job Template.", 13 | ) 14 | 15 | add_common_arguments(summary_parser, {CommonArgument.PATH, CommonArgument.JOB_PARAMS}) 16 | add_summary_arguments(summary_parser) 17 | summary_parser.set_defaults(func=do_summary) 18 | -------------------------------------------------------------------------------- /src/openjd/cli/_summary/_summary_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser, Namespace 4 | 5 | from ._summary_output import output_summary_result 6 | from .._common import ( 7 | add_extensions_argument, 8 | OpenJDCliResult, 9 | generate_job, 10 | process_extensions_argument, 11 | print_cli_result, 12 | ) 13 | 14 | 15 | def add_summary_arguments(summary_parser: ArgumentParser) -> None: 16 | # `step` is *technically* a shared argument, 17 | # but the help string and `required` attribute are 18 | # different among commands 19 | summary_parser.add_argument( 20 | "--step", 21 | action="store", 22 | type=str, 23 | metavar="STEP_NAME", 24 | help="Prints information about the Step with this name within the Job Template.", 25 | ) 26 | add_extensions_argument(summary_parser) 27 | 28 | 29 | @print_cli_result 30 | def do_summary(args: Namespace) -> OpenJDCliResult: 31 | """ 32 | Given a Job Template and applicable parameters, generates a Job and outputs information about it. 33 | """ 34 | extensions = process_extensions_argument(args.extensions) 35 | 36 | try: 37 | # Raises: RuntimeError 38 | sample_job, _ = generate_job(args, supported_extensions=extensions) 39 | except RuntimeError as rte: 40 | return OpenJDCliResult(status="error", message=str(rte)) 41 | 42 | return output_summary_result(sample_job, args.step) 43 | -------------------------------------------------------------------------------- /src/openjd/cli/_summary/_summary_output.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | from __future__ import annotations 3 | 4 | from dataclasses import dataclass, field 5 | from typing import Callable, Optional 6 | 7 | from .._common import OpenJDCliResult 8 | from openjd.model import Job, Step, StepParameterSpaceIterator 9 | 10 | 11 | def _populate_summary_list(source: list, to_summary_func: Callable) -> list: 12 | """ 13 | Given a list of elements, uses the `to_summary_func` argument 14 | to transform each element into a summary dataclass, returning 15 | the list of summary objects. 16 | """ 17 | summary_list: list = [] 18 | for item in source: 19 | summary_list.append(to_summary_func(item)) 20 | return summary_list 21 | 22 | 23 | def _format_summary_list(data: list, padding: int = 0) -> str: 24 | """ 25 | Prints the supplied list of summary objects as a bulleted list. 26 | """ 27 | formatted_list: str = "" 28 | for item in data: 29 | formatted_list += " " * padding + f"- {str(item)}\n" 30 | 31 | return formatted_list 32 | 33 | 34 | @dataclass 35 | class ParameterSummary: 36 | """ 37 | Organizes Parameter information in a dataclass. 38 | """ 39 | 40 | name: str 41 | description: Optional[str] 42 | type: str 43 | value: Optional[str] 44 | 45 | def __str__(self) -> str: 46 | readable_string = f"{self.name} ({self.type})" 47 | 48 | if self.value: 49 | readable_string += f": {self.value}" 50 | 51 | return readable_string 52 | 53 | 54 | @dataclass 55 | class EnvironmentSummary: 56 | """ 57 | Organizes Environment information in a dataclass. 58 | """ 59 | 60 | name: str 61 | description: Optional[str] 62 | parent: str = field(default="root") 63 | 64 | def __str__(self) -> str: 65 | readable_string = f"{self.name} (from '{self.parent}')" 66 | if self.description: 67 | readable_string += f"\n {self.description}" 68 | 69 | return readable_string 70 | 71 | 72 | @dataclass 73 | class DependencySummary: 74 | """ 75 | Organizes Step dependency information in a dataclass. 76 | Will include more fields when Step dependencies are further expanded on! 77 | """ 78 | 79 | step_name: str 80 | 81 | def __str__(self) -> str: 82 | return f"'{self.step_name}'" 83 | 84 | 85 | @dataclass 86 | class StepSummary: 87 | """ 88 | Organizes Step information in a dataclass. 89 | """ 90 | 91 | name: str 92 | description: Optional[str] 93 | parameter_definitions: Optional[list[ParameterSummary]] 94 | total_tasks: int 95 | environments: Optional[list[EnvironmentSummary]] 96 | dependencies: Optional[list[DependencySummary]] 97 | 98 | def __str__(self) -> str: 99 | summary_str = f"'{self.name}' ({self.total_tasks} total Tasks)\n" 100 | 101 | if self.parameter_definitions: 102 | summary_str += ( 103 | f" Task parameters:\n{_format_summary_list(self.parameter_definitions, padding=4)}" 104 | ) 105 | 106 | if self.environments: 107 | summary_str += f" {len(self.environments)} environments\n" 108 | 109 | if self.dependencies: 110 | summary_str += f" {len(self.dependencies)} dependencies\n" 111 | 112 | return summary_str 113 | 114 | 115 | @dataclass 116 | class OpenJDJobSummaryResult(OpenJDCliResult): 117 | """ 118 | A CLI result object with information specific to invoking the `summary` command on a Job. 119 | """ 120 | 121 | name: str 122 | parameter_definitions: Optional[list[ParameterSummary]] 123 | total_steps: int 124 | total_tasks: int 125 | total_environments: int 126 | root_environments: Optional[list[EnvironmentSummary]] 127 | steps: list[StepSummary] 128 | 129 | def __str__(self) -> str: 130 | summary_str = f"\n--- {self.message} ---\n" 131 | 132 | # For each parameter, print its name and its value (may be default or user-provided) 133 | if self.parameter_definitions: 134 | summary_str += ( 135 | f"\nParameters:\n{_format_summary_list(self.parameter_definitions, padding=2)}" 136 | ) 137 | 138 | summary_str += f""" 139 | Total steps: {self.total_steps} 140 | Total tasks: {self.total_tasks} 141 | Total environments: {self.total_environments} 142 | """ 143 | 144 | summary_str += f"\n--- Steps in '{self.name}' ---\n\n" 145 | for index, step in enumerate(self.steps): 146 | summary_str += f"{index+1}. {str(step)}\n" 147 | 148 | if self.total_environments: 149 | summary_str += f"\n--- Environments in '{self.name}' ---\n" 150 | if self.root_environments: 151 | summary_str += _format_summary_list(self.root_environments, padding=2) 152 | 153 | for step in self.steps: 154 | if step.environments: 155 | summary_str += _format_summary_list(step.environments, padding=2) 156 | 157 | return summary_str 158 | 159 | 160 | @dataclass 161 | class OpenJDStepSummaryResult(OpenJDCliResult): 162 | """ 163 | A CLI result with fields specific to invoking the `summary` command on a Step. 164 | """ 165 | 166 | job_name: str 167 | step_name: str 168 | total_parameters: int 169 | parameter_definitions: Optional[list[ParameterSummary]] 170 | total_tasks: int 171 | total_environments: int 172 | environments: Optional[list[EnvironmentSummary]] 173 | dependencies: Optional[list[DependencySummary]] 174 | 175 | def __str__(self) -> str: 176 | summary_str = f""" 177 | --- {self.message} --- 178 | 179 | Total tasks: {self.total_tasks} 180 | Total task parameters: {self.total_parameters} 181 | Total environments: {self.total_environments} 182 | """ 183 | 184 | if self.dependencies: 185 | summary_str += f"\nDependencies ({len(self.dependencies)}):\n{_format_summary_list(self.dependencies)}" 186 | 187 | if self.parameter_definitions: 188 | summary_str += f"\nParameters:\n{_format_summary_list(self.parameter_definitions)}" 189 | 190 | if self.environments: 191 | summary_str += f"\nEnvironments:\n{_format_summary_list(self.environments)}" 192 | 193 | return summary_str 194 | 195 | 196 | def _get_step_summary(step: Step) -> StepSummary: 197 | """ 198 | Given a Step object, transforms its relevant attributes 199 | into a StepSummary dataclass. 200 | """ 201 | 202 | parameter_definitions: Optional[list[ParameterSummary]] = None 203 | environments: Optional[list[EnvironmentSummary]] = None 204 | dependencies: Optional[list[DependencySummary]] = None 205 | total_tasks = 1 206 | 207 | parameter_definitions = [] 208 | if step.parameterSpace: 209 | parameter_definitions = _populate_summary_list( 210 | [(name, param) for name, param in step.parameterSpace.taskParameterDefinitions.items()], 211 | lambda param_tuple: ParameterSummary( 212 | name=param_tuple[0], description=None, type=param_tuple[1].type.value, value=None 213 | ), 214 | ) 215 | total_tasks = len( 216 | StepParameterSpaceIterator(space=step.parameterSpace, chunks_task_count_override=1) 217 | ) 218 | 219 | environments = [] 220 | if step.stepEnvironments: 221 | environments = _populate_summary_list( 222 | step.stepEnvironments, 223 | lambda env: EnvironmentSummary( 224 | name=env.name, parent=step.name, description=env.description 225 | ), 226 | ) 227 | 228 | dependencies = [] 229 | if step.dependencies: 230 | dependencies = _populate_summary_list( 231 | step.dependencies, lambda dep: DependencySummary(step_name=dep.dependsOn) 232 | ) 233 | 234 | return StepSummary( 235 | name=step.name, 236 | description=step.description, 237 | parameter_definitions=parameter_definitions, 238 | total_tasks=total_tasks, 239 | environments=environments, 240 | dependencies=dependencies, 241 | ) 242 | 243 | 244 | def output_summary_result(job: Job, step_name: str | None = None) -> OpenJDCliResult: 245 | """ 246 | Returns a CLI result object with information about this Job. 247 | """ 248 | 249 | steps_list: list[StepSummary] = [_get_step_summary(step) for step in job.steps] 250 | step_envs = sum(len(step.environments) if step.environments else 0 for step in steps_list) 251 | 252 | if not step_name: 253 | # We only need information about parameters and root environments 254 | # if we're summarizing an entire Job 255 | 256 | params_list: list[ParameterSummary] = [] 257 | if job.parameters: 258 | params_list = _populate_summary_list( 259 | [(name, param) for name, param in job.parameters.items()], 260 | lambda param_tuple: ParameterSummary( 261 | name=param_tuple[0], 262 | description=param_tuple[1].description, 263 | type=param_tuple[1].type.name, 264 | value=param_tuple[1].value, 265 | ), 266 | ) 267 | 268 | envs_list: list[EnvironmentSummary] = [] 269 | if job.jobEnvironments: 270 | envs_list = _populate_summary_list( 271 | job.jobEnvironments, 272 | lambda env: EnvironmentSummary(name=env.name, description=env.description), 273 | ) 274 | 275 | return OpenJDJobSummaryResult( 276 | status="success", 277 | message=f"Summary for '{job.name}'", 278 | name=job.name, 279 | parameter_definitions=params_list if params_list else None, 280 | total_steps=len(steps_list), 281 | total_tasks=sum(step.total_tasks for step in steps_list), 282 | total_environments=len(envs_list) + step_envs if envs_list else step_envs, 283 | root_environments=envs_list if envs_list else None, 284 | steps=steps_list, 285 | ) 286 | 287 | for step in steps_list: 288 | if step.name == step_name: 289 | return OpenJDStepSummaryResult( 290 | status="success", 291 | message=f"Summary for Step '{step.name}' in Job '{job.name}'", 292 | job_name=job.name, 293 | step_name=step.name, 294 | total_parameters=( 295 | len(step.parameter_definitions) if step.parameter_definitions else 0 296 | ), 297 | parameter_definitions=step.parameter_definitions, 298 | total_tasks=step.total_tasks, 299 | total_environments=len(step.environments) if step.environments else 0, 300 | environments=step.environments, 301 | dependencies=step.dependencies, 302 | ) 303 | 304 | return OpenJDCliResult( 305 | status="error", message=f"Step '{step_name}' does not exist in Job '{job.name}'." 306 | ) 307 | -------------------------------------------------------------------------------- /src/openjd/cli/py.typed: -------------------------------------------------------------------------------- 1 | # Marker file that indicates this package supports typing 2 | -------------------------------------------------------------------------------- /test/openjd/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | import pytest 4 | from enum import Enum 5 | from typing import Any 6 | from pathlib import Path 7 | 8 | import yaml 9 | 10 | from openjd.model import ParameterValue, ParameterValueType 11 | from openjd.cli import main as openjd_cli_main 12 | 13 | 14 | def run_openjd_cli_main(capsys, *, args: list[str], expected_exit_code: int) -> Any: 15 | """Wraps the logic to run the OpenJD CLI within a capsys environment""" 16 | try: 17 | openjd_cli_main(args) 18 | exit_code = 0 19 | except SystemExit as e: 20 | exit_code = e.code # type: ignore 21 | 22 | outerr = capsys.readouterr() 23 | 24 | assert ( 25 | exit_code == expected_exit_code 26 | ), f"Expected exit code {expected_exit_code}, but got {exit_code}:\n{format_capsys_outerr(outerr)}" 27 | 28 | return outerr 29 | 30 | 31 | def format_capsys_outerr(outerr: Any) -> str: 32 | """Formats the capsys stdout and stderr to insert in an assertion message""" 33 | return f"\nstdout:\n{outerr.out}\nstderr:\n{outerr.err}" 34 | 35 | 36 | # Catch-all sample template with different cases per Step 37 | 38 | MOCK_TEMPLATE = yaml.safe_load( 39 | (Path(__file__).parent / "templates" / "job_with_test_steps.yaml").read_text(encoding="utf8") 40 | ) 41 | 42 | # Map of Step names to Step indices for more readable test cases 43 | 44 | 45 | class SampleSteps(int, Enum): 46 | NormalStep = 0 47 | LongCommand = 1 48 | BareStep = 2 49 | DependentStep = 3 50 | TaskParamStep = 4 51 | ExtraDependentStep = 5 52 | DependentParamStep = 6 53 | StepDepHasStepEnv = 7 54 | BadCommand = 8 55 | 56 | 57 | # Sample dictionaries for tests using Job parameters 58 | 59 | MOCK_TEMPLATE_REQUIRES_PARAMS = { 60 | "specificationVersion": "jobtemplate-2023-09", 61 | "name": "{{Param.Title}}", 62 | "parameterDefinitions": [ 63 | {"name": "Title", "type": "STRING", "minLength": 3, "default": "my job"}, 64 | {"name": "RequiredParam", "type": "INT", "minValue": 3, "maxValue": 8}, 65 | ], 66 | "steps": [ 67 | { 68 | "name": "step1", 69 | "script": { 70 | "actions": { 71 | "onRun": { 72 | "command": "python", 73 | "args": ["-c", "print('{{Param.RequiredParam}}')"], 74 | } 75 | } 76 | }, 77 | }, 78 | { 79 | "name": "step2", 80 | "script": { 81 | "actions": { 82 | "onRun": {"command": "python", "args": ["-c", "print('Hello, world!'}"]} 83 | } 84 | }, 85 | "stepEnvironments": [ 86 | {"name": "my-step1-environment", "variables": {"variable": "value"}} 87 | ], 88 | "dependencies": [{"dependsOn": "step1"}], 89 | }, 90 | ], 91 | } 92 | 93 | MOCK_PARAM_ARGUMENTS = ["Title=overwrite", "RequiredParam=5"] 94 | MOCK_PARAM_VALUES = {"Title": "overwrite", "RequiredParam": "5"} 95 | 96 | # Shared parameters for `LocalSession` tests to be used with `@pytest.mark.parametrize` 97 | 98 | SESSION_PARAMETERS = ( 99 | "step_index,maximum_tasks, parameter_sets,num_expected_tasks", 100 | [ 101 | pytest.param(SampleSteps.NormalStep, -1, None, 1, id="Basic step"), 102 | pytest.param( 103 | SampleSteps.DependentStep, 104 | -1, 105 | None, 106 | 1, 107 | id="Direct dependency", 108 | ), 109 | pytest.param( 110 | SampleSteps.ExtraDependentStep, 111 | -1, 112 | None, 113 | 1, 114 | id="Dependencies and Task parameters", 115 | ), 116 | pytest.param( 117 | SampleSteps.ExtraDependentStep, 118 | 1, 119 | None, 120 | 1, 121 | id="No maximum on dependencies' Tasks", 122 | ), 123 | pytest.param(SampleSteps.TaskParamStep, 1, None, 1, id="Limit on maximum Task parameters"), 124 | pytest.param( 125 | SampleSteps.TaskParamStep, 126 | 100, 127 | None, 128 | 6, 129 | id="Maximum Task parameters more than defined", 130 | ), 131 | pytest.param( 132 | SampleSteps.TaskParamStep, 133 | -1, 134 | [ 135 | { 136 | "TaskNumber": ParameterValue(type=ParameterValueType.INT, value="2"), 137 | "TaskMessage": ParameterValue(type=ParameterValueType.STRING, value="Bye!"), 138 | }, 139 | { 140 | "TaskNumber": ParameterValue(type=ParameterValueType.INT, value="1"), 141 | "TaskMessage": ParameterValue(type=ParameterValueType.STRING, value="Hi!"), 142 | }, 143 | ], 144 | 2, 145 | id="Custom parameter sets", 146 | ), 147 | pytest.param( 148 | SampleSteps.BareStep, 149 | -1, 150 | [ 151 | {"Why": ParameterValue(type=ParameterValueType.STRING, value="Am")}, 152 | {"I": ParameterValue(type=ParameterValueType.STRING, value="Here")}, 153 | ], 154 | 2, 155 | id="Task parameters for step not requiring them", 156 | ), 157 | pytest.param( 158 | SampleSteps.DependentParamStep, 159 | -1, 160 | [ 161 | {"Adjective": ParameterValue(type=ParameterValueType.STRING, value="extremely")}, 162 | {"Adjective": ParameterValue(type=ParameterValueType.STRING, value="most")}, 163 | ], 164 | 2, 165 | id="Custom Task parameters not applied to dependency", 166 | ), 167 | pytest.param( 168 | SampleSteps.TaskParamStep, 169 | 1, 170 | [ 171 | { 172 | "TaskNumber": ParameterValue(type=ParameterValueType.INT, value="2"), 173 | "TaskMessage": ParameterValue(type=ParameterValueType.STRING, value="Hi!"), 174 | }, 175 | { 176 | "TaskNumber": ParameterValue(type=ParameterValueType.INT, value="2"), 177 | "TaskMessage": ParameterValue(type=ParameterValueType.STRING, value="Bye!"), 178 | }, 179 | ], 180 | 1, 181 | id="Maximum Tasks less than number of parameter sets", 182 | ), 183 | ], 184 | ) 185 | -------------------------------------------------------------------------------- /test/openjd/cli/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | import os 4 | import pytest 5 | import tempfile 6 | from pathlib import Path 7 | from unittest.mock import patch 8 | 9 | from . import MOCK_TEMPLATE, SampleSteps 10 | from openjd.cli._common._job_from_template import job_from_template 11 | from openjd.cli._run._local_session._session_manager import LocalSession 12 | from openjd.model import decode_job_template 13 | 14 | 15 | @pytest.fixture(scope="function", params=[[], ["Message=A new message!"]]) 16 | def sample_job_and_dirs(request): 17 | """ 18 | Uses the MOCK_TEMPLATE object to create a Job, once 19 | with default parameters and once with user-specified parameters. 20 | 21 | This fixture also manages the life time of a temporary directory that's 22 | used for the job template dir and the current working directory. 23 | """ 24 | with tempfile.TemporaryDirectory() as tmpdir: 25 | template_dir = Path(tmpdir) / "template_dir" 26 | current_working_dir = Path(tmpdir) / "current_working_dir" 27 | os.makedirs(template_dir) 28 | os.makedirs(current_working_dir) 29 | 30 | template = decode_job_template(template=MOCK_TEMPLATE) 31 | yield ( 32 | *job_from_template( 33 | template=template, 34 | environments=[], 35 | parameter_args=request.param, 36 | job_template_dir=template_dir, 37 | current_working_dir=current_working_dir, 38 | ), 39 | template_dir, 40 | current_working_dir, 41 | ) 42 | 43 | 44 | @pytest.fixture(scope="function") 45 | def sample_step_map(sample_job_and_dirs): 46 | return {step.name: step for step in sample_job_and_dirs[0].steps} 47 | 48 | 49 | @pytest.fixture( 50 | scope="function", 51 | params=[ 52 | pytest.param([], SampleSteps.NormalStep, -1, 2, 1, id="Basic step"), 53 | pytest.param( 54 | [SampleSteps.BareStep], SampleSteps.DependentStep, -1, 1, 2, id="Direct dependency" 55 | ), 56 | pytest.param( 57 | [ 58 | SampleSteps.BareStep, 59 | SampleSteps.DependentStep, 60 | SampleSteps.TaskParamStep, 61 | ], 62 | SampleSteps.ExtraDependentStep, 63 | -1, 64 | 1, 65 | 6, 66 | id="Dependencies and Task parameters", 67 | ), 68 | pytest.param( 69 | [SampleSteps.BareStep, SampleSteps.DependentStep, SampleSteps.TaskParamStep], 70 | SampleSteps.ExtraDependentStep, 71 | 1, 72 | 1, 73 | 6, 74 | id="No maximum on dependencies' Tasks", 75 | ), 76 | pytest.param([], SampleSteps.TaskParamStep, 1, 1, 1, id="Limit on maximum Task parameters"), 77 | pytest.param( 78 | [], SampleSteps.TaskParamStep, 100, 1, 3, id="Maximum Task parameters more than defined" 79 | ), 80 | ], 81 | ) 82 | def session_parameters(request): 83 | """ 84 | Tests for `LocalSession.initialize` and `LocalSession.run` use the same 85 | test parameters, so we create a fixture that returns them. 86 | """ 87 | return request.param 88 | 89 | 90 | @pytest.fixture(scope="function") 91 | def patched_session_cleanup(): 92 | """ 93 | Patches the `cleanup` function in a LocalSession, but uses the 94 | original function as a side effect to track how many times it 95 | gets called. 96 | We use this to verify that Sessions are properly cleaned up 97 | after exiting on a success or error. 98 | """ 99 | with patch.object( 100 | LocalSession, "cleanup", autospec=True, side_effect=LocalSession.cleanup 101 | ) as patched_cleanup: 102 | yield patched_cleanup 103 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/basic.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: "jobtemplate-2023-09" 2 | name: Job 3 | parameterDefinitions: 4 | - name: J 5 | type: STRING 6 | jobEnvironments: 7 | - name: J1 8 | script: 9 | actions: 10 | onEnter: 11 | command: python 12 | args: ["-c", "print('J1 Enter')"] 13 | onExit: 14 | command: python 15 | args: ["-c", "print('J1 Exit')"] 16 | - name: J2 17 | script: 18 | actions: 19 | onEnter: 20 | command: python 21 | args: ["-c", "print('J2 Enter')"] 22 | onExit: 23 | command: python 24 | args: ["-c", "print('J2 Exit')"] 25 | steps: 26 | - name: First 27 | parameterSpace: 28 | taskParameterDefinitions: 29 | - name: Foo 30 | type: INT 31 | range: "1" 32 | - name: Bar 33 | type: STRING 34 | range: ["Bar1", "Bar2"] 35 | stepEnvironments: 36 | - name: FirstS, 37 | script: 38 | actions: 39 | onEnter: 40 | command: python 41 | args: ["-c", "print('FirstS Enter')"] 42 | onExit: 43 | command: python 44 | args: ["-c", "print('FirstS Exit')"] 45 | script: 46 | actions: 47 | onRun: 48 | command: python 49 | args: 50 | - "-c" 51 | - "print('J={{Param.J}} Foo={{Task.Param.Foo}}. Bar={{Task.Param.Bar}}')" 52 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/basic_dependency_job.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: jobtemplate-2023-09 2 | name: Job 3 | parameterDefinitions: 4 | - name: J 5 | type: STRING 6 | jobEnvironments: 7 | - name: J1 8 | script: 9 | actions: 10 | onEnter: 11 | command: python 12 | args: 13 | - -c 14 | - print('J1 Enter') 15 | onExit: 16 | command: python 17 | args: 18 | - -c 19 | - print('J1 Exit') 20 | steps: 21 | - name: First 22 | parameterSpace: 23 | taskParameterDefinitions: 24 | - name: Foo 25 | type: INT 26 | range: '1' 27 | - name: Bar 28 | type: STRING 29 | range: 30 | - Bar1 31 | - Bar2 32 | script: 33 | actions: 34 | onRun: 35 | command: python 36 | args: 37 | - -c 38 | - print('J={{Param.J}} Foo={{Task.Param.Foo}}. Bar={{Task.Param.Bar}}') 39 | - name: Second 40 | dependencies: 41 | - dependsOn: First 42 | parameterSpace: 43 | taskParameterDefinitions: 44 | - name: Fuz 45 | type: INT 46 | range: 1-2 47 | script: 48 | actions: 49 | onRun: 50 | command: python 51 | args: 52 | - -c 53 | - print('J={{Param.J}} Fuz={{Task.Param.Fuz}}.') -------------------------------------------------------------------------------- /test/openjd/cli/templates/chunked_job.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: 'jobtemplate-2023-09' 2 | extensions: 3 | - TASK_CHUNKING 4 | name: Chunked Job 5 | 6 | parameterDefinitions: 7 | - name: Items 8 | type: STRING 9 | default: 1-40 10 | - name: ChunkSize 11 | type: INT 12 | default: 10 13 | - name: TargetRuntime 14 | type: INT 15 | default: 0 16 | - name: TaskSleepTime 17 | type: FLOAT 18 | default: 0.01 19 | 20 | steps: 21 | - name: Chunked Step 22 | 23 | parameterSpace: 24 | taskParameterDefinitions: 25 | - name: Item 26 | type: CHUNK[INT] 27 | range: "{{Param.Items}}" 28 | chunks: 29 | defaultTaskCount: "{{Param.ChunkSize}}" 30 | targetRuntimeSeconds: "{{Param.TargetRuntime}}" 31 | rangeConstraint: NONCONTIGUOUS 32 | 33 | script: 34 | actions: 35 | onRun: 36 | command: bash 37 | args: ['{{Task.File.Run}}'] 38 | embeddedFiles: 39 | - name: GetSleepTime 40 | type: TEXT 41 | filename: get_sleep_time.py 42 | data: | 43 | """ 44 | Converts an Open Job Description range expression into a sleep time according to the job parameters. 45 | * https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#34111-intrangeexpr 46 | """ 47 | 48 | import sys, re 49 | 50 | task_sleep_time = float(sys.argv[2]) 51 | 52 | def range_expr_to_list(range_expr): 53 | # Regex that matches "", "-", or "-:" 54 | int_pat = r"\s*(-?[0-9]+)\s*" 55 | part_re = re.compile(f"^{int_pat}(?:-{int_pat}(?::{int_pat})?)?$") 56 | result = [] 57 | for part in range_expr.split(","): 58 | if m := part_re.match(part): 59 | start, end, step = m.groups() 60 | if step is not None: 61 | # Linear sequence "3-7:2" means the values [3, 5, 7]. 62 | result.extend(range(int(start), int(end) + (int(step)//abs(int(step))), int(step))) 63 | elif end is not None: 64 | # Interval "3-6" means the values [3, 4, 5, 6]. 65 | result.extend(range(int(start), int(end) + 1)) 66 | else: 67 | # Integer "3" means the values [3]. 68 | result.append(int(start)) 69 | else: 70 | raise ValueError(f"Invalid frame range expression: {range_expr}") 71 | return result 72 | 73 | print(task_sleep_time * len(range_expr_to_list(sys.argv[1]))) 74 | - name: Run 75 | type: TEXT 76 | data: | 77 | set -xeuo pipefail 78 | 79 | sleep "$(python '{{Task.File.GetSleepTime}}' '{{Task.Param.Item}}' '{{Param.TaskSleepTime}}')" 80 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/env_1.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: environment-2023-09 2 | environment: 3 | name: Env1 4 | script: 5 | actions: 6 | onEnter: 7 | command: python 8 | args: 9 | - -c 10 | - print('Env1 Enter') 11 | onExit: 12 | command: python 13 | args: 14 | - -c 15 | - print('Env1 Exit') 16 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/env_2.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: environment-2023-09 2 | environment: 3 | name: Env2 4 | script: 5 | actions: 6 | onEnter: 7 | command: python 8 | args: 9 | - -c 10 | - print('Env2 Enter') 11 | onExit: 12 | command: python 13 | args: 14 | - -c 15 | - print('Env2 Exit') 16 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/env_fails_enter.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: environment-2023-09 2 | environment: 3 | name: EnvEnterFail 4 | script: 5 | actions: 6 | onEnter: 7 | command: python 8 | args: 9 | - -c 10 | - import sys; print('EnvEnterFail Enter'); sys.exit(1) 11 | onExit: 12 | command: python 13 | args: 14 | - -c 15 | - print('EnvEnterFail Exit') 16 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/env_fails_exit.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: environment-2023-09 2 | environment: 3 | name: EnvExitFail 4 | script: 5 | actions: 6 | onEnter: 7 | command: python 8 | args: 9 | - -c 10 | - print('EnvExitFail Enter') 11 | onExit: 12 | command: python 13 | args: 14 | - -c 15 | - import sys; print('EnvExitFail Exit'); sys.exit(1) 16 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/env_with_param.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: environment-2023-09 2 | parameterDefinitions: 3 | - name: EnvParam 4 | type: STRING 5 | default: DefaultForEnvParam 6 | environment: 7 | name: EnvWithParam 8 | script: 9 | actions: 10 | onEnter: 11 | command: python 12 | args: 13 | - -c 14 | - print('EnvWithParam Enter {{Param.EnvParam}}') 15 | onExit: 16 | command: python 17 | args: 18 | - -c 19 | - print('EnvWithParam Exit {{Param.EnvParam}}') 20 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/job_sleep_exit_normal.yaml: -------------------------------------------------------------------------------- 1 | { 2 | "specificationVersion": "jobtemplate-2023-09", 3 | "name": "TimeoutTest", 4 | "parameterDefinitions": [{"name": "J", "type": "STRING"}], 5 | "steps": [ 6 | { 7 | "name": "Timeout", 8 | "script": { 9 | "actions": { 10 | "onRun": { 11 | "command": "python", 12 | "args": [ 13 | "-c", 14 | # Obfuscate "EXIT_NORMAL" so it doesn't appear in the log when Windows prints the command that's run to the log. 15 | "import time,sys; print('SLEEP'); sys.stdout.flush(); time.sleep(5); print(chr(69)+'XIT_NORMAL')", 16 | ], 17 | "timeout": 2, 18 | } 19 | } 20 | }, 21 | } 22 | ], 23 | } -------------------------------------------------------------------------------- /test/openjd/cli/templates/job_with_test_steps.yaml: -------------------------------------------------------------------------------- 1 | # Catch-all sample template with different cases per Step 2 | specificationVersion: jobtemplate-2023-09 3 | name: my-job 4 | parameterDefinitions: 5 | - name: Message 6 | type: STRING 7 | default: Hello, world! 8 | jobEnvironments: 9 | - name: rootEnv 10 | variables: 11 | rootVar: rootVal 12 | steps: 13 | # VALID STEPS 14 | # Basic step; uses Job parameters and has an environment 15 | - name: NormalStep 16 | script: 17 | actions: 18 | onRun: 19 | command: python 20 | args: 21 | - -c 22 | - print('{{Param.Message}}') 23 | stepEnvironments: 24 | - name: env1 25 | script: 26 | actions: 27 | onEnter: 28 | command: python 29 | args: 30 | - -c 31 | - print('EnteringEnv') 32 | # Step that will wait for one minute before completing its Task 33 | - name: LongCommand 34 | script: 35 | actions: 36 | onRun: 37 | command: sleep 38 | args: 39 | - '60' 40 | # Step with the bare minimum information, i.e., no Task parameters, environments, or dependencies 41 | - name: BareStep 42 | script: 43 | actions: 44 | onRun: 45 | command: python 46 | args: 47 | - -c 48 | - print('zzz') 49 | # Step with a direct dependency on a previous Step 50 | - name: DependentStep 51 | script: 52 | actions: 53 | onRun: 54 | command: python 55 | args: 56 | - -c 57 | - print('I am dependent!') 58 | dependencies: 59 | - dependsOn: BareStep 60 | # Step with Task parameters 61 | - name: TaskParamStep 62 | parameterSpace: 63 | taskParameterDefinitions: 64 | - name: TaskNumber 65 | type: INT 66 | range: 67 | - 1 68 | - 2 69 | - 3 70 | - name: TaskMessage 71 | type: STRING 72 | range: 73 | - Hi! 74 | - Bye! 75 | script: 76 | actions: 77 | onRun: 78 | command: python 79 | args: 80 | - -c 81 | - print('{{Task.Param.TaskNumber}}.{{Task.Param.TaskMessage}}') 82 | # Step with a transitive dependency and a direct dependency 83 | - name: ExtraDependentStep 84 | script: 85 | actions: 86 | onRun: 87 | command: python 88 | args: 89 | - -c 90 | - print('I am extra dependent!') 91 | dependencies: 92 | - dependsOn: DependentStep 93 | - dependsOn: TaskParamStep 94 | # Step with dependencies and Task parameters 95 | - name: DependentParamStep 96 | parameterSpace: 97 | taskParameterDefinitions: 98 | - name: Adjective 99 | type: STRING 100 | range: 101 | - really 102 | - very 103 | - super 104 | script: 105 | actions: 106 | onRun: 107 | command: python 108 | args: 109 | - -c 110 | - print('I am {{Task.Param.Adjective}} dependent!') 111 | dependencies: 112 | - dependsOn: TaskParamStep 113 | # Step whose dependency has a step environment 114 | - name: StepDepHasStepEnv 115 | script: 116 | actions: 117 | onRun: 118 | command: python 119 | args: 120 | - -c 121 | - print('I have a dependency with a step environment!') 122 | dependencies: 123 | - dependsOn: NormalStep 124 | # ERROR STEPS 125 | # Step with a non-existent command that will throw an error when run 126 | - name: BadCommand 127 | script: 128 | actions: 129 | onRun: 130 | command: aaaaaaaaaa 131 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/redacted_env.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: "jobtemplate-2023-09" 2 | extensions: 3 | - REDACTED_ENV_VARS 4 | name: Test Redacted Env 5 | description: Test redacted environment variables 6 | 7 | jobEnvironments: 8 | - name: RedactedEnv 9 | script: 10 | actions: 11 | onEnter: 12 | command: python 13 | args: ["{{Env.File.Enter}}"] 14 | onExit: 15 | command: python 16 | args: ["{{Env.File.Exit}}"] 17 | embeddedFiles: 18 | - name: Enter 19 | type: TEXT 20 | data: | 21 | print("Setting redacted vars..") 22 | print(f"openjd_redacted_env: SECRETVAR=SECRETVAL") 23 | print(f"openjd_redacted_env: KEYSPACE =SECRETVAL") 24 | print(f"openjd_redacted_env: VALSPACE= SPACEVAL") 25 | print(f'openjd_redacted_env: "MULTILINE=first_line\\nsecond_line\\nthird_line"') 26 | - name: Exit 27 | type: TEXT 28 | data: | 29 | import os 30 | print(f"SECRETVAR is {os.environ.get('SECRETVAR')}") 31 | print(f"KEYSPACE is {os.environ.get('KEYSPACE')}") 32 | print(f"VALSPACE is {os.environ.get('VALSPACE')}") 33 | print(f"MULTILINE is {os.environ.get('VALSPACE')} END") 34 | print("first_line") 35 | print("second_line") 36 | print("third_line") 37 | steps: 38 | - name: CheckVars 39 | script: 40 | actions: 41 | onRun: 42 | command: python 43 | args: ["{{Task.File.Run}}"] 44 | embeddedFiles: 45 | - name: Run 46 | type: TEXT 47 | data: | 48 | import os 49 | print(f"SECRETVAR is {os.environ.get('SECRETVAR')}") 50 | print(f"KEYSPACE is {os.environ.get('KEYSPACE')}") 51 | print(f"VALSPACE is {os.environ.get('VALSPACE')}") 52 | print(f"MULTILINE is {os.environ.get('VALSPACE')} END") 53 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/simple_with_j_param.yaml: -------------------------------------------------------------------------------- 1 | { 2 | "specificationVersion": "jobtemplate-2023-09", 3 | "name": "Test", 4 | "parameterDefinitions": [{"name": "J", "type": "STRING"}], 5 | "steps": [ 6 | { 7 | "name": "SimpleStep", 8 | "script": { 9 | "actions": { 10 | "onRun": { 11 | "command": "python", 12 | "args": ["-c", "print('DoTask {{Param.J}}')"], 13 | } 14 | } 15 | }, 16 | } 17 | ], 18 | } -------------------------------------------------------------------------------- /test/openjd/cli/templates/simple_with_j_param_exit_1.yaml: -------------------------------------------------------------------------------- 1 | { 2 | "specificationVersion": "jobtemplate-2023-09", 3 | "name": "Test", 4 | "parameterDefinitions": [{"name": "J", "type": "STRING"}], 5 | "steps": [ 6 | { 7 | "name": "SimpleStep", 8 | "script": { 9 | "actions": { 10 | "onRun": { 11 | "command": "python", 12 | "args": ["-c", "import sys; print('DoTask'); sys.exit(1)"] 13 | } 14 | } 15 | } 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /test/openjd/cli/test_check_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import Namespace 4 | from pathlib import Path 5 | from typing import Callable 6 | import json 7 | import pytest 8 | import tempfile 9 | import yaml 10 | 11 | from . import MOCK_TEMPLATE 12 | from openjd.cli._check._check_command import do_check 13 | 14 | 15 | @pytest.mark.parametrize( 16 | "tempfile_extension,doc_serializer", 17 | [ 18 | pytest.param(".template.json", json.dump, id="Successful JSON"), 19 | pytest.param(".template.yaml", yaml.dump, id="Successful YAML"), 20 | ], 21 | ) 22 | def test_do_check_file_success(tempfile_extension: str, doc_serializer: Callable): 23 | """ 24 | Execution should succeed given a correct filepath and JSON/YAML body 25 | """ 26 | with tempfile.NamedTemporaryFile( 27 | mode="w+t", suffix=tempfile_extension, encoding="utf8", delete=False 28 | ) as temp_template: 29 | doc_serializer(MOCK_TEMPLATE, temp_template.file) 30 | 31 | mock_args = Namespace(path=Path(temp_template.name), output="human-readable", extensions="") 32 | do_check(mock_args) 33 | 34 | Path(temp_template.name).unlink() 35 | 36 | 37 | def test_do_check_file_error(): 38 | """ 39 | Raise a SystemExit on an error 40 | (RunTime and DecodeValidation errors are treated the same; 41 | in this case we just test an incorrect filename that gets 42 | handled in read_template) 43 | """ 44 | mock_args = Namespace(path=Path("error-file.json"), output="human-readable", extensions="") 45 | with pytest.raises(SystemExit): 46 | do_check(mock_args) 47 | 48 | 49 | def test_do_check_bundle_error(): 50 | """ 51 | Test that passing a bundle with no template file yields a SystemError 52 | """ 53 | with tempfile.TemporaryDirectory() as temp_bundle: 54 | mock_args = Namespace(path=Path(temp_bundle), output="human-readable", extensions="") 55 | with pytest.raises(SystemExit): 56 | do_check(mock_args) 57 | -------------------------------------------------------------------------------- /test/openjd/cli/test_chunked_job.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from pathlib import Path 4 | import re 5 | 6 | import pytest 7 | 8 | from . import run_openjd_cli_main, format_capsys_outerr 9 | 10 | CHUNKED_JOB_TEMPLATE_FILE = str(Path(__file__).parent / "templates" / "chunked_job.yaml") 11 | 12 | 13 | @pytest.mark.parametrize( 14 | "cli_options, expected_exit_code, expected_message_regex_list", 15 | [ 16 | ([], 0, [r"Template at '.*[\\/]chunked_job.yaml' passes validation checks."]), 17 | (["--extensions", ""], 1, ["Unsupported extension names: TASK_CHUNKING"]), 18 | ( 19 | ["--extensions", "TASK_CHUNKING"], 20 | 0, 21 | [r"Template at '.*[\\/]chunked_job.yaml' passes validation checks."], 22 | ), 23 | ], 24 | ) 25 | def test_openjd_check_on_chunked_job( 26 | capsys, cli_options: list[str], expected_exit_code: int, expected_message_regex_list: list[str] 27 | ) -> None: 28 | # Test that "openjd check" validates the chunked_job.yaml appropriately 29 | 30 | outerr = run_openjd_cli_main( 31 | capsys, 32 | args=["check", CHUNKED_JOB_TEMPLATE_FILE, *cli_options], 33 | expected_exit_code=expected_exit_code, 34 | ) 35 | 36 | for expected_message_regex in expected_message_regex_list: 37 | assert re.search( 38 | expected_message_regex, outerr.out 39 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 40 | 41 | 42 | def test_openjd_summary_on_chunked_job(capsys): 43 | # Test that "openjd summary" prints out correct information for chunked_job.yaml 44 | 45 | expected_message_regex_list = [ 46 | "Summary for 'Chunked Job'", 47 | "Total tasks: 40", 48 | r"1. 'Chunked Step' \(40 total Tasks\)", 49 | r"Item \(CHUNK\[INT\]\)", 50 | ] 51 | 52 | outerr = run_openjd_cli_main( 53 | capsys, args=["summary", CHUNKED_JOB_TEMPLATE_FILE], expected_exit_code=0 54 | ) 55 | 56 | for expected_message_regex in expected_message_regex_list: 57 | assert re.search( 58 | expected_message_regex, outerr.out, re.MULTILINE 59 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 60 | 61 | 62 | def test_openjd_run_on_chunked_job_default_options(capsys): 63 | # Test that "openjd run" runs the chunked_job.yaml with the expected chunks 64 | 65 | expected_message_regex_list = [ 66 | r"Item\(CHUNK\[INT\]\) = 1-10$", 67 | r"Item\(CHUNK\[INT\]\) = 11-20$", 68 | r"Item\(CHUNK\[INT\]\) = 21-30$", 69 | r"Item\(CHUNK\[INT\]\) = 31-40$", 70 | "Chunks run: 4$", 71 | ] 72 | 73 | outerr = run_openjd_cli_main( 74 | capsys, 75 | args=["run", CHUNKED_JOB_TEMPLATE_FILE, "--step", "Chunked Step"], 76 | expected_exit_code=0, 77 | ) 78 | 79 | for expected_message_regex in expected_message_regex_list: 80 | assert re.search( 81 | expected_message_regex, outerr.out, re.MULTILINE 82 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 83 | 84 | 85 | def test_openjd_run_on_chunked_job_adaptive_chunking(capsys): 86 | # Test that running chunked_job.yaml with adaptive chunking and the TargetRuntime cranked really high 87 | # results in two chunks, the first being one task, and the second being the remainder. 88 | 89 | expected_message_regex_list = [ 90 | r"Item\(CHUNK\[INT\]\) = 1$", 91 | r"Item\(CHUNK\[INT\]\) = 2-40$", 92 | "Chunks run: 2$", 93 | ] 94 | 95 | outerr = run_openjd_cli_main( 96 | capsys, 97 | args=[ 98 | "run", 99 | CHUNKED_JOB_TEMPLATE_FILE, 100 | "--step", 101 | "Chunked Step", 102 | "-p", 103 | "ChunkSize=1", 104 | "-p", 105 | "TargetRuntime=10000", 106 | ], 107 | expected_exit_code=0, 108 | ) 109 | 110 | for expected_message_regex in expected_message_regex_list: 111 | assert re.search( 112 | expected_message_regex, outerr.out, re.MULTILINE 113 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 114 | 115 | 116 | @pytest.mark.parametrize("target_runtime", [0, 1]) 117 | def test_openjd_run_on_chunked_job_maximum_task_count(capsys, target_runtime): 118 | # Test that running chunked_job.yaml with small chunks and a maximum task count will run the max count 119 | # Runs with TargetRuntime=0 (fixed chunk size) and TargetRuntime=1 (adaptive chunk size) to 120 | # exercise both task running inner loops. 121 | expected_message_regex_list = [ 122 | "Chunks run: 3$", 123 | ] 124 | 125 | outerr = run_openjd_cli_main( 126 | capsys, 127 | args=[ 128 | "run", 129 | CHUNKED_JOB_TEMPLATE_FILE, 130 | "--step", 131 | "Chunked Step", 132 | "-p", 133 | "ChunkSize=3", 134 | "-p", 135 | f"TargetRuntime={target_runtime}", 136 | "--maximum-tasks", 137 | "3", 138 | ], 139 | expected_exit_code=0, 140 | ) 141 | 142 | for expected_message_regex in expected_message_regex_list: 143 | assert re.search( 144 | expected_message_regex, outerr.out, re.MULTILINE 145 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 146 | 147 | 148 | PARAMETRIZE_CASES: tuple = ( 149 | pytest.param( 150 | ["-tp", "Item=0"], 151 | [ 152 | r"Parameter Item of type CHUNK\[INT\] value 0 is not a subset of the range in the parameter space." 153 | ], 154 | id="Item single value out of range", 155 | ), 156 | pytest.param( 157 | ["-tp", "Item=1;2"], 158 | [r"Parameter Item of type CHUNK\[INT\] value 1;2 is not a valid range expression"], 159 | id="Item is not a range expr", 160 | ), 161 | pytest.param( 162 | ["-tp", "Item=30-41"], 163 | [ 164 | r"Parameter Item of type CHUNK\[INT\] value 30-41 is not a subset of the range in the parameter space." 165 | ], 166 | id="Item interval out of range", 167 | ), 168 | ) 169 | 170 | 171 | @pytest.mark.parametrize("bad_task_params,expected_message_regex_list", PARAMETRIZE_CASES) 172 | def test_openjd_run_on_chunked_job_bad_task_params( 173 | capsys, bad_task_params, expected_message_regex_list 174 | ): 175 | # Test that running chunked_job.yaml with various bad task parameters fails with expected messages 176 | 177 | outerr = run_openjd_cli_main( 178 | capsys, 179 | args=[ 180 | "run", 181 | CHUNKED_JOB_TEMPLATE_FILE, 182 | "--step", 183 | "Chunked Step", 184 | *bad_task_params, 185 | ], 186 | expected_exit_code=1, 187 | ) 188 | 189 | for expected_message_regex in expected_message_regex_list: 190 | assert re.search( 191 | expected_message_regex, outerr.out, re.MULTILINE 192 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 193 | -------------------------------------------------------------------------------- /test/openjd/cli/test_local_session.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | import pytest 4 | from unittest.mock import call, patch 5 | import signal 6 | 7 | from . import SampleSteps, SESSION_PARAMETERS 8 | from openjd.model import StepParameterSpaceIterator 9 | from openjd.sessions import Session, SessionState 10 | from openjd.cli._run._local_session._session_manager import ( 11 | LocalSession, 12 | EnvironmentType, 13 | LocalSessionFailed, 14 | ) 15 | import openjd.cli._run._local_session._session_manager as local_session_mod 16 | 17 | 18 | @pytest.fixture(scope="function", autouse=True) 19 | def patched_actions(): 20 | """ 21 | Patch the `Session` actions to keep track of how many times 22 | they're called, but set their side effects to the original method 23 | so it has the same functionality. 24 | 25 | We also patch the action callback to make sure it's being called. 26 | 27 | (This is because the subprocesses in Sessions causes tests to 28 | hang when mocking actions directly, so we just run the Session 29 | to completion with short sample Jobs) 30 | """ 31 | with ( 32 | patch.object( 33 | Session, "enter_environment", autospec=True, side_effect=Session.enter_environment 34 | ) as patched_enter, 35 | patch.object( 36 | Session, "run_task", autospec=True, side_effect=Session.run_task 37 | ) as patched_run, 38 | patch.object( 39 | Session, "exit_environment", autospec=True, side_effect=Session.exit_environment 40 | ) as patched_exit, 41 | patch.object( 42 | LocalSession, 43 | "_action_callback", 44 | autospec=True, 45 | side_effect=LocalSession._action_callback, 46 | ) as patched_callback, 47 | ): 48 | yield patched_enter, patched_run, patched_exit, patched_callback 49 | 50 | 51 | @pytest.mark.usefixtures("sample_job_and_dirs") 52 | @pytest.mark.parametrize(*SESSION_PARAMETERS) 53 | def test_localsession_initialize( 54 | sample_job_and_dirs: tuple, 55 | step_index: int, 56 | maximum_tasks: int, 57 | parameter_sets: list[dict], 58 | num_expected_tasks: int, 59 | ): 60 | """ 61 | Test that initializing the local Session enters external and job environments, and is ready to run tasks. 62 | """ 63 | sample_job, sample_job_parameters, template_dir, current_working_dir = sample_job_and_dirs 64 | with ( 65 | patch.object( 66 | LocalSession, 67 | "run_environment_enters", 68 | autospec=True, 69 | side_effect=LocalSession.run_environment_enters, 70 | ) as patched_run_environment_enters, 71 | patch.object( 72 | LocalSession, "run_step", autospec=True, side_effect=LocalSession.run_step 73 | ) as patched_run_step, 74 | ): 75 | with LocalSession( 76 | job=sample_job, job_parameter_values=sample_job_parameters, session_id="my-session" 77 | ) as session: 78 | assert session._openjd_session.state == SessionState.READY 79 | 80 | # It should have entered the external and job environments in order 81 | assert patched_run_environment_enters.call_count == 2 82 | assert patched_run_environment_enters.call_args_list[0] == call( 83 | session, None, EnvironmentType.EXTERNAL 84 | ) 85 | assert patched_run_environment_enters.call_args_list[1] == call( 86 | session, sample_job.jobEnvironments, EnvironmentType.JOB 87 | ) 88 | 89 | # It should not have run any steps 90 | assert patched_run_step.call_count == 0 91 | 92 | 93 | @pytest.mark.usefixtures("sample_job_and_dirs") 94 | def test_localsession_traps_sigint(sample_job_and_dirs: tuple): 95 | # Make sure that we hook up, and remove the signal handler when using the local session 96 | sample_job, sample_job_parameters, template_dir, current_working_dir = sample_job_and_dirs 97 | 98 | # GIVEN 99 | with patch.object(local_session_mod, "signal") as signal_mod: 100 | # WHEN 101 | with LocalSession( 102 | job=sample_job, job_parameter_values=sample_job_parameters, session_id="test-id" 103 | ) as localsession: 104 | pass 105 | 106 | # THEN 107 | assert signal_mod.call_count == 4 108 | signal_mod.assert_has_calls( 109 | [ 110 | call(signal.SIGINT, localsession._sigint_handler), 111 | call(signal.SIGTERM, localsession._sigint_handler), 112 | call(signal.SIGINT, signal.SIG_DFL), 113 | call(signal.SIGTERM, signal.SIG_DFL), 114 | ] 115 | ) 116 | 117 | 118 | @pytest.mark.usefixtures("sample_job_and_dirs", "capsys") 119 | @pytest.mark.parametrize(*SESSION_PARAMETERS) 120 | def test_localsession_run_success( 121 | sample_job_and_dirs: tuple, 122 | capsys: pytest.CaptureFixture, 123 | step_index: int, 124 | maximum_tasks: int, 125 | parameter_sets: list[dict], 126 | num_expected_tasks: int, 127 | ): 128 | """ 129 | Test that calling `run_step` causes the local Session to run the tasks requested in that step. 130 | """ 131 | sample_job, sample_job_parameters, template_dir, current_working_dir = sample_job_and_dirs 132 | 133 | if parameter_sets is None: 134 | parameter_sets = StepParameterSpaceIterator( 135 | space=sample_job.steps[step_index].parameterSpace 136 | ) 137 | 138 | with ( 139 | patch.object( 140 | LocalSession, 141 | "run_environment_enters", 142 | autospec=True, 143 | side_effect=LocalSession.run_environment_enters, 144 | ) as patched_run_environment_enters, 145 | patch.object( 146 | LocalSession, 147 | "run_environment_exits", 148 | autospec=True, 149 | side_effect=LocalSession.run_environment_exits, 150 | ) as patched_run_environment_exits, 151 | patch.object( 152 | LocalSession, "run_step", autospec=True, side_effect=LocalSession.run_step 153 | ) as patched_run_step, 154 | patch.object( 155 | LocalSession, "run_task", autospec=True, side_effect=LocalSession.run_task 156 | ) as patched_run_task, 157 | ): 158 | with LocalSession( 159 | job=sample_job, job_parameter_values=sample_job_parameters, session_id="my-session" 160 | ) as session: 161 | session.run_step( 162 | sample_job.steps[step_index], 163 | task_parameters=parameter_sets, 164 | maximum_tasks=maximum_tasks, 165 | ) 166 | 167 | # It should have entered the environments in order 168 | assert patched_run_environment_enters.call_args_list == [ 169 | call(session, None, EnvironmentType.EXTERNAL), 170 | call(session, sample_job.jobEnvironments, EnvironmentType.JOB), 171 | call(session, sample_job.steps[step_index].stepEnvironments, EnvironmentType.STEP), 172 | ] 173 | # It should have run one step 174 | assert patched_run_step.call_args_list == [ 175 | call( 176 | session, 177 | sample_job.steps[step_index], 178 | task_parameters=parameter_sets, 179 | maximum_tasks=maximum_tasks, 180 | ) 181 | ] 182 | # It should have exited the environments in reverse order 183 | assert patched_run_environment_exits.call_args_list == [ 184 | call(session, type=EnvironmentType.STEP, keep_session_running=True), 185 | call(session, type=EnvironmentType.ALL, keep_session_running=False), 186 | ] 187 | 188 | assert patched_run_task.call_count == num_expected_tasks 189 | 190 | assert ( 191 | "Open Job Description CLI: All actions completed successfully!" 192 | in capsys.readouterr().out 193 | ) 194 | 195 | 196 | @pytest.mark.usefixtures("sample_job_and_dirs", "capsys") 197 | def test_localsession_run_failed(sample_job_and_dirs: tuple, capsys: pytest.CaptureFixture): 198 | """ 199 | Test that a LocalSession can gracefully handle an error in its inner Session. 200 | """ 201 | sample_job, sample_job_parameters, template_dir, current_working_dir = sample_job_and_dirs 202 | with ( 203 | patch.object( 204 | LocalSession, 205 | "run_environment_enters", 206 | autospec=True, 207 | side_effect=LocalSession.run_environment_enters, 208 | ) as patched_run_environment_enters, 209 | ): 210 | with LocalSession( 211 | job=sample_job, job_parameter_values=sample_job_parameters, session_id="bad-session" 212 | ) as session: 213 | with pytest.raises(LocalSessionFailed): 214 | session.run_step(sample_job.steps[SampleSteps.BadCommand]) 215 | 216 | # The Task has failed. That means that we've entered the one environment and also exited it. 217 | assert patched_run_environment_enters.call_args_list == [ 218 | call(session, None, EnvironmentType.EXTERNAL), 219 | call(session, sample_job.jobEnvironments, EnvironmentType.JOB), 220 | call( 221 | session, 222 | sample_job.steps[SampleSteps.BadCommand].stepEnvironments, 223 | EnvironmentType.STEP, 224 | ), 225 | ] 226 | session._openjd_session.exit_environment.assert_called_once() # type: ignore 227 | assert session.failed 228 | assert session._cleanup_called 229 | assert "Open Job Description CLI: ERROR" in capsys.readouterr().out 230 | -------------------------------------------------------------------------------- /test/openjd/cli/test_redacted_env.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from pathlib import Path 4 | import re 5 | 6 | from . import run_openjd_cli_main, format_capsys_outerr 7 | 8 | TEMPLATE_DIR = Path(__file__).parent / "templates" 9 | 10 | 11 | def test_run_job_with_redacted_env(capsys): 12 | """Test that environment variables set with openjd_redacted_env are properly handled.""" 13 | outerr = run_openjd_cli_main( 14 | capsys, 15 | args=[ 16 | "run", 17 | str(TEMPLATE_DIR / "redacted_env.yaml"), 18 | ], 19 | expected_exit_code=0, 20 | ) 21 | 22 | # Verify the environment variables were set 23 | for expected_message_regex in [ 24 | "Setting redacted vars", 25 | "SECRETVAR is \\*\\*\\*\\*\\*\\*\\*\\*", 26 | "KEYSPACE is None", 27 | "VALSPACE is \\*\\*\\*\\*\\*\\*\\*\\*", 28 | "MULTILINE is \\*\\*\\*\\*\\*\\*\\*\\*", 29 | ]: 30 | assert re.search( 31 | expected_message_regex, outerr.out 32 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 33 | 34 | # Verify the openjd_redacted_env lines are not in the output 35 | for unexpected_message in [ 36 | "openjd_redacted_env: SECRETVAR=SECRETVAL", 37 | "openjd_redacted_env: KEYSPACE =SECRETVAL", 38 | "openjd_redacted_env: VALSPACE= SPACEVAL", 39 | "first_line", 40 | "second_line", 41 | "third_line", 42 | ]: 43 | assert ( 44 | unexpected_message not in outerr.out 45 | ), f"Found unexpected line in output:\n{format_capsys_outerr(outerr)}" 46 | -------------------------------------------------------------------------------- /test/openjd/cli/test_run_with_env.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from pathlib import Path 4 | import re 5 | 6 | from . import run_openjd_cli_main, format_capsys_outerr 7 | 8 | TEMPLATE_DIR = Path(__file__).parent / "templates" 9 | 10 | 11 | def test_run_job_with_env_default_params(capsys): 12 | # Run a job with env_with_param as an external environment, 13 | # leaving the environment's parameter at its default value 14 | 15 | outerr = run_openjd_cli_main( 16 | capsys, 17 | args=[ 18 | "run", 19 | str(TEMPLATE_DIR / "simple_with_j_param.yaml"), 20 | "-p", 21 | "J=Jvalue", 22 | "--environment", 23 | str(TEMPLATE_DIR / "env_with_param.yaml"), 24 | ], 25 | expected_exit_code=0, 26 | ) 27 | 28 | for expected_message_regex in [ 29 | "EnvWithParam Enter DefaultForEnvParam", 30 | "DoTask Jvalue", 31 | "EnvWithParam Exit DefaultForEnvParam", 32 | ]: 33 | assert re.search( 34 | expected_message_regex, outerr.out 35 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 36 | 37 | 38 | def test_run_job_with_env_provide_env_param(capsys): 39 | # Run a job with env_with_param as an external environment, 40 | # explicitly providing the env parameter 41 | 42 | outerr = run_openjd_cli_main( 43 | capsys, 44 | args=[ 45 | "run", 46 | str(TEMPLATE_DIR / "simple_with_j_param.yaml"), 47 | "-p", 48 | "J=Jvalue", 49 | "-p", 50 | "EnvParam=EnvParamValue", 51 | "--environment", 52 | str(TEMPLATE_DIR / "env_with_param.yaml"), 53 | ], 54 | expected_exit_code=0, 55 | ) 56 | 57 | for expected_message_regex in [ 58 | "EnvWithParam Enter EnvParamValue", 59 | "DoTask Jvalue", 60 | "EnvWithParam Exit EnvParamValue", 61 | ]: 62 | assert re.search( 63 | expected_message_regex, outerr.out 64 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 65 | -------------------------------------------------------------------------------- /test/openjd/cli/test_schema_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from openjd.cli._schema._schema_command import do_get_schema, _process_regex 4 | from openjd.model import TemplateSpecificationVersion 5 | import openjd.model.v2023_09 6 | 7 | from argparse import Namespace 8 | import json 9 | import pytest 10 | from unittest.mock import Mock, patch 11 | 12 | 13 | @pytest.mark.parametrize( 14 | "target,expected_result", 15 | [ 16 | pytest.param( 17 | {"attr": "value", "pattern": r"(?-m:^[^\u0000-\u001F\u007F-\u009F]+\Z)"}, 18 | {"attr": "value", "pattern": r"(?:^[^\u0000-\u001F\u007F-\u009F]+$)"}, 19 | id="Standard dictionary with pattern attribute", 20 | ), 21 | pytest.param( 22 | {"nested_dict": {"pattern": r"(?-m:^[^\u0000-\u001F\u007F-\u009F]+\Z)"}}, 23 | {"nested_dict": {"pattern": r"(?:^[^\u0000-\u001F\u007F-\u009F]+$)"}}, 24 | id="Pattern in nested dictionary", 25 | ), 26 | pytest.param( 27 | { 28 | "pattern": r"(?-m:^[^\u0000-\u001F\u007F-\u009F]+\Z)", 29 | "nested_dict": {"pattern": r"(?-m:^[^\u0000-\u001F\u007F-\u009F]+\Z)"}, 30 | }, 31 | { 32 | "pattern": r"(?:^[^\u0000-\u001F\u007F-\u009F]+$)", 33 | "nested_dict": {"pattern": r"(?:^[^\u0000-\u001F\u007F-\u009F]+$)"}, 34 | }, 35 | id="Patterns in multiple levels", 36 | ), 37 | pytest.param( 38 | {"pattern": "NotRealRegex"}, {"pattern": "NotRealRegex"}, id="Unaffected pattern" 39 | ), 40 | pytest.param( 41 | {"pattern": ["not", "a", "string"]}, 42 | {"pattern": ["not", "a", "string"]}, 43 | id="Non-string pattern attribute", 44 | ), 45 | pytest.param({}, {}, id="Empty dictionary"), 46 | ], 47 | ) 48 | def test_process_regex(target: dict, expected_result: dict): 49 | """ 50 | Test that the `schema` command can process Python-specific regex 51 | into JSON-compatible regex. 52 | """ 53 | _process_regex(target) 54 | 55 | assert target == expected_result 56 | 57 | 58 | @pytest.mark.usefixtures("capsys") 59 | def test_do_get_schema_success(capsys: pytest.CaptureFixture): 60 | """ 61 | Test that the `schema` command returns a correctly-formed 62 | JSON body with specific Job template attributes. 63 | """ 64 | with patch( 65 | "openjd.cli._schema._schema_command._process_regex", new=Mock(side_effect=_process_regex) 66 | ) as patched_process_regex: 67 | do_get_schema( 68 | Namespace( 69 | version=TemplateSpecificationVersion.JOBTEMPLATE_v2023_09.value, 70 | output="human-readable", 71 | ) 72 | ) 73 | patched_process_regex.assert_called() 74 | 75 | model_output = capsys.readouterr().out 76 | model_json = json.loads(model_output) 77 | 78 | assert model_json is not None 79 | assert model_json["title"] == "JobTemplate" 80 | assert "specificationVersion" in model_json["properties"] 81 | assert "name" in model_json["properties"] 82 | assert "steps" in model_json["properties"] 83 | 84 | 85 | @pytest.mark.usefixtures("capsys") 86 | def test_do_get_schema_success_environment(capsys: pytest.CaptureFixture): 87 | """ 88 | Test that the `schema` command returns a correctly-formed 89 | JSON body with specific Environment template attributes. 90 | """ 91 | with patch( 92 | "openjd.cli._schema._schema_command._process_regex", new=Mock(side_effect=_process_regex) 93 | ) as patched_process_regex: 94 | do_get_schema( 95 | Namespace( 96 | version=TemplateSpecificationVersion.ENVIRONMENT_v2023_09.value, 97 | output="human-readable", 98 | ) 99 | ) 100 | patched_process_regex.assert_called() 101 | 102 | model_output = capsys.readouterr().out 103 | model_json = json.loads(model_output) 104 | 105 | assert model_json is not None 106 | assert model_json["title"] == "EnvironmentTemplate" 107 | assert "specificationVersion" in model_json["properties"] 108 | 109 | 110 | @pytest.mark.usefixtures("capsys") 111 | def test_do_get_schema_incorrect_version(capsys: pytest.CaptureFixture): 112 | """ 113 | Test that the `schema` command fails if an unsupported version string 114 | is supplied. 115 | """ 116 | 117 | with pytest.raises(SystemExit): 118 | do_get_schema(Namespace(version="badversion", output="human-readable")) 119 | output = capsys.readouterr().out 120 | 121 | assert "Cannot generate schema for version 'badversion'" in output 122 | 123 | 124 | @pytest.mark.usefixtures("capsys") 125 | def test_do_get_schema_error(capsys: pytest.CaptureFixture): 126 | """ 127 | Test that the `schema` command can recover from an error 128 | when generating the JSON schema. 129 | """ 130 | 131 | with ( 132 | patch.object(openjd.model.v2023_09, "JobTemplate") as job_template_class, 133 | pytest.raises(SystemExit), 134 | ): 135 | job_template_class.model_json_schema.side_effect = RuntimeError("Test error") 136 | do_get_schema( 137 | Namespace( 138 | version=TemplateSpecificationVersion.JOBTEMPLATE_v2023_09.value, 139 | output="human-readable", 140 | ) 141 | ) 142 | output = capsys.readouterr().out 143 | 144 | assert "Test error" in output 145 | -------------------------------------------------------------------------------- /test/openjd/cli/test_summary_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import Namespace 4 | from pathlib import Path 5 | from unittest.mock import Mock, patch 6 | from typing import Optional 7 | import json 8 | import pytest 9 | import tempfile 10 | 11 | from . import MOCK_TEMPLATE, MOCK_TEMPLATE_REQUIRES_PARAMS 12 | from openjd.cli._summary._summary_command import do_summary 13 | from openjd.cli._summary._summary_output import ( 14 | OpenJDJobSummaryResult, 15 | OpenJDStepSummaryResult, 16 | output_summary_result, 17 | ) 18 | 19 | from openjd.model import ( 20 | JobParameterValues, 21 | ParameterValue, 22 | ParameterValueType, 23 | create_job, 24 | decode_job_template, 25 | ) 26 | 27 | 28 | @pytest.mark.parametrize( 29 | "mock_params,mock_step,template", 30 | [ 31 | pytest.param(None, None, MOCK_TEMPLATE, id="No extra options"), 32 | pytest.param(None, "NormalStep", MOCK_TEMPLATE, id="Step given"), 33 | pytest.param( 34 | ["RequiredParam=5"], 35 | None, 36 | MOCK_TEMPLATE_REQUIRES_PARAMS, 37 | id="Job params given", 38 | ), 39 | pytest.param( 40 | ["RequiredParam=5"], 41 | "step1", 42 | MOCK_TEMPLATE_REQUIRES_PARAMS, 43 | id="Step & Job params given", 44 | ), 45 | ], 46 | ) 47 | def test_do_summary_success( 48 | mock_params: Optional[list[str]], 49 | mock_step: Optional[str], 50 | template: dict, 51 | ): 52 | """ 53 | Test that the `summary` command succeeds with various argument options. 54 | """ 55 | with tempfile.NamedTemporaryFile( 56 | mode="w+t", suffix=".template.json", encoding="utf8", delete=False 57 | ) as temp_template: 58 | json.dump(template, temp_template.file) 59 | 60 | mock_args = Namespace( 61 | path=Path(temp_template.name), 62 | job_params=mock_params, 63 | step=mock_step, 64 | output="human-readable", 65 | extensions="", 66 | ) 67 | do_summary(mock_args) 68 | 69 | Path(temp_template.name).unlink() 70 | 71 | 72 | def test_do_summary_error(): 73 | """ 74 | Test that the `summary` command exits on any error (in this case, we mock an error in `read_template`) 75 | """ 76 | mock_args = Namespace(path=Path("some-file.json"), output="human-readable", extensions="") 77 | with ( 78 | patch("openjd.cli._common.read_template", new=Mock(side_effect=RuntimeError())), 79 | pytest.raises(SystemExit), 80 | ): 81 | do_summary(mock_args) 82 | 83 | 84 | PARAMETRIZE_CASES: tuple = ( 85 | pytest.param( 86 | JobParameterValues({}), 87 | "my-job", 88 | "BareStep", 89 | 1, 90 | [], 91 | 0, 92 | MOCK_TEMPLATE, 93 | id="No Job parameters, dependencies, or environments", 94 | ), 95 | pytest.param( 96 | JobParameterValues({}), 97 | "my-job", 98 | "DependentStep", 99 | 1, 100 | ["NormalStep"], 101 | 0, 102 | MOCK_TEMPLATE, 103 | id="With dependencies", 104 | ), 105 | pytest.param( 106 | JobParameterValues({}), 107 | "my-job", 108 | "NormalStep", 109 | 1, 110 | [], 111 | 1, 112 | MOCK_TEMPLATE, 113 | id="With environments", 114 | ), 115 | pytest.param( 116 | JobParameterValues( 117 | { 118 | "Title": ParameterValue(type=ParameterValueType.STRING, value="new title"), 119 | "RequiredParam": ParameterValue(type=ParameterValueType.INT, value="5"), 120 | } 121 | ), 122 | "new title", 123 | "step1", 124 | 1, 125 | [], 126 | 0, 127 | MOCK_TEMPLATE_REQUIRES_PARAMS, 128 | id="Job parameters supplied", 129 | ), 130 | pytest.param( 131 | {}, 132 | "template", 133 | "step1", 134 | 5, 135 | [], 136 | 0, 137 | { 138 | "specificationVersion": "jobtemplate-2023-09", 139 | "name": "template", 140 | "steps": [ 141 | { 142 | "name": "step1", 143 | "parameterSpace": { 144 | "taskParameterDefinitions": [ 145 | {"name": "taskNumber", "type": "INT", "range": [1, 2, 3, 4, 5]} 146 | ] 147 | }, 148 | "script": { 149 | "actions": { 150 | "onRun": {"command": 'echo "Task ran {{Task.Param.taskNumber}} times"'} 151 | } 152 | }, 153 | } 154 | ], 155 | }, 156 | id="With Task parameters", 157 | ), 158 | pytest.param( 159 | JobParameterValues({"Runs": ParameterValue(type=ParameterValueType.INT, value="7")}), 160 | "template", 161 | "step1", 162 | 8, 163 | [], 164 | 0, 165 | { 166 | "specificationVersion": "jobtemplate-2023-09", 167 | "name": "template", 168 | "parameterDefinitions": [{"name": "Runs", "type": "INT", "default": 1}], 169 | "steps": [ 170 | { 171 | "name": "step1", 172 | "parameterSpace": { 173 | "taskParameterDefinitions": [ 174 | {"name": "taskNumber", "type": "INT", "range": "0-{{Param.Runs}}"} 175 | ] 176 | }, 177 | "script": { 178 | "actions": { 179 | "onRun": {"command": 'echo "Task ran {{Task.Param.taskNumber}} times"'} 180 | } 181 | }, 182 | } 183 | ], 184 | }, 185 | id="Task parameters set by Job parameter", 186 | ), 187 | pytest.param( 188 | {}, 189 | "template", 190 | "step1", 191 | 10, 192 | [], 193 | 0, 194 | { 195 | "specificationVersion": "jobtemplate-2023-09", 196 | "name": "template", 197 | "steps": [ 198 | { 199 | "name": "step1", 200 | "parameterSpace": { 201 | "taskParameterDefinitions": [ 202 | {"name": "param1", "type": "INT", "range": [1, 2, 3, 4, 5]}, 203 | {"name": "param2", "type": "INT", "range": [6, 7, 8, 9, 10]}, 204 | {"name": "param3", "type": "STRING", "range": ["yes", "no"]}, 205 | ], 206 | "combination": "(param1, param2) * param3", 207 | }, 208 | "script": { 209 | "actions": { 210 | "onRun": { 211 | "command": 'echo "{{Task.Param.param1}} {{Task.Param.param2}} {{Task.Param.param3}}"' 212 | } 213 | } 214 | }, 215 | }, 216 | ], 217 | }, 218 | id="Task parameters with combination expression", 219 | ), 220 | ) 221 | 222 | 223 | @pytest.mark.parametrize( 224 | "mock_job_params,expected_job_name,step_name,expected_tasks,expected_dependencies,expected_total_envs,template_dict", 225 | PARAMETRIZE_CASES, 226 | ) 227 | def test_get_output_step_summary_success( 228 | mock_job_params: JobParameterValues, 229 | expected_job_name: str, 230 | step_name: str, 231 | expected_tasks: int, 232 | expected_dependencies: list, 233 | expected_total_envs: int, 234 | template_dict: dict, 235 | ) -> None: 236 | """ 237 | Test that `output_summary_result` returns an object with the expected values when called with a Step. 238 | """ 239 | template = decode_job_template(template=template_dict, supported_extensions=[]) 240 | job = create_job(job_template=template, job_parameter_values=mock_job_params) 241 | 242 | response = output_summary_result(job, step_name) 243 | assert isinstance(response, OpenJDStepSummaryResult) 244 | assert response.status == "success" 245 | assert response.job_name == expected_job_name 246 | assert response.step_name == step_name 247 | assert response.total_tasks == expected_tasks 248 | assert response.total_environments == expected_total_envs 249 | 250 | if response.dependencies: 251 | assert (dep.step_name in expected_dependencies for dep in response.dependencies) 252 | 253 | 254 | def test_output_step_summary_result_error(): 255 | """ 256 | Test that `output_summary_result` throws an error if a non-existent Step name is provided. 257 | (function only has one error state) 258 | """ 259 | template = decode_job_template(template=MOCK_TEMPLATE, supported_extensions=[]) 260 | job = create_job(job_template=template, job_parameter_values={}) 261 | 262 | response = output_summary_result(job, "no step") 263 | assert response.status == "error" 264 | assert "Step 'no step' does not exist in Job 'my-job'" in response.message 265 | 266 | 267 | PARAMETRIZE_CASES = ( 268 | pytest.param( 269 | {}, 270 | "template", 271 | [], 272 | ["step1"], 273 | 1, 274 | 0, 275 | [], 276 | { 277 | "specificationVersion": "jobtemplate-2023-09", 278 | "name": "template", 279 | "steps": [ 280 | { 281 | "name": "step1", 282 | "script": {"actions": {"onRun": {"command": 'echo "Hello, world!"'}}}, 283 | } 284 | ], 285 | }, 286 | id="No parameters or environments", 287 | ), 288 | pytest.param( 289 | {}, 290 | "DefaultValue", 291 | [("NameParam", "DefaultValue")], 292 | ["step1"], 293 | 1, 294 | 0, 295 | [], 296 | { 297 | "specificationVersion": "jobtemplate-2023-09", 298 | "name": "{{Param.NameParam}}", 299 | "parameterDefinitions": [ 300 | {"name": "NameParam", "type": "STRING", "default": "DefaultValue"} 301 | ], 302 | "steps": [ 303 | { 304 | "name": "step1", 305 | "script": {"actions": {"onRun": {"command": 'echo "Hello, world!"'}}}, 306 | } 307 | ], 308 | }, 309 | id="Default parameters", 310 | ), 311 | pytest.param( 312 | JobParameterValues( 313 | {"NameParam": ParameterValue(type=ParameterValueType.STRING, value="NewName")} 314 | ), 315 | "NewName", 316 | [("NameParam", "NewName")], 317 | ["step1"], 318 | 1, 319 | 0, 320 | [], 321 | { 322 | "specificationVersion": "jobtemplate-2023-09", 323 | "name": "{{Param.NameParam}}", 324 | "parameterDefinitions": [ 325 | {"name": "NameParam", "type": "STRING", "default": "DefaultValue"} 326 | ], 327 | "steps": [ 328 | { 329 | "name": "step1", 330 | "script": {"actions": {"onRun": {"command": 'echo "Hello, world!"'}}}, 331 | } 332 | ], 333 | }, 334 | id="Overwritten parameters", 335 | ), 336 | pytest.param( 337 | {}, 338 | "template", 339 | [], 340 | ["step1"], 341 | 1, 342 | 1, 343 | ["aRootEnv"], 344 | { 345 | "specificationVersion": "jobtemplate-2023-09", 346 | "name": "template", 347 | "jobEnvironments": [{"name": "aRootEnv", "variables": {"variable": "value"}}], 348 | "steps": [ 349 | { 350 | "name": "step1", 351 | "script": {"actions": {"onRun": {"command": 'echo "Hello, world!"'}}}, 352 | } 353 | ], 354 | }, 355 | id="Root environments only", 356 | ), 357 | pytest.param( 358 | {}, 359 | "template", 360 | [], 361 | ["step1"], 362 | 1, 363 | 1, 364 | [], 365 | { 366 | "specificationVersion": "jobtemplate-2023-09", 367 | "name": "template", 368 | "steps": [ 369 | { 370 | "name": "step1", 371 | "script": {"actions": {"onRun": {"command": 'echo "Hello, world!"'}}}, 372 | "stepEnvironments": [{"name": "aStepEnv", "variables": {"variable": "value"}}], 373 | } 374 | ], 375 | }, 376 | id="Step environments only", 377 | ), 378 | pytest.param( 379 | {}, 380 | "template", 381 | [], 382 | ["step1"], 383 | 1, 384 | 2, 385 | ["aRootEnv"], 386 | { 387 | "specificationVersion": "jobtemplate-2023-09", 388 | "name": "template", 389 | "jobEnvironments": [{"name": "aRootEnv", "variables": {"variable": "value"}}], 390 | "steps": [ 391 | { 392 | "name": "step1", 393 | "script": {"actions": {"onRun": {"command": 'echo "Hello, world!"'}}}, 394 | "stepEnvironments": [{"name": "aStepEnv", "variables": {"variable": "value"}}], 395 | } 396 | ], 397 | }, 398 | id="Root and Step level environments", 399 | ), 400 | pytest.param( 401 | {}, 402 | "template", 403 | [], 404 | ["step1", "step2"], 405 | 2, 406 | 2, 407 | [], 408 | { 409 | "specificationVersion": "jobtemplate-2023-09", 410 | "name": "template", 411 | "steps": [ 412 | { 413 | "name": "step1", 414 | "script": {"actions": {"onRun": {"command": 'echo "We can have lots of fun"'}}}, 415 | "stepEnvironments": [{"name": "step1Env", "variables": {"variable": "value"}}], 416 | }, 417 | { 418 | "name": "step2", 419 | "script": { 420 | "actions": {"onRun": {"command": 'echo "There\'s so much we can do"'}} 421 | }, 422 | "stepEnvironments": [{"name": "step2Env", "variables": {"variable": "value"}}], 423 | }, 424 | ], 425 | }, 426 | id="Environments in multiple steps", 427 | ), 428 | ) 429 | 430 | 431 | @pytest.mark.parametrize( 432 | "mock_params,expected_name,expected_params,expected_steps,expected_total_tasks,expected_total_envs,expected_root_envs,template_dict", 433 | PARAMETRIZE_CASES, 434 | ) 435 | def test_output_job_summary_result_success( 436 | mock_params: JobParameterValues, 437 | expected_name: str, 438 | expected_params: list, 439 | expected_steps: list, 440 | expected_total_tasks: int, 441 | expected_total_envs: int, 442 | expected_root_envs: list, 443 | template_dict: dict, 444 | ): 445 | """ 446 | Test that `output_summary_result` returns an object with the expected values when called on a Job. 447 | """ 448 | template = decode_job_template(template=template_dict, supported_extensions=[]) 449 | job = create_job(job_template=template, job_parameter_values=mock_params) 450 | 451 | response = output_summary_result(job) 452 | assert isinstance(response, OpenJDJobSummaryResult) 453 | assert response.status == "success" 454 | assert response.name == expected_name 455 | 456 | if response.parameter_definitions: 457 | assert [ 458 | (param.name, param.value) for param in response.parameter_definitions 459 | ] == expected_params 460 | 461 | assert response.total_steps == len(expected_steps) 462 | assert [step.name for step in response.steps] == expected_steps 463 | 464 | assert response.total_tasks == expected_total_tasks 465 | 466 | assert response.total_environments == expected_total_envs 467 | if response.root_environments: 468 | assert [env.name for env in response.root_environments] == expected_root_envs 469 | -------------------------------------------------------------------------------- /test/openjd/test_main.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | """Tests for __main__""" 4 | 5 | from unittest.mock import Mock, patch 6 | import pytest 7 | import sys 8 | 9 | from openjd import __main__ 10 | from openjd.model import TemplateSpecificationVersion 11 | 12 | 13 | @patch("openjd.cli._check.do_check") 14 | def test_cli_check_success(mock_check: Mock): 15 | """ 16 | Test that we can call the `check` command at the entrypoint. 17 | """ 18 | 19 | mock_check.assert_not_called() 20 | mock_args = ["openjd", "check", "some-file.json"] 21 | with patch.object(sys, "argv", new=mock_args): 22 | __main__.main() 23 | mock_check.assert_called_once() 24 | 25 | 26 | @patch("openjd.cli._summary.do_summary") 27 | @pytest.mark.parametrize( 28 | "mock_args", 29 | [ 30 | pytest.param( 31 | ["some-template.json"], 32 | id="Base summary command", 33 | ), 34 | pytest.param(["some-template.json", "--step", "step-name"], id="Summary command with step"), 35 | pytest.param( 36 | ["some-template.json", "--job-param", "param=value"], 37 | id="Summary command with job parameters", 38 | ), 39 | pytest.param( 40 | [ 41 | "some-template.json", 42 | "--job-param", 43 | "param=value", 44 | "--step", 45 | "step-name", 46 | "--output", 47 | "json", 48 | ], 49 | id="Summary command with step, job params, and output", 50 | ), 51 | pytest.param( 52 | [ 53 | "some-template.json", 54 | "--job-param", 55 | "param1=value1", 56 | "--job-param", 57 | "param2=value2", 58 | ], 59 | id="Multiple Job parameters", 60 | ), 61 | ], 62 | ) 63 | def test_cli_summary_success(mock_summary: Mock, mock_args: list): 64 | """ 65 | Test that we can call the `summary` command at the entrypoint. 66 | """ 67 | 68 | mock_summary.assert_not_called() 69 | with patch.object(sys, "argv", new=(["openjd", "summary"] + mock_args)): 70 | __main__.main() 71 | mock_summary.assert_called_once() 72 | 73 | 74 | @patch("openjd.cli._run.do_run") 75 | @pytest.mark.parametrize( 76 | "mock_args", 77 | [ 78 | pytest.param(["some-template.json", "--step", "step1"], id="Base run command"), 79 | pytest.param( 80 | ["some-template.json", "--step", "step1", "-p", "param=value", "-p", "param2=value2"], 81 | id="With multiple Job parameters", 82 | ), 83 | pytest.param( 84 | ["some-template.json", "--step", "step1", "-p", "param=value1", "-p", "param2="], 85 | id="With an empty string Job parameter value", 86 | ), 87 | pytest.param( 88 | [ 89 | "some-template.json", 90 | "--step", 91 | "step1", 92 | "-tp", 93 | "param1=value1 param2=value2", 94 | "-tp", 95 | "param1=newvalue1 param2=newvalue2", 96 | ], 97 | id="With Task parameter sets", 98 | ), 99 | pytest.param( 100 | [ 101 | "some-template.json", 102 | "--step", 103 | "step1", 104 | "-p", 105 | "jobparam=paramvalue", 106 | "-tp", 107 | "taskparam=paramvalue", 108 | "--run-dependencies", 109 | "--path-mapping-rules", 110 | '[{"source_os": "someOS", "source_path": "some\\path", "destination_path": "some/new/path"}]', 111 | "--output", 112 | "json", 113 | ], 114 | id="With all optional arguments (-tp)", 115 | ), 116 | pytest.param( 117 | [ 118 | "some-template.json", 119 | "--step", 120 | "step1", 121 | "-p", 122 | "jobparam=paramvalue", 123 | "--tasks", 124 | '[{"taskparam": "paramvalue"}]', 125 | "--run-dependencies", 126 | "--path-mapping-rules", 127 | '[{"source_os": "someOS", "source_path": "some\\path", "destination_path": "some/new/path"}]', 128 | "--output", 129 | "json", 130 | ], 131 | id="With all optional arguments (--tasks)", 132 | ), 133 | pytest.param( 134 | [ 135 | "some-template.json", 136 | "--step", 137 | "step1", 138 | "-p", 139 | "jobparam=paramvalue", 140 | "--maximum-tasks", 141 | "1", 142 | "--run-dependencies", 143 | "--path-mapping-rules", 144 | '[{"source_os": "someOS", "source_path": "some\\path", "destination_path": "some/new/path"}]', 145 | "--output", 146 | "json", 147 | ], 148 | id="With all optional arguments (--maximum-tasks)", 149 | ), 150 | ], 151 | ) 152 | def test_cli_run_success(mock_run: Mock, mock_args: list): 153 | """ 154 | Test that we can call the `run` command at the entrypoint. 155 | """ 156 | 157 | mock_run.assert_not_called() 158 | with patch.object(sys, "argv", new=(["openjd", "run"] + mock_args)): 159 | __main__.main() 160 | mock_run.assert_called_once() 161 | 162 | 163 | @patch("openjd.cli._schema.do_get_schema") 164 | def test_cli_schema_success(mock_schema: Mock): 165 | """ 166 | Test that we can call the `schema` command at the entrypoint. 167 | """ 168 | 169 | mock_schema.assert_not_called() 170 | # "UNDEFINED" should always be a valid TemplateSpecificationVersion option, even though the unpatched 171 | # `do_get_schema` function throws an error on receiving it 172 | with patch.object( 173 | sys, "argv", new=(["openjd", "schema", "--version", TemplateSpecificationVersion.UNDEFINED]) 174 | ): 175 | __main__.main() 176 | mock_schema.assert_called_once() 177 | 178 | 179 | @pytest.mark.parametrize( 180 | "mock_args", 181 | [ 182 | pytest.param( 183 | ["notarealcommand", "some-file.json"], 184 | id="Non-existent command", 185 | ), 186 | pytest.param(["check"], id="Not enough arguments"), 187 | pytest.param(["summary", "template.json", "--job-param"], id="Missing argument value"), 188 | pytest.param(["summary", "template.json", "notarealarg"], id="Unexpected argument"), 189 | pytest.param( 190 | ["run", "somefile.json", "-tp", "Foo=Bar", "--tasks", '[{"Foo": "Bar"}]'], 191 | id="-tp/--tasks mutually exclusive", 192 | ), 193 | pytest.param( 194 | ["run", "somefile.json", "-tp", "Foo=Bar", "--maximum-tasks", "1"], 195 | id="-tp/--maximum-tasks mutually exclusive", 196 | ), 197 | pytest.param( 198 | ["run", "somefile.json", "--tasks", '[{"Foo": "Bar"}]', "--maximum-tasks", "1"], 199 | id="--tasks/--maximum-tasks mutually exclusive", 200 | ), 201 | ], 202 | ) 203 | def test_cli_argument_errors(mock_args: list): 204 | """ 205 | Tests that various formatting errors with Argparse cause the program to exit with an error. 206 | """ 207 | 208 | with patch.object(sys, "argv", new=(["openjd"] + mock_args)), pytest.raises(SystemExit): 209 | __main__.main() 210 | --------------------------------------------------------------------------------