├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── doc.yml │ ├── maintenance.yml │ ├── feature_request.yml │ └── bug.yml ├── workflows │ ├── record_pr.yml │ ├── responded.yml │ ├── codeql.yml │ ├── stale_prs_and_issues.yml │ ├── auto_approve.yml │ ├── release_bump.yml │ ├── code_quality.yml │ ├── on_opened_pr.yml │ └── release_publish.yml ├── dependabot.yml ├── scripts │ └── get_latest_changelog.py └── PULL_REQUEST_TEMPLATE.md ├── requirements-release.txt ├── requirements-development.txt ├── NOTICE ├── src └── openjd │ ├── cli │ ├── py.typed │ ├── _run │ │ ├── _local_session │ │ │ ├── __init__.py │ │ │ ├── _logs.py │ │ │ └── _actions.py │ │ ├── __init__.py │ │ └── _help_formatter.py │ ├── __init__.py │ ├── _schema │ │ ├── __init__.py │ │ └── _schema_command.py │ ├── _summary │ │ ├── __init__.py │ │ ├── _summary_command.py │ │ └── _summary_output.py │ ├── _check │ │ ├── __init__.py │ │ └── _check_command.py │ ├── _common │ │ ├── _extensions.py │ │ ├── _validation_utils.py │ │ ├── _job_from_template.py │ │ └── __init__.py │ └── _create_argparser.py │ └── __main__.py ├── pipeline ├── publish.sh └── build.sh ├── requirements-testing.txt ├── .gitignore ├── CODE_OF_CONDUCT.md ├── test └── openjd │ ├── cli │ ├── templates │ │ ├── env_1.yaml │ │ ├── env_2.yaml │ │ ├── env_fails_exit.yaml │ │ ├── env_fails_enter.yaml │ │ ├── env_with_param.yaml │ │ ├── simple_with_j_param.yaml │ │ ├── simple_with_j_param_exit_1.yaml │ │ ├── job_sleep_exit_normal.yaml │ │ ├── basic_dependency_job.yaml │ │ ├── basic.yaml │ │ ├── redacted_env.yaml │ │ ├── chunked_job.yaml │ │ └── job_with_test_steps.yaml │ ├── test_redacted_env.py │ ├── test_check_command.py │ ├── test_run_with_env.py │ ├── conftest.py │ ├── test_schema_command.py │ ├── __init__.py │ ├── test_chunked_job.py │ ├── test_local_session.py │ └── test_summary_command.py │ └── test_main.py ├── hatch.toml ├── .semantic_release └── CHANGELOG.md.j2 ├── scripts └── add_copyright_headers.sh ├── pyproject.toml ├── CHANGELOG.md ├── hatch_version_hook.py ├── CONTRIBUTING.md ├── DEVELOPMENT.md ├── LICENSE └── README.md /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @OpenJobDescription/Developers -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false -------------------------------------------------------------------------------- /requirements-release.txt: -------------------------------------------------------------------------------- 1 | python-semantic-release == 10.5.* 2 | -------------------------------------------------------------------------------- /requirements-development.txt: -------------------------------------------------------------------------------- 1 | hatch == 1.15.* 2 | hatch-vcs == 0.5.* -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /src/openjd/cli/py.typed: -------------------------------------------------------------------------------- 1 | # Marker file that indicates this package supports typing 2 | -------------------------------------------------------------------------------- /src/openjd/cli/_run/_local_session/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /pipeline/publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Set the -e option 3 | set -e 4 | 5 | ./pipeline/build.sh 6 | twine upload --repository codeartifact dist/* --verbose -------------------------------------------------------------------------------- /src/openjd/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from .cli._create_argparser import main 4 | 5 | 6 | main() 7 | -------------------------------------------------------------------------------- /src/openjd/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from ._create_argparser import main 4 | 5 | __all__ = ["main"] 6 | -------------------------------------------------------------------------------- /pipeline/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Set the -e option 3 | set -e 4 | 5 | pip install --upgrade pip 6 | pip install --upgrade hatch 7 | pip install --upgrade twine 8 | hatch run lint 9 | hatch run test 10 | hatch build 11 | -------------------------------------------------------------------------------- /requirements-testing.txt: -------------------------------------------------------------------------------- 1 | coverage[toml] == 7.* 2 | pytest == 8.4.* 3 | pytest-cov == 7.0.* 4 | pytest-timeout == 2.4.* 5 | pytest-xdist == 3.8.* 6 | types-PyYAML == 6.* 7 | black == 25.* 8 | mypy == 1.19.* 9 | ruff == 0.14.* 10 | -------------------------------------------------------------------------------- /.github/workflows/record_pr.yml: -------------------------------------------------------------------------------- 1 | name: Record PR 2 | 3 | on: 4 | pull_request: 5 | types: [opened, reopened] 6 | 7 | jobs: 8 | call-record-workflow: 9 | uses: OpenJobDescription/.github/.github/workflows/reusable_record_pr_details.yml@mainline -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *# 3 | *.swp 4 | 5 | *.DS_Store 6 | 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | /.coverage 12 | /.coverage.* 13 | /.cache 14 | /.pytest_cache 15 | /.mypy_cache 16 | /.ruff_cache 17 | /.attach_pid* 18 | /.venv 19 | 20 | /doc/_apidoc/ 21 | /build 22 | /dist 23 | _version.py -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /.github/workflows/responded.yml: -------------------------------------------------------------------------------- 1 | name: Contributor Responded 2 | on: 3 | issue_comment: 4 | types: [created, edited] 5 | 6 | jobs: 7 | check-for-response: 8 | uses: OpenJobDescription/.github/.github/workflows/reusable_responded.yml@mainline 9 | permissions: 10 | issues: write 11 | pull-requests: write 12 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/env_1.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: environment-2023-09 2 | environment: 3 | name: Env1 4 | script: 5 | actions: 6 | onEnter: 7 | command: python 8 | args: 9 | - -c 10 | - print('Env1 Enter') 11 | onExit: 12 | command: python 13 | args: 14 | - -c 15 | - print('Env1 Exit') 16 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/env_2.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: environment-2023-09 2 | environment: 3 | name: Env2 4 | script: 5 | actions: 6 | onEnter: 7 | command: python 8 | args: 9 | - -c 10 | - print('Env2 Enter') 11 | onExit: 12 | command: python 13 | args: 14 | - -c 15 | - print('Env2 Exit') 16 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ "mainline" ] 6 | pull_request: 7 | branches: [ "mainline" ] 8 | schedule: 9 | - cron: '0 8 * * MON' 10 | 11 | jobs: 12 | Analysis: 13 | name: Analysis 14 | uses: OpenJobDescription/.github/.github/workflows/reusable_codeql.yml@mainline 15 | permissions: 16 | security-events: write 17 | -------------------------------------------------------------------------------- /.github/workflows/stale_prs_and_issues.yml: -------------------------------------------------------------------------------- 1 | name: 'Check stale issues/PRs.' 2 | on: 3 | schedule: 4 | # Run every hour on the hour 5 | - cron: '0 * * * *' 6 | 7 | jobs: 8 | check-for-stales: 9 | uses: OpenJobDescription/.github/.github/workflows/reusable_stale_prs_and_issues.yml@mainline 10 | permissions: 11 | contents: read 12 | issues: write 13 | pull-requests: write 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/doc.yml: -------------------------------------------------------------------------------- 1 | 2 | name: "📕 Documentation Issue" 3 | description: Issue in the documentation 4 | title: "Docs: (short description of the issue)" 5 | labels: ["documentation", "needs triage"] 6 | body: 7 | - type: textarea 8 | id: documentation_issue 9 | attributes: 10 | label: Documentation Issue 11 | description: Describe the issue 12 | validations: 13 | required: true -------------------------------------------------------------------------------- /test/openjd/cli/templates/env_fails_exit.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: environment-2023-09 2 | environment: 3 | name: EnvExitFail 4 | script: 5 | actions: 6 | onEnter: 7 | command: python 8 | args: 9 | - -c 10 | - print('EnvExitFail Enter') 11 | onExit: 12 | command: python 13 | args: 14 | - -c 15 | - import sys; print('EnvExitFail Exit'); sys.exit(1) 16 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/env_fails_enter.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: environment-2023-09 2 | environment: 3 | name: EnvEnterFail 4 | script: 5 | actions: 6 | onEnter: 7 | command: python 8 | args: 9 | - -c 10 | - import sys; print('EnvEnterFail Enter'); sys.exit(1) 11 | onExit: 12 | command: python 13 | args: 14 | - -c 15 | - print('EnvEnterFail Exit') 16 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/env_with_param.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: environment-2023-09 2 | parameterDefinitions: 3 | - name: EnvParam 4 | type: STRING 5 | default: DefaultForEnvParam 6 | environment: 7 | name: EnvWithParam 8 | script: 9 | actions: 10 | onEnter: 11 | command: python 12 | args: 13 | - -c 14 | - print('EnvWithParam Enter {{Param.EnvParam}}') 15 | onExit: 16 | command: python 17 | args: 18 | - -c 19 | - print('EnvWithParam Exit {{Param.EnvParam}}') 20 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/simple_with_j_param.yaml: -------------------------------------------------------------------------------- 1 | { 2 | "specificationVersion": "jobtemplate-2023-09", 3 | "name": "Test", 4 | "parameterDefinitions": [{"name": "J", "type": "STRING"}], 5 | "steps": [ 6 | { 7 | "name": "SimpleStep", 8 | "script": { 9 | "actions": { 10 | "onRun": { 11 | "command": "python", 12 | "args": ["-c", "print('DoTask {{Param.J}}')"], 13 | } 14 | } 15 | }, 16 | } 17 | ], 18 | } -------------------------------------------------------------------------------- /test/openjd/cli/templates/simple_with_j_param_exit_1.yaml: -------------------------------------------------------------------------------- 1 | { 2 | "specificationVersion": "jobtemplate-2023-09", 3 | "name": "Test", 4 | "parameterDefinitions": [{"name": "J", "type": "STRING"}], 5 | "steps": [ 6 | { 7 | "name": "SimpleStep", 8 | "script": { 9 | "actions": { 10 | "onRun": { 11 | "command": "python", 12 | "args": ["-c", "import sys; print('DoTask'); sys.exit(1)"] 13 | } 14 | } 15 | } 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/maintenance.yml: -------------------------------------------------------------------------------- 1 | name: "🛠️ Maintenance" 2 | description: Some type of improvement 3 | title: "Maintenance: (short description of the issue)" 4 | labels: ["maintenance", "needs triage"] 5 | body: 6 | - type: textarea 7 | id: description 8 | attributes: 9 | label: Description 10 | description: Describe the improvement and why it is important to do. 11 | validations: 12 | required: true 13 | - type: textarea 14 | id: solution 15 | attributes: 16 | label: Solution 17 | description: Provide any ideas you have for how the suggestion can be implemented. 18 | validations: 19 | required: true 20 | -------------------------------------------------------------------------------- /.github/workflows/auto_approve.yml: -------------------------------------------------------------------------------- 1 | name: Dependabot auto-approve 2 | on: pull_request 3 | 4 | permissions: 5 | pull-requests: write 6 | 7 | jobs: 8 | dependabot: 9 | runs-on: ubuntu-latest 10 | if: ${{ github.actor == 'dependabot[bot]' }} 11 | steps: 12 | - name: Dependabot metadata 13 | id: metadata 14 | uses: dependabot/fetch-metadata@v2 15 | with: 16 | github-token: "${{ secrets.GITHUB_TOKEN }}" 17 | - name: Approve a PR 18 | run: gh pr review --approve "$PR_URL" 19 | env: 20 | PR_URL: ${{ github.event.pull_request.html_url }} 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/release_bump.yml: -------------------------------------------------------------------------------- 1 | name: "Release: Bump" 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | force_version_bump: 7 | required: false 8 | default: "" 9 | type: choice 10 | options: 11 | - "" 12 | - patch 13 | - minor 14 | - major 15 | 16 | concurrency: 17 | group: release 18 | 19 | jobs: 20 | Bump: 21 | name: Version Bump 22 | uses: OpenJobDescription/.github/.github/workflows/reusable_bump.yml@mainline 23 | permissions: 24 | contents: write 25 | pull-requests: write 26 | secrets: inherit 27 | with: 28 | force_version_bump: ${{ inputs.force_version_bump }} -------------------------------------------------------------------------------- /src/openjd/cli/_schema/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from .._common import SubparserGroup, add_common_arguments 4 | from ._schema_command import add_schema_arguments, do_get_schema 5 | 6 | 7 | def populate_argparser(subcommands: SubparserGroup) -> None: 8 | """ 9 | Adds the `schema` command to the given parser. 10 | """ 11 | schema_parser = subcommands.add( 12 | "schema", 13 | description="Returns a JSON Schema document for the Job template model.", 14 | ) 15 | add_common_arguments(schema_parser, set()) 16 | add_schema_arguments(schema_parser) 17 | schema_parser.set_defaults(func=do_get_schema) 18 | -------------------------------------------------------------------------------- /.github/workflows/code_quality.yml: -------------------------------------------------------------------------------- 1 | name: Code Quality 2 | 3 | on: 4 | pull_request: 5 | branches: [ mainline, release, 'patch_*' ] 6 | workflow_call: 7 | inputs: 8 | branch: 9 | required: false 10 | type: string 11 | tag: 12 | required: false 13 | type: string 14 | 15 | jobs: 16 | Test: 17 | name: Python 18 | strategy: 19 | matrix: 20 | os: [ubuntu-latest, windows-latest, macos-latest] 21 | python-version: ['3.9', '3.10', '3.11', '3.12'] 22 | uses: OpenJobDescription/.github/.github/workflows/reusable_python_build.yml@mainline 23 | with: 24 | os: ${{ matrix.os }} 25 | python-version: ${{ matrix.python-version }} 26 | ref: ${{inputs.tag}} 27 | -------------------------------------------------------------------------------- /src/openjd/cli/_summary/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from .._common import add_common_arguments, CommonArgument, SubparserGroup 4 | from ._summary_command import add_summary_arguments, do_summary 5 | 6 | 7 | def populate_argparser(subcommands: SubparserGroup) -> None: 8 | """Adds the `summary` command's arguments to the given subcommand parser.""" 9 | summary_parser = subcommands.add( 10 | "summary", 11 | usage="openjd summary JOB_TEMPLATE_PATH [arguments]", 12 | description="Print summary information about a Job Template.", 13 | ) 14 | 15 | add_common_arguments(summary_parser, {CommonArgument.PATH, CommonArgument.JOB_PARAMS}) 16 | add_summary_arguments(summary_parser) 17 | summary_parser.set_defaults(func=do_summary) 18 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | day: "monday" 13 | commit-message: 14 | prefix: "chore(deps):" 15 | - package-ecosystem: "github-actions" 16 | directory: "/" # Location of package manifests 17 | schedule: 18 | interval: "weekly" 19 | day: "monday" 20 | commit-message: 21 | prefix: "chore(github):" -------------------------------------------------------------------------------- /hatch.toml: -------------------------------------------------------------------------------- 1 | [envs.default] 2 | pre-install-commands = [ 3 | "pip install -r requirements-testing.txt" 4 | ] 5 | 6 | [envs.default.scripts] 7 | sync = "pip install -r requirements-testing.txt" 8 | test = "pytest --cov-config pyproject.toml {args:test}" 9 | typing = "mypy {args:src test}" 10 | style = [ 11 | "ruff check {args:.}", 12 | "black --check --diff {args:.}", 13 | ] 14 | fmt = [ 15 | "black {args:.}", 16 | "style", 17 | ] 18 | lint = [ 19 | "style", 20 | "typing", 21 | ] 22 | 23 | [[envs.all.matrix]] 24 | python = ["3.9", "3.10", "3.11", "3.12", "3.13"] 25 | 26 | [envs.release] 27 | detached = true 28 | 29 | [envs.release.scripts] 30 | deps = "pip install -r requirements-release.txt" 31 | bump = "semantic-release -v --strict version --no-push --no-commit --no-tag --skip-build {args}" 32 | version = "semantic-release -v --strict version --print {args}" 33 | -------------------------------------------------------------------------------- /src/openjd/cli/_check/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from .._common import add_common_arguments, CommonArgument, SubparserGroup 4 | from ._check_command import add_check_arguments, do_check 5 | 6 | 7 | def populate_argparser(subcommands: SubparserGroup) -> None: 8 | """Adds the `check` command and all of its arguments to the given parser.""" 9 | check_parser = subcommands.add( 10 | "check", 11 | usage="openjd check JOB_TEMPLATE_PATH [arguments]", 12 | description="Given an Open Job Description template file, parse the file and run validation checks against it to ensure that it is correctly formed.", 13 | ) 14 | 15 | # add all arguments through `add_common_arguments` 16 | add_common_arguments(check_parser, {CommonArgument.PATH}) 17 | add_check_arguments(check_parser) 18 | check_parser.set_defaults(func=do_check) 19 | -------------------------------------------------------------------------------- /.github/workflows/on_opened_pr.yml: -------------------------------------------------------------------------------- 1 | name: On Opened PR 2 | 3 | on: 4 | workflow_run: 5 | workflows: ["Record PR"] 6 | types: 7 | - completed 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | get-pr-details: 14 | permissions: 15 | actions: read # download PR artifact 16 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 17 | uses: OpenJobDescription/.github/.github/workflows/reusable_extract_pr_details.yml@mainline 18 | with: 19 | record_pr_workflow_id: ${{ github.event.workflow_run.id }} 20 | artifact_name: "pr-info" 21 | workflow_origin: ${{ github.repository }} 22 | 23 | label-pr: 24 | needs: get-pr-details 25 | if: ${{ needs.get-pr-details.outputs.pr_action == 'opened' || needs.get-pr-details.outputs.pr_action == 'reopened' }} 26 | uses: OpenJobDescription/.github/.github/workflows/reusable_label_pr.yml@mainline 27 | with: 28 | pr_number: ${{ needs.get-pr-details.outputs.pr_number }} 29 | label_name: "waiting-on-maintainers" 30 | permissions: 31 | pull-requests: write -------------------------------------------------------------------------------- /test/openjd/cli/templates/job_sleep_exit_normal.yaml: -------------------------------------------------------------------------------- 1 | { 2 | "specificationVersion": "jobtemplate-2023-09", 3 | "name": "TimeoutTest", 4 | "parameterDefinitions": [{"name": "J", "type": "STRING"}], 5 | "steps": [ 6 | { 7 | "name": "Timeout", 8 | "script": { 9 | "actions": { 10 | "onRun": { 11 | "command": "python", 12 | "args": [ 13 | "-c", 14 | # Obfuscate "EXIT_NORMAL" so it doesn't appear in the log when Windows prints the command that's run to the log. 15 | "import time,sys; print('SLEEP'); sys.stdout.flush(); time.sleep(5); print(chr(69)+'XIT_NORMAL')", 16 | ], 17 | "timeout": 2, 18 | } 19 | } 20 | }, 21 | } 22 | ], 23 | } -------------------------------------------------------------------------------- /test/openjd/cli/templates/basic_dependency_job.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: jobtemplate-2023-09 2 | name: Job 3 | parameterDefinitions: 4 | - name: J 5 | type: STRING 6 | jobEnvironments: 7 | - name: J1 8 | script: 9 | actions: 10 | onEnter: 11 | command: python 12 | args: 13 | - -c 14 | - print('J1 Enter') 15 | onExit: 16 | command: python 17 | args: 18 | - -c 19 | - print('J1 Exit') 20 | steps: 21 | - name: First 22 | parameterSpace: 23 | taskParameterDefinitions: 24 | - name: Foo 25 | type: INT 26 | range: '1' 27 | - name: Bar 28 | type: STRING 29 | range: 30 | - Bar1 31 | - Bar2 32 | script: 33 | actions: 34 | onRun: 35 | command: python 36 | args: 37 | - -c 38 | - print('J={{Param.J}} Foo={{Task.Param.Foo}}. Bar={{Task.Param.Bar}}') 39 | - name: Second 40 | dependencies: 41 | - dependsOn: First 42 | parameterSpace: 43 | taskParameterDefinitions: 44 | - name: Fuz 45 | type: INT 46 | range: 1-2 47 | script: 48 | actions: 49 | onRun: 50 | command: python 51 | args: 52 | - -c 53 | - print('J={{Param.J}} Fuz={{Task.Param.Fuz}}.') -------------------------------------------------------------------------------- /src/openjd/cli/_run/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from ._run_command import add_run_arguments, do_run 4 | from ._help_formatter import JobTemplateHelpAction 5 | from .._common import add_common_arguments, CommonArgument, SubparserGroup 6 | 7 | 8 | def populate_argparser(subcommands: SubparserGroup) -> None: 9 | """Adds the `run` command and all of its arguments to the given parser.""" 10 | run_parser = subcommands.add( 11 | "run", 12 | description="Takes a Job Template and runs the entire job or a selected Step from the job.", 13 | usage="openjd run JOB_TEMPLATE_PATH [arguments]", 14 | add_help=False, # Disable default help to use custom action 15 | ) 16 | add_common_arguments(run_parser, {CommonArgument.PATH, CommonArgument.JOB_PARAMS}) 17 | add_run_arguments(run_parser) 18 | 19 | # Add custom help action that provides context-aware help based on job template 20 | run_parser.add_argument( 21 | "-h", 22 | "--help", 23 | action=JobTemplateHelpAction, 24 | help="Show help message. When a job template path is provided, displays job-specific help including parameter definitions.", 25 | ) 26 | 27 | run_parser.set_defaults(func=do_run) 28 | -------------------------------------------------------------------------------- /.github/scripts/get_latest_changelog.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | """ 3 | This script gets the changelog notes for the latest version of this package. It makes the following assumptions 4 | 1. A file called CHANGELOG.md is in the current directory that has the changelog 5 | 2. The changelog file is formatted in a way such that level 2 headers are: 6 | a. The only indication of the beginning of a version's changelog notes. 7 | b. Always begin with `## ` 8 | 3. The changelog file contains the newest version's changelog notes at the top of the file. 9 | 10 | Example CHANGELOG.md: 11 | ``` 12 | ## 1.0.0 (2024-02-06) 13 | 14 | ### BREAKING CHANGES 15 | * **api**: rename all APIs 16 | 17 | ## 0.1.0 (2024-02-06) 18 | 19 | ### Features 20 | * **api**: add new api 21 | ``` 22 | 23 | Running this script on the above CHANGELOG.md should return the following contents: 24 | ``` 25 | ## 1.0.0 (2024-02-06) 26 | 27 | ### BREAKING CHANGES 28 | * **api**: rename all APIs 29 | 30 | ``` 31 | """ 32 | import re 33 | 34 | h2 = r"^##\s.*$" 35 | with open("CHANGELOG.md") as f: 36 | contents = f.read() 37 | matches = re.findall(h2, contents, re.MULTILINE) 38 | changelog = contents[: contents.find(matches[1]) - 1] if len(matches) > 1 else contents 39 | print(changelog) 40 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.yml: -------------------------------------------------------------------------------- 1 | name: "\U0001F680 Feature Request" 2 | description: Request a new feature 3 | title: "Feature request: (short description of the feature)" 4 | labels: ["enhancement", "needs triage"] 5 | body: 6 | - type: textarea 7 | id: problem 8 | attributes: 9 | label: Describe the problem 10 | description: | 11 | Help us understand the problem that you are trying to solve, and why it is important to you. 12 | Provide as much detail as you are able. 13 | validations: 14 | required: true 15 | 16 | - type: textarea 17 | id: proposed_solution 18 | attributes: 19 | label: Proposed Solution 20 | description: | 21 | Describe your proposed feature that you see solving this problem for you. If you have a 22 | full or partial prototype implementation then please open a draft pull request and link to 23 | it here as well. 24 | validations: 25 | required: true 26 | 27 | - type: textarea 28 | id: use_case 29 | attributes: 30 | label: Example Use Cases 31 | description: | 32 | Provide some sample code snippets or shell scripts that show how **you** would use this feature as 33 | you have proposed it. 34 | validations: 35 | required: true 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/openjd/cli/_common/_extensions.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser 4 | from typing import Optional 5 | 6 | # This is the list of Open Job Description extensions with implemented support 7 | SUPPORTED_EXTENSIONS = ["TASK_CHUNKING", "REDACTED_ENV_VARS"] 8 | 9 | 10 | def add_extensions_argument(run_parser: ArgumentParser): 11 | run_parser.add_argument( 12 | "--extensions", 13 | help=f"A comma-separated list of Open Job Description extension names to enable. Defaults to all that are implemented: {','.join(SUPPORTED_EXTENSIONS)}", 14 | ) 15 | 16 | 17 | def process_extensions_argument(extensions: Optional[str]) -> list[str]: 18 | """Process the comma-separated extensions argument and return a list of supported extensions.""" 19 | 20 | # If the option is not provided, default to all the supported extensions. 21 | if extensions is None: 22 | return SUPPORTED_EXTENSIONS 23 | 24 | extensions_list = [ 25 | extension.strip().upper() for extension in extensions.split(",") if extension.strip() != "" 26 | ] 27 | 28 | unsupported_extensions = set(extensions_list) - set(SUPPORTED_EXTENSIONS) 29 | if unsupported_extensions: 30 | raise ValueError( 31 | f"Unsupported Open Job Description extension(s): {', '.join(sorted(unsupported_extensions))}" 32 | ) 33 | 34 | return extensions_list 35 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/basic.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: "jobtemplate-2023-09" 2 | name: Job 3 | parameterDefinitions: 4 | - name: J 5 | type: STRING 6 | jobEnvironments: 7 | - name: J1 8 | script: 9 | actions: 10 | onEnter: 11 | command: python 12 | args: ["-c", "print('J1 Enter')"] 13 | onExit: 14 | command: python 15 | args: ["-c", "print('J1 Exit')"] 16 | - name: J2 17 | script: 18 | actions: 19 | onEnter: 20 | command: python 21 | args: ["-c", "print('J2 Enter')"] 22 | onExit: 23 | command: python 24 | args: ["-c", "print('J2 Exit')"] 25 | steps: 26 | - name: First 27 | parameterSpace: 28 | taskParameterDefinitions: 29 | - name: Foo 30 | type: INT 31 | range: "1" 32 | - name: Bar 33 | type: STRING 34 | range: ["Bar1", "Bar2"] 35 | stepEnvironments: 36 | - name: FirstS, 37 | script: 38 | actions: 39 | onEnter: 40 | command: python 41 | args: ["-c", "print('FirstS Enter')"] 42 | onExit: 43 | command: python 44 | args: ["-c", "print('FirstS Exit')"] 45 | script: 46 | actions: 47 | onRun: 48 | command: python 49 | args: 50 | - "-c" 51 | - "print('J={{Param.J}} Foo={{Task.Param.Foo}}. Bar={{Task.Param.Bar}}')" 52 | -------------------------------------------------------------------------------- /src/openjd/cli/_summary/_summary_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser, Namespace 4 | 5 | from ._summary_output import output_summary_result 6 | from .._common import ( 7 | add_extensions_argument, 8 | OpenJDCliResult, 9 | generate_job, 10 | process_extensions_argument, 11 | print_cli_result, 12 | ) 13 | 14 | 15 | def add_summary_arguments(summary_parser: ArgumentParser) -> None: 16 | # `step` is *technically* a shared argument, 17 | # but the help string and `required` attribute are 18 | # different among commands 19 | summary_parser.add_argument( 20 | "--step", 21 | action="store", 22 | type=str, 23 | metavar="STEP_NAME", 24 | help="Prints information about the Step with this name within the Job Template.", 25 | ) 26 | add_extensions_argument(summary_parser) 27 | 28 | 29 | @print_cli_result 30 | def do_summary(args: Namespace) -> OpenJDCliResult: 31 | """ 32 | Given a Job Template and applicable parameters, generates a Job and outputs information about it. 33 | """ 34 | extensions = process_extensions_argument(args.extensions) 35 | 36 | try: 37 | # Raises: RuntimeError 38 | sample_job, _ = generate_job(args, supported_extensions=extensions) 39 | except RuntimeError as rte: 40 | return OpenJDCliResult(status="error", message=str(rte)) 41 | 42 | return output_summary_result(sample_job, args.step) 43 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Fixes: ** 2 | 3 | ### What was the problem/requirement? (What/Why) 4 | 5 | ### What was the solution? (How) 6 | 7 | ### What is the impact of this change? 8 | 9 | ### How was this change tested? 10 | 11 | See [DEVELOPMENT.md](https://github.com/OpenJobDescription/openjd-cli/blob/mainline/DEVELOPMENT.md#testing) for information on running tests. 12 | 13 | - Have you run the tests? 14 | 15 | ### Was this change documented? 16 | 17 | - Are relevant docstrings in the code base updated? 18 | 19 | ### Is this a breaking change? 20 | 21 | A breaking change is one that modifies a public contract in a way that is not backwards compatible. See the 22 | [Public Interfaces](https://github.com/OpenJobDescription/openjd-cli/blob/mainline/DEVELOPMENT.md#the-packages-public-interface) section 23 | of the DEVELOPMENT.md for more information on the public contracts. 24 | 25 | If so, then please describe the changes that users of this package must make to update their scripts, or Python applications. 26 | 27 | ### Does this change impact security? 28 | 29 | - Does the change need to be threat modeled? For example, does it create or modify files/directories that must only be readable by the process owner? 30 | - If so, then please label this pull request with the "security" label. We'll work with you to analyze the threats. 31 | 32 | ---- 33 | 34 | *By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.* -------------------------------------------------------------------------------- /test/openjd/cli/test_redacted_env.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from pathlib import Path 4 | import re 5 | 6 | from . import run_openjd_cli_main, format_capsys_outerr 7 | 8 | TEMPLATE_DIR = Path(__file__).parent / "templates" 9 | 10 | 11 | def test_run_job_with_redacted_env(capsys): 12 | """Test that environment variables set with openjd_redacted_env are properly handled.""" 13 | outerr = run_openjd_cli_main( 14 | capsys, 15 | args=[ 16 | "run", 17 | str(TEMPLATE_DIR / "redacted_env.yaml"), 18 | ], 19 | expected_exit_code=0, 20 | ) 21 | 22 | # Verify the environment variables were set 23 | for expected_message_regex in [ 24 | "Setting redacted vars", 25 | "SECRETVAR is \\*\\*\\*\\*\\*\\*\\*\\*", 26 | "KEYSPACE is None", 27 | "VALSPACE is \\*\\*\\*\\*\\*\\*\\*\\*", 28 | "MULTILINE is \\*\\*\\*\\*\\*\\*\\*\\*", 29 | ]: 30 | assert re.search( 31 | expected_message_regex, outerr.out 32 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 33 | 34 | # Verify the openjd_redacted_env lines are not in the output 35 | for unexpected_message in [ 36 | "openjd_redacted_env: SECRETVAR=SECRETVAL", 37 | "openjd_redacted_env: KEYSPACE =SECRETVAL", 38 | "openjd_redacted_env: VALSPACE= SPACEVAL", 39 | "first_line", 40 | "second_line", 41 | "third_line", 42 | ]: 43 | assert ( 44 | unexpected_message not in outerr.out 45 | ), f"Found unexpected line in output:\n{format_capsys_outerr(outerr)}" 46 | -------------------------------------------------------------------------------- /.semantic_release/CHANGELOG.md.j2: -------------------------------------------------------------------------------- 1 | {% for version, release in context.history.released.items() %} 2 | ## {{ version.as_tag() }} ({{ release.tagged_date.strftime("%Y-%m-%d") }}) 3 | 4 | {% if "breaking" in release["elements"] %} 5 | ### BREAKING CHANGES 6 | {% for commit in release["elements"]["breaking"] %} 7 | * {% if commit.scope %}**{{ commit.scope }}**: {% endif %}{{ commit.commit.summary[commit.commit.summary.find(": ")+1:].strip() }} ([`{{ commit.short_hash }}`]({{ commit.commit.hexsha | commit_hash_url }})) 8 | {% endfor %} 9 | {% endif %} 10 | 11 | {% if "features" in release["elements"] %} 12 | ### Features 13 | {% for commit in release["elements"]["features"] %} 14 | * {% if commit.scope %}**{{ commit.scope }}**: {% endif %}{{ commit.commit.summary[commit.commit.summary.find(": ")+1:].strip() }} ([`{{ commit.short_hash }}`]({{ commit.commit.hexsha | commit_hash_url }})) 15 | {% endfor %} 16 | {% endif %} 17 | 18 | {% if "bug fixes" in release["elements"] %} 19 | ### Bug Fixes 20 | {% for commit in release["elements"]["bug fixes"] %} 21 | * {% if commit.scope %}**{{ commit.scope }}**: {% endif %}{{ commit.commit.summary[commit.commit.summary.find(":")+1:].strip() }} ([`{{ commit.short_hash }}`]({{ commit.commit.hexsha | commit_hash_url }})) 22 | {% endfor %} 23 | {% endif %} 24 | 25 | {% if "performance improvements" in release["elements"] %} 26 | ### Performance Improvements 27 | {% for commit in release["elements"]["performance improvements"] %} 28 | * {% if commit.scope %}**{{ commit.scope }}**: {% endif %}{{ commit.commit.summary[commit.commit.summary.find(":")+1:].strip() }} ([`{{ commit.short_hash }}`]({{ commit.commit.hexsha | commit_hash_url }})) 29 | {% endfor %} 30 | {% endif %} 31 | 32 | {% endfor %} -------------------------------------------------------------------------------- /test/openjd/cli/templates/redacted_env.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: "jobtemplate-2023-09" 2 | extensions: 3 | - REDACTED_ENV_VARS 4 | name: Test Redacted Env 5 | description: Test redacted environment variables 6 | 7 | jobEnvironments: 8 | - name: RedactedEnv 9 | script: 10 | actions: 11 | onEnter: 12 | command: python 13 | args: ["{{Env.File.Enter}}"] 14 | onExit: 15 | command: python 16 | args: ["{{Env.File.Exit}}"] 17 | embeddedFiles: 18 | - name: Enter 19 | type: TEXT 20 | data: | 21 | print("Setting redacted vars..") 22 | print(f"openjd_redacted_env: SECRETVAR=SECRETVAL") 23 | print(f"openjd_redacted_env: KEYSPACE =SECRETVAL") 24 | print(f"openjd_redacted_env: VALSPACE= SPACEVAL") 25 | print(f'openjd_redacted_env: "MULTILINE=first_line\\nsecond_line\\nthird_line"') 26 | - name: Exit 27 | type: TEXT 28 | data: | 29 | import os 30 | print(f"SECRETVAR is {os.environ.get('SECRETVAR')}") 31 | print(f"KEYSPACE is {os.environ.get('KEYSPACE')}") 32 | print(f"VALSPACE is {os.environ.get('VALSPACE')}") 33 | print(f"MULTILINE is {os.environ.get('VALSPACE')} END") 34 | print("first_line") 35 | print("second_line") 36 | print("third_line") 37 | steps: 38 | - name: CheckVars 39 | script: 40 | actions: 41 | onRun: 42 | command: python 43 | args: ["{{Task.File.Run}}"] 44 | embeddedFiles: 45 | - name: Run 46 | type: TEXT 47 | data: | 48 | import os 49 | print(f"SECRETVAR is {os.environ.get('SECRETVAR')}") 50 | print(f"KEYSPACE is {os.environ.get('KEYSPACE')}") 51 | print(f"VALSPACE is {os.environ.get('VALSPACE')}") 52 | print(f"MULTILINE is {os.environ.get('VALSPACE')} END") 53 | -------------------------------------------------------------------------------- /test/openjd/cli/test_check_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import Namespace 4 | from pathlib import Path 5 | from typing import Callable 6 | import json 7 | import pytest 8 | import tempfile 9 | import yaml 10 | 11 | from . import MOCK_TEMPLATE 12 | from openjd.cli._check._check_command import do_check 13 | 14 | 15 | @pytest.mark.parametrize( 16 | "tempfile_extension,doc_serializer", 17 | [ 18 | pytest.param(".template.json", json.dump, id="Successful JSON"), 19 | pytest.param(".template.yaml", yaml.dump, id="Successful YAML"), 20 | ], 21 | ) 22 | def test_do_check_file_success(tempfile_extension: str, doc_serializer: Callable): 23 | """ 24 | Execution should succeed given a correct filepath and JSON/YAML body 25 | """ 26 | with tempfile.NamedTemporaryFile( 27 | mode="w+t", suffix=tempfile_extension, encoding="utf8", delete=False 28 | ) as temp_template: 29 | doc_serializer(MOCK_TEMPLATE, temp_template.file) 30 | 31 | mock_args = Namespace(path=Path(temp_template.name), output="human-readable", extensions="") 32 | do_check(mock_args) 33 | 34 | Path(temp_template.name).unlink() 35 | 36 | 37 | def test_do_check_file_error(): 38 | """ 39 | Raise a SystemExit on an error 40 | (RunTime and DecodeValidation errors are treated the same; 41 | in this case we just test an incorrect filename that gets 42 | handled in read_template) 43 | """ 44 | mock_args = Namespace(path=Path("error-file.json"), output="human-readable", extensions="") 45 | with pytest.raises(SystemExit): 46 | do_check(mock_args) 47 | 48 | 49 | def test_do_check_bundle_error(): 50 | """ 51 | Test that passing a bundle with no template file yields a SystemError 52 | """ 53 | with tempfile.TemporaryDirectory() as temp_bundle: 54 | mock_args = Namespace(path=Path(temp_bundle), output="human-readable", extensions="") 55 | with pytest.raises(SystemExit): 56 | do_check(mock_args) 57 | -------------------------------------------------------------------------------- /src/openjd/cli/_create_argparser.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser 4 | import sys 5 | import traceback 6 | from typing import Optional 7 | 8 | from ._version import version 9 | 10 | from ._common import SubparserGroup 11 | 12 | from ._check import populate_argparser as populate_check_subparser 13 | from ._summary import populate_argparser as populate_summary_subparser 14 | from ._run import populate_argparser as populate_run_subparser 15 | from ._schema import populate_argparser as populate_schema_subparser 16 | 17 | 18 | # Our CLI subcommand construction requires that all leaf subcommands define a default 19 | # 'func' property which is a Callable[[],None] that implements the subcommand. 20 | # After parsing, we call that `func` argument of the resulting args object. 21 | 22 | 23 | def create_argparser() -> ArgumentParser: 24 | """Generate the root argparser for the CLI""" 25 | parser = ArgumentParser(prog="openjd", usage="openjd [arguments]") 26 | parser.set_defaults(func=lambda _: parser.print_help()) 27 | 28 | parser.add_argument( 29 | "--version", action="version", version=f"Open Job Description CLI {version}" 30 | ) 31 | 32 | subcommands = SubparserGroup( 33 | parser, 34 | title="commands", 35 | ) 36 | populate_check_subparser(subcommands) 37 | populate_summary_subparser(subcommands) 38 | populate_run_subparser(subcommands) 39 | populate_schema_subparser(subcommands) 40 | return parser 41 | 42 | 43 | def main(arg_list: Optional[list[str]] = None) -> None: 44 | """Main function for invoking the CLI""" 45 | parser = create_argparser() 46 | 47 | if arg_list is None: 48 | arg_list = sys.argv[1:] 49 | 50 | args = parser.parse_args(arg_list) 51 | try: 52 | # Raises: 53 | # SystemExit - on failure 54 | args.func(args) 55 | except Exception as exc: 56 | print(f"ERROR: {str(exc)}", file=sys.stderr) 57 | traceback.print_exc() 58 | sys.exit(1) 59 | -------------------------------------------------------------------------------- /test/openjd/cli/test_run_with_env.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from pathlib import Path 4 | import re 5 | 6 | from . import run_openjd_cli_main, format_capsys_outerr 7 | 8 | TEMPLATE_DIR = Path(__file__).parent / "templates" 9 | 10 | 11 | def test_run_job_with_env_default_params(capsys): 12 | # Run a job with env_with_param as an external environment, 13 | # leaving the environment's parameter at its default value 14 | 15 | outerr = run_openjd_cli_main( 16 | capsys, 17 | args=[ 18 | "run", 19 | str(TEMPLATE_DIR / "simple_with_j_param.yaml"), 20 | "-p", 21 | "J=Jvalue", 22 | "--environment", 23 | str(TEMPLATE_DIR / "env_with_param.yaml"), 24 | ], 25 | expected_exit_code=0, 26 | ) 27 | 28 | for expected_message_regex in [ 29 | "EnvWithParam Enter DefaultForEnvParam", 30 | "DoTask Jvalue", 31 | "EnvWithParam Exit DefaultForEnvParam", 32 | ]: 33 | assert re.search( 34 | expected_message_regex, outerr.out 35 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 36 | 37 | 38 | def test_run_job_with_env_provide_env_param(capsys): 39 | # Run a job with env_with_param as an external environment, 40 | # explicitly providing the env parameter 41 | 42 | outerr = run_openjd_cli_main( 43 | capsys, 44 | args=[ 45 | "run", 46 | str(TEMPLATE_DIR / "simple_with_j_param.yaml"), 47 | "-p", 48 | "J=Jvalue", 49 | "-p", 50 | "EnvParam=EnvParamValue", 51 | "--environment", 52 | str(TEMPLATE_DIR / "env_with_param.yaml"), 53 | ], 54 | expected_exit_code=0, 55 | ) 56 | 57 | for expected_message_regex in [ 58 | "EnvWithParam Enter EnvParamValue", 59 | "DoTask Jvalue", 60 | "EnvWithParam Exit EnvParamValue", 61 | ]: 62 | assert re.search( 63 | expected_message_regex, outerr.out 64 | ), f"Regex r'{expected_message_regex}' not matched in:\n{format_capsys_outerr(outerr)}" 65 | -------------------------------------------------------------------------------- /scripts/add_copyright_headers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | if [ $# -eq 0 ]; then 5 | echo "Usage: add-copyright-headers ..." >&2 6 | exit 1 7 | fi 8 | 9 | for file in "$@"; do 10 | if ! head -1 | grep 'Copyright ' "$file" >/dev/null; then 11 | case "$file" in 12 | *.java) 13 | CONTENT=$(cat "$file") 14 | cat > "$file" </dev/null; then 23 | CONTENT=$(tail -n +2 "$file") 24 | cat > "$file" < 27 | $CONTENT 28 | EOF 29 | else 30 | CONTENT=$(cat "$file") 31 | cat > "$file" < 33 | $CONTENT 34 | EOF 35 | fi 36 | ;; 37 | *.py) 38 | CONTENT=$(cat "$file") 39 | cat > "$file" < "$file" < "$file" < "$file" <&2 71 | exit 1 72 | ;; 73 | esac 74 | fi 75 | done -------------------------------------------------------------------------------- /src/openjd/cli/_check/_check_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser, Namespace 4 | from openjd.model import ( 5 | DecodeValidationError, 6 | TemplateSpecificationVersion, 7 | decode_job_template, 8 | decode_environment_template, 9 | ) 10 | 11 | from .._common import ( 12 | add_extensions_argument, 13 | read_template, 14 | OpenJDCliResult, 15 | print_cli_result, 16 | process_extensions_argument, 17 | ) 18 | 19 | 20 | def add_check_arguments(run_parser: ArgumentParser): 21 | add_extensions_argument(run_parser) 22 | 23 | 24 | @print_cli_result 25 | def do_check(args: Namespace) -> OpenJDCliResult: 26 | """Open a provided template file and check its schema for errors.""" 27 | 28 | extensions = process_extensions_argument(args.extensions) 29 | 30 | try: 31 | # Raises: RuntimeError 32 | template_object = read_template(args.path) 33 | 34 | # Raises: KeyError 35 | document_version = template_object["specificationVersion"] 36 | 37 | # Raises: ValueError 38 | template_version = TemplateSpecificationVersion(document_version) 39 | 40 | # Raises: DecodeValidationError 41 | if TemplateSpecificationVersion.is_job_template(template_version): 42 | decode_job_template(template=template_object, supported_extensions=extensions) 43 | elif TemplateSpecificationVersion.is_environment_template(template_version): 44 | decode_environment_template(template=template_object) 45 | else: 46 | return OpenJDCliResult( 47 | status="error", 48 | message=f"Unknown template 'specificationVersion' ({document_version}).", 49 | ) 50 | 51 | except KeyError: 52 | return OpenJDCliResult( 53 | status="error", message="ERROR: Missing field 'specificationVersion'" 54 | ) 55 | except RuntimeError as exc: 56 | return OpenJDCliResult(status="error", message=f"ERROR: {str(exc)}") 57 | except DecodeValidationError as exc: 58 | return OpenJDCliResult( 59 | status="error", message=f"ERROR: '{str(args.path)}' failed checks: {str(exc)}" 60 | ) 61 | 62 | return OpenJDCliResult( 63 | status="success", message=f"Template at '{str(args.path)}' passes validation checks." 64 | ) 65 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug.yml: -------------------------------------------------------------------------------- 1 | name: "\U0001F41B Bug Report" 2 | description: Report a bug 3 | title: "Bug: (short bug description)" 4 | labels: ["bug", "needs triage"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thank you for taking the time to fill out this bug report! 10 | 11 | ⚠️ If the bug that you are reporting is a security-related issue or security vulnerability, 12 | then please do not create a report via this template. Instead please 13 | notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) 14 | or directly via email to [AWS Security](aws-security@amazon.com). 15 | 16 | - type: textarea 17 | id: description 18 | attributes: 19 | label: Describe the bug 20 | description: What is the problem? A clear and concise description of the bug. 21 | validations: 22 | required: true 23 | 24 | - type: textarea 25 | id: expected_behaviour 26 | attributes: 27 | label: Expected Behaviour 28 | description: What did you expect to happen? 29 | validations: 30 | required: true 31 | 32 | - type: textarea 33 | id: current_behaviour 34 | attributes: 35 | label: Current Behaviour 36 | description: What actually happened? Please include as much detail as you can. 37 | validations: 38 | required: true 39 | 40 | - type: textarea 41 | id: reproduction_steps 42 | attributes: 43 | label: Reproduction Steps 44 | description: | 45 | Please provide as much detail as you can to help us understand how we can reproduce the bug. 46 | Step by step instructions and self-contained code snippets are ideal. 47 | validations: 48 | required: true 49 | 50 | - type: textarea 51 | id: environment 52 | attributes: 53 | label: Environment 54 | description: Please provide information on the environment and software versions that you are using to reproduce the bug. 55 | value: | 56 | At minimum: 57 | 1. Operating system (e.g. Windows Server 2022; Amazon Linux 2023; etc.) 58 | 2. Output of `python3 --version` 59 | 3. Version of your openjd-cli package (see: `pip show openjd-cli`): 60 | 61 | Please share other details about your environment that you think might be relevant to reproducing the bug. 62 | validations: 63 | required: true 64 | -------------------------------------------------------------------------------- /src/openjd/cli/_schema/_schema_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from argparse import ArgumentParser, Namespace 4 | import json 5 | from typing import Union 6 | 7 | from .._common import OpenJDCliResult, print_cli_result 8 | from openjd.model import EnvironmentTemplate, JobTemplate, TemplateSpecificationVersion 9 | 10 | 11 | def add_schema_arguments(schema_parser: ArgumentParser) -> None: 12 | allowed_values = [ 13 | v.value 14 | for v in TemplateSpecificationVersion 15 | if TemplateSpecificationVersion.is_job_template(v) 16 | or TemplateSpecificationVersion.is_environment_template(v) 17 | ] 18 | schema_parser.add_argument( 19 | "--version", 20 | action="store", 21 | type=TemplateSpecificationVersion, 22 | required=True, 23 | help=f"The specification version to return a JSON schema document for. Allowed values: {', '.join(allowed_values)}", 24 | ) 25 | 26 | 27 | def _process_regex(target: dict) -> None: 28 | """ 29 | Translates Python's language-specific regex into a JSON-compatible format. 30 | """ 31 | 32 | if "pattern" in target and isinstance(target["pattern"], str): 33 | target["pattern"] = target["pattern"].replace("(?-m:", "(?:") 34 | target["pattern"] = target["pattern"].replace("\\Z", "$") 35 | 36 | for attr in target.keys(): 37 | if isinstance(target[attr], dict): 38 | _process_regex(target[attr]) 39 | 40 | 41 | @print_cli_result 42 | def do_get_schema(args: Namespace) -> OpenJDCliResult: 43 | """ 44 | Uses Pydantic to convert the Open Job Description Job template model 45 | into a JSON schema document to compare in-development 46 | Job templates against. 47 | """ 48 | 49 | Template: Union[type[JobTemplate], type[EnvironmentTemplate]] 50 | if args.version == TemplateSpecificationVersion.JOBTEMPLATE_v2023_09: 51 | from openjd.model.v2023_09 import JobTemplate as Template 52 | elif args.version == TemplateSpecificationVersion.ENVIRONMENT_v2023_09: 53 | from openjd.model.v2023_09 import EnvironmentTemplate as Template 54 | else: 55 | return OpenJDCliResult( 56 | status="error", message=f"ERROR: Cannot generate schema for version '{args.version}'." 57 | ) 58 | 59 | schema_doc: dict = {} 60 | 61 | try: 62 | schema_doc = Template.model_json_schema() 63 | _process_regex(schema_doc) 64 | except Exception as e: 65 | return OpenJDCliResult(status="error", message=f"ERROR generating schema: {str(e)}") 66 | 67 | return OpenJDCliResult(status="success", message=json.dumps(schema_doc, indent=4)) 68 | -------------------------------------------------------------------------------- /src/openjd/cli/_run/_local_session/_logs.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from dataclasses import dataclass 4 | from logging import Handler, LogRecord 5 | from datetime import datetime, timezone 6 | from enum import Enum 7 | 8 | 9 | class LoggingTimestampFormat(str, Enum): 10 | """ 11 | Different formats for the timestamp of each log entry 12 | """ 13 | 14 | RELATIVE = "relative" 15 | LOCAL = "local" 16 | UTC = "utc" 17 | 18 | 19 | @dataclass 20 | class LogEntry: 21 | """ 22 | Log information from a sample Session to return in a CLI result. 23 | """ 24 | 25 | timestamp: str 26 | message: str 27 | 28 | def __str__(self) -> str: 29 | return f"{self.timestamp}\t{self.message}" 30 | 31 | 32 | class LocalSessionLogHandler(Handler): 33 | """ 34 | A custom Handler that formats and records logs in a dataclass. 35 | Used to print logs to `stdout` in real time while also storing 36 | them in memory. 37 | 38 | It prints a timestamp that is relative to the session start. 39 | """ 40 | 41 | messages: list[LogEntry] = [] 42 | _should_print: bool 43 | _session_start_timestamp: datetime 44 | _timestamp_format: LoggingTimestampFormat 45 | 46 | def __init__( 47 | self, 48 | should_print: bool, 49 | session_start_timestamp: datetime, 50 | timestamp_format: LoggingTimestampFormat, 51 | ): 52 | super(LocalSessionLogHandler, self).__init__() 53 | self._should_print = should_print 54 | self._session_start_timestamp = session_start_timestamp 55 | self._timestamp_format = timestamp_format 56 | 57 | def handle(self, record: LogRecord) -> bool: 58 | if self._timestamp_format == LoggingTimestampFormat.RELATIVE: 59 | timestamp = str( 60 | datetime.fromtimestamp(record.created, timezone.utc) - self._session_start_timestamp 61 | ) 62 | elif self._timestamp_format == LoggingTimestampFormat.LOCAL: 63 | timestamp = str( 64 | datetime.fromtimestamp(record.created, timezone.utc).astimezone().isoformat() 65 | ) 66 | else: 67 | timestamp = str(datetime.fromtimestamp(record.created, timezone.utc).isoformat()) 68 | 69 | record.created = datetime.fromtimestamp(record.created, timezone.utc).timestamp() 70 | new_record = LogEntry( 71 | timestamp=timestamp, 72 | message=record.getMessage(), 73 | ) 74 | self.messages.append(new_record) 75 | 76 | if self._should_print: 77 | print(new_record) 78 | 79 | # No filters are applied to the message, so always return True 80 | return True 81 | -------------------------------------------------------------------------------- /.github/workflows/release_publish.yml: -------------------------------------------------------------------------------- 1 | name: "Release: Publish" 2 | run-name: "Release: ${{ github.event.head_commit.message || inputs.tag }}" 3 | 4 | on: 5 | push: 6 | branches: 7 | - mainline 8 | paths: 9 | - CHANGELOG.md 10 | workflow_dispatch: 11 | inputs: 12 | tag: 13 | required: true 14 | type: string 15 | description: Specify a tag to re-run a release. 16 | 17 | concurrency: 18 | group: release 19 | 20 | permissions: 21 | contents: read 22 | 23 | jobs: 24 | TagRelease: 25 | uses: OpenJobDescription/.github/.github/workflows/reusable_tag_release.yml@mainline 26 | secrets: inherit 27 | with: 28 | tag: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || '' }} 29 | 30 | UnitTests: 31 | name: Unit Tests 32 | needs: TagRelease 33 | uses: ./.github/workflows/code_quality.yml 34 | with: 35 | tag: ${{ needs.TagRelease.outputs.tag }} 36 | 37 | PreRelease: 38 | needs: [TagRelease, UnitTests] 39 | uses: OpenJobDescription/.github/.github/workflows/reusable_prerelease.yml@mainline 40 | permissions: 41 | id-token: write 42 | contents: write 43 | secrets: inherit 44 | with: 45 | tag: ${{ needs.TagRelease.outputs.tag }} 46 | 47 | Release: 48 | needs: [TagRelease, PreRelease] 49 | uses: OpenJobDescription/.github/.github/workflows/reusable_release.yml@mainline 50 | secrets: inherit 51 | permissions: 52 | id-token: write 53 | contents: write 54 | with: 55 | tag: ${{ needs.TagRelease.outputs.tag }} 56 | 57 | Publish: 58 | needs: [TagRelease, Release] 59 | uses: OpenJobDescription/.github/.github/workflows/reusable_publish_python.yml@mainline 60 | permissions: 61 | id-token: write 62 | secrets: inherit 63 | with: 64 | tag: ${{ needs.TagRelease.outputs.tag }} 65 | 66 | PublishToPyPI: 67 | needs: [TagRelease, Publish] 68 | runs-on: ubuntu-latest 69 | environment: release 70 | permissions: 71 | id-token: write 72 | steps: 73 | - name: Checkout 74 | uses: actions/checkout@v6 75 | with: 76 | ref: ${{ needs.TagRelease.outputs.tag }} 77 | fetch-depth: 0 78 | - name: Set up Python 79 | uses: actions/setup-python@v6 80 | with: 81 | python-version: '3.9' 82 | - name: Install dependencies 83 | run: | 84 | pip install --upgrade hatch 85 | - name: Build 86 | run: hatch -v build 87 | # # See https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-pypi 88 | - name: Publish to PyPI 89 | uses: pypa/gh-action-pypi-publish@release/v1 90 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/chunked_job.yaml: -------------------------------------------------------------------------------- 1 | specificationVersion: 'jobtemplate-2023-09' 2 | extensions: 3 | - TASK_CHUNKING 4 | name: Chunked Job 5 | 6 | parameterDefinitions: 7 | - name: Items 8 | type: STRING 9 | default: 1-40 10 | - name: ChunkSize 11 | type: INT 12 | default: 10 13 | - name: TargetRuntime 14 | type: INT 15 | default: 0 16 | - name: TaskSleepTime 17 | type: FLOAT 18 | default: 0.01 19 | 20 | steps: 21 | - name: Chunked Step 22 | 23 | parameterSpace: 24 | taskParameterDefinitions: 25 | - name: Item 26 | type: CHUNK[INT] 27 | range: "{{Param.Items}}" 28 | chunks: 29 | defaultTaskCount: "{{Param.ChunkSize}}" 30 | targetRuntimeSeconds: "{{Param.TargetRuntime}}" 31 | rangeConstraint: NONCONTIGUOUS 32 | 33 | script: 34 | actions: 35 | onRun: 36 | command: bash 37 | args: ['{{Task.File.Run}}'] 38 | embeddedFiles: 39 | - name: GetSleepTime 40 | type: TEXT 41 | filename: get_sleep_time.py 42 | data: | 43 | """ 44 | Converts an Open Job Description range expression into a sleep time according to the job parameters. 45 | * https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#34111-intrangeexpr 46 | """ 47 | 48 | import sys, re 49 | 50 | task_sleep_time = float(sys.argv[2]) 51 | 52 | def range_expr_to_list(range_expr): 53 | # Regex that matches "", "-", or "-:" 54 | int_pat = r"\s*(-?[0-9]+)\s*" 55 | part_re = re.compile(f"^{int_pat}(?:-{int_pat}(?::{int_pat})?)?$") 56 | result = [] 57 | for part in range_expr.split(","): 58 | if m := part_re.match(part): 59 | start, end, step = m.groups() 60 | if step is not None: 61 | # Linear sequence "3-7:2" means the values [3, 5, 7]. 62 | result.extend(range(int(start), int(end) + (int(step)//abs(int(step))), int(step))) 63 | elif end is not None: 64 | # Interval "3-6" means the values [3, 4, 5, 6]. 65 | result.extend(range(int(start), int(end) + 1)) 66 | else: 67 | # Integer "3" means the values [3]. 68 | result.append(int(start)) 69 | else: 70 | raise ValueError(f"Invalid frame range expression: {range_expr}") 71 | return result 72 | 73 | print(task_sleep_time * len(range_expr_to_list(sys.argv[1]))) 74 | - name: Run 75 | type: TEXT 76 | data: | 77 | set -xeuo pipefail 78 | 79 | sleep "$(python '{{Task.File.GetSleepTime}}' '{{Task.Param.Item}}' '{{Param.TaskSleepTime}}')" 80 | -------------------------------------------------------------------------------- /src/openjd/cli/_run/_local_session/_actions.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from enum import Enum 4 | 5 | from openjd.model import Step, TaskParameterSet 6 | from openjd.model.v2023_09 import Environment 7 | from openjd.sessions import Session 8 | 9 | 10 | class EnvironmentType(str, Enum): 11 | """ 12 | The three different types of environment types that can be entered/exited in a session. 13 | """ 14 | 15 | EXTERNAL = "EXTERNAL" 16 | JOB = "JOB" 17 | STEP = "STEP" 18 | ALL = "ALL" 19 | 20 | def matches(self, other: "EnvironmentType") -> bool: 21 | """Environment types match if they are equal, or one of them is ALL.""" 22 | return self == other or self == EnvironmentType.ALL or other == EnvironmentType.ALL 23 | 24 | 25 | class SessionAction: 26 | _session: Session 27 | duration: float 28 | 29 | def __init__(self, session: Session): 30 | self._session = session 31 | 32 | def run(self): 33 | """ 34 | Subclasses of `SessionAction` should have 35 | custom implementations of this depending on their type. 36 | """ 37 | 38 | 39 | class RunTaskAction(SessionAction): 40 | _step: Step 41 | _parameters: TaskParameterSet 42 | 43 | def __init__(self, session: Session, step: Step, parameters: TaskParameterSet): 44 | super(RunTaskAction, self).__init__(session) 45 | self._step = step 46 | self._parameters = parameters 47 | 48 | def run(self): 49 | self._session.run_task( 50 | step_script=self._step.script, task_parameter_values=self._parameters 51 | ) 52 | 53 | def __str__(self): 54 | parameters = {name: parameter.value for name, parameter in self._parameters.items()} 55 | return f"Run Step '{self._step.name}' with Task parameters '{str(parameters)}'" 56 | 57 | 58 | class EnterEnvironmentAction(SessionAction): 59 | _environment: Environment 60 | _id: str 61 | 62 | def __init__(self, session: Session, environment: Environment, env_id: str): 63 | super(EnterEnvironmentAction, self).__init__(session) 64 | self._environment = environment 65 | self._id = env_id 66 | 67 | def run(self): 68 | self._session.enter_environment(environment=self._environment, identifier=self._id) 69 | 70 | def __str__(self): 71 | return f"Enter Environment '{self._environment.name}'" 72 | 73 | 74 | class ExitEnvironmentAction(SessionAction): 75 | _id: str 76 | _keep_session_running: bool 77 | 78 | def __init__(self, session: Session, id: str, keep_session_running: bool): 79 | super(ExitEnvironmentAction, self).__init__(session) 80 | self._id = id 81 | self._keep_session_running = keep_session_running 82 | 83 | def run(self): 84 | self._session.exit_environment( 85 | identifier=self._id, keep_session_running=self._keep_session_running 86 | ) 87 | 88 | def __str__(self): 89 | return f"Exit Environment '{self._id}'" 90 | -------------------------------------------------------------------------------- /src/openjd/cli/_common/_validation_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from typing import Any 4 | from pathlib import Path 5 | 6 | from openjd.model import ( 7 | DecodeValidationError, 8 | DocumentType, 9 | EnvironmentTemplate, 10 | JobTemplate, 11 | document_string_to_object, 12 | decode_environment_template, 13 | decode_job_template, 14 | ) 15 | 16 | 17 | def get_doc_type(filepath: Path) -> DocumentType: 18 | # If the file has a .json extension, treat it strictly 19 | # as JSON, otherwise treat it as YAML. 20 | if filepath.suffix.lower() == ".json": 21 | return DocumentType.JSON 22 | else: 23 | return DocumentType.YAML 24 | 25 | 26 | def read_template(template_file: Path) -> dict[str, Any]: 27 | """Open a JSON or YAML-formatted file and attempt to parse it into a JobTemplate object. 28 | Raises a RuntimeError if the file doesn't exist or can't be opened, and raises a 29 | DecodeValidationError if its contents can't be parsed into a valid JobTemplate. 30 | """ 31 | 32 | if not template_file.exists(): 33 | raise RuntimeError(f"'{str(template_file)}' does not exist.") 34 | 35 | if template_file.is_file(): 36 | # Raises: RuntimeError 37 | filetype = get_doc_type(template_file) 38 | else: 39 | raise RuntimeError(f"'{str(template_file)}' is not a file.") 40 | 41 | try: 42 | template_string = template_file.read_text(encoding="utf-8") 43 | except OSError as exc: 44 | raise RuntimeError(f"Could not open file '{str(template_file)}': {str(exc)}") 45 | 46 | try: 47 | # Raises: DecodeValidationError 48 | template_object = document_string_to_object( 49 | document=template_string, document_type=filetype 50 | ) 51 | except DecodeValidationError as exc: 52 | raise RuntimeError(f"'{str(template_file)}' failed checks: {str(exc)}") 53 | 54 | return template_object 55 | 56 | 57 | def read_job_template(template_file: Path, *, supported_extensions: list[str]) -> JobTemplate: 58 | """Open a JSON or YAML-formatted file and attempt to parse it into a JobTemplate object. 59 | Raises a RuntimeError if the file doesn't exist or can't be opened, and raises a 60 | DecodeValidationError if its contents can't be parsed into a valid JobTemplate. 61 | """ 62 | # Raises RuntimeError 63 | template_object = read_template(template_file) 64 | 65 | # Raises: DecodeValidationError 66 | template = decode_job_template( 67 | template=template_object, supported_extensions=supported_extensions 68 | ) 69 | 70 | return template 71 | 72 | 73 | def read_environment_template(template_file: Path) -> EnvironmentTemplate: 74 | """Open a JSON or YAML-formatted file and attempt to parse it into an EnvironmentTemplate object. 75 | Raises a RuntimeError if the file doesn't exist or can't be opened, and raises a 76 | DecodeValidationError if its contents can't be parsed into a valid EnvironmentTemplate. 77 | """ 78 | # Raises RuntimeError 79 | template_object = read_template(template_file) 80 | 81 | # Raises: DecodeValidationError 82 | template = decode_environment_template(template=template_object) 83 | 84 | return template 85 | -------------------------------------------------------------------------------- /test/openjd/cli/templates/job_with_test_steps.yaml: -------------------------------------------------------------------------------- 1 | # Catch-all sample template with different cases per Step 2 | specificationVersion: jobtemplate-2023-09 3 | name: my-job 4 | parameterDefinitions: 5 | - name: Message 6 | type: STRING 7 | default: Hello, world! 8 | jobEnvironments: 9 | - name: rootEnv 10 | variables: 11 | rootVar: rootVal 12 | steps: 13 | # VALID STEPS 14 | # Basic step; uses Job parameters and has an environment 15 | - name: NormalStep 16 | script: 17 | actions: 18 | onRun: 19 | command: python 20 | args: 21 | - -c 22 | - print('{{Param.Message}}') 23 | stepEnvironments: 24 | - name: env1 25 | script: 26 | actions: 27 | onEnter: 28 | command: python 29 | args: 30 | - -c 31 | - print('EnteringEnv') 32 | # Step that will wait for one minute before completing its Task 33 | - name: LongCommand 34 | script: 35 | actions: 36 | onRun: 37 | command: sleep 38 | args: 39 | - '60' 40 | # Step with the bare minimum information, i.e., no Task parameters, environments, or dependencies 41 | - name: BareStep 42 | script: 43 | actions: 44 | onRun: 45 | command: python 46 | args: 47 | - -c 48 | - print('zzz') 49 | # Step with a direct dependency on a previous Step 50 | - name: DependentStep 51 | script: 52 | actions: 53 | onRun: 54 | command: python 55 | args: 56 | - -c 57 | - print('I am dependent!') 58 | dependencies: 59 | - dependsOn: BareStep 60 | # Step with Task parameters 61 | - name: TaskParamStep 62 | parameterSpace: 63 | taskParameterDefinitions: 64 | - name: TaskNumber 65 | type: INT 66 | range: 67 | - 1 68 | - 2 69 | - 3 70 | - name: TaskMessage 71 | type: STRING 72 | range: 73 | - Hi! 74 | - Bye! 75 | script: 76 | actions: 77 | onRun: 78 | command: python 79 | args: 80 | - -c 81 | - print('{{Task.Param.TaskNumber}}.{{Task.Param.TaskMessage}}') 82 | # Step with a transitive dependency and a direct dependency 83 | - name: ExtraDependentStep 84 | script: 85 | actions: 86 | onRun: 87 | command: python 88 | args: 89 | - -c 90 | - print('I am extra dependent!') 91 | dependencies: 92 | - dependsOn: DependentStep 93 | - dependsOn: TaskParamStep 94 | # Step with dependencies and Task parameters 95 | - name: DependentParamStep 96 | parameterSpace: 97 | taskParameterDefinitions: 98 | - name: Adjective 99 | type: STRING 100 | range: 101 | - really 102 | - very 103 | - super 104 | script: 105 | actions: 106 | onRun: 107 | command: python 108 | args: 109 | - -c 110 | - print('I am {{Task.Param.Adjective}} dependent!') 111 | dependencies: 112 | - dependsOn: TaskParamStep 113 | # Step whose dependency has a step environment 114 | - name: StepDepHasStepEnv 115 | script: 116 | actions: 117 | onRun: 118 | command: python 119 | args: 120 | - -c 121 | - print('I have a dependency with a step environment!') 122 | dependencies: 123 | - dependsOn: NormalStep 124 | # ERROR STEPS 125 | # Step with a non-existent command that will throw an error when run 126 | - name: BadCommand 127 | script: 128 | actions: 129 | onRun: 130 | command: aaaaaaaaaa 131 | -------------------------------------------------------------------------------- /test/openjd/cli/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | import os 4 | import pytest 5 | import tempfile 6 | from pathlib import Path 7 | from unittest.mock import patch 8 | 9 | from . import MOCK_TEMPLATE, SampleSteps 10 | from openjd.cli._common._job_from_template import job_from_template 11 | from openjd.cli._run._local_session._session_manager import LocalSession 12 | from openjd.model import decode_job_template 13 | 14 | 15 | @pytest.fixture(scope="function", params=[[], ["Message=A new message!"]]) 16 | def sample_job_and_dirs(request): 17 | """ 18 | Uses the MOCK_TEMPLATE object to create a Job, once 19 | with default parameters and once with user-specified parameters. 20 | 21 | This fixture also manages the life time of a temporary directory that's 22 | used for the job template dir and the current working directory. 23 | """ 24 | with tempfile.TemporaryDirectory() as tmpdir: 25 | template_dir = Path(tmpdir) / "template_dir" 26 | current_working_dir = Path(tmpdir) / "current_working_dir" 27 | os.makedirs(template_dir) 28 | os.makedirs(current_working_dir) 29 | 30 | template = decode_job_template(template=MOCK_TEMPLATE) 31 | yield ( 32 | *job_from_template( 33 | template=template, 34 | environments=[], 35 | parameter_args=request.param, 36 | job_template_dir=template_dir, 37 | current_working_dir=current_working_dir, 38 | ), 39 | template_dir, 40 | current_working_dir, 41 | ) 42 | 43 | 44 | @pytest.fixture(scope="function") 45 | def sample_step_map(sample_job_and_dirs): 46 | return {step.name: step for step in sample_job_and_dirs[0].steps} 47 | 48 | 49 | @pytest.fixture( 50 | scope="function", 51 | params=[ 52 | pytest.param([], SampleSteps.NormalStep, -1, 2, 1, id="Basic step"), 53 | pytest.param( 54 | [SampleSteps.BareStep], SampleSteps.DependentStep, -1, 1, 2, id="Direct dependency" 55 | ), 56 | pytest.param( 57 | [ 58 | SampleSteps.BareStep, 59 | SampleSteps.DependentStep, 60 | SampleSteps.TaskParamStep, 61 | ], 62 | SampleSteps.ExtraDependentStep, 63 | -1, 64 | 1, 65 | 6, 66 | id="Dependencies and Task parameters", 67 | ), 68 | pytest.param( 69 | [SampleSteps.BareStep, SampleSteps.DependentStep, SampleSteps.TaskParamStep], 70 | SampleSteps.ExtraDependentStep, 71 | 1, 72 | 1, 73 | 6, 74 | id="No maximum on dependencies' Tasks", 75 | ), 76 | pytest.param([], SampleSteps.TaskParamStep, 1, 1, 1, id="Limit on maximum Task parameters"), 77 | pytest.param( 78 | [], SampleSteps.TaskParamStep, 100, 1, 3, id="Maximum Task parameters more than defined" 79 | ), 80 | ], 81 | ) 82 | def session_parameters(request): 83 | """ 84 | Tests for `LocalSession.initialize` and `LocalSession.run` use the same 85 | test parameters, so we create a fixture that returns them. 86 | """ 87 | return request.param 88 | 89 | 90 | @pytest.fixture(scope="function") 91 | def patched_session_cleanup(): 92 | """ 93 | Patches the `cleanup` function in a LocalSession, but uses the 94 | original function as a side effect to track how many times it 95 | gets called. 96 | We use this to verify that Sessions are properly cleaned up 97 | after exiting on a success or error. 98 | """ 99 | with patch.object( 100 | LocalSession, "cleanup", autospec=True, side_effect=LocalSession.cleanup 101 | ) as patched_cleanup: 102 | yield patched_cleanup 103 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling", "hatch-vcs"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "openjd-cli" 7 | authors = [ 8 | {name = "Amazon Web Services"}, 9 | ] 10 | dynamic = ["version"] 11 | readme = "README.md" 12 | license = "Apache-2.0" 13 | requires-python = ">=3.9" 14 | description = "Provides a command-line interface for working with Open Job Description templates." 15 | # https://pypi.org/classifiers/ 16 | classifiers = [ 17 | "Development Status :: 5 - Production/Stable", 18 | "Programming Language :: Python", 19 | "Programming Language :: Python :: 3 :: Only", 20 | "Programming Language :: Python :: 3.9", 21 | "Programming Language :: Python :: 3.10", 22 | "Programming Language :: Python :: 3.11", 23 | "Programming Language :: Python :: 3.12", 24 | "Operating System :: POSIX :: Linux", 25 | "Operating System :: Microsoft :: Windows", 26 | "Operating System :: MacOS", 27 | "License :: OSI Approved :: Apache Software License", 28 | "Intended Audience :: Developers", 29 | "Intended Audience :: End Users/Desktop" 30 | ] 31 | dependencies = [ 32 | "openjd-sessions >= 0.10.3,< 0.11", 33 | "openjd-model >= 0.8,< 0.9" 34 | ] 35 | 36 | [project.urls] 37 | Homepage = "https://github.com/OpenJobDescription/openjd-cli" 38 | Source = "https://github.com/OpenJobDescription/openjd-cli" 39 | 40 | 41 | [project.scripts] 42 | openjd = "openjd.cli:main" 43 | 44 | 45 | [tool.hatch.build] 46 | artifacts = [ 47 | "*_version.py", 48 | ] 49 | 50 | [tool.hatch.version] 51 | source = "vcs" 52 | 53 | [tool.hatch.version.raw-options] 54 | version_scheme = "post-release" 55 | 56 | [tool.hatch.build.hooks.vcs] 57 | version-file = "_version.py" 58 | 59 | [tool.hatch.build.hooks.custom] 60 | path = "hatch_version_hook.py" 61 | 62 | [[tool.hatch.build.hooks.custom.copy_map]] 63 | sources = [ 64 | "_version.py", 65 | ] 66 | destinations = [ 67 | "src/openjd/cli", 68 | ] 69 | 70 | [tool.hatch.build.targets.sdist] 71 | include = [ 72 | "src/openjd", 73 | "hatch_version_hook.py", 74 | ] 75 | 76 | [tool.hatch.build.targets.wheel] 77 | packages = [ 78 | "src/openjd", 79 | ] 80 | only-include = [ 81 | "src/openjd", 82 | ] 83 | 84 | [tool.hatch.build.targets.wheel.force-include] 85 | "src/openjd/__main__.py" = "openjd/__main__.py" 86 | 87 | [tool.mypy] 88 | check_untyped_defs = false 89 | show_error_codes = false 90 | pretty = true 91 | ignore_missing_imports = true 92 | disallow_incomplete_defs = false 93 | disallow_untyped_calls = false 94 | show_error_context = true 95 | strict_equality = false 96 | python_version = "3.9" 97 | warn_redundant_casts = true 98 | warn_unused_configs = true 99 | warn_unused_ignores = false 100 | # Tell mypy that there's a namespace package at src/openjd 101 | namespace_packages = true 102 | explicit_package_bases = true 103 | mypy_path = "src" 104 | 105 | # See: https://docs.pydantic.dev/mypy_plugin/ 106 | # - Helps mypy understand pydantic typing. 107 | plugins = "pydantic.mypy" 108 | 109 | [tool.ruff] 110 | line-length = 100 111 | 112 | [tool.ruff.lint] 113 | ignore = [ 114 | "E501", 115 | # Double Check if this should be fixed 116 | "E731", 117 | ] 118 | 119 | [tool.ruff.lint.pep8-naming] 120 | classmethod-decorators = [ 121 | "classmethod", 122 | # pydantic decorators are classmethod decorators 123 | # suppress N805 errors on classes decorated with them 124 | "pydantic.validator", 125 | "pydantic.root_validator", 126 | ] 127 | 128 | [tool.ruff.lint.isort] 129 | known-first-party = [ 130 | "openjd", 131 | ] 132 | 133 | [tool.black] 134 | line-length = 100 135 | 136 | [tool.pytest.ini_options] 137 | xfail_strict = false 138 | addopts = [ 139 | "-rfEx", 140 | "--durations=5", 141 | "--cov=src/openjd/cli", 142 | "--color=yes", 143 | "--cov-report=html:build/coverage", 144 | "--cov-report=xml:build/coverage/coverage.xml", 145 | "--cov-report=term-missing", 146 | "--numprocesses=auto", 147 | "--timeout=60" 148 | ] 149 | 150 | 151 | [tool.coverage.run] 152 | branch = true 153 | parallel = true 154 | 155 | 156 | [tool.coverage.paths] 157 | source = [ 158 | "src/" 159 | ] 160 | 161 | [tool.coverage.report] 162 | show_missing = true 163 | fail_under = 92 164 | omit= [ 165 | "src/openjd/cli/_version.py" 166 | ] 167 | 168 | [tool.semantic_release] 169 | # Can be removed or set to true once we are v1 170 | major_on_zero = false 171 | tag_format = "{version}" 172 | allow_zero_version = true 173 | 174 | [tool.semantic_release.commit_parser_options] 175 | allowed_tags = [ 176 | "build", 177 | "chore", 178 | "ci", 179 | "docs", 180 | "feat", 181 | "fix", 182 | "perf", 183 | "style", 184 | "refactor", 185 | "test", 186 | ] 187 | minor_tags = [] 188 | patch_tags = [ 189 | "chore", 190 | "feat", 191 | "fix", 192 | "refactor", 193 | "perf", 194 | ] 195 | 196 | [tool.semantic_release.publish] 197 | upload_to_vcs_release = false 198 | 199 | [tool.semantic_release.changelog] 200 | template_dir = ".semantic_release" 201 | 202 | [tool.semantic_release.changelog.environment] 203 | trim_blocks = true 204 | lstrip_blocks = true 205 | 206 | [tool.semantic_release.branches.release] 207 | match = "(mainline|release|patch_.*)" 208 | -------------------------------------------------------------------------------- /test/openjd/cli/test_schema_command.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | from openjd.cli._schema._schema_command import do_get_schema, _process_regex 4 | from openjd.model import TemplateSpecificationVersion 5 | import openjd.model.v2023_09 6 | 7 | from argparse import Namespace 8 | import json 9 | import pytest 10 | from unittest.mock import Mock, patch 11 | 12 | 13 | @pytest.mark.parametrize( 14 | "target,expected_result", 15 | [ 16 | pytest.param( 17 | {"attr": "value", "pattern": r"(?-m:^[^\u0000-\u001F\u007F-\u009F]+\Z)"}, 18 | {"attr": "value", "pattern": r"(?:^[^\u0000-\u001F\u007F-\u009F]+$)"}, 19 | id="Standard dictionary with pattern attribute", 20 | ), 21 | pytest.param( 22 | {"nested_dict": {"pattern": r"(?-m:^[^\u0000-\u001F\u007F-\u009F]+\Z)"}}, 23 | {"nested_dict": {"pattern": r"(?:^[^\u0000-\u001F\u007F-\u009F]+$)"}}, 24 | id="Pattern in nested dictionary", 25 | ), 26 | pytest.param( 27 | { 28 | "pattern": r"(?-m:^[^\u0000-\u001F\u007F-\u009F]+\Z)", 29 | "nested_dict": {"pattern": r"(?-m:^[^\u0000-\u001F\u007F-\u009F]+\Z)"}, 30 | }, 31 | { 32 | "pattern": r"(?:^[^\u0000-\u001F\u007F-\u009F]+$)", 33 | "nested_dict": {"pattern": r"(?:^[^\u0000-\u001F\u007F-\u009F]+$)"}, 34 | }, 35 | id="Patterns in multiple levels", 36 | ), 37 | pytest.param( 38 | {"pattern": "NotRealRegex"}, {"pattern": "NotRealRegex"}, id="Unaffected pattern" 39 | ), 40 | pytest.param( 41 | {"pattern": ["not", "a", "string"]}, 42 | {"pattern": ["not", "a", "string"]}, 43 | id="Non-string pattern attribute", 44 | ), 45 | pytest.param({}, {}, id="Empty dictionary"), 46 | ], 47 | ) 48 | def test_process_regex(target: dict, expected_result: dict): 49 | """ 50 | Test that the `schema` command can process Python-specific regex 51 | into JSON-compatible regex. 52 | """ 53 | _process_regex(target) 54 | 55 | assert target == expected_result 56 | 57 | 58 | @pytest.mark.usefixtures("capsys") 59 | def test_do_get_schema_success(capsys: pytest.CaptureFixture): 60 | """ 61 | Test that the `schema` command returns a correctly-formed 62 | JSON body with specific Job template attributes. 63 | """ 64 | with patch( 65 | "openjd.cli._schema._schema_command._process_regex", new=Mock(side_effect=_process_regex) 66 | ) as patched_process_regex: 67 | do_get_schema( 68 | Namespace( 69 | version=TemplateSpecificationVersion.JOBTEMPLATE_v2023_09.value, 70 | output="human-readable", 71 | ) 72 | ) 73 | patched_process_regex.assert_called() 74 | 75 | model_output = capsys.readouterr().out 76 | model_json = json.loads(model_output) 77 | 78 | assert model_json is not None 79 | assert model_json["title"] == "JobTemplate" 80 | assert "specificationVersion" in model_json["properties"] 81 | assert "name" in model_json["properties"] 82 | assert "steps" in model_json["properties"] 83 | 84 | 85 | @pytest.mark.usefixtures("capsys") 86 | def test_do_get_schema_success_environment(capsys: pytest.CaptureFixture): 87 | """ 88 | Test that the `schema` command returns a correctly-formed 89 | JSON body with specific Environment template attributes. 90 | """ 91 | with patch( 92 | "openjd.cli._schema._schema_command._process_regex", new=Mock(side_effect=_process_regex) 93 | ) as patched_process_regex: 94 | do_get_schema( 95 | Namespace( 96 | version=TemplateSpecificationVersion.ENVIRONMENT_v2023_09.value, 97 | output="human-readable", 98 | ) 99 | ) 100 | patched_process_regex.assert_called() 101 | 102 | model_output = capsys.readouterr().out 103 | model_json = json.loads(model_output) 104 | 105 | assert model_json is not None 106 | assert model_json["title"] == "EnvironmentTemplate" 107 | assert "specificationVersion" in model_json["properties"] 108 | 109 | 110 | @pytest.mark.usefixtures("capsys") 111 | def test_do_get_schema_incorrect_version(capsys: pytest.CaptureFixture): 112 | """ 113 | Test that the `schema` command fails if an unsupported version string 114 | is supplied. 115 | """ 116 | 117 | with pytest.raises(SystemExit): 118 | do_get_schema(Namespace(version="badversion", output="human-readable")) 119 | output = capsys.readouterr().out 120 | 121 | assert "Cannot generate schema for version 'badversion'" in output 122 | 123 | 124 | @pytest.mark.usefixtures("capsys") 125 | def test_do_get_schema_error(capsys: pytest.CaptureFixture): 126 | """ 127 | Test that the `schema` command can recover from an error 128 | when generating the JSON schema. 129 | """ 130 | 131 | with ( 132 | patch.object(openjd.model.v2023_09, "JobTemplate") as job_template_class, 133 | pytest.raises(SystemExit), 134 | ): 135 | job_template_class.model_json_schema.side_effect = RuntimeError("Test error") 136 | do_get_schema( 137 | Namespace( 138 | version=TemplateSpecificationVersion.JOBTEMPLATE_v2023_09.value, 139 | output="human-readable", 140 | ) 141 | ) 142 | output = capsys.readouterr().out 143 | 144 | assert "Test error" in output 145 | -------------------------------------------------------------------------------- /src/openjd/cli/_common/_job_from_template.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | from __future__ import annotations 3 | 4 | import json 5 | from pathlib import Path 6 | import re 7 | from typing import Optional, Union 8 | import yaml 9 | 10 | from ._validation_utils import get_doc_type 11 | from openjd.model import ( 12 | DecodeValidationError, 13 | DocumentType, 14 | EnvironmentTemplate, 15 | Job, 16 | JobParameterValues, 17 | JobTemplate, 18 | create_job, 19 | preprocess_job_parameters, 20 | ) 21 | 22 | 23 | def get_params_from_file(parameter_string: str) -> Union[dict, list]: 24 | """ 25 | Resolves the supplied Job Parameter filepath into a JSON object with its contents. 26 | 27 | Raises: RuntimeError if the file can't be opened 28 | """ 29 | parameter_file = Path(parameter_string.removeprefix("file://")).expanduser() 30 | 31 | if not parameter_file.exists(): 32 | raise RuntimeError(f"Provided parameter file '{str(parameter_file)}' does not exist.") 33 | if not parameter_file.is_file(): 34 | raise RuntimeError(f"Provided parameter file '{str(parameter_file)}' is not a file.") 35 | 36 | # Raises: RuntimeError 37 | doc_type = get_doc_type(parameter_file) 38 | 39 | try: 40 | parameter_string = parameter_file.read_text() 41 | except OSError: 42 | raise RuntimeError(f"Could not open parameter file '{str(parameter_file)}'.") 43 | 44 | try: 45 | if doc_type == DocumentType.YAML: 46 | # Raises: YAMLError 47 | parameters = yaml.safe_load(parameter_string) 48 | else: 49 | # Raises: JSONDecodeError 50 | parameters = json.loads(parameter_string) 51 | except (yaml.YAMLError, json.JSONDecodeError) as exc: 52 | raise RuntimeError( 53 | f"Parameter file '{str(parameter_file)}' is formatted incorrectly: {str(exc)}" 54 | ) 55 | 56 | return parameters 57 | 58 | 59 | def get_job_params(parameter_args: Optional[list[str]]) -> dict: 60 | """ 61 | Resolves Job Parameters from a list of command-line arguments. 62 | Arguments may be a filepath or a string with format 'Key=Value'. 63 | 64 | Raises: RuntimeError if the provided Parameters are formatted incorrectly or can't be opened 65 | """ 66 | parameter_dict: dict = {} 67 | 68 | for arg in parameter_args or []: 69 | arg = arg.strip() 70 | # Case 1: Provided argument is a filepath 71 | if arg.startswith("file://"): 72 | # Raises: RuntimeError 73 | parameters = get_params_from_file(arg) 74 | 75 | if isinstance(parameters, dict): 76 | parameter_dict.update(parameters) 77 | else: 78 | raise RuntimeError(f"Job parameter file '{arg}' should contain a dictionary.") 79 | 80 | # Case 2: Provided as a JSON string 81 | elif re.match("^{(.*)}$", arg): 82 | try: 83 | # Raises: JSONDecodeError 84 | parameters = json.loads(arg) 85 | except (json.JSONDecodeError, TypeError): 86 | raise RuntimeError( 87 | f"Job parameter string ('{arg}') not formatted correctly. It must be key=value pairs, inline JSON, or a path to a JSON or YAML document prefixed with 'file://'." 88 | ) 89 | if not isinstance(parameters, dict): 90 | # This should never happen. Including it out of a sense of paranoia. 91 | raise RuntimeError( 92 | f"Job parameter ('{arg}') must contain a dictionary mapping job parameters to their value." 93 | ) 94 | parameter_dict.update(parameters) 95 | 96 | # Case 3: Provided argument is a Key=Value string 97 | elif regex_match := re.match("^([^=]+)=(.*)$", arg): 98 | parameter_dict.update({regex_match[1]: regex_match[2]}) 99 | 100 | else: 101 | raise RuntimeError( 102 | f"Job parameter string ('{arg}') not formatted correctly. It must be key=value pairs, inline JSON, or a path to a JSON or YAML document prefixed with 'file://'." 103 | ) 104 | 105 | return parameter_dict 106 | 107 | 108 | def job_from_template( 109 | template: JobTemplate, 110 | environments: list[EnvironmentTemplate], 111 | parameter_args: list[str] | None, 112 | job_template_dir: Path, 113 | current_working_dir: Path, 114 | ) -> tuple[Job, JobParameterValues]: 115 | """ 116 | Given a decoded Job Template and a user-input parameter dictionary, 117 | generates a Job object and the parameter values for running the job. 118 | 119 | Raises: RuntimeError if parameters are an unsupported type or don't correspond to the template 120 | """ 121 | parameter_dict = get_job_params(parameter_args) 122 | 123 | try: 124 | parameter_values = preprocess_job_parameters( 125 | job_template=template, 126 | job_parameter_values=parameter_dict, 127 | job_template_dir=job_template_dir, 128 | current_working_dir=current_working_dir, 129 | environment_templates=environments, 130 | ) 131 | except ValueError as ve: 132 | raise RuntimeError(str(ve)) 133 | 134 | try: 135 | job = create_job( 136 | job_template=template, 137 | job_parameter_values=parameter_values, 138 | environment_templates=environments, 139 | ) 140 | return (job, parameter_values) 141 | except DecodeValidationError as dve: 142 | raise RuntimeError(f"Could not generate Job from template and parameters: {str(dve)}") 143 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.7.4 (2025-11-14) 2 | 3 | 4 | ### Features 5 | * Make `openjd run