├── .binny.yaml ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ └── feature_request.md ├── actions │ └── bootstrap │ │ └── action.yaml ├── dependabot.yml ├── scripts │ ├── ci-check.sh │ └── trigger-release.sh └── workflows │ ├── dependabot-automation.yaml │ ├── oss-project-board-add.yaml │ ├── release.yaml │ ├── remove-awaiting-response-label.yaml │ └── validations.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── DEVELOPING.md ├── LICENSE ├── Makefile ├── README.md ├── RELEASE.md ├── __main__.py ├── pyproject.toml ├── src └── yardstick │ ├── __init__.py │ ├── arrange.py │ ├── artifact.py │ ├── capture.py │ ├── cli │ ├── __init__.py │ ├── cli.py │ ├── config.py │ ├── display.py │ ├── explore │ │ ├── __init__.py │ │ ├── image_labels │ │ │ ├── __init__.py │ │ │ ├── cve_provider.py │ │ │ ├── display_pane.py │ │ │ ├── edit_note_dialog.py │ │ │ ├── history.py │ │ │ ├── label_json_editor_dialog.py │ │ │ ├── label_manager.py │ │ │ ├── label_margin.py │ │ │ ├── label_selection_pane.py │ │ │ ├── result_selection_pane.py │ │ │ ├── text_area.py │ │ │ └── text_dialog.py │ │ └── result.py │ ├── label.py │ ├── result.py │ └── validate.py │ ├── comparison.py │ ├── label.py │ ├── store │ ├── __init__.py │ ├── config.py │ ├── image_lineage.py │ ├── label_stats.py │ ├── labels.py │ ├── naming.py │ ├── result_set.py │ ├── scan_result.py │ └── tool.py │ ├── tool │ ├── __init__.py │ ├── grype.py │ ├── plugin.py │ ├── sbom_generator.py │ ├── syft.py │ └── vulnerability_scanner.py │ ├── utils │ ├── __init__.py │ ├── github.py │ └── grype_db.py │ └── validate │ ├── __init__.py │ ├── delta.py │ ├── gate.py │ └── validate.py ├── tests ├── cli │ ├── .gitignore │ ├── .yardstick.yaml │ ├── .yardstick │ │ └── labels │ │ │ └── docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da │ │ │ ├── 215d9c7a-0fb2-4313-bcf4-26bda1bec916.json │ │ │ ├── 23e358c1-3956-4774-9af2-09ac4cbb7931.json │ │ │ ├── 2e4c5ab8-7d94-4dc2-a0b1-78d9fd5ab78b.json │ │ │ ├── 34a3e6ce-59b7-46a2-b369-448805ce5919.json │ │ │ ├── 57d681b5-1080-496e-9bf1-b755b8851791.json │ │ │ ├── 5866cffe-c48c-4ce7-8ddb-6c4ac952e69b.json │ │ │ ├── 59c7cc8e-fdf8-49e0-be70-a66146db6750.json │ │ │ ├── 6d4faa73-8ac7-4b2a-9252-cd8b31dfc8b7.json │ │ │ ├── 797d4932-d64e-4d52-a59a-1128fdc2dde0.json │ │ │ ├── 9dd57c41-d9e9-40e8-9359-af640b3dc9b0.json │ │ │ ├── a2d926f2-9de7-4480-a2c0-9516d81d1d29.json │ │ │ ├── a6f0668e-1ca4-4857-b50a-32b56a4d38a1.json │ │ │ ├── b1901af4-524b-4248-afc8-3d0df1303266.json │ │ │ ├── d95890a6-5934-43e3-8826-ce086ec5f1e0.json │ │ │ ├── db463105-6378-4173-81c8-f0942f78e179.json │ │ │ ├── eb8254f5-baf3-43e0-84d6-ff958990de63.json │ │ │ ├── fc872e76-fa2c-43a3-a24b-84cc612366d6.json │ │ │ └── fcc3a71e-8f7a-4503-910e-de2b7ed861c5.json │ ├── Makefile │ └── run.sh └── unit │ ├── __init__.py │ ├── cli │ ├── __init__.py │ ├── explore_labels │ │ ├── __init__.py │ │ └── test_history.py │ └── test_config.py │ ├── store │ ├── __init__.py │ ├── test_naming.py │ └── test_scan_result.py │ ├── test_artifact.py │ ├── test_comparison.py │ ├── test_label.py │ ├── tool │ └── test_grype.py │ ├── utils │ ├── __init__.py │ └── test_utils.py │ └── validate │ ├── __init__.py │ ├── test_delta.py │ ├── test_gate.py │ └── test_validate.py └── uv.lock /.binny.yaml: -------------------------------------------------------------------------------- 1 | tools: 2 | # we want to use a pinned version of binny to manage the toolchain (so binny manages itself!) 3 | - name: binny 4 | version: 5 | want: v0.8.0 6 | method: github-release 7 | with: 8 | repo: anchore/binny 9 | 10 | # used for showing the changelog at release 11 | - name: glow 12 | version: 13 | want: v2.0.0 14 | method: github-release 15 | with: 16 | repo: charmbracelet/glow 17 | 18 | # used at release to generate the changelog 19 | - name: chronicle 20 | version: 21 | want: v0.8.0 22 | method: github-release 23 | with: 24 | repo: anchore/chronicle 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What happened**: 11 | 12 | **What you expected to happen**: 13 | 14 | **How to reproduce it (as minimally and precisely as possible)**: 15 | 16 | **Anything else we need to know?**: 17 | 18 | **Environment**: 19 | - Output of `yardstick version`: 20 | - OS (e.g: `cat /etc/os-release` or similar): 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | contact_links: 2 | 3 | - name: Join the Slack community 💬 4 | # link to our community Slack registration page 5 | url: https://anchore.com/slack 6 | about: 'Come chat with us! Ask for help, join our software development efforts, or just give us feedback!' 7 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What would you like to be added**: 11 | 12 | **Why is this needed**: 13 | 14 | **Additional context**: 15 | 16 | -------------------------------------------------------------------------------- /.github/actions/bootstrap/action.yaml: -------------------------------------------------------------------------------- 1 | name: "Bootstrap" 2 | description: "Bootstrap all tools and dependencies" 3 | inputs: 4 | uv-version: 5 | description: "UV version to install" 6 | required: true 7 | default: "0.5.16" 8 | cache-key-prefix: 9 | description: "Prefix all cache keys with this value" 10 | required: true 11 | default: "9c833ef7" 12 | tools: 13 | description: "whether to install tools" 14 | default: "true" 15 | bootstrap-apt-packages: 16 | description: "Space delimited list of tools to install via apt" 17 | default: "" 18 | 19 | runs: 20 | using: "composite" 21 | steps: 22 | 23 | - name: Install uv 24 | uses: astral-sh/setup-uv@v5 25 | with: 26 | enable-cache: true 27 | 28 | - name: "Set up Python" 29 | uses: actions/setup-python@v5 30 | with: 31 | python-version-file: "pyproject.toml" 32 | 33 | - name: Restore tool cache 34 | if: inputs.tools == 'true' 35 | id: tool-cache 36 | uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 37 | with: 38 | path: ${{ github.workspace }}/.tool 39 | key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-tool-${{ hashFiles('.binny.yaml') }} 40 | 41 | - name: Install project tools 42 | shell: bash 43 | if: inputs.tools == 'true' 44 | run: make tools 45 | 46 | - name: Install apt packages 47 | if: inputs.bootstrap-apt-packages != '' 48 | shell: bash 49 | run: | 50 | DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y ${{ inputs.bootstrap-apt-packages }} 51 | 52 | - name: Install project + dependencies 53 | shell: bash 54 | run: uv sync --all-extras --dev 55 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | - package-ecosystem: "github-actions" 5 | open-pull-requests-limit: 10 6 | directory: "/.github/actions/bootstrap" 7 | schedule: 8 | interval: "daily" 9 | 10 | - package-ecosystem: "github-actions" 11 | open-pull-requests-limit: 10 12 | directory: "/.github/workflows" 13 | schedule: 14 | interval: "daily" 15 | 16 | - package-ecosystem: "pip" 17 | directory: "/" 18 | schedule: 19 | interval: daily 20 | -------------------------------------------------------------------------------- /.github/scripts/ci-check.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | red=$(tput setaf 1) 4 | bold=$(tput bold) 5 | normal=$(tput sgr0) 6 | 7 | # assert we are running in CI (or die!) 8 | if [[ -z "$CI" ]]; then 9 | echo "${bold}${red}This script should ONLY be run in CI. Exiting...${normal}" 10 | exit 1 11 | fi 12 | -------------------------------------------------------------------------------- /.github/scripts/trigger-release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | 4 | bold=$(tput bold) 5 | normal=$(tput sgr0) 6 | 7 | if ! [ -x "$(command -v gh)" ]; then 8 | echo "The GitHub CLI could not be found. To continue follow the instructions at https://github.com/cli/cli#installation" 9 | exit 1 10 | fi 11 | 12 | gh auth status 13 | 14 | # we need all of the git state to determine the next version. Since tagging is done by 15 | # the release pipeline it is possible to not have all of the tags from previous releases. 16 | git fetch --tags 17 | 18 | # populates the CHANGELOG.md and VERSION files 19 | echo "${bold}Generating changelog...${normal}" 20 | make changelog 2> /dev/null 21 | 22 | NEXT_VERSION=$(cat VERSION) 23 | 24 | if [[ "$NEXT_VERSION" == "" || "${NEXT_VERSION}" == "(Unreleased)" ]]; then 25 | echo "Could not determine the next version to release. Exiting..." 26 | exit 1 27 | fi 28 | 29 | while true; do 30 | read -p "${bold}Do you want to trigger a release for version '${NEXT_VERSION}'?${normal} [y/n] " yn 31 | case $yn in 32 | [Yy]* ) echo; break;; 33 | [Nn]* ) echo; echo "Cancelling release..."; exit;; 34 | * ) echo "Please answer yes or no.";; 35 | esac 36 | done 37 | 38 | echo "${bold}Kicking off release for ${NEXT_VERSION}${normal}..." 39 | echo 40 | gh workflow run release.yaml -f version=${NEXT_VERSION} 41 | 42 | echo 43 | echo "${bold}Waiting for release to start...${normal}" 44 | sleep 10 45 | 46 | set +e 47 | 48 | echo "${bold}Head to the release workflow to monitor the release:${normal} $(gh run list --workflow=release.yaml --limit=1 --json url --jq '.[].url')" 49 | id=$(gh run list --workflow=release.yaml --limit=1 --json databaseId --jq '.[].databaseId') 50 | gh run watch $id --exit-status || (echo ; echo "${bold}Logs of failed step:${normal}" && GH_PAGER="" gh run view $id --log-failed) 51 | -------------------------------------------------------------------------------- /.github/workflows/dependabot-automation.yaml: -------------------------------------------------------------------------------- 1 | name: Dependabot Automation 2 | on: 3 | pull_request: 4 | 5 | permissions: 6 | pull-requests: write 7 | 8 | jobs: 9 | run: 10 | uses: anchore/workflows/.github/workflows/dependabot-automation.yaml@main 11 | -------------------------------------------------------------------------------- /.github/workflows/oss-project-board-add.yaml: -------------------------------------------------------------------------------- 1 | name: Add to OSS board 2 | 3 | permissions: 4 | contents: read 5 | 6 | on: 7 | issues: 8 | types: 9 | - opened 10 | - reopened 11 | - transferred 12 | - labeled 13 | 14 | jobs: 15 | 16 | run: 17 | uses: "anchore/workflows/.github/workflows/oss-project-board-add.yaml@main" 18 | secrets: 19 | token: ${{ secrets.OSS_PROJECT_GH_TOKEN }} 20 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: "Release" 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | version: 6 | description: tag the latest commit on main with the given version (prefixed with v) 7 | required: true 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | 14 | quality-gate: 15 | runs-on: ubuntu-24.04 16 | steps: 17 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 18 | - name: Check if tag already exists 19 | # note: this will fail if the tag already exists 20 | run: | 21 | [[ "${{ github.event.inputs.version }}" == v* ]] || (echo "version '${{ github.event.inputs.version }}' does not have a 'v' prefix" && exit 1) 22 | git tag ${{ github.event.inputs.version }} 23 | 24 | # we don't want to release commits that have been pushed and tagged, but not necessarily merged onto main 25 | - name: Ensure tagged commit is on main 26 | run: | 27 | echo "Tag: ${GITHUB_REF##*/}" 28 | git fetch origin main 29 | git merge-base --is-ancestor ${GITHUB_REF##*/} origin/main && echo "${GITHUB_REF##*/} is a commit on main!" 30 | 31 | - name: Check validation results 32 | uses: fountainhead/action-wait-for-check@5a908a24814494009c4bb27c242ea38c93c593be # v1.2.0 33 | id: validations 34 | with: 35 | token: ${{ secrets.GITHUB_TOKEN }} 36 | # This check name is defined as the github action job name (in .github/workflows/validations.yaml) 37 | checkName: "validations" 38 | ref: ${{ github.event.pull_request.head.sha || github.sha }} 39 | 40 | - name: Quality gate 41 | if: steps.validations.conclusion != 'success' 42 | run: | 43 | echo "Validations Status: ${{ steps.validations.conclusion }}" 44 | false 45 | 46 | tag: 47 | needs: 48 | - quality-gate 49 | runs-on: ubuntu-22.04 50 | permissions: 51 | contents: write 52 | steps: 53 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 54 | with: 55 | # in order to properly resolve the version from git 56 | fetch-depth: 0 57 | 58 | - name: Tag release 59 | run: | 60 | git config --global user.name "anchoreci" 61 | git config --global user.email "anchoreci@users.noreply.github.com" 62 | git tag ${{ github.event.inputs.version }} 63 | git push origin --tags 64 | env: 65 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 66 | 67 | release-pypi: 68 | needs: 69 | - tag 70 | runs-on: ubuntu-22.04 71 | environment: release 72 | permissions: 73 | contents: read 74 | # required to authenticate with PyPI via OIDC token 75 | id-token: write 76 | steps: 77 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 78 | with: 79 | # in order to properly resolve the version from git 80 | fetch-depth: 0 81 | 82 | - name: Bootstrap environment 83 | uses: ./.github/actions/bootstrap 84 | 85 | # note: authentication is via the OIDC token 86 | - name: Publish to PyPI 87 | run: make ci-publish-pypi 88 | 89 | release-github: 90 | needs: 91 | - tag 92 | runs-on: ubuntu-22.04 93 | permissions: 94 | contents: write 95 | packages: write 96 | issues: read 97 | pull-requests: read 98 | steps: 99 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 100 | with: 101 | # in order to properly resolve the version from git 102 | fetch-depth: 0 103 | 104 | - name: Bootstrap environment 105 | uses: ./.github/actions/bootstrap 106 | 107 | - name: Create github release 108 | run: | 109 | make changelog 110 | gh release create ${{ github.event.inputs.version }} -F CHANGELOG.md -t ${{ github.event.inputs.version }} 111 | env: 112 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 113 | -------------------------------------------------------------------------------- /.github/workflows/remove-awaiting-response-label.yaml: -------------------------------------------------------------------------------- 1 | name: "Manage Awaiting Response Label" 2 | 3 | on: 4 | issue_comment: 5 | types: [created] 6 | 7 | jobs: 8 | run: 9 | uses: "anchore/workflows/.github/workflows/remove-awaiting-response-label.yaml@main" 10 | secrets: 11 | token: ${{ secrets.OSS_PROJECT_GH_TOKEN }} 12 | -------------------------------------------------------------------------------- /.github/workflows/validations.yaml: -------------------------------------------------------------------------------- 1 | name: "Validations" 2 | on: 3 | workflow_dispatch: 4 | pull_request: 5 | push: 6 | branches: 7 | - main 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | 14 | # note: changing the job name requires a quality gate reference change in .github/workflows/release.yaml 15 | validations: 16 | runs-on: ubuntu-24.04 17 | steps: 18 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 19 | 20 | - name: Bootstrap environment 21 | uses: ./.github/actions/bootstrap 22 | 23 | - name: Run static analysis 24 | run: make static-analysis 25 | 26 | - name: Run unit tests 27 | run: make unit 28 | 29 | - name: Build test 30 | run: make build 31 | 32 | - name: Run CLI tests 33 | run: make cli 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | CHANGELOG.md 2 | VERSION 3 | .tmp/ 4 | .tool-versions 5 | .mise.toml 6 | .tool 7 | 8 | .idea/ 9 | .vscode/ 10 | scratch/ 11 | /.yardstick/ 12 | /.yardstick* 13 | /results/ 14 | /.yardstick.profiles.yaml 15 | /.yardstick.yaml 16 | /.grype.yaml 17 | 18 | # Byte-compiled / optimized / DLL files 19 | __pycache__/ 20 | *.py[cod] 21 | *$py.class 22 | 23 | # C extensions 24 | *.so 25 | 26 | # Distribution / packaging 27 | .Python 28 | build/ 29 | develop-eggs/ 30 | dist/ 31 | downloads/ 32 | eggs/ 33 | .eggs/ 34 | lib/ 35 | lib64/ 36 | parts/ 37 | sdist/ 38 | var/ 39 | wheels/ 40 | pip-wheel-metadata/ 41 | share/python-wheels/ 42 | *.egg-info/ 43 | .installed.cfg 44 | *.egg 45 | MANIFEST 46 | 47 | # PyInstaller 48 | # Usually these files are written by a python script from a template 49 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 50 | *.manifest 51 | *.spec 52 | 53 | # Installer logs 54 | pip-log.txt 55 | pip-delete-this-directory.txt 56 | 57 | # Unit test / coverage reports 58 | htmlcov/ 59 | .tox/ 60 | .nox/ 61 | .coverage 62 | .coverage.* 63 | .cache 64 | nosetests.xml 65 | coverage.xml 66 | *.cover 67 | *.py,cover 68 | .hypothesis/ 69 | .pytest_cache/ 70 | 71 | # Translations 72 | *.mo 73 | *.pot 74 | 75 | # Django stuff: 76 | *.log 77 | local_settings.py 78 | db.sqlite3 79 | db.sqlite3-journal 80 | 81 | # Flask stuff: 82 | instance/ 83 | .webassets-cache 84 | 85 | # Scrapy stuff: 86 | .scrapy 87 | 88 | # Sphinx documentation 89 | docs/_build/ 90 | 91 | # PyBuilder 92 | target/ 93 | 94 | # Jupyter Notebook 95 | .ipynb_checkpoints 96 | 97 | # IPython 98 | profile_default/ 99 | ipython_config.py 100 | 101 | # pyenv 102 | .python-version 103 | 104 | # pipenv 105 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 106 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 107 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 108 | # install all needed dependencies. 109 | #Pipfile.lock 110 | 111 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 112 | __pypackages__/ 113 | 114 | # Celery stuff 115 | celerybeat-schedule 116 | celerybeat.pid 117 | 118 | # SageMath parsed files 119 | *.sage.py 120 | 121 | # Environments 122 | .env 123 | .venv 124 | env/ 125 | venv/ 126 | ENV/ 127 | env.bak/ 128 | venv.bak/ 129 | 130 | # Spyder project settings 131 | .spyderproject 132 | .spyproject 133 | 134 | # Rope project settings 135 | .ropeproject 136 | 137 | # mkdocs documentation 138 | /site 139 | 140 | # mypy 141 | .mypy_cache/ 142 | .dmypy.json 143 | dmypy.json 144 | 145 | # Pyre type checker 146 | .pyre/ 147 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # the default is to run these on commit + push 2 | default_stages: 3 | - pre-push 4 | 5 | repos: 6 | - repo: https://github.com/pre-commit/pre-commit-hooks 7 | rev: v2.4.0 8 | hooks: 9 | 10 | # Prevent giant files from being committed. 11 | - id: check-added-large-files 12 | stages: 13 | - pre-push 14 | 15 | # Checks for a common error of placing code before the docstring. 16 | - id: check-docstring-first 17 | stages: 18 | - pre-push 19 | 20 | # Attempts to load all yaml files to verify syntax. 21 | - id: check-yaml 22 | stages: 23 | - pre-push 24 | 25 | # Attempts to load all json files to verify syntax. 26 | - id: check-json 27 | stages: 28 | - pre-push 29 | 30 | # Makes sure files end in a newline and only a newline. 31 | - id: end-of-file-fixer 32 | stages: 33 | - pre-push 34 | 35 | # Trims trailing whitespace. 36 | - id: trailing-whitespace 37 | stages: 38 | - pre-push 39 | 40 | # Check for files that contain merge conflict strings. 41 | - id: check-merge-conflict 42 | stages: 43 | - pre-push 44 | 45 | # Simply check whether files parse as valid python. 46 | - id: check-ast 47 | stages: 48 | - pre-push 49 | 50 | # Check for files with names that would conflict on a case-insensitive filesystem like MacOS HFS+ or Windows FAT. 51 | - id: check-case-conflict 52 | stages: 53 | - pre-push 54 | 55 | # why use the local repo instead of hosted hooks? so that dependencies are centrally managed through uv 56 | - repo: local 57 | hooks: 58 | 59 | # this is used in lieu of several flake8 plugins, isort, unimport, pulint, pyupgrade, pydocstyle, autoflake, mccabe, black, autopep8 and yapf 60 | - id: ruff 61 | name: ruff 62 | entry: make lint-fix 63 | pass_filenames: false 64 | language: system 65 | 66 | - id: format 67 | name: format 68 | entry: make format 69 | pass_filenames: false 70 | language: system 71 | 72 | - id: mypy 73 | name: mypy 74 | entry: make check-types 75 | pass_filenames: false 76 | language: system 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to yardstick 2 | 3 | If you are looking to contribute to this project and want to open a GitHub pull request ("PR"), there are a few guidelines of what we are looking for in patches. Make sure you go through this document and ensure that your code proposal is aligned. 4 | 5 | For development instructions see the [DEVELOPING.md](DEVELOPING.md) 6 | 7 | ## Adding a feature or fix 8 | 9 | If you look at the [list of yardstick issues](https://github.com/anchore/yardstick/issues) there are plenty of bugs and feature requests. 10 | 11 | ## Commit guidelines 12 | 13 | In the yardstick project we like commits and pull requests (PR) to be easy to understand and review. Open source thrives best when everything happening is over documented and small enough to be understood. 14 | 15 | ### Granular commits 16 | 17 | Please try to make every commit as simple as possible, but no simpler. The idea is that each commit should be a logical unit of code. Try not to commit too many tiny changes, for example every line changed in a file as a separate commit. And also try not to make a commit enormous, for example committing all your work at the end of the day. 18 | 19 | Rather than try to follow a strict guide on what is or is not best, we try to be flexible and simple in this space. Do what makes the most sense for the changes you are trying to include. 20 | 21 | ### Commit title and description 22 | 23 | Remember that the message you leave for a commit is for the reviewer in the present, and for someone (maybe you) changing something in the future. Please make sure the title and description used is easy to understand and explains what was done. Jokes and clever comments generally don't age well in commit messages, try to stick to the facts please. 24 | 25 | ## Sign off your work 26 | 27 | The `sign-off` is an added line at the end of the explanation for the commit, certifying that you wrote it or otherwise have the right to submit it as an open-source patch. By submitting a contribution, you agree to be bound by the terms of the DCO Version 1.1 and Apache License Version 2.0. 28 | 29 | Signing off a commit certifies the below Developer's Certificate of Origin (DCO): 30 | 31 | ```text 32 | Developer's Certificate of Origin 1.1 33 | 34 | By making a contribution to this project, I certify that: 35 | 36 | (a) The contribution was created in whole or in part by me and I 37 | have the right to submit it under the open source license 38 | indicated in the file; or 39 | 40 | (b) The contribution is based upon previous work that, to the best 41 | of my knowledge, is covered under an appropriate open source 42 | license and I have the right under that license to submit that 43 | work with modifications, whether created in whole or in part 44 | by me, under the same open source license (unless I am 45 | permitted to submit under a different license), as indicated 46 | in the file; or 47 | 48 | (c) The contribution was provided directly to me by some other 49 | person who certified (a), (b) or (c) and I have not modified 50 | it. 51 | 52 | (d) I understand and agree that this project and the contribution 53 | are public and that a record of the contribution (including all 54 | personal information I submit with it, including my sign-off) is 55 | maintained indefinitely and may be redistributed consistent with 56 | this project or the open source license(s) involved. 57 | ``` 58 | 59 | All contributions to this project are licensed under the [Apache License Version 2.0, January 2004](http://www.apache.org/licenses/). 60 | 61 | When committing your change, you can add the required line manually so that it looks like this: 62 | 63 | ```text 64 | Signed-off-by: John Doe 65 | ``` 66 | 67 | Creating a signed-off commit is then possible with `-s` or `--signoff`: 68 | 69 | ```text 70 | $ git commit -s -m "this is a commit message" 71 | ``` 72 | 73 | To double-check that the commit was signed-off, look at the log output: 74 | 75 | ```text 76 | $ git log -1 77 | commit 37ceh170e4hb283bb73d958f2036ee5k07e7fde7 (HEAD -> issue-35, origin/main, main) 78 | Author: John Doe 79 | Date: Mon Aug 1 11:27:13 2020 -0400 80 | 81 | this is a commit message 82 | 83 | Signed-off-by: John Doe 84 | ``` 85 | 86 | ## Test your changes 87 | 88 | This project has a `Makefile` which includes many helpers running both unit and integration tests. You can run `make help` to see all the options. Although PRs will have automatic checks for these, it is useful to run them locally, ensuring they pass before submitting changes. 89 | 90 | You can run the static analysis and tests: 91 | 92 | ```bash 93 | $ make 94 | 95 | # same as 96 | make static-analysis unit cli 97 | ``` 98 | 99 | ## Pull Request 100 | 101 | If you made it this far and all the tests are passing, it's time to submit a Pull Request (PR) for yardstick. Submitting a PR is always a scary moment as what happens next can be an unknown. The yardstick project strives to be easy to work with, we appreciate all contributions. Nobody is going to yell at you or try to make you feel bad. We love contributions and know how scary that first PR can be. 102 | 103 | ### PR Title and Description 104 | 105 | Just like the commit title and description mentioned above, the PR title and description is very important for letting others know what's happening. Please include any details you think a reviewer will need to more properly review your PR. 106 | 107 | A PR that is very large or poorly described has a higher likelihood of being pushed to the end of the list. Reviewers like PRs they can understand and quickly review. 108 | 109 | ### What to expect next 110 | 111 | Please be patient with the project. We try to review PRs in a timely manner, but this is highly dependent on all the other tasks we have going on. It's OK to ask for a status update every week or two, it's not OK to ask for a status update every day. 112 | 113 | It's very likely the reviewer will have questions and suggestions for changes to your PR. If your changes don't match the current style and flow of the other code, expect a request to change what you've done. 114 | 115 | ## Document your changes 116 | 117 | And lastly, when proposed changes are modifying user-facing functionality or output, it is expected the PR will include updates to the documentation as well. yardstick is not a project that is heavy on documentation. This will mostly be updating the README and help for the tool. 118 | 119 | If nobody knows new features exist, they can't use them! 120 | -------------------------------------------------------------------------------- /DEVELOPING.md: -------------------------------------------------------------------------------- 1 | # Developing 2 | 3 | ## Getting Started 4 | 5 | This project requires: 6 | - python (>= 3.11) 7 | - uv (>= 0.5): see [installation instructions](https://docs.astral.sh/uv/getting-started/installation/) 8 | 9 | Once you have python and uv installed, get the project bootstrapped: 10 | 11 | ```bash 12 | # get basic project tooling 13 | make tools 14 | 15 | # install project dependencies 16 | uv sync 17 | ``` 18 | 19 | [Pre-commit](https://pre-commit.com/) is used to help enforce static analysis checks with git hooks: 20 | 21 | ```bash 22 | uv run pre-commit install --hook-type pre-push 23 | ``` 24 | 25 | ## Developing 26 | 27 | If you want to use a locally-editable copy of yardstick while you develop: 28 | 29 | ```bash 30 | uv pip uninstall yardstick #... if you already have yardstick installed in this virtual env 31 | uv pip install -e . 32 | ``` 33 | 34 | To run all static-analysis and tests: 35 | 36 | ```bash 37 | make 38 | ``` 39 | 40 | Or run them individually: 41 | 42 | ```bash 43 | make static-analysis 44 | make unit 45 | make cli 46 | ``` 47 | 48 | If you want to see all of the things you can do: 49 | 50 | ```bash 51 | make help 52 | ``` 53 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TOOL_DIR = .tool 2 | 3 | # Command templates ################################# 4 | 5 | BINNY = $(TOOL_DIR)/binny 6 | CHRONICLE = $(TOOL_DIR)/chronicle 7 | GLOW = $(TOOL_DIR)/glow 8 | ENV = uv run 9 | 10 | 11 | # Formatting variables ################################# 12 | 13 | BOLD := $(shell tput -T linux bold) 14 | PURPLE := $(shell tput -T linux setaf 5) 15 | GREEN := $(shell tput -T linux setaf 2) 16 | CYAN := $(shell tput -T linux setaf 6) 17 | RED := $(shell tput -T linux setaf 1) 18 | RESET := $(shell tput -T linux sgr0) 19 | TITLE := $(BOLD)$(PURPLE) 20 | SUCCESS := $(BOLD)$(GREEN) 21 | ERROR := $(BOLD)$(RED) 22 | 23 | 24 | .DEFAULT_GOAL := all 25 | 26 | .PHONY: all 27 | all: static-analysis test ## Run all validations 28 | 29 | .PHONY: static-analysis 30 | static-analysis: ## Run all static analyses 31 | $(ENV) pre-commit run -a --hook-stage push 32 | 33 | .PHONY: test 34 | test: unit cli ## Run all tests 35 | 36 | ## Bootstrapping targets ################################# 37 | 38 | $(TOOL_DIR): 39 | mkdir -p $(TOOL_DIR) 40 | 41 | .PHONY: tools 42 | tools: $(TOOL_DIR) ## Download and install all tooling dependencies 43 | @[ -f .tool/binny ] || curl -sSfL https://raw.githubusercontent.com/anchore/binny/main/install.sh | sh -s -- -b $(TOOL_DIR) 44 | @$(BINNY) install -v 45 | 46 | 47 | ## Static analysis targets ################################# 48 | 49 | .PHONY: lint 50 | lint: ## Show linting issues (ruff) 51 | $(ENV) ruff check src 52 | 53 | .PHONY: lint-fix 54 | lint-fix: ## Fix linting issues (ruff) 55 | $(ENV) ruff check src --fix 56 | 57 | format: ## Format code (ruff format) 58 | $(ENV) ruff format src tests 59 | 60 | .PHONY: check-types 61 | check-types: ## Run type checks (mypy) 62 | $(ENV) mypy --config-file ./pyproject.toml src/yardstick 63 | 64 | 65 | ## Testing targets ################################# 66 | 67 | .PHONY: unit 68 | unit: ## Run unit tests 69 | $(ENV) pytest --cov-report html --cov yardstick -v tests/unit/ 70 | 71 | .PHONY: cli 72 | cli: ## Run CLI tests 73 | cd ./tests/cli && make 74 | 75 | 76 | ## Build-related targets ################################# 77 | 78 | .PHONY: build 79 | build: clean-dist ## Run build assets 80 | git fetch --tags 81 | rm -rf dist 82 | uv build -v 83 | 84 | 85 | ## Release ################################# 86 | 87 | .PHONY: changelog 88 | changelog: 89 | @$(CHRONICLE) -vvv -n --version-file VERSION > CHANGELOG.md 90 | @$(GLOW) CHANGELOG.md 91 | 92 | .PHONY: release 93 | release: 94 | @.github/scripts/trigger-release.sh 95 | 96 | .PHONY: ci-check 97 | ci-check: 98 | @.github/scripts/ci-check.sh 99 | 100 | .PHONY: ci-publish-pypi 101 | ci-publish-pypi: ci-check build 102 | uv publish 103 | 104 | 105 | ## Cleanup ################################# 106 | 107 | .PHONY: clean-dist 108 | clean-dist: 109 | rm -rf dist 110 | 111 | 112 | ## Halp! ################################# 113 | 114 | .PHONY: help 115 | help: 116 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}' 117 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # yardstick 2 | 3 | A tool that can parse and compare the results of vulnerability scanner tools. 4 | 5 | Manage and explore scan results: 6 | ``` 7 | # capture a new scan result for a specific tool and image 8 | yardstick result capture --image ubuntu:20.04 -t grype@v0.11.0 9 | 10 | # list all scan results that have been captured 11 | yardstick result list 12 | 13 | # explore the scan results interactively 14 | yardstick result explore 15 | ``` 16 | 17 | Manage true positive / false positive labels for images: 18 | ``` 19 | # explore labels applied to specific scan-result matches for an image and tool pair 20 | yardstick label explore 21 | 22 | # list all managed labels 23 | yardstick label list 24 | ``` 25 | 26 | Supported scanners: 27 | - `grype` 28 | - `syft` 29 | 30 | ### F.A.Q. 31 | 32 | *"Why is syft on this list? It's not a vulnerability scanner!"* 33 | 34 | Right you are, however, capturing SBOM results that can be fed into grype or for 35 | reference during analysis is quite useful! 36 | 37 | 38 | *"Yardstick doesn't support vulnerability scanner X..."* 39 | 40 | PR's are welcome! The goal of this tool is to provide the analysis capabilities 41 | to understand how we can make these scanners better. 42 | 43 | 44 | 45 | ## Result Sets 46 | 47 | Result sets can be useful to operate on and track results from scans taken at the same time. For instance: 48 | ```yaml 49 | # .yardstick.yaml 50 | result-sets: 51 | example: 52 | matrix: 53 | images: 54 | - ubuntu:20.04 55 | tools: 56 | - name: grype 57 | version: v0.32.0 58 | - name: grype 59 | version: v0.48.0 60 | ``` 61 | 62 | ```bash 63 | # capture results for all tools 64 | $ yardstick result capture -r example 65 | 66 | # see the specific result details 67 | $ yardstick result list -r example 68 | 69 | # perform a label comparison using all tooling 70 | $ yardstick label compare -r example 71 | ``` 72 | 73 | 74 | ## Configuration 75 | 76 | Sample application config: 77 | ```yaml 78 | # .yardstick.yaml 79 | 80 | x-ref: 81 | images: &images 82 | - docker.io/cloudbees/cloudbees-core-mm:2.346.4.1@sha256:b8ec61aad2f5f9be2dc9c68923eab1de0e8b026176093ad2e0742fca310bf3bf 83 | 84 | result-sets: 85 | pr-vs-latest: 86 | description: "latest released grype vs grype from the current build" 87 | matrix: 88 | images: *images 89 | tools: 90 | - name: syft # go ahead and capture an SBOM each time to help analysis later 91 | version: v0.54.0 92 | produces: SBOM 93 | 94 | - name: grype # from the latest published github release 95 | version: latest 96 | takes: SBOM 97 | 98 | - name: grype:pr # from a local PR checkout install (feed via an environment variable) 99 | version: env:CURRENT_GRYPE_COMMIT 100 | takes: SBOM 101 | ``` 102 | 103 | ## CLI Commands 104 | 105 | ``` 106 | config show the application config 107 | 108 | label manage match labels 109 | 110 | add add a match label indication for an image 111 | apply see which labels apply to the given image and... 112 | compare compare a scan result against labeled data 113 | compare-by-ecosystem show TPs/FPs/Precision from label comparison... 114 | explore interact with an label results for a single image... 115 | images show all images derived from label data 116 | list show all labels 117 | remove remove a match label indication for an image 118 | set-image-parent set the parent image for a given image 119 | show-image-lineage show all parents and children for the given image 120 | 121 | result manage image scan results 122 | 123 | capture capture all tool output for the given image 124 | clear remove all results and result sets 125 | compare show a comparison between tool output 126 | explore interact with an image scan result 127 | images list images in results 128 | import import results for a tool that were run externally 129 | list list stored results 130 | sets list configured result sets 131 | show show a the results for a single scan + tool 132 | tools list tools in results 133 | ``` 134 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | ## Releasing 2 | 3 | Yardstick is published as a git tag in the repo, not to PyPI or elsewhere. 4 | 5 | ## Creating a new release 6 | 7 | You can release Yardstick by running `make release` (if you have the appropriate repo permissions). 8 | 9 | You do **not** need to use this local trigger script. You can always kick off the release from the GitHub actions UI as a workflow_dispatch, inputting the desired new version for the release. This approach acts as a manual override for the version if `chronicle` is non-functional or the issue/PR labels are not ready but a release is urgently needed. Remember, if you go this approach you will need to check the release notes afterwards and manually tailor as-needed. 10 | 11 | ## Managing release versions 12 | 13 | This project uses [`chronicle`](https://github.com/anchore/chronicle) to determine the next release from the current set of issues closed and PRs merged since the last release. The kind of change is determined by the set of issue labels, for example the `enhancement` label applied to a closed issue will bump the minor version and the label `bug` applied to a closed issue will bump the patch version. See the [default chronicle change definitions](https://github.com/anchore/chronicle#default-github-change-definitions) for more guidance on how labels affect the changelog. PRs can also be directly included, however, they are superseded by any closed issues that are also linked. 14 | 15 | The changelog is also generated with chronicle, collecting changes of particular kinds together with the issue/PR title acting as the changelog entry summary. 16 | 17 | If you close an issue with the label `wont-fix` or `changelog-ignore` then the issue will be excluded from consideration while creating the version and changelog. 18 | 19 | Why go this approach? The basic idea is that **if you keep issues and PRs well organized and linked (as should be done anyway) then you get version management and changelogs for free**! 20 | 21 | The python package version is managed by [`dunamai`](https://github.com/mtkennerly/dunamai) and derives the answer from a single-source of truth (the git tag) but additionally manages how that version propagates to the `pyproject.toml` and other places in the final build. The version within the `pyproject.toml` in the git repo should remain as `v0.0.0`. 22 | -------------------------------------------------------------------------------- /__main__.py: -------------------------------------------------------------------------------- 1 | from yardstick.cli.cli import cli 2 | 3 | cli() 4 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "yardstick" 3 | requires-python = "<3.14,>=3.11" 4 | authors = [ 5 | {name = "Alex Goodman", email = "alex.goodman@anchore.com"}, 6 | ] 7 | license = {text = "Apache 2.0"} 8 | dependencies = [ 9 | "click<9,>=8", 10 | "dataclasses-json<1.0.0,>=0.6.7", 11 | "tabulate<1.0.0,>=0.9.0", 12 | "prompt-toolkit<4.0.0,>=3.0.48", 13 | "Pygments<3.0.0,>=2.18.0", 14 | "requests<3.0.0,>=2.32.3", 15 | "GitPython<4.0.0,>=3.1.43", 16 | "rfc3339<7.0,>=6.2", 17 | "omitempty<1.0.0,>=0.1.1", 18 | "importlib-metadata<9.0.0,>=7.0.1", 19 | "mergedeep<2.0.0,>=1.3.4", 20 | "dataclass-wizard<1.0.0,>=0.30.1", 21 | "PyYAML<7.0,>=6.0.0", 22 | "zstandard<1.0.0,>=0.23.0", 23 | "xxhash<4.0.0,>=3.5.0", 24 | ] 25 | dynamic = ["version"] 26 | description = "Tool for comparing the results from vulnerability scanners" 27 | readme = "README.md" 28 | keywords = [ 29 | "vulnerability", 30 | "grype", 31 | ] 32 | classifiers = [ 33 | # derived from https://pypi.org/classifiers/ 34 | "Development Status :: 5 - Production/Stable", 35 | "Intended Audience :: Developers", 36 | "Intended Audience :: Information Technology", 37 | "Intended Audience :: System Administrators", 38 | "Natural Language :: English", 39 | "Operating System :: POSIX :: Linux", 40 | "Operating System :: MacOS", 41 | "Topic :: Security", 42 | "Topic :: Software Development :: Libraries :: Python Modules", 43 | "Topic :: Utilities", 44 | ] 45 | 46 | [project.urls] 47 | repository = "https://github.com/anchore/yardstick" 48 | 49 | [project.scripts] 50 | yardstick = "yardstick.cli.cli:cli" 51 | 52 | [build-system] 53 | build-backend = "hatchling.build" 54 | requires = ["hatchling", "uv-dynamic-versioning"] 55 | 56 | [tool.hatch.version] 57 | source = "uv-dynamic-versioning" 58 | 59 | [tool.uv-dynamic-versioning] 60 | vcs = "git" 61 | style = "semver" 62 | 63 | [tool.uv] 64 | trusted-publishing = "always" 65 | 66 | [[tool.uv.index]] 67 | explicit = true 68 | name = "testpypi" 69 | url = "https://test.pypi.org/simple/" 70 | publish-url = "https://test.pypi.org/legacy/" 71 | 72 | [tool.mypy] 73 | check_untyped_defs = 0 74 | ignore_missing_imports = 1 75 | ignore_errors = 0 76 | strict_optional = 0 77 | warn_unused_ignores = 0 78 | warn_redundant_casts = 1 79 | warn_unused_configs = 1 80 | 81 | [tool.pytest.ini_options] 82 | cache_dir = ".cache/pytest" 83 | 84 | [tool.ruff] 85 | cache-dir = ".cache/ruff" 86 | # allow for a wide-birth relative to what black will correct to 87 | line-length = 150 88 | 89 | 90 | [lint] 91 | ignore = [ 92 | "ARG001", # unused args are ok, as they communicate intent in interfaces, even if not used in impls. 93 | "ARG002", # unused args are ok, as they communicate intent in interfaces, even if not used in impls. 94 | "G004", # it's ok to use formatted strings for logging 95 | "PGH004", # no blanket "noqa" usage, can be improved over time, but not now 96 | "PLR2004", # a little too agressive, not allowing any magic numbers 97 | "PLW2901", # "Outer for loop variable X overwritten by inner assignment target", not useful in most cases 98 | "RUF100", # no blanket "noqa" usage, can be improved over time, but not now 99 | "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` -- not compatible with python 3.9 (even with __future__ import) 100 | "S603", # subprocess calls are common and necessary in this codebase 101 | "S607", # we cannot rely on exact paths to all tools invoked for all users so must rely on PATH resolution 102 | ] 103 | 104 | select = [ 105 | "A", # flake8-builtins 106 | # "ANN", # flake8-annotations # this is great, but let mypy handle this so it can honor type:ignore comments without noqa comments too 107 | "ARG", # flake8-unused-arguments 108 | "B", # flake8-bugbear 109 | "C", # mccabe 110 | "C4", # flake8-comprehensions 111 | "COM", # flake8-commas 112 | "DTZ", # flake8-datetimez 113 | "E", # pycodestyle, errors 114 | # "EM", # flake8-errmsg # seems to aggressive relative to the current codebase 115 | # "ERA", # flake8-eradicate # not all code in comments should be removed 116 | "EXE", # flake8-executable 117 | "F", # pyflakes 118 | "G", # flake8-logging-format 119 | "I", # isort 120 | "ICN", # flake8-import-conventions 121 | "INP", # flake8-no-pep420 122 | "ISC", # flake8-implicit-str-concat 123 | "PGH", # pygrep-hooks 124 | "PIE", # flake8-pie 125 | "PL", # pylint (this can be broken down into more checks if needed) 126 | "PT", # flake8-pytest-style 127 | # "PTH", # flake8-use-pathlib # the codebase is not ready for this yet, but would be nice to add 128 | "Q", # flake8-quotes 129 | "RET", # flake8-return 130 | "RSE", # flake8-raise 131 | "RUF", # ruff specific rules 132 | "S", # flake8-bandit 133 | "SIM", # flake8-simplify 134 | "SLF", # flakes8-self 135 | "T10", # flake8-debugger 136 | "TCH", # flake8-type-checking 137 | "TID", # flake8-tidy-imports 138 | # "TRY", # tryceratops # seems to aggressive relative to the current codebase 139 | "UP", # pyupgrade 140 | "W", # pycodestyle, warnings 141 | "YTT", # flake8-2020 142 | ] 143 | 144 | [dependency-groups] 145 | dev = [ 146 | "pytest-mock<4.0.0,>=3.14.0", 147 | "pytest<9.0.0,>=8.3.4", 148 | "pre-commit<5.0.0,>=4.0.1", 149 | "mypy<2.0,>=1.13", 150 | "isort<6.0.0,>=5.13.2", 151 | "pylint<4.0.0,>=3.3.2", 152 | "autoflake<3.0,>=2.3", 153 | "ruff>=0.8.1,<1.0.0", 154 | "pytest-unordered<1.0.0,>=0.6.1", 155 | "pytest-sugar<2.0.0,>=1.0.0", 156 | "pytest-clarity<2.0.0,>=1.0.1", 157 | "pytest-cov<7.0.0,>=6.0.0", 158 | "pytest-picked<1.0.0,>=0.5.0", 159 | "pytest-xdist<4.0.0,>=3.6.1", 160 | "types-requests<3.0.0.0,>=2.32.0.20241016", 161 | "types-tabulate<1.0.0.0,>=0.9.0.20240106", 162 | "types-pyyaml<7.0.0.0,>=6.0.12.20240917", 163 | "types-pygments<3.0.0.0,>=2.18.0.20240506", 164 | "hatchling>=1.27.0", 165 | "hatch-vcs>=0.4.0", 166 | ] 167 | -------------------------------------------------------------------------------- /src/yardstick/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Callable, Optional 3 | 4 | from . import ( 5 | arrange, 6 | artifact, 7 | capture, 8 | cli, 9 | comparison, 10 | label, 11 | store, 12 | tool, 13 | validate, 14 | utils, 15 | ) 16 | 17 | __all__ = [ 18 | "arrange", 19 | "artifact", 20 | "capture", 21 | "cli", 22 | "comparison", 23 | "label", 24 | "store", 25 | "tool", 26 | "validate", 27 | "utils", 28 | ] 29 | 30 | 31 | def compare_results( 32 | descriptions: list[str], 33 | year_max_limit: Optional[int] = None, 34 | year_from_cve_only: bool = False, 35 | matches_filter: Optional[Callable] = None, 36 | store_root: Optional[str] = None, 37 | ) -> comparison.ByPreservedMatch: 38 | results = store.scan_result.load_by_descriptions( 39 | descriptions=descriptions, 40 | year_max_limit=year_max_limit, 41 | year_from_cve_only=year_from_cve_only, 42 | skip_sbom_results=True, 43 | store_root=store_root, 44 | ) 45 | 46 | if matches_filter: 47 | for result in results: 48 | result.matches = matches_filter(result.matches) 49 | 50 | digests = {r.config.image_digest for r in results} 51 | if len(digests) != 1: 52 | raise RuntimeError(f"image digests being compared do not match: {digests}") 53 | 54 | return comparison.ByPreservedMatch(results=results) 55 | 56 | 57 | def compare_results_against_labels( # noqa: PLR0913 58 | descriptions: list[str], 59 | result_set: Optional[str] = None, 60 | fuzzy: bool = False, 61 | year_max_limit: Optional[int] = None, 62 | year_from_cve_only: bool = False, 63 | label_entries: Optional[list[artifact.LabelEntry]] = None, 64 | matches_filter: Optional[Callable] = None, 65 | store_root: Optional[str] = None, 66 | ) -> tuple[ 67 | list[artifact.ScanResult], 68 | list[artifact.LabelEntry], 69 | dict[str, comparison.AgainstLabels], 70 | comparison.ImageToolLabelStats, 71 | ]: 72 | descriptions = list(descriptions) 73 | 74 | # this is a description of what was done to the results before comparison 75 | # we want to keep this on the comparison to evaluate if the comparison result 76 | # can be compared to other comparison results (are they apples to apples). 77 | compare_configuration = { 78 | "year_max_limit": year_max_limit, 79 | "year_from_cve_only": year_from_cve_only, 80 | } 81 | 82 | if result_set: 83 | descriptions.extend(store.result_set.load(result_set).descriptions) 84 | 85 | if not descriptions: 86 | raise RuntimeError("no descriptions provided") 87 | 88 | logging.debug(f"running label comparison with {descriptions}") 89 | 90 | results = store.scan_result.load_by_descriptions( 91 | descriptions, 92 | skip_sbom_results=True, 93 | year_max_limit=year_max_limit, 94 | year_from_cve_only=year_from_cve_only, 95 | store_root=store_root, 96 | ) 97 | 98 | if matches_filter: 99 | for result in results: 100 | result.matches = matches_filter(result.matches) 101 | 102 | if label_entries is None: 103 | label_entries = store.labels.load_all( 104 | year_max_limit=year_max_limit, 105 | year_from_cve_only=year_from_cve_only, 106 | store_root=store_root, 107 | ) 108 | 109 | ( 110 | comparisons_by_result_id, 111 | stats_by_image_tool_pair, 112 | ) = comparison.of_results_against_label( 113 | *results, 114 | fuzzy_package_match=fuzzy, 115 | label_entries=label_entries, 116 | compare_configuration=compare_configuration, 117 | ) 118 | 119 | return results, label_entries, comparisons_by_result_id, stats_by_image_tool_pair 120 | 121 | 122 | def compare_results_against_labels_by_ecosystem( 123 | result_set: str, 124 | fuzzy: bool = False, 125 | year_max_limit: Optional[int] = None, 126 | year_from_cve_only: bool = False, 127 | label_entries: Optional[list[artifact.LabelEntry]] = None, 128 | ) -> tuple[ 129 | dict[str, list[artifact.ScanResult]], 130 | list[artifact.LabelEntry], 131 | comparison.ToolLabelStatsByEcosystem, 132 | ]: 133 | results = store.result_set.load_scan_results( 134 | result_set, 135 | year_max_limit=year_max_limit, 136 | skip_sbom_results=True, 137 | ) 138 | results_by_image = arrange.scan_results_by_image(results) 139 | 140 | if label_entries is None: 141 | label_entries = store.labels.load_all( 142 | year_max_limit=year_max_limit, 143 | year_from_cve_only=year_from_cve_only, 144 | ) 145 | 146 | stats = comparison.of_results_against_label_by_ecosystem( 147 | results_by_image, 148 | fuzzy_package_match=fuzzy, 149 | label_entries=label_entries, 150 | ) 151 | return results_by_image, label_entries, stats 152 | -------------------------------------------------------------------------------- /src/yardstick/arrange.py: -------------------------------------------------------------------------------- 1 | import collections 2 | from typing import Dict, List, Set 3 | 4 | from yardstick import artifact 5 | 6 | 7 | def packages_by_vulnerability( 8 | matches: List[artifact.Match], 9 | ) -> Dict[artifact.Vulnerability, Set[artifact.Package]]: 10 | result: Dict[artifact.Vulnerability, Set[artifact.Package]] = {m.vulnerability: set() for m in matches} 11 | for match in matches: 12 | result[match.vulnerability].add(match.package) 13 | return result 14 | 15 | 16 | def scan_results_by_image( 17 | results: list[artifact.ScanResult], 18 | ) -> Dict[str, list[artifact.ScanResult]]: 19 | results_by_image = collections.defaultdict(list) 20 | for r in results: 21 | results_by_image[r.config.image].append(r) 22 | return results_by_image 23 | -------------------------------------------------------------------------------- /src/yardstick/capture.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | from typing import Any, Dict, Optional, Tuple, Union 4 | 5 | from yardstick import artifact, store 6 | from yardstick.tool import get_tool, sbom_generator, vulnerability_scanner 7 | 8 | 9 | class Timer: 10 | start = None 11 | end = None 12 | 13 | def __enter__(self): 14 | self.start = datetime.datetime.now(datetime.timezone.utc) 15 | return self 16 | 17 | def __exit__(self, ty, value, traceback): 18 | self.end = datetime.datetime.now(datetime.timezone.utc) 19 | 20 | 21 | def run_scan( 22 | config: artifact.ScanConfiguration, 23 | tool: Optional[Union[vulnerability_scanner.VulnerabilityScanner, sbom_generator.SBOMGenerator]] = None, 24 | reinstall: bool = False, 25 | **kwargs, 26 | ) -> Tuple[artifact.ScanResult, str]: 27 | logging.debug( 28 | f"capturing via run config image={config.image} tool={config.tool_name}@{config.tool_version}", 29 | ) 30 | 31 | tool_cls = get_tool(str(config.tool_name)) 32 | if not tool_cls: 33 | raise RuntimeError(f"unknown tool: {config.tool.name}") 34 | 35 | if not tool: 36 | path = store.tool.install_path(config=config) 37 | tool = tool_cls.install( 38 | version=config.tool_version, 39 | path=path, 40 | use_cache=not reinstall, 41 | **kwargs, 42 | ) 43 | 44 | # some tools will have additional metadata... persist this on the config 45 | if hasattr(tool, "version_detail"): 46 | installed_version = tool.version_detail 47 | if installed_version != config.tool_version: 48 | config.detail["version_detail"] = installed_version 49 | config.tool_version = installed_version 50 | 51 | with Timer() as timer: 52 | raw_json = tool.capture(image=config.full_image, tool_input=config.tool_input) 53 | result = tool.parse(raw_json, config=config) 54 | 55 | config.timestamp = timer.start 56 | 57 | keys = {} 58 | if issubclass(tool_cls, vulnerability_scanner.VulnerabilityScanner): 59 | keys["matches"] = result 60 | elif issubclass(tool_cls, sbom_generator.SBOMGenerator): 61 | keys["packages"] = result 62 | else: 63 | raise RuntimeError("unknown tool type") 64 | 65 | metadata = artifact.ScanMetadata( 66 | timestamp=config.timestamp, 67 | elapsed=(timer.end - timer.start).microseconds / 100000.0, 68 | image_digest=config.image_digest, 69 | ) 70 | return ( 71 | artifact.ScanResult(config=config, metadata=metadata, **keys), # type: ignore[arg-type] 72 | raw_json, 73 | ) 74 | 75 | 76 | def intake(config: artifact.ScanConfiguration, raw_results: str) -> artifact.ScanResult: 77 | logging.info(f"capturing via intake config={config}") 78 | 79 | tool_cls = get_tool(config.tool_name) 80 | if not tool_cls: 81 | raise RuntimeError(f"unknown tool: {config.tool_name}") 82 | 83 | result = tool_cls.parse(raw_results, config=config) 84 | keys = {} 85 | if issubclass(tool_cls, vulnerability_scanner.VulnerabilityScanner): 86 | keys["matches"] = result 87 | elif issubclass(tool_cls, sbom_generator.SBOMGenerator): 88 | keys["packages"] = result 89 | else: 90 | raise RuntimeError("unknown tool type") 91 | 92 | metadata = artifact.ScanMetadata( 93 | timestamp=datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0), 94 | ) 95 | return artifact.ScanResult(config=config, metadata=metadata, **keys) # type: ignore[arg-type] 96 | 97 | 98 | def one( 99 | request: artifact.ScanRequest, 100 | producer_state: Optional[str] = None, 101 | profiles: Optional[Dict[str, Dict[str, Any]]] = None, 102 | ) -> artifact.ScanConfiguration: 103 | logging.debug( 104 | f"capturing data image={request.image} tool={request.tool} profile={request.profile}", 105 | ) 106 | 107 | if not profiles: 108 | profiles = {} 109 | 110 | scan_config = artifact.ScanConfiguration.new( 111 | image=request.image, 112 | tool=request.tool, 113 | label=request.label, 114 | ) 115 | 116 | if producer_state: 117 | scan_config.tool_input = producer_state 118 | 119 | profile_obj = None 120 | if request.profile: 121 | profile_obj = profiles.get(scan_config.tool_name, {}).get(request.profile, {}) 122 | if not profile_obj: 123 | raise RuntimeError(f"no profile found for tool {scan_config.tool_name}") 124 | 125 | match_results, raw_json = run_scan(config=scan_config, profile=profile_obj) 126 | store.scan_result.save( 127 | raw_json, 128 | match_results, 129 | ) 130 | 131 | return scan_config 132 | 133 | 134 | def result_set( # noqa: C901, PLR0912 135 | result_set: str, 136 | scan_requests: list[artifact.ScanRequest], 137 | only_producers: bool = False, 138 | profiles=Optional[Dict[str, Dict[str, Any]]], 139 | ) -> artifact.ResultSet: 140 | logging.info(f"capturing data result_set={result_set}") 141 | 142 | if not profiles: 143 | profiles = {} 144 | 145 | existing_result_set_obj = None 146 | if store.result_set.exists(name=result_set): 147 | existing_result_set_obj = store.result_set.load(result_set) 148 | 149 | result_set_obj = artifact.ResultSet(name=result_set) 150 | total = len(scan_requests) 151 | for idx, scan_request in enumerate(scan_requests): 152 | logging.info(f"capturing data for request {idx + 1} of {total}") 153 | producer_data_path = None 154 | if scan_request.takes: 155 | producer = result_set_obj.provider( 156 | image=scan_request.image, 157 | provides=scan_request.takes, 158 | ) 159 | if not producer: 160 | raise RuntimeError( 161 | f"unable to find result state for the requested tool {scan_request}", 162 | ) 163 | 164 | if producer.config: 165 | producer_scan_config = store.scan_result.find_one( 166 | by_description=producer.config.path, 167 | ) 168 | producer_data_path, _ = store.scan_result.store_paths( 169 | producer_scan_config, 170 | ) 171 | 172 | if only_producers and not scan_request.provides: 173 | logging.info(f"skipping non-producer tool {scan_request.tool}") 174 | continue 175 | 176 | refresh = scan_request.refresh 177 | scan_config = None 178 | 179 | if existing_result_set_obj and not refresh: 180 | result_state = existing_result_set_obj.get( 181 | image=scan_request.image, 182 | tool=scan_request.tool, 183 | ) 184 | if result_state and result_state.config: 185 | try: 186 | scan_config = store.scan_result.find_one( 187 | by_description=result_state.config.path, 188 | ) 189 | except RuntimeError: 190 | logging.warning( 191 | f"unable to find scan config for result state, will refresh: {result_state.config.path}", 192 | ) 193 | scan_config = None 194 | 195 | if scan_config: 196 | logging.info(f"using existing scan result {scan_config.ID}") 197 | 198 | if refresh or not scan_config: 199 | scan_config = one( 200 | scan_request, 201 | producer_state=producer_data_path, 202 | profiles=profiles, 203 | ) 204 | 205 | if not scan_config: 206 | raise RuntimeError(f"unable to find scan configuration for {scan_request}") 207 | 208 | result_set_obj.add(request=scan_request, scan_config=scan_config) 209 | store.result_set.save(result_set_obj) 210 | 211 | return result_set_obj 212 | -------------------------------------------------------------------------------- /src/yardstick/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # note: don't import anything here to prevent the prompt_toolkit from being loaded by default 2 | -------------------------------------------------------------------------------- /src/yardstick/cli/cli.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import enum 3 | import logging 4 | from typing import Any 5 | 6 | import click 7 | import importlib_metadata 8 | import yaml 9 | 10 | from yardstick import store 11 | from yardstick.cli import config, label, result, validate 12 | 13 | 14 | @click.option("--verbose", "-v", default=False, help="show logs", is_flag=True) 15 | @click.option("--config", "-c", "config_path", default="", help="override config path") 16 | @click.group( 17 | help="Tool for parsing and comparing the vulnerability report output from multiple tools.", 18 | ) 19 | @click.pass_context 20 | def cli(ctx, verbose: bool, config_path: str): 21 | import logging.config 22 | 23 | # initialize yardstick based on the current configuration and 24 | # set the config object to click context to pass to subcommands 25 | ctx.obj = config.load(config_path) 26 | store.config.set_values(store_root=ctx.obj.store_root) 27 | 28 | log_level = "INFO" 29 | if verbose: 30 | log_level = "DEBUG" 31 | 32 | logging.config.dictConfig( 33 | { 34 | "version": 1, 35 | "formatters": { 36 | "standard": { 37 | "format": "%(asctime)s [%(levelname)s] %(message)s", 38 | "datefmt": "", 39 | }, 40 | }, 41 | "handlers": { 42 | "default": { 43 | "level": log_level, 44 | "formatter": "standard", 45 | "class": "logging.StreamHandler", 46 | "stream": "ext://sys.stderr", 47 | }, 48 | }, 49 | "loggers": { 50 | "": { # root logger 51 | "handlers": ["default"], 52 | "level": log_level, 53 | }, 54 | }, 55 | }, 56 | ) 57 | 58 | 59 | @cli.command(name="config", help="show the application config") 60 | @click.pass_obj 61 | def show_config(cfg: config.Application): 62 | logging.info("showing application config") 63 | 64 | class IndentDumper(yaml.Dumper): 65 | def increase_indent( 66 | self, 67 | flow: bool = False, 68 | indentless: bool = False, 69 | ) -> None: 70 | return super().increase_indent(flow, False) 71 | 72 | def enum_asdict_factory(data: list[tuple[str, Any]]) -> dict[Any, Any]: 73 | # prevents showing oddities such as 74 | # 75 | # wolfi: 76 | # request_timeout: 125 77 | # runtime: 78 | # existing_input: !!python/object/apply:vunnel.provider.InputStatePolicy 79 | # - keep 80 | # existing_results: !!python/object/apply:vunnel.provider.ResultStatePolicy 81 | # - delete-before-write 82 | # on_error: 83 | # action: !!python/object/apply:vunnel.provider.OnErrorAction 84 | # - fail 85 | # input: !!python/object/apply:vunnel.provider.InputStatePolicy 86 | # - keep 87 | # results: !!python/object/apply:vunnel.provider.ResultStatePolicy 88 | # - keep 89 | # retry_count: 3 90 | # retry_delay: 5 91 | # result_store: !!python/object/apply:vunnel.result.StoreStrategy 92 | # - flat-file 93 | # 94 | # and instead preferring: 95 | # 96 | # wolfi: 97 | # request_timeout: 125 98 | # runtime: 99 | # existing_input: keep 100 | # existing_results: delete-before-write 101 | # on_error: 102 | # action: fail 103 | # input: keep 104 | # results: keep 105 | # retry_count: 3 106 | # retry_delay: 5 107 | # result_store: flat-file 108 | 109 | def convert_value(obj: Any) -> Any: 110 | if isinstance(obj, enum.Enum): 111 | return obj.value 112 | return obj 113 | 114 | return {k: convert_value(v) for k, v in data} 115 | 116 | cfg_dict = dataclasses.asdict(cfg, dict_factory=enum_asdict_factory) 117 | print(yaml.dump(cfg_dict, Dumper=IndentDumper, default_flow_style=False)) 118 | 119 | 120 | @cli.command(name="version", help="show the installed version of yardstick") 121 | @click.pass_obj 122 | def version(_: config.Application): 123 | d = importlib_metadata.distribution("yardstick") 124 | if not d: 125 | raise RuntimeError("yardstick install information could not be found") 126 | print(f"{d.name} {d.version} ({d.locate_file(d.name).parent})") 127 | 128 | 129 | cli.add_command(validate.validate) 130 | cli.add_command(result.group) 131 | cli.add_command(label.group) 132 | -------------------------------------------------------------------------------- /src/yardstick/cli/config.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | from dataclasses import dataclass, field 5 | from typing import Any, Sequence 6 | 7 | import mergedeep # type: ignore[import] 8 | import re 9 | import yaml 10 | from dataclass_wizard import asdict, fromdict # type: ignore[import] 11 | 12 | from yardstick import artifact, validate 13 | from yardstick.store import config as store_config 14 | 15 | DEFAULT_CONFIGS = ( 16 | ".yardstick.yaml", 17 | "yardstick.yaml", 18 | ) 19 | 20 | 21 | @dataclass() 22 | class Profiles: 23 | data: dict[str, dict[str, str]] = field(default_factory=dict) 24 | 25 | def get(self, tool_name: str, profile: str): 26 | return self.data.get(tool_name, {}).get(profile, {}) 27 | 28 | 29 | @dataclass() 30 | class Tool: 31 | name: str 32 | version: str 33 | label: str | None = None 34 | produces: str | None = None 35 | takes: str | None = None 36 | profile: str | None = None 37 | refresh: bool = True 38 | 39 | @property 40 | def short(self): 41 | return f"{self.name}@{self.version}" 42 | 43 | 44 | @dataclass() 45 | class ScanMatrix: 46 | images: list[str] = field(default_factory=list) 47 | tools: list[Tool] = field(default_factory=list) 48 | 49 | DIGEST_REGEX = re.compile(r"(?Psha256:[a-fA-F0-9]{64})") 50 | 51 | def __post_init__(self): 52 | for idx, tool in enumerate(self.tools): 53 | ( 54 | self.tools[idx].name, 55 | self.tools[idx].version, 56 | ) = artifact.ScanRequest.render_tool(tool.short).split("@", 1) 57 | 58 | # flatten elements in images (in case yaml anchores are used) 59 | images = [] 60 | for image in self.images: 61 | if isinstance(image, list): 62 | images += image 63 | if image.startswith("["): 64 | # technically yaml anchors to lists of lists are interpreted as strings... which is terrible 65 | images += yaml.safe_load(image) 66 | else: 67 | images += [image] 68 | self.images = images 69 | invalid = [image for image in images if not ScanMatrix.is_valid_oci_reference(image)] 70 | if invalid: 71 | raise ValueError(f"all images must be complete OCI references, but {' '.join(invalid)} are not") 72 | 73 | @staticmethod 74 | def is_valid_oci_reference(image: str) -> bool: 75 | host, _, repository, _, digest = ScanMatrix.parse_oci_reference(image) 76 | return all([host, repository, digest]) and bool(ScanMatrix.DIGEST_REGEX.match(digest or "")) and ("." in host or "localhost" in host) 77 | 78 | @staticmethod 79 | def parse_oci_reference(image: str) -> tuple[str, str, str, str, str]: 80 | host = "" 81 | path = "" 82 | host_and_path = "" 83 | repository = "" 84 | tag = "" 85 | digest = "" 86 | 87 | if "@" in image: 88 | pre_digest, digest = image.rsplit("@", 1) 89 | else: 90 | pre_digest = image 91 | 92 | if ":" in pre_digest: 93 | pre_tag, tag = pre_digest.rsplit(":", 1) 94 | else: 95 | pre_tag = pre_digest 96 | 97 | if "/" in pre_tag: 98 | host_and_path, repository = pre_tag.rsplit("/", 1) 99 | else: 100 | repository = pre_tag 101 | 102 | if host_and_path: 103 | parts = host_and_path.split("/") 104 | host = parts[0] 105 | path = "/".join(parts[1:]) if len(parts) > 1 else "" 106 | 107 | return host, path, repository, tag, digest 108 | 109 | 110 | @dataclass() 111 | class Validation(validate.GateConfig): 112 | name: str = "default" 113 | 114 | 115 | @dataclass() 116 | class ResultSet: 117 | description: str = "" 118 | declared: list[artifact.ScanRequest] = field(default_factory=list) 119 | matrix: ScanMatrix = field(default_factory=ScanMatrix) 120 | validations: list[Validation] = field(default_factory=list) 121 | 122 | def images(self) -> list[str]: 123 | return self.matrix.images + [req.image for req in self.declared] 124 | 125 | def scan_requests(self) -> list[artifact.ScanRequest]: 126 | rendered = [] 127 | for image in self.matrix.images: 128 | for tool in self.matrix.tools: 129 | rendered.append( 130 | artifact.ScanRequest( 131 | image=image, 132 | tool=tool.short, 133 | label=tool.label, 134 | profile=tool.profile, 135 | provides=tool.produces, 136 | takes=tool.takes, 137 | refresh=tool.refresh, 138 | ), 139 | ) 140 | return self.declared + rendered 141 | 142 | 143 | @dataclass() 144 | class Application: 145 | store_root: str = store_config.DEFAULT_STORE_ROOT 146 | profile_path: str = ".yardstick.profiles.yaml" 147 | profiles: Profiles = field(default_factory=Profiles) 148 | result_sets: dict[str, ResultSet] = field(default_factory=dict) 149 | default_max_year: int | None = None 150 | derive_year_from_cve_only: bool = False 151 | 152 | def max_year_for_any_result_set(self, result_sets: list[str]) -> int | None: 153 | years = [] 154 | for result_set in result_sets: 155 | m = self.max_year_for_result_set(result_set) 156 | if m is not None: 157 | years.append(m) 158 | 159 | if not years: 160 | return None 161 | 162 | return max(years) 163 | 164 | def max_year_for_result_set(self, result_set: str) -> int | None: 165 | """return the max year needed by any validation on the result set, or default_max_year""" 166 | rs = self.result_sets.get(result_set, None) 167 | years = [] 168 | if rs is not None: 169 | for gate in rs.validations: 170 | if gate.max_year is not None: 171 | years.append(gate.max_year) 172 | elif self.default_max_year is not None: 173 | years.append(self.default_max_year) 174 | 175 | if years: 176 | return max(years) 177 | 178 | return self.default_max_year 179 | 180 | 181 | def clean_dict_keys(d): 182 | new = {} 183 | for k, v in d.items(): 184 | if isinstance(v, dict): 185 | v = clean_dict_keys(v) 186 | new[k.replace("-", "_")] = v 187 | return new 188 | 189 | 190 | def yaml_decoder(data) -> dict[Any, Any]: 191 | return clean_dict_keys(yaml.safe_load(data)) 192 | 193 | 194 | def load( 195 | path: str | Sequence[str] = DEFAULT_CONFIGS, 196 | ) -> Application: 197 | cfg = _load_paths(path) 198 | 199 | if not cfg: 200 | msg = "no config found" 201 | raise RuntimeError(msg) 202 | 203 | if cfg.profile_path: 204 | try: 205 | with open(cfg.profile_path, encoding="utf-8") as yaml_file: 206 | profile = Profiles(yaml_decoder(yaml_file)) 207 | except FileNotFoundError: 208 | profile = Profiles({}) 209 | cfg.profiles = profile 210 | 211 | return cfg 212 | 213 | 214 | def _load_paths( 215 | path: str | Sequence[str], 216 | ) -> Application | None: 217 | if not path: 218 | path = DEFAULT_CONFIGS 219 | 220 | if isinstance(path, str): 221 | if path == "": 222 | path = DEFAULT_CONFIGS 223 | else: 224 | return _load(path) 225 | 226 | if isinstance(path, (list, tuple)): 227 | for p in path: 228 | if not os.path.exists(p): 229 | continue 230 | 231 | return _load(p) 232 | 233 | # use the default application config 234 | return Application() 235 | 236 | msg = f"invalid path type {type(path)}" 237 | raise ValueError(msg) 238 | 239 | 240 | def _load(path: str) -> Application: 241 | with open(path, encoding="utf-8") as f: 242 | app_object = yaml.load(f.read(), yaml.SafeLoader) or {} # noqa: S506 (since our loader is using the safe loader) 243 | # we need a full default application config first then merge the loaded config on top. 244 | # Why? dataclass_wizard.fromdict() will create instances from the dataclass default 245 | # and NOT the field definition from the container. So it is possible to specify a 246 | # single field in the config and all other fields would be set to the default value 247 | # based on the dataclass definition and not any field(default_factory=...) hints 248 | # from the containing class. 249 | instance = asdict(Application()) 250 | 251 | mergedeep.merge(instance, app_object) 252 | cfg = fromdict( 253 | Application, 254 | instance, 255 | ) 256 | 257 | if cfg is None: 258 | msg = "parsed empty config" 259 | raise FileNotFoundError(msg) 260 | 261 | return cfg 262 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/__init__.py: -------------------------------------------------------------------------------- 1 | from . import image_labels, result 2 | 3 | __all__ = ["image_labels", "result"] 4 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/image_labels/cve_provider.py: -------------------------------------------------------------------------------- 1 | from yardstick import utils 2 | 3 | 4 | class CveDescriptions: 5 | def __init__(self): 6 | self.grype_db = utils.grype_db.GrypeDBManager() 7 | self.cache = {} 8 | 9 | def is_cached(self, cve: str): 10 | return cve in self.cache 11 | 12 | def get(self, cve: str): 13 | if cve in self.cache: 14 | return self.cache[cve] 15 | 16 | if not self.grype_db.enabled: 17 | description = "could not connect to grype db:\n" + self.grype_db.message 18 | else: 19 | description = self.grype_db.get_all_vulnerability_descriptions(cve) 20 | 21 | if not description: 22 | description = "Unable to get CVE description" 23 | 24 | self.cache[cve] = description 25 | return description 26 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/image_labels/display_pane.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from prompt_toolkit.formatted_text import split_lines, to_formatted_text 4 | from prompt_toolkit.key_binding import KeyBindings 5 | from prompt_toolkit.layout import ( 6 | Container, 7 | Dimension, 8 | FormattedTextControl, 9 | HSplit, 10 | Window, 11 | ) 12 | from prompt_toolkit.layout.containers import ConditionalContainer 13 | from prompt_toolkit.layout.margins import ScrollbarMargin 14 | from prompt_toolkit.layout.screen import Point 15 | 16 | 17 | class DisplayPane: 18 | def __init__( 19 | self, 20 | get_formatted_text, 21 | filter=None, # noqa: A002 22 | title="", 23 | height=None, 24 | ) -> None: 25 | self.cursor = Point(0, 0) 26 | self.get_formatted_text = get_formatted_text 27 | self.container: Union[HSplit, ConditionalContainer] = HSplit( 28 | [ 29 | Window( 30 | content=FormattedTextControl( 31 | text=to_formatted_text(title, style="bold reverse"), 32 | focusable=False, 33 | ), 34 | style="class:pane-title", 35 | height=Dimension.exact(1), 36 | cursorline=True, 37 | always_hide_cursor=True, 38 | wrap_lines=False, 39 | ), 40 | Window( 41 | content=FormattedTextControl( 42 | text=get_formatted_text, 43 | focusable=True, 44 | key_bindings=self._get_key_bindings(), 45 | get_cursor_position=self._cursor_position, 46 | ), 47 | height=height, 48 | style="class:details-box", 49 | always_hide_cursor=False, 50 | right_margins=[ 51 | ScrollbarMargin(display_arrows=False), 52 | ], 53 | wrap_lines=True, 54 | ), 55 | ], 56 | width=Dimension(preferred=80), 57 | ) 58 | 59 | if filter is not None: 60 | self.container = ConditionalContainer(content=self.container, filter=filter) 61 | 62 | def _cursor_position(self): 63 | # don't allow the cursor to go beyond the rendered content length 64 | lines = list(split_lines(self.get_formatted_text()())) 65 | if self.cursor.y >= len(lines) - 1: 66 | self.cursor = Point(self.cursor.x, len(lines) - 1) 67 | elif self.cursor.y < 0: 68 | self.cursor = Point(self.cursor.x, 0) 69 | return self.cursor 70 | 71 | def _get_key_bindings(self) -> KeyBindings: 72 | kb = KeyBindings() 73 | 74 | @kb.add("up") 75 | def _go_up(event) -> None: 76 | if self.cursor.y > 0: 77 | self.cursor = Point(self.cursor.x, self.cursor.y - 1) 78 | 79 | @kb.add("down") 80 | def _go_down(event) -> None: 81 | self.cursor = Point(self.cursor.x, self.cursor.y + 1) 82 | 83 | return kb 84 | 85 | def __pt_container__(self) -> Container: 86 | return self.container 87 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/image_labels/edit_note_dialog.py: -------------------------------------------------------------------------------- 1 | from asyncio import Future 2 | 3 | from prompt_toolkit.application.current import get_app 4 | from prompt_toolkit.key_binding import KeyBindings 5 | from prompt_toolkit.layout import Dimension 6 | from prompt_toolkit.layout.containers import HSplit 7 | from prompt_toolkit.widgets import Button, Dialog, Label, TextArea 8 | 9 | 10 | class EditNoteDialog: 11 | def __init__( # noqa: PLR0913 12 | self, 13 | title="", 14 | label_text="", 15 | completer=None, 16 | text_area=True, 17 | default_value="", 18 | ok_button_text="OK", 19 | cancel_button_text="Cancel", 20 | ): 21 | self.future = Future() 22 | 23 | def accept_text(buf): 24 | get_app().layout.focus(ok_button) 25 | buf.complete_state = None 26 | return True 27 | 28 | def accept(): 29 | self.future.set_result(self.text_area.text) 30 | 31 | kb = KeyBindings() 32 | 33 | kb.add("escape") 34 | 35 | def cancel(): 36 | self.future.set_result(None) 37 | 38 | if text_area: 39 | self.text_area = TextArea( 40 | text=default_value or "", 41 | completer=completer, 42 | multiline=True, 43 | width=Dimension(preferred=40), 44 | height=Dimension(preferred=5), 45 | accept_handler=accept_text, 46 | ) 47 | 48 | ok_button = Button( 49 | text=ok_button_text, 50 | left_symbol="[", 51 | right_symbol="]", 52 | handler=accept, 53 | ) 54 | cancel_button = Button( 55 | text=cancel_button_text, 56 | left_symbol="[", 57 | right_symbol="]", 58 | handler=cancel, 59 | ) 60 | 61 | layout_items = [Label(text=label_text, style="bold")] 62 | if text_area: 63 | layout_items.append(self.text_area) 64 | 65 | self.dialog = Dialog( 66 | title=title, 67 | body=HSplit(layout_items, key_bindings=kb), 68 | buttons=[ok_button, cancel_button], 69 | width=Dimension(preferred=150), 70 | modal=True, 71 | ) 72 | 73 | def __pt_container__(self): 74 | return self.dialog 75 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/image_labels/history.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Callable, List 3 | 4 | 5 | @dataclass 6 | class Command: 7 | undo: Callable 8 | redo: Callable 9 | 10 | 11 | class History: 12 | history: List[Command] 13 | index: int 14 | 15 | def __init__(self): 16 | self.history = [] 17 | self.index = 0 18 | 19 | def record(self, command: Command): 20 | # clear any history after the current index, but only if the index has moved (we've done undo's since the last record) 21 | if self.index + 1 != len(self.history): 22 | self.history = self.history[: self.index] 23 | # record the new event 24 | self.history.append(command) 25 | self.redo() # perform the action for the first time 26 | 27 | def undo(self): 28 | if self.index > 0: 29 | self.index -= 1 30 | self.history[self.index].undo() 31 | 32 | def redo(self): 33 | if self.index <= len(self.history) - 1: 34 | self.history[self.index].redo() 35 | self.index += 1 36 | 37 | def total_events(self): 38 | return len(self.history) 39 | 40 | def undone_events(self): 41 | return len(self.history) - (self.index) 42 | 43 | def reset(self): 44 | self.history = [] 45 | self.index = 0 46 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/image_labels/label_json_editor_dialog.py: -------------------------------------------------------------------------------- 1 | from asyncio import Future 2 | 3 | from prompt_toolkit.application.current import get_app 4 | from prompt_toolkit.formatted_text import StyleAndTextTuples 5 | from prompt_toolkit.key_binding import KeyBindings 6 | from prompt_toolkit.layout import Dimension 7 | from prompt_toolkit.layout.containers import Container, HSplit, Window 8 | from prompt_toolkit.layout.controls import FormattedTextControl 9 | from prompt_toolkit.lexers import PygmentsLexer 10 | from prompt_toolkit.widgets import Button, Dialog, TextArea 11 | from pygments.lexers.data import JsonLexer 12 | 13 | from yardstick import artifact 14 | 15 | 16 | class LabelJsonEditorDialog: 17 | future: Future 18 | 19 | def __init__( 20 | self, 21 | entry: artifact.LabelEntry, 22 | title="Edit label JSON", 23 | ok_button_text="Save", 24 | cancel_button_text="Cancel", 25 | ): 26 | self.future = Future() 27 | 28 | def accept_text(buf): 29 | get_app().layout.focus(ok_button) 30 | buf.complete_state = None 31 | return True 32 | 33 | def accept(): 34 | if self.validation_toolbar.is_value: 35 | self.future.set_result(self.text_area.text) 36 | 37 | kb = KeyBindings() 38 | 39 | kb.add("escape") 40 | 41 | def cancel(): 42 | self.future.set_result(None) 43 | 44 | formatted_json = entry.to_json(indent=2) # type: ignore[attr-defined] 45 | 46 | ok_button = Button( 47 | text=ok_button_text, 48 | left_symbol="[", 49 | right_symbol="]", 50 | handler=accept, 51 | ) 52 | cancel_button = Button( 53 | text=cancel_button_text, 54 | left_symbol="[", 55 | right_symbol="]", 56 | handler=cancel, 57 | ) 58 | 59 | self.text_area = TextArea( 60 | text=formatted_json, 61 | multiline=True, 62 | line_numbers=True, 63 | scrollbar=True, 64 | lexer=PygmentsLexer(JsonLexer), 65 | height=Dimension(preferred=50), 66 | accept_handler=accept_text, 67 | style="bg:#333333", 68 | complete_while_typing=True, 69 | ) 70 | 71 | self.validation_toolbar = ValidationToolbar(self.text_area) 72 | 73 | layout_items = [self.text_area, self.validation_toolbar] 74 | 75 | self.dialog = Dialog( 76 | title=title, 77 | # layout_items is a list of MagicContainer, which is only exported when checking types unfortunately 78 | body=HSplit(layout_items, key_bindings=kb), # type: ignore[arg-type] 79 | buttons=[ok_button, cancel_button], 80 | width=Dimension(preferred=100), 81 | modal=True, 82 | ) 83 | 84 | def __pt_container__(self): 85 | return self.dialog 86 | 87 | 88 | class ValidationToolbar: 89 | def __init__(self, text_area) -> None: 90 | self.is_value = True 91 | 92 | def get_formatted_text() -> StyleAndTextTuples: 93 | try: 94 | artifact.LabelEntry.from_json( # type: ignore[attr-defined] 95 | text_area.text, 96 | ) 97 | self.is_value = True 98 | except Exception as e: 99 | self.is_value = False 100 | return [("class:validation-toolbar", "Invalid LabelEntry: " + repr(e))] 101 | 102 | return [] 103 | 104 | self.control = FormattedTextControl(get_formatted_text) 105 | 106 | self.container = Window( 107 | self.control, 108 | height=1, 109 | always_hide_cursor=True, 110 | wrap_lines=True, 111 | ) 112 | 113 | def __pt_container__(self) -> Container: 114 | return self.container 115 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/image_labels/label_margin.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Callable 2 | 3 | from prompt_toolkit.formatted_text import StyleAndTextTuples, fragment_list_width 4 | from prompt_toolkit.layout.controls import UIContent 5 | from prompt_toolkit.layout.margins import Margin 6 | 7 | if TYPE_CHECKING: 8 | from prompt_toolkit.layout.containers import WindowRenderInfo 9 | 10 | 11 | class LabelMargin(Margin): 12 | def __init__(self, label_manager) -> None: 13 | self.label_manager = label_manager 14 | self.minwidth = 8 15 | self.maxwidth = 8 * 3 # no more than about 3 tags worth 16 | self.width = self.minwidth 17 | 18 | def get_width(self, get_ui_content: Callable[[], UIContent]) -> int: 19 | return min(self.maxwidth, self.width * 2) 20 | 21 | def create_margin( 22 | self, 23 | window_render_info: "WindowRenderInfo", 24 | width: int, 25 | height: int, 26 | ) -> StyleAndTextTuples: 27 | # Get current line number. 28 | current_lineno = window_render_info.ui_content.cursor_position.y 29 | 30 | # Construct margin. 31 | result: StyleAndTextTuples = [] 32 | last_lineno = None 33 | 34 | entries = self.label_manager.match_select_entries 35 | 36 | self.width = self.minwidth 37 | for _, lineno in enumerate(window_render_info.displayed_lines): 38 | # Only display line number if this line is not a continuation of the previous line. 39 | if lineno != last_lineno: 40 | if lineno is None: 41 | pass 42 | 43 | offset = 0 44 | if lineno == current_lineno: 45 | offset = 2 46 | 47 | if lineno < len(entries): 48 | text = entries[lineno].get_formatted_annotations() 49 | 50 | content_width = fragment_list_width(text) 51 | self.width = max(self.width, content_width, self.minwidth) 52 | padding = width - content_width 53 | if padding > 0: 54 | result.append(("", " " * (padding - offset))) 55 | 56 | if lineno == current_lineno: 57 | result.append(("fg:#888888", "● ")) 58 | result.extend(text) 59 | 60 | last_lineno = lineno 61 | result.append(("", "\n")) 62 | 63 | return result 64 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/image_labels/label_selection_pane.py: -------------------------------------------------------------------------------- 1 | from asyncio import Future 2 | from typing import List, Optional, Union 3 | 4 | from prompt_toolkit.formatted_text import ( 5 | AnyFormattedText, 6 | merge_formatted_text, 7 | to_formatted_text, 8 | ) 9 | from prompt_toolkit.key_binding import KeyBindings 10 | from prompt_toolkit.layout import ( 11 | ConditionalContainer, 12 | Container, 13 | Dimension, 14 | FormattedTextControl, 15 | ) 16 | from prompt_toolkit.layout.containers import HSplit, Window 17 | from prompt_toolkit.layout.margins import ScrollbarMargin 18 | 19 | from yardstick import artifact 20 | from yardstick.cli.explore.image_labels.edit_note_dialog import EditNoteDialog 21 | from yardstick.cli.explore.image_labels.label_json_editor_dialog import ( 22 | LabelJsonEditorDialog, 23 | ) 24 | from yardstick.cli.explore.image_labels.label_manager import LabelManager 25 | 26 | 27 | class LabelSelectionPane: 28 | def __init__( 29 | self, 30 | label_manager: LabelManager, 31 | dialog_executor, 32 | filter=None, # noqa: A002 33 | ) -> None: 34 | self.label_manager = label_manager 35 | self.dialog_executor = dialog_executor 36 | self.entries: List[artifact.LabelEntry] = [] 37 | self.selected_entry = 0 38 | self.match: Optional[artifact.Match] = None 39 | # self.width = 80 40 | self.container: Union[HSplit, ConditionalContainer] = HSplit( 41 | [ 42 | Window( 43 | content=FormattedTextControl( 44 | text=to_formatted_text( 45 | "Match Label Details", 46 | style="bold reverse", 47 | ), 48 | focusable=False, 49 | ), 50 | style="class:pane-title", 51 | height=Dimension.exact(1), 52 | cursorline=True, 53 | always_hide_cursor=True, 54 | wrap_lines=False, 55 | ), 56 | Window( 57 | content=FormattedTextControl( 58 | text=self.get_formatted_result_list, 59 | focusable=True, 60 | key_bindings=self._get_key_bindings(), 61 | ), 62 | style="class:select-label-box", 63 | # width=Dimension.exact(self.width), 64 | height=Dimension(preferred=15), 65 | cursorline=True, 66 | always_hide_cursor=False, 67 | wrap_lines=True, 68 | right_margins=[ 69 | ScrollbarMargin(display_arrows=False), 70 | ], 71 | ), 72 | ], 73 | ) 74 | 75 | if filter is not None: 76 | self.container = ConditionalContainer(content=self.container, filter=filter) 77 | 78 | def set_match(self, match: artifact.Match): 79 | self.selected_entry = 0 80 | self.match = match 81 | self._update() 82 | 83 | def _update(self): 84 | self.entries = self.label_manager.get_label_entries_by_match(match=self.match) 85 | 86 | def get_formatted_result_list(self) -> AnyFormattedText: 87 | result: List[AnyFormattedText] = [] 88 | 89 | for i, entry in enumerate(self.entries): 90 | if i == self.selected_entry: 91 | result.append([("[SetCursorPosition]", "")]) 92 | 93 | note = entry.note 94 | formatted_note = to_formatted_text("[no note provided]", style="italic") if not note else to_formatted_text(note) 95 | 96 | label_style = "" 97 | if entry.label == artifact.Label.TruePositive: 98 | label_style = "#428bff" 99 | elif entry.label == artifact.Label.FalsePositive: 100 | label_style = "#ff0066" 101 | elif entry.label == artifact.Label.Unclear: 102 | label_style = "#888888" 103 | 104 | result += [ 105 | to_formatted_text(f"{i + 1}. "), 106 | to_formatted_text(entry.label.display, label_style), 107 | to_formatted_text(f" [{entry.ID}]"), 108 | to_formatted_text(f" {entry.vulnerability_id}"), 109 | ] 110 | 111 | result += [ 112 | to_formatted_text(f" from {entry.user}"), 113 | ": ", 114 | formatted_note, 115 | "\n", 116 | # "\n" + "━" * (self.width - 1) + "\n", 117 | ] 118 | 119 | if not result: 120 | result.append(to_formatted_text("[no labels for match]", style="italic")) 121 | return merge_formatted_text(result) 122 | 123 | def get_selected_entry(self): 124 | if self.selected_entry > len(self.entries) - 1: 125 | return None 126 | return self.entries[self.selected_entry] 127 | 128 | def _get_key_bindings(self) -> KeyBindings: # noqa: C901 129 | kb = KeyBindings() 130 | 131 | @kb.add("up") 132 | def _go_up(event) -> None: 133 | if self.entries: 134 | self.selected_entry = (self.selected_entry - 1) % len(self.entries) 135 | 136 | @kb.add("down") 137 | def _go_down(event) -> None: 138 | if self.entries: 139 | self.selected_entry = (self.selected_entry + 1) % len(self.entries) 140 | 141 | @kb.add("backspace") 142 | @kb.add("delete") 143 | def _delete_entry(event) -> None: 144 | entry = self.get_selected_entry() 145 | if not entry: 146 | return 147 | 148 | self.label_manager.remove_label_entry(entry.ID) 149 | 150 | self._update() 151 | 152 | @kb.add("e") 153 | def _edit_note(event) -> None: 154 | entry = self.get_selected_entry() 155 | if not entry: 156 | return 157 | 158 | def done(fut: Future): 159 | if fut.done() and fut.result() is not None: 160 | note = fut.result() 161 | self.label_manager.edit_label_entry_note(entry.ID, note) 162 | self._update() 163 | 164 | self.dialog_executor( 165 | dialog=EditNoteDialog( 166 | title="Edit note", 167 | label_text="Note:", 168 | text_area=True, 169 | ok_button_text="Update", 170 | default_value=entry.note, 171 | ), 172 | done_callback=done, 173 | ) 174 | 175 | @kb.add("j") 176 | def _edit_json(event) -> None: 177 | entry = self.get_selected_entry() 178 | if not entry: 179 | return 180 | 181 | def done(fut: Future): 182 | if fut.done() and fut.result() is not None: 183 | self.label_manager.edit_label_entry_json(entry.ID, fut.result()) 184 | self._update() 185 | 186 | self.dialog_executor( 187 | dialog=LabelJsonEditorDialog(entry), 188 | done_callback=done, 189 | ) 190 | 191 | return kb 192 | 193 | def __pt_container__(self) -> Container: 194 | return self.container 195 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/image_labels/text_area.py: -------------------------------------------------------------------------------- 1 | """ 2 | Collection of reusable components for building full screen applications. 3 | 4 | All of these widgets implement the ``__pt_container__`` method, which makes 5 | them usable in any situation where we are expecting a `prompt_toolkit` 6 | container object. 7 | 8 | .. warning:: 9 | 10 | At this point, the API for these widgets is considered unstable, and can 11 | potentially change between minor releases (we try not too, but no 12 | guarantees are made yet). The public API in 13 | `prompt_toolkit.shortcuts.dialogs` on the other hand is considered stable. 14 | """ 15 | 16 | from typing import List, Optional 17 | 18 | from prompt_toolkit.auto_suggest import AutoSuggest, DynamicAutoSuggest 19 | from prompt_toolkit.buffer import Buffer, BufferAcceptHandler 20 | from prompt_toolkit.completion import Completer, DynamicCompleter 21 | from prompt_toolkit.document import Document 22 | from prompt_toolkit.filters import ( 23 | Condition, 24 | FilterOrBool, 25 | has_focus, 26 | is_done, 27 | is_true, 28 | to_filter, 29 | ) 30 | from prompt_toolkit.formatted_text import AnyFormattedText 31 | from prompt_toolkit.history import History 32 | from prompt_toolkit.key_binding import KeyBindingsBase 33 | from prompt_toolkit.key_binding.key_processor import KeyPressEvent 34 | from prompt_toolkit.layout.containers import Container, Window 35 | from prompt_toolkit.layout.controls import BufferControl, GetLinePrefixCallable 36 | from prompt_toolkit.layout.dimension import AnyDimension 37 | from prompt_toolkit.layout.dimension import Dimension as D 38 | from prompt_toolkit.layout.margins import Margin, NumberedMargin, ScrollbarMargin 39 | from prompt_toolkit.layout.processors import ( 40 | AppendAutoSuggestion, 41 | BeforeInput, 42 | ConditionalProcessor, 43 | PasswordProcessor, 44 | Processor, 45 | ) 46 | from prompt_toolkit.lexers import DynamicLexer, Lexer 47 | from prompt_toolkit.validation import DynamicValidator, Validator 48 | from prompt_toolkit.widgets.toolbars import SearchToolbar 49 | 50 | __all__ = [ 51 | "TextArea", 52 | ] 53 | 54 | E = KeyPressEvent 55 | 56 | 57 | class TextArea: 58 | def __init__( # noqa: PLR0913 59 | self, 60 | text: AnyFormattedText = "", 61 | multiline: FilterOrBool = True, 62 | password: FilterOrBool = False, 63 | lexer: Optional[Lexer] = None, 64 | auto_suggest: Optional[AutoSuggest] = None, 65 | completer: Optional[Completer] = None, 66 | complete_while_typing: FilterOrBool = True, 67 | validator: Optional[Validator] = None, 68 | accept_handler: Optional[BufferAcceptHandler] = None, 69 | history: Optional[History] = None, 70 | focusable: FilterOrBool = True, 71 | focus_on_click: FilterOrBool = False, 72 | wrap_lines: FilterOrBool = True, 73 | read_only: FilterOrBool = False, 74 | width: AnyDimension = None, 75 | height: AnyDimension = None, 76 | dont_extend_height: FilterOrBool = False, 77 | dont_extend_width: FilterOrBool = False, 78 | line_numbers: bool = False, 79 | get_line_prefix: Optional[GetLinePrefixCallable] = None, 80 | scrollbar: bool = False, 81 | style: str = "", 82 | search_field: Optional[SearchToolbar] = None, 83 | preview_search: FilterOrBool = True, 84 | prompt: AnyFormattedText = "", 85 | input_processors: Optional[List[Processor]] = None, 86 | cursorline: bool = False, 87 | cursorcolumn: bool = False, 88 | key_bindings: Optional[KeyBindingsBase] = None, 89 | right_margins: Optional[List[Margin]] = None, 90 | ) -> None: 91 | if search_field is None: 92 | search_control = None 93 | elif isinstance(search_field, SearchToolbar): 94 | search_control = search_field.control 95 | 96 | if input_processors is None: 97 | input_processors = [] 98 | 99 | # Writeable attributes. 100 | self.completer = completer 101 | self.complete_while_typing = complete_while_typing 102 | self.lexer = lexer 103 | self.auto_suggest = auto_suggest 104 | self.read_only = read_only 105 | self.wrap_lines = wrap_lines 106 | self.validator = validator 107 | 108 | self.buffer = Buffer( 109 | document=Document(text, 0), # type: ignore[arg-type] 110 | multiline=multiline, 111 | read_only=Condition(lambda: is_true(self.read_only)), 112 | completer=DynamicCompleter(lambda: self.completer), 113 | complete_while_typing=Condition( 114 | lambda: is_true(self.complete_while_typing), 115 | ), 116 | validator=DynamicValidator(lambda: self.validator), 117 | auto_suggest=DynamicAutoSuggest(lambda: self.auto_suggest), 118 | accept_handler=accept_handler, 119 | history=history, 120 | ) 121 | 122 | self.control = BufferControl( 123 | buffer=self.buffer, 124 | lexer=DynamicLexer(lambda: self.lexer), 125 | input_processors=[ 126 | ConditionalProcessor( 127 | AppendAutoSuggestion(), 128 | has_focus(self.buffer) & ~is_done, 129 | ), 130 | ConditionalProcessor( 131 | processor=PasswordProcessor(), 132 | filter=to_filter(password), 133 | ), 134 | BeforeInput(prompt, style="class:text-area.prompt"), 135 | *input_processors, 136 | ], 137 | search_buffer_control=search_control, 138 | preview_search=preview_search, 139 | focusable=focusable, 140 | focus_on_click=focus_on_click, 141 | key_bindings=key_bindings, 142 | ) 143 | 144 | if multiline: 145 | _right_margins: list[Margin] = [ScrollbarMargin(display_arrows=False)] if scrollbar else [] 146 | 147 | if right_margins: 148 | _right_margins = right_margins + _right_margins 149 | 150 | left_margins = [NumberedMargin()] if line_numbers else [] 151 | else: 152 | height = D.exact(1) 153 | left_margins = [] 154 | _right_margins = [] 155 | 156 | style = "class:text-area " + style 157 | 158 | # If no height was given, guarantee height of at least 1. 159 | if height is None: 160 | height = D(min=1) 161 | 162 | self.window = Window( 163 | height=height, 164 | width=width, 165 | dont_extend_height=dont_extend_height, 166 | dont_extend_width=dont_extend_width, 167 | content=self.control, 168 | style=style, 169 | wrap_lines=Condition(lambda: is_true(self.wrap_lines)), 170 | left_margins=left_margins, 171 | right_margins=_right_margins, 172 | get_line_prefix=get_line_prefix, 173 | cursorline=cursorline, 174 | cursorcolumn=cursorcolumn, 175 | ) 176 | 177 | @property 178 | def text(self) -> str: 179 | """ 180 | The `Buffer` text. 181 | """ 182 | return self.buffer.text 183 | 184 | @text.setter 185 | def text(self, value: str) -> None: 186 | self.document = Document(value, 0) 187 | 188 | @property 189 | def document(self) -> Document: 190 | """ 191 | The `Buffer` document (text + cursor position). 192 | """ 193 | return self.buffer.document 194 | 195 | @document.setter 196 | def document(self, value: Document) -> None: 197 | self.buffer.set_document(value, bypass_readonly=True) 198 | 199 | @property 200 | def accept_handler(self) -> Optional[BufferAcceptHandler]: 201 | """ 202 | The accept handler. Called when the user accepts the input. 203 | """ 204 | return self.buffer.accept_handler 205 | 206 | @accept_handler.setter 207 | def accept_handler(self, value: BufferAcceptHandler) -> None: 208 | self.buffer.accept_handler = value 209 | 210 | def __pt_container__(self) -> Container: 211 | return self.window 212 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/image_labels/text_dialog.py: -------------------------------------------------------------------------------- 1 | from asyncio import Future 2 | 3 | from prompt_toolkit.application.current import get_app 4 | from prompt_toolkit.key_binding import KeyBindings 5 | from prompt_toolkit.layout import Dimension 6 | from prompt_toolkit.layout.containers import HSplit 7 | from prompt_toolkit.widgets import Button, Dialog, Label, TextArea 8 | 9 | 10 | class TextDialog: 11 | def __init__( # noqa: PLR0913 12 | self, 13 | title="", 14 | label_text="", 15 | completer=None, 16 | text_area=True, 17 | default_value="", 18 | ok_button_text="OK", 19 | cancel_button_text="Cancel", 20 | ): 21 | self.future = Future() 22 | 23 | def accept_text(buf): 24 | get_app().layout.focus(ok_button) 25 | buf.complete_state = None 26 | return True 27 | 28 | def accept(): 29 | if text_area: 30 | self.future.set_result(self.text_area.text) 31 | else: 32 | self.future.set_result(True) 33 | 34 | kb = KeyBindings() 35 | 36 | kb.add("escape") 37 | 38 | def cancel(): 39 | self.future.set_result(None) 40 | 41 | if text_area: 42 | self.text_area = TextArea( 43 | text=default_value or "", 44 | completer=completer, 45 | multiline=True, 46 | width=Dimension(preferred=40), 47 | height=Dimension(preferred=5), 48 | accept_handler=accept_text, 49 | ) 50 | 51 | ok_button = Button( 52 | text=ok_button_text, 53 | left_symbol="[", 54 | right_symbol="]", 55 | handler=accept, 56 | ) 57 | cancel_button = Button( 58 | text=cancel_button_text, 59 | left_symbol="[", 60 | right_symbol="]", 61 | handler=cancel, 62 | ) 63 | 64 | layout_items = [Label(text=label_text)] 65 | if text_area: 66 | layout_items.append(self.text_area) 67 | 68 | self.dialog = Dialog( 69 | title=title, 70 | body=HSplit(layout_items, key_bindings=kb), 71 | buttons=[ok_button, cancel_button], 72 | width=Dimension(preferred=80), 73 | modal=True, 74 | ) 75 | 76 | def __pt_container__(self): 77 | return self.dialog 78 | -------------------------------------------------------------------------------- /src/yardstick/cli/explore/result.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Callable, Dict, Optional 3 | 4 | import pygments 5 | from prompt_toolkit import PromptSession, print_formatted_text 6 | from prompt_toolkit.completion import ( 7 | CompleteEvent, 8 | Completer, 9 | Completion, 10 | merge_completers, 11 | ) 12 | from prompt_toolkit.document import Document 13 | from prompt_toolkit.formatted_text import HTML, PygmentsTokens 14 | from prompt_toolkit.styles import style_from_pygments_cls 15 | from prompt_toolkit.validation import ValidationError, Validator 16 | from pygments.lexers.data import JsonLexer 17 | from pygments.styles.monokai import MonokaiStyle 18 | 19 | from yardstick import artifact 20 | from yardstick.tool import get_tool 21 | 22 | 23 | def display_match_table_row(match: artifact.Match) -> str: 24 | if not match.config: 25 | return "" 26 | t = get_tool(match.config.tool_name) 27 | package_type = t.parse_package_type(match.fullentry) # type: ignore[union-attr] 28 | return f"{match.vulnerability.id:20} {match.package.name}@{match.package.version} {package_type}" 29 | 30 | 31 | def display_match(match: artifact.Match) -> str: 32 | if not match.config: 33 | return "" 34 | t = get_tool(match.config.tool_name) 35 | package_type = t.parse_package_type(match.fullentry) # type: ignore[union-attr] 36 | pkg = f"{match.package.name}@{match.package.version}" 37 | return f"match vuln='{match.vulnerability.id}', cve='{match.vulnerability.cve_id}', package='{pkg}', type='{package_type}', id='{match.ID}'" 38 | 39 | 40 | class MatchCollection: 41 | def __init__(self, result: artifact.ScanResult): 42 | self.result = result 43 | if not self.result.matches: 44 | raise ValueError("no matches provided") 45 | 46 | self.match_display_text_by_id = {m.ID: display_match(m) for m in self.result.matches} 47 | self.match_by_id = {m.ID: m for m in self.result.matches} 48 | self.match_id_by_display_text = {v: k for k, v in self.match_display_text_by_id.items()} 49 | 50 | def has_display_text(self, text): 51 | return text in self.match_display_text_by_id.values() 52 | 53 | def get_match_display_text(self, match): 54 | return self.match_display_text_by_id[match.ID] 55 | 56 | def get_match(self, text) -> Optional[artifact.Match]: 57 | match_id = self.match_id_by_display_text.get(text, None) 58 | if match_id: 59 | return self.match_by_id.get(match_id, None) 60 | return None 61 | 62 | def get_matches(self, filter_text=None): 63 | if filter_text: 64 | filter_text = filter_text.lower() 65 | 66 | def condition(match: artifact.Match) -> bool: 67 | if not match.config: 68 | return False 69 | t = get_tool(match.config.tool_name) 70 | package_type = t.parse_package_type(match.fullentry) # type: ignore[union-attr] 71 | 72 | return ( 73 | filter_text in match.vulnerability.id.lower() 74 | or filter_text in match.package.name.lower() 75 | or filter_text in match.package.version.lower() 76 | or filter_text in package_type 77 | ) 78 | 79 | else: 80 | 81 | def condition( 82 | match: artifact.Match, 83 | ) -> bool: 84 | return True 85 | 86 | return [match for match in sorted(self.result.matches) if condition(match)] 87 | 88 | 89 | class ResultCompleter(Completer): 90 | def __init__(self, matches: MatchCollection): 91 | self.matches = matches 92 | 93 | def get_completions(self, document: Document, complete_event: CompleteEvent): 94 | matches = self.matches.get_matches(filter_text=document.text.strip()) 95 | if document.text.lower().startswith("match"): 96 | matches += self.matches.get_matches( 97 | filter_text=document.text.lstrip("match").strip(), 98 | ) 99 | 100 | for match in matches: 101 | yield Completion( 102 | self.matches.get_match_display_text(match), 103 | start_position=-len(document.text_before_cursor), 104 | display=display_match_table_row(match), 105 | display_meta="match", 106 | ) 107 | 108 | 109 | class ExploreValidator(Validator): 110 | def __init__(self, completers, matches): 111 | self.completers = completers 112 | self.matches = matches 113 | 114 | def validate(self, document: Document): 115 | # did we find a match or command? then error out 116 | if not self.completers.get_completions( 117 | document, 118 | None, 119 | ) and not self.matches.has_display_text(document.text): 120 | raise ValidationError(message="Not matches found") 121 | 122 | 123 | class Executor(Completer): 124 | def __init__(self, matches: MatchCollection): 125 | self.matches = matches 126 | self.commands: Dict[str, Callable] = { 127 | "list": self.list, 128 | "help": self.help, 129 | "match": self.match, 130 | } 131 | self.display: Dict[str, HTML] = { 132 | "list": HTML("list [optional filter text]"), 133 | "help": HTML("help"), 134 | "match": HTML( 135 | "match vuln=str package=str id=str", 136 | ), 137 | } 138 | self.command_descriptions: Dict[str, Optional[str]] = {cmd: fn.__doc__ for cmd, fn in self.commands.items()} 139 | 140 | def get_completions(self, document: Document, complete_event: CompleteEvent): 141 | text = document.text.lower() 142 | for name in self.commands: 143 | if text.startswith(name) or name.startswith(text): 144 | yield Completion( 145 | name, 146 | start_position=-len(document.text_before_cursor), 147 | display=self.display[name], 148 | display_meta="command", 149 | ) 150 | 151 | def execute(self, text): 152 | cmd = self.commands.get(text.split(" ").pop(0), None) 153 | if cmd: 154 | cmd(text) 155 | else: 156 | print(f"could not parse input '{text}'") 157 | 158 | def match(self, text: str): 159 | """ 160 | show original json entry for a given match (prompt matches partial CVE, package name, and package version)) 161 | """ 162 | match = self.matches.get_match(text) 163 | if match: 164 | json_str = json.dumps(match.fullentry, indent=2) 165 | tokens = list(pygments.lex(json_str, lexer=JsonLexer())) 166 | style = style_from_pygments_cls(pygments_style_cls=MonokaiStyle) 167 | print_formatted_text(PygmentsTokens(tokens), style=style) 168 | 169 | def list(self, text: Optional[str] = None): # noqa: A003 170 | """ 171 | list all vulnerability matches (accepts optional filter argument) 172 | """ 173 | if text: 174 | text = text.lstrip("list").strip() 175 | 176 | matches = [f"{num + 1!s:3} | " + display_match(match) for num, match in enumerate(sorted(self.matches.get_matches(text)))] 177 | print_formatted_text("\n".join(matches)) 178 | 179 | def help(self, _: Optional[str] = None): # noqa: A003 180 | """ 181 | show available commands 182 | """ 183 | messages = [] 184 | for k in sorted(self.commands): 185 | description = "" 186 | command_description = self.command_descriptions.get(k, None) 187 | if command_description: 188 | description = ": " + command_description 189 | messages.append(" " + k + "" + description) 190 | print_formatted_text(HTML("\n".join(["Commands:", *messages]))) 191 | 192 | 193 | def bottom_toolbar(result: artifact.ScanResult): 194 | def _render(): 195 | info = f"{result.config.image} {result.config.tool_name}@{result.config.tool_version}" 196 | stats = [f"[matches ]"] 197 | 198 | unique = len(set(result.matches)) 199 | if unique != len(result.matches): 200 | stats += [f"[unique matches ]"] 201 | 202 | return HTML(f"{info} {' '.join(stats)}") 203 | 204 | return _render 205 | 206 | 207 | def run(result: artifact.ScanResult): 208 | matches = MatchCollection(result) 209 | executor = Executor(matches) 210 | completer = ResultCompleter(matches) 211 | all_completers = merge_completers((completer, executor)) 212 | vaidator = ExploreValidator(all_completers, matches) 213 | config = { 214 | "bottom_toolbar": bottom_toolbar(result), 215 | # mouse_support=True, # with mouse support, scrolling is disabled 216 | "completer": all_completers, 217 | "validator": vaidator, 218 | } 219 | session: PromptSession = PromptSession() 220 | 221 | executor.help() 222 | while True: 223 | text = session.prompt("> ", **config) 224 | executor.execute(text) 225 | -------------------------------------------------------------------------------- /src/yardstick/cli/validate.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sys 3 | 4 | import click 5 | from tabulate import tabulate 6 | 7 | import yardstick 8 | from yardstick import store 9 | from yardstick import validate as val 10 | from yardstick.cli import config, display 11 | from yardstick.validate import Gate, GateInputDescription 12 | 13 | 14 | class bcolors: 15 | HEADER: str = "\033[95m" 16 | OKBLUE: str = "\033[94m" 17 | OKCYAN: str = "\033[96m" 18 | OKGREEN: str = "\033[92m" 19 | WARNING: str = "\033[93m" 20 | FAIL: str = "\033[91m" 21 | BOLD: str = "\033[1m" 22 | UNDERLINE: str = "\033[4m" 23 | RESET: str = "\033[0m" 24 | 25 | 26 | if not sys.stdout.isatty(): 27 | bcolors.HEADER = "" 28 | bcolors.OKBLUE = "" 29 | bcolors.OKCYAN = "" 30 | bcolors.OKGREEN = "" 31 | bcolors.WARNING = "" 32 | bcolors.FAIL = "" 33 | bcolors.BOLD = "" 34 | bcolors.UNDERLINE = "" 35 | bcolors.RESET = "" 36 | 37 | 38 | @click.command() 39 | @click.pass_obj 40 | @click.option( 41 | "--image", 42 | "-i", 43 | "images", 44 | multiple=True, 45 | help="filter down to one or more images to validate with (don't use the full result set)", 46 | ) 47 | @click.option( 48 | "--label-comparison", 49 | "-l", 50 | "always_run_label_comparison", 51 | is_flag=True, 52 | help="run label comparison irregardless of relative comparison results", 53 | ) 54 | @click.option( 55 | "--breakdown-by-ecosystem", 56 | "-e", 57 | is_flag=True, 58 | help="show label comparison results broken down by ecosystem", 59 | ) 60 | @click.option("--verbose", "-v", "verbosity", count=True, help="show details of all comparisons") 61 | @click.option( 62 | "--result-set", 63 | "-r", 64 | "result_sets", 65 | multiple=True, 66 | default=[], 67 | help="the result set to use for the quality gate", 68 | ) 69 | @click.option( 70 | "--all", 71 | "all_result_sets", 72 | is_flag=True, 73 | default=False, 74 | help="validate all known result sets", 75 | ) 76 | def validate( 77 | cfg: config.Application, 78 | images: list[str], 79 | always_run_label_comparison: bool, 80 | breakdown_by_ecosystem: bool, 81 | verbosity: int, 82 | result_sets: list[str], 83 | all_result_sets: bool, 84 | ): 85 | # TODO: don't artificially inflate logging; figure out what to print 86 | setup_logging(verbosity + 3) 87 | if all_result_sets and result_sets and len(result_sets) > 0: # default result set will be present anyway 88 | raise ValueError(f"cannot pass --all and -r / --result-set: {all_result_sets} {result_sets}") 89 | 90 | if all_result_sets: 91 | result_sets = [r for r in cfg.result_sets.keys()] 92 | 93 | if not result_sets: 94 | raise ValueError("must pass --result-set / -r at least once or --all to validate all result sets") 95 | 96 | # let's not load any more labels than we need to, base this off of the images we're validating 97 | if not images: 98 | unique_images = set() 99 | for r in result_sets: 100 | result_set_obj = store.result_set.load(name=r) 101 | for state in result_set_obj.state: 102 | if state and state.config and state.config.image: 103 | unique_images.add(state.config.image) 104 | images = sorted(list(unique_images)) 105 | 106 | click.echo("Loading label entries...", nl=False) 107 | label_entries = store.labels.load_for_image(images, year_max_limit=cfg.max_year_for_any_result_set(result_sets)) 108 | click.echo(f"done! {len(label_entries)} entries loaded") 109 | 110 | gates = [] 111 | for result_set in result_sets: 112 | rs_config = cfg.result_sets[result_set] 113 | for gate_config in rs_config.validations: 114 | if gate_config.max_year is None: 115 | gate_config.max_year = cfg.default_max_year 116 | 117 | click.echo(f"{bcolors.HEADER}{bcolors.BOLD}Validating with {result_set!r}{bcolors.RESET}") 118 | new_gates = val.validate_result_set( 119 | gate_config, 120 | result_set, 121 | images=images, 122 | always_run_label_comparison=always_run_label_comparison, 123 | verbosity=verbosity, 124 | label_entries=label_entries, 125 | ) 126 | for gate in new_gates: 127 | show_results_for_image(gate.input_description, gate) 128 | 129 | gates.extend(new_gates) 130 | click.echo() 131 | 132 | if breakdown_by_ecosystem: 133 | click.echo( 134 | f"{bcolors.HEADER}Breaking down label comparison by ecosystem performance...{bcolors.RESET}", 135 | ) 136 | results_by_image, label_entries, stats = yardstick.compare_results_against_labels_by_ecosystem( 137 | result_set=result_set, 138 | year_max_limit=cfg.max_year_for_result_set(result_set), 139 | label_entries=label_entries, 140 | ) 141 | display.labels_by_ecosystem_comparison( 142 | results_by_image, 143 | stats, 144 | show_images_used=False, 145 | ) 146 | click.echo() 147 | 148 | failure = not all([gate.passed() for gate in gates]) 149 | if failure: 150 | click.echo("Reasons for quality gate failure:") 151 | for gate in gates: 152 | for reason in gate.reasons: 153 | click.echo(f" - {reason}") 154 | 155 | if failure: 156 | click.echo() 157 | click.echo(f"{bcolors.FAIL}{bcolors.BOLD}Quality gate FAILED{bcolors.RESET}") 158 | sys.exit(1) 159 | else: 160 | click.echo(f"{bcolors.OKGREEN}{bcolors.BOLD}Quality gate passed!{bcolors.RESET}") 161 | 162 | 163 | def setup_logging(verbosity: int): 164 | # pylint: disable=redefined-outer-name, import-outside-toplevel 165 | import logging.config 166 | 167 | if verbosity in [0, 1, 2]: 168 | log_level = "WARN" 169 | elif verbosity == 3: 170 | log_level = "INFO" 171 | else: 172 | log_level = "DEBUG" 173 | 174 | logging.config.dictConfig( 175 | { 176 | "version": 1, 177 | "formatters": { 178 | "standard": { 179 | # [%(module)s.%(funcName)s] 180 | "format": "%(asctime)s [%(levelname)s] %(message)s", 181 | "datefmt": "", 182 | }, 183 | }, 184 | "handlers": { 185 | "default": { 186 | "level": log_level, 187 | "formatter": "standard", 188 | "class": "logging.StreamHandler", 189 | "stream": "ext://sys.stderr", 190 | }, 191 | }, 192 | "loggers": { 193 | "": { # root logger 194 | "handlers": ["default"], 195 | "level": log_level, 196 | }, 197 | }, 198 | } 199 | ) 200 | 201 | 202 | def show_delta_commentary(gate: Gate): 203 | if not gate.deltas: 204 | click.echo("No differences found between tooling (with labels)") 205 | return 206 | 207 | header_row = ["TOOL PARTITION", "PACKAGE", "VULNERABILITY", "LABEL", "COMMENTARY"] 208 | 209 | all_rows = [] 210 | for delta in gate.deltas: 211 | color = "" 212 | if delta.is_improved: 213 | color = bcolors.OKBLUE 214 | elif delta.is_improved is not None and not delta.is_improved: 215 | color = bcolors.FAIL 216 | all_rows.append( 217 | [ 218 | f"{color}{delta.tool} ONLY{bcolors.RESET}", 219 | f"{color}{delta.package_name}@{delta.package_version}{bcolors.RESET}", 220 | f"{color}{delta.vulnerability_id}{bcolors.RESET}", 221 | f"{color}{delta.label}{bcolors.RESET}", 222 | f"{delta.commentary}", 223 | ] 224 | ) 225 | 226 | def escape_ansi(line): 227 | ansi_escape = re.compile(r"(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]") 228 | return ansi_escape.sub("", line) 229 | 230 | # sort but don't consider ansi escape codes 231 | all_rows = sorted(all_rows, key=lambda x: escape_ansi(str(x[0] + x[1] + x[2] + x[3]))) 232 | click.echo("Match differences between tooling (with labels):") 233 | indent = " " 234 | click.echo( 235 | indent 236 | + tabulate( 237 | [header_row] + all_rows, 238 | tablefmt="plain", 239 | ).replace("\n", "\n" + indent) 240 | + "\n" 241 | ) 242 | 243 | 244 | def show_results_for_image(input_description: GateInputDescription, gate: Gate): 245 | click.echo(f" Results used for image {input_description.image}:") 246 | for idx, description in enumerate(input_description.configs): 247 | branch = "├──" 248 | if idx == len(input_description.configs) - 1: 249 | branch = "└──" 250 | label = " " 251 | if description.tool_label and len(description.tool_label) > 0: 252 | label = f" ({description.tool_label}) " 253 | click.echo(f" {branch} {description.id} : {description.tool}{label} against {input_description.image}") 254 | if gate.deltas: 255 | click.echo(f"Deltas for {input_description.image}:") 256 | show_delta_commentary(gate) 257 | click.echo("-" * 80) 258 | -------------------------------------------------------------------------------- /src/yardstick/label.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from yardstick.artifact import LabelEntry, Match 4 | 5 | # note: this cannot be changed, as there is persisted values with this value + store logic based on this 6 | MANUAL_SOURCE = "manual" 7 | 8 | 9 | def label_entry_matches_image_lineage( 10 | label_entry: LabelEntry, 11 | image: Optional[str], 12 | lineage: Optional[List[str]] = None, 13 | ): 14 | if not lineage: 15 | lineage = [] 16 | return any(label_entry.matches_image(i) for i in [image, *lineage]) 17 | 18 | 19 | def find_labels_for_match( # noqa: PLR0913, PLR0912, C901 20 | image: Optional[str], 21 | match: Match, 22 | label_entries: List[LabelEntry], 23 | lineage: Optional[List[str]] = None, 24 | must_match_image=True, 25 | fuzzy_package_match=False, 26 | ) -> List[LabelEntry]: 27 | matched_label_entries: List[LabelEntry] = [] 28 | for label_entry in label_entries: 29 | # this field must be matched to continue 30 | if not has_overlapping_vulnerability_id(label_entry, match): 31 | continue 32 | 33 | # this field must be matched to continue 34 | if must_match_image and not label_entry_matches_image_lineage( 35 | label_entry, 36 | image, 37 | lineage, 38 | ): 39 | continue 40 | 41 | # we need at least one more field to match to consider this label valid for the given match... 42 | matched_fields = 0 43 | 44 | if label_entry.package: 45 | if label_entry.package != match.package: 46 | # if fuzzy mathcing isn't enabled, bail now 47 | if not fuzzy_package_match: 48 | continue 49 | 50 | # names must minimally match with some normalization 51 | if label_entry.package.name.replace( 52 | "-", 53 | "_", 54 | ) != match.package.name.replace("-", "_"): 55 | continue 56 | 57 | # version must be a subset or superset of other 58 | if ( 59 | label_entry.package.version != match.package.version 60 | and label_entry.package.version not in match.package.version 61 | and match.package.version not in label_entry.package.version 62 | ): 63 | continue 64 | 65 | matched_fields += 1 66 | 67 | if label_entry.fullentry_fields: 68 | mismatched = False 69 | for value in label_entry.fullentry_fields: 70 | if not _contains_as_value(match.fullentry, value): 71 | mismatched = True 72 | break 73 | if mismatched: 74 | continue 75 | matched_fields += 1 76 | 77 | if matched_fields > 0: 78 | # we should match on a minimum number of fields, otherwise a blank entry with a vuln ID will match, which is wrong 79 | matched_label_entries.append(label_entry) 80 | return matched_label_entries 81 | 82 | 83 | def has_overlapping_vulnerability_id(label_entry: LabelEntry, match: Match) -> bool: 84 | left_ids = {label_entry.vulnerability_id, label_entry.effective_cve} 85 | right_ids = {match.vulnerability.id, match.vulnerability.cve_id} 86 | 87 | if "" in left_ids: 88 | left_ids.remove("") 89 | 90 | if "" in right_ids: 91 | right_ids.remove("") 92 | 93 | return bool(left_ids & right_ids) 94 | 95 | 96 | def _contains_as_value(o, target): 97 | if isinstance(o, dict): 98 | values = [v for k, v in o.items()] 99 | elif isinstance(o, list): 100 | values = o 101 | else: 102 | return target in o 103 | for v in values: 104 | if v == target: 105 | return True 106 | if isinstance(v, dict) and _contains_as_value(v, target): 107 | return True 108 | if isinstance(v, list) and _contains_as_value(v, target): 109 | return True 110 | return False 111 | 112 | 113 | def merge_label_entries( 114 | original_entries: List[LabelEntry], 115 | new_and_modified_entries: List[LabelEntry], 116 | deleted_ids: Optional[List[str]] = None, 117 | ) -> List[LabelEntry]: 118 | # keep a copy to prevent mutating the argument 119 | original_entries = original_entries[:] 120 | new_and_modified_entries = new_and_modified_entries[:] 121 | 122 | # keep list indexes by label entry IDs 123 | new_and_modified_id_idx = {le.ID: idx for idx, le in enumerate(new_and_modified_entries)} 124 | 125 | # step 1: take potentially mutated entries and overwrite the original entries 126 | for original_idx, _ in enumerate(original_entries): 127 | original_id = original_entries[original_idx].ID 128 | if original_id in new_and_modified_id_idx: 129 | # overwrite the old entry with the new data 130 | new_idx = new_and_modified_id_idx[original_id] 131 | original_entries[original_idx] = new_and_modified_entries[new_idx] 132 | # mark this index as not new for later skips 133 | new_and_modified_entries[new_idx] = None # type: ignore[call-overload] 134 | 135 | # step 2: there may be entries that are None, which should be deleted (unexpected, but possible) 136 | original_entries = [entry for entry in original_entries if entry] 137 | 138 | # step 3: everything left behind is a new entry, add it to the list 139 | new_entries = [entry for entry in new_and_modified_entries if entry] 140 | 141 | # step 4: remove all ids which are explicitly deleted 142 | if deleted_ids: 143 | original_entries = [entry for entry in original_entries if entry.ID not in deleted_ids] 144 | 145 | # add the new elements to the final result 146 | return original_entries + new_entries 147 | -------------------------------------------------------------------------------- /src/yardstick/store/__init__.py: -------------------------------------------------------------------------------- 1 | from . import ( 2 | config, 3 | image_lineage, 4 | label_stats, 5 | labels, 6 | naming, 7 | result_set, 8 | scan_result, 9 | tool, 10 | ) 11 | 12 | __all__ = [ 13 | "config", 14 | "image_lineage", 15 | "label_stats", 16 | "labels", 17 | "naming", 18 | "result_set", 19 | "scan_result", 20 | "tool", 21 | ] 22 | -------------------------------------------------------------------------------- /src/yardstick/store/config.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | DEFAULT_STORE_ROOT = ".yardstick" 4 | 5 | 6 | @dataclass 7 | class StoreConfig: 8 | store_root: str = DEFAULT_STORE_ROOT 9 | 10 | 11 | _config = StoreConfig() 12 | 13 | 14 | def get() -> StoreConfig: 15 | return _config 16 | 17 | 18 | def set_values(**kwargs): 19 | for k, v in kwargs.items(): 20 | setattr(get(), k, v) 21 | -------------------------------------------------------------------------------- /src/yardstick/store/image_lineage.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import os 5 | from dataclasses import dataclass 6 | 7 | from dataclasses_json import dataclass_json 8 | 9 | from yardstick.store import config as store_config 10 | 11 | IMAGE_LINEAGE_DIR = os.path.join("image-lineage") 12 | SUFFIX = ".json" 13 | 14 | 15 | @dataclass_json 16 | @dataclass(frozen=True, eq=True, order=True) 17 | class ImageLineageDocument: 18 | lineage: dict[str, list[str]] 19 | 20 | 21 | def store_path(suffix: str = SUFFIX, store_root: str | None = None) -> str: 22 | if not store_root: 23 | store_root = store_config.get().store_root 24 | 25 | return os.path.join(store_root, IMAGE_LINEAGE_DIR, "image-lineage" + suffix) 26 | 27 | 28 | def add(image: str, lineage: list[str], store_root: str | None = None): 29 | data_path = store_path(store_root=store_root) 30 | logging.debug(f"storing image lineage to {data_path!r}") 31 | 32 | os.makedirs(os.path.dirname(data_path), exist_ok=True) 33 | 34 | existing = load(store_root=store_root) 35 | existing[image] = lineage 36 | 37 | with open(data_path, "w", encoding="utf-8") as data_file: 38 | data_file.write( 39 | ImageLineageDocument(lineage=existing).to_json(indent=2), # type: ignore[attr-defined] 40 | ) 41 | 42 | 43 | def get_parents(image: str, store_root: str | None = None) -> list[str]: 44 | return load(store_root).get(image, []) 45 | 46 | 47 | def get(image: str, store_root: str | None = None) -> list[str]: 48 | result = [] 49 | parents = get_parents(image, store_root=store_root) 50 | result += parents 51 | 52 | for parent in parents: 53 | for ancestor in get(parent, store_root=store_root): 54 | if ancestor not in result: 55 | result += [ancestor] 56 | return result 57 | 58 | 59 | def load(store_root: str | None = None) -> dict[str, list[str]]: 60 | data_path = store_path(store_root=store_root) 61 | logging.debug(f"loading image lineage location={data_path!r}") 62 | 63 | if not os.path.exists(data_path): 64 | return {} 65 | 66 | with open(data_path, encoding="utf-8") as data_file: 67 | data_json = data_file.read() 68 | return ImageLineageDocument.from_json( # type: ignore[attr-defined] 69 | data_json, 70 | ).lineage 71 | -------------------------------------------------------------------------------- /src/yardstick/store/label_stats.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import contextlib 4 | import hashlib 5 | import json 6 | import logging 7 | import os 8 | import shutil 9 | from typing import Any 10 | 11 | from yardstick import comparison 12 | from yardstick.store import config as store_config 13 | 14 | LABEL_STATS_DIR = "label-stats" 15 | 16 | 17 | def _store_root(store_root: str | None = None) -> str: 18 | if not store_root: 19 | store_root = store_config.get().store_root 20 | return os.path.join(store_root, LABEL_STATS_DIR) 21 | 22 | 23 | def store_path( 24 | ids: list[str], 25 | configuration: list[dict[str, Any]] | None, 26 | store_root: str | None = None, 27 | ) -> str: 28 | config_str = _configuration_string(configuration) 29 | 30 | filename = "_".join(sorted(ids)) + "_" + config_str + ".json" 31 | 32 | return os.path.join(_store_root(store_root=store_root), filename) 33 | 34 | 35 | def clear(store_root: str | None = None): 36 | with contextlib.suppress(FileNotFoundError): 37 | shutil.rmtree(_store_root(store_root=store_root)) 38 | 39 | 40 | def save(result: comparison.ImageToolLabelStats, store_root: str | None = None): 41 | if not isinstance(result, comparison.ImageToolLabelStats): 42 | raise RuntimeError( 43 | f"only ImageToolLabelStats is supported, given {type(result)}", 44 | ) 45 | 46 | ids = [c.ID for c in result.configs] 47 | path = store_path(ids, result.compare_configs, store_root=store_root) 48 | logging.debug(f"storing label comparison state for {ids!r}") 49 | 50 | os.makedirs(os.path.dirname(path), exist_ok=True) 51 | 52 | with open(path, "w", encoding="utf-8") as data_file: 53 | data_file.write(json.dumps(result.to_dict(), indent=2)) # type: ignore[attr-defined] 54 | 55 | 56 | def _configuration_string(configurations: list[dict[str, str]] | None) -> str: 57 | if configurations is None: 58 | return "no-configuration" 59 | 60 | config_strs = [] 61 | for configuration in configurations: 62 | config_str = ",".join([f"{k}={v}" for k, v in sorted(configuration.items())]) 63 | config_strs.append(config_str) 64 | 65 | return hashlib.sha256(",".join(config_strs).encode("utf-8")).hexdigest() 66 | 67 | 68 | def load( 69 | ids: list[str], 70 | configurations: list[dict[str, Any]] | None = None, 71 | store_root: str | None = None, 72 | ) -> comparison.ImageToolLabelStats: 73 | data_path = store_path(ids, configurations, store_root=store_root) 74 | 75 | logging.debug( 76 | f"loading label comparison state for {ids!r} with detailed configurations location={data_path!r}", 77 | ) 78 | 79 | with open(data_path, encoding="utf-8") as data_file: 80 | data_json = data_file.read() 81 | 82 | return comparison.ImageToolLabelStats.from_json( # type: ignore[attr-defined] 83 | data_json, 84 | ) 85 | -------------------------------------------------------------------------------- /src/yardstick/store/naming.py: -------------------------------------------------------------------------------- 1 | SLASH_REPLACEMENT = "+" 2 | SUFFIX = ".json" 3 | 4 | 5 | class image: 6 | @staticmethod 7 | def encode(img: str) -> str: 8 | # allow for tags to have the slash replacement, but image names still get replaced 9 | 10 | if ":" in img: 11 | fields = img.split(":") 12 | image_name = ":".join(fields[:-1]) 13 | tag = fields[-1] 14 | image_name = image_name.replace("/", SLASH_REPLACEMENT) 15 | return f"{image_name}:{tag}" 16 | 17 | if "@" in img: 18 | fields = img.split("@") 19 | image_name = "@".join(fields[:-1]) 20 | digest = fields[-1] 21 | image_name = image_name.replace("/", SLASH_REPLACEMENT) 22 | return f"{image_name}@{digest}" 23 | 24 | return img.replace("/", SLASH_REPLACEMENT) 25 | 26 | @staticmethod 27 | def decode(name: str) -> str: 28 | # allow for tags to have the slash replacement, but image names still get replaced 29 | 30 | if ":" in name: 31 | fields = name.split(":") 32 | image_name = ":".join(fields[:-1]) 33 | tag = fields[-1] 34 | image_name = image_name.replace(SLASH_REPLACEMENT, "/") 35 | return f"{image_name}:{tag}" 36 | 37 | if "@" in name: 38 | fields = name.split("@") 39 | image_name = "@".join(fields[:-1]) 40 | digest = fields[-1] 41 | image_name = image_name.replace(SLASH_REPLACEMENT, "/") 42 | return f"{image_name}@{digest}" 43 | 44 | return name.replace(SLASH_REPLACEMENT, "/") 45 | -------------------------------------------------------------------------------- /src/yardstick/store/result_set.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import logging 5 | import os 6 | import shutil 7 | 8 | from yardstick import artifact 9 | from yardstick.store import config as store_config 10 | from yardstick.store import scan_result, tool 11 | 12 | 13 | def _store_root(store_root: str | None = None): 14 | if not store_root: 15 | store_root = store_config.get().store_root 16 | 17 | return tool.result_set_path(store_root=store_root) 18 | 19 | 20 | def clear(store_root: str | None = None): 21 | shutil.rmtree(_store_root(store_root=store_root), ignore_errors=True) 22 | 23 | 24 | def store_paths(name: str, store_root: str | None = None) -> str: 25 | parent_dir = _store_root(store_root=store_root) 26 | 27 | return os.path.join(parent_dir, f"{name}.json") 28 | 29 | 30 | def save(results: artifact.ResultSet, store_root: str | None = None): 31 | if not isinstance(results, artifact.ResultSet): 32 | raise RuntimeError(f"only ResultSet is supported, given {type(results)}") 33 | 34 | path = store_paths(results.name, store_root=store_root) 35 | logging.debug(f"storing result set state {results.name!r}") 36 | 37 | os.makedirs(os.path.dirname(path), exist_ok=True) 38 | 39 | with open(path, "w", encoding="utf-8") as data_file: 40 | data_file.write(json.dumps(results.to_dict(), indent=2)) # type: ignore[attr-defined] 41 | 42 | 43 | def load(name: str, store_root: str | None = None) -> artifact.ResultSet: 44 | data_path = store_paths(name, store_root=store_root) 45 | logging.debug(f"loading result set {name!r} location={data_path!r}") 46 | 47 | with open(data_path, encoding="utf-8") as data_file: 48 | data_json = data_file.read() 49 | return artifact.ResultSet.from_json(data_json) # type: ignore[attr-defined] 50 | 51 | 52 | def load_scan_results( 53 | name: str, 54 | year_max_limit: int | None = None, 55 | store_root: str | None = None, 56 | skip_sbom_results: bool = False, 57 | ) -> list[artifact.ScanResult]: 58 | data_path = store_paths(name, store_root=store_root) 59 | logging.debug( 60 | f"loading scan results from result set {name!r} location={data_path!r}", 61 | ) 62 | 63 | result_set = load(name, store_root=store_root) 64 | 65 | descriptions = [result_state.config.path for result_state in result_set.state if result_state.config] 66 | return scan_result.load_by_descriptions( 67 | descriptions, 68 | year_max_limit=year_max_limit, 69 | store_root=store_root, 70 | skip_sbom_results=skip_sbom_results, 71 | ) 72 | 73 | 74 | def exists(name: str, store_root: str | None = None) -> bool: 75 | data_path = store_paths(name, store_root=store_root) 76 | return os.path.exists(data_path) 77 | 78 | 79 | def load_all(store_root: str | None = None) -> list[artifact.ResultSet]: 80 | parent_dir = _store_root(store_root=store_root) 81 | result_sets = [] 82 | 83 | for file_name in os.listdir(parent_dir): 84 | if file_name.endswith(".json"): 85 | result_sets.append(load(file_name[:-5], store_root=store_root)) 86 | 87 | return result_sets 88 | -------------------------------------------------------------------------------- /src/yardstick/store/tool.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | from typing import TYPE_CHECKING 5 | 6 | from yardstick.store import config as store_config 7 | 8 | if TYPE_CHECKING: 9 | from yardstick import artifact 10 | 11 | TOOL_DIR = "tools" 12 | RESULT_DIR = os.path.join("result", "store") 13 | RESULT_SET_DIR = os.path.join("result", "sets") 14 | 15 | 16 | def install_base(name: str, store_root: str | None = None) -> str: 17 | if not store_root: 18 | store_root = store_config.get().store_root 19 | 20 | return os.path.join(store_config.get().store_root, TOOL_DIR, name.replace("/", "_")) 21 | 22 | 23 | def install_path( 24 | config: artifact.ScanConfiguration, 25 | store_root: str | None = None, 26 | ) -> str: 27 | if not store_root: 28 | store_root = store_config.get().store_root 29 | 30 | store_base = install_base(name=config.tool_name, store_root=store_root) 31 | 32 | version = config.tool_version 33 | if config.tool_name.startswith("grype"): 34 | version = version.split("+import-db=")[0] 35 | 36 | return os.path.join( 37 | store_base, 38 | version.replace("/", "_"), 39 | ) 40 | 41 | 42 | def results_path(store_root: str | None = None): 43 | if not store_root: 44 | store_root = store_config.get().store_root 45 | 46 | return os.path.join(store_root, RESULT_DIR) 47 | 48 | 49 | def result_set_path(store_root: str | None = None): 50 | if not store_root: 51 | store_root = store_config.get().store_root 52 | 53 | return os.path.join(store_root, RESULT_SET_DIR) 54 | -------------------------------------------------------------------------------- /src/yardstick/tool/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Type, Union 2 | 3 | from .grype import Grype 4 | from .plugin import load_plugins 5 | from .sbom_generator import SBOMGenerator 6 | from .syft import Syft 7 | from .vulnerability_scanner import VulnerabilityScanner 8 | 9 | tools: dict[str, Union[Type[SBOMGenerator], Type[VulnerabilityScanner]]] = { 10 | # vulnerability scanners 11 | "grype": Grype, 12 | # sbom generators 13 | "syft": Syft, 14 | } 15 | 16 | 17 | def Register( 18 | name: str, 19 | tool: Union[Type[SBOMGenerator], Type[VulnerabilityScanner]], 20 | ) -> None: 21 | tools[name] = tool 22 | 23 | 24 | def get_tool( 25 | name: str, 26 | ) -> Optional[Union[Type[SBOMGenerator], Type[VulnerabilityScanner]]]: 27 | # this normalizes the name and removes labels in [brackets] 28 | return tools.get(name.split("[")[0].lower()) 29 | 30 | 31 | load_plugins() 32 | -------------------------------------------------------------------------------- /src/yardstick/tool/plugin.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | if sys.version_info < (3, 10): 5 | from importlib_metadata import entry_points 6 | else: 7 | from importlib.metadata import entry_points 8 | 9 | 10 | def load_plugins(): 11 | tool_plugins = entry_points(group="yardstick.plugins.tools") 12 | logging.debug(f"discovered plugin entrypoints: {tool_plugins}") 13 | 14 | for tool in tool_plugins: 15 | try: 16 | logging.info(f"Loading tool plugin {tool.name}") 17 | tool.load() 18 | except: # noqa: E722 19 | logging.exception(f"Failed loading tool plugin {tool.name}") 20 | -------------------------------------------------------------------------------- /src/yardstick/tool/sbom_generator.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from typing import List, Optional 3 | 4 | from yardstick import artifact 5 | 6 | 7 | class SBOMGenerator(abc.ABC): 8 | @staticmethod 9 | @abc.abstractmethod 10 | def install(version: str, path: Optional[str] = None, **kwargs) -> "SBOMGenerator": 11 | raise NotImplementedError 12 | 13 | @abc.abstractmethod 14 | def capture(self, image: str, tool_input: Optional[str]) -> str: 15 | raise NotImplementedError 16 | 17 | @staticmethod 18 | @abc.abstractmethod 19 | def parse( 20 | result: str, 21 | config: artifact.ScanConfiguration, 22 | ) -> List[artifact.Package]: 23 | raise NotImplementedError 24 | -------------------------------------------------------------------------------- /src/yardstick/tool/vulnerability_scanner.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from typing import Any, Dict, List, Optional 3 | 4 | from yardstick import artifact 5 | 6 | 7 | class VulnerabilityScanner(abc.ABC): 8 | @staticmethod 9 | @abc.abstractmethod 10 | def install( 11 | version: str, 12 | path: Optional[str] = None, 13 | use_cache: Optional[bool] = True, 14 | **kwargs, 15 | ) -> "VulnerabilityScanner": 16 | raise NotImplementedError 17 | 18 | @abc.abstractmethod 19 | def capture(self, image: str, tool_input: Optional[str]) -> str: 20 | raise NotImplementedError 21 | 22 | @staticmethod 23 | @abc.abstractmethod 24 | def parse(result: str, config: artifact.ScanConfiguration) -> List[artifact.Match]: 25 | raise NotImplementedError 26 | 27 | @staticmethod 28 | @abc.abstractmethod 29 | def parse_package_type(full_entry: Optional[Dict[str, Any]]) -> str: 30 | raise NotImplementedError 31 | -------------------------------------------------------------------------------- /src/yardstick/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import hashlib 4 | import logging 5 | import os 6 | 7 | import git 8 | 9 | 10 | def local_build_version_suffix(src_path: str) -> str: 11 | src_path = os.path.abspath(os.path.expanduser(src_path)) 12 | git_desc = "" 13 | diff_digest = "clean" 14 | try: 15 | repo = git.Repo(src_path) 16 | except: 17 | logging.error(f"failed to open existing repo at {src_path!r}") 18 | raise 19 | git_desc = repo.git.describe("--tags", "--always", "--long", "--dirty") 20 | if repo.is_dirty(): 21 | # note on S324 usage: this is currently only used for deriving a unique, content-sensitive 22 | # value to use for identifying local builds. This is not used for cryptographic purposes. 23 | hash_obj = hashlib.sha1() # noqa: S324 24 | for untracked in repo.untracked_files: 25 | hash_obj.update( 26 | hash_file(os.path.join(repo.working_dir, untracked)).encode(), 27 | ) 28 | hash_obj.update(repo.git.diff("HEAD").encode()) 29 | diff_digest = hash_obj.hexdigest()[:8] 30 | return f"{git_desc}-{diff_digest}" 31 | 32 | 33 | def hash_file(path: str) -> str: 34 | # note on S324 usage: this is currently only used for deriving a unique, content-sensitive 35 | # value to use for identifying local builds. This is not used for cryptographic purposes. 36 | hash_obj = hashlib.sha1() # noqa: S324 37 | with open(path, "rb") as f: 38 | while True: 39 | data = f.read(4096) 40 | if not data: 41 | break 42 | hash_obj.update(data) 43 | return hash_obj.hexdigest() 44 | 45 | 46 | def dig(target, *keys, **kwargs): 47 | """ 48 | Traverse a nested set of dictionaries, tuples, or lists similar to ruby's dig function. 49 | """ 50 | end_of_chain = target 51 | for key in keys: 52 | if (isinstance(end_of_chain, dict) and key in end_of_chain) or (isinstance(end_of_chain, (list, tuple)) and isinstance(key, int)): 53 | end_of_chain = end_of_chain[key] 54 | else: 55 | if "fail" in kwargs and kwargs["fail"] is True: 56 | if isinstance(end_of_chain, dict): 57 | raise KeyError 58 | raise IndexError 59 | if "default" in kwargs: 60 | return kwargs["default"] 61 | end_of_chain = None 62 | break 63 | 64 | # we may have found a falsy value in the collection at the given key 65 | # and the caller has specified to return a default value in this case in it's place. 66 | if not end_of_chain and "falsy_default" in kwargs: 67 | end_of_chain = kwargs["falsy_default"] 68 | 69 | return end_of_chain 70 | 71 | 72 | def safe_div(one, two): 73 | if two == 0: 74 | return 0 75 | return float(one) / float(two) 76 | 77 | 78 | # CVE prefix + Year + Arbitrary Digits 79 | # CVE-YYYY-NNNNN 80 | def is_cve_vuln_id(vuln_id: str | None) -> bool: 81 | if not vuln_id: 82 | return False 83 | return vuln_id.lower().startswith("cve-") 84 | 85 | 86 | def parse_year_from_id(vuln_id: str) -> int | None: 87 | def try_convert_year(s: str) -> int | None: 88 | try: 89 | value = int(s) 90 | if value < 1990 or digits_in_number(value) != 4: 91 | return None 92 | return value 93 | except ValueError: 94 | return None 95 | 96 | components = vuln_id.split("-") 97 | if not components: 98 | return None 99 | 100 | first_component = components[0].lower() 101 | 102 | if len(components) == 3 and first_component in {"cve", "alas", "elsa"}: 103 | return try_convert_year(components[1]) 104 | 105 | # there are cases in the amazon data that are considered "extras" and the vulnerability ID is augmented 106 | # in a way that portrays the application scope. For instance, ALASRUBY3.0-2023-003 or ALASSELINUX-NG-2023-001. 107 | # fore more information on the "extras" feature for amazon linux, see: https://aws.amazon.com/amazon-linux-2/faqs/#Amazon_Linux_Extras 108 | if first_component.startswith("alas") and len(components) >= 3: 109 | # note that we need to reference the compoents from the end since the ID may contain a dynamic number of hyphens. 110 | return try_convert_year(components[-2]) 111 | 112 | return None 113 | 114 | 115 | def digits_in_number(n: int) -> int: 116 | count = 0 117 | while n > 0: 118 | count = count + 1 119 | n = n // 10 120 | return count 121 | 122 | 123 | def remove_prefix(s: str, prefix: str, /) -> str: 124 | if s.startswith(prefix): 125 | return s[len(prefix) :] 126 | return s[:] 127 | -------------------------------------------------------------------------------- /src/yardstick/utils/github.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | import requests 5 | 6 | 7 | def get_latest_release_version(project: str, owner: str = "anchore") -> str: 8 | headers = {} 9 | token = os.environ.get("GITHUB_TOKEN") 10 | if token: 11 | headers["Authorization"] = "Bearer " + token 12 | 13 | response = requests.get( 14 | f"https://api.github.com/repos/{owner}/{project}/releases/latest", 15 | headers=headers, 16 | timeout=15.0, 17 | ) 18 | 19 | if response.status_code >= 400: 20 | logging.error( 21 | f"error while fetching latest {project} version: {response.status_code}: {response.reason} {response.text}", 22 | ) 23 | 24 | response.raise_for_status() 25 | 26 | return response.json()["name"] 27 | -------------------------------------------------------------------------------- /src/yardstick/utils/grype_db.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import sqlite3 5 | import subprocess 6 | import sys 7 | import threading 8 | from contextlib import closing 9 | from typing import Optional 10 | 11 | 12 | def remove_prefix(text, prefix): 13 | return text[text.startswith(prefix) and len(prefix) :] 14 | 15 | 16 | class GrypeDBManager: 17 | enabled: bool 18 | message: str 19 | db_location: Optional[str] 20 | connections: dict[int, sqlite3.Connection] 21 | 22 | def __init__(self, db_location: Optional[str] = None): 23 | self.enabled = False 24 | self.message = "" 25 | self.db_location = db_location 26 | self.connections = {} 27 | 28 | if self.db_location: 29 | try: 30 | self.connect() 31 | except: # noqa: E722 32 | self.db_location = None 33 | logging.error( 34 | f"unable to open grype DB at {self.db_location}. Falling back to system grype DB.", 35 | ) 36 | 37 | if not self.db_location: 38 | self.set_db_to_system_grype_db() 39 | 40 | if self.db_location: 41 | self.enabled = True 42 | 43 | def close(self): 44 | for conn in self.connections.values(): 45 | conn.close() 46 | self.connections = {} 47 | 48 | def set_db_to_system_grype_db(self): 49 | try: 50 | logging.debug("using system grype DB...") 51 | out = subprocess.check_output( 52 | ["grype", "db", "status"], 53 | ).decode( 54 | sys.stdout.encoding, 55 | ) 56 | for line in out.split("\n"): 57 | if line.startswith("Location:"): 58 | self.db_location = remove_prefix(line, "Location:").strip() 59 | except Exception as e: 60 | self.message = str(e) 61 | logging.error("unable to open grype DB %s", e) 62 | 63 | def connect(self): 64 | # sqlite3 is not thread safe, so we need to create a connection per thread 65 | tid = threading.get_ident() 66 | if tid in self.connections: 67 | return self.connections[tid] 68 | 69 | conn = sqlite3.connect(os.path.join(self.db_location, "vulnerability.db")) 70 | self.connections[tid] = conn 71 | return conn 72 | 73 | def get_upstream_vulnerability(self, vuln_id: str) -> Optional[str]: 74 | with closing(self.connect().cursor()) as cur: 75 | cur.execute( 76 | "select related_vulnerabilities from vulnerability where id == ? ;", 77 | (vuln_id,), 78 | ) 79 | vulnerability_info = cur.fetchall() 80 | 81 | for info in vulnerability_info: 82 | if info and len(info) > 0 and info[0]: 83 | loaded_info = json.loads(info[0]) 84 | if len(loaded_info) > 0: 85 | return loaded_info[0]["id"] 86 | return None 87 | 88 | def get_vuln_description(self, vuln_id: str) -> str: 89 | with closing(self.connect().cursor()) as cur: 90 | cur.execute( 91 | "select description from vulnerability_metadata where id == ? ;", 92 | (vuln_id,), 93 | ) 94 | results = cur.fetchall() 95 | 96 | for result in results: 97 | if result and len(result) > 0 and result[0]: 98 | return result[0] 99 | return "" 100 | 101 | # get vulnerability description of a vulnerability and all its related vulnerabilities 102 | def get_all_vulnerability_descriptions(self, vuln_id: str) -> str: 103 | if not self.db_location: 104 | return "" 105 | 106 | upstream = self.get_upstream_vulnerability(vuln_id) 107 | 108 | message = "" 109 | 110 | if upstream and upstream != vuln_id: 111 | vuln_desc = self.get_vuln_description(upstream) 112 | if vuln_desc: 113 | message += f"Upstream Vulnerability: {upstream}\n{vuln_desc}\n\n" 114 | 115 | vuln_desc = self.get_vuln_description(vuln_id) 116 | if vuln_desc: 117 | message += f"Vulnerability: {vuln_id}\n{vuln_desc}\n" 118 | 119 | return message 120 | 121 | 122 | _instance = None 123 | _raise_on_failure = False 124 | 125 | 126 | def raise_on_failure(value: bool): 127 | global _raise_on_failure # noqa: PLW0603 128 | _raise_on_failure = value 129 | 130 | 131 | def use(location: str): 132 | global _instance # noqa: PLW0603 133 | 134 | if _instance: 135 | _instance.close() 136 | 137 | _instance = GrypeDBManager(location) 138 | 139 | 140 | def normalize_to_cve(vuln_id: str): 141 | global _instance # noqa: PLW0603 142 | if vuln_id.lower().startswith("cve-"): 143 | return vuln_id 144 | 145 | try: 146 | if not _instance: 147 | _instance = GrypeDBManager() 148 | 149 | upstream = _instance.get_upstream_vulnerability(vuln_id) 150 | except: # noqa: E722 151 | if _raise_on_failure: 152 | raise 153 | return vuln_id 154 | 155 | if upstream and upstream.lower().startswith("cve-"): 156 | return upstream 157 | 158 | # unable to normalize, return the original 159 | return vuln_id 160 | -------------------------------------------------------------------------------- /src/yardstick/validate/__init__.py: -------------------------------------------------------------------------------- 1 | from .delta import DeltaType, Delta 2 | from .gate import Gate, GateConfig, GateInputResultConfig, GateInputDescription 3 | from .validate import validate_image, validate_result_set 4 | 5 | __all__ = [ 6 | "GateConfig", 7 | "GateInputResultConfig", 8 | "GateInputDescription", 9 | "DeltaType", 10 | "Delta", 11 | "Gate", 12 | "validate_image", 13 | "validate_result_set", 14 | ] 15 | -------------------------------------------------------------------------------- /src/yardstick/validate/delta.py: -------------------------------------------------------------------------------- 1 | import enum 2 | from dataclasses import dataclass 3 | 4 | from yardstick import artifact, comparison 5 | 6 | 7 | class DeltaType(enum.Enum): 8 | Unknown = "Unknown" 9 | FixedFalseNegative = "FixedFalseNegative" 10 | FixedFalsePositive = "FixedFalsePositive" 11 | NewFalseNegative = "NewFalseNegative" 12 | NewFalsePositive = "NewFalsePositive" 13 | 14 | 15 | @dataclass 16 | class Delta: 17 | tool: str 18 | package_name: str 19 | package_version: str 20 | vulnerability_id: str 21 | added: bool 22 | label: str | None = None 23 | 24 | @property 25 | def is_improved(self) -> bool | None: 26 | if self.outcome in {DeltaType.FixedFalseNegative, DeltaType.FixedFalsePositive}: 27 | return True 28 | if self.outcome in {DeltaType.NewFalseNegative, DeltaType.NewFalsePositive}: 29 | return False 30 | return None 31 | 32 | @property 33 | def commentary(self) -> str: 34 | commentary = "" 35 | # if self.is_improved and self.label == artifact.Label.TruePositive.name: 36 | if self.outcome == DeltaType.FixedFalseNegative: 37 | commentary = "(this is a new TP 🙌)" 38 | elif self.outcome == DeltaType.FixedFalsePositive: 39 | commentary = "(got rid of a former FP 🙌)" 40 | elif self.outcome == DeltaType.NewFalsePositive: 41 | commentary = "(this is a new FP 😱)" 42 | elif self.outcome == DeltaType.NewFalseNegative: 43 | commentary = "(this is a new FN 😱)" 44 | 45 | return commentary 46 | 47 | @property 48 | def outcome(self) -> DeltaType: 49 | # TODO: this would be better handled post init and set I think 50 | if not self.label: 51 | return DeltaType.Unknown 52 | 53 | if not self.added: 54 | # the tool which found the unique result is the reference tool... 55 | if self.label == artifact.Label.TruePositive.name: 56 | # drats! we missed a case (this is a new FN) 57 | return DeltaType.NewFalseNegative 58 | elif artifact.Label.FalsePositive.name in self.label: 59 | # we got rid of a FP! ["hip!", "hip!"] 60 | return DeltaType.FixedFalsePositive 61 | else: 62 | # the tool which found the unique result is the current tool... 63 | if self.label == artifact.Label.TruePositive.name: 64 | # highest of fives! we found a new TP that the previous tool release missed! 65 | return DeltaType.FixedFalseNegative 66 | elif artifact.Label.FalsePositive.name in self.label: 67 | # welp, our changes resulted in a new FP... not great, maybe not terrible? 68 | return DeltaType.NewFalsePositive 69 | 70 | return DeltaType.Unknown 71 | 72 | 73 | def compute_deltas( 74 | comparisons_by_result_id: dict[str, comparison.AgainstLabels], 75 | reference_tool: str, 76 | relative_comparison: comparison.ByPreservedMatch, 77 | ): 78 | deltas = [] 79 | for result in relative_comparison.results: 80 | label_comparison = comparisons_by_result_id[result.ID] 81 | for unique_match in relative_comparison.unique[result.ID]: 82 | labels = label_comparison.labels_by_match[unique_match.ID] 83 | if not labels: 84 | label = "(unknown)" 85 | elif len(set(labels)) > 1: 86 | label = ", ".join([la.name for la in labels]) 87 | else: 88 | label = labels[0].name 89 | 90 | delta = Delta( 91 | tool=result.config.tool, 92 | package_name=unique_match.package.name, 93 | package_version=unique_match.package.version, 94 | vulnerability_id=unique_match.vulnerability.id, 95 | added=result.config.tool != reference_tool, 96 | label=label, 97 | ) 98 | deltas.append(delta) 99 | return deltas 100 | -------------------------------------------------------------------------------- /src/yardstick/validate/gate.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field, InitVar 2 | from typing import Optional 3 | 4 | from yardstick import comparison 5 | from yardstick.validate.delta import Delta 6 | 7 | 8 | @dataclass 9 | class GateConfig: 10 | max_f1_regression: float = 0.0 11 | max_new_false_negatives: int = 0 12 | max_unlabeled_percent: int = 0 13 | max_year: int | None = None 14 | reference_tool_label: str = "reference" 15 | candidate_tool_label: str = "candidate" 16 | # only consider matches from these namespaces when judging results 17 | allowed_namespaces: list[str] = field(default_factory=list) 18 | # fail this gate unless all of these namespaces are present 19 | required_namespaces: list[str] = field(default_factory=list) 20 | fail_on_empty_match_set: bool = True 21 | 22 | 23 | @dataclass 24 | class GateInputResultConfig: 25 | id: str 26 | tool: str 27 | tool_label: str 28 | 29 | 30 | @dataclass 31 | class GateInputDescription: 32 | image: str 33 | configs: list[GateInputResultConfig] = field(default_factory=list) 34 | 35 | 36 | @dataclass 37 | class Gate: 38 | reference_comparison: InitVar[Optional[comparison.LabelComparisonSummary]] 39 | candidate_comparison: InitVar[Optional[comparison.LabelComparisonSummary]] 40 | 41 | config: GateConfig 42 | 43 | input_description: GateInputDescription 44 | reasons: list[str] = field(default_factory=list) 45 | deltas: list[Delta] = field(default_factory=list) 46 | 47 | def __post_init__( 48 | self, 49 | reference_comparison: Optional[comparison.LabelComparisonSummary], 50 | candidate_comparison: Optional[comparison.LabelComparisonSummary], 51 | ): 52 | if not reference_comparison or not candidate_comparison: 53 | return 54 | 55 | reasons = [] 56 | 57 | reference_f1_score = reference_comparison.f1_score 58 | current_f1_score = candidate_comparison.f1_score 59 | if current_f1_score < reference_f1_score - self.config.max_f1_regression: 60 | reasons.append( 61 | f"current F1 score is lower than the latest release F1 score: candidate_score={current_f1_score:0.2f} reference_score={reference_f1_score:0.2f} image={self.input_description.image}" 62 | ) 63 | 64 | if candidate_comparison.indeterminate_percent > self.config.max_unlabeled_percent: 65 | reasons.append( 66 | f"current indeterminate matches % is greater than {self.config.max_unlabeled_percent}%: candidate={candidate_comparison.indeterminate_percent:0.2f}% image={self.input_description.image}" 67 | ) 68 | 69 | reference_fns = reference_comparison.false_negatives 70 | candidate_fns = candidate_comparison.false_negatives 71 | if candidate_fns > reference_fns + self.config.max_new_false_negatives: 72 | reasons.append( 73 | f"current false negatives is greater than the latest release false negatives: candidate={candidate_fns} reference={reference_fns} image={self.input_description.image}" 74 | ) 75 | 76 | self.reasons = reasons 77 | 78 | def passed(self) -> bool: 79 | return len(self.reasons) == 0 80 | 81 | @classmethod 82 | def failing(cls, reasons: list[str], input_description: GateInputDescription): 83 | """failing bypasses Gate's normal validation calculating and returns a 84 | gate that is failing for the reasons given.""" 85 | return cls( 86 | reference_comparison=None, 87 | candidate_comparison=None, 88 | config=GateConfig(), 89 | reasons=reasons, 90 | input_description=input_description, 91 | ) 92 | 93 | @classmethod 94 | def passing(cls, input_description: GateInputDescription): 95 | """passing bypasses a Gate's normal validation and returns a gate that is passing.""" 96 | return cls( 97 | reference_comparison=None, 98 | candidate_comparison=None, 99 | config=GateConfig(), 100 | reasons=[], # a gate with no reason to fail is considered passing 101 | input_description=input_description, 102 | ) 103 | -------------------------------------------------------------------------------- /tests/cli/.gitignore: -------------------------------------------------------------------------------- 1 | .yardstick/tools 2 | .yardstick/result 3 | *.tar.gz 4 | -------------------------------------------------------------------------------- /tests/cli/.yardstick.yaml: -------------------------------------------------------------------------------- 1 | default-max-year: 2020 2 | 3 | result-sets: 4 | test: 5 | description: "test" 6 | matrix: 7 | images: 8 | - docker.io/anchore/test_images:java-56d52bc@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da 9 | tools: 10 | - name: syft 11 | version: v0.54.0 12 | produces: SBOM 13 | refresh: False 14 | 15 | - name: grype 16 | version: v0.27.0 17 | 18 | - name: grype 19 | version: v0.72.0+import-db=db.tar.gz 20 | takes: SBOM 21 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/215d9c7a-0fb2-4313-bcf4-26bda1bec916.json: -------------------------------------------------------------------------------- 1 | {"ID": "215d9c7a-0fb2-4313-bcf4-26bda1bec916", "effective_cve": "CVE-2021-42383", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:45-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42383"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/23e358c1-3956-4774-9af2-09ac4cbb7931.json: -------------------------------------------------------------------------------- 1 | {"ID": "23e358c1-3956-4774-9af2-09ac4cbb7931", "effective_cve": "CVE-2021-42375", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:43-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42375"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/2e4c5ab8-7d94-4dc2-a0b1-78d9fd5ab78b.json: -------------------------------------------------------------------------------- 1 | {"ID": "2e4c5ab8-7d94-4dc2-a0b1-78d9fd5ab78b", "effective_cve": "CVE-2021-42379", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:44-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42379"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/34a3e6ce-59b7-46a2-b369-448805ce5919.json: -------------------------------------------------------------------------------- 1 | {"ID": "34a3e6ce-59b7-46a2-b369-448805ce5919", "effective_cve": "CVE-2021-42381", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:44-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42381"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/57d681b5-1080-496e-9bf1-b755b8851791.json: -------------------------------------------------------------------------------- 1 | {"ID": "57d681b5-1080-496e-9bf1-b755b8851791", "effective_cve": "CVE-2020-1945", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "FP", "package": {"name": "ant-launcher", "version": "1.8.0"}, "timestamp": "2022-09-27T16:55:54-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2020-1945"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/5866cffe-c48c-4ce7-8ddb-6c4ac952e69b.json: -------------------------------------------------------------------------------- 1 | {"ID": "5866cffe-c48c-4ce7-8ddb-6c4ac952e69b", "effective_cve": "CVE-2020-1945", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "ant", "version": "1.8.0"}, "timestamp": "2022-09-27T16:55:57-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2020-1945"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/59c7cc8e-fdf8-49e0-be70-a66146db6750.json: -------------------------------------------------------------------------------- 1 | {"ID": "59c7cc8e-fdf8-49e0-be70-a66146db6750", "effective_cve": "CVE-2021-42380", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:44-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42380"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/6d4faa73-8ac7-4b2a-9252-cd8b31dfc8b7.json: -------------------------------------------------------------------------------- 1 | {"ID": "6d4faa73-8ac7-4b2a-9252-cd8b31dfc8b7", "effective_cve": "CVE-2021-42374", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:43-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42374"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/797d4932-d64e-4d52-a59a-1128fdc2dde0.json: -------------------------------------------------------------------------------- 1 | {"ID": "797d4932-d64e-4d52-a59a-1128fdc2dde0", "effective_cve": "CVE-2010-3700", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "acegi-security", "version": "1.0.5"}, "timestamp": "2022-09-27T16:56:05-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2010-3700"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/9dd57c41-d9e9-40e8-9359-af640b3dc9b0.json: -------------------------------------------------------------------------------- 1 | {"ID": "9dd57c41-d9e9-40e8-9359-af640b3dc9b0", "effective_cve": "CVE-2021-42382", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:45-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42382"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/a2d926f2-9de7-4480-a2c0-9516d81d1d29.json: -------------------------------------------------------------------------------- 1 | {"ID": "a2d926f2-9de7-4480-a2c0-9516d81d1d29", "effective_cve": "CVE-2021-42385", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:45-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42385"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/a6f0668e-1ca4-4857-b50a-32b56a4d38a1.json: -------------------------------------------------------------------------------- 1 | {"ID": "a6f0668e-1ca4-4857-b50a-32b56a4d38a1", "effective_cve": "CVE-2010-3700", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "acegi-security", "version": "1.0.5"}, "timestamp": "2022-09-27T16:56:02-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "GHSA-3295-h9qx-r82x"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/b1901af4-524b-4248-afc8-3d0df1303266.json: -------------------------------------------------------------------------------- 1 | {"ID": "b1901af4-524b-4248-afc8-3d0df1303266", "effective_cve": "CVE-2022-28391", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:47-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2022-28391"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/d95890a6-5934-43e3-8826-ce086ec5f1e0.json: -------------------------------------------------------------------------------- 1 | {"ID": "d95890a6-5934-43e3-8826-ce086ec5f1e0", "effective_cve": "CVE-2021-21681", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "nomad", "version": "0.7.4"}, "timestamp": "2022-09-27T16:56:24-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-21681"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/db463105-6378-4173-81c8-f0942f78e179.json: -------------------------------------------------------------------------------- 1 | {"ID": "db463105-6378-4173-81c8-f0942f78e179", "effective_cve": "CVE-2021-42386", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:46-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42386"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/eb8254f5-baf3-43e0-84d6-ff958990de63.json: -------------------------------------------------------------------------------- 1 | {"ID": "eb8254f5-baf3-43e0-84d6-ff958990de63", "effective_cve": "CVE-2019-1003092", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "nomad", "version": "0.7.4"}, "timestamp": "2022-09-27T16:56:23-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2019-1003092"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/fc872e76-fa2c-43a3-a24b-84cc612366d6.json: -------------------------------------------------------------------------------- 1 | {"ID": "fc872e76-fa2c-43a3-a24b-84cc612366d6", "effective_cve": "CVE-2021-42384", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:45-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42384"} 2 | -------------------------------------------------------------------------------- /tests/cli/.yardstick/labels/docker.io+anchore+test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da/fcc3a71e-8f7a-4503-910e-de2b7ed861c5.json: -------------------------------------------------------------------------------- 1 | {"ID": "fcc3a71e-8f7a-4503-910e-de2b7ed861c5", "effective_cve": "CVE-2021-42378", "image": {"exact": "docker.io/anchore/test_images@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da"}, "label": "TP", "package": {"name": "busybox", "version": "1.33.1-r3"}, "timestamp": "2022-09-27T16:55:44-04:00", "tool": "grype@v0.50.2", "user": "wagoodman", "vulnerability_id": "CVE-2021-42378"} 2 | -------------------------------------------------------------------------------- /tests/cli/Makefile: -------------------------------------------------------------------------------- 1 | ACTIVATE_VENV = . venv/bin/activate && 2 | 3 | TEST_DB_URL = https://toolbox-data.anchore.io/grype/databases/vulnerability-db_v5_2023-10-25T01:27:28Z_fd5a911f9285633c57e3.tar.gz 4 | TEST_DB = db.tar.gz 5 | 6 | # formatting variables 7 | BOLD := $(shell tput -T linux bold) 8 | PURPLE := $(shell tput -T linux setaf 5) 9 | GREEN := $(shell tput -T linux setaf 2) 10 | CYAN := $(shell tput -T linux setaf 6) 11 | RED := $(shell tput -T linux setaf 1) 12 | RESET := $(shell tput -T linux sgr0) 13 | TITLE := $(BOLD)$(PURPLE) 14 | SUCCESS := $(BOLD)$(GREEN) 15 | 16 | test: venv $(TEST_DB) ## Run CLI tests 17 | $(ACTIVATE_VENV) ./run.sh 18 | 19 | $(TEST_DB): 20 | curl -o $(TEST_DB) -SsL $(TEST_DB_URL) 21 | 22 | venv: venv/touchfile ## Create a python virtual environment 23 | 24 | venv/touchfile: ../../pyproject.toml 25 | test -d venv || python3 -m venv venv 26 | $(ACTIVATE_VENV) pip install -e ../../ 27 | touch venv/touchfile 28 | 29 | .PHONY: clean 30 | clean: ## Clear all existing yardstick results and delete python environment 31 | rm -rf venv 32 | 33 | help: 34 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}' 35 | -------------------------------------------------------------------------------- /tests/cli/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ERROR="\033[1;31m" 4 | SUCCESS="\033[1;32m" 5 | TITLE="\033[1;35m" 6 | RESET="\033[0m" 7 | 8 | i=0 9 | 10 | temp_files=() 11 | 12 | function run() { 13 | tmp_file=$(mktemp /tmp/yardstick-test.XXXXXX) 14 | temp_files+=( $tmp_file ) 15 | echo -e "${TITLE}$i| Running $@${RESET}" 16 | $@ | tee $tmp_file 17 | rc=${PIPESTATUS[0]} 18 | if [ $rc -eq 0 ]; then 19 | echo -e "${SUCCESS}Success${RESET}" 20 | else 21 | echo -e "${ERROR}Exited with $rc${RESET}" 22 | exit 1 23 | fi 24 | ((i++)) 25 | } 26 | 27 | function last_output_file() { 28 | echo ${temp_files[${#temp_files[@]} - 1]} 29 | } 30 | 31 | function last_output() { 32 | cat $(last_output_file) 33 | } 34 | 35 | function assert_last_output_length() { 36 | expected=$1 37 | len=$(last_output | wc -l | tr -d ' ') 38 | if [[ "$len" == "$expected" ]]; then 39 | return 40 | fi 41 | echo -e "${ERROR}Unexpected length $len != $expected${RESET}" 42 | exit 1 43 | } 44 | 45 | function assert_last_output_contains() { 46 | target=$1 47 | is_in_file=$(cat $(last_output_file) | grep -c "$target") 48 | if [ $is_in_file -eq 0 ]; then 49 | echo -e "${ERROR}Target not found in contents '$target'${RESET}" 50 | echo -e "${ERROR}...contents:\n$(last_output)${RESET}" 51 | exit 1 52 | fi 53 | } 54 | 55 | run yardstick result clear 56 | 57 | run yardstick result capture -r test 58 | 59 | run yardstick result list -r test 60 | 61 | assert_last_output_length 3 62 | assert_last_output_contains "grype@v0.27.0" 63 | assert_last_output_contains "grype@v0.72.0" 64 | assert_last_output_contains "syft@v0.54.0" 65 | assert_last_output_contains "docker.io/anchore/test_images:java-56d52bc@sha256:10008791acbc5866de04108746a02a0c4029ce3a4400a9b3dad45d7f2245f9da" 66 | 67 | run yardstick result compare $(yardstick result list -r test --ids -t grype) 68 | 69 | assert_last_output_contains "grype-v0.72.0-only" 70 | assert_last_output_contains "commons-collections" 71 | assert_last_output_contains "dom4j" 72 | assert_last_output_contains "log4j" 73 | assert_last_output_contains "spring-core" 74 | 75 | run yardstick label apply $(yardstick result list -r test --ids -t grype@v0.72.0) 76 | 77 | assert_last_output_contains "label: TruePositive" 78 | 79 | ID_TO_REMOVE=$(yardstick label add -i foo -c CVE-1234-ASDF -p test-package -v 1.2.3 -n "testing" --label TP) 80 | run yardstick label remove ${ID_TO_REMOVE} 81 | assert_last_output_contains ${ID_TO_REMOVE} 82 | 83 | echo "cleaning up temp files created:" 84 | for i in ${!temp_files[@]}; do 85 | echo " " ${temp_files[$i]} 86 | rm ${temp_files[$i]} 87 | done 88 | 89 | 90 | echo -e "\n${SUCCESS}PASS${RESET}" 91 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anchore/yardstick/f19314d2d44ec262a1d52792c47e15759a90559a/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anchore/yardstick/f19314d2d44ec262a1d52792c47e15759a90559a/tests/unit/cli/__init__.py -------------------------------------------------------------------------------- /tests/unit/cli/explore_labels/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anchore/yardstick/f19314d2d44ec262a1d52792c47e15759a90559a/tests/unit/cli/explore_labels/__init__.py -------------------------------------------------------------------------------- /tests/unit/cli/explore_labels/test_history.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from yardstick.cli.explore.image_labels.history import Command, History 3 | 4 | 5 | @pytest.fixture() 6 | def counter_command_state(): 7 | items = [] 8 | 9 | def redo(count): 10 | def do(): 11 | print("adding", count, "to", items) 12 | items.append(count) 13 | 14 | return do 15 | 16 | def undo(count): 17 | def do(): 18 | print("removing", count, "from", items) 19 | items.remove(count) 20 | 21 | return do 22 | 23 | return undo, redo, items 24 | 25 | 26 | @pytest.fixture() 27 | def history_of_5(counter_command_state): 28 | undo, redo, state = counter_command_state 29 | history = History() 30 | 31 | for i in range(5): 32 | history.record(Command(undo=undo(i), redo=redo(i))) 33 | return history, undo, redo, state 34 | 35 | 36 | def test_history_record(history_of_5): 37 | history, undo, redo, state = history_of_5 38 | assert state == [0, 1, 2, 3, 4] 39 | 40 | 41 | def test_history_undo(history_of_5): 42 | history, undo, redo, state = history_of_5 43 | # first undo... 44 | history.undo() 45 | assert state == [0, 1, 2, 3] 46 | 47 | # second undo... 48 | history.undo() 49 | assert state == [0, 1, 2] 50 | 51 | # undo until empty... 52 | history.undo() 53 | history.undo() 54 | history.undo() 55 | assert state == [] 56 | 57 | # check we don't go path the boundary 58 | history.undo() 59 | assert state == [] 60 | 61 | 62 | def test_history_redo(history_of_5): 63 | history, undo, redo, state = history_of_5 64 | # prep undo... 65 | history.undo() 66 | history.undo() 67 | history.undo() 68 | assert state == [0, 1] 69 | 70 | # first redo... 71 | history.redo() 72 | assert state == [0, 1, 2] 73 | 74 | # redo all... 75 | history.redo() 76 | history.redo() 77 | assert state == [0, 1, 2, 3, 4] 78 | 79 | # check we don't go path the boundary 80 | history.redo() 81 | assert state == [0, 1, 2, 3, 4] 82 | 83 | 84 | def test_history_undo_redo_rewrite(history_of_5): 85 | history, undo, redo, state = history_of_5 86 | # prep undo... 87 | history.undo() 88 | history.undo() 89 | history.undo() 90 | assert state == [0, 1] 91 | 92 | # first redo... 93 | history.redo() 94 | assert state == [0, 1, 2] 95 | 96 | # record new history 97 | history.record(Command(undo=undo(42), redo=redo(42))) 98 | 99 | assert state == [0, 1, 2, 42] 100 | 101 | # redo does nothing... 102 | history.redo() 103 | history.redo() 104 | assert state == [0, 1, 2, 42] 105 | 106 | # undo still works 107 | history.undo() 108 | history.undo() 109 | assert state == [0, 1] 110 | 111 | # a few more cases... 112 | history.record(Command(undo=undo(72), redo=redo(72))) 113 | 114 | assert state == [0, 1, 72] 115 | 116 | history.undo() 117 | history.undo() 118 | history.undo() 119 | history.undo() 120 | assert state == [] 121 | 122 | history.redo() 123 | history.redo() 124 | history.redo() 125 | history.redo() 126 | history.redo() 127 | history.redo() 128 | history.redo() 129 | 130 | assert state == [0, 1, 72] 131 | -------------------------------------------------------------------------------- /tests/unit/cli/test_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from yardstick.cli import config 4 | 5 | 6 | def test_config(tmp_path): 7 | profile_file = tmp_path / ".yardstick.profiles.yaml" 8 | subject = f""" 9 | store_root: . 10 | profile_path: {profile_file} 11 | default-max-year: 2021 12 | 13 | x-ref: 14 | full-label-set-images: &full-label-set-images 15 | - docker.io/cloudbees/cloudbees-core-agent:2.289.2.2@sha256:d48f0546b4cf5ef4626136242ce302f94a42751156b7be42f4b1b75a66608880 16 | - docker.io/cloudbees/cloudbees-core-mm:2.277.3.1@sha256:4c564f473d38f23da1caa48c4ef53b958ef03d279232007ad3319b1f38584bdb 17 | - docker.io/cloudbees/cloudbees-core-oc:2.289.2.2@sha256:9cd85ee84e401dc27e3a8268aae67b594a651b2f4c7fc056ca14c7b0a0a6b82d 18 | - docker.io/anchore/test_images:grype-quality-node-d89207b@sha256:f56164678054e5eb59ab838367373a49df723b324617b1ba6de775749d7f91d4 19 | 20 | partial-label-set-images: &partial-label-set-images 21 | - docker.io/vulhub/cve-2017-1000353:latest@sha256:da2a59314b9ccfb428a313a7f163adcef77a74a393b8ebadeca8223b8cea9797 22 | 23 | result-sets: 24 | 25 | sboms: 26 | 27 | description: "SBOMs for images that should be fully labeled" 28 | matrix: 29 | images: 30 | - *full-label-set-images 31 | - *partial-label-set-images 32 | 33 | tools: 34 | 35 | - name: syft 36 | # note: we want to use a fixed version of syft for capturing all results (NOT "latest") 37 | version: v0.68.1 38 | # once we have results captured, don't re-capture them 39 | refresh: false 40 | """ 41 | file = tmp_path / "config.yaml" 42 | file.write_text(subject) 43 | 44 | profile_text = """ 45 | test_profile: 46 | something: 47 | name: jim 48 | config_path: .abc/xyx.conf 49 | refresh: false 50 | """ 51 | profile_file.write_text(profile_text) 52 | 53 | cfg = config.load(str(file)) 54 | 55 | assert cfg.result_sets["sboms"].matrix.images == [ 56 | "docker.io/cloudbees/cloudbees-core-agent:2.289.2.2@sha256:d48f0546b4cf5ef4626136242ce302f94a42751156b7be42f4b1b75a66608880", 57 | "docker.io/cloudbees/cloudbees-core-mm:2.277.3.1@sha256:4c564f473d38f23da1caa48c4ef53b958ef03d279232007ad3319b1f38584bdb", 58 | "docker.io/cloudbees/cloudbees-core-oc:2.289.2.2@sha256:9cd85ee84e401dc27e3a8268aae67b594a651b2f4c7fc056ca14c7b0a0a6b82d", 59 | "docker.io/anchore/test_images:grype-quality-node-d89207b@sha256:f56164678054e5eb59ab838367373a49df723b324617b1ba6de775749d7f91d4", 60 | "docker.io/vulhub/cve-2017-1000353:latest@sha256:da2a59314b9ccfb428a313a7f163adcef77a74a393b8ebadeca8223b8cea9797", 61 | ] 62 | 63 | assert cfg.profiles == config.Profiles( 64 | { 65 | "test_profile": { 66 | "something": { 67 | "name": "jim", 68 | "config_path": ".abc/xyx.conf", 69 | "refresh": False, 70 | }, 71 | }, 72 | }, 73 | ) 74 | 75 | 76 | @pytest.mark.parametrize( 77 | "name, image, expected_valid", 78 | [ 79 | # valid: everything present 80 | ( 81 | "valid", 82 | "registry.example.com/repo/image:latest@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 83 | True, 84 | ), 85 | ( 86 | "valid: vulhub", 87 | "docker.io/vulhub/cve-2017-1000353:latest@sha256:da2a59314b9ccfb428a313a7f163adcef77a74a393b8ebadeca8223b8cea9797", 88 | True, 89 | ), 90 | ( 91 | "valid: alpine", 92 | "docker.io/alpine:3.2@sha256:ddac200f3ebc9902fb8cfcd599f41feb2151f1118929da21bcef57dc276975f9", 93 | True, 94 | ), 95 | # valid: localhost with port as repo host 96 | ( 97 | "valid: localhost with port as repo host", 98 | "localhost:5555/repo/image:latest@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 99 | True, 100 | ), 101 | ( 102 | "valid: missing tag is allowed but discouraged", 103 | "registry.access.redhat.com/ubi8@sha256:68fecea0d255ee253acbf0c860eaebb7017ef5ef007c25bee9eeffd29ce85b29", 104 | True, 105 | ), 106 | ( 107 | "invalid: missing host", 108 | "repo/image:latest@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 109 | False, 110 | ), 111 | ("invalid: missing digest", "registry.example.com/repo/image:latest", False), 112 | ("invalid: missing everything", "repo/image", False), 113 | ("invalid: empty string", "", False), 114 | ( 115 | "invalid: missing repo", 116 | "registry.example.com/:latest@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 117 | False, 118 | ), 119 | ("invalid: missing repo and tag", "registry.example.com/", False), 120 | ("invalid: missing digest", "registry.example.com/repo/image:stable", False), 121 | ( 122 | "invalid: digest does not look like sha256", 123 | "registry.example.com/repo/image:latest@sha256:invaliddigest", 124 | False, 125 | ), 126 | ( 127 | "invalid: bad sha256 (too short)", 128 | "docker.io/alpine:3.2@sha256:ddac200f3ebc9902fb8cfcd599f41feb2151f1118929da21bcef57dc27697", 129 | False, 130 | ), 131 | ], 132 | ) 133 | def test_is_valid_oci_reference(name, image, expected_valid): 134 | result = config.ScanMatrix.is_valid_oci_reference(image) 135 | assert result == expected_valid, f"Test case {name}: Expected {expected_valid} but got {result} for image '{image}'" 136 | 137 | 138 | @pytest.mark.parametrize( 139 | "image, expected_output", 140 | [ 141 | ( 142 | "docker.io/anchore/test_images:some-tag@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 143 | ( 144 | "docker.io", 145 | "anchore", 146 | "test_images", 147 | "some-tag", 148 | "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 149 | ), 150 | ), 151 | # Localhost reference with path, repository, tag, and digest 152 | ( 153 | "localhost/anchore/test_images:some-tag@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 154 | ( 155 | "localhost", 156 | "anchore", 157 | "test_images", 158 | "some-tag", 159 | "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 160 | ), 161 | ), 162 | # Localhost with port, path, repository, tag, and digest 163 | ( 164 | "localhost:5000/anchore/test_images:some-tag@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 165 | ( 166 | "localhost:5000", 167 | "anchore", 168 | "test_images", 169 | "some-tag", 170 | "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 171 | ), 172 | ), 173 | # Missing digest 174 | ( 175 | "docker.io/anchore/test_images:some-tag", 176 | ("docker.io", "anchore", "test_images", "some-tag", ""), 177 | ), 178 | # Missing tag 179 | ( 180 | "docker.io/anchore/test_images@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 181 | ( 182 | "docker.io", 183 | "anchore", 184 | "test_images", 185 | "", 186 | "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", 187 | ), 188 | ), 189 | # Only repository 190 | ("test_images", ("", "", "test_images", "", "")), 191 | ], 192 | ) 193 | def test_parse_oci_reference(image, expected_output): 194 | result = config.ScanMatrix.parse_oci_reference(image) 195 | assert result == expected_output, f"Expected {expected_output} but got {result} for image '{image}'" 196 | -------------------------------------------------------------------------------- /tests/unit/store/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anchore/yardstick/f19314d2d44ec262a1d52792c47e15759a90559a/tests/unit/store/__init__.py -------------------------------------------------------------------------------- /tests/unit/store/test_naming.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from yardstick import store 3 | 4 | 5 | class TestNamingImage: 6 | @pytest.mark.parametrize( 7 | ("image", "expected"), 8 | [ 9 | ("ubuntu:20.04", "ubuntu:20.04"), 10 | ("anchore/anchore-engine:latest", "anchore+anchore-engine:latest"), 11 | ("something/nested/image:latest", "something+nested+image:latest"), 12 | ], 13 | ) 14 | def test_encode_decode(self, image, expected): 15 | assert expected == store.naming.image.encode(image) 16 | assert image == store.naming.image.decode(expected) 17 | -------------------------------------------------------------------------------- /tests/unit/test_comparison.py: -------------------------------------------------------------------------------- 1 | from yardstick import artifact, comparison 2 | 3 | # TODO: include test with lineage 4 | 5 | 6 | def test_comparison_against_labels(): 7 | config = artifact.ScanConfiguration( 8 | image_repo="myimage", 9 | image_digest="123456", 10 | tool_name="grype", 11 | tool_version="main", 12 | ) 13 | 14 | package_bash_5 = artifact.Package(name="bash", version="5.0-6ubuntu1.1") 15 | package_coreutils_8 = artifact.Package(name="coreutils", version="8.30-3ubuntu2") 16 | package_libc_2 = artifact.Package(name="libc-bin", version="2.31-0ubuntu9.2") 17 | package_libsystemd_245 = artifact.Package( 18 | name="libsystemd0", 19 | version="245.4-4ubuntu3.2", 20 | ) 21 | package_libsystemd_25 = artifact.Package(name="libsystemd0", version="25") 22 | 23 | expected_tp_matches = [ 24 | artifact.Match( 25 | vulnerability=artifact.Vulnerability(id="CVE-2019-18276"), 26 | package=package_bash_5, 27 | config=config, 28 | ), 29 | artifact.Match( 30 | vulnerability=artifact.Vulnerability(id="CVE-2016-2781"), 31 | package=package_coreutils_8, 32 | config=config, 33 | ), 34 | ] 35 | 36 | expected_fp_matches = [ 37 | artifact.Match( 38 | vulnerability=artifact.Vulnerability(id="CVE-2016-10228"), 39 | package=package_libc_2, 40 | config=config, 41 | ), 42 | artifact.Match( 43 | vulnerability=artifact.Vulnerability(id="CVE-2018-20839"), 44 | package=package_libsystemd_245, 45 | config=config, 46 | ), 47 | artifact.Match( 48 | vulnerability=artifact.Vulnerability(id="CVE-2018-29999"), 49 | package=package_libsystemd_25, 50 | config=config, 51 | ), 52 | ] 53 | 54 | matches = [ 55 | *expected_tp_matches, 56 | *expected_fp_matches, 57 | ] 58 | 59 | result = artifact.ScanResult( 60 | config=config, 61 | matches=matches, 62 | ) 63 | 64 | common_label_options = { 65 | "image": artifact.ImageSpecifier(exact=config.image), 66 | "source": "manual", 67 | } 68 | 69 | false_negative_label_entries = [ 70 | artifact.LabelEntry( 71 | label=artifact.Label.TruePositive, 72 | vulnerability_id="CVE-2016-NM111", 73 | package=package_libc_2, 74 | **common_label_options, 75 | ), 76 | # do not have matches and already are covered with effective CVE... but the package mismatches 77 | artifact.LabelEntry( 78 | label=artifact.Label.TruePositive, 79 | vulnerability_id="ELSA-2020-123567", 80 | effective_cve="CVE-2019-18276", 81 | package=package_libsystemd_25, 82 | **common_label_options, 83 | ), 84 | ] 85 | 86 | label_entries = [ 87 | # have matches 88 | artifact.LabelEntry( 89 | label=artifact.Label.TruePositive, 90 | vulnerability_id="CVE-2019-18276", 91 | package=package_bash_5, 92 | **common_label_options, 93 | ), 94 | artifact.LabelEntry( 95 | label=artifact.Label.TruePositive, 96 | vulnerability_id="CVE-2016-2781", 97 | package=package_coreutils_8, 98 | **common_label_options, 99 | ), 100 | artifact.LabelEntry( 101 | label=artifact.Label.FalsePositive, 102 | vulnerability_id="CVE-2016-10228", 103 | package=package_libc_2, 104 | **common_label_options, 105 | ), 106 | artifact.LabelEntry( 107 | label=artifact.Label.FalsePositive, 108 | vulnerability_id="CVE-2018-20839", 109 | package=package_libsystemd_245, 110 | **common_label_options, 111 | ), 112 | artifact.LabelEntry( 113 | label=artifact.Label.FalsePositive, 114 | vulnerability_id="CVE-2018-29999", 115 | package=package_libsystemd_25, 116 | **common_label_options, 117 | ), 118 | # do not have matches 119 | *false_negative_label_entries, 120 | artifact.LabelEntry( 121 | label=artifact.Label.FalsePositive, 122 | vulnerability_id="CVE-2016-NM222", 123 | package=package_libc_2, 124 | **common_label_options, 125 | ), 126 | artifact.LabelEntry( 127 | label=artifact.Label.FalsePositive, 128 | vulnerability_id="CVE-2016-NM333", 129 | package=package_libsystemd_25, 130 | **common_label_options, 131 | ), 132 | # do not have matches and already are covered with effective CVE 133 | artifact.LabelEntry( 134 | label=artifact.Label.TruePositive, 135 | vulnerability_id="ELSA-2020-123567", 136 | effective_cve="CVE-2019-18276", 137 | package=package_bash_5, 138 | **common_label_options, 139 | ), 140 | ] 141 | 142 | actual = comparison.AgainstLabels( 143 | result=result, 144 | label_entries=label_entries, 145 | lineage=[], 146 | ) 147 | 148 | assert actual.summary.true_positives == len(expected_tp_matches) 149 | assert actual.summary.false_positives == len(expected_fp_matches) 150 | assert actual.summary.false_negatives == len(false_negative_label_entries) 151 | assert actual.summary.f1_score == 0.4444444444444444 152 | 153 | assert set(actual.true_positive_matches) == set(expected_tp_matches) 154 | assert set(actual.false_positive_matches) == set(expected_fp_matches) 155 | assert set(actual.false_negative_label_entries) == set(false_negative_label_entries) 156 | 157 | 158 | def test_comparison_against_labels_indeterminate(): 159 | config = artifact.ScanConfiguration( 160 | image_repo="myimage", 161 | image_digest="123456", 162 | tool_name="grype", 163 | tool_version="main", 164 | ) 165 | 166 | package_bash_5 = artifact.Package(name="bash", version="5.0-6ubuntu1.1") 167 | 168 | m1 = artifact.Match( 169 | vulnerability=artifact.Vulnerability(id="CVE-2019-18276"), 170 | package=package_bash_5, 171 | config=config, 172 | ) 173 | m2 = artifact.Match( 174 | vulnerability=artifact.Vulnerability(id="CVE-2020-12000"), 175 | package=package_bash_5, 176 | config=config, 177 | ) 178 | m3 = artifact.Match( 179 | vulnerability=artifact.Vulnerability(id="CVE-2020-2222"), 180 | package=package_bash_5, 181 | config=config, 182 | ) 183 | 184 | matches = [m1, m2, m3] 185 | 186 | result = artifact.ScanResult( 187 | config=config, 188 | matches=matches, 189 | ) 190 | 191 | common_label_options = { 192 | "image": artifact.ImageSpecifier(exact=config.image), 193 | "source": "manual", 194 | } 195 | 196 | m2_indeterminate = [ 197 | artifact.LabelEntry( 198 | label=artifact.Label.TruePositive, 199 | vulnerability_id="CVE-2020-12000", 200 | package=package_bash_5, 201 | **common_label_options, 202 | ), 203 | artifact.LabelEntry( 204 | label=artifact.Label.FalsePositive, 205 | vulnerability_id="CVE-2020-12000", 206 | package=package_bash_5, 207 | **common_label_options, 208 | ), 209 | ] 210 | 211 | m3_indeterminate = [ 212 | artifact.LabelEntry( 213 | label=artifact.Label.Unclear, 214 | vulnerability_id="CVE-2020-2222", 215 | package=package_bash_5, 216 | **common_label_options, 217 | ), 218 | ] 219 | 220 | label_entries = [ 221 | # not indeterminate 222 | artifact.LabelEntry( 223 | label=artifact.Label.TruePositive, 224 | vulnerability_id="CVE-2019-18276", 225 | package=package_bash_5, 226 | **common_label_options, 227 | ), 228 | artifact.LabelEntry( 229 | label=artifact.Label.TruePositive, 230 | vulnerability_id="CVE-2019-18276", 231 | package=package_bash_5, 232 | **common_label_options, 233 | ), 234 | # indeterminate 235 | *m2_indeterminate, 236 | *m3_indeterminate, 237 | ] 238 | 239 | actual = comparison.AgainstLabels( 240 | result=result, 241 | label_entries=label_entries, 242 | lineage=[], 243 | ) 244 | 245 | assert actual.summary.indeterminate == 1 246 | assert set(actual.matches_with_indeterminate_labels) == {m2} 247 | assert actual.summary.f1_score == 1 248 | assert actual.summary.f1_score_lower_confidence == 0.6666666666666666 249 | assert actual.summary.f1_score_upper_confidence == 1 250 | -------------------------------------------------------------------------------- /tests/unit/tool/test_grype.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import tarfile 3 | import os 4 | import json 5 | from tempfile import TemporaryDirectory 6 | from unittest import mock 7 | 8 | import zstandard as zstd 9 | import xxhash 10 | 11 | from yardstick.tool.grype import ( 12 | Grype, 13 | GrypeProfile, 14 | handle_legacy_archive, 15 | handle_zstd_archive, 16 | ) 17 | 18 | 19 | def test_grype_profiles(): 20 | profile_arg = {"name": "test-profile", "config_path": "test-config-path"} 21 | profile = GrypeProfile(**profile_arg) 22 | with mock.patch("subprocess.check_output") as check_output: 23 | check_output.return_value = bytes("test-output", "utf-8") 24 | tool = Grype(path="test-path", profile=profile, db_identity="oss") 25 | tool.capture(image="test-image", tool_input=None) 26 | assert check_output.call_args.args[0] == [ 27 | "test-path/grype", 28 | "-o", 29 | "json", 30 | "test-image", 31 | "-c", 32 | "test-config-path", 33 | ] 34 | 35 | 36 | def test_grype_no_profile(): 37 | with mock.patch("subprocess.check_output") as check_output: 38 | check_output.return_value = bytes("test-output", "utf-8") 39 | tool = Grype(path="test-path", db_identity="oss") 40 | tool.capture(image="test-image", tool_input=None) 41 | assert check_output.call_args.args[0] == [ 42 | "test-path/grype", 43 | "-o", 44 | "json", 45 | "test-image", 46 | ] 47 | 48 | 49 | def test_install_from_path(): 50 | with ( 51 | mock.patch("subprocess.check_call") as check_call, 52 | mock.patch( 53 | "git.Repo", 54 | ) as repo, 55 | mock.patch("os.path.exists") as exists, 56 | mock.patch( 57 | "os.makedirs", 58 | ), 59 | mock.patch( 60 | "os.chmod", 61 | ), 62 | ): 63 | check_call.return_value = bytes("test-output", "utf-8") 64 | exists.return_value = True 65 | fake_repo = mock.Mock() 66 | fake_repo.git = mock.Mock() 67 | fake_repo.untracked_files = [] 68 | git_describe_val = "v0.65.1-1-g74a7a67-dirty" 69 | hash_of_git_diff = "a29864cf5600b481056b6fa30a21cdbabc15287d"[:8] 70 | fake_repo.git.describe.return_value = git_describe_val 71 | fake_repo.git.diff.return_value = "test-diff" # hash is 'a29864cf5600b481056b6fa30a21cdbabc15287d' 72 | repo.return_value = fake_repo 73 | version_str = "path:/where/grype/is/cloned" 74 | normalized_version_str = version_str.replace("/", "_").removeprefix("path:") 75 | expected_grype_path = f".yardstick/tools/grype/{normalized_version_str}/{git_describe_val}-{hash_of_git_diff}/local_install" 76 | tool = Grype.install( 77 | version=version_str, 78 | path=".yardstick/tools/grype/path:_where_grype_is_cloned", 79 | update_db=False, 80 | ) 81 | assert tool.path == expected_grype_path 82 | 83 | 84 | def create_legacy_archive_with_metadata(archive_path, metadata): 85 | with tarfile.open(archive_path, "w:gz") as tar: 86 | metadata_path = os.path.join(os.path.dirname(archive_path), "metadata.json") 87 | with open(metadata_path, "w") as f: 88 | json.dump(metadata, f) 89 | tar.add(metadata_path, arcname="metadata.json") 90 | os.remove(metadata_path) 91 | 92 | 93 | def create_zstd_archive_with_db(archive_path, db_content): 94 | with TemporaryDirectory() as temp_dir: 95 | db_path = os.path.join(temp_dir, "vulnerability.db") 96 | with open(db_path, "wb") as f: 97 | f.write(db_content) 98 | 99 | tar_path = os.path.join(temp_dir, "archive.tar") 100 | with tarfile.open(tar_path, "w") as tar: 101 | tar.add(db_path, arcname="vulnerability.db") 102 | 103 | with open(tar_path, "rb") as tar_file, open(archive_path, "wb") as zstd_file: 104 | dctx = zstd.ZstdCompressor() 105 | dctx.copy_stream(tar_file, zstd_file) 106 | 107 | 108 | @pytest.fixture 109 | def legacy_archive(tmp_path): 110 | archive_path = tmp_path / "db.tar.gz" 111 | metadata = {"checksum": "12345"} 112 | create_legacy_archive_with_metadata(archive_path, metadata) 113 | return archive_path, metadata["checksum"] 114 | 115 | 116 | @pytest.fixture 117 | def zstd_archive(tmp_path): 118 | archive_path = tmp_path / "db.tar.zst" 119 | db_content = b"dummy database content" 120 | hasher = xxhash.xxh64() 121 | hasher.update(db_content) 122 | expected_checksum = hasher.hexdigest() 123 | create_zstd_archive_with_db(archive_path, db_content) 124 | return archive_path, expected_checksum 125 | 126 | 127 | def test_handle_legacy_archive(legacy_archive): 128 | archive_path, expected_checksum = legacy_archive 129 | assert handle_legacy_archive(str(archive_path)) == expected_checksum 130 | 131 | 132 | def test_handle_zstd_archive(zstd_archive): 133 | archive_path, expected_checksum = zstd_archive 134 | assert handle_zstd_archive(str(archive_path)) == expected_checksum 135 | -------------------------------------------------------------------------------- /tests/unit/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anchore/yardstick/f19314d2d44ec262a1d52792c47e15759a90559a/tests/unit/utils/__init__.py -------------------------------------------------------------------------------- /tests/unit/utils/test_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from yardstick import utils 4 | 5 | 6 | @pytest.mark.parametrize( 7 | ("input", "expected_year"), 8 | [ 9 | ("CVE-2016-2781", 2016), 10 | ("CVE-1989-18276", None), 11 | ("CVE-20222-18276", None), 12 | ("ALAS-2019-1234", 2019), 13 | ("ALASRUBY2.6-2023-006", 2023), 14 | ("ALASSELINUX-NG-2023-001", 2023), 15 | ("ALASKERNEL-5.4-2023-043", 2023), 16 | ("ELSA-2023-6162", 2023), 17 | ], 18 | ) 19 | def test_parse_year_from_id(input, expected_year): 20 | assert utils.parse_year_from_id(input) == expected_year 21 | -------------------------------------------------------------------------------- /tests/unit/validate/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anchore/yardstick/f19314d2d44ec262a1d52792c47e15759a90559a/tests/unit/validate/__init__.py -------------------------------------------------------------------------------- /tests/unit/validate/test_delta.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from yardstick.artifact import Label, Package 4 | from yardstick.validate.delta import Delta, DeltaType, compute_deltas 5 | 6 | import pytest 7 | from unittest.mock import MagicMock 8 | from yardstick.comparison import AgainstLabels, ByPreservedMatch 9 | 10 | 11 | @pytest.mark.parametrize( 12 | "tool, package_name, package_version, vulnerability_id, added, label, expected_outcome, expected_is_improved, expected_commentary", 13 | [ 14 | ( 15 | "scanner1", 16 | "libc", 17 | "2.29", 18 | "CVE-2023-1234", 19 | True, 20 | Label.TruePositive.name, 21 | DeltaType.FixedFalseNegative, 22 | True, 23 | "(this is a new TP 🙌)", 24 | ), 25 | ( 26 | "scanner1", 27 | "nginx", 28 | "1.17", 29 | "CVE-2023-0002", 30 | False, 31 | Label.FalsePositive.name, 32 | DeltaType.FixedFalsePositive, 33 | True, 34 | "(got rid of a former FP 🙌)", 35 | ), 36 | ( 37 | "scanner2", 38 | "bash", 39 | "5.0", 40 | "CVE-2023-5678", 41 | False, 42 | Label.TruePositive.name, 43 | DeltaType.NewFalseNegative, 44 | False, 45 | "(this is a new FN 😱)", 46 | ), 47 | ( 48 | "scanner3", 49 | "zlib", 50 | "1.2.11", 51 | "CVE-2023-8888", 52 | True, 53 | Label.FalsePositive.name, 54 | DeltaType.NewFalsePositive, 55 | False, 56 | "(this is a new FP 😱)", 57 | ), 58 | ( 59 | "scanner4", 60 | "openssl", 61 | "1.1.1", 62 | "CVE-2023-0001", 63 | True, 64 | None, 65 | DeltaType.Unknown, 66 | None, 67 | "", 68 | ), 69 | ], 70 | ) 71 | def test_delta_properties( 72 | tool, 73 | package_name, 74 | package_version, 75 | vulnerability_id, 76 | added, 77 | label, 78 | expected_outcome, 79 | expected_is_improved, 80 | expected_commentary, 81 | ): 82 | """Test Delta properties is_improved, outcome, and commentary based on logical combinations.""" 83 | 84 | delta = Delta( 85 | tool=tool, 86 | package_name=package_name, 87 | package_version=package_version, 88 | vulnerability_id=vulnerability_id, 89 | added=added, 90 | label=label, 91 | ) 92 | 93 | assert delta.outcome == expected_outcome 94 | assert delta.is_improved == expected_is_improved 95 | assert delta.commentary == expected_commentary 96 | 97 | 98 | @pytest.fixture 99 | def reference_result(): 100 | """Fixture for creating a mock reference result.""" 101 | return MagicMock(name="reference_results", ID="reference", config=MagicMock(tool="reference")) 102 | 103 | 104 | @pytest.fixture 105 | def candidate_result(): 106 | """Fixture for creating a mock candidate result.""" 107 | return MagicMock(name="candidate_results", ID="candidate", config=MagicMock(tool="candidate")) 108 | 109 | 110 | @pytest.fixture 111 | def comparisons_by_result_id(): 112 | """Fixture for setting up comparisons with expected label data (source of truth).""" 113 | comparison = { 114 | # skip post init calculations on against labels, since 115 | # we're setting the comparison results directly below 116 | "reference": typing.cast(AgainstLabels, object.__new__(AgainstLabels)), 117 | "candidate": typing.cast(AgainstLabels, object.__new__(AgainstLabels)), 118 | } 119 | comparison["reference"].labels_by_match = { 120 | "match1": [Label.TruePositive], 121 | "match2": [Label.TruePositive], 122 | "match3": [Label.FalsePositive], 123 | "match4": [Label.FalsePositive], 124 | } 125 | comparison["candidate"].labels_by_match = { 126 | "match1": [Label.TruePositive], 127 | "match2": [Label.TruePositive], 128 | "match3": [Label.FalsePositive], 129 | "match4": [Label.FalsePositive], 130 | } 131 | return comparison 132 | 133 | 134 | @pytest.fixture 135 | def relative_comparison(reference_result, candidate_result): 136 | """Fixture for creating a mock relative comparison of reference and candidate.""" 137 | match1 = MagicMock( 138 | name="match1", 139 | ID="match1", 140 | package=Package(name="libc", version="2.29"), 141 | vulnerability=MagicMock(id="CVE-2023-1234"), 142 | ) 143 | match2 = MagicMock( 144 | name="match2", 145 | ID="match2", 146 | package=Package(name="nginx", version="1.17"), 147 | vulnerability=MagicMock(id="CVE-2023-0002"), 148 | ) 149 | match3 = MagicMock( 150 | name="match3", 151 | ID="match3", 152 | package=Package(name="openssl", version="1.1.1"), 153 | vulnerability=MagicMock(id="CVE-2023-5678"), 154 | ) 155 | match4 = MagicMock( 156 | name="match4", 157 | ID="match4", 158 | package=Package(name="zlib", version="1.2.11"), 159 | vulnerability=MagicMock(id="CVE-2023-8888"), 160 | ) 161 | 162 | result = ByPreservedMatch( 163 | results=[reference_result, candidate_result], 164 | ) 165 | result.unique = { 166 | "reference": [match2, match3], 167 | "candidate": [match1, match4], 168 | } 169 | return result 170 | 171 | 172 | def test_compute_deltas(comparisons_by_result_id, relative_comparison): 173 | """Test compute_deltas with realistic comparisons between reference and candidate results.""" 174 | deltas = compute_deltas( 175 | comparisons_by_result_id=comparisons_by_result_id, 176 | reference_tool="reference", 177 | relative_comparison=relative_comparison, 178 | ) 179 | 180 | expected_deltas = [ 181 | Delta( 182 | tool="reference", 183 | package_name="nginx", 184 | package_version="1.17", 185 | vulnerability_id="CVE-2023-0002", 186 | added=False, 187 | label="TruePositive", 188 | ), 189 | Delta( 190 | tool="reference", 191 | package_name="openssl", 192 | package_version="1.1.1", 193 | vulnerability_id="CVE-2023-5678", 194 | added=False, 195 | label="FalsePositive", 196 | ), 197 | Delta( 198 | tool="candidate", 199 | package_name="libc", 200 | package_version="2.29", 201 | vulnerability_id="CVE-2023-1234", 202 | added=True, 203 | label="TruePositive", 204 | ), 205 | Delta( 206 | tool="candidate", 207 | package_name="zlib", 208 | package_version="1.2.11", 209 | vulnerability_id="CVE-2023-8888", 210 | added=True, 211 | label="FalsePositive", 212 | ), 213 | ] 214 | 215 | assert len(deltas) == len(expected_deltas) 216 | for idx, actual in enumerate(deltas): 217 | assert actual == expected_deltas[idx], f"unequal at {idx}" 218 | -------------------------------------------------------------------------------- /tests/unit/validate/test_gate.py: -------------------------------------------------------------------------------- 1 | from yardstick.validate import Gate, GateConfig, GateInputDescription, Delta 2 | from yardstick import artifact, comparison 3 | 4 | 5 | import pytest 6 | from unittest.mock import MagicMock 7 | 8 | 9 | @pytest.fixture 10 | def mock_label_comparison(): 11 | """Fixture to create a mock LabelComparisonSummary with defaults.""" 12 | summary = MagicMock() 13 | summary.f1_score = 0.9 14 | summary.false_negatives = 5 15 | summary.indeterminate_percent = 2.0 16 | return summary 17 | 18 | 19 | @pytest.mark.parametrize( 20 | "config, reference_summary, candidate_summary, expected_reasons", 21 | [ 22 | # Case 1: Candidate has a lower F1 score beyond the allowed threshold -> gate fails 23 | ( 24 | GateConfig( 25 | max_f1_regression=0.1, 26 | max_new_false_negatives=5, 27 | max_unlabeled_percent=10, 28 | ), 29 | MagicMock(f1_score=0.9, false_negatives=5, indeterminate_percent=2.0), 30 | MagicMock(f1_score=0.7, false_negatives=5, indeterminate_percent=2.0), 31 | ["current F1 score is lower than the latest release F1 score"], 32 | ), 33 | # Case 2: Candidate has too many false negatives -> gate fails 34 | ( 35 | GateConfig( 36 | max_f1_regression=0.1, 37 | max_new_false_negatives=1, 38 | max_unlabeled_percent=10, 39 | ), 40 | MagicMock(f1_score=0.9, false_negatives=5, indeterminate_percent=2.0), 41 | MagicMock(f1_score=0.85, false_negatives=7, indeterminate_percent=2.0), 42 | ["current false negatives is greater than the latest release false negatives"], 43 | ), 44 | # Case 3: Candidate has too high indeterminate percent -> gate fails 45 | ( 46 | GateConfig( 47 | max_f1_regression=0.1, 48 | max_new_false_negatives=5, 49 | max_unlabeled_percent=5, 50 | ), 51 | MagicMock(f1_score=0.9, false_negatives=5, indeterminate_percent=2.0), 52 | MagicMock(f1_score=0.85, false_negatives=5, indeterminate_percent=6.0), 53 | ["current indeterminate matches % is greater than"], 54 | ), 55 | # Case 4: Candidate passes all thresholds -> gate passes (no reasons) 56 | ( 57 | GateConfig( 58 | max_f1_regression=0.1, 59 | max_new_false_negatives=5, 60 | max_unlabeled_percent=10, 61 | ), 62 | MagicMock(f1_score=0.9, false_negatives=5, indeterminate_percent=2.0), 63 | MagicMock(f1_score=0.85, false_negatives=5, indeterminate_percent=3.0), 64 | [], 65 | ), 66 | ], 67 | ) 68 | def test_gate(config, reference_summary, candidate_summary, expected_reasons): 69 | """Parameterized test for the Gate class that checks different pass/fail conditions.""" 70 | 71 | # Create the Gate instance with the given parameters 72 | gate = Gate( 73 | reference_comparison=reference_summary, 74 | candidate_comparison=candidate_summary, 75 | config=config, 76 | input_description=MagicMock(image="test_image"), 77 | ) 78 | 79 | # Check that the reasons list matches the expected outcome 80 | assert len(gate.reasons) == len(expected_reasons) 81 | for reason, expected_reason in zip(gate.reasons, expected_reasons): 82 | assert expected_reason in reason 83 | 84 | 85 | def test_gate_failing(): 86 | input_description = GateInputDescription(image="some-image", configs=[]) 87 | gate = Gate.failing(["sample failure reason"], input_description) 88 | assert not gate.passed() 89 | assert gate.reasons == ["sample failure reason"] 90 | 91 | 92 | def test_gate_passing(): 93 | input_description = GateInputDescription(image="some-image", configs=[]) 94 | gate = Gate.passing(input_description) 95 | assert gate.passed() 96 | -------------------------------------------------------------------------------- /tests/unit/validate/test_validate.py: -------------------------------------------------------------------------------- 1 | # Sample images 2 | from unittest.mock import patch, MagicMock 3 | 4 | import pytest 5 | 6 | from yardstick import comparison 7 | from yardstick.artifact import ( 8 | ScanResult, 9 | ScanConfiguration, 10 | Package, 11 | Vulnerability, 12 | LabelEntry, 13 | Label, 14 | Match, 15 | ) 16 | from yardstick.validate import validate_image, GateConfig, Delta 17 | 18 | 19 | @pytest.fixture() 20 | def compare_results_no_matches(): 21 | return MagicMock(results=[MagicMock(matches=[]), MagicMock(matches=[])]) 22 | 23 | 24 | @pytest.fixture() 25 | def compare_results_identical_matches(): 26 | return MagicMock( 27 | results=[ 28 | MagicMock( 29 | matches=[MagicMock()], 30 | unique={}, 31 | ), 32 | MagicMock( 33 | matches=[MagicMock()], 34 | unique={}, 35 | ), 36 | ] 37 | ) 38 | 39 | 40 | @patch("yardstick.compare_results") 41 | def test_validate_fail_on_empty_matches(mock_compare_results, compare_results_no_matches): 42 | mock_compare_results.return_value = compare_results_no_matches 43 | gate = validate_image( 44 | "some image", 45 | GateConfig(fail_on_empty_match_set=True), 46 | descriptions=["some-str", "another-str"], 47 | always_run_label_comparison=False, 48 | verbosity=0, 49 | ) 50 | assert not gate.passed() 51 | assert "gate configured to fail on empty matches, and no matches found" in gate.reasons 52 | mock_compare_results.assert_called_once_with( 53 | descriptions=["some-str", "another-str"], 54 | year_max_limit=None, 55 | matches_filter=None, 56 | ) 57 | 58 | 59 | @patch("yardstick.compare_results") 60 | def test_validate_dont_fail_on_empty_matches(mock_compare_results, compare_results_no_matches): 61 | mock_compare_results.return_value = compare_results_no_matches 62 | gate = validate_image( 63 | "some image", 64 | GateConfig(fail_on_empty_match_set=False), 65 | descriptions=["some-str", "another-str"], 66 | always_run_label_comparison=False, 67 | verbosity=0, 68 | ) 69 | assert gate.passed() 70 | mock_compare_results.assert_called_once_with( 71 | descriptions=["some-str", "another-str"], 72 | year_max_limit=None, 73 | matches_filter=None, 74 | ) 75 | 76 | 77 | @patch("yardstick.compare_results") 78 | def test_validate_pass_early_identical_match_sets(mock_compare_results, compare_results_identical_matches): 79 | mock_compare_results.return_value = compare_results_identical_matches 80 | gate = validate_image( 81 | "some image", 82 | GateConfig(fail_on_empty_match_set=False), 83 | descriptions=["some-str", "another-str"], 84 | always_run_label_comparison=False, 85 | verbosity=0, 86 | ) 87 | assert gate.passed() 88 | mock_compare_results.assert_called_once_with( 89 | descriptions=["some-str", "another-str"], 90 | year_max_limit=None, 91 | matches_filter=None, 92 | ) 93 | 94 | 95 | @pytest.fixture() 96 | def reference_config(): 97 | return ScanConfiguration( 98 | image_repo="docker.io/anchore/test_images", 99 | image_digest="f" * 64, 100 | tool_name="grype", 101 | tool_version="123", 102 | tool_label="reference", 103 | ID="reference-config-uuid", 104 | ) 105 | 106 | 107 | @pytest.fixture() 108 | def candidate_config(): 109 | return ScanConfiguration( 110 | image_repo="docker.io/anchore/test_images", 111 | image_digest="f" * 64, 112 | tool_name="grype", 113 | tool_version="1234", 114 | tool_label="candidate", 115 | ID="candidate-config-uuid", 116 | ) 117 | 118 | 119 | @pytest.fixture() 120 | def matches(packages, vulns): 121 | libc, nginx, openssl, zlib = packages 122 | vuln1, vuln2, vuln3, vuln4 = vulns 123 | match1 = Match( 124 | package=libc, 125 | vulnerability=vuln1, 126 | ) 127 | match2 = Match( 128 | package=nginx, 129 | vulnerability=vuln2, 130 | ) 131 | match3 = Match( 132 | package=openssl, 133 | vulnerability=vuln3, 134 | ) 135 | match4 = Match( 136 | package=zlib, 137 | vulnerability=vuln4, 138 | ) 139 | return [match1, match2, match3, match4] 140 | 141 | 142 | @pytest.fixture() 143 | def reference_results(reference_config, packages, matches): 144 | match1, match2, match3, match4 = matches 145 | return ScanResult( 146 | config=reference_config, 147 | matches=[match1, match2, match3], 148 | packages=packages, 149 | ) 150 | 151 | 152 | @pytest.fixture() 153 | def candidate_results(candidate_config, packages, matches): 154 | match1, match2, match3, match4 = matches 155 | return ScanResult( 156 | config=candidate_config, 157 | matches=[match1, match2, match3, match4], 158 | packages=packages, 159 | ) 160 | 161 | 162 | @pytest.fixture() 163 | def non_identical_results(reference_results, candidate_results): 164 | return comparison.ByPreservedMatch(results=[reference_results, reference_results]) 165 | 166 | 167 | @pytest.fixture() 168 | def vulns(): 169 | vuln1 = Vulnerability(id="CVE-2021-1234") 170 | vuln2 = Vulnerability(id="CVE-2021-0002") 171 | vuln3 = Vulnerability(id="CVE-2021-5678") 172 | vuln4 = Vulnerability(id="CVE-2021-8888") 173 | return vuln1, vuln2, vuln3, vuln4 174 | 175 | 176 | @pytest.fixture() 177 | def packages(): 178 | libc = Package(name="libc", version="2.29") 179 | nginx = Package(name="nginx", version="1.17") 180 | openssl = Package(name="openssl", version="1.1.1") 181 | zlib = Package(name="zlib", version="1.2.11") 182 | return [libc, nginx, openssl, zlib] 183 | 184 | 185 | @pytest.fixture() 186 | def deltas(): 187 | return [ 188 | MagicMock(spec=Delta), 189 | MagicMock(spec=Delta), 190 | ] 191 | 192 | 193 | @pytest.fixture() 194 | def label_entries(matches): 195 | match1, match2, match3, match4 = matches 196 | return [ 197 | LabelEntry( 198 | Label.TruePositive, 199 | vulnerability_id=match1.vulnerability.id, 200 | package=match1.package, 201 | ), 202 | LabelEntry( 203 | Label.FalsePositive, 204 | vulnerability_id=match2.vulnerability.id, 205 | package=match2.package, 206 | ), 207 | LabelEntry( 208 | Label.TruePositive, 209 | vulnerability_id=match3.vulnerability.id, 210 | package=match3.package, 211 | ), 212 | LabelEntry( 213 | Label.TruePositive, 214 | vulnerability_id=match4.vulnerability.id, 215 | package=match4.package, 216 | ), 217 | ] 218 | 219 | 220 | @pytest.fixture() 221 | def label_comparison_results(reference_results, candidate_results, label_entries): 222 | compare_configuration = { 223 | "year_max_limit": 2021, 224 | "year_from_cve_only": True, 225 | } 226 | return ( 227 | [reference_results, candidate_results], 228 | [], # label_entries is not used 229 | { 230 | reference_results.ID: comparison.AgainstLabels( 231 | result=reference_results, 232 | label_entries=label_entries, 233 | lineage=[], 234 | compare_configuration=compare_configuration, 235 | ), 236 | candidate_results.ID: comparison.AgainstLabels( 237 | result=candidate_results, 238 | label_entries=label_entries, 239 | lineage=[], 240 | compare_configuration=compare_configuration, 241 | ), 242 | }, 243 | MagicMock(name="stats_by_image_tool_pair"), 244 | ) 245 | 246 | 247 | @patch("yardstick.compare_results") 248 | @patch("yardstick.compare_results_against_labels") 249 | @patch("yardstick.validate.delta.compute_deltas") 250 | def test_validate_non_identical_match_sets( 251 | mock_compute_deltas, 252 | mock_compare_against_labels, 253 | mock_compare_results, 254 | non_identical_results, 255 | deltas, 256 | label_comparison_results, 257 | ): 258 | mock_compare_results.return_value = non_identical_results 259 | mock_compare_against_labels.return_value = label_comparison_results 260 | mock_compute_deltas.return_value = deltas 261 | gate = validate_image( 262 | f"docker.io/anchore/test_images@{'f' * 64}", 263 | GateConfig(fail_on_empty_match_set=False), 264 | descriptions=["some-str", "another-str"], 265 | always_run_label_comparison=False, 266 | verbosity=0, 267 | ) 268 | assert gate.passed() 269 | --------------------------------------------------------------------------------