├── .github ├── dependabot.yml ├── labeler.yml └── workflows │ ├── build.yml │ ├── labeler.yml │ ├── publish.yml │ └── security.yml ├── .gitignore ├── .python-version ├── .release.yml ├── .vimignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── docs ├── api │ ├── codeql │ │ ├── codeql-cli.rst │ │ ├── databases.rst │ │ ├── index.rst │ │ ├── packs.rst │ │ └── results.rst │ ├── index.rst │ ├── octokit │ │ ├── codescanning.rst │ │ ├── github.rst │ │ ├── index.rst │ │ ├── octokit.rst │ │ ├── repository.rst │ │ ├── secretscanning.rst │ │ ├── security-advisories.rst │ │ └── supplychain.rst │ └── supplychain │ │ ├── advisories.rst │ │ ├── dependencies.rst │ │ ├── dependency.rst │ │ └── index.rst ├── cli │ ├── codeql-packs.md │ ├── index.rst │ └── supplychain.md ├── conf.py ├── examples │ ├── advisories.md │ ├── codeql-packs.md │ ├── codeql.md │ ├── codescanning.md │ ├── dependencies.md │ └── index.rst ├── index.rst └── static │ ├── README.md │ ├── custom.css │ └── ghastoolkit.png ├── examples ├── advisories.py ├── advisories │ └── log4shell.json ├── clearlydefined.py ├── codeql-databases.py ├── codeql-packs.py ├── codeql.py ├── codescanning.py ├── dependabot.py ├── dependencies-org.py ├── dependencies.py ├── github.py ├── licenses.py ├── packs │ ├── codeql-pack.lock.yml │ └── qlpack.yml ├── repository.py └── secrets.py ├── pyproject.toml ├── src └── ghastoolkit │ ├── __init__.py │ ├── __main__.py │ ├── billing │ └── __main__.py │ ├── codeql │ ├── __init__.py │ ├── __main__.py │ ├── cli.py │ ├── consts.py │ ├── databases.py │ ├── dataextensions │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── ext.py │ │ └── models.py │ ├── packs │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── pack.py │ │ └── packs.py │ └── results.py │ ├── errors.py │ ├── octokit │ ├── __init__.py │ ├── advisories.py │ ├── billing.py │ ├── clearlydefined.py │ ├── codescanning.py │ ├── dependabot.py │ ├── dependencygraph.py │ ├── enterprise.py │ ├── github.py │ ├── graphql │ │ ├── GetDependencyAlerts.graphql │ │ ├── GetDependencyInfo.graphql │ │ └── __init__.py │ ├── octokit.py │ ├── repository.py │ └── secretscanning.py │ ├── secretscanning │ ├── __init__.py │ └── secretalerts.py │ ├── supplychain │ ├── __init__.py │ ├── __main__.py │ ├── advisories.py │ ├── dependencies.py │ ├── dependency.py │ ├── dependencyalert.py │ └── licensing.py │ └── utils │ ├── __init__.py │ ├── cache.py │ └── cli.py ├── tests ├── data │ └── advisories.yml ├── responses │ ├── codescanning.json │ ├── dependabot.json │ └── restrequests.json ├── test_advisories.py ├── test_clearlydefined.py ├── test_codeql_dataext.py ├── test_codeql_packs.py ├── test_codeqldb.py ├── test_codescanning.py ├── test_default.py ├── test_dependabot.py ├── test_dependencies.py ├── test_depgraph.py ├── test_github.py ├── test_licenses.py ├── test_octokit.py ├── test_restrequest.py ├── test_secretscanning.py └── utils.py └── uv.lock /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" 9 | directory: "/" 10 | schedule: 11 | interval: "weekly" 12 | reviewers: 13 | - "geekmasher" 14 | groups: 15 | production-dependencies: 16 | dependency-type: "production" 17 | development-dependencies: 18 | dependency-type: "development" 19 | 20 | - package-ecosystem: "github-actions" 21 | directory: "/" 22 | schedule: 23 | interval: "weekly" 24 | reviewers: 25 | - "geekmasher" 26 | groups: 27 | production-dependencies: 28 | dependency-type: "production" 29 | development-dependencies: 30 | dependency-type: "development" 31 | 32 | -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | 2 | docs: 3 | - changed-files: 4 | - any-glob-to-any-file: 5 | - "docs/**" 6 | - "examples/**" 7 | 8 | actions: 9 | - changed-files: 10 | - any-glob-to-any-file: 11 | - ".github/workflows/**" 12 | 13 | version: 14 | - changed-files: 15 | - any-glob-to-any-file: 16 | - "pyproject.toml" 17 | 18 | codeql: 19 | - changed-files: 20 | - any-glob-to-any-file: 21 | - "src/ghastoolkit/codeql/**" 22 | - "src/ghastoolkit/octokit/codescanning.py" 23 | 24 | supplychain: 25 | - changed-files: 26 | - any-glob-to-any-file: 27 | - "src/ghastoolkit/supplychain/**" 28 | - "src/ghastoolkit/octokit/dependabot.py" 29 | - "src/ghastoolkit/octokit/dependencygraph.py" 30 | - "src/ghastoolkit/octokit/clearlydefined.py" 31 | 32 | octokit: 33 | - changed-files: 34 | - any-glob-to-any-file: 35 | - "src/ghastoolkit/octokit/**" 36 | 37 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Python Build and Test 2 | 3 | on: 4 | push: 5 | branches: [ "main", "develop" ] 6 | pull_request: 7 | branches: [ "main", "develop" ] 8 | schedule: 9 | - cron: "0 15 * * *" 10 | 11 | permissions: 12 | contents: read 13 | security-events: read 14 | 15 | env: 16 | PYTHON_VERSION: "3.13" 17 | 18 | jobs: 19 | build: 20 | runs-on: ubuntu-latest 21 | 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | python-version: ["3.10", "3.11", "3.12", "3.13"] 26 | 27 | concurrency: 28 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.python-version }} 29 | cancel-in-progress: true 30 | 31 | env: 32 | UV_PYTHON: ${{ matrix.python-version }} 33 | 34 | steps: 35 | - uses: actions/checkout@v4 36 | - name: Set up Python ${{ matrix.python-version }} 37 | uses: actions/setup-python@v5 38 | with: 39 | python-version: ${{ matrix.python-version }} 40 | 41 | - name: Install uv 42 | uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0 43 | 44 | - name: Install dependencies 45 | run: | 46 | uv sync --all-extras --dev 47 | 48 | - name: Building 49 | run: | 50 | uv build 51 | 52 | - name: Linting 53 | run: | 54 | uv run black --check ./src 55 | 56 | - name: Testing 57 | run: | 58 | export PYTHONPATH=$PWD/src 59 | uv run python -m unittest discover -v -s ./tests -p 'test_*.py' 60 | 61 | - name: Documentation building 62 | run: | 63 | export PYTHONPATH=$PWD/src 64 | uv run sphinx-build -b html ./docs ./public 65 | 66 | # Only run examples on push to main branch 67 | # examples: 68 | # runs-on: ubuntu-latest 69 | # if: github.event_name == 'push' && github.ref == 'refs/heads/main' 70 | # needs: build 71 | 72 | # steps: 73 | # - uses: actions/checkout@v4 74 | # - name: Set up Python ${{ env.PYTHON_VERSION }} 75 | # uses: actions/setup-python@v5 76 | # with: 77 | # python-version: ${{ env.PYTHON_VERSION }} 78 | # - name: Install uv 79 | # uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0 80 | 81 | # - name: Install dependencies 82 | # run: | 83 | # uv sync --all-extras --dev 84 | 85 | # - name: Run Examples 86 | # env: 87 | # GITHUB_TOKEN: "${{ secrets.GHASTOOLKIT_PAT }}" 88 | # GHASTOOLKIT_ORG_PAT: "${{ secrets.GHASTOOLKIT_ORG_PAT }}" 89 | # run: | 90 | # set -e 91 | # export PYTHONPATH=$PWD/src 92 | 93 | # for f in examples/*.py; do 94 | # echo "[+] Running :: $f" 95 | # uv run python $f 96 | # done 97 | 98 | cli: 99 | runs-on: ubuntu-latest 100 | needs: build 101 | 102 | steps: 103 | - uses: actions/checkout@v4 104 | - name: Set up Python ${{ env.PYTHON_VERSION }} 105 | uses: actions/setup-python@v5 106 | with: 107 | python-version: ${{ env.PYTHON_VERSION }} 108 | - name: Install uv 109 | uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0 110 | 111 | - name: Install dependencies 112 | run: | 113 | uv sync --all-extras --dev 114 | 115 | - name: Run CLI 116 | env: 117 | GITHUB_TOKEN: "${{ github.token }}" 118 | run: | 119 | set -e 120 | export PYTHONPATH=$PWD/src 121 | 122 | uv run python -m ghastoolkit --help 123 | -------------------------------------------------------------------------------- /.github/workflows/labeler.yml: -------------------------------------------------------------------------------- 1 | # This workflow will triage pull requests and apply a label based on the 2 | # paths that are modified in the pull request. 3 | # 4 | # To use this workflow, you will need to set up a .github/labeler.yml 5 | # file with configuration. For more information, see: 6 | # https://github.com/actions/labeler 7 | 8 | name: Labeler 9 | on: [pull_request_target] 10 | 11 | jobs: 12 | label: 13 | 14 | runs-on: ubuntu-latest 15 | permissions: 16 | contents: read 17 | pull-requests: write 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | - uses: actions/labeler@v5 22 | with: 23 | repo-token: "${{ secrets.GITHUB_TOKEN }}" 24 | 25 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | permissions: 8 | contents: read 9 | 10 | env: 11 | PYTHON_VERSION: "3.13" 12 | 13 | jobs: 14 | publish: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Set up Python 20 | uses: actions/setup-python@v5 21 | with: 22 | python-version: '${{ env.PYTHON_VERSION }}' 23 | 24 | - name: Install uv 25 | uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0 26 | 27 | - name: Install dependencies 28 | run: | 29 | uv sync --all-extras --dev 30 | 31 | - name: Build and Test Package 32 | env: 33 | GITHUB_TOKEN: ${{ github.token }} 34 | run: | 35 | set -e 36 | export PYTHONPATH=$PWD/src 37 | 38 | uv build 39 | uv run python -m unittest discover -v -s ./tests -p 'test_*.py' 40 | 41 | - name: Publish package 42 | uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4 43 | with: 44 | user: __token__ 45 | password: ${{ secrets.PYPI_API_TOKEN }} 46 | 47 | docs: 48 | runs-on: ubuntu-latest 49 | needs: [ publish ] 50 | permissions: 51 | contents: write 52 | 53 | steps: 54 | - uses: actions/checkout@v4 55 | - name: Set up Python 56 | uses: actions/setup-python@v5 57 | with: 58 | python-version: '3.13' 59 | 60 | - name: Install uv 61 | uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0 62 | 63 | - name: Build docs 64 | run: | 65 | export PYTHONPATH=$PWD/src 66 | uv run sphinx-build -b html ./docs ./public 67 | 68 | - name: Publish 69 | uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 70 | with: 71 | github_token: ${{ secrets.GITHUB_TOKEN }} 72 | publish_dir: ./public 73 | -------------------------------------------------------------------------------- /.github/workflows/security.yml: -------------------------------------------------------------------------------- 1 | name: 'Security' 2 | on: 3 | push: 4 | branches: [main] 5 | pull_request: 6 | branches: [main] 7 | 8 | jobs: 9 | dependency-review: 10 | runs-on: ubuntu-latest 11 | # Only run in a pull request 12 | if: github.event_name == 'pull_request' 13 | 14 | permissions: 15 | contents: read 16 | pull-requests: write 17 | 18 | steps: 19 | - name: 'Checkout Repository' 20 | uses: actions/checkout@v4 21 | 22 | - name: 'Dependency Review' 23 | uses: actions/dependency-review-action@v4 24 | with: 25 | fail-on-severity: moderate 26 | fail-on-scopes: runtime 27 | comment-summary-in-pr: 'on-failure' 28 | 29 | analyze: 30 | name: Analyze 31 | runs-on: ubuntu-latest 32 | permissions: 33 | actions: read 34 | contents: read 35 | security-events: write 36 | 37 | strategy: 38 | fail-fast: false 39 | matrix: 40 | language: [ 'python', 'actions' ] 41 | 42 | steps: 43 | - name: Checkout repository 44 | uses: actions/checkout@v4 45 | 46 | - name: Initialize CodeQL 47 | uses: github/codeql-action/init@v3 48 | with: 49 | languages: ${{ matrix.language }} 50 | config-file: geekmasher/security-codeql/config/default.yml@main 51 | 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v3 54 | 55 | - name: Perform CodeQL Analysis 56 | uses: github/codeql-action/analyze@v3 57 | with: 58 | category: "/language:${{matrix.language}}" 59 | 60 | semgrep: 61 | name: semgrep/ci 62 | runs-on: ubuntu-latest 63 | permissions: 64 | actions: read 65 | contents: read 66 | security-events: write 67 | 68 | container: 69 | image: returntocorp/semgrep 70 | 71 | steps: 72 | - name: Checkout repository 73 | uses: actions/checkout@v4 74 | 75 | - name: Run Semgrep 76 | run: semgrep --config auto . --sarif --output semgrep.sarif 77 | 78 | - name: Upload SARIF file 79 | uses: github/codeql-action/upload-sarif@v3 80 | with: 81 | sarif_file: semgrep.sarif 82 | if: always() 83 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | public/ 3 | .data/ 4 | test.py 5 | *.spdx 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | cover/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | .pybuilder/ 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/#use-with-ide 116 | .pdm.toml 117 | 118 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 119 | __pypackages__/ 120 | 121 | # Celery stuff 122 | celerybeat-schedule 123 | celerybeat.pid 124 | 125 | # SageMath parsed files 126 | *.sage.py 127 | 128 | # Environments 129 | .env 130 | .venv 131 | env/ 132 | venv/ 133 | ENV/ 134 | env.bak/ 135 | venv.bak/ 136 | 137 | # Spyder project settings 138 | .spyderproject 139 | .spyproject 140 | 141 | # Rope project settings 142 | .ropeproject 143 | 144 | # mkdocs documentation 145 | /site 146 | 147 | # mypy 148 | .mypy_cache/ 149 | .dmypy.json 150 | dmypy.json 151 | 152 | # Pyre type checker 153 | .pyre/ 154 | 155 | # pytype static type analyzer 156 | .pytype/ 157 | 158 | # Cython debug symbols 159 | cython_debug/ 160 | 161 | # PyCharm 162 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 163 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 164 | # and can be added to the global gitignore or merged into this file. For a more nuclear 165 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 166 | #.idea/ 167 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.13 2 | -------------------------------------------------------------------------------- /.release.yml: -------------------------------------------------------------------------------- 1 | name: "ghastoolkit" 2 | version: "0.17.6" 3 | 4 | ecosystems: 5 | - Docs 6 | - Python 7 | -------------------------------------------------------------------------------- /.vimignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | src/ghastoolkit.egg-info 3 | 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to GHASToolkit 2 | 3 | The GHASToolkit project is an open-source library and CLI for working with the different features of GitHub Advance Security. We welcome contributions from the community! Here are some guidelines to help you get started. 4 | 5 | ## Issues 6 | 7 | The easiest way to contribute is to report issues. If you find a bug or have a feature request, please open an issue on the [GitHub repository](https://github.com/geekmasher/ghastoolkit/issues). 8 | 9 | ## Requirements 10 | 11 | - `python` 3.10 or higher 12 | - [`uv`](https://github.com/astral-sh/uv) 13 | 14 | GHASToolkit needs to be able to run on all supported versions of Python. Please make sure to test your changes on all supported versions. 15 | 16 | ## Building 17 | 18 | To build the project, you just need to run the following command: 19 | 20 | ```bash 21 | uv build 22 | ``` 23 | 24 | If you are having issues building / running the project, you might need to set the `PYTHONPATH` environment variable to the root of the project. You can do this by running the following command: 25 | 26 | ```bash 27 | export PYTHONPATH=$PWD/src 28 | ``` 29 | 30 | ### Code Formatting 31 | 32 | GHASToolkit uses `black` for code formatting. To format the code, you can use the following command: 33 | 34 | ```bash 35 | uv run black . 36 | ``` 37 | 38 | ## Testing 39 | 40 | GHASToolkit uses `unittest` for testing. To run the tests, you can use the following command: 41 | 42 | ```bash 43 | uv run python -m unittest discover -v -s ./tests -p 'test_*.py' 44 | ``` 45 | 46 | ### API Requests / Responses 47 | 48 | When writing tests for API requests, please make sure to use the `responses` library along with the `utils` module to mock the requests. 49 | This is important to ensure that the tests are not dependent on the actual API responses and can be run in isolation. 50 | 51 | **Example:** 52 | 53 | ```python 54 | import responses 55 | import utils 56 | 57 | class TestMyClass(unittest.TestCase): 58 | @responses.activate 59 | def test_my_method(self): 60 | # This will add all the mocked 61 | utils.loadResponses('mytests', 'my_method') 62 | 63 | # Call the method 64 | result = utils.my_method() 65 | 66 | # Assert the result 67 | self.assertEqual(result, {'key': 'value'}) 68 | ``` 69 | 70 | ## Running CLI 71 | 72 | To run the CLI, you can use the following command: 73 | 74 | ```bash 75 | uv run python -m ghastoolkit --help 76 | ``` 77 | 78 | ## Documentation 79 | 80 | GHASToolkit uses `sphinx` for documentation. To build the documentation, run the following command: 81 | 82 | ```bash 83 | uv run sphinx-build -b html ./docs ./public 84 | ``` 85 | 86 | *Note:* This might change in the future, but for now, the documentation is built using `sphinx` and hosted on GitHub Pages. You can find the documentation at [https://ghastoolkit.github.io/](https://ghastoolkit.github.io/). 87 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Mathew Payne 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |
3 |

GHASToolkit

4 | 5 | 6 | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)][github] 7 | [![GitHub Actions](https://img.shields.io/github/actions/workflow/status/geekmasher/ghastoolkit/python-package.yml?style=for-the-badge)][github] 8 | [![GitHub Issues](https://img.shields.io/github/issues/geekmasher/ghastoolkit?style=for-the-badge)][github-issues] 9 | [![GitHub Stars](https://img.shields.io/github/stars/geekmasher/ghastoolkit?style=for-the-badge)][github] 10 | [![Python Versions](https://img.shields.io/pypi/pyversions/ghastoolkit?style=for-the-badge)][pypi] 11 | [![License](https://img.shields.io/github/license/Ileriayo/markdown-badges?style=for-the-badge)][license] 12 | 13 |
14 | 15 | 16 | ## Overview 17 | 18 | [GitHub Advanced Security (GHAS)][advanced-security] Python Toolkit to make the lives of everyone that uses GHAS a little easier. 19 | 20 | ## ✨ Features 21 | 22 | - API Client for all GHAS Features 23 | - Code Scanning 24 | - Secret Scanning 25 | - Dependency Graph 26 | - Security Advisories 27 | - Dependabot / Security Alerts 28 | - CodeQL Management 29 | - Database Management 30 | - Packs / Query Management 31 | 32 | ## 📦 Installing 33 | 34 | To install `ghastoolkit`, you can use `pip` or one of `pypi` family's of tools to install: 35 | 36 | ```bash 37 | # pip 38 | pip install ghastoolkit 39 | 40 | # pipenv 41 | pipenv install ghastoolkit 42 | 43 | # poetry 44 | poetry add ghastoolkit 45 | ``` 46 | 47 | ## 🏃 Usage 48 | 49 | To see how to use `ghastoolkit`, [take a look at the docs][docs]. 50 | 51 | ## 🧑‍🤝‍🧑 Maintainers / Contributors 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | ## 🦸 Support 63 | 64 | Please create [GitHub Issues][github-issues] if there are bugs or feature requests. 65 | 66 | This project uses [Semantic Versioning (v2)][semver] and with major releases, breaking changes will occur. 67 | 68 | ## 📓 License 69 | 70 | This project is licensed under the terms of the MIT open source license. 71 | Please refer to [MIT][license] for the full terms. 72 | 73 | 74 | 75 | [license]: ./LICENSE 76 | [pypi]: https://pypi.org/project/ghastoolkit 77 | [github]: https://github.com/GeekMasher/ghastoolkit 78 | [github-issues]: https://github.com/GeekMasher/ghastoolkit/issues 79 | 80 | [docs]: https://geekmasher.github.io/ghastoolkit 81 | [advanced-security]: https://github.com/features/security 82 | 83 | [semver]: https://semver.org/ 84 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | GHASToolkit only officially supports its latest version. 6 | If a security issue is found in an older version, this should be reported to `ghastoolkit`. 7 | 8 | ## Reporting a Vulnerability 9 | 10 | Please report security vulnerabilities via [the repositories GitHub Advisories](https://github.com/GeekMasher/ghastoolkit/security/advisories)https://github.com/GeekMasher/ghastoolkit/security/advisories. 11 | These can be created privately so the maintainer and the community can create patches for end users. 12 | -------------------------------------------------------------------------------- /docs/api/codeql/codeql-cli.rst: -------------------------------------------------------------------------------- 1 | .. _codeql_cli: 2 | 3 | CodeQL CLI 4 | ========== 5 | 6 | CodeQL CLI module to make it easier to run CodeQL queries. 7 | 8 | CodeQL 9 | ------ 10 | .. module:: ghastoolkit 11 | .. autoclass:: CodeQL 12 | :members: 13 | 14 | -------------------------------------------------------------------------------- /docs/api/codeql/databases.rst: -------------------------------------------------------------------------------- 1 | .. _codeqldatabases: 2 | 3 | CodeQL Databases 4 | ================ 5 | 6 | This is the CodeQL databases API which allows you to load, run, and a number of other things on CodeQL databases. 7 | 8 | CodeQL Database 9 | --------------- 10 | .. module:: ghastoolkit 11 | .. autoclass:: CodeQLDatabase 12 | :members: 13 | 14 | 15 | CodeQL Databases 16 | ---------------- 17 | .. module:: ghastoolkit 18 | .. autoclass:: CodeQLDatabases 19 | :members: 20 | 21 | -------------------------------------------------------------------------------- /docs/api/codeql/index.rst: -------------------------------------------------------------------------------- 1 | .. _codeql: 2 | 3 | CodeQL 4 | ====== 5 | 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | codeql-cli 11 | databases 12 | packs 13 | results 14 | 15 | -------------------------------------------------------------------------------- /docs/api/codeql/packs.rst: -------------------------------------------------------------------------------- 1 | .. _codeql_packs: 2 | 3 | CodeQL Packs 4 | ============= 5 | 6 | 7 | CodeQL Pack 8 | ----------- 9 | .. module:: ghastoolkit 10 | .. autoclass:: CodeQLPack 11 | :members: 12 | 13 | 14 | CodeQL Packs 15 | ------------ 16 | .. module:: ghastoolkit 17 | .. autoclass:: CodeQLPacks 18 | :members: 19 | 20 | -------------------------------------------------------------------------------- /docs/api/codeql/results.rst: -------------------------------------------------------------------------------- 1 | .. _codeql_results: 2 | 3 | CodeQL Result Models 4 | ==================== 5 | 6 | 7 | CodeQLResults 8 | ------------- 9 | .. module:: ghastoolkit 10 | .. autoclass:: CodeQLResults 11 | :members: 12 | 13 | CodeLocation 14 | ------------ 15 | .. module:: ghastoolkit 16 | .. autoclass:: CodeLocation 17 | :members: 18 | 19 | 20 | CodeResult 21 | ---------- 22 | .. module:: ghastoolkit 23 | .. autoclass:: CodeResult 24 | :members: 25 | 26 | -------------------------------------------------------------------------------- /docs/api/index.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | API Reference 4 | ============= 5 | 6 | This part of the documentation covers all the APIs which are part of `ghastoolkit`. 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | octokit/index 12 | codeql/index 13 | supplychain/index 14 | 15 | -------------------------------------------------------------------------------- /docs/api/octokit/codescanning.rst: -------------------------------------------------------------------------------- 1 | .. _codescanning: 2 | 3 | Code Scanning 4 | ============= 5 | 6 | This part of the documentation covers all the `codescanning` APIs. 7 | 8 | 9 | CodeScanning 10 | ------------ 11 | .. module:: ghastoolkit 12 | .. autoclass:: CodeScanning 13 | :members: 14 | 15 | 16 | Code Alert 17 | ---------- 18 | .. module:: ghastoolkit 19 | .. autoclass:: CodeAlert 20 | :members: 21 | 22 | -------------------------------------------------------------------------------- /docs/api/octokit/github.rst: -------------------------------------------------------------------------------- 1 | .. _github: 2 | 3 | GitHub 4 | ====== 5 | 6 | This part of the documentation covers all the `github` APIs. 7 | 8 | 9 | GitHub 10 | ------ 11 | .. module:: ghastoolkit 12 | .. autoclass:: GitHub 13 | :members: 14 | 15 | -------------------------------------------------------------------------------- /docs/api/octokit/index.rst: -------------------------------------------------------------------------------- 1 | .. _octokit_apis: 2 | 3 | Octokit 4 | ============= 5 | 6 | This part of the documentation covers all the `octokit` APIs. 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | github 12 | repository 13 | codescanning 14 | supplychain 15 | secretscanning 16 | security-advisories 17 | octokit 18 | 19 | -------------------------------------------------------------------------------- /docs/api/octokit/octokit.rst: -------------------------------------------------------------------------------- 1 | .. _octokitclass: 2 | 3 | Octokit (internals) 4 | =================== 5 | 6 | This part of the documentation covers all the `octokit` APIs. 7 | 8 | 9 | RestRequest API 10 | --------------- 11 | 12 | `RestRequest` is for all the Octokit APIs that require a REST interface with GitHub. 13 | 14 | .. module:: ghastoolkit 15 | .. autoclass:: RestRequest 16 | :members: 17 | 18 | 19 | GraphQLRequest API 20 | ------------------ 21 | 22 | `GraphQLRequest` is for all the Octokit APIs that require a GraphQL interface with GitHub. 23 | 24 | .. module:: ghastoolkit 25 | .. autoclass:: GraphQLRequest 26 | :members: 27 | 28 | OctoItem Class 29 | -------------- 30 | 31 | .. module:: ghastoolkit.octokit.octokit 32 | .. autoclass:: OctoItem 33 | :members: 34 | 35 | -------------------------------------------------------------------------------- /docs/api/octokit/repository.rst: -------------------------------------------------------------------------------- 1 | .. _github_repository: 2 | 3 | GitHub Repository 4 | ================= 5 | 6 | This is the GitHub `Repository` API to do all things to do with GitHub repositories. 7 | 8 | Repository API 9 | -------------- 10 | .. module:: ghastoolkit 11 | .. autoclass:: Repository 12 | :members: 13 | 14 | -------------------------------------------------------------------------------- /docs/api/octokit/secretscanning.rst: -------------------------------------------------------------------------------- 1 | .. _secretscanning: 2 | 3 | Secret Scanning 4 | =============== 5 | 6 | This part of the documentation covers all the `secretscanning` APIs. 7 | 8 | 9 | SecretScanning 10 | -------------- 11 | .. module:: ghastoolkit 12 | .. autoclass:: SecretScanning 13 | :members: 14 | 15 | 16 | SecretAlert 17 | ----------- 18 | .. module:: ghastoolkit 19 | .. autoclass:: SecretAlert 20 | :members: 21 | 22 | -------------------------------------------------------------------------------- /docs/api/octokit/security-advisories.rst: -------------------------------------------------------------------------------- 1 | .. _security_advisories: 2 | 3 | Security Advisories 4 | =================== 5 | 6 | 7 | SecurityAdvisories 8 | ------------------ 9 | .. module:: ghastoolkit 10 | .. autoclass:: SecurityAdvisories 11 | :members: 12 | 13 | -------------------------------------------------------------------------------- /docs/api/octokit/supplychain.rst: -------------------------------------------------------------------------------- 1 | .. _supplychain: 2 | 3 | Supply Chain 4 | ============ 5 | 6 | This part of the documentation covers all the `supplychain` APIs. 7 | 8 | 9 | Dependency Graph API 10 | -------------------- 11 | .. module:: ghastoolkit 12 | .. autoclass:: DependencyGraph 13 | :members: 14 | 15 | 16 | Dependabot API 17 | -------------- 18 | .. module:: ghastoolkit 19 | .. autoclass:: Dependabot 20 | :members: 21 | 22 | -------------------------------------------------------------------------------- /docs/api/supplychain/advisories.rst: -------------------------------------------------------------------------------- 1 | .. _advisories: 2 | 3 | Advisories 4 | ========== 5 | 6 | Advisory 7 | -------- 8 | .. module:: ghastoolkit 9 | .. autoclass:: Advisory 10 | :members: 11 | 12 | Advisories 13 | ---------- 14 | .. module:: ghastoolkit 15 | .. autoclass:: Advisories 16 | :members: 17 | 18 | AdvisoryAffect 19 | -------------- 20 | .. module:: ghastoolkit.supplychain.advisories 21 | .. autoclass:: AdvisoryAffect 22 | :members: 23 | 24 | -------------------------------------------------------------------------------- /docs/api/supplychain/dependencies.rst: -------------------------------------------------------------------------------- 1 | .. _dependencies: 2 | 3 | Dependencies 4 | ========== 5 | 6 | Dependencies 7 | ------------ 8 | .. module:: ghastoolkit 9 | .. autoclass:: Dependencies 10 | :members: 11 | 12 | -------------------------------------------------------------------------------- /docs/api/supplychain/dependency.rst: -------------------------------------------------------------------------------- 1 | .. _dependency: 2 | 3 | Dependency 4 | ========== 5 | 6 | Dependency 7 | ---------- 8 | .. module:: ghastoolkit 9 | .. autoclass:: Dependency 10 | :members: 11 | -------------------------------------------------------------------------------- /docs/api/supplychain/index.rst: -------------------------------------------------------------------------------- 1 | .. _supplychain: 2 | 3 | Supply Chain 4 | ============ 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | advisories 10 | dependency 11 | dependencies 12 | 13 | -------------------------------------------------------------------------------- /docs/cli/codeql-packs.md: -------------------------------------------------------------------------------- 1 | # CodeQL Packs 2 | 3 | ## Subcommands 4 | 5 | ### Version Bumping 6 | 7 | To update your packs version number, you can use the `version` subcommand. 8 | 9 | ```bash 10 | python -m ghastoolkit.codeql.packs version --help 11 | ``` 12 | 13 | You can also set the type of bumping using the `--bump [type]` command. 14 | 15 | ```bash 16 | python -m ghastoolkit.codeql.packs version --bump patch 17 | ``` 18 | 19 | This will result in the CodeQL Pack being version bumped correctly based on the type. 20 | 21 | ## Features 22 | 23 | ### Updating Dependencies 24 | 25 | To update your packs dependencies to the latest version, you can use the `--latest` 26 | argument. 27 | 28 | ```bash 29 | python -m ghastoolkit.codeql.packs --latest 30 | ``` 31 | 32 | Updating dependencies can work along with `version` subcommand to both update and 33 | bump your packs version. 34 | -------------------------------------------------------------------------------- /docs/cli/index.rst: -------------------------------------------------------------------------------- 1 | .. _cli: 2 | 3 | GHASToolkit CLI 4 | =============== 5 | 6 | This is the command line interface for the GHASToolkit. It is a wrapper around 7 | the various tools that are part of the GHASToolkit. It is designed to allow a user 8 | to do various tasks without having to know the details of the underlying tools. 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | codeql-packs 14 | 15 | supplychain 16 | 17 | -------------------------------------------------------------------------------- /docs/cli/supplychain.md: -------------------------------------------------------------------------------- 1 | # Supply Chain CLI 2 | 3 | ```bash 4 | python -m ghastoolkit.supplychain --help 5 | ``` 6 | 7 | ## Organization Audit 8 | 9 | The CLI mode allows you to audit an entire organization to see if: 10 | 11 | 1. If a repository has an unwanted license 12 | 2. If a repository has an unknown license by GitHub 13 | 14 | To use this, we need to enable the `org-audit` mode in the supplychain cli: 15 | 16 | ```bash 17 | python -m ghastoolkit.supplychain org-audit \ 18 | -r "org/repo" 19 | ``` 20 | 21 | The only required argument is the `-r/--repository` which sets the owner and 22 | repository for `ghastoolkit`. 23 | 24 | You can also update the licenses you want to check for using `--licenses`, 25 | using `,` as a separater, and widecards to help with versions of licenses. 26 | 27 | ```bash 28 | python -m ghastoolkit.supplychain org-audit \ 29 | -r "org/repo" \ 30 | --licenses "MIT*,Apache*" 31 | ``` 32 | 33 | Finally you can also set the `--debug` flag to see the different repositories 34 | being analysed: 35 | 36 | ```bash 37 | python -m ghastoolkit.supplychain org-audit \ 38 | -r "org/repo" \ 39 | --debug 40 | ``` 41 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from sphinx.application import Sphinx 4 | 5 | sys.path.append(os.path.join(os.getcwd(), "..")) 6 | 7 | 8 | # -- Project information ----------------------------------------------------- 9 | 10 | from ghastoolkit import ( 11 | __title__ as project, 12 | __copyright__ as copyright, 13 | __version__ as release, 14 | __author__ as author, 15 | ) 16 | 17 | # -- General configuration --------------------------------------------------- 18 | extensions = [ 19 | "myst_parser", 20 | "sphinx.ext.autodoc", 21 | "sphinx.ext.doctest", 22 | "sphinx.ext.todo", 23 | "sphinx.ext.coverage", 24 | "sphinx.ext.githubpages", 25 | "sphinx.ext.napoleon", 26 | "sphinx.ext.autosectionlabel", 27 | ] 28 | 29 | master_doc = "index" 30 | 31 | templates_path = ["_templates"] 32 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 33 | 34 | source_suffix = { 35 | ".rst": "restructuredtext", 36 | ".md": "markdown", 37 | } 38 | 39 | pygments_style = "sphinx" 40 | 41 | # -- Options for HTML output ------------------------------------------------- 42 | html_theme = "alabaster" 43 | html_static_path = ["static"] 44 | 45 | # Toolbox icons created by Smashicons - Flaticon 46 | # https://www.flaticon.com/free-icons/toolbox 47 | html_logo = "static/ghastoolkit.png" 48 | 49 | htmlhelp_basename = f"{project}Doc" 50 | 51 | # -- Options for Napoleon output ------------------------------------------------ 52 | 53 | napoleon_google_docstring = True 54 | napoleon_numpy_docstring = False 55 | napoleon_include_init_with_doc = True 56 | napoleon_include_private_with_doc = False 57 | napoleon_include_special_with_doc = True 58 | napoleon_use_admonition_for_examples = True 59 | napoleon_use_admonition_for_notes = False 60 | napoleon_use_admonition_for_references = False 61 | napoleon_use_ivar = False 62 | napoleon_use_param = True 63 | napoleon_use_rtype = True 64 | 65 | # -- Options for manual page output ------------------------------------------ 66 | man_pages = [ 67 | (master_doc, project, f"{project} Documentation", [author], 1) 68 | ] 69 | 70 | # -- Options for Texinfo output ---------------------------------------------- 71 | texinfo_documents = [ 72 | ( 73 | master_doc, 74 | project, 75 | f"{project} Documentation", 76 | author, 77 | project, 78 | "One line description of project.", 79 | "Miscellaneous", 80 | ), 81 | ] 82 | 83 | 84 | def setup(app: Sphinx): 85 | def cut_module_meta(app, what, name, obj, options, lines): 86 | """Remove metadata from autodoc output.""" 87 | if what != "module": 88 | return 89 | 90 | lines[:] = [ 91 | line for line in lines if not line.startswith((":copyright:", ":license:")) 92 | ] 93 | 94 | app.connect("autodoc-process-docstring", cut_module_meta) 95 | 96 | -------------------------------------------------------------------------------- /docs/examples/advisories.md: -------------------------------------------------------------------------------- 1 | # Custom Supply Chain Advisories 2 | 3 | First lets import and load our dependency (in our test case, `log4j-core`) 4 | 5 | ```python 6 | from ghastoolkit import Advisories, Advisory, Dependency 7 | 8 | # Load Dependency from PURL 9 | dependency = Dependency.fromPurl( 10 | "pkg:maven/org.apache.logging/log4j:log4j-core@1.12.0" 11 | ) 12 | ``` 13 | 14 | Then we want to create and load our advisories (in our case, `log4shell` / `CVE-2021-44228`) 15 | 16 | ```python 17 | # create a new list of Advisories 18 | advisories = Advisories() 19 | # load advisories from path 20 | advisories.loadAdvisories(".") 21 | 22 | print(f"Advisories :: {len(advisories)}") 23 | ``` 24 | 25 | Another option is to have your advisories in a repository and pull them from 26 | GitHub using `SecurityAdvisories`. 27 | 28 | ```python 29 | # initialise SecurityAdvisories 30 | security_advisories = SecurityAdvisories() 31 | # get all the remote advisories 32 | advisories = security_advisories.getAdvisories() 33 | 34 | print(f"Advisories :: {len(advisories)}") 35 | ``` 36 | 37 | Now lets find and display the advisory to make sure its found. 38 | 39 | ```python 40 | # find an advisories by GHSA ID ('CVE-2021-44228' would be the same) 41 | log4shell: Advisory = advisories.find("GHSA-jfh8-c2jp-5v3q") 42 | 43 | print(f"Advisory({log4shell.ghsa_id}, {log4shell.severity})") 44 | ``` 45 | 46 | Finally, lets check to see if the dependency has a known advisories associated 47 | with it. 48 | 49 | ```python 50 | print(f"Dependency :: {dependency.name} ({dependency.version})") 51 | 52 | # check in the advisories list if dependency is affected 53 | print("Advisories Found::") 54 | for adv in advisories.check(dependency): 55 | # display GHSA ID and aliases 56 | print(f" >>> Advisory({adv.ghsa_id}, `{','.join(adv.aliases)}`)") 57 | 58 | ``` 59 | 60 | In our case, it shows the following: 61 | 62 | ```text 63 | Dependency :: log4j:log4j-core (1.12.0) 64 | Advisories Found: 65 | >>> Advisory(GHSA-jfh8-c2jp-5v3q, `CVE-2021-44228`) 66 | ``` 67 | 68 | See all [examples here](https://github.com/GeekMasher/ghastoolkit/tree/main/examples) 69 | -------------------------------------------------------------------------------- /docs/examples/codeql-packs.md: -------------------------------------------------------------------------------- 1 | # CodeQL Pack 2 | 3 | First lets import and download the `codeql/python-queries` pack with the version 4 | set to `0.8.0`. This will automatically download the pack for us. 5 | 6 | ```python 7 | from ghastoolkit import CodeQLPack, CodeQLPacks 8 | 9 | pack = CodeQLPack.download("codeql/python-queries", "0.8.0") 10 | print(f"Pack :: {pack}") 11 | ``` 12 | 13 | Otherwise you can load via a path 14 | 15 | ```python 16 | pack = CodeQLPack("~/.codeql/packages/codeql/python-queries/0.8.0") 17 | print(f"Pack :: {pack}") 18 | ``` 19 | 20 | Or load a collection of packs using the `CodeQLPacks` API. 21 | 22 | ```python 23 | packs = CodeQLPacks("~/.codeql/packages") 24 | print(f"Packs :: {len(packs)}") 25 | for pack in packs: 26 | print(f" -> {pack}") 27 | ``` 28 | 29 | ## Custom Packs 30 | 31 | If you are creating custom packs and want to do all the things, you can use the 32 | easy to use APIs to make your life easier. 33 | 34 | ### Install Pack Dependencies 35 | 36 | To resolve and install the pack dependencies, you can use the following: 37 | 38 | ```python 39 | pack.install() 40 | ``` 41 | 42 | ### Create Pack and Install it locally 43 | 44 | To create a pack and install it locally on the current system: 45 | 46 | ```python 47 | path = pack.create() 48 | print(f"Pack Install Path :: {path}") 49 | ``` 50 | 51 | ### Resolve Pack Queries 52 | 53 | To get a list of all the queries in the pack you can use the `resolveQueries()` API. 54 | 55 | ```python 56 | queries = pack.resolveQueries() 57 | print(f"# Queries :: {len(queries)}") 58 | for query in queries: 59 | print(f" - {query}") 60 | ``` 61 | -------------------------------------------------------------------------------- /docs/examples/codeql.md: -------------------------------------------------------------------------------- 1 | # CodeQL 2 | 3 | ## CLI 4 | 5 | To use the CodeQL CLI in Python, you need to first import `CodeQL` and set it up: 6 | 7 | ```python 8 | from ghastoolkit import CodeQL, CodeQLDatabases 9 | 10 | codeql = CodeQL() 11 | # load all local databases on system in common locations 12 | databases = CodeQLDatabases.loadLocalDatabase() 13 | 14 | print(f"CodeQL :: {codeql}") 15 | print(f"Databases :: {len(databases)}") 16 | ``` 17 | 18 | You can also download databases remotely. 19 | 20 | ## Running Queries 21 | 22 | By default you can run the default queries on a database which will run the 23 | standard CodeQL query pack `codeql/$LANGUAGE-queries`. 24 | 25 | ```python 26 | # get a single database by name 27 | db = databases.get("ghastoolkit") 28 | 29 | results = codeql.runQuery(db) 30 | print(f"Results :: {len(results)}") 31 | ``` 32 | 33 | If you want to run a suites from the default packs on the database, use one of 34 | the built-in suites: 35 | 36 | ```python 37 | # security-extended 38 | results = codeql.runQuery(db, "security-extended") 39 | 40 | # security-and-quality 41 | results = codeql.runQuery(db, "security-and-quality") 42 | 43 | # security-experimental 44 | results = codeql.runQuery(db, "security-experimental") 45 | ``` 46 | 47 | You can also output the command to the console using `display` versus it being 48 | hidden by default. 49 | 50 | ```python 51 | codeql.runQuery(db, display=True) 52 | ``` 53 | 54 | ## Custom Packs 55 | 56 | To run a query from a custom pack, you can use the following pattern. 57 | 58 | ```python 59 | from ghastoolkit import CodeQL, CodeQLDatabases, CodeQLPack 60 | 61 | codeql = CodeQL() 62 | databases = CodeQLDatabases.loadLocalDatabase() 63 | 64 | # download the latest pack 65 | pack = CodeQLPack.download("geekmasher/codeql-python") 66 | print(f"Pack: {pack} (queries: {len(pack.resolveQueries())})") 67 | 68 | for db in databases: 69 | results = codeql.runQuery(db, pack.name) 70 | print(f" >> {db} :: {len(results)}") 71 | ``` 72 | -------------------------------------------------------------------------------- /docs/examples/codescanning.md: -------------------------------------------------------------------------------- 1 | # Code Scanning API Examples 2 | 3 | ### Get Code Scanning Alerts 4 | 5 | ```python 6 | from ghastoolkit import GitHub, CodeScanning 7 | # Initialise GitHub with repository `owner/name` 8 | GitHub.init("GeekMasher/ghastoolkit") 9 | 10 | # Initialise a CodeScanning instance 11 | cs = CodeScanning() 12 | 13 | # Get all the open alerts 14 | alerts = cs.getAlerts("open") 15 | 16 | print(f"Alerts :: {alerts}") 17 | ``` 18 | -------------------------------------------------------------------------------- /docs/examples/dependencies.md: -------------------------------------------------------------------------------- 1 | # Dependency Graph 2 | 3 | ## Organization 4 | 5 | To get a list of all the dependencies in an organization, your'll need to setup `DependencyGraph`. 6 | 7 | ```python 8 | from ghastoolkit import GitHub, DependencyGraph 9 | 10 | GitHub.init("GeekMasher/ghastoolkit") 11 | print(f"Owner :: {GitHub.repository.owner}") 12 | ``` 13 | 14 | After that, we can use the graph to pull a dict of repositories and associated dependencies: 15 | 16 | ```python 17 | depgraph = DependencyGraph() 18 | 19 | dependencies = depgraph.getOrganizationDependencies() 20 | 21 | for repo, deps in dependencies.items(): 22 | print(f" > {repo.display} :: {len(deps)}") 23 | ``` 24 | 25 | Once you have this information, you can look up dependencies usages across the 26 | organization or get other data from it. 27 | 28 | ## Dependencies and associated repositories 29 | 30 | One of the features of the Dependency Graph is it links a Dependency with a repository. 31 | This has to be done via the GraphQL API but this data can help with tracking and linking repositories being used. 32 | 33 | ```python 34 | depgraph = DependencyGraph() 35 | 36 | dependencies = depgraph.getDependenciesGraphQL() 37 | 38 | for dependency in dependencies: 39 | print(f" > {dependency} :: {dependency.repository}") 40 | ``` 41 | 42 | ## Snapshots 43 | 44 | To upload a snapshot to the Dependency Graph you can use the `DependencyGraph` octokit API. 45 | 46 | Lets start by importing and setting up `GitHub`. 47 | 48 | ```python 49 | import json 50 | from ghastoolkit import DependencyGraph, Dependencies 51 | 52 | # initialise GitHub 53 | GitHub.init("GeekMasher/ghastoolkit") 54 | 55 | # create DependencyGraph API 56 | depgraph = DependencyGraph() 57 | ``` 58 | 59 | Once you have a DependencyGraph ready, you with need to create a list of `Dependencies`. 60 | 61 | ```python 62 | dependencies = Dependencies() 63 | # ... create list of dependencies 64 | 65 | ``` 66 | 67 | Once you have the list, you can simply call the following API: 68 | 69 | ```python 70 | depgraph.submitDependencies(dependencies) 71 | ``` 72 | 73 | This function converts the list of dependencies to a SPDX and submits the data to the Dependency Graph API. 74 | -------------------------------------------------------------------------------- /docs/examples/index.rst: -------------------------------------------------------------------------------- 1 | .. _examples: 2 | 3 | Examples 4 | ============= 5 | 6 | All Examples can be found in the `examples` directory of the repository. 7 | 8 | https://github.com/GeekMasher/ghastoolkit/tree/main/examples 9 | 10 | 11 | **Working Examples:** 12 | 13 | .. toctree:: 14 | :maxdepth: 1 15 | 16 | codescanning 17 | dependencies 18 | advisories 19 | 20 | codeql 21 | codeql-packs 22 | 23 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to GHASToolkit! 2 | ========================================== 3 | 4 | GitHub Advanced Security (GHAS) Toolkit is a Python based library to things easier when working with GHAS features. 5 | 6 | 7 | Installation 8 | ================== 9 | 10 | To install `ghastoolkit` you can use the following commands: 11 | 12 | .. code-block:: bash 13 | 14 | # pip 15 | pip install ghastoolkit 16 | 17 | # pipenv 18 | pipenv install ghastoolkit 19 | 20 | # poetry 21 | poetry add ghastoolkit 22 | 23 | 24 | User's Guide 25 | ================== 26 | 27 | .. toctree:: 28 | :maxdepth: 3 29 | 30 | api/index 31 | 32 | 33 | .. toctree:: 34 | :maxdepth: 2 35 | 36 | cli/index 37 | 38 | .. toctree:: 39 | :maxdepth: 2 40 | 41 | examples/index 42 | 43 | -------------------------------------------------------------------------------- /docs/static/README.md: -------------------------------------------------------------------------------- 1 | # Docs Static Files 2 | 3 | ## References 4 | 5 | - [Toolbox icons created by Smashicons - Flaticon](https://www.flaticon.com/free-icons/toolbox) 6 | 7 | -------------------------------------------------------------------------------- /docs/static/custom.css: -------------------------------------------------------------------------------- 1 | 2 | div.sphinxsidebarwrapper p.logo { 3 | margin-bottom: 1rem; 4 | } 5 | 6 | div.sphinxsidebarwrapper h1.logo { 7 | text-align: center; 8 | margin-bottom: 2rem; 9 | font-size: 3rem; 10 | } 11 | 12 | div.sphinxsidebar h3 { 13 | text-align: center; 14 | } 15 | 16 | -------------------------------------------------------------------------------- /docs/static/ghastoolkit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GeekMasher/ghastoolkit/6d1ce533eebc886d40a580c357776a47fd87d138/docs/static/ghastoolkit.png -------------------------------------------------------------------------------- /examples/advisories.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ghastoolkit import Advisories, Advisory, SecurityAdvisories 3 | from ghastoolkit.octokit.github import GitHub 4 | from ghastoolkit.supplychain.dependencies import Dependency 5 | 6 | # Load GitHub 7 | GitHub.init( 8 | os.environ.get("GITHUB_REPOSITORY", "GeekMasher/ghastoolkit"), 9 | ) 10 | print(f"{GitHub.repository}") 11 | 12 | # SecurityAdvisories REST API helper 13 | security_advisories = SecurityAdvisories() 14 | 15 | # get remote security advisories 16 | advisories: Advisories = security_advisories.getAdvisories() 17 | print(f"Remote Advisories :: {len(advisories)}") 18 | # load local (json) advisories 19 | advisories.loadAdvisories("./examples") 20 | 21 | print(f"Total Advisories :: {len(advisories)}") 22 | 23 | for a in advisories.advisories: 24 | print(f"Advisory :: {a.ghsa_id} ({a.summary})") 25 | 26 | # get log4shell advisory 27 | log4shell: Advisory = advisories.find("ghas-jfh8-c2jp-5v3q") 28 | 29 | if log4shell: 30 | print(f"Advisory :: {log4shell.ghsa_id} ({log4shell.summary})") 31 | 32 | # load Dependency from PURL (log4j vulnerable to log4shell) 33 | dependency = Dependency.fromPurl("pkg:maven/org.apache.logging/log4j:log4j-core@1.12.0") 34 | print(f"Dependency :: {dependency.name} ({dependency.version})") 35 | 36 | # display all advisories which affect the dependency 37 | print("Advisories Found:") 38 | for adv in advisories.check(dependency): 39 | print(f" >>> Advisory({adv.ghsa_id}, `{','.join(adv.aliases)}`)") 40 | -------------------------------------------------------------------------------- /examples/advisories/log4shell.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema_version": "1.4.0", 3 | "id": "GHSA-jfh8-c2jp-5v3q", 4 | "modified": "2022-03-25T20:56:18Z", 5 | "published": "2021-12-10T00:40:56Z", 6 | "aliases": [ 7 | "CVE-2021-44228" 8 | ], 9 | "summary": "Remote code injection in Log4j", 10 | "details": "# Summary\n\nLog4j versions prior to 2.16.0 are subject to a remote code execution vulnerability via the ldap JNDI parser.\nAs per [Apache's Log4j security guide](https://logging.apache.org/log4j/2.x/security.html): Apache Log4j2 <=2.14.1 JNDI features used in configuration, log messages, and parameters do not protect against attacker controlled LDAP and other JNDI related endpoints. An attacker who can control log messages or log message parameters can execute arbitrary code loaded from LDAP servers when message lookup substitution is enabled. From log4j 2.16.0, this behavior has been disabled by default.\n\nLog4j version 2.15.0 contained an earlier fix for the vulnerability, but that patch did not disable attacker-controlled JNDI lookups in all situations. For more information, see the `Updated advice for version 2.16.0` section of this advisory.\n\n# Impact\n\nLogging untrusted or user controlled data with a vulnerable version of Log4J may result in Remote Code Execution (RCE) against your application. This includes untrusted data included in logged errors such as exception traces, authentication failures, and other unexpected vectors of user controlled input. \n\n# Affected versions\n\nAny Log4J version prior to v2.15.0 is affected to this specific issue.\n\nThe v1 branch of Log4J which is considered End Of Life (EOL) is vulnerable to other RCE vectors so the recommendation is to still update to 2.16.0 where possible.\n\n## Security releases\nAdditional backports of this fix have been made available in versions 2.3.1, 2.12.2, and 2.12.3\n\n## Affected packages\nOnly the `org.apache.logging.log4j:log4j-core` package is directly affected by this vulnerability. The `org.apache.logging.log4j:log4j-api` should be kept at the same version as the `org.apache.logging.log4j:log4j-core` package to ensure compatability if in use.\n\n# Remediation Advice\n\n## Updated advice for version 2.16.0\n\nThe Apache Logging Services team provided updated mitigation advice upon the release of version 2.16.0, which [disables JNDI by default and completely removes support for message lookups](https://logging.apache.org/log4j/2.x/changes-report.html#a2.16.0).\nEven in version 2.15.0, lookups used in layouts to provide specific pieces of context information will still recursively resolve, possibly triggering JNDI lookups. This problem is being tracked as [CVE-2021-45046](https://nvd.nist.gov/vuln/detail/CVE-2021-45046). More information is available on the [GitHub Security Advisory for CVE-2021-45046](https://github.com/advisories/GHSA-7rjr-3q55-vv33).\n\nUsers who want to avoid attacker-controlled JNDI lookups but cannot upgrade to 2.16.0 must [ensure that no such lookups resolve to attacker-provided data and ensure that the the JndiLookup class is not loaded](https://issues.apache.org/jira/browse/LOG4J2-3221).\n\nPlease note that Log4J v1 is End Of Life (EOL) and will not receive patches for this issue. Log4J v1 is also vulnerable to other RCE vectors and we recommend you migrate to Log4J 2.16.0 where possible.\n\n", 11 | "severity": [ 12 | { 13 | "type": "CVSS_V3", 14 | "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H" 15 | } 16 | ], 17 | "affected": [ 18 | { 19 | "package": { 20 | "ecosystem": "Maven", 21 | "name": "org.apache.logging.log4j:log4j-core" 22 | }, 23 | "ranges": [ 24 | { 25 | "type": "ECOSYSTEM", 26 | "events": [ 27 | { 28 | "introduced": "2.13.0" 29 | }, 30 | { 31 | "fixed": "2.15.0" 32 | } 33 | ] 34 | } 35 | ] 36 | }, 37 | { 38 | "package": { 39 | "ecosystem": "Maven", 40 | "name": "org.apache.logging.log4j:log4j-core" 41 | }, 42 | "ranges": [ 43 | { 44 | "type": "ECOSYSTEM", 45 | "events": [ 46 | { 47 | "introduced": "0" 48 | }, 49 | { 50 | "fixed": "2.3.1" 51 | } 52 | ] 53 | } 54 | ] 55 | }, 56 | { 57 | "package": { 58 | "ecosystem": "Maven", 59 | "name": "org.apache.logging.log4j:log4j-core" 60 | }, 61 | "ranges": [ 62 | { 63 | "type": "ECOSYSTEM", 64 | "events": [ 65 | { 66 | "introduced": "2.4" 67 | }, 68 | { 69 | "fixed": "2.12.2" 70 | } 71 | ] 72 | } 73 | ] 74 | } 75 | ], 76 | "references": [ 77 | { 78 | "type": "ADVISORY", 79 | "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-44228" 80 | }, 81 | { 82 | "type": "WEB", 83 | "url": "https://github.com/apache/logging-log4j2/pull/608" 84 | } 85 | ], 86 | "database_specific": { 87 | "cwe_ids": [ 88 | "CWE-20", 89 | "CWE-400", 90 | "CWE-502", 91 | "CWE-917" 92 | ], 93 | "severity": "CRITICAL", 94 | "github_reviewed": true, 95 | "github_reviewed_at": "2021-12-10T00:40:41Z", 96 | "nvd_published_at": "2021-12-10T10:15:00Z" 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /examples/clearlydefined.py: -------------------------------------------------------------------------------- 1 | 2 | from ghastoolkit.octokit.clearlydefined import ClearlyDefined 3 | from ghastoolkit.supplychain.dependencies import Dependency 4 | 5 | dependency = Dependency("requests", manager="pypi") 6 | 7 | clearly = ClearlyDefined() 8 | 9 | licenses = clearly.getLicenses(dependency) 10 | print(licenses) 11 | 12 | -------------------------------------------------------------------------------- /examples/codeql-databases.py: -------------------------------------------------------------------------------- 1 | """CodeQL Database examples.""" 2 | import os 3 | from ghastoolkit import CodeQLDatabase, CodeQLDatabases, GitHub 4 | 5 | GitHub.init("GeekMasher/ghastoolkit") 6 | 7 | path = os.path.expanduser("~/.codeql/databases/") 8 | os.makedirs(path, exist_ok=True) # create if not present 9 | 10 | codeqldb = CodeQLDatabase("ghastoolkit", "python", GitHub.repository) 11 | print(f"- {codeqldb} (existis: {codeqldb.exists()})") 12 | print(f" [{codeqldb.path}]") 13 | 14 | if not codeqldb.exists(): 15 | codeqldb.downloadDatabase(codeqldb.path) 16 | 17 | 18 | # Load All Local Databases 19 | local_databases = CodeQLDatabases() 20 | local_databases.findDatabases(path) 21 | 22 | print("\nAll Local Databases:") 23 | for database in local_databases: 24 | print(f" - {database}") 25 | 26 | 27 | # Load Remote Database 28 | remote_databases = CodeQLDatabases.loadRemoteDatabases(GitHub.repository) 29 | # remote_databases.downloadDatabases() 30 | 31 | print("\nAll Remote Databases:") 32 | for database in remote_databases: 33 | print(f" - {database}") 34 | -------------------------------------------------------------------------------- /examples/codeql-packs.py: -------------------------------------------------------------------------------- 1 | """CodeQL Packs examples.""" 2 | from ghastoolkit import CodeQLPack 3 | 4 | # download 5 | pack = CodeQLPack.download("codeql/python-queries", "0.7.4") 6 | print(f"Remote Pack :: {pack} ({pack.path})") 7 | 8 | # local loading 9 | pack = CodeQLPack("./examples/packs") 10 | print(f"Local Pack :: {pack}") 11 | 12 | # install dependencies 13 | pack.install(True) 14 | 15 | path = pack.create() 16 | print(f"Pack Path :: {path}") 17 | 18 | queries = pack.resolveQueries() 19 | print("") 20 | print(f"# Queries :: {len(queries)}") 21 | for query in queries: 22 | print(f" - {query}") 23 | -------------------------------------------------------------------------------- /examples/codeql.py: -------------------------------------------------------------------------------- 1 | """CodeQL Example.""" 2 | import os 3 | from ghastoolkit import CodeQL, CodeQLDatabase 4 | from ghastoolkit.octokit.github import GitHub 5 | 6 | GitHub.init(os.environ.get("GITHUB_REPOSITORY", "GeekMasher/ghastoolkit")) 7 | 8 | codeql = CodeQL() 9 | print(codeql) 10 | 11 | 12 | db = CodeQLDatabase("ghastoolkit", "python", GitHub.repository) 13 | codeql.createDatabase(db, display=True) 14 | 15 | print("") 16 | if not db: 17 | print("Failed to load Database...") 18 | exit(1) 19 | 20 | print(f"Database :: {db} ({db.path})") 21 | if not db.exists(): 22 | print("Downloading database...") 23 | db.downloadDatabase() 24 | 25 | results = codeql.runQuery(db, "security-extended", display=True) 26 | 27 | print(f"\nResults: {len(results)}\n") 28 | 29 | for result in results: 30 | print(f"- {result}") 31 | -------------------------------------------------------------------------------- /examples/codescanning.py: -------------------------------------------------------------------------------- 1 | """Example showing how to connect and use the Code Scanning API. 2 | """ 3 | 4 | import os 5 | from ghastoolkit import GitHub, CodeScanning 6 | 7 | GitHub.init( 8 | os.environ.get("GITHUB_REPOSITORY", "GeekMasher/ghastoolkit"), 9 | reference=os.environ.get("GITHUB_REF", "refs/heads/main"), 10 | ) 11 | 12 | cs = CodeScanning() 13 | 14 | print(f"Repository :: {GitHub.repository}") 15 | 16 | # requires "Repository Administration" repository permissions (read) 17 | if not cs.isEnabled(): 18 | print("Code Scanning is not enabled :(") 19 | exit() 20 | 21 | if GitHub.repository.isInPullRequest(): 22 | # Get list of the delta alerts in a PR 23 | print(f"Alerts from PR :: {GitHub.repository.getPullRequestNumber()}") 24 | alerts = cs.getAlertsInPR(GitHub.repository.reference or "") 25 | 26 | else: 27 | # Get all alerts 28 | print("Alerts from default Branch") 29 | alerts = cs.getAlerts("open") 30 | 31 | print(f"Alert Count :: {len(alerts)}") 32 | 33 | for alert in alerts: 34 | print(f" >> {alert} ({alert.severity})") 35 | 36 | # get latest analyses for each tool 37 | analyses = cs.getLatestAnalyses() 38 | print(f"Analyses :: {len(analyses)}") 39 | 40 | # get list of tools 41 | tools = cs.getTools() 42 | print(f"Tools :: {tools}") 43 | -------------------------------------------------------------------------------- /examples/dependabot.py: -------------------------------------------------------------------------------- 1 | """Dependabot API example.""" 2 | import os 3 | from ghastoolkit import Dependabot, GitHub 4 | 5 | GitHub.init( 6 | os.environ.get("GITHUB_REPOSITORY", "GeekMasher/ghastoolkit"), 7 | reference=os.environ.get("GITHUB_REF", "refs/heads/main"), 8 | ) 9 | 10 | dependabot = Dependabot() 11 | 12 | if not dependabot.isEnabled(): 13 | print("Dependabot is not enabled") 14 | exit(1) 15 | 16 | if GitHub.repository.isInPullRequest(): 17 | print("Dependabot Alerts from Pull Request") 18 | alerts = dependabot.getAlertsInPR() 19 | else: 20 | print("Dependabot Alerts from Repository") 21 | alerts = dependabot.getAlerts() 22 | 23 | print(f"Total Alerts :: {len(alerts)}") 24 | 25 | for alert in alerts: 26 | print(f"Alert :: {alert}") 27 | -------------------------------------------------------------------------------- /examples/dependencies-org.py: -------------------------------------------------------------------------------- 1 | """Example fetching all of the dependencies for an organization. 2 | 3 | This example also caches the results to a local folder. 4 | """ 5 | 6 | import os 7 | import logging 8 | from ghastoolkit import GitHub, DependencyGraph 9 | from ghastoolkit.utils.cache import Cache, CACHE_WEEK 10 | 11 | # Set the logging for ghastoolkit 12 | ghastoolkit_logger = logging.getLogger("ghastoolkit") 13 | ghastoolkit_logger.setLevel(logging.DEBUG) 14 | # Set the cache to be for a week 15 | Cache.cache_age = CACHE_WEEK 16 | 17 | # Initialize GitHub with the token 18 | GitHub.init("GeekMasherOrg", token=os.environ.get("GITHUB_TOKEN")) 19 | 20 | print(f"GitHub :: {GitHub}") 21 | print(f"Owner :: {GitHub.owner}") 22 | print(f"Token :: {GitHub.getToken()} ({GitHub.token_type})") 23 | 24 | # GraphQL is required to get the full dependencies details 25 | depgraph = DependencyGraph(cache=True, enable_graphql=True) 26 | print(f"Cache :: {depgraph.cache.cache_path}") 27 | 28 | # Get the list of unique dependencies (ignoring versions) 29 | unique = depgraph.getUniqueOrgDependencies(version=False) 30 | 31 | print(f"Unique dependencies: {len(unique)}") 32 | 33 | direct = unique.findDirectDependencies() 34 | print(f"Direct dependencies: {len(direct)}\n") 35 | 36 | for dep in direct: 37 | # Every dependencies has a list of repositories that use it 38 | print(f"{dep}") 39 | for r in dep.repositories: 40 | print(f"\t> {r}") 41 | -------------------------------------------------------------------------------- /examples/dependencies.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from ghastoolkit import DependencyGraph, GitHub 4 | 5 | GitHub.init(repository=os.environ.get("GITHUB_REPOSITORY", "GeekMasher/ghastoolkit")) 6 | print(f"Repository :: {GitHub.repository}") 7 | 8 | depgraph = DependencyGraph() 9 | dependencies = depgraph.getDependencies() 10 | 11 | print(f"Total Dependencies :: {len(dependencies)}") 12 | 13 | # or you can get the data from the GraphQL API as well 14 | # This can be useful if you want to get more information about the dependencies 15 | dependencies = depgraph.getDependenciesGraphQL() 16 | print(f"Total Dependencies (GraphQL) :: {len(dependencies)}") 17 | 18 | gpl = dependencies.findLicenses(["GPL-*", "AGPL-*"]) 19 | print(f"Total GPL Dependencies :: {len(gpl)}") 20 | 21 | print("Downloaded SBOM...") 22 | path = "./sbom.spdx" 23 | sbom = depgraph.exportBOM() 24 | with open(path, "w") as handle: 25 | json.dump(sbom, handle, indent=2) 26 | 27 | print(f"Stored SBOM :: {path}") 28 | -------------------------------------------------------------------------------- /examples/github.py: -------------------------------------------------------------------------------- 1 | """GitHub Instance example.""" 2 | 3 | from ghastoolkit import GitHub 4 | 5 | GitHub.init( 6 | repository="GeekMasher/ghastoolkit", 7 | ) 8 | 9 | print(f"GitHub Repository :: {GitHub.repository}") 10 | 11 | # By default, the GitHub instance is set to https://github.com 12 | print(f"GitHub Instance :: {GitHub.instance}") 13 | print(f"GitHub API REST :: {GitHub.api_rest}") 14 | print(f"GitHub API GraphQL :: {GitHub.api_graphql}") 15 | 16 | # Initialise the GitHub will also load defaults values for access GitHub 17 | # resources 18 | if token := GitHub.getToken(): 19 | # Only show the first 5 characters of the token 20 | print(f"GitHub Token :: {token} [masked for security]") 21 | else: 22 | print("GitHub Token :: Not set") 23 | -------------------------------------------------------------------------------- /examples/licenses.py: -------------------------------------------------------------------------------- 1 | """Licenses using custom data.""" 2 | from ghastoolkit import Licenses 3 | 4 | # load custom license data 5 | licenses = Licenses() 6 | print(f" >> {len(licenses)}") 7 | 8 | # find dependency 9 | result = licenses.find("com.google.guava/guava") 10 | print(f"License :: {result}") 11 | -------------------------------------------------------------------------------- /examples/packs/codeql-pack.lock.yml: -------------------------------------------------------------------------------- 1 | --- 2 | lockVersion: 1.0.0 3 | dependencies: 4 | codeql/python-all: 5 | version: 0.9.4 6 | codeql/regex: 7 | version: 0.0.15 8 | codeql/tutorial: 9 | version: 0.0.12 10 | codeql/util: 11 | version: 0.0.12 12 | codeql/yaml: 13 | version: 0.0.4 14 | compiled: false 15 | -------------------------------------------------------------------------------- /examples/packs/qlpack.yml: -------------------------------------------------------------------------------- 1 | --- 2 | library: false 3 | name: geekmasher/ghastoolkit-python 4 | version: 0.1.0 5 | dependencies: 6 | codeql/python-all: "^0.9.2" 7 | defaultSuiteFile: default.qls 8 | 9 | -------------------------------------------------------------------------------- /examples/repository.py: -------------------------------------------------------------------------------- 1 | 2 | from ghastoolkit import GitHub 3 | 4 | GitHub.init("GeekMasher/ghastoolkit") 5 | # repository from the loading GitHub 6 | repository = GitHub.repository 7 | 8 | # clone repo 9 | repository.clone() 10 | print(f" >> {repository.clone_path}") 11 | 12 | # get a file from the repo 13 | path = repository.getFile("README.md") 14 | print(f" >> {path}") 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /examples/secrets.py: -------------------------------------------------------------------------------- 1 | """Secret Scanning API example.""" 2 | 3 | import os 4 | from ghastoolkit import GitHub, SecretScanning 5 | 6 | GitHub.init(os.environ.get("GITHUB_REPOSITORY", "GeekMasher/ghastoolkit")) 7 | 8 | # Setup Secret Scanning 9 | secret_scanning = SecretScanning() 10 | 11 | if not secret_scanning.isEnabled(): 12 | print("Secret Scanning is disabled :(") 13 | exit(1) 14 | 15 | try: 16 | alerts = secret_scanning.getAlerts("open") 17 | except: 18 | print("[!] Error getting alerts, check access") 19 | exit(0) 20 | 21 | print(f"Alert Count :: {len(alerts)}") 22 | 23 | # Display Secrets 24 | for alert in alerts: 25 | print(f"- Secret :: {alert}") 26 | # locations 27 | for loc in alert.locations: 28 | print(f" >> {loc}") 29 | 30 | print("Finished!") 31 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "ghastoolkit" 3 | version = "0.17.6" 4 | authors = [{ name = "GeekMasher" }] 5 | description = "GitHub Advanced Security Python Toolkit" 6 | readme = "README.md" 7 | requires-python = ">=3.10" 8 | classifiers = [ 9 | "Development Status :: 4 - Beta", 10 | "License :: OSI Approved :: MIT License", 11 | "Programming Language :: Python :: 3", 12 | "Programming Language :: Python :: 3.10", 13 | "Programming Language :: Python :: 3.11", 14 | "Programming Language :: Python :: 3.12", 15 | "Programming Language :: Python :: 3.13", 16 | "Operating System :: OS Independent", 17 | ] 18 | 19 | dependencies = [ 20 | "pyyaml>=6.0.2", 21 | "ratelimit>=2.2.1", 22 | "requests>=2.32.3", 23 | "semantic-version>=2.10.0", 24 | ] 25 | 26 | [dependency-groups] 27 | dev = [ 28 | "black>=25.1.0", 29 | "myst-parser>=4.0.1", 30 | "responses>=0.25.7", 31 | "sphinx==7.4.7", 32 | "typing-extensions>=4.12.2", 33 | ] 34 | 35 | [project.urls] 36 | "Homepage" = "https://github.com/GeekMasher/ghastoolkit" 37 | "Bug Tracker" = "https://github.com/GeekMasher/ghastoolkit/issues" 38 | -------------------------------------------------------------------------------- /src/ghastoolkit/__init__.py: -------------------------------------------------------------------------------- 1 | """GitHub Advanced Security Toolkit.""" 2 | 3 | __name__ = "ghastoolkit" 4 | __title__ = "GHAS Toolkit" 5 | 6 | __version__ = "0.17.6" 7 | 8 | __description__ = "GitHub Advanced Security Python Toolkit" 9 | __summary__ = """\ 10 | GitHub Advanced Security Python Toolkit 11 | """ 12 | 13 | __url__ = "https://github.com/GeekMasher/ghastoolkit" 14 | 15 | __license__ = "MIT License" 16 | __copyright__ = "Copyright (c) 2023, GeekMasher" 17 | 18 | __author__ = "GeekMasher" 19 | 20 | __banner__ = f"""\ 21 | _____ _ _ ___ _____ _____ _ _ _ _ 22 | | __ \\| | | | / _ \\ / ___|_ _| | | | (_) | 23 | | | \\/| |_| |/ /_\\ \\\\ `--. | | ___ ___ | | | ___| |_ 24 | | | __ | _ || _ | `--. \\ | |/ _ \\ / _ \\| | |/ / | __| 25 | | |_\\ \\| | | || | | |/\\__/ / | | (_) | (_) | | <| | |_ 26 | \\____/\\_| |_/\\_| |_/\\____/ \\_/\\___/ \\___/|_|_|\\_\\_|\\__| v{__version__} by {__author__} 27 | """ 28 | 29 | 30 | from ghastoolkit.errors import * 31 | 32 | # Octokit 33 | from ghastoolkit.octokit.github import GitHub 34 | from ghastoolkit.octokit.repository import Repository 35 | from ghastoolkit.octokit.enterprise import Enterprise, Organization 36 | from ghastoolkit.octokit.octokit import Octokit, RestRequest, GraphQLRequest 37 | from ghastoolkit.octokit.codescanning import CodeScanning, CodeAlert 38 | from ghastoolkit.octokit.secretscanning import SecretScanning, SecretAlert 39 | from ghastoolkit.octokit.dependencygraph import DependencyGraph 40 | from ghastoolkit.octokit.dependabot import Dependabot 41 | from ghastoolkit.octokit.advisories import SecurityAdvisories 42 | 43 | # Supply Chain 44 | from ghastoolkit.supplychain.advisories import Advisory, Advisories 45 | from ghastoolkit.supplychain.dependencyalert import DependencyAlert 46 | from ghastoolkit.supplychain.dependencies import Dependencies 47 | from ghastoolkit.supplychain.dependency import Dependency 48 | from ghastoolkit.supplychain.licensing import Licenses 49 | 50 | # CodeQL 51 | from ghastoolkit.codeql.databases import CodeQLDatabases, CodeQLDatabase 52 | from ghastoolkit.codeql.cli import CodeQL 53 | from ghastoolkit.codeql.packs.pack import CodeQLPack 54 | from ghastoolkit.codeql.packs.packs import CodeQLPacks 55 | from ghastoolkit.codeql.results import CodeQLResults, CodeLocation, CodeResult 56 | 57 | # CodeQL Data Extensions / Models as Data 58 | from ghastoolkit.codeql.dataextensions.ext import DataExtensions 59 | -------------------------------------------------------------------------------- /src/ghastoolkit/__main__.py: -------------------------------------------------------------------------------- 1 | """ghastoolkit main workflow.""" 2 | 3 | from argparse import Namespace 4 | import logging 5 | 6 | from ghastoolkit import __name__ as name, __banner__, __version__ 7 | from ghastoolkit.octokit.codescanning import CodeScanning 8 | from ghastoolkit.octokit.dependencygraph import DependencyGraph 9 | from ghastoolkit.octokit.github import GitHub 10 | from ghastoolkit.utils.cli import CommandLine 11 | from ghastoolkit.supplychain.__main__ import ( 12 | runDefault as runSCDefault, 13 | runOrgAudit as runSCOrgAudit, 14 | ) 15 | 16 | 17 | def header(name: str, width: int = 32): 18 | logging.info("#" * width) 19 | logging.info(f"{name:^32}") 20 | logging.info("#" * width) 21 | logging.info("") 22 | 23 | 24 | def runCodeScanning(arguments): 25 | codescanning = CodeScanning(GitHub.repository) 26 | 27 | alerts = codescanning.getAlerts() 28 | 29 | logging.info(f"Total Alerts :: {len(alerts)}") 30 | 31 | analyses = codescanning.getLatestAnalyses(GitHub.repository.reference) 32 | logging.info(f"\nTools:") 33 | 34 | for analyse in analyses: 35 | tool = analyse.get("tool", {}).get("name") 36 | version = analyse.get("tool", {}).get("version") 37 | created_at = analyse.get("created_at") 38 | 39 | logging.info(f" - {tool} v{version} ({created_at})") 40 | 41 | 42 | class MainCli(CommandLine): 43 | """Main CLI.""" 44 | 45 | def arguments(self): 46 | """Adding additional parsers from submodules.""" 47 | self.addModes(["all"]) 48 | 49 | def run(self, arguments: Namespace): 50 | """Run main CLI.""" 51 | if arguments.version: 52 | logging.info(f"v{__version__}") 53 | return 54 | 55 | print(__banner__) 56 | 57 | if not arguments.token or arguments.token == "": 58 | logging.error("Missing GitHub token.") 59 | return 60 | 61 | if arguments.mode in ["all", "codescanning"]: 62 | logging.info("") 63 | header("Code Scanning") 64 | runCodeScanning(arguments) 65 | 66 | if arguments.mode in ["all", "dependencygraph"]: 67 | logging.info("") 68 | header("Dependency Graph") 69 | runSCDefault(arguments) 70 | 71 | if arguments.mode == "org-audit": 72 | # run org audit with all products 73 | # supplychain 74 | runSCOrgAudit(arguments) 75 | return 76 | 77 | 78 | if __name__ == "__main__": 79 | # Arguments 80 | parser = MainCli(name) 81 | 82 | parser.run(parser.parse_args()) 83 | -------------------------------------------------------------------------------- /src/ghastoolkit/billing/__main__.py: -------------------------------------------------------------------------------- 1 | """CodeQL CLI for ghastoolkit.""" 2 | 3 | import csv 4 | import logging 5 | from argparse import Namespace 6 | from typing import List 7 | from ghastoolkit.octokit.github import GitHub 8 | from ghastoolkit.octokit.enterprise import Organization 9 | from ghastoolkit.octokit.billing import Billing 10 | from ghastoolkit.utils.cli import CommandLine 11 | 12 | logger = logging.getLogger("ghastoolkit-billing") 13 | 14 | 15 | class CostCenter: 16 | """Cost Center.""" 17 | 18 | def __init__(self, name: str, repositories: list[str] = []) -> None: 19 | """Initialize Cost Center.""" 20 | self.name = name 21 | self.repositories = set(repositories) 22 | 23 | def addRepository(self, repo: str): 24 | """Add a Repository.""" 25 | self.repositories.add(repo) 26 | 27 | 28 | def loadCostCenterCsv(path: str) -> List[CostCenter]: 29 | cost_centers = {} 30 | 31 | with open(path, "r") as csv_file: 32 | csv_reader = csv.DictReader(csv_file) 33 | 34 | for row in csv_reader: 35 | cost_center = row["Cost Center"] 36 | repo = row["Repository"] 37 | 38 | if cost_centers.get(cost_center): 39 | cost_centers[cost_center].addRepository(repo) 40 | else: 41 | cost_centers[cost_center] = CostCenter(cost_center, [repo]) 42 | 43 | return cost_centers.values() 44 | 45 | 46 | class BillingCommandLine(CommandLine): 47 | """Billing CLI.""" 48 | 49 | def arguments(self): 50 | """Billing arguments.""" 51 | if self.subparser: 52 | # self.addModes([""]) 53 | 54 | parser = self.parser.add_argument_group("billing") 55 | parser.add_argument( 56 | "--csv", 57 | help="Input CSV Billing File", 58 | ) 59 | parser.add_argument( 60 | "--cost-center", 61 | help="Cost Center CSV File", 62 | ) 63 | 64 | def run(self, arguments: Namespace): 65 | self.default_logger() 66 | 67 | org = Organization(GitHub.owner) 68 | 69 | if arguments.csv: 70 | logging.info(f"Loading GHAS Billing from {arguments.csv}") 71 | 72 | ghas = Billing.loadFromCsv(arguments.csv) 73 | else: 74 | if GitHub.token is None: 75 | logger.error("No GitHub Token provided") 76 | return 77 | billing = Billing(org) 78 | ghas = billing.getGhasBilling() 79 | 80 | if not ghas: 81 | logger.error("No GHAS Billing found") 82 | return 83 | 84 | print(f"GHAS Active Committers :: {ghas.active}") 85 | print(f"GHAS Maximum Committers :: {ghas.maximum}") 86 | print(f"GHAS Purchased Committers :: {ghas.purchased}") 87 | 88 | if arguments.cost_center: 89 | cost_centers = loadCostCenterCsv(arguments.cost_center) 90 | print(f"\nCost Centers :: {len(cost_centers)}\n") 91 | total = 0 92 | 93 | for center in cost_centers: 94 | active = set() 95 | repos = 0 96 | 97 | for repo in center.repositories: 98 | r = ghas.getRepository(repo, org.name) 99 | if r: 100 | repos += 1 101 | active.update(r.activeCommitterNames()) 102 | else: 103 | logger.warning(f"Repository cost center not found: {repo}") 104 | 105 | print(f" > {center.name} (active: {len(active)}, repos: {repos})") 106 | total += len(active) 107 | 108 | print(f"\nShared Cost Center Licenses :: {total - ghas.active}") 109 | 110 | 111 | if __name__ == "__main__": 112 | parser = BillingCommandLine("ghastoolkit-billing") 113 | parser.run(parser.parse_args()) 114 | logging.info(f"Finished!") 115 | -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GeekMasher/ghastoolkit/6d1ce533eebc886d40a580c357776a47fd87d138/src/ghastoolkit/codeql/__init__.py -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/__main__.py: -------------------------------------------------------------------------------- 1 | """CodeQL CLI for ghastoolkit.""" 2 | 3 | import logging 4 | from argparse import Namespace 5 | from ghastoolkit.codeql.cli import CodeQL 6 | from ghastoolkit.utils.cli import CommandLine 7 | 8 | 9 | class CodeQLCommandLine(CommandLine): 10 | """CodeQL CLI.""" 11 | 12 | def arguments(self): 13 | """CodeQL arguments.""" 14 | if self.subparser: 15 | self.addModes(["init", "analyze", "update"]) 16 | 17 | parser = self.parser.add_argument_group("codeql") 18 | parser.add_argument("-b", "--binary") 19 | parser.add_argument("-c", "--command", type=str) 20 | 21 | def run(self, arguments: Namespace): 22 | codeql = CodeQL() 23 | 24 | if not codeql.exists(): 25 | logging.error(f"Failed to find codeql on system") 26 | exit(1) 27 | 28 | logging.debug(f"Found codeql on system :: '{' '.join(codeql.path_binary)}'") 29 | 30 | if arguments.version: 31 | logging.info(f"CodeQL Version :: v{codeql.version}") 32 | exit(0) 33 | 34 | 35 | if __name__ == "__main__": 36 | parser = CodeQLCommandLine("ghastoolkit-codeql") 37 | parser.run(parser.parse_args()) 38 | logging.info(f"Finished!") 39 | -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/consts.py: -------------------------------------------------------------------------------- 1 | # Dict of CodeQL supported Languages 2 | CODEQL_LANGUAGES = { 3 | "actions": "actions", 4 | "c": "cpp", 5 | "cpp": "cpp", 6 | "csharp": "csharp", 7 | "java": "java", 8 | "kotlin": "java", 9 | "javascript": "javascript", 10 | "typescript": "javascript", 11 | "go": "go", 12 | "python": "python", 13 | "rust": "rust", 14 | "swift": "swift", 15 | "ruby": "ruby", 16 | } 17 | -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/dataextensions/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/dataextensions/__main__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import argparse 4 | from ghastoolkit.codeql.dataextensions.ext import DataExtensions 5 | 6 | logging.basicConfig(format="%(message)s") 7 | parser = argparse.ArgumentParser("ghastoolkit-codeql-dataextensions") 8 | parser.add_argument("-l", "--language", required=True) 9 | parser.add_argument("-i", "--input", required=True) 10 | 11 | args = parser.parse_args() 12 | 13 | de = DataExtensions(args.language) 14 | 15 | if os.path.isfile(args.input): 16 | de.load(args.input) 17 | elif os.path.isdir(args.input): 18 | for root, dirs, files in os.walk(args.input): 19 | for fl in files: 20 | path = os.path.join(root, fl) 21 | _, ext = os.path.splitext(fl) 22 | if ext in [".yml", ".yaml"]: 23 | de.load(path) 24 | 25 | logging.info(f" Language :: {args.language} (loaded: {len(de.paths)})") 26 | logging.info(f" Sources :: {len(de.sources)}") 27 | logging.info(f" Sinks :: {len(de.sinks)}") 28 | logging.info(f" Summaries :: {len(de.summaries)}") 29 | logging.info(f" Types :: {len(de.types)}") 30 | logging.info(f" Neutrals :: {len(de.neutrals)}") 31 | -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/dataextensions/ext.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | from typing import List, Optional, Union 4 | from dataclasses import dataclass, field 5 | 6 | import yaml 7 | 8 | from ghastoolkit.codeql.dataextensions.models import ( 9 | CompiledSinks, 10 | CompiledSources, 11 | CompiledSummaries, 12 | CompiledNeutrals, 13 | __MODELES__, 14 | InterpretedSink, 15 | InterpretedSource, 16 | InterpretedSummary, 17 | InterpretedType, 18 | InterpretedTypeVariable, 19 | ) 20 | 21 | LANGUAGE_TYPES = {"csharp": "Compiled", "java": "Compiled", "javascript": "Interpreted"} 22 | 23 | logger = logging.getLogger("ghastoolkit.codeql.dataextensions") 24 | 25 | 26 | @dataclass 27 | class DataExtensions: 28 | language: str 29 | pack: Optional[str] = None 30 | paths: List[str] = field(default_factory=list) 31 | 32 | sources: List[Union[CompiledSources, InterpretedSource]] = field( 33 | default_factory=list 34 | ) 35 | 36 | sinks: List[Union[CompiledSinks, InterpretedSink]] = field(default_factory=list) 37 | 38 | summaries: List[Union[CompiledSummaries, InterpretedSummary]] = field( 39 | default_factory=list 40 | ) 41 | 42 | types: List[Union[InterpretedType, InterpretedTypeVariable]] = field( 43 | default_factory=list 44 | ) 45 | 46 | neutrals: List[CompiledNeutrals] = field(default_factory=list) 47 | 48 | def __post_init__(self): 49 | if not self.pack: 50 | self.pack = f"codeql/{self.language}-queries" 51 | 52 | def load(self, path: str): 53 | if not os.path.exists(path): 54 | raise Exception(f"Path does not exist :: {path}") 55 | logger.debug(f"Loading data extension from path :: '{path}'") 56 | with open(path, "r") as handle: 57 | data = yaml.safe_load(handle) 58 | 59 | language_type = LANGUAGE_TYPES.get(self.language) 60 | 61 | for ext in data.get("extensions"): 62 | extensible = ext.get("addsTo", {}).get("extensible", "") 63 | ext_name = extensible.replace("Model", "") 64 | class_name = f"{language_type}{ext_name.title()}" 65 | clss = __MODELES__.get(class_name) 66 | if not clss: 67 | logger.error(f"Unknown class :: {class_name}") 68 | continue 69 | 70 | for data_ext in ext.get("data", []): 71 | i = clss(*data_ext) 72 | if ext_name == "source": 73 | self.sources.append(i) 74 | elif ext_name == "sink": 75 | self.sinks.append(i) 76 | elif ext_name == "summary": 77 | self.summaries.append(i) 78 | elif ext_name == "neutral": 79 | self.neutrals.append(i) 80 | elif ext_name == "type": 81 | self.types.append(i) 82 | elif ext_name == "typeVariable": 83 | self.types.append(i) 84 | else: 85 | logger.warning(f"Unknown data extension :: {ext_name}") 86 | 87 | self.paths.append(path) 88 | -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/dataextensions/models.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List 3 | 4 | MODELS_AS_DATA = { 5 | "CompiledSources": [ 6 | "package", 7 | "type", 8 | "subtypes", 9 | "name", 10 | "signature", 11 | "ext", 12 | "output", 13 | "kind", 14 | "provenance", 15 | ], 16 | "CompiledSinks": [ 17 | "package", 18 | "type", 19 | "subtypes", 20 | "name", 21 | "signature", 22 | "ext", 23 | "input", 24 | "kind", 25 | "provenance", 26 | ], 27 | "CompiledSummaries": [ 28 | "package", 29 | "type", 30 | "subtypes", 31 | "name", 32 | "signature", 33 | "ext", 34 | "input", 35 | "output", 36 | "kind", 37 | "provenance", 38 | ], 39 | "CompiledNeutrals": ["package", "type", "name", "signature", "kind", "provenance"], 40 | } 41 | 42 | 43 | class ModelAsData: 44 | def generateMad(self, headers: List[str]) -> List[str]: 45 | result = [] 46 | for header in headers: 47 | if hasattr(self, header): 48 | result.append(getattr(self, header)) 49 | elif hasattr(self, f"object_{header}"): 50 | result.append(getattr(self, f"object_{header}")) 51 | return result 52 | 53 | def generate(self) -> List[str]: 54 | return self.generateMad(MODELS_AS_DATA.get(self.__class__.__name__, [])) 55 | 56 | 57 | @dataclass 58 | class CompiledSources(ModelAsData): 59 | """Compile Sources""" 60 | 61 | package: str 62 | object_type: str 63 | subtypes: bool 64 | name: str 65 | signature: str 66 | ext: str 67 | output: str 68 | kind: str 69 | provenance: str = "manual" 70 | 71 | 72 | @dataclass 73 | class CompiledSinks(ModelAsData): 74 | """Compile Sources""" 75 | 76 | package: str 77 | object_type: str 78 | subtypes: bool 79 | name: str 80 | signature: str 81 | ext: str 82 | object_input: str 83 | kind: str 84 | provenance: str = "manual" 85 | 86 | 87 | @dataclass 88 | class CompiledSummaries(ModelAsData): 89 | """Compiled Summaries""" 90 | 91 | package: str 92 | object_type: str 93 | subtypes: bool 94 | name: str 95 | signature: str 96 | ext: str 97 | object_input: str 98 | output: str 99 | kind: str 100 | provenance: str = "manual" 101 | 102 | 103 | @dataclass 104 | class CompiledNeutrals(ModelAsData): 105 | """Compiled Neutrals""" 106 | 107 | package: str 108 | object_type: str 109 | name: str 110 | signature: str 111 | kind: str 112 | provenance: str = "manual" 113 | 114 | 115 | @dataclass 116 | class InterpretedSource(ModelAsData): 117 | """Interpreted Source""" 118 | 119 | object_type: str 120 | path: str 121 | kind: str 122 | 123 | 124 | @dataclass 125 | class InterpretedSink(ModelAsData): 126 | """Interpreted Sink""" 127 | 128 | object_type: str 129 | path: str 130 | kind: str 131 | 132 | 133 | @dataclass 134 | class InterpretedSummary(ModelAsData): 135 | """Interpreted Summary""" 136 | 137 | object_type: str 138 | path: str 139 | object_input: str 140 | output: str 141 | kind: str 142 | 143 | 144 | @dataclass 145 | class InterpretedType(ModelAsData): 146 | """Interpreted Type""" 147 | 148 | object_type1: str 149 | object_type2: str 150 | path: str 151 | 152 | 153 | @dataclass 154 | class InterpretedTypeVariable(ModelAsData): 155 | """Interpreted Type""" 156 | 157 | object_type1: str 158 | object_type2: str 159 | 160 | 161 | __MODELES__ = { 162 | "CompiledSink": CompiledSinks, 163 | "CompiledSource": CompiledSources, 164 | "CompiledSummary": CompiledSummaries, 165 | "CompiledNeutral": CompiledNeutrals, 166 | "InterpretedSource": InterpretedSource, 167 | "InterpretedSink": InterpretedSink, 168 | "InterpretedSummary": InterpretedSummary, 169 | "InterpretedType": InterpretedType, 170 | "InterpretedTypevariable": InterpretedTypeVariable, 171 | } 172 | -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/packs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GeekMasher/ghastoolkit/6d1ce533eebc886d40a580c357776a47fd87d138/src/ghastoolkit/codeql/packs/__init__.py -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/packs/__main__.py: -------------------------------------------------------------------------------- 1 | """GitHub CodeQL Packs CLI.""" 2 | 3 | import os 4 | import logging 5 | from argparse import Namespace 6 | from typing import Optional 7 | 8 | from yaml import parse 9 | from ghastoolkit.codeql.packs.packs import CodeQLPacks 10 | from ghastoolkit.utils.cli import CommandLine 11 | 12 | 13 | def codeqlPackPublish(arguments: Namespace, packs: CodeQLPacks): 14 | if not arguments.packs or arguments.packs == "": 15 | logging.error(f"CodeQL Pack path must be provided") 16 | exit(1) 17 | 18 | for pack in packs: 19 | remote = pack.remote_version 20 | logging.info(f"CodeQL Pack Remote Version :: {remote}") 21 | 22 | if pack.version != remote: 23 | logging.info("Publishing CodeQL Pack...") 24 | pack.publish() 25 | logging.info(f"CodeQL Pack published :: {pack}") 26 | else: 27 | logging.info(f"CodeQL Pack is up to date :: {pack}") 28 | 29 | 30 | class CodeQLPacksCommandLine(CommandLine): 31 | def arguments(self): 32 | self.addModes(["publish", "queries", "compile", "version"]) 33 | default_pack_path = os.path.expanduser("~/.codeql/packages") 34 | 35 | parser = self.parser.add_argument_group("codeql-packs") 36 | parser.add_argument( 37 | "--packs", 38 | type=str, 39 | default=os.environ.get("CODEQL_PACKS_PATH", default_pack_path), 40 | help="CodeQL Packs Path", 41 | ) 42 | parser.add_argument( 43 | "--bump", 44 | type=str, 45 | default="patch", 46 | help="CodeQL Pack Version Bump", 47 | ) 48 | parser.add_argument( 49 | "--suite", 50 | type=str, 51 | default="default", 52 | help="CodeQL Pack Suite", 53 | ) 54 | parser.add_argument( 55 | "--latest", 56 | action="store_true", 57 | help="Update to latest CodeQL Pack Dependencies", 58 | ) 59 | parser.add_argument("--warnings", action="store_true", help="Enable Warnings") 60 | 61 | def run(self, arguments: Optional[Namespace] = None): 62 | if not arguments: 63 | arguments = self.parse_args() 64 | 65 | logging.info(f"CodeQL Packs Path :: {arguments.packs}") 66 | packs = CodeQLPacks(arguments.packs) 67 | 68 | if arguments.latest: 69 | logging.info("Updating CodeQL Pack Dependencies...") 70 | for pack in packs: 71 | pack.updateDependencies() 72 | 73 | if arguments.mode == "publish": 74 | codeqlPackPublish(arguments, packs) 75 | 76 | elif arguments.mode == "version": 77 | logging.info(f"Loading packs from :: {arguments.packs}") 78 | 79 | for pack in packs: 80 | old_version = pack.version 81 | pack.updateVersion(arguments.bump) 82 | pack.updatePack() 83 | logging.info( 84 | f"CodeQL Pack :: {pack.name} :: {old_version} -> {pack.version}" 85 | ) 86 | 87 | elif arguments.mode == "queries": 88 | suite = arguments.suite or "code-scanning" 89 | for pack in packs: 90 | logging.info(f"CodeQL Pack :: {pack}") 91 | 92 | if not pack.library: 93 | if suite == "default" and pack.default_suite: 94 | suite = pack.default_suite 95 | 96 | queries = pack.resolveQueries(suite) 97 | logging.info(f"Queries: {len(queries)}") 98 | for query in queries: 99 | logging.info(f"- {query}") 100 | 101 | elif arguments.mode == "compile": 102 | for pack in packs: 103 | logging.info(f"CodeQL Pack :: {pack}") 104 | 105 | else: 106 | logging.info("CodeQL Packs") 107 | for pack in packs: 108 | logging.info(f"- {pack}") 109 | 110 | for dep in pack.dependencies: 111 | logging.info(f" |-> {dep}") 112 | 113 | 114 | if __name__ == "__main__": 115 | parser = CodeQLPacksCommandLine("ghastoolkit-codeql-packs") 116 | parser.run(parser.parse_args()) 117 | logging.info(f"Finished!") 118 | -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/packs/pack.py: -------------------------------------------------------------------------------- 1 | """CodeQL Packs.""" 2 | 3 | import os 4 | import json 5 | import glob 6 | import logging 7 | from typing import Any, List, Optional 8 | from semantic_version import Version 9 | import yaml 10 | 11 | from ghastoolkit.codeql.cli import CodeQL 12 | 13 | 14 | logger = logging.getLogger("ghastoolkit.codeql.packs") 15 | 16 | 17 | class CodeQLPack: 18 | """CodeQL Pack class.""" 19 | 20 | codeql_packages: str = os.path.join(os.path.expanduser("~"), ".codeql", "packages") 21 | """CodeQL Packages Location""" 22 | 23 | def __init__( 24 | self, 25 | path: Optional[str] = None, 26 | library: Optional[bool] = None, 27 | name: Optional[str] = None, 28 | version: Optional[str] = None, 29 | cli: Optional[CodeQL] = None, 30 | ) -> None: 31 | """Initialise CodeQL Pack.""" 32 | self.cli = cli or CodeQL() 33 | 34 | self.path = path # dir 35 | self.library: bool = library or False 36 | self.name: str = name or "" 37 | self.version: str = version or "0.0.0" 38 | self.dependencies: List["CodeQLPack"] = [] 39 | 40 | self.default_suite: Optional[str] = None 41 | self.warnOnImplicitThis: Optional[bool] = None 42 | self.dbscheme: Optional[str] = None 43 | self.extractor: Optional[str] = None 44 | self.upgrades: Optional[str] = None 45 | self.groups: Optional[list[str]] = None 46 | 47 | if path: 48 | # if its a file 49 | if os.path.isfile(path) and path.endswith("qlpack.yml"): 50 | path = os.path.realpath(os.path.dirname(path)) 51 | 52 | self.path = os.path.realpath(os.path.expanduser(path)) 53 | 54 | if os.path.exists(self.qlpack): 55 | self.load() 56 | 57 | logger.debug(f"Finished loading Pack :: {self}") 58 | 59 | @property 60 | def qlpack(self) -> str: 61 | """QL Pack Location.""" 62 | if self.path: 63 | return os.path.join(self.path, "qlpack.yml") 64 | return "qlpack.yml" 65 | 66 | def validate(self) -> bool: 67 | """Validate and check if the path is a valid CodeQL Pack.""" 68 | return os.path.exists(self.qlpack) 69 | 70 | def load(self): 71 | """Load QLPack file.""" 72 | if not os.path.exists(self.qlpack): 73 | logger.warning(f"Pack Path :: {self.path}") 74 | raise Exception(f"Failed to find qlpack file") 75 | 76 | logger.debug(f"Loading Pack from path :: {self.path}") 77 | with open(self.qlpack, "r") as handle: 78 | data = yaml.safe_load(handle) 79 | 80 | self.library = bool(data.get("library")) 81 | self.name = data.get("name", "") 82 | self.version = data.get("version", "") 83 | self.default_suite = data.get("defaultSuiteFile") 84 | 85 | self.warnOnImplicitThis = data.get("warnOnImplicitThis") 86 | self.dbscheme = data.get("dbscheme") 87 | self.extractor = data.get("extractor") 88 | self.upgrades = data.get("upgrades") 89 | self.groups = data.get("groups") 90 | 91 | for name, version in data.get("dependencies", {}).items(): 92 | self.dependencies.append(CodeQLPack(name=name, version=version)) 93 | 94 | @staticmethod 95 | def findByQuery(query_path: str) -> Optional["CodeQLPack"]: 96 | """Find Pack by query path.""" 97 | stack = query_path.split("/") 98 | if query_path.startswith("/"): 99 | stack.insert(0, "/") 100 | 101 | while len(stack) != 0: 102 | path = os.path.join(*stack, "qlpack.yml") 103 | if os.path.exists(path): 104 | return CodeQLPack(path) 105 | 106 | stack.pop(-1) 107 | return 108 | 109 | def run(self, *args, display: bool = False) -> Optional[str]: 110 | """Run Pack command.""" 111 | return self.cli.runCommand("pack", *args, display=display) 112 | 113 | def create(self) -> str: 114 | """Create / Compile a CodeQL Pack.""" 115 | logger.debug(f"Creating CodeQL Pack :: {self.name}") 116 | home = os.path.expanduser("~") 117 | packages = os.path.join(home, ".codeql", "packages") 118 | self.run("create", "--output", packages, self.path) 119 | return os.path.join(packages, self.name, self.version) 120 | 121 | def publish(self): 122 | """Publish a CodeQL Pack to a remote registry.""" 123 | self.run("publish", self.path) 124 | 125 | @staticmethod 126 | def download(name: str, version: Optional[str] = None) -> "CodeQLPack": 127 | """Download a CodeQL Pack.""" 128 | cli = CodeQL() 129 | full_name = f"{name}@{version}" if version else name 130 | logger.debug(f"Download Pack :: {full_name}") 131 | 132 | cli.runCommand("pack", "download", full_name) 133 | base = os.path.join(CodeQLPack.codeql_packages, name) 134 | if version: 135 | return CodeQLPack(os.path.join(base, version)) 136 | else: 137 | return CodeQLPack(glob.glob(f"{base}/**/")[0]) 138 | 139 | def install(self, display: bool = False): 140 | """Install Dependencies for a CodeQL Pack.""" 141 | self.run("install", self.path, display=display) 142 | 143 | def updateDependencies(self, version: str = "latest"): 144 | for dep in self.dependencies: 145 | if version == "latest": 146 | dep.version = dep.remote_version 147 | self.updatePack() 148 | 149 | def resolveQueries(self, suite: Optional[str] = None) -> List[str]: 150 | """Resolve all the queries in a Pack and return them.""" 151 | results = [] 152 | if self.path: 153 | pack = os.path.join(self.path, suite) if suite else self.path 154 | else: 155 | pack = f"{self.name}:{suite}" if suite else self.name 156 | 157 | result = self.cli.runCommand( 158 | "resolve", "queries", "--format", "bylanguage", pack 159 | ) 160 | if result: 161 | for _, queries in json.loads(result).get("byLanguage", {}).items(): 162 | results.extend(list(queries.keys())) 163 | return results 164 | 165 | @property 166 | def remote_version(self) -> Optional[str]: 167 | """Gets the remote version of the pack if possible.""" 168 | from ghastoolkit import CodeScanning 169 | 170 | try: 171 | cs = CodeScanning() 172 | latest_remote = cs.getLatestPackVersion(self.name) 173 | latest_version = ( 174 | latest_remote.get("metadata", {}) 175 | .get("container", {}) 176 | .get("tags", ["NA"])[0] 177 | ) 178 | return latest_version 179 | except Exception: 180 | logging.debug(f"Error getting remote version") 181 | return None 182 | 183 | def updatePack(self) -> dict[str, Any]: 184 | """Update Local CodeQL Pack.""" 185 | data = { 186 | "library": self.library, 187 | "name": self.name, 188 | "version": self.version, 189 | "defaultSuiteFile": self.default_suite, 190 | "warnOnImplicitThis": self.warnOnImplicitThis, 191 | "dbscheme": self.dbscheme, 192 | "extractor": self.extractor, 193 | "upgrades": self.upgrades, 194 | "groups": self.groups, 195 | } 196 | data = {k: v for k, v in data.items() if v is not None} 197 | 198 | if self.dependencies: 199 | data["dependencies"] = {} 200 | for dep in self.dependencies: 201 | data["dependencies"][dep.name] = dep.version 202 | 203 | if self.path: 204 | logger.debug(f"Saving pack to path :: {self.path}") 205 | with open(self.qlpack, "w") as handle: 206 | yaml.safe_dump(data, handle, sort_keys=False) 207 | 208 | return data 209 | 210 | def updateVersion(self, name: str = "patch", version: Optional[str] = None) -> str: 211 | """Update CodeQL Pack version.""" 212 | if version: 213 | self.version = version 214 | return version 215 | 216 | v = Version(self.version) 217 | if name == "major": 218 | v = v.next_major() 219 | elif name == "minor": 220 | v = v.next_minor() 221 | elif name == "patch": 222 | v = v.next_patch() 223 | self.version = str(v) 224 | return self.version 225 | 226 | def __str__(self) -> str: 227 | """To String.""" 228 | if self.name != "": 229 | return f"CodeQLPack('{self.name}', '{self.version}')" 230 | return f"CodeQLPack('{self.path}')" 231 | -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/packs/packs.py: -------------------------------------------------------------------------------- 1 | """CodeQL Packs.""" 2 | 3 | import os 4 | import logging 5 | from typing import List, Optional 6 | 7 | from ghastoolkit.codeql.packs.pack import CodeQLPack 8 | 9 | 10 | logger = logging.getLogger("ghastoolkit.codeql.packs") 11 | 12 | 13 | class CodeQLPacks: 14 | """CodeQL List of Packs.""" 15 | 16 | def __init__(self, path: Optional[str] = None) -> None: 17 | """Initialise CodeQLPacks.""" 18 | self.packs: List[CodeQLPack] = [] 19 | 20 | if path: 21 | self.load(os.path.realpath(os.path.expanduser(path))) 22 | 23 | def append(self, pack: CodeQLPack): 24 | """Append a CodeQLPack.""" 25 | self.packs.append(pack) 26 | 27 | def load(self, path: str): 28 | """Load packs from path.""" 29 | if not os.path.exists(path): 30 | raise Exception("Path does not exist") 31 | 32 | logger.debug(f"Loading from path :: {path}") 33 | lib_path = os.path.join(".codeql", "libraries") 34 | 35 | for root, _, files in os.walk(path): 36 | for file in files: 37 | if file == "qlpack.yml": 38 | fpath = os.path.join(root, file) 39 | 40 | if lib_path in fpath: 41 | continue 42 | self.append(CodeQLPack(fpath)) 43 | 44 | def __iter__(self): 45 | return self.packs.__iter__() 46 | 47 | def __len__(self) -> int: 48 | """Get length / amount of loaded packs.""" 49 | return len(self.packs) 50 | 51 | def __str__(self) -> str: 52 | """To String.""" 53 | return f"CodeQLPacks('{len(self)}')" 54 | -------------------------------------------------------------------------------- /src/ghastoolkit/codeql/results.py: -------------------------------------------------------------------------------- 1 | """CodeQL Results.""" 2 | 3 | from dataclasses import dataclass, field 4 | from typing import Optional 5 | 6 | 7 | @dataclass 8 | class CodeLocation: 9 | """Code Location Module.""" 10 | 11 | uri: str 12 | """URI to the location where the result occurs""" 13 | 14 | start_line: int 15 | """Start line of the result""" 16 | start_column: Optional[int] = None 17 | """Start column of the result""" 18 | end_line: Optional[int] = None 19 | """End line of the result""" 20 | end_column: Optional[int] = None 21 | """End line of the result""" 22 | 23 | def __str__(self) -> str: 24 | """To String.""" 25 | return f"{self.uri}#{self.start_line}" 26 | 27 | 28 | @dataclass 29 | class CodeResult: 30 | """Code Result.""" 31 | 32 | rule_id: str 33 | """Rule ID""" 34 | message: str 35 | """Message of the result""" 36 | 37 | locations: list[CodeLocation] = field(default_factory=list) 38 | """Locations of the results""" 39 | 40 | def __str__(self) -> str: 41 | """To String.""" 42 | if len(self.locations) == 1: 43 | return f"CodeResult('{self.rule_id}', '{self.locations[0]}')" 44 | return f"CodeResult('{self.rule_id}', {len(self.locations)})" 45 | 46 | @staticmethod 47 | def loadSarifLocations(data: list[dict]) -> list["CodeLocation"]: 48 | """Load SARIF Locations.""" 49 | locations = [] 50 | for loc in data: 51 | physical = loc.get("physicalLocation", {}) 52 | region = physical.get("region", {}) 53 | locations.append( 54 | CodeLocation( 55 | physical.get("artifactLocation", {}).get("uri", ""), 56 | start_line=region.get("startLine", "0"), 57 | start_column=region.get("startColumn"), 58 | end_line=region.get("endLine"), 59 | end_column=region.get("endColumn"), 60 | ) 61 | ) 62 | return locations 63 | 64 | 65 | class CodeQLResults(list): 66 | """CodeQL Results.""" 67 | 68 | @staticmethod 69 | def loadSarifResults(results: list[dict]) -> "CodeQLResults": 70 | """Load SARIF Results.""" 71 | result = CodeQLResults() 72 | 73 | for alert in results: 74 | result.append( 75 | CodeResult( 76 | alert.get("ruleId", "NA"), 77 | alert.get("message", {}).get("text", "NA"), 78 | locations=CodeResult.loadSarifLocations(alert.get("locations", [])), 79 | ) 80 | ) 81 | 82 | return result 83 | -------------------------------------------------------------------------------- /src/ghastoolkit/errors.py: -------------------------------------------------------------------------------- 1 | # GHASToolkit Errors 2 | 3 | 4 | from typing import List, Optional 5 | 6 | 7 | class GHASToolkitError(Exception): 8 | """Base class for GHASToolkit errors.""" 9 | 10 | def __init__( 11 | self, 12 | message: Optional[str] = None, 13 | docs: Optional[str] = None, 14 | permissions: Optional[List[str]] = [], 15 | status: Optional[int] = None, 16 | ) -> None: 17 | self.message = message 18 | self.docs = docs 19 | self.permissions = permissions 20 | self.status = status 21 | 22 | super().__init__(message) 23 | 24 | def __str__(self) -> str: 25 | msg = "" 26 | 27 | if hasattr(self, "message"): 28 | msg = self.message 29 | else: 30 | msg = "An error occurred" 31 | 32 | if status := self.status: 33 | msg += f" (status code: {status})" 34 | 35 | if permissions := self.permissions: 36 | msg += "\n\nPermissions Required:" 37 | for perm in permissions: 38 | msg += f"\n- {perm}" 39 | if docs := self.docs: 40 | msg += f"\n\nFor more information, see: {docs}" 41 | 42 | return msg 43 | 44 | 45 | class GHASToolkitTypeError(GHASToolkitError): 46 | """Raised when an invalid type is passed.""" 47 | 48 | 49 | class GHASToolkitAuthenticationError(GHASToolkitError): 50 | """Raised when an authentication error occurs.""" 51 | -------------------------------------------------------------------------------- /src/ghastoolkit/octokit/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/ghastoolkit/octokit/advisories.py: -------------------------------------------------------------------------------- 1 | """GitHub Security Advisories API.""" 2 | 3 | from typing import Dict, Optional 4 | from ghastoolkit.errors import GHASToolkitError, GHASToolkitTypeError 5 | from ghastoolkit.octokit.github import GitHub, Repository 6 | from ghastoolkit.octokit.octokit import RestRequest 7 | from ghastoolkit.supplychain.advisories import Advisories, Advisory, AdvisoryAffect 8 | 9 | 10 | class SecurityAdvisories: 11 | """Security Advisories.""" 12 | 13 | def __init__(self, repository: Optional[Repository] = None) -> None: 14 | """Security Advisories REST API. 15 | 16 | https://docs.github.com/en/rest/security-advisories/repository-advisories 17 | """ 18 | self.repository = repository or GitHub.repository 19 | if not self.repository: 20 | raise Exception("SecurityAdvisories requires Repository to be set") 21 | self.rest = RestRequest(self.repository) 22 | 23 | def getAdvisories(self) -> Advisories: 24 | """Get list of security advisories from a repository. 25 | 26 | https://docs.github.com/en/rest/security-advisories/repository-advisories#list-repository-security-advisories 27 | """ 28 | results = self.rest.get( 29 | "/repos/{owner}/{repo}/security-advisories", authenticated=True 30 | ) 31 | if isinstance(results, list): 32 | advisories = Advisories() 33 | for advisory in results: 34 | advisories.append(self.loadAdvisoryData(advisory)) 35 | return advisories 36 | 37 | raise GHASToolkitTypeError( 38 | f"Error getting advisories from repository", 39 | docs="https://docs.github.com/en/rest/security-advisories/repository-advisories#list-repository-security-advisories", 40 | ) 41 | 42 | def getAdvisory(self, ghsa_id: str) -> Advisory: 43 | """Get advisory by ghsa id. 44 | 45 | https://docs.github.com/en/rest/security-advisories/repository-advisories#get-a-repository-security-advisory 46 | """ 47 | result = self.rest.get( 48 | "/repos/{owner}/{repo}/security-advisories/{ghsa_id}", 49 | {"ghsa_id": ghsa_id}, 50 | authenticated=True, 51 | ) 52 | if isinstance(result, dict): 53 | return self.loadAdvisoryData(result) 54 | 55 | raise GHASToolkitTypeError( 56 | f"Error getting advisory by id", 57 | docs="https://docs.github.com/en/rest/security-advisories/repository-advisories#get-a-repository-security-advisory", 58 | ) 59 | 60 | def createAdvisory( 61 | self, advisory: Advisory, repository: Optional[Repository] = None 62 | ): 63 | """Create a GitHub Security Advisories for a repository. 64 | 65 | https://docs.github.com/en/rest/security-advisories/repository-advisories#create-a-repository-security-advisory 66 | """ 67 | raise GHASToolkitError("Unsupported feature") 68 | 69 | def createPrivateAdvisory( 70 | self, advisory: Advisory, repository: Optional[Repository] = None 71 | ): 72 | """Create a GitHub Security Advisories for a repository.""" 73 | raise Exception("Unsupported feature") 74 | 75 | def updateAdvisory( 76 | self, advisory: Advisory, repository: Optional[Repository] = None 77 | ): 78 | """Update GitHub Security Advisory. 79 | 80 | https://docs.github.com/en/rest/security-advisories/repository-advisories#update-a-repository-security-advisory 81 | """ 82 | raise GHASToolkitError("Unsupported feature") 83 | 84 | def loadAdvisoryData(self, data: Dict) -> Advisory: 85 | """Load Advisory from API data.""" 86 | ghsa_id = data.get("ghsa_id") 87 | severity = data.get("severity") 88 | 89 | if not ghsa_id or not severity: 90 | raise Exception("Data is not an advisory") 91 | 92 | aliases = [] 93 | if data.get("cve_id"): 94 | aliases.append(data.get("cve_id")) 95 | 96 | adv = Advisory( 97 | ghsa_id, 98 | severity, 99 | aliases=aliases, 100 | summary=data.get("summary", ""), 101 | cwes=data.get("cwe_ids", []), 102 | ) 103 | # affected 104 | for vuln in data.get("vulnerabilities", []): 105 | introduced = vuln.get("vulnerable_version_range") 106 | if introduced == "": 107 | introduced = None 108 | fixed = vuln.get("patched_versions") 109 | if fixed == "": 110 | fixed = None 111 | 112 | affect = AdvisoryAffect( 113 | ecosystem=vuln.get("package", {}).get("ecosystem", ""), 114 | package=vuln.get("package", {}).get("name", ""), 115 | introduced=introduced, 116 | fixed=fixed, 117 | ) 118 | adv.affected.append(affect) 119 | 120 | return adv 121 | -------------------------------------------------------------------------------- /src/ghastoolkit/octokit/billing.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import csv 3 | from dataclasses import dataclass, field 4 | from typing import Optional, List, Set 5 | 6 | from ghastoolkit.errors import GHASToolkitError 7 | from ghastoolkit.octokit.github import GitHub 8 | from ghastoolkit.octokit.octokit import RestRequest, OctoItem, loadOctoItem 9 | from ghastoolkit.octokit.enterprise import Organization 10 | 11 | 12 | logger = logging.getLogger("ghastoolkit.octokit.github") 13 | 14 | 15 | @dataclass 16 | class BillingUser(OctoItem): 17 | """Billing User.""" 18 | 19 | user_login: str 20 | """Login.""" 21 | last_pushed_date: str 22 | """Last Pushed Date.""" 23 | last_pushed_email: str 24 | """Last Pushed Email.""" 25 | 26 | @property 27 | def login(self) -> str: 28 | """Login.""" 29 | return self.user_login 30 | 31 | 32 | @dataclass 33 | class BillingRepository(OctoItem): 34 | """Billing Repository.""" 35 | 36 | name: str 37 | """Repository Name.""" 38 | advanced_security_committers: int 39 | """Advanced Security Committers.""" 40 | advanced_security_committers_breakdown: List[BillingUser] = field( 41 | default_factory=list 42 | ) 43 | """Advanced Security Committers Breakdown.""" 44 | 45 | def activeCommitterCount(self) -> int: 46 | """Count of Active Committers.""" 47 | return len(self.advanced_security_committers_breakdown) 48 | 49 | def activeCommitterNames(self) -> Set[str]: 50 | """Active Committer Names.""" 51 | results = set() 52 | for commiter in self.advanced_security_committers_breakdown: 53 | results.add(commiter.login) 54 | return results 55 | 56 | def activeCommitterEmails(self) -> Set[str]: 57 | """Active Committer Emails.""" 58 | results = set() 59 | for commiter in self.advanced_security_committers_breakdown: 60 | results.add(commiter.last_pushed_email) 61 | return results 62 | 63 | 64 | @dataclass 65 | class GhasBilling(OctoItem): 66 | """Billing Response.""" 67 | 68 | repositories: List[BillingRepository] = field(default_factory=list) 69 | """Repositories (required).""" 70 | 71 | total_advanced_security_committers: Optional[int] = None 72 | """Total Advanced Security Committers.""" 73 | total_count: Optional[int] = None 74 | """Total Count.""" 75 | maximum_advanced_security_committers: Optional[int] = None 76 | """Maximum Advanced Security Committers.""" 77 | purchased_advanced_security_committers: Optional[int] = None 78 | """Purchased Advanced Security Committers.""" 79 | 80 | @property 81 | def active(self) -> int: 82 | """Active Advanced Security Committers.""" 83 | return self.total_advanced_security_committers or 0 84 | 85 | @property 86 | def maximum(self) -> int: 87 | """Maximum Advanced Security Committers.""" 88 | return self.maximum_advanced_security_committers or 0 89 | 90 | @property 91 | def purchased(self) -> int: 92 | """Purchased Advanced Security Committers.""" 93 | return self.purchased_advanced_security_committers or 0 94 | 95 | def getRepository( 96 | self, name: str, org: Optional[str] = None 97 | ) -> Optional[BillingRepository]: 98 | """Get Repository by Name.""" 99 | for repo in self.repositories: 100 | org, repo_name = repo.name.split("/", 1) 101 | if repo_name == name: 102 | return repo 103 | 104 | return None 105 | 106 | def activeCommitterNames(self) -> Set[str]: 107 | """Active Committer Names.""" 108 | results = set() 109 | for repo in self.repositories: 110 | results.update(repo.activeCommitterNames()) 111 | return results 112 | 113 | def activeCommitterEmails(self) -> Set[str]: 114 | """Active Committer Emails.""" 115 | results = set() 116 | for repo in self.repositories: 117 | results.update(repo.activeCommitterEmails()) 118 | return results 119 | 120 | 121 | class Billing: 122 | """GitHub Billing API""" 123 | 124 | def __init__(self, organization: Optional[Organization] = None) -> None: 125 | """Initialise Billing API.""" 126 | if organization is not None: 127 | self.org = organization.name 128 | else: 129 | self.org = GitHub.owner 130 | self.rest = RestRequest() 131 | self.state = None 132 | 133 | def getGhasBilling(self) -> GhasBilling: 134 | """Get GitHub Advanced Security Billing.""" 135 | if self.org is None: 136 | logger.error("No organization provided") 137 | raise GHASToolkitError( 138 | "No organization provided", 139 | ) 140 | result = self.rest.get(f"/orgs/{self.org}/settings/billing/advanced-security") 141 | 142 | if isinstance(result, dict): 143 | return loadOctoItem(GhasBilling, result) 144 | 145 | logger.error("Error getting billing") 146 | raise GHASToolkitError( 147 | "Error getting billing", 148 | permissions=['"Administration" organization permissions (read)'], 149 | docs="https://docs.github.com/en/enterprise-cloud@latest/rest/billing/billing#get-github-advanced-security-active-committers-for-an-organization", 150 | ) 151 | 152 | @staticmethod 153 | def loadFromCsv(path: str) -> GhasBilling: 154 | """Load Billing from CSV.""" 155 | # name: { 156 | repositories: dict[str, List[BillingUser]] = {} 157 | unique_committers = [] 158 | 159 | with open(path, mode="r") as csv_file: 160 | csv_reader = csv.DictReader(csv_file) 161 | 162 | for row in csv_reader: 163 | repo = row["Organization / repository"] 164 | # if exists, add user to list 165 | user = BillingUser( 166 | row["User login"], 167 | row["Last pushed date"], 168 | row["Last pushed email"], 169 | ) 170 | if repositories.get(repo): 171 | repositories[repo].append(user) 172 | else: 173 | repositories[repo] = [user] 174 | 175 | if user.login not in unique_committers: 176 | unique_committers.append(user.login) 177 | 178 | result = GhasBilling([]) 179 | result.total_count = len(unique_committers) 180 | result.total_advanced_security_committers = len(unique_committers) 181 | 182 | for repo, usrs in repositories.items(): 183 | result.repositories.append(BillingRepository(repo, len(usrs), usrs)) 184 | 185 | return result 186 | -------------------------------------------------------------------------------- /src/ghastoolkit/octokit/clearlydefined.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Any, Optional 3 | from requests import Session 4 | 5 | from ghastoolkit.errors import GHASToolkitError 6 | from ghastoolkit.supplychain.dependencies import Dependency 7 | 8 | 9 | logger = logging.getLogger("ghastoolkit.octokit.clearlydefined") 10 | 11 | PROVIDEDERS = { 12 | "cocoapods": "cocoapods", 13 | "cratesio": "cratesio", 14 | "deb": "debian", 15 | "github": "github", 16 | "githubactions": "github", 17 | "gitlab": "gitlab", 18 | "maven": "mavencentral", 19 | "npm": "npmjs", 20 | "nuget": "nuget", 21 | # packagist, 22 | "pypi": "pypi", 23 | "gems": "rubygems", 24 | } 25 | 26 | 27 | class ClearlyDefined: 28 | def __init__(self) -> None: 29 | self.api = "https://api.clearlydefined.io" 30 | self.session = Session() 31 | self.session.headers = {"Accept": "*/*"} 32 | 33 | def createCurationUrl(self, dependency: Dependency) -> Optional[str]: 34 | if not dependency.manager: 35 | return 36 | provider = PROVIDEDERS.get(dependency.manager, dependency.manager) 37 | 38 | url = f"{self.api}/curations/{dependency.manager}/{provider}/" 39 | url += dependency.namespace or "-" 40 | url += f"/{dependency.name}" 41 | return url 42 | 43 | def getCurations(self, dependency: Dependency) -> dict[str, Any]: 44 | if not dependency.manager: 45 | raise GHASToolkitError(f"Dependency manager / type must be set") 46 | 47 | url = self.createCurationUrl(dependency) 48 | if not url: 49 | logger.warning(f"Url failed to be created from dependency :: {dependency}") 50 | return {} 51 | 52 | resp = self.session.get(url) 53 | if resp.status_code != 200: 54 | raise Exception(f"Failed to access API") 55 | 56 | return resp.json() 57 | 58 | def getLicenses(self, dependency: Dependency) -> list[str]: 59 | licenses = set() 60 | try: 61 | data = self.getCurations(dependency) 62 | for _, curation in data.get("curations", {}).items(): 63 | curlicense = curation.get("licensed", {}).get("declared") 64 | if curlicense: 65 | licenses.add(curlicense) 66 | except KeyboardInterrupt: 67 | raise Exception("Keyboard Interrupt") 68 | except Exception as err: 69 | logger.warning(f"Error getting curation data :: {dependency}") 70 | logger.warning(f"Error :: {err}") 71 | 72 | return list(licenses) 73 | -------------------------------------------------------------------------------- /src/ghastoolkit/octokit/enterprise.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List, Optional 3 | 4 | from semantic_version import Version 5 | 6 | from ghastoolkit.errors import GHASToolkitError 7 | from ghastoolkit.octokit.github import GitHub 8 | from ghastoolkit.octokit.octokit import Octokit, RestRequest 9 | from ghastoolkit.octokit.repository import Repository 10 | 11 | logger = logging.getLogger("ghastoolkit.octokit.enterprise") 12 | 13 | 14 | class Organization: 15 | """Organization.""" 16 | 17 | def __init__( 18 | self, organization: Optional[str] = None, identifier: Optional[int] = None 19 | ) -> None: 20 | """Initialise Organization.""" 21 | self.name = organization 22 | self.identifier = identifier 23 | 24 | self.rest = RestRequest(GitHub.repository) 25 | 26 | def getRepositories(self) -> List[Repository]: 27 | """Get Repositories. 28 | 29 | https://docs.github.com/en/rest/repos/repos#list-organization-repositories 30 | """ 31 | repositories = [] 32 | result = self.rest.get(f"/orgs/{self.name}/repos") 33 | if not isinstance(result, list): 34 | logger.error("Error getting repositories") 35 | raise GHASToolkitError( 36 | "Error getting repositories", 37 | permissions=["Metadata repository permissions (read)"], 38 | docs="https://docs.github.com/en/rest/repos/repos#list-organization-repositories", 39 | ) 40 | 41 | for repository in result: 42 | repositories.append(Repository.parseRepository(repository.get("full_name"))) 43 | 44 | logger.debug(f"Found {len(repositories)} repositories in {self.name}") 45 | return repositories 46 | 47 | def enableAllSecurityProduct(self) -> bool: 48 | """Enable all security products.""" 49 | products = [ 50 | "advanced_security", 51 | "dependency_graph", 52 | "dependabot_alerts", 53 | "dependabot_security_updates", 54 | "code_scanning_default_setup", 55 | "secret_scanning", 56 | "secret_scanning_push_protection", 57 | ] 58 | for product in products: 59 | rslt = self.enableSecurityProduct(product) 60 | if not rslt: 61 | return False 62 | 63 | return True 64 | 65 | def enableSecurityProduct(self, security_product: str) -> bool: 66 | """Enable Advanced Security.""" 67 | url = Octokit.route( 68 | f"/orgs/{self.name}/{security_product}/enable_all", GitHub.repository 69 | ) 70 | result = self.rest.session.post(url) 71 | if result.status_code != 204: 72 | logger.error("Error enabling security product") 73 | return False 74 | 75 | return True 76 | 77 | def enableDefaultSetup(self) -> bool: 78 | """Enable Code Scanning Default Setup on all repositories in an organization. 79 | Assumes that advanced-security is enabled on all repositories. 80 | 81 | - GHE cloud: supported 82 | - GHE server: 3.8 or lower: not supported 83 | - GHE server: 3.9 or 3.10: uses repo level setup (may take a while) 84 | - GHE server: 3.11 or above: not supported 85 | """ 86 | 87 | if GitHub.isEnterpriseServer(): 88 | # version 3.8 or lower 89 | if GitHub.server_version and GitHub.server_version < Version("3.9.0"): 90 | logger.error( 91 | "Enterprise Server 3.8 or lower does not support default setup" 92 | ) 93 | raise GHASToolkitError( 94 | "Enterprise Server 3.8 or lower does not support default setup" 95 | ) 96 | 97 | elif GitHub.server_version and GitHub.server_version < Version("3.11.0"): 98 | from ghastoolkit.octokit.codescanning import CodeScanning 99 | 100 | logger.debug("Enterprise Server 3.9/3.10 supports repo level setup") 101 | 102 | for repo in self.getRepositories(): 103 | logger.debug(f"Enabling default setup for {repo.repo}") 104 | 105 | code_scanning = CodeScanning(repo) 106 | code_scanning.enableDefaultSetup() 107 | return True 108 | else: 109 | logger.error( 110 | "Enterprise Server 3.11 or above isn't supported by this version of the toolkit" 111 | ) 112 | else: 113 | self.enableSecurityProduct("code_scanning_default_setup") 114 | return True 115 | return False 116 | 117 | def __str__(self) -> str: 118 | """Return string representation.""" 119 | return f"Organization('{self.name}')" 120 | 121 | 122 | class Enterprise: 123 | """Enterprise API.""" 124 | 125 | def __init__( 126 | self, 127 | enterprise: Optional[str] = None, 128 | ) -> None: 129 | """Initialise Enterprise.""" 130 | self.enterprise = enterprise or GitHub.enterprise 131 | self.rest = RestRequest(GitHub.repository) 132 | 133 | def getOrganizations(self, include_github: bool = False) -> List[Organization]: 134 | """Get all the Organizations in an enterprise. 135 | 136 | You will need to be authenticated as an enterprise owner to use this API. 137 | """ 138 | github_orgs = ["github", "actions"] 139 | organizations = [] 140 | url = Octokit.route("/organizations", GitHub.repository) 141 | # pagination uses a different API versus the rest of the API 142 | # https://docs.github.com/en/enterprise-cloud@latest/rest/orgs/orgs#list-organizations 143 | last_org_id = 1 144 | 145 | while True: 146 | response = self.rest.session.get( 147 | url, params={"since": last_org_id, "per_page": 100} 148 | ) 149 | 150 | if response.status_code != 200: 151 | logger.error("Error getting organizations") 152 | raise GHASToolkitError( 153 | "Error getting organizations", 154 | permissions=["Metadata repository permissions (read)"], 155 | docs="https://docs.github.com/en/rest/orgs/orgs#list-organizations", 156 | ) 157 | 158 | result = response.json() 159 | 160 | if not isinstance(result, list): 161 | logger.error("Error getting organizations") 162 | raise GHASToolkitError( 163 | "Error getting organizations", 164 | permissions=["Metadata repository permissions (read)"], 165 | docs="https://docs.github.com/en/rest/orgs/orgs#list-organizations", 166 | ) 167 | 168 | for org in result: 169 | if not include_github and org.get("login") in github_orgs: 170 | continue 171 | organizations.append(Organization(org.get("login"), org.get("id"))) 172 | 173 | if len(result) < 100: 174 | break 175 | 176 | if len(organizations) == 0: 177 | logger.error("Error getting last org in organizations") 178 | logger.error("Only GitHub orgs might be returned") 179 | break 180 | 181 | # set last org ID 182 | last_org_id = organizations[-1].identifier 183 | 184 | return organizations 185 | 186 | def enableDefaultSetup(self): 187 | """Enable Code Scanning default setup on all repositories in an enterprise. 188 | 189 | Assumes that advanced-security is enabled on all repositories. 190 | 191 | - GHE cloud: supported 192 | - GHE server: 3.8 or lower: not supported 193 | - GHE server: 3.9 or 3.10: uses repo level setup 194 | - GHE server: 3.11 or above: uses default setup 195 | """ 196 | 197 | for organization in self.getOrganizations(): 198 | organization.enableDefaultSetup() 199 | -------------------------------------------------------------------------------- /src/ghastoolkit/octokit/github.py: -------------------------------------------------------------------------------- 1 | """GitHub and Repository APIs.""" 2 | 3 | import logging 4 | import os 5 | from typing import Dict, Optional, Tuple 6 | from urllib.parse import urlparse 7 | 8 | from semantic_version import Version 9 | 10 | from ghastoolkit.errors import GHASToolkitError 11 | from ghastoolkit.octokit.repository import Repository 12 | 13 | 14 | logger = logging.getLogger("ghastoolkit.octokit.github") 15 | 16 | 17 | class GitHub: 18 | """The GitHub Class. 19 | 20 | This API is used to configure the state for all Octokit apis. 21 | Its a standard interface across all projects. 22 | """ 23 | 24 | repository: Repository = Repository("GeekMasher", "ghastoolkit") 25 | """Repository""" 26 | 27 | owner: Optional[str] = None 28 | """Owner / Organisation""" 29 | 30 | enterprise: Optional[str] = None 31 | """Enterprise Name""" 32 | 33 | token: Optional[str] = None 34 | """GitHub Access Token 35 | This is used to authenticate with the GitHub API. 36 | 37 | This can be set using the GITHUB_TOKEN environment variable or 38 | passed in as a parameter. 39 | """ 40 | 41 | token_type: Optional[str] = None 42 | """GitHub Token Type""" 43 | 44 | # URLs 45 | instance: str = "https://github.com" 46 | """Instance""" 47 | api_rest: str = "https://api.github.com" 48 | """REST API URL""" 49 | api_graphql: str = "https://api.github.com/graphql" 50 | """GraphQL API URL""" 51 | 52 | server_version: Optional[Version] = None 53 | """GitHub Enterprise Server Version""" 54 | 55 | @staticmethod 56 | def init( 57 | repository: Optional[str] = None, 58 | owner: Optional[str] = None, 59 | repo: Optional[str] = None, 60 | reference: Optional[str] = None, 61 | branch: Optional[str] = None, 62 | token: Optional[str] = None, 63 | instance: Optional[str] = None, 64 | enterprise: Optional[str] = None, 65 | retrieve_metadata: bool = True, 66 | ) -> None: 67 | """Initialise a GitHub class using a number of properties.""" 68 | if repository and "/" in repository: 69 | GitHub.repository = Repository.parseRepository(repository) 70 | GitHub.owner = GitHub.repository.owner 71 | elif repository or owner: 72 | GitHub.owner = owner or repository 73 | elif owner and repo: 74 | GitHub.repository = Repository(owner, repo) 75 | GitHub.owner = owner 76 | 77 | if GitHub.repository: 78 | if reference: 79 | GitHub.repository.reference = reference 80 | if branch: 81 | GitHub.repository.branch = branch 82 | 83 | # Set token or load from environment 84 | if token: 85 | GitHub.token = token 86 | else: 87 | GitHub.loadToken() 88 | GitHub.token_type = GitHub.validateTokenType(GitHub.token) 89 | 90 | if not instance: 91 | instance = os.environ.get("GITHUB_SERVER_URL") 92 | 93 | # instance 94 | if instance and instance != "": 95 | GitHub.instance = instance 96 | GitHub.api_rest, GitHub.api_graphql = GitHub.parseInstance(instance) 97 | 98 | if GitHub.isEnterpriseServer() and retrieve_metadata: 99 | # Get the server version 100 | GitHub.getMetaInformation() 101 | 102 | GitHub.enterprise = enterprise 103 | 104 | return 105 | 106 | @staticmethod 107 | def parseInstance(instance: str) -> Tuple[str, str]: 108 | """Parse GitHub Instance.""" 109 | url = urlparse(instance) 110 | 111 | # GitHub Cloud (.com) 112 | if url.netloc == "github.com": 113 | api = url.scheme + "://api." + url.netloc 114 | return (api, f"{api}/graphql") 115 | # GitHub Ent Server 116 | api = url.scheme + "://" + url.netloc + "/api" 117 | 118 | return (f"{api}/v3", f"{api}/graphql") 119 | 120 | @staticmethod 121 | def isEnterpriseServer() -> bool: 122 | """Is the GitHub instance an Enterprise Server.""" 123 | return GitHub.instance != "https://github.com" 124 | 125 | @staticmethod 126 | def display() -> str: 127 | """Display the GitHub Settings.""" 128 | return f"GitHub('{GitHub.repository.display()}', '{GitHub.instance}')" 129 | 130 | @staticmethod 131 | def getOrganization() -> str: 132 | """Get the Organization.""" 133 | return GitHub.owner or GitHub.repository.owner 134 | 135 | @staticmethod 136 | def getMetaInformation() -> Dict: 137 | """Get the GitHub Meta Information.""" 138 | from ghastoolkit.octokit.octokit import RestRequest 139 | 140 | response = RestRequest().session.get(f"{GitHub.api_rest}/meta") 141 | 142 | if response.headers.get("X-GitHub-Enterprise-Version"): 143 | version = response.headers.get("X-GitHub-Enterprise-Version") 144 | GitHub.server_version = Version(version) 145 | 146 | return response.json() 147 | 148 | @staticmethod 149 | def loadToken(): 150 | """Load the GitHub token from the environment variable.""" 151 | if envvar := os.environ.get("GITHUB_TOKEN"): 152 | GitHub.token = envvar 153 | logger.debug("Loaded GITHUB_TOKEN from environment variable") 154 | 155 | GitHub.validateTokenType(GitHub.token) 156 | elif envvar := os.environ.get("GH_TOKEN"): 157 | # This is sometimes set by GitHub CLI 158 | GitHub.token = envvar 159 | logger.debug("Loaded GH_TOKEN from environment variable") 160 | 161 | else: 162 | # TODO: Load from GH CLI? 163 | logger.debug("Failed to load GitHub token") 164 | 165 | @staticmethod 166 | def getToken(masked: bool = True) -> Optional[str]: 167 | """Get the GitHub token. 168 | 169 | Masking the token will only show the first 5 and all the other 170 | characters as `#`. 171 | 172 | Args: 173 | masked (bool): Mask the token. Defaults to True. 174 | 175 | Returns: 176 | str: The GitHub token. 177 | """ 178 | if not GitHub.token: 179 | return None 180 | 181 | if masked: 182 | last = len(GitHub.token) - 5 183 | return f"{GitHub.token[0:5]}{'#' * last}" 184 | return GitHub.token 185 | 186 | @property 187 | def github_app(self) -> bool: 188 | """Check if the token is a GitHub App token.""" 189 | # This is for backwards compatibility 190 | if ttype := self.token_type: 191 | return ttype == "OAUTH" 192 | return False 193 | 194 | @staticmethod 195 | def validateTokenType(token: Optional[str]) -> Optional[str]: 196 | """Check what type of token is being used. 197 | 198 | Returns: 199 | str: The type of token being used. 200 | - "PAT" for Personal Access Token 201 | - "OAUTH" for GitHub App token / OAuth token 202 | - "ACTIONS" for GitHub Actions token 203 | - "SERVICES" for Server-to-Server token 204 | - "UNKNOWN" for unknown token type 205 | 206 | https://github.blog/engineering/behind-githubs-new-authentication-token-formats/ 207 | """ 208 | if not token or not isinstance(token, str): 209 | return None 210 | 211 | # GitHub Actions sets the GITHUB_SECRET_SOURCE environment variable 212 | if secret_source := os.environ.get("GITHUB_SECRET_SOURCE"): 213 | if secret_source != "None": 214 | return secret_source.upper() 215 | 216 | if token.startswith("ghp_") or token.startswith("github_pat_"): 217 | return "PAT" 218 | elif token.startswith("gho_"): 219 | # GitHub OAuth tokens are used for GitHub Apps or GH CLI 220 | return "OAUTH" 221 | elif token.startswith("ghs_"): 222 | # GitHub Actions token are Server-to-Server tokens 223 | if os.environ.get("CI") == "true": 224 | return "ACTIONS" 225 | return "SERVICES" 226 | -------------------------------------------------------------------------------- /src/ghastoolkit/octokit/graphql/GetDependencyAlerts.graphql: -------------------------------------------------------------------------------- 1 | { 2 | repository(owner: "$owner", name: "$repo") { 3 | vulnerabilityAlerts(first: 100, states: [OPEN], $cursor) { 4 | totalCount 5 | pageInfo { 6 | hasNextPage 7 | endCursor 8 | } 9 | edges { 10 | node { 11 | number 12 | state 13 | createdAt 14 | dismissReason 15 | securityVulnerability { 16 | package { 17 | ecosystem 18 | name 19 | } 20 | } 21 | securityAdvisory { 22 | ghsaId 23 | severity 24 | cwes(first: 100) { 25 | edges { 26 | node { 27 | cweId 28 | } 29 | } 30 | } 31 | } 32 | } 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/ghastoolkit/octokit/graphql/GetDependencyInfo.graphql: -------------------------------------------------------------------------------- 1 | { 2 | repository(owner: "$owner", name: "$repo") { 3 | name 4 | licenseInfo { 5 | name 6 | } 7 | dependencyGraphManifests(first: 1, $manifests_cursor) { 8 | totalCount 9 | pageInfo { 10 | hasNextPage 11 | endCursor 12 | } 13 | edges { 14 | node { 15 | filename 16 | dependencies(first: $dependencies_first, $dependencies_cursor) { 17 | totalCount 18 | pageInfo { 19 | hasNextPage 20 | endCursor 21 | } 22 | edges { 23 | node { 24 | packageName 25 | packageManager 26 | requirements 27 | repository { 28 | name 29 | isArchived 30 | isDisabled 31 | isEmpty 32 | isFork 33 | isSecurityPolicyEnabled 34 | isInOrganization 35 | licenseInfo { 36 | name 37 | } 38 | } 39 | } 40 | } 41 | } 42 | } 43 | } 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /src/ghastoolkit/octokit/graphql/__init__.py: -------------------------------------------------------------------------------- 1 | DEPENDENCY_GRAPH_STATUS = """\ 2 | { 3 | repository(owner: "$owner", name: "$repo") { 4 | hasVulnerabilityAlertsEnabled 5 | } 6 | } 7 | """ 8 | 9 | DEPENDENCY_GRAPH_ALERTS = """\ 10 | { 11 | repository(owner: "$owner", name: "$repo") { 12 | vulnerabilityAlerts(first: 100, states: [OPEN], $cursor) { 13 | totalCount 14 | pageInfo { 15 | hasNextPage 16 | endCursor 17 | } 18 | edges { 19 | node { 20 | number 21 | state 22 | createdAt 23 | dismissReason 24 | securityVulnerability { 25 | package { 26 | ecosystem 27 | name 28 | } 29 | } 30 | securityAdvisory { 31 | ghsaId 32 | severity 33 | cwes(first: 100) { 34 | edges { 35 | node { 36 | cweId 37 | } 38 | } 39 | } 40 | } 41 | } 42 | } 43 | } 44 | } 45 | } 46 | """ 47 | 48 | DEPENDENCY_GRAPH_INFO = """\ 49 | { 50 | repository(owner: "$owner", name: "$repo") { 51 | name 52 | licenseInfo { 53 | name 54 | } 55 | dependencyGraphManifests(first: 1, $manifests_cursor) { 56 | totalCount 57 | pageInfo { 58 | hasNextPage 59 | endCursor 60 | } 61 | edges { 62 | node { 63 | filename 64 | dependencies(first: $dependencies_first, $dependencies_cursor) { 65 | totalCount 66 | pageInfo { 67 | hasNextPage 68 | endCursor 69 | } 70 | edges { 71 | node { 72 | packageName 73 | packageManager 74 | requirements 75 | repository { 76 | nameWithOwner 77 | isArchived 78 | isDisabled 79 | isEmpty 80 | isFork 81 | isSecurityPolicyEnabled 82 | isInOrganization 83 | licenseInfo { 84 | name 85 | } 86 | } 87 | } 88 | } 89 | } 90 | } 91 | } 92 | } 93 | } 94 | } 95 | """ 96 | 97 | 98 | QUERIES = { 99 | "GetDependencyStatus": DEPENDENCY_GRAPH_STATUS, 100 | "GetDependencyAlerts": DEPENDENCY_GRAPH_ALERTS, 101 | "GetDependencyInfo": DEPENDENCY_GRAPH_INFO, 102 | } 103 | -------------------------------------------------------------------------------- /src/ghastoolkit/octokit/secretscanning.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from dataclasses import dataclass, field 3 | from datetime import datetime, timedelta 4 | from typing import Any, Optional 5 | 6 | from ghastoolkit.errors import ( 7 | GHASToolkitAuthenticationError, 8 | GHASToolkitError, 9 | GHASToolkitTypeError, 10 | ) 11 | from ghastoolkit.octokit.github import GitHub, Repository 12 | from ghastoolkit.octokit.octokit import OctoItem, RestRequest, loadOctoItem 13 | 14 | 15 | logger = logging.getLogger("ghastoolkit.octokit.secretscanning") 16 | 17 | 18 | @dataclass 19 | class SecretAlert(OctoItem): 20 | """Secret Scanning Alert.""" 21 | 22 | number: int 23 | """Number / Identifier""" 24 | state: str 25 | """Alert State""" 26 | 27 | secret_type: str 28 | """Secret Scanning type""" 29 | secret_type_display_name: str 30 | """Secret Scanning type display name""" 31 | secret: str 32 | """Secret value (sensitive)""" 33 | 34 | created_at: str 35 | """Created Timestamp""" 36 | resolved_at: Optional[str] = None 37 | """Resolved Timestamp""" 38 | resolved_by: Optional[dict[str, Any]] = None 39 | """Resolved By""" 40 | 41 | push_protection_bypassed: bool = False 42 | """Push Protection Bypassed""" 43 | push_protection_bypassed_by: Optional[dict[str, Any]] = None 44 | """Push Protection Bypassed By""" 45 | push_protection_bypassed_at: Optional[str] = None 46 | """Push Protection Bypassed At""" 47 | 48 | resolution_comment: Optional[str] = None 49 | """Resolution Comment""" 50 | 51 | validity: str = "unknown" 52 | """Validity of secret""" 53 | 54 | _locations: list[dict] = field(default_factory=list) 55 | _sha: Optional[str] = None 56 | 57 | @property 58 | def locations(self) -> list[dict]: 59 | """Get Alert locations. Uses a cached version or request from API.""" 60 | if not self._locations: 61 | self._locations = SecretScanning().getAlertLocations(self.number) 62 | return self._locations 63 | 64 | @property 65 | def commit_sha(self) -> Optional[str]: 66 | """Get commit sha if present.""" 67 | if self._sha is None: 68 | for loc in self.locations: 69 | if loc.get("type") == "commit": 70 | self._sha = loc.get("details", {}).get("blob_sha") 71 | break 72 | return self._sha 73 | 74 | @property 75 | def mttr(self) -> Optional[timedelta]: 76 | """Calculate Mean Time To Resolution / Remidiate (MTTR) for a closed/fixed alert.""" 77 | if self.created_at and self.resolved_at: 78 | # GitHub returns ISO 8601 timestamps with a Z at the end 79 | # datetime.fromisoformat() doesn't like the Z 80 | created = self.created_at.replace("Z", "+00:00") 81 | resolved = self.resolved_at.replace("Z", "+00:00") 82 | return datetime.fromisoformat(resolved) - datetime.fromisoformat(created) 83 | return None 84 | 85 | def __str__(self) -> str: 86 | return f"SecretAlert({self.number}, '{self.secret_type}')" 87 | 88 | 89 | class SecretScanning: 90 | """Secret Scanning API.""" 91 | 92 | def __init__(self, repository: Optional[Repository] = None) -> None: 93 | """Initialise Secret Scanning API.""" 94 | self.repository = repository or GitHub.repository 95 | if not self.repository: 96 | raise GHASToolkitError("SecretScanning requires Repository to be set") 97 | 98 | self.rest = RestRequest(self.repository) 99 | 100 | self.state = None 101 | 102 | def isEnabled(self) -> bool: 103 | """Check to see if Secret Scanning is enabled or not via the repository status. 104 | 105 | Permissions: 106 | - "Administration" repository permissions (read) 107 | 108 | https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#get-a-repository 109 | """ 110 | if not self.state: 111 | self.state = self.getStatus() 112 | 113 | if self.state.get("visibility") == "public": 114 | logger.debug("All public repositories have secret scanning enabled") 115 | return True 116 | if saa := self.state.get("security_and_analysis"): 117 | return saa.get("secret_scanning", {}).get("status", "disabled") == "enabled" 118 | 119 | raise GHASToolkitAuthenticationError( 120 | "Failed to fetch Secret Scanning repository settings", 121 | docs="https://docs.github.com/en/enterprise-cloud@latest/rest/repos/repos#get-a-repository", 122 | permissions=["Repository Administration (read)"], 123 | ) 124 | 125 | def isPushProtectionEnabled(self) -> bool: 126 | """Check if Push Protection is enabled. 127 | 128 | Permissions: 129 | - "Administration" repository permissions (read) 130 | 131 | https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#get-a-repository 132 | """ 133 | if not self.state: 134 | self.state = self.getStatus() 135 | 136 | if ssa := self.state.get("security_and_analysis"): 137 | return ( 138 | ssa.get("secret_scanning_push_protection", {}).get("status", "disabled") 139 | == "enabled" 140 | ) 141 | 142 | raise GHASToolkitAuthenticationError( 143 | "Failed to get Push Protection status", 144 | permissions=["Repository Administration (read)"], 145 | docs="https://docs.github.com/en/rest/repos/repos#get-a-repository", 146 | ) 147 | 148 | def getStatus(self) -> dict: 149 | """Get Status of GitHub Advanced Security.""" 150 | result = self.rest.get("/repos/{owner}/{repo}") 151 | if isinstance(result, dict): 152 | return result 153 | raise GHASToolkitTypeError( 154 | "Failed to get the current state of secret scanning", 155 | permissions=["Repository Administration (read)"], 156 | docs="https://docs.github.com/en/rest/repos/repos#get-a-repository", 157 | ) 158 | 159 | def getOrganizationAlerts(self, state: Optional[str] = None) -> list[dict]: 160 | """Get Organization Alerts. 161 | 162 | Permissions: 163 | - "Secret scanning alerts" repository permissions (read) 164 | 165 | https://docs.github.com/en/rest/secret-scanning#list-secret-scanning-alerts-for-an-organization 166 | """ 167 | results = self.rest.get("/orgs/{org}/secret-scanning/alerts", {"state": state}) 168 | if isinstance(results, list): 169 | return results 170 | 171 | raise GHASToolkitTypeError( 172 | f"Error getting organization secret scanning results", 173 | permissions=["Secret scanning alerts (read)"], 174 | docs="https://docs.github.com/en/rest/secret-scanning#list-secret-scanning-alerts-for-an-organization", 175 | ) 176 | 177 | def getAlerts(self, state: str = "open") -> list[SecretAlert]: 178 | """Get Repository alerts. 179 | 180 | Permissions: 181 | - "Secret scanning alerts" repository permissions (read) 182 | 183 | https://docs.github.com/en/rest/secret-scanning#list-secret-scanning-alerts-for-a-repository 184 | """ 185 | 186 | results = self.rest.get( 187 | "/repos/{owner}/{repo}/secret-scanning/alerts", {"state": state} 188 | ) 189 | if isinstance(results, list): 190 | return [loadOctoItem(SecretAlert, item) for item in results] 191 | 192 | raise GHASToolkitTypeError( 193 | "Error getting secret scanning alerts", 194 | docs="https://docs.github.com/en/rest/secret-scanning#list-secret-scanning-alerts-for-a-repository", 195 | ) 196 | 197 | def getAlert( 198 | self, alert_number: int, state: Optional[str] = None 199 | ) -> Optional[SecretAlert]: 200 | """Get Alert by `alert_number`. 201 | 202 | Permissions: 203 | - "Secret scanning alerts" repository permissions (read) 204 | 205 | https://docs.github.com/en/rest/secret-scanning#get-a-secret-scanning-alert 206 | """ 207 | results = self.rest.get( 208 | "/repos/{owner}/{repo}/secret-scanning/alerts/{alert_number}", 209 | {"alert_number": alert_number, "state": state}, 210 | ) 211 | if isinstance(results, dict): 212 | return loadOctoItem(SecretAlert, results) 213 | raise GHASToolkitTypeError( 214 | "Error getting secret scanning alert", 215 | docs="https://docs.github.com/en/rest/secret-scanning#get-a-secret-scanning-alert", 216 | ) 217 | 218 | def getAlertsInPR(self) -> list[SecretAlert]: 219 | """Get Alerts in a Pull Request. 220 | 221 | Permissions: 222 | - "Secret scanning alerts" repository permissions (read) 223 | - "Pull requests" repository permissions (read) 224 | """ 225 | results = [] 226 | pr_commits = self.repository.getPullRequestCommits() 227 | logger.debug(f"Number of Commits in PR :: {len(pr_commits)}") 228 | 229 | for alert in self.getAlerts("open"): 230 | if alert.commit_sha in pr_commits: 231 | results.append(alert) 232 | 233 | return results 234 | 235 | def getAlertLocations(self, alert_number: int) -> list[dict]: 236 | """Get Alert Locations by `alert_number`. 237 | 238 | Permissions: 239 | - "Secret scanning alerts" repository permissions (read) 240 | 241 | https://docs.github.com/en/rest/secret-scanning#list-locations-for-a-secret-scanning-alert 242 | """ 243 | results = self.rest.get( 244 | "/repos/{owner}/{repo}/secret-scanning/alerts/{alert_number}/locations", 245 | {"alert_number": alert_number}, 246 | ) 247 | if isinstance(results, list): 248 | return results 249 | raise GHASToolkitTypeError( 250 | f"Error getting alert locations", 251 | docs="https://docs.github.com/en/rest/secret-scanning#list-locations-for-a-secret-scanning-alert", 252 | ) 253 | -------------------------------------------------------------------------------- /src/ghastoolkit/secretscanning/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GeekMasher/ghastoolkit/6d1ce533eebc886d40a580c357776a47fd87d138/src/ghastoolkit/secretscanning/__init__.py -------------------------------------------------------------------------------- /src/ghastoolkit/secretscanning/secretalerts.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | from typing import Optional 3 | 4 | from ghastoolkit.octokit.octokit import OctoItem 5 | 6 | 7 | @dataclass 8 | class SecretAlert(OctoItem): 9 | number: int 10 | state: str 11 | 12 | created_at: str 13 | 14 | secret_type: str 15 | secret_type_display_name: str 16 | secret: str 17 | 18 | _locations: list[dict] = field(default_factory=list) 19 | _sha: Optional[str] = None 20 | 21 | @property 22 | def locations(self) -> list[dict]: 23 | """Get Alert locations (use cache or request from API)""" 24 | if not self._locations: 25 | from ghastoolkit.octokit.secretscanning import SecretScanning 26 | 27 | self._locations = SecretScanning().getAlertLocations(self.number) 28 | return self._locations 29 | 30 | @property 31 | def commit_sha(self) -> Optional[str]: 32 | """Get commit sha if present""" 33 | if self._sha is None: 34 | for loc in self.locations: 35 | if loc.get("type") == "commit": 36 | self._sha = loc.get("details", {}).get("commit_sha") 37 | break 38 | return self._sha 39 | 40 | def __str__(self) -> str: 41 | return f"SecretAlert({self.number}, '{self.secret_type}')" 42 | -------------------------------------------------------------------------------- /src/ghastoolkit/supplychain/__init__.py: -------------------------------------------------------------------------------- 1 | from .advisories import Advisory, Advisories, AdvisoryAffect 2 | from .dependencies import Dependencies, uniqueDependencies 3 | from .dependency import Dependency 4 | from .licensing import Licenses 5 | from .dependencyalert import DependencyAlert 6 | -------------------------------------------------------------------------------- /src/ghastoolkit/supplychain/__main__.py: -------------------------------------------------------------------------------- 1 | """Supply Chain Toolkit CLI.""" 2 | 3 | from argparse import Namespace 4 | import logging 5 | 6 | from ghastoolkit.octokit.dependencygraph import DependencyGraph 7 | from ghastoolkit.octokit.github import GitHub 8 | from ghastoolkit.utils.cli import CommandLine 9 | 10 | 11 | def runDefault(arguments): 12 | depgraph = DependencyGraph(GitHub.repository) 13 | bom = depgraph.exportBOM() 14 | packages = bom.get("sbom", {}).get("packages", []) 15 | 16 | logging.info(f"Total Dependencies :: {len(packages)}") 17 | 18 | info = bom.get("sbom", {}).get("creationInfo", {}) 19 | logging.info(f"SBOM Created :: {info.get('created')}") 20 | 21 | logging.info("\nTools:") 22 | for tool in info.get("creators", []): 23 | logging.info(f" - {tool}") 24 | 25 | 26 | def runOrgAudit(arguments): 27 | """Run an audit on an organization.""" 28 | licenses = arguments.licenses.split(",") 29 | logging.info(f"Licenses :: {','.join(licenses)}") 30 | 31 | if arguments.debug: 32 | logging.getLogger("ghastoolkit.octokit.dependencygraph").setLevel(logging.DEBUG) 33 | 34 | depgraph = DependencyGraph() 35 | 36 | dependencies = depgraph.getOrganizationDependencies() 37 | 38 | for repo, deps in dependencies.items(): 39 | # get a list of deps that match the licenses 40 | violations = deps.findLicenses(licenses) 41 | # get a list of deps with no license data 42 | unknowns = deps.findUnknownLicenses() 43 | 44 | if len(violations) == 0 and len(unknowns) == 0: 45 | continue 46 | 47 | logging.info(f" > {repo} :: {len(deps)}") 48 | logging.info(f" |-> Unknowns :: {len(unknowns)}") 49 | for unknown in unknowns: 50 | logging.warning(f" |---> {unknown.getPurl()}") 51 | 52 | logging.info(f" |-> Violations :: {len(violations)}") 53 | for violation in violations: 54 | logging.warning(f" |---> {violation.getPurl()}") 55 | 56 | 57 | class SupplyChainCLI(CommandLine): 58 | def arguments(self): 59 | """CLI for Supply Chain Toolkit.""" 60 | if self.subparser: 61 | self.addModes(["org-audit"]) 62 | 63 | parser = self.parser.add_argument_group("supplychain") 64 | parser.add_argument( 65 | "--licenses", 66 | default="GPL-*,AGPL-*,LGPL-*", 67 | help="License(s) to check for (default: 'GPL-*,AGPL-*,LGPL-*')", 68 | ) 69 | 70 | def run(self, arguments: Namespace): 71 | """Run Supply Chain Toolkit.""" 72 | if arguments.mode == "default": 73 | runDefault(arguments) 74 | elif arguments.mode == "org-audit": 75 | runOrgAudit(arguments) 76 | else: 77 | self.parser.print_help() 78 | exit(1) 79 | 80 | 81 | if __name__ == "__main__": 82 | parser = SupplyChainCLI() 83 | parser.run(parser.parse_args()) 84 | logging.info("Done.") 85 | -------------------------------------------------------------------------------- /src/ghastoolkit/supplychain/dependency.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from dataclasses import dataclass, field 3 | from typing import Optional, Union, Dict 4 | 5 | from ghastoolkit.supplychain.dependencyalert import DependencyAlert 6 | from ghastoolkit.octokit.github import Repository 7 | 8 | logger = logging.getLogger("ghastoolkit.supplychain.dependency") 9 | 10 | 11 | @dataclass 12 | class Dependency: 13 | """Dependency.""" 14 | 15 | name: str 16 | """Name of the Dependency""" 17 | 18 | namespace: Optional[str] = None 19 | """Namespace of the Dependency""" 20 | 21 | version: Optional[str] = None 22 | """Version of the Dependency""" 23 | 24 | manager: Optional[str] = None 25 | """Package Manager""" 26 | 27 | path: Optional[str] = None 28 | """Path to the Dependency""" 29 | 30 | qualifiers: dict[str, str] = field(default_factory=dict) 31 | """Qualifiers""" 32 | 33 | relationship: Optional[str] = None 34 | """Relationship to the Dependency. 35 | 36 | This can be direct or indirect/transitive. 37 | """ 38 | 39 | license: Optional[str] = None 40 | """License information""" 41 | 42 | alerts: list[DependencyAlert] = field(default_factory=list) 43 | """Security Alerts""" 44 | 45 | repository: Optional[Union[str, Repository]] = None 46 | """GitHub Repository for the dependency""" 47 | 48 | repositories: set[Repository] = field(default_factory=set) 49 | """List of repositories for the dependency""" 50 | 51 | def __post_init__(self): 52 | # normalize manager 53 | if self.manager: 54 | self.manager = self.manager.lower() 55 | if self.repository and isinstance(self.repository, str): 56 | self.repository = Repository.parseRepository(self.repository) 57 | 58 | if self.version: 59 | self.version = self.version.strip() 60 | if self.version.startswith("v"): 61 | # normalize version 62 | self.version = self.version[1:] 63 | 64 | def getPurl(self, version: bool = True) -> str: 65 | """Create a PURL from the Dependency. 66 | 67 | https://github.com/package-url/purl-spec 68 | """ 69 | result = f"pkg:" 70 | if self.manager: 71 | result += f"{self.manager.lower()}/" 72 | if self.namespace: 73 | result += f"{self.namespace}/" 74 | result += f"{self.name}" 75 | if version and self.version: 76 | result += f"@{self.version}" 77 | 78 | return result 79 | 80 | @staticmethod 81 | def fromPurl(purl: str) -> "Dependency": 82 | """Create a Dependency from a PURL.""" 83 | dep = Dependency("") 84 | # version (at end) 85 | if "@" in purl: 86 | pkg, dep.version = purl.split("@", 1) 87 | else: 88 | pkg = purl 89 | 90 | slashes = pkg.count("/") 91 | if slashes == 0 and pkg.count(":", 1): 92 | # basic purl `npm:name` 93 | manager, dep.name = pkg.split(":", 1) 94 | elif slashes == 2: 95 | manager, dep.namespace, dep.name = pkg.split("/", 3) 96 | elif slashes == 1: 97 | manager, dep.name = pkg.split("/", 2) 98 | elif slashes > 2: 99 | manager, dep.namespace, dep.name = pkg.split("/", 2) 100 | else: 101 | raise Exception(f"Unable to parse PURL :: {purl}") 102 | 103 | if manager.startswith("pkg:"): 104 | _, dep.manager = manager.split(":", 1) 105 | else: 106 | dep.manager = manager 107 | 108 | return dep 109 | 110 | @property 111 | def fullname(self) -> str: 112 | """Full Name of the Dependency.""" 113 | if self.namespace: 114 | sep = "/" 115 | if self.manager == "maven": 116 | sep = ":" 117 | return f"{self.namespace}{sep}{self.name}" 118 | return self.name 119 | 120 | def isDirect(self) -> bool: 121 | """Is this a direct dependency? 122 | 123 | This is a bit of a hack to determine if this is a direct dependency or not. 124 | In the future we will have this data as part of the API (SBOM). 125 | 126 | **Supports:** 127 | 128 | - `npm` 129 | - `maven` 130 | - `pip` 131 | """ 132 | if self.relationship and self.relationship.lower() == "direct": 133 | return True 134 | if manifest_file := self.path: 135 | # Use the manifest file to determine if this is a direct dependency 136 | if self.manager == "npm" and manifest_file.endswith("package.json"): 137 | return True 138 | elif self.manager == "maven" and manifest_file.endswith("pom.xml"): 139 | return True 140 | elif self.manager == "pip" and manifest_file.endswith("requirements.txt"): 141 | return True 142 | 143 | return False 144 | 145 | def __str__(self) -> str: 146 | """To String (PURL).""" 147 | return self.getPurl() 148 | 149 | def __repr__(self) -> str: 150 | return self.getPurl() 151 | 152 | def __hash__(self) -> int: 153 | return hash(self.getPurl()) 154 | -------------------------------------------------------------------------------- /src/ghastoolkit/supplychain/dependencyalert.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from datetime import datetime 3 | from typing import Optional 4 | 5 | from ghastoolkit.octokit.octokit import OctoItem 6 | from ghastoolkit.supplychain.advisories import Advisory 7 | 8 | 9 | @dataclass 10 | class DependencyAlert(OctoItem): 11 | number: int 12 | """Number / Identifier""" 13 | state: str 14 | """Alert State""" 15 | severity: str 16 | """Alert Severity""" 17 | advisory: Advisory 18 | """GitHub Security Advisory""" 19 | 20 | purl: str 21 | """Package URL""" 22 | 23 | created_at: Optional[str] = None 24 | """Created Timestamp""" 25 | 26 | manifest: Optional[str] = None 27 | """Manifest""" 28 | 29 | def __init_post__(self): 30 | if not self.created_at: 31 | self.created_at = datetime.now().strftime("%Y-%m-%dT%XZ") 32 | 33 | @property 34 | def cwes(self) -> list[str]: 35 | return self.advisory.cwes 36 | 37 | def createdAt(self) -> Optional[datetime]: 38 | if self.created_at: 39 | return datetime.strptime(self.created_at, "%Y-%m-%dT%XZ") 40 | 41 | def __str__(self) -> str: 42 | return f"DependencyAlert({self.advisory.ghsa_id}, {self.severity})" 43 | -------------------------------------------------------------------------------- /src/ghastoolkit/supplychain/licensing.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import os 3 | import json 4 | import logging 5 | from typing import Optional, Union 6 | 7 | from ghastoolkit.octokit.github import Repository 8 | 9 | 10 | logger = logging.getLogger("ghastoolkit.supplychain.licenses") 11 | 12 | NO_LICENSES = ["None", "NA", "NOASSERTION"] 13 | 14 | 15 | class Licenses: 16 | def __init__(self, path: Optional[str] = None) -> None: 17 | """Licenses""" 18 | self.data: dict[str, list[str]] = {} 19 | self.sources: list[str] = [] 20 | 21 | if path: 22 | self.load(path) 23 | 24 | def load(self, path: str): 25 | """Load a licenses file""" 26 | if not os.path.exists(path): 27 | raise Exception(f"License path does not exist: {path}") 28 | if not os.path.isfile(path): 29 | raise Exception("Path provided needs to be a file") 30 | 31 | logger.debug(f"Loading licenseing file :: {path}") 32 | with open(path, "r") as handle: 33 | data = json.load(handle) 34 | # TODO validate the data before loading? 35 | 36 | self.data.update(data) 37 | 38 | self.sources.append(path) 39 | logger.debug(f"Loaded licenses :: {len(self.data)}") 40 | 41 | def add(self, purl: str, licenses: Union[str, list]): 42 | """Add license""" 43 | if self.data.get(purl): 44 | return 45 | licenses = licenses if isinstance(licenses, list) else [licenses] 46 | self.data[purl] = licenses 47 | 48 | def find(self, purl: str) -> Optional[list[str]]: 49 | """Find by PURL""" 50 | return self.data.get(purl) 51 | 52 | def export(self, path: str): 53 | """Export licenses file""" 54 | with open(path, "w") as handle: 55 | json.dump(self.data, handle) 56 | 57 | def generateLockfile(self, path: str, repository: Optional[Repository] = None): 58 | """Generate Lockfile for the current licenses""" 59 | lock_data = {"total": len(self.data), "created": datetime.now().isoformat()} 60 | if repository: 61 | lock_data["repository"] = str(repository.display()) 62 | lock_data["version"] = repository.gitsha() or repository.sha 63 | 64 | with open(path, "w") as handle: 65 | json.dump(lock_data, handle, indent=2, sort_keys=True) 66 | 67 | def __len__(self) -> int: 68 | return len(self.data) 69 | 70 | 71 | if __name__ == "__main__": 72 | import yaml 73 | import argparse 74 | from ghastoolkit import Repository, Dependency 75 | 76 | logging.basicConfig( 77 | level=logging.DEBUG, 78 | format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", 79 | ) 80 | parser = argparse.ArgumentParser("ghastoolkit.supplychain.licenses") 81 | parser.add_argument("-o", "--output", help="Output") 82 | 83 | arguments = parser.parse_args() 84 | 85 | logging.info(f"Output :: {arguments.output}") 86 | 87 | licenses = Licenses() 88 | 89 | repository = Repository.parseRepository("clearlydefined/curated-data") 90 | logging.info(f"Cloning / Using `clearlydefined` repo: {repository.clone_path}") 91 | repository.clone(clobber=True, depth=1) 92 | 93 | lock_content = { 94 | "repository": repository.display(), 95 | "version": repository.gitsha(), 96 | } 97 | 98 | # https://github.com/clearlydefined/curated-data/tree/master/curations 99 | curations = repository.getFile("curations") 100 | 101 | for root, dirs, files in os.walk(curations): 102 | for filename in files: 103 | name, ext = os.path.splitext(filename) 104 | if ext not in [".yml", ".yaml"]: 105 | continue 106 | 107 | path = os.path.join(root, filename) 108 | 109 | with open(path, "r") as handle: 110 | curation_data = yaml.safe_load(handle) 111 | 112 | coordinates = curation_data.get("coordinates", {}) 113 | purl = Dependency( 114 | coordinates.get("name"), 115 | coordinates.get("namespace"), 116 | manager=coordinates.get("type"), 117 | ).getPurl() 118 | 119 | revision_licenses = set() 120 | for _, revision in curation_data.get("revisions", {}).items(): 121 | l = revision.get("licensed") 122 | if l and l.get("declaired"): 123 | revision_licenses.add(l.get("declaired")) 124 | 125 | licenses.add(purl, list(revision_licenses)) 126 | 127 | logging.info(f"Licenses Loaded :: {len(licenses)}") 128 | 129 | # lock 130 | lock_path = arguments.output.replace(".json", ".lock.json") 131 | logging.info(f"Saving lock file :: {lock_path}") 132 | licenses.generateLockfile(lock_path, repository=repository) 133 | 134 | # export 135 | logging.info(f"Exporting Output :: {arguments.output}") 136 | licenses.export(arguments.output) 137 | -------------------------------------------------------------------------------- /src/ghastoolkit/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GeekMasher/ghastoolkit/6d1ce533eebc886d40a580c357776a47fd87d138/src/ghastoolkit/utils/__init__.py -------------------------------------------------------------------------------- /src/ghastoolkit/utils/cache.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import logging 4 | from typing import Any, Dict, Optional, Union 5 | from datetime import datetime, timedelta 6 | 7 | logger = logging.getLogger("ghastoolkit.utils.cache") 8 | 9 | # A month in minutes 10 | CACHE_MONTH = 30 * 24 * 60 11 | # A week in minutes 12 | CACHE_WEEK = 7 * 24 * 60 13 | # A day in minutes 14 | CACHE_DAY = 24 * 60 15 | 16 | 17 | class Cache: 18 | """Cache class for storing and retrieving data.""" 19 | 20 | cache_age: int = CACHE_DAY 21 | """Default cache age in minutes.""" 22 | 23 | def __init__( 24 | self, 25 | root: Optional[str] = None, 26 | store: Optional[str] = None, 27 | age: Union[int, str] = CACHE_DAY, 28 | ): 29 | """Initialize Cache. 30 | 31 | Args: 32 | root (str, optional): Root directory for cache. Defaults to ~/.ghastoolkit/cache. 33 | store (str, optional): Subdirectory for cache. Defaults to None. 34 | age (int, str): Cache expiration age in hours. Defaults to 1440mins (24hrs). 35 | """ 36 | if root is None: 37 | root = os.path.join(os.path.expanduser("~"), ".ghastoolkit", "cache") 38 | self.root = root 39 | self.store = store 40 | self.cache: Dict[str, Any] = {} 41 | 42 | if isinstance(age, str): 43 | if age.upper() == "MONTH": 44 | Cache.cache_age = CACHE_MONTH 45 | elif age.upper() == "WEEK": 46 | Cache.cache_age = CACHE_WEEK 47 | elif age.upper() == "DAY": 48 | Cache.cache_age = CACHE_DAY 49 | else: 50 | Cache.cache_age = CACHE_DAY 51 | else: 52 | Cache.cache_age = age 53 | 54 | logger.debug(f"Cache root: {self.root}") 55 | 56 | if not os.path.exists(self.cache_path): 57 | os.makedirs(self.cache_path, exist_ok=True) 58 | 59 | @property 60 | def cache_path(self) -> str: 61 | if self.store is None: 62 | return self.root 63 | return os.path.join(self.root, self.store) 64 | 65 | def get_file_age(self, path: str) -> Optional[float]: 66 | """Get the age of a file in hours.""" 67 | if not os.path.exists(path): 68 | return None 69 | 70 | file_mtime = os.path.getmtime(path) 71 | file_time = datetime.fromtimestamp(file_mtime) 72 | current_time = datetime.now() 73 | 74 | age_hours = (current_time - file_time).total_seconds() / 3600 75 | logger.debug(f"Cache file age: {age_hours:.2f} hours for {path}") 76 | 77 | return age_hours 78 | 79 | def is_cache_expired(self, path: str, max_age_hours: float = 24.0) -> bool: 80 | """Check if cache file is expired (older than max_age_hours).""" 81 | age = self.get_file_age(path) 82 | if age is None: 83 | return True 84 | 85 | return age > max_age_hours 86 | 87 | def read( 88 | self, key: str, file_type: Optional[str] = None, max_age_hours: float = 24.0 89 | ) -> Optional[Any]: 90 | """Read from cache.""" 91 | path = os.path.join(self.cache_path, key) 92 | if file_type: 93 | path = f"{path}.{file_type}" 94 | 95 | if os.path.exists(path): 96 | if self.is_cache_expired(path, max_age_hours): 97 | logger.debug(f"Cache expired ({max_age_hours} hours): {path}") 98 | return None 99 | 100 | logger.debug(f"Cache hit: {path}") 101 | with open(path, "r") as file: 102 | return file.read() 103 | return None 104 | 105 | def write(self, key: str, value: Any, file_type: Optional[str] = None): 106 | """Write to cache.""" 107 | if not isinstance(key, str): 108 | raise ValueError("Key must be a string") 109 | # Convert value to string if it's not already 110 | if isinstance(value, str): 111 | pass 112 | elif isinstance(value, dict): 113 | value = json.dumps(value) 114 | else: 115 | raise ValueError(f"Value is a unsupported type: {type(value)}") 116 | 117 | path = os.path.join(self.cache_path, key) 118 | # the key might be a owner/repo 119 | parent = os.path.dirname(path) 120 | if not os.path.exists(parent): 121 | os.makedirs(parent, exist_ok=True) 122 | 123 | if ftype := file_type: 124 | path = f"{path}.{ftype}" 125 | 126 | logger.debug(f"Cache write: {path}") 127 | with open(path, "w") as file: 128 | file.write(value) 129 | -------------------------------------------------------------------------------- /src/ghastoolkit/utils/cli.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from argparse import ArgumentParser, Namespace 4 | from typing import Optional 5 | 6 | from ghastoolkit.octokit.github import GitHub 7 | 8 | 9 | class CommandLine: 10 | def __init__( 11 | self, 12 | name: Optional[str] = None, 13 | parser: Optional[ArgumentParser] = None, 14 | default_logger: bool = True, 15 | ) -> None: 16 | """Initialize CommandLine.""" 17 | self.parser = parser or ArgumentParser(name or "ghastoolkit") 18 | self.subparser: bool = parser is None 19 | 20 | if not parser: 21 | self.default() 22 | 23 | self.modes = set() 24 | self.modes.add("default") 25 | 26 | self.arguments() 27 | 28 | if not parser: 29 | self.parser.add_argument( 30 | "mode", 31 | const="default", 32 | nargs="?", 33 | default="default", 34 | choices=list(self.modes), 35 | ) 36 | 37 | if default_logger: 38 | self.default_logger() 39 | 40 | def default(self): 41 | """Setup default arguments.""" 42 | self.parser.add_argument( 43 | "--debug", dest="debug", action="store_true", help="Enable Debugging" 44 | ) 45 | self.parser.add_argument( 46 | "--version", dest="version", action="store_true", help="Output version" 47 | ) 48 | self.parser.add_argument( 49 | "--cwd", 50 | "--working-directory", 51 | dest="cwd", 52 | default=os.getcwd(), 53 | help="Working directory", 54 | ) 55 | 56 | github = self.parser.add_argument_group("github") 57 | 58 | github.add_argument( 59 | "-r", 60 | "--github-repository", 61 | dest="repository", 62 | default=os.environ.get("GITHUB_REPOSITORY"), 63 | help="GitHub Repository (default: GITHUB_REPOSITORY)", 64 | ) 65 | github.add_argument( 66 | "--github-instance", 67 | dest="instance", 68 | default=os.environ.get("GITHUB_SERVER_URL", "https://github.com"), 69 | help="GitHub Instance URL (default: GITHUB_SERVER_URL)", 70 | ) 71 | github.add_argument( 72 | "--github-owner", dest="owner", help="GitHub Owner (Org/User)" 73 | ) 74 | github.add_argument( 75 | "--github-enterprise", dest="enterprise", help="GitHub Enterprise" 76 | ) 77 | github.add_argument( 78 | "-t", 79 | "--github-token", 80 | dest="token", 81 | default=os.environ.get("GITHUB_TOKEN"), 82 | help="GitHub API Token (default: GITHUB_TOKEN)", 83 | ) 84 | 85 | github.add_argument( 86 | "--sha", default=os.environ.get("GITHUB_SHA"), help="Commit SHA" 87 | ) 88 | github.add_argument( 89 | "--ref", default=os.environ.get("GITHUB_REF"), help="Commit ref" 90 | ) 91 | 92 | def addModes(self, modes: list[str]): 93 | """Set modes.""" 94 | self.modes.update(modes) 95 | 96 | def arguments(self): 97 | """Set custom arguments.""" 98 | return 99 | 100 | def run(self, arguments: Optional[Namespace] = None): 101 | """Run CLI.""" 102 | raise Exception("Not implemented") 103 | 104 | def default_logger(self): 105 | """Setup default logger.""" 106 | arguments = self.parse_args() 107 | logging.basicConfig( 108 | level=( 109 | logging.DEBUG 110 | if arguments.debug or os.environ.get("DEBUG") 111 | else logging.INFO 112 | ), 113 | format="%(message)s", 114 | ) 115 | 116 | def parse_args(self) -> Namespace: 117 | """Parse arguments.""" 118 | arguments = self.parser.parse_args() 119 | # GitHub Init 120 | GitHub.init( 121 | repository=arguments.repository, 122 | reference=arguments.ref, 123 | owner=arguments.owner, 124 | instance=arguments.instance, 125 | token=arguments.token, 126 | enterprise=arguments.enterprise, 127 | ) 128 | 129 | return arguments 130 | -------------------------------------------------------------------------------- /tests/data/advisories.yml: -------------------------------------------------------------------------------- 1 | 2 | - ghas_id: "test1" 3 | severity: "high" 4 | affected: 5 | - ecosystem: "maven" 6 | package: "com.geekmasher.ghastoolkit" 7 | affected: 8 | - ">=0.1" 9 | 10 | -------------------------------------------------------------------------------- /tests/responses/codescanning.json: -------------------------------------------------------------------------------- 1 | { 2 | "default": [ 3 | { 4 | "url": "https://api.github.com/repos/GeekMasher/ghastoolkit/code-scanning/analyses", 5 | "json": [ 6 | { 7 | "ref": "refs/head/main", 8 | "commit_sha": "abcdef", 9 | "analysis_key": "dynamic/github-code-scanning/codeql:analyze", 10 | "environment": "{\"language\": \"python\"}", 11 | "category": "/language:python", 12 | "error": "", 13 | "created_at": "2024-08-27T10:35:05Z", 14 | "results_count": 0, 15 | "rules_count": 50, 16 | "id": 1234, 17 | "url": "https://api.github.com/repos/...", 18 | "sarif_id": "absdef", 19 | "tool": { 20 | "name": "CodeQL", 21 | "guid": null, 22 | "version": "2.18.2" 23 | }, 24 | "deletable": true, 25 | "warning": "" 26 | } 27 | ] 28 | } 29 | ], 30 | "errors": [ 31 | { 32 | "url": "https://api.github.com/repos/GeekMasher/ghastoolkit/code-scanning/analyses", 33 | "status": 404, 34 | "json": { 35 | "message": "Not Found", 36 | "documentation_url": "https://docs.github.com/rest/reference/repos#list-code-scanning-analyses" 37 | } 38 | } 39 | ], 40 | "retries": [ 41 | { 42 | "url": "https://api.github.com/repos/GeekMasher/ghastoolkit/code-scanning/analyses", 43 | "json": [] 44 | }, 45 | { 46 | "url": "https://api.github.com/repos/GeekMasher/ghastoolkit/code-scanning/analyses", 47 | "json": [ 48 | { 49 | "ref": "refs/pull/1/head", 50 | "commit_sha": "abcdef", 51 | "analysis_key": "dynamic/github-code-scanning/codeql:analyze", 52 | "environment": "{\"language\": \"python\"}", 53 | "category": "/language:python", 54 | "error": "", 55 | "created_at": "2024-08-27T10:35:05Z", 56 | "results_count": 0, 57 | "rules_count": 50, 58 | "id": 1234, 59 | "url": "https://api.github.com/repos/...", 60 | "sarif_id": "absdef", 61 | "tool": { 62 | "name": "CodeQL", 63 | "guid": null, 64 | "version": "2.18.2" 65 | }, 66 | "deletable": true, 67 | "warning": "" 68 | } 69 | ] 70 | } 71 | ] 72 | } -------------------------------------------------------------------------------- /tests/responses/restrequests.json: -------------------------------------------------------------------------------- 1 | { 2 | "errors": [ 3 | { 4 | "url": "https://api.github.com/repos/GeekMasher/ghastoolkit/secret-scanning/alerts", 5 | "status": 404, 6 | "json": { 7 | "message": "Secret scanning is disabled on this repository.", 8 | "documentation_url": "https://docs.github.com/rest/secret-scanning/secret-scanning" 9 | } 10 | }, 11 | { 12 | "url": "https://api.github.com/repos/GeekMasher/ghastoolkit/secret-scanning/alerts/1", 13 | "status": 404, 14 | "json": { 15 | "message": "Not Found", 16 | "documentation_url": "https://docs.github.com/rest/secret-scanning/secret-scanning" 17 | } 18 | } 19 | ], 20 | "error_handler": [ 21 | { 22 | "url": "https://api.github.com/repos/GeekMasher/ghastoolkit/secret-scanning/alerts", 23 | "status": 404, 24 | "json": { 25 | "message": "Secret scanning is disabled on this repository.", 26 | "documentation_url": "https://docs.github.com/rest/secret-scanning/secret-scanning" 27 | } 28 | } 29 | ] 30 | } -------------------------------------------------------------------------------- /tests/test_advisories.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from ghastoolkit.supplychain.advisories import ( 3 | Advisory, 4 | Advisories, 5 | AdvisoryAffect, 6 | parseVersion, 7 | ) 8 | from ghastoolkit.supplychain.dependencies import Dependency 9 | 10 | 11 | class TestAdvisories(unittest.TestCase): 12 | def setUp(self) -> None: 13 | self.advisories = Advisories() 14 | return super().setUp() 15 | 16 | def test_advisories(self): 17 | ad = Advisory("rand", "high") 18 | self.advisories.append(ad) 19 | self.assertEqual(len(self.advisories), 1) 20 | 21 | def test_advisory_check(self): 22 | affected = [ 23 | AdvisoryAffect( 24 | "maven", "com.geekmasher.ghastoolkit", introduced="0", fixed="1" 25 | ) 26 | ] 27 | ad = Advisory("rand", "high", affected=affected) 28 | self.advisories.append(ad) 29 | self.assertEqual(len(self.advisories), 1) 30 | 31 | dep = Dependency("ghastoolkit", "com.geekmasher", "0.8", "maven") 32 | 33 | alert = self.advisories.check(dep) 34 | self.assertEqual(alert, [ad]) 35 | 36 | def test_advisory_cwes(self): 37 | ad = Advisory("rand", "high", cwes=["CWE-1234"]) 38 | self.assertEqual(ad.cwes, ["CWE-1234"]) 39 | 40 | ad = Advisory("rand", "high", cwes=[{"cwe_id": "CWE-1234"}]) 41 | self.assertEqual(ad.cwes, ["CWE-1234"]) 42 | 43 | def test_advisory_cvss(self): 44 | ad = Advisory( 45 | "rand", 46 | "high", 47 | cvss={ 48 | "vector_string": "CVSS:3.1/AV:N/AC:H/PR:H/UI:R/S:C/C:H/I:H/A:H", 49 | "score": 7.6 50 | } 51 | ) 52 | self.assertEqual(ad.cvss_score(), 7.6) 53 | 54 | ad = Advisory( 55 | "rand", 56 | "high", 57 | cvss_severities={ 58 | "cvss_v3": { 59 | "vector_string": "CVSS:3.1/AV:N/AC:H/PR:H/UI:R/S:C/C:H/I:H/A:H", 60 | "score": 7.6 61 | }, 62 | "cvss_v4": { 63 | "vector_string": "CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N", 64 | "score": 9.3 65 | } 66 | } 67 | ) 68 | self.assertEqual(ad.cvss_score(3), 7.6) 69 | self.assertEqual(ad.cvss_score(4), 9.3) 70 | 71 | def test_affect_check(self): 72 | dep = Dependency("ghastoolkit", "com.geekmasher", "0.8", "maven") 73 | affect = AdvisoryAffect( 74 | "maven", "com.geekmasher.ghastoolkit", introduced="0", fixed="1" 75 | ) 76 | 77 | self.assertTrue(affect.check(dep)) 78 | 79 | def test_post_init(self): 80 | affect = AdvisoryAffect( 81 | "maven", "com.geekmasher.ghastoolkit", introduced="0", fixed="1" 82 | ) 83 | self.assertIsNotNone(affect.introduced) 84 | self.assertIsNotNone(affect.fixed) 85 | if affect.package_dependency: 86 | self.assertEqual(affect.package_dependency.name, "ghastoolkit") 87 | self.assertEqual(affect.package_dependency.namespace, "com.geekmasher") 88 | else: 89 | self.assertIsNotNone(affect.package_dependency) 90 | 91 | affect = AdvisoryAffect("pypi", "ghastoolkit", introduced="0", fixed="1") 92 | if affect.package_dependency: 93 | self.assertEqual(affect.package_dependency.name, "ghastoolkit") 94 | self.assertIsNone(affect.package_dependency.namespace) 95 | else: 96 | self.assertIsNotNone(affect.package_dependency) 97 | 98 | def test_affect_check_version(self): 99 | affect = AdvisoryAffect("", "", introduced="0.2", fixed="1") 100 | 101 | # too early 102 | self.assertFalse(affect.checkVersion("0.1")) 103 | self.assertFalse(affect.checkVersion("0.1.1")) 104 | # inside range 105 | self.assertTrue(affect.checkVersion("0.2")) 106 | self.assertTrue(affect.checkVersion("0.4.2")) 107 | self.assertTrue(affect.checkVersion("0.1111")) 108 | 109 | # fixed 110 | self.assertFalse(affect.checkVersion("1")) 111 | # later versions 112 | self.assertFalse(affect.checkVersion("1.1")) 113 | self.assertFalse(affect.checkVersion("10")) 114 | 115 | def test_parse_version(self): 116 | self.assertEqual(parseVersion("1"), "1.0.0") 117 | self.assertEqual(parseVersion("1.0"), "1.0.0") 118 | self.assertEqual(parseVersion("1.1.1"), "1.1.1") 119 | 120 | def test_affect_versions(self): 121 | affect = AdvisoryAffect("", "", introduced="0.2", fixed="1") 122 | self.assertEqual(affect.introduced, "0.2.0") 123 | self.assertEqual(affect.fixed, "1.0.0") 124 | -------------------------------------------------------------------------------- /tests/test_clearlydefined.py: -------------------------------------------------------------------------------- 1 | 2 | import unittest 3 | 4 | from ghastoolkit.octokit.clearlydefined import ClearlyDefined 5 | from ghastoolkit.supplychain.dependencies import Dependency 6 | 7 | 8 | class TestClearly(unittest.TestCase): 9 | def test_url_builder(self): 10 | clearly = ClearlyDefined() 11 | dep = Dependency("requests", manager="pypi") 12 | 13 | url = clearly.createCurationUrl(dep) 14 | self.assertEqual(url, f"{clearly.api}/curations/pypi/pypi/-/requests") 15 | 16 | 17 | -------------------------------------------------------------------------------- /tests/test_codeql_dataext.py: -------------------------------------------------------------------------------- 1 | 2 | import unittest 3 | from ghastoolkit.codeql.dataextensions.ext import DataExtensions 4 | 5 | from ghastoolkit.codeql.dataextensions.models import CompiledSinks 6 | 7 | 8 | class TestDataExtModels(unittest.TestCase): 9 | def test_dataext(self): 10 | de = DataExtensions("python") 11 | self.assertEqual(de.pack, f"codeql/python-queries") 12 | 13 | 14 | def test_generation_compiled(self): 15 | mad = ["java.net", "Socket", True, "Socket", "(String,int)", "", "Argument[0]", "request-forgery", "manual"] 16 | model = CompiledSinks(*mad) 17 | 18 | self.assertEqual(model.package, "java.net") 19 | self.assertEqual(model.object_type, "Socket") 20 | self.assertEqual(model.subtypes, True) 21 | self.assertEqual(model.name, "Socket") 22 | self.assertEqual(model.signature, "(String,int)") 23 | self.assertEqual(model.ext, "") 24 | self.assertEqual(model.object_input, "Argument[0]") 25 | self.assertEqual(model.kind, "request-forgery") 26 | self.assertEqual(model.provenance, "manual") 27 | 28 | self.assertEqual(model.generate(), mad) 29 | 30 | -------------------------------------------------------------------------------- /tests/test_codeql_packs.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from ghastoolkit.codeql.packs.pack import CodeQLPack 4 | 5 | 6 | class TestCodeQLPacks(unittest.TestCase): 7 | def setUp(self) -> None: 8 | self.pack = CodeQLPack(name="geekmasher/test", version="1.0.0") 9 | return super().setUp() 10 | 11 | def test_loaded(self): 12 | self.assertEqual(self.pack.name, "geekmasher/test") 13 | self.assertEqual(self.pack.version, "1.0.0") 14 | 15 | def test_qlpack(self): 16 | # assume without path is default 17 | self.assertEqual(self.pack.qlpack, "qlpack.yml") 18 | 19 | def test_update_version_major(self): 20 | version = self.pack.updateVersion("major") 21 | self.assertEqual(version, "2.0.0") 22 | self.assertEqual(self.pack.version, "2.0.0") 23 | 24 | def test_update_version_minor(self): 25 | version = self.pack.updateVersion("minor") 26 | self.assertEqual(version, "1.1.0") 27 | self.assertEqual(self.pack.version, "1.1.0") 28 | 29 | def test_update_version_patch(self): 30 | version = self.pack.updateVersion("patch") 31 | self.assertEqual(version, "1.0.1") 32 | self.assertEqual(self.pack.version, "1.0.1") 33 | 34 | def test_update_pack(self): 35 | self.pack.path = None 36 | data = self.pack.updatePack() 37 | self.assertTrue(isinstance(data, dict)) 38 | self.assertEqual(data.get("name"), "geekmasher/test") 39 | self.assertEqual(data.get("version"), "1.0.0") 40 | self.assertFalse(data.get("library")) 41 | 42 | # by default, don't export it 43 | self.assertIsNone(data.get("dependencies")) 44 | self.assertIsNone(data.get("defaultSuiteFile")) 45 | -------------------------------------------------------------------------------- /tests/test_codeqldb.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import os 3 | import tempfile 4 | import unittest 5 | 6 | import yaml 7 | 8 | from ghastoolkit.codeql.databases import CodeQLDatabase, Repository 9 | 10 | 11 | class TestCodeQLDb(unittest.TestCase): 12 | def setUp(self): 13 | self.repo = Repository("GeekMasher", "ghastoolkit") 14 | 15 | self.codeql_path_yml = os.path.join(tempfile.gettempdir(), "codeql.yml") 16 | 17 | def test_path(self): 18 | codeql = CodeQLDatabase("db", "java", self.repo) 19 | self.assertEqual( 20 | codeql.createDownloadPath("/tmp"), 21 | os.path.join("/tmp", "java", "GeekMasher", "ghastoolkit"), 22 | ) 23 | 24 | codeql = CodeQLDatabase("db", "java") 25 | self.assertEqual( 26 | codeql.createDownloadPath("/tmp"), os.path.join("/tmp", "java", "db") 27 | ) 28 | 29 | def test_pack(self): 30 | codeql = CodeQLDatabase("db", "java", self.repo) 31 | self.assertEqual(codeql.default_pack, "codeql/java-queries") 32 | 33 | def test_suite(self): 34 | codeql = CodeQLDatabase("db", "java", self.repo) 35 | self.assertEqual(codeql.getSuite("code-scanning"), "codeql/java-queries:codeql-suites/java-code-scanning.qls") 36 | 37 | def test_yml_loading(self): 38 | data = { 39 | "sourceLocationPrefix": "/tmp/ghastoolkit", 40 | "baselineLinesOfCode": 42069, 41 | "primaryLanguage": "python", 42 | "creationMetadata": { 43 | "creationTime": "2023-01-01T16:20:00.000000000Z" 44 | } 45 | } 46 | with open(self.codeql_path_yml, 'w') as handle: 47 | yaml.safe_dump(data, handle) 48 | 49 | self.assertTrue(os.path.exists(self.codeql_path_yml)) 50 | 51 | db = CodeQLDatabase.loadFromYml(self.codeql_path_yml) 52 | self.assertEqual(db.name, "ghastoolkit") 53 | self.assertEqual(db.language, "python") 54 | self.assertEqual(db.loc_baseline, 42069) 55 | 56 | time = datetime.fromisoformat("2023-01-01T16:20:00") 57 | self.assertEqual(db.created, time) 58 | 59 | -------------------------------------------------------------------------------- /tests/test_codescanning.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import responses 3 | import utils 4 | 5 | from ghastoolkit import GHASToolkitError, CodeScanning, GitHub 6 | from ghastoolkit.octokit.codescanning import ( 7 | CodeScanningAnalysis, 8 | CodeScanningAnalysisEnvironment, 9 | CodeScanningTool, 10 | ) 11 | 12 | from ghastoolkit.octokit.codescanning import CodeScanning 13 | from ghastoolkit.octokit.github import GitHub 14 | from ghastoolkit.octokit.repository import Repository 15 | 16 | 17 | class TestCodeScanning(unittest.TestCase): 18 | def setUp(self) -> None: 19 | GitHub.init("GeekMasher/ghastoolkit") 20 | return super().setUp() 21 | 22 | def test_codescanning_default(self): 23 | cs = CodeScanning() 24 | self.assertEqual(cs.repository.display(), "GeekMasher/ghastoolkit") 25 | 26 | cs = CodeScanning(GitHub.repository) 27 | self.assertEqual(cs.repository.display(), "GeekMasher/ghastoolkit") 28 | 29 | GitHub.init("Sample/Repo") 30 | cs = CodeScanning(GitHub.repository) 31 | self.assertEqual(cs.repository.display(), "Sample/Repo") 32 | 33 | @responses.activate 34 | def test_default(self): 35 | utils.loadResponses("codescanning.json", "default") 36 | 37 | repo = Repository("GeekMasher", "ghastoolkit", reference="refs/heads/main") 38 | 39 | codescanning = CodeScanning(repo) 40 | analyses = codescanning.getAnalyses(GitHub.repository.reference) 41 | 42 | self.assertEqual(len(analyses), 1) 43 | 44 | analysis = analyses[0] 45 | self.assertEqual(type(analysis), CodeScanningAnalysis) 46 | self.assertEqual(analysis.ref, "refs/head/main") 47 | 48 | @responses.activate 49 | def test_errors(self): 50 | utils.loadResponses("codescanning.json", "errors") 51 | 52 | repo = Repository("GeekMasher", "ghastoolkit", reference="refs/pull/1/head") 53 | 54 | codescanning = CodeScanning(repo) 55 | 56 | with self.assertRaises(GHASToolkitError): 57 | codescanning.getAnalyses(repo.reference) 58 | 59 | @responses.activate 60 | def test_retries(self): 61 | utils.loadResponses("codescanning.json", "retries") 62 | 63 | # Reference is a Default Setup for a Pull Request 64 | repo = Repository("GeekMasher", "ghastoolkit", reference="refs/pull/1/head") 65 | self.assertTrue(repo.isInPullRequest()) 66 | 67 | # Enable retries 68 | codescanning = CodeScanning(repo, retry_count=5, retry_sleep=0) 69 | self.assertEqual(codescanning.retry_count, 5) 70 | self.assertEqual(codescanning.retry_sleep, 0) 71 | 72 | analyses = codescanning.getAnalyses(repo.reference) 73 | 74 | self.assertEqual(len(analyses), 1) 75 | analysis = analyses[0] 76 | self.assertEqual(type(analysis), CodeScanningAnalysis) 77 | self.assertEqual(analysis.ref, "refs/pull/1/head") 78 | 79 | # Test the dicts are all loaded correctly 80 | self.assertEqual(type(analysis.tool), CodeScanningTool) 81 | self.assertEqual(analysis.tool.name, "CodeQL") 82 | 83 | self.assertEqual(type(analysis.environment), CodeScanningAnalysisEnvironment) 84 | self.assertEqual(analysis.environment.language, "python") 85 | -------------------------------------------------------------------------------- /tests/test_default.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from ghastoolkit import * 4 | 5 | 6 | class TestDefault(unittest.TestCase): 7 | def test_supplychain(self): 8 | gp = DependencyGraph() 9 | deps = Dependencies() 10 | dep = Dependency("ghastoolkit") 11 | 12 | alert = DependencyAlert( 13 | 0, 14 | "open", 15 | "high", 16 | advisory=Advisory("0000", "high"), 17 | purl="pypi/ghastoolkit", 18 | ) 19 | 20 | advisory = Advisory("ghas-0000-0000", "high") 21 | 22 | def test_codescanning(self): 23 | cs = CodeScanning() 24 | alert = CodeAlert(0, "open", "", {}, {}) 25 | 26 | def test_codeql(self): 27 | codeql = CodeQL("codeql") 28 | alerts = CodeQLResults() 29 | 30 | dataext = DataExtensions("python") 31 | 32 | pack = CodeQLPack() 33 | packs = CodeQLPacks() 34 | 35 | def test_secretscanning(self): 36 | ss = SecretScanning() 37 | alert = SecretAlert( 38 | 0, "open", "", "geekmasher_token", "GeekMasher Token", "ABCD" 39 | ) 40 | 41 | def test_licenses(self): 42 | l = Licenses() 43 | -------------------------------------------------------------------------------- /tests/test_dependabot.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import responses 4 | import utils 5 | from ghastoolkit import Dependabot, Dependency, DependencyAlert, Advisory 6 | 7 | class TestDependabot(unittest.TestCase): 8 | 9 | @responses.activate 10 | def test_api(self): 11 | utils.loadResponses("dependabot.json", "alerts") 12 | dependabot = Dependabot() 13 | 14 | alerts = dependabot.getAlerts("open") 15 | 16 | self.assertEqual(len(alerts), 2) 17 | 18 | alert1 = alerts[0] 19 | self.assertIsInstance(alert1, DependencyAlert) 20 | self.assertIsInstance(alert1.advisory, Advisory) 21 | 22 | # EPSS 23 | self.assertIsInstance(alert1.advisory.epss, list) 24 | self.assertIsInstance(alert1.advisory.epss_percentage, float) 25 | self.assertEqual(alert1.advisory.epss_percentage, 0.00045) 26 | self.assertIsInstance(alert1.advisory.epss_percentile, str) 27 | self.assertEqual(alert1.advisory.epss_percentile, "0.16001e0") 28 | -------------------------------------------------------------------------------- /tests/test_dependencies.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from ghastoolkit import Dependencies, Dependency, Licenses 4 | 5 | 6 | class TestDependencies(unittest.TestCase): 7 | def setUp(self) -> None: 8 | self.deps = Dependencies() 9 | self.deps.append(Dependency("urllib3", manager="pypi", license="MIT")) 10 | self.deps.append(Dependency("rich", manager="pypi", license="NOASSERTION")) 11 | self.deps.append(Dependency("pyyaml", manager="pypi", license="GPL-3.0")) 12 | self.deps.append(Dependency("pyproject-hooks", manager="pypi", license="Apache-2.0")) 13 | self.deps.append(Dependency("requests", manager="pypi", license="GPL-2.0")) 14 | return super().setUp() 15 | 16 | def test_license(self): 17 | mit = self.deps.findLicenses(["MIT"]) 18 | self.assertEqual(len(mit), 1) 19 | self.assertTrue(isinstance(self.deps.pop("urllib3"), Dependency)) 20 | 21 | gpl = self.deps.findLicenses(["GPL-3.0", "GPL-2.0"]) 22 | self.assertEqual(len(gpl), 2) 23 | self.assertTrue(isinstance(self.deps.pop("pyyaml"), Dependency)) 24 | self.assertTrue(isinstance(self.deps.pop("requests"), Dependency)) 25 | 26 | def test_license_wildcard(self): 27 | gpl = self.deps.findLicenses(["GPL-*"]) 28 | self.assertEqual(len(gpl), 2) 29 | self.assertTrue(isinstance(self.deps.pop('pyyaml'), Dependency)) 30 | self.assertTrue(isinstance(self.deps.pop('requests'), Dependency)) 31 | 32 | def test_findName(self): 33 | pys = self.deps.findNames(["py*"]) 34 | self.assertEqual(len(pys), 2) 35 | self.assertTrue(isinstance(self.deps.pop("pyyaml"), Dependency)) 36 | self.assertTrue(isinstance(self.deps.pop("pyproject-hooks"), Dependency)) 37 | 38 | def test_find(self): 39 | dep = self.deps.find("pyyaml") 40 | self.assertIsNotNone(dep) 41 | assert dep is not None 42 | self.assertEqual(dep.name, "pyyaml") 43 | 44 | def test_apply_license(self): 45 | deps = self.deps.findUnknownLicenses() 46 | self.assertEqual(len(deps), 1) 47 | 48 | licenses = Licenses() 49 | licenses.add("pkg:pypi/rich", ["MIT"]) 50 | 51 | self.deps.applyLicenses(licenses) 52 | 53 | deps = self.deps.findUnknownLicenses() 54 | self.assertEqual(len(deps), 0) 55 | 56 | dep = self.deps.find("rich") 57 | self.assertEqual(dep.name, "rich") 58 | self.assertEqual(dep.license, "MIT") 59 | 60 | def test_update_dep(self): 61 | dep = Dependency("urllib3", manager="pypi", license="Apache-2") 62 | 63 | self.deps.updateDependency(dep) 64 | 65 | urllib_dep = self.deps.find("urllib3") 66 | self.assertEqual(urllib_dep.name, "urllib3") 67 | self.assertEqual(urllib_dep.license, "Apache-2") 68 | 69 | def test_hashable(self): 70 | dep = Dependency("urllib3", manager="pypi", license="MIT") 71 | self.assertEqual(hash(dep), hash(dep.getPurl())) 72 | 73 | def test_contains(self): 74 | dep = Dependency("urllib3", manager="pypi", license="MIT") 75 | self.assertTrue(self.deps.contains(dep)) 76 | 77 | dep = Dependency("random-lib", manager="pypi", license="MIT") 78 | self.assertFalse(self.deps.contains(dep)) 79 | 80 | # version is ignored 81 | dep = Dependency("urllib3", manager="pypi", license="MIT", version="1.26.5") 82 | self.assertTrue(self.deps.contains(dep)) 83 | 84 | self.assertFalse(self.deps.contains(dep, version=True)) 85 | 86 | def test_extends(self): 87 | deps = Dependencies() 88 | deps.append(Dependency("random-lib", manager="pypi", license="MIT")) 89 | deps.append(Dependency("random-lib2", manager="pypi", license="MIT")) 90 | 91 | self.deps.extend(deps) 92 | 93 | self.assertEqual(len(self.deps), 7) 94 | self.assertTrue(isinstance(self.deps.pop("random-lib"), Dependency)) 95 | self.assertTrue(isinstance(self.deps.pop("random-lib2"), Dependency)) 96 | 97 | def test_is_direct(self): 98 | # Test with relationship="direct" 99 | dep = Dependency("direct-dep", relationship="direct") 100 | self.assertTrue(dep.isDirect()) 101 | 102 | dep = Dependency("indirect-dep", relationship="indirect") 103 | self.assertFalse(dep.isDirect()) 104 | 105 | # Test npm ecosystem 106 | dep = Dependency("npm-direct", manager="npm", path="package.json") 107 | self.assertTrue(dep.isDirect()) 108 | 109 | dep = Dependency("npm-indirect", manager="npm", path="node_modules/some-lib/package.json") 110 | self.assertTrue(dep.isDirect()) 111 | 112 | # Test maven ecosystem 113 | dep = Dependency("maven-direct", manager="maven", path="pom.xml") 114 | self.assertTrue(dep.isDirect()) 115 | 116 | dep = Dependency("maven-indirect", manager="maven") 117 | self.assertFalse(dep.isDirect()) 118 | 119 | # Test pip ecosystem 120 | dep = Dependency("pip-direct", manager="pip", path="requirements.txt") 121 | self.assertTrue(dep.isDirect()) 122 | 123 | dep = Dependency("pip-indirect", manager="pip", path="venv/lib/something.txt") 124 | self.assertFalse(dep.isDirect()) 125 | 126 | # Test without path 127 | dep = Dependency("no-path", manager="npm") 128 | self.assertFalse(dep.isDirect()) 129 | -------------------------------------------------------------------------------- /tests/test_depgraph.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from ghastoolkit.octokit.dependencygraph import Dependency, Dependencies 4 | from ghastoolkit.supplychain.licensing import Licenses 5 | 6 | 7 | class TestDepGraph(unittest.TestCase): 8 | def test_dependency(self): 9 | dep = Dependency("django", version="1.11.1", manager="pypi") 10 | 11 | self.assertEqual(dep.name, "django") 12 | self.assertEqual(dep.manager, "pypi") 13 | self.assertEqual(dep.version, "1.11.1") 14 | 15 | def test_fullname_maven(self): 16 | dep = Dependency("express", manager="npm") 17 | self.assertEqual(dep.fullname, "express") 18 | 19 | dep = Dependency( 20 | "spring-boot-starter-web", "org.springframework.boot", manager="maven" 21 | ) 22 | self.assertEqual( 23 | dep.fullname, "org.springframework.boot:spring-boot-starter-web" 24 | ) 25 | 26 | def test_purl(self): 27 | # python 28 | dep = Dependency("django", version="1.11.1", manager="pypi") 29 | self.assertEqual(dep.getPurl(), "pkg:pypi/django@1.11.1") 30 | 31 | # go 32 | dep = Dependency("genproto", "google.golang.org", manager="golang") 33 | self.assertEqual(dep.getPurl(), "pkg:golang/google.golang.org/genproto") 34 | 35 | def test_purl_from(self): 36 | # GitHub Actions 37 | dep = Dependency.fromPurl("pkg:githubactions/actions/setup-python@3") 38 | self.assertEqual(dep.name, "setup-python") 39 | self.assertEqual(dep.namespace, "actions") 40 | self.assertEqual(dep.manager, "githubactions") 41 | self.assertEqual(dep.version, "3") 42 | 43 | dep = Dependency.fromPurl("pkg:githubactions/github/codeql-action/analyze@2") 44 | self.assertEqual(dep.name, "codeql-action/analyze") 45 | self.assertEqual(dep.namespace, "github") 46 | self.assertEqual(dep.manager, "githubactions") 47 | self.assertEqual(dep.version, "2") 48 | 49 | # PyPi 50 | dep = Dependency.fromPurl("pkg:pypi/requests@2.28.2") 51 | self.assertEqual(dep.name, "requests") 52 | self.assertEqual(dep.manager, "pypi") 53 | self.assertEqual(dep.version, "2.28.2") 54 | 55 | def test_purl_from_basic(self): 56 | dep = Dependency.fromPurl("npm/ini") 57 | self.assertEqual(dep.name, "ini") 58 | self.assertEqual(dep.manager, "npm") 59 | 60 | -------------------------------------------------------------------------------- /tests/test_github.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from ghastoolkit.octokit.github import GitHub, Repository 4 | 5 | 6 | class TestGitHub(unittest.TestCase): 7 | def setUp(self) -> None: 8 | return super().setUp() 9 | 10 | def tearDown(self) -> None: 11 | # reset 12 | GitHub.owner = None 13 | GitHub.instance = "https://github.com" 14 | GitHub.api_rest = "https://api.github.com" 15 | GitHub.api_graphql = "https://api.github.com/graphql" 16 | 17 | return super().tearDown() 18 | 19 | def test_default(self): 20 | GitHub.init("GeekMasher/ghastoolkit") 21 | 22 | self.assertEqual(GitHub.instance, "https://github.com") 23 | self.assertEqual(GitHub.api_rest, "https://api.github.com") 24 | self.assertEqual(GitHub.api_graphql, "https://api.github.com/graphql") 25 | 26 | def test_server(self): 27 | GitHub.init("GeekMasher/ghastoolkit", instance="https://github.geekmasher.dev") 28 | 29 | self.assertEqual(GitHub.instance, "https://github.geekmasher.dev") 30 | self.assertEqual(GitHub.api_rest, "https://github.geekmasher.dev/api/v3") 31 | self.assertEqual( 32 | GitHub.api_graphql, "https://github.geekmasher.dev/api/graphql" 33 | ) 34 | 35 | def test_parseReference(self): 36 | repo = Repository.parseRepository("GeekMasher/ghastoolkit") 37 | self.assertEqual(repo.owner, "GeekMasher") 38 | self.assertEqual(repo.repo, "ghastoolkit") 39 | 40 | repo = Repository.parseRepository("GeekMasher/ghastoolkit@main") 41 | self.assertEqual(repo.owner, "GeekMasher") 42 | self.assertEqual(repo.repo, "ghastoolkit") 43 | self.assertEqual(repo.branch, "main") 44 | self.assertEqual(repo.reference, "refs/heads/main") 45 | 46 | def test_owner(self): 47 | GitHub.init("MyOrg") 48 | self.assertEqual(GitHub.owner, "MyOrg") 49 | self.assertEqual(GitHub.getOrganization(), "MyOrg") 50 | 51 | GitHub.init("MyOtherOrg/repo") 52 | self.assertEqual(GitHub.owner, "MyOtherOrg") 53 | self.assertEqual(GitHub.getOrganization(), "MyOtherOrg") 54 | 55 | def test_token_type(self): 56 | GitHub.init(token="github_pat_1234567890") 57 | self.assertEqual(GitHub.token, "github_pat_1234567890") 58 | self.assertEqual(GitHub.validateTokenType(GitHub.token or ""), "PAT") 59 | 60 | GitHub.init(token="gho_1234567890") 61 | self.assertEqual(GitHub.token, "gho_1234567890") 62 | self.assertEqual(GitHub.validateTokenType(GitHub.token or ""), "OAUTH") 63 | 64 | GitHub.token = None 65 | 66 | 67 | class TestRepository(unittest.TestCase): 68 | def setUp(self) -> None: 69 | GitHub.token = None 70 | GitHub.github_app = False 71 | 72 | return super().setUp() 73 | 74 | def test_parse_repository(self): 75 | repo = Repository.parseRepository("GeekMasher/ghastoolkit") 76 | self.assertEqual(repo.owner, "GeekMasher") 77 | self.assertEqual(repo.repo, "ghastoolkit") 78 | 79 | repo = Repository.parseRepository("GeekMasher/ghastoolkit@develop") 80 | self.assertEqual(repo.owner, "GeekMasher") 81 | self.assertEqual(repo.repo, "ghastoolkit") 82 | self.assertEqual(repo.branch, "develop") 83 | self.assertEqual(repo.reference, "refs/heads/develop") 84 | 85 | def test_parse_repository_path(self): 86 | repo = Repository.parseRepository("GeekMasher/ghastoolkit:sub/folder") 87 | self.assertEqual(repo.owner, "GeekMasher") 88 | self.assertEqual(repo.repo, "ghastoolkit") 89 | self.assertEqual(repo.path, "sub/folder") 90 | 91 | repo = Repository.parseRepository( 92 | "GeekMasher/ghastoolkit:this/other/file.yml@develop" 93 | ) 94 | self.assertEqual(repo.owner, "GeekMasher") 95 | self.assertEqual(repo.repo, "ghastoolkit") 96 | self.assertEqual(repo.path, "this/other/file.yml") 97 | self.assertEqual(repo.branch, "develop") 98 | 99 | repo = Repository.parseRepository("GeekMasher/ghas.toolkit") 100 | self.assertEqual(repo.owner, "GeekMasher") 101 | self.assertEqual(repo.repo, "ghas.toolkit") 102 | 103 | def test_parse_repository_path_alt(self): 104 | repo = Repository.parseRepository("GeekMasher/ghastoolkit/sub/folder") 105 | self.assertEqual(repo.owner, "GeekMasher") 106 | self.assertEqual(repo.repo, "ghastoolkit") 107 | self.assertEqual(repo.path, "sub/folder") 108 | 109 | def test_parse_repository_invalid(self): 110 | # only owner 111 | with self.assertRaises(SyntaxError): 112 | Repository.parseRepository("GeekMasher") 113 | # multiple branches 114 | with self.assertRaises(SyntaxError): 115 | Repository.parseRepository("GeekMasher/ghastoolkit@develop@main") 116 | # invalid path separator 117 | with self.assertRaises(SyntaxError): 118 | Repository.parseRepository("GeekMasher/ghastoolkit\\test") 119 | 120 | def test_branch(self): 121 | repo = Repository("GeekMasher", "ghastoolkit", reference="refs/heads/main") 122 | self.assertEqual(repo.reference, "refs/heads/main") 123 | self.assertEqual(repo.branch, "main") 124 | 125 | repo = Repository( 126 | "GeekMasher", "ghastoolkit", reference="refs/heads/random-branch/name" 127 | ) 128 | self.assertEqual(repo.reference, "refs/heads/random-branch/name") 129 | self.assertEqual(repo.branch, "random-branch/name") 130 | 131 | def test_branch_tag(self): 132 | repo = Repository("GeekMasher", "ghastoolkit", reference="refs/tags/0.4.0") 133 | self.assertEqual(repo.reference, "refs/tags/0.4.0") 134 | self.assertEqual(repo.branch, "0.4.0") 135 | 136 | def test_pull_request(self): 137 | repo = Repository("GeekMasher", "ghastoolkit", reference="refs/heads/main") 138 | self.assertFalse(repo.isInPullRequest()) 139 | 140 | repo = Repository("GeekMasher", "ghastoolkit", reference="refs/pull/1/merge") 141 | self.assertTrue(repo.isInPullRequest()) 142 | self.assertEqual(repo.getPullRequestNumber(), 1) 143 | 144 | def test_clone_url(self): 145 | repo = Repository("GeekMasher", "ghastoolkit") 146 | self.assertEqual( 147 | repo.clone_url, "https://github.com/GeekMasher/ghastoolkit.git" 148 | ) 149 | 150 | GitHub.token = "test_token" 151 | self.assertEqual( 152 | repo.clone_url, "https://test_token@github.com/GeekMasher/ghastoolkit.git" 153 | ) 154 | 155 | GitHub.github_app = True 156 | GitHub.token = "test_token" 157 | self.assertEqual( 158 | repo.clone_url, 159 | "https://x-access-token:test_token@github.com/GeekMasher/ghastoolkit.git", 160 | ) 161 | 162 | def test_clone_cmd(self): 163 | path = "/tml/ghastoolkit" 164 | repo = Repository("GeekMasher", "ghastoolkit") 165 | 166 | cmd = ["git", "clone", repo.clone_url, path] 167 | self.assertEqual(repo._cloneCmd(path), cmd) 168 | 169 | cmd = ["git", "clone", "--depth", "1", repo.clone_url, path] 170 | self.assertEqual(repo._cloneCmd(path, depth=1), cmd) 171 | 172 | repo.branch = "main" 173 | cmd = ["git", "clone", "-b", "main", repo.clone_url, path] 174 | self.assertEqual(repo._cloneCmd(path), cmd) 175 | 176 | def test_clone_file(self): 177 | path = "README.md" 178 | repo = Repository("GeekMasher", "ghastoolkit") 179 | repo.getFile(path) 180 | -------------------------------------------------------------------------------- /tests/test_licenses.py: -------------------------------------------------------------------------------- 1 | 2 | import unittest 3 | 4 | from ghastoolkit.supplychain.dependencies import Dependencies, Dependency 5 | from ghastoolkit.supplychain.licensing import Licenses 6 | 7 | 8 | class TestLicensing(unittest.TestCase): 9 | def setUp(self) -> None: 10 | self.licenses = Licenses() 11 | self.licenses.add("pkg:maven/com.geekmasher/ghastoolkit", ["MIT"]) 12 | 13 | return super().setUp() 14 | 15 | def test_find(self): 16 | result = self.licenses.find("pkg:maven/com.geekmasher/ghastoolkit") 17 | self.assertEqual(result, ["MIT"]) 18 | 19 | def test_apply(self): 20 | dependencies = Dependencies() 21 | dependencies.append(Dependency("ghastoolkit", "com.geekmasher", manager="maven")) 22 | self.assertEqual(len(dependencies), 1) 23 | 24 | dependencies.applyLicenses(self.licenses) 25 | 26 | dep = dependencies.pop('ghastoolkit') 27 | self.assertTrue(isinstance(dep, Dependency)) 28 | self.assertEqual(dep.license, "MIT") 29 | -------------------------------------------------------------------------------- /tests/test_octokit.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, is_dataclass 2 | import unittest 3 | 4 | from ghastoolkit.octokit.octokit import ( 5 | OctoItem, 6 | Octokit, 7 | GraphQLRequest, 8 | loadOctoItem, 9 | ) 10 | from ghastoolkit.octokit.github import GitHub 11 | 12 | 13 | class TestOctokit(unittest.TestCase): 14 | def setUp(self) -> None: 15 | GitHub.init(repository="GeekMasher/ghastoolkit@main", token="1234567890") 16 | return super().setUp() 17 | 18 | def test_route(self): 19 | route = Octokit.route( 20 | "/repos/{owner}/{repo}/secret-scanning/alerts", GitHub.repository 21 | ) 22 | self.assertEqual( 23 | route, 24 | "https://api.github.com/repos/GeekMasher/ghastoolkit/secret-scanning/alerts", 25 | ) 26 | 27 | 28 | @dataclass 29 | class Example(OctoItem): 30 | number: int 31 | 32 | 33 | class TestLoadOctoItem(unittest.TestCase): 34 | def test_load(self): 35 | item = loadOctoItem(Example, {"number": 5}) 36 | 37 | self.assertTrue(isinstance(item, Example)) 38 | self.assertTrue(is_dataclass(item)) 39 | 40 | self.assertEqual(item.number, 5) 41 | 42 | 43 | class TestOctokitGraphQL(unittest.TestCase): 44 | def setUp(self) -> None: 45 | GitHub.init(repository="GeekMasher/ghastoolkit@main") 46 | return super().setUp() 47 | 48 | def test_loading_defaults(self): 49 | gql = GraphQLRequest() 50 | # load 3 default queries 51 | self.assertEqual(len(gql.queries.keys()), 3) 52 | 53 | query1 = gql.queries.get("GetDependencyAlerts") 54 | self.assertIsNotNone(query1) 55 | 56 | query2 = gql.queries.get("GetDependencyInfo") 57 | self.assertIsNotNone(query2) 58 | -------------------------------------------------------------------------------- /tests/test_restrequest.py: -------------------------------------------------------------------------------- 1 | """Test RestRequest class.""" 2 | 3 | import unittest 4 | import utils 5 | from ghastoolkit.errors import GHASToolkitError 6 | import responses 7 | 8 | from ghastoolkit.octokit.octokit import RestRequest 9 | from ghastoolkit.octokit.github import GitHub 10 | 11 | 12 | class TestRestRequest(unittest.TestCase): 13 | def setUp(self) -> None: 14 | GitHub.init(repository="GeekMasher/ghastoolkit@main") 15 | 16 | self.rest = RestRequest() 17 | 18 | return super().setUp() 19 | 20 | @responses.activate 21 | def test_errors(self): 22 | utils.loadResponses("restrequests.json", "errors") 23 | 24 | with self.assertRaises(GHASToolkitError): 25 | self.rest.get("/repos/{owner}/{repo}/secret-scanning/alerts") 26 | 27 | with self.assertRaises(GHASToolkitError): 28 | self.rest.get("/repos/{owner}/{repo}/secret-scanning/alerts/1") 29 | 30 | @responses.activate 31 | def test_error_handler(self): 32 | utils.loadResponses("restrequests.json", "error_handler") 33 | 34 | def handle(code, _): 35 | self.assertEqual(code, 404) 36 | 37 | self.rest.get( 38 | "/repos/{owner}/{repo}/secret-scanning/alerts", error_handler=handle 39 | ) 40 | -------------------------------------------------------------------------------- /tests/test_secretscanning.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from datetime import datetime, timedelta 3 | 4 | from ghastoolkit.octokit.github import GitHub 5 | from ghastoolkit.octokit.secretscanning import ( 6 | SecretAlert, 7 | SecretScanning, 8 | ) 9 | 10 | 11 | class TestSecretScanning(unittest.TestCase): 12 | def setUp(self) -> None: 13 | GitHub.init("GeekMasher/ghastoolkit") 14 | return super().setUp() 15 | 16 | def test_secretscanning_default(self): 17 | ss = SecretScanning() 18 | self.assertEqual(ss.repository.display(), "GeekMasher/ghastoolkit") 19 | 20 | ss = SecretScanning(GitHub.repository) 21 | self.assertEqual(ss.repository.display(), "GeekMasher/ghastoolkit") 22 | 23 | GitHub.init("Sample/Repo") 24 | ss = SecretScanning(GitHub.repository) 25 | self.assertEqual(ss.repository.display(), "Sample/Repo") 26 | 27 | 28 | class TestSecretAlert(unittest.TestCase): 29 | def test_load_alert(self): 30 | data = { 31 | "number": 23, 32 | "created_at": "2020-11-06T18:18:30Z", 33 | "state": "open", 34 | "secret_type": "mailchimp_api_key", 35 | "secret_type_display_name": "Mailchimp API Key", 36 | "secret": "ABCDEFG", 37 | "validity": "active", 38 | } 39 | alert = SecretAlert(**data) 40 | self.assertEqual(alert.number, 23) 41 | self.assertEqual(alert.state, "open") 42 | self.assertEqual(alert.secret_type, "mailchimp_api_key") 43 | self.assertEqual(alert.secret, "ABCDEFG") 44 | self.assertEqual(alert.validity, "active") 45 | 46 | def test_mttr(self): 47 | data = { 48 | "number": 23, 49 | "created_at": "2020-11-06T18:18:30Z", 50 | "state": "open", 51 | "secret_type": "mailchimp_api_key", 52 | "secret_type_display_name": "Mailchimp API Key", 53 | "secret": "ABCDEFG", 54 | "validity": "active", 55 | } 56 | alert = SecretAlert(**data) 57 | self.assertEqual(alert.mttr, None) 58 | 59 | data["resolved_at"] = "2020-11-06T18:18:30Z" 60 | alert = SecretAlert(**data) 61 | self.assertEqual(alert.mttr, timedelta(seconds=0)) 62 | 63 | data["resolved_at"] = "2020-11-06T18:18:31Z" 64 | alert = SecretAlert(**data) 65 | self.assertEqual(alert.mttr, timedelta(seconds=1)) 66 | 67 | data["resolved_at"] = "2020-11-06T18:19:30Z" 68 | alert = SecretAlert(**data) 69 | self.assertEqual(alert.mttr, timedelta(seconds=60)) 70 | 71 | data["resolved_at"] = "2020-11-06T19:19:30Z" 72 | alert = SecretAlert(**data) 73 | self.assertEqual(alert.mttr, timedelta(seconds=3660)) 74 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import responses 4 | 5 | __HERE__ = os.path.dirname(os.path.abspath(__file__)) 6 | 7 | def loadResponses(file: str, test_name: str): 8 | """ 9 | Load the responses from a file and add them to the responses library. 10 | """ 11 | path = os.path.join(__HERE__, "responses", file) 12 | if not os.path.exists(path): 13 | raise FileNotFoundError(f"File {file} not found in responses directory") 14 | with open(path, "r") as f: 15 | data = json.load(f) 16 | 17 | resps = data.get(test_name) 18 | if not resps: 19 | raise ValueError(f"Test name '{test_name}' not found in {file}") 20 | 21 | # Array of responses 22 | for resp in resps: 23 | method = resp.get("method", "GET") 24 | url = resp.get("url") 25 | content_type = resp.get("content_type", "application/json") 26 | status = resp.get("status", 200) 27 | json_data = resp.get("json", {}) 28 | 29 | responses.add( 30 | method, 31 | url, 32 | content_type=content_type, 33 | status=status, 34 | json=json_data, 35 | ) 36 | --------------------------------------------------------------------------------