├── .coveragerc ├── .dockerignore ├── .github ├── dependabot.yml └── workflows │ ├── codeql.yml │ └── gating.yaml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── bin ├── cachito-download.sh └── pip_find_builddeps.py ├── cachito ├── __init__.py ├── common │ ├── __init__.py │ ├── checksum.py │ ├── packages_data.py │ ├── paths.py │ └── utils.py ├── errors.py ├── web │ ├── __init__.py │ ├── api_v1.py │ ├── app.py │ ├── auth.py │ ├── config.py │ ├── content_manifest.py │ ├── docs.py │ ├── errors.py │ ├── manage.py │ ├── metrics.py │ ├── migrations │ │ ├── alembic.ini │ │ ├── env.py │ │ ├── script.py.mako │ │ └── versions │ │ │ ├── 01bb0873ddcb_add_the_remove_unsafe_symlinks_flag.py │ │ │ ├── 02229e089b24_delete_user_username_index.py │ │ │ ├── 044548f3f83a_add_pip_dev_preview_flag.py │ │ │ ├── 07d89c5778f2_add_cgo_disable_flag.py │ │ │ ├── 15c4aa0c4144_rename_dependency_table.py │ │ │ ├── 1714d8e3002b_map_deps_to_pkg.py │ │ │ ├── 193baf9d7cbf_config_file_base64.py │ │ │ ├── 2f83b3e4c5cc_add_request_package_subpath.py │ │ │ ├── 3c208b05d703_request_state_fk.py │ │ │ ├── 418241dba06c_add_force_gomod_tidy_flag.py │ │ │ ├── 491454f79a8b_add_packages_and_deps_count.py │ │ │ ├── 4a64656ba27f_add_git_submodule.py │ │ │ ├── 4d17dec0cfc3_adjust_flag_unique_constraint.py │ │ │ ├── 5854e700a35e_dependency_replacements.py │ │ │ ├── 615c19a1cee1_add_npm.py │ │ │ ├── 71909d479045_add_submitted_by.py │ │ │ ├── 7d979987402d_delete_dependencies_and_packages.py │ │ │ ├── 9118b23629ef_add_indexes.py │ │ │ ├── 92f0d370ba4d_add_requesterror_table.py │ │ │ ├── 976b7ef3ec86_add_created_field.py │ │ │ ├── 97d5df7fca86_add_include_git_dir_flag.py │ │ │ ├── a655a299e967_request_flags.py │ │ │ ├── b46cf36806d7_add_gomod_vendor_flag.py │ │ │ ├── c6ac095d8e9f_add_gomod_vendor_check_flag.py │ │ │ ├── c8b2a3a26191_initial_migration.py │ │ │ ├── cb6bdbe533cc_add_env_var_types.py │ │ │ ├── cdf17fad3edb_request_env_vars.py │ │ │ ├── cfbbf7675e3b_add_pip.py │ │ │ ├── e16de598d00d_add_rubygems.py │ │ │ ├── eff9db96576e_add_yarn.py │ │ │ ├── f133002ffdb4_disable_pip_dev_review.py │ │ │ ├── f201f05a95a7_add_index_to_request_repo_and_request_ref.py │ │ │ └── fdd6d6978386_add_request_package_table.py │ ├── models.py │ ├── purl.py │ ├── static │ │ ├── api_v1.yaml │ │ └── docs.html │ ├── status.py │ ├── utils.py │ ├── validation.py │ └── wsgi.py └── workers │ ├── __init__.py │ ├── celery_logging.py │ ├── cleanup_job.py │ ├── config.py │ ├── errors.py │ ├── nexus.py │ ├── nexus_scripts │ ├── js_after_content_staged.groovy │ ├── js_before_content_staged.groovy │ ├── js_cleanup.groovy │ ├── pip_after_content_staged.groovy │ ├── pip_before_content_staged.groovy │ ├── pip_cleanup.groovy │ ├── rubygems_after_content_staged.groovy │ ├── rubygems_before_content_staged.groovy │ └── rubygems_cleanup.groovy │ ├── paths.py │ ├── pkg_managers │ ├── __init__.py │ ├── general.py │ ├── general_js.py │ ├── gomod.py │ ├── npm.py │ ├── pip.py │ ├── rubygems.py │ └── yarn.py │ ├── prune_archives.py │ ├── requests.py │ ├── scm.py │ └── tasks │ ├── __init__.py │ ├── celery.py │ ├── general.py │ ├── gitsubmodule.py │ ├── gomod.py │ ├── npm.py │ ├── pip.py │ ├── rubygems.py │ ├── utils.py │ └── yarn.py ├── docker-compose.yml ├── docker ├── Dockerfile-api ├── Dockerfile-workers ├── cachito-httpd.conf ├── configure-nexus.groovy ├── configure-nexus.py └── rabbitmq.conf ├── docs ├── dependency_confusion.md ├── metadata.md ├── pip.md ├── pull_request_template.md ├── tracing.md └── using_requests_locally.md ├── hack └── mock-unittest-data │ └── gomod.sh ├── pyproject.toml ├── requirements-test.in ├── requirements-test.txt ├── requirements-web.in ├── requirements-web.txt ├── requirements.in ├── requirements.txt ├── setup.py ├── test_env_vars.yaml ├── tests ├── __init__.py ├── conftest.py ├── helper_utils │ └── __init__.py ├── integration │ ├── README.md │ ├── __init__.py │ ├── conftest.py │ ├── test_check_downloaded_output.py │ ├── test_content_manifest.py │ ├── test_creating_new_request.py │ ├── test_data │ │ ├── cached_dependencies.yaml │ │ ├── git_submodule_packages.yaml │ │ ├── go_generate_packages.yaml │ │ ├── gomod_packages.yaml │ │ ├── gomod_vendor_check.yaml │ │ ├── npm_packages.yaml │ │ ├── pip_packages.yaml │ │ ├── private_repo_packages.yaml │ │ ├── rubygems_packages.yaml │ │ └── yarn_packages.yaml │ ├── test_dependency_replacement.py │ ├── test_get_latest_request.py │ ├── test_gomod_packages.py │ ├── test_http_get_all.py │ ├── test_packages.py │ ├── test_pip_packages.py │ ├── test_private_repos.py │ ├── test_request_error.py │ ├── test_request_metrics.py │ ├── test_run_app_from_bundle.py │ ├── test_using_cached_dependencies.py │ ├── test_valid_data_in_request.py │ └── utils.py ├── test_api_v1.py ├── test_cachito_config.py ├── test_checksum.py ├── test_common │ ├── __init__.py │ └── test_utils.py ├── test_content_manifest.py ├── test_docs.py ├── test_healthcheck.py ├── test_models.py ├── test_packages_data.py ├── test_status.py ├── test_utils.py └── test_workers │ ├── test_celery_logging.py │ ├── test_cleanup_job.py │ ├── test_config.py │ ├── test_nexus.py │ ├── test_pkg_managers │ ├── __init__.py │ ├── data │ │ ├── gomod-mocks │ │ │ ├── expected-results │ │ │ │ ├── resolve_gomod.json │ │ │ │ └── resolve_gomod_vendored.json │ │ │ ├── non-vendored │ │ │ │ ├── go_list_deps_all.json │ │ │ │ ├── go_list_deps_threedot.json │ │ │ │ ├── go_list_modules.json │ │ │ │ └── go_mod_download.json │ │ │ └── vendored │ │ │ │ ├── go_list_deps_all.json │ │ │ │ ├── go_list_deps_threedot.json │ │ │ │ └── modules.txt │ │ ├── myapp-0.1.tar │ │ ├── myapp-0.1.tar.Z │ │ ├── myapp-0.1.tar.bz2 │ │ ├── myapp-0.1.tar.fake.zip │ │ ├── myapp-0.1.tar.gz │ │ ├── myapp-0.1.tar.xz │ │ ├── myapp-0.1.zip │ │ ├── myapp-0.1.zip.fake.tar │ │ ├── myapp-without-pkg-info.tar.Z │ │ └── myapp-without-pkg-info.tar.gz │ ├── golang_git_repo.tar.gz │ ├── test_general.py │ ├── test_general_js.py │ ├── test_gomod.py │ ├── test_npm.py │ ├── test_pip.py │ ├── test_rubygems.py │ └── test_yarn.py │ ├── test_prune_archives.py │ ├── test_scm.py │ ├── test_tasks │ ├── __init__.py │ ├── test_general.py │ ├── test_gitsubmodule.py │ ├── test_gomod.py │ ├── test_npm.py │ ├── test_pip.py │ ├── test_rubygems.py │ ├── test_utils.py │ └── test_yarn.py │ └── test_worker_utils.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | skip_covered = True 3 | show_missing = True 4 | fail_under = 90 5 | exclude_lines = 6 | pragma: no cover 7 | if __name__ == .__main__.: 8 | def __repr__ 9 | omit = 10 | cachito/web/manage.py 11 | cachito/web/wsgi.py 12 | cachito/web/config.py 13 | cachito/web/migrations* 14 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .env/ 2 | .idea/ 3 | .pytest_cache/ 4 | .tox/ 5 | .vscode/ 6 | build/ 7 | dist/ 8 | htmlcov/ 9 | tmp/ 10 | venv/ 11 | *.egg-info/ 12 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Please see the documentation for all configuration options: 2 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 3 | 4 | version: 2 5 | updates: 6 | - package-ecosystem: "github-actions" 7 | directory: "/" 8 | schedule: 9 | # Check for updates to GitHub Actions every week 10 | interval: "weekly" 11 | - package-ecosystem: "pip" 12 | directory: "/" 13 | schedule: 14 | interval: "weekly" 15 | groups: 16 | patch-and-minor-updates: 17 | update-types: 18 | - "patch" 19 | - "minor" 20 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | schedule: 9 | - cron: "56 22 * * 2" 10 | 11 | jobs: 12 | analyze: 13 | name: Analyze 14 | runs-on: ubuntu-latest 15 | permissions: 16 | actions: read 17 | contents: read 18 | security-events: write 19 | 20 | strategy: 21 | fail-fast: false 22 | matrix: 23 | language: [ python ] 24 | 25 | steps: 26 | - name: Checkout 27 | uses: actions/checkout@v4 28 | 29 | - name: Initialize CodeQL 30 | uses: github/codeql-action/init@v3 31 | with: 32 | languages: ${{ matrix.language }} 33 | queries: +security-and-quality 34 | 35 | - name: Autobuild 36 | uses: github/codeql-action/autobuild@v3 37 | 38 | - name: Perform CodeQL Analysis 39 | uses: github/codeql-action/analyze@v3 40 | with: 41 | category: "/language:${{ matrix.language }}" 42 | -------------------------------------------------------------------------------- /.github/workflows/gating.yaml: -------------------------------------------------------------------------------- 1 | name: Gating 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - master 8 | workflow_dispatch: 9 | inputs: {} 10 | 11 | jobs: 12 | tests: 13 | name: Unit tests 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | python-version: ["3.11"] 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Install required packages 22 | run: | 23 | sudo apt-get update && sudo apt-get install -y libkrb5-dev 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v5 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | pip install tox tox-gh-actions 32 | - name: Test with tox 33 | run: tox 34 | - name: Run coveralls-python 35 | env: 36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 37 | COVERALLS_FLAG_NAME: python-${{ matrix.python-version }} 38 | COVERALLS_PARALLEL: true 39 | run: | 40 | pip3 install --upgrade pip 41 | pip3 install --upgrade setuptools 42 | pip3 install --upgrade coveralls==3.2.0 43 | coveralls --service=github 44 | 45 | coveralls-finish: 46 | name: Finish coveralls-python 47 | needs: tests 48 | runs-on: ubuntu-latest 49 | steps: 50 | - name: Finished 51 | run: | 52 | pip3 install --upgrade pip 53 | pip3 install --upgrade setuptools 54 | pip3 install --upgrade coveralls 55 | coveralls --finish --service=github 56 | env: 57 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 58 | 59 | linters: 60 | name: Linters 61 | strategy: 62 | fail-fast: false 63 | matrix: 64 | tox_env: 65 | - bandit 66 | - black 67 | - isort 68 | - flake8 69 | - mypy 70 | runs-on: ubuntu-latest 71 | steps: 72 | - uses: actions/checkout@v4 73 | with: 74 | fetch-depth: 0 75 | - name: Install required packages 76 | run: | 77 | sudo apt-get update && sudo apt-get install -y libkrb5-dev 78 | - name: Set up Python 79 | uses: actions/setup-python@v5 80 | with: 81 | python-version: "3.11" 82 | - name: Install dependencies 83 | run: | 84 | python -m pip install --upgrade pip 85 | pip install tox 86 | - name: Test '${{ matrix.tox_env }}' with tox 87 | run: tox -e ${{ matrix.tox_env }} 88 | 89 | hadolint: 90 | name: Hadolint 91 | runs-on: ubuntu-latest 92 | strategy: 93 | fail-fast: false 94 | matrix: 95 | dockerfile: 96 | - Dockerfile-api 97 | - Dockerfile-workers 98 | steps: 99 | - uses: actions/checkout@v4 100 | - uses: hadolint/hadolint-action@v3.1.0 101 | with: 102 | dockerfile: docker/${{ matrix.dockerfile }} 103 | # Ignore list: 104 | # * DL3041 - Specify version with dnf install -y - 105 | ignore: DL3041 106 | failure-threshold: warning 107 | 108 | test_swagger_editor_validator_remote: 109 | runs-on: ubuntu-latest 110 | name: Swagger Editor Validator Remote 111 | 112 | steps: 113 | - uses: actions/checkout@v4 114 | - name: Validate OpenAPI definition 115 | uses: swaggerexpert/swagger-editor-validate@v1.4.2 116 | with: 117 | definition-file: cachito/web/static/api_v1.yaml 118 | 119 | integration_tests: 120 | name: Integration tests 121 | runs-on: ubuntu-latest 122 | needs: 123 | - tests 124 | - linters 125 | - hadolint 126 | steps: 127 | - uses: actions/checkout@v4 128 | with: 129 | fetch-depth: 0 130 | - name: Install required packages 131 | run: | 132 | sudo apt-get update && sudo apt-get install -y libkrb5-dev 133 | - name: Set up Python 134 | uses: actions/setup-python@v5 135 | with: 136 | python-version: "3.11" 137 | - name: Install dependencies 138 | run: | 139 | python -m pip install --upgrade pip 140 | pip install tox 141 | - name: Start cachito services 142 | run: make run-start UP_OPTS=-d 143 | - name: Run integration tests 144 | run: tox -e integration 145 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # VS Code settings 101 | .vscode 102 | 103 | # mkdocs documentation 104 | /site 105 | 106 | # mypy 107 | .mypy_cache/ 108 | 109 | # Flask 110 | settings.py 111 | 112 | # Local application cache 113 | /dest 114 | 115 | # Local deployment storage 116 | /tmp 117 | 118 | # idea files 119 | .idea/ 120 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include cachito/web/templates * 2 | recursive-include cachito/web/static * 3 | recursive-include cachito/web/migrations * 4 | recursive-include cachito/workers/nexus_scripts * 5 | include requirements.txt requirements-web.txt LICENSE 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CACHITO_COMPOSE_ENGINE ?= docker compose 2 | PYTHON_VERSION_VENV ?= python3.11 3 | TOX_ENVLIST ?= python3.11 4 | TOX_ARGS ?= 5 | 6 | PODMAN_COMPOSE_AUTO_URL ?= https://raw.githubusercontent.com/containers/podman-compose/devel/podman_compose.py 7 | PODMAN_COMPOSE_TMP ?= tmp/podman_compose.py 8 | 9 | ifeq (podman-compose-auto,$(CACHITO_COMPOSE_ENGINE)) 10 | ifeq (,$(wildcard $(PODMAN_COMPOSE_TMP))) 11 | $(shell mkdir -p `dirname $(PODMAN_COMPOSE_TMP)`) 12 | $(shell curl -sL $(PODMAN_COMPOSE_AUTO_URL) -o $(PODMAN_COMPOSE_TMP)) 13 | $(shell chmod +x $(PODMAN_COMPOSE_TMP)) 14 | endif 15 | override CACHITO_COMPOSE_ENGINE = $(PODMAN_COMPOSE_TMP) 16 | endif 17 | 18 | # Older versions of podman-compose do not support deleting volumes via -v 19 | DOWN_HELP := $(shell ${CACHITO_COMPOSE_ENGINE} down --help) 20 | ifeq (,$(findstring volume,$(DOWN_HELP))) 21 | DOWN_OPTS := 22 | else 23 | DOWN_OPTS := -v 24 | endif 25 | 26 | UP_OPTS ?= 27 | 28 | all: venv run-start 29 | 30 | clean: run-down 31 | rm -rf venv && rm -rf *.egg-info && rm -rf dist && rm -rf *.log* && rm -rf .tox && rm -rf tmp 32 | 33 | .PHONY: venv 34 | venv: 35 | virtualenv --python=${PYTHON_VERSION_VENV} venv 36 | venv/bin/pip install --upgrade pip 37 | venv/bin/pip install -r requirements.txt -r requirements-web.txt 38 | venv/bin/pip install tox 39 | venv/bin/python setup.py develop 40 | 41 | # Keep run target for backwards compatibility 42 | run run-start: 43 | # Create the nexus volume before running (podman compatibility) 44 | mkdir -p ./tmp/nexus-data 45 | # Let everyone write to the temp directory 46 | # - nexus needs to write to ./tmp/nexus-data 47 | # - integration tests need to write to (and create) ./tmp/cachito-archives 48 | chmod -R 0777 ./tmp 49 | $(CACHITO_COMPOSE_ENGINE) up $(UP_OPTS) 50 | 51 | run-down run-stop: 52 | $(CACHITO_COMPOSE_ENGINE) down $(DOWN_OPTS) 53 | 54 | run-build run-rebuild: run-down 55 | $(CACHITO_COMPOSE_ENGINE) build 56 | 57 | # stop any containers, rebuild containers, and start it again 58 | run-build-start: run-rebuild run-start 59 | 60 | # Keep test target for backwards compatibility 61 | test test-unit: 62 | PATH="${PWD}/venv/bin:${PATH}" tox 63 | 64 | test-integration: 65 | PATH="${PWD}/venv/bin:${PATH}" tox -e integration 66 | 67 | test-suite test-tox: 68 | PATH="${PWD}/venv/bin:${PATH}" tox -e $(TOX_ENVLIST) -- $(TOX_ARGS) 69 | 70 | test-all: test-unit test-integration 71 | 72 | mock-unittest-data: 73 | hack/mock-unittest-data/gomod.sh 74 | 75 | pip-compile: 76 | venv/bin/pip install -U pip-tools 77 | # --allow-unsafe: we use pkg_resources (provided by setuptools) as a runtime dependency 78 | venv/bin/pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements.in 79 | venv/bin/pip-compile --generate-hashes --output-file=requirements-web.txt requirements-web.in 80 | venv/bin/pip-compile --generate-hashes --output-file=requirements-test.txt requirements-test.in 81 | -------------------------------------------------------------------------------- /bin/cachito-download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | usage () { 5 | echo "Usage: $(basename "$0") " 6 | } 7 | 8 | description () { 9 | cat << EOF 10 | Download a Cachito request, inject its configuration files and cachito.env 11 | 12 | $(usage) 13 | 14 | Example: 15 | $(basename "$0") localhost:8080/api/v1/requests/1 /tmp/cachito-test 16 | EOF 17 | } 18 | 19 | if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then 20 | description 21 | exit 0 22 | fi 23 | 24 | if [ $# -ne 2 ]; then 25 | usage >&2 26 | exit 1 27 | fi 28 | 29 | request_url=$1 30 | output_dir=$2 31 | 32 | error () { 33 | echo "$1" >&2 34 | return 1 35 | } 36 | 37 | check_dependencies () { 38 | command -v jq >/dev/null || error "Missing dependency: jq" 39 | } 40 | 41 | prepare_output_dir () { 42 | local output_dir=$1 43 | 44 | if [ -e "$output_dir" ]; then 45 | if [ ! -d "$output_dir" ]; then 46 | error "Cannot download to $output_dir: not a directory" 47 | fi 48 | 49 | if [ -n "$(ls -A "$output_dir")" ]; then 50 | error "Cannot download to $output_dir: already exists and is not empty" 51 | fi 52 | 53 | echo "Using existing output directory $output_dir" 54 | else 55 | echo "Using new output directory $output_dir" 56 | mkdir -p "$output_dir" 57 | fi 58 | } 59 | 60 | download_and_extract () { 61 | local request_url=$1 62 | local output_dir=$2 63 | 64 | echo "Downloading archive" 65 | # -f: fail on HTTP error code, -s: silent, -S: show error even when silent 66 | curl -fsS "$request_url/download" > "$output_dir/remote-source.tar.gz" 67 | 68 | echo "Extracting downloaded archive to remote-source/" 69 | mkdir "$output_dir/remote-source" 70 | tar -xf "$output_dir/remote-source.tar.gz" -C "$output_dir/remote-source" 71 | } 72 | 73 | inject_config_files () { 74 | local request_url=$1 75 | local output_dir=$2 76 | 77 | local config_json="$output_dir/configuration-files.json" 78 | 79 | echo "Getting configuration files" 80 | curl -fsS "$request_url/configuration-files" > "$config_json" 81 | 82 | echo "Injecting configuration files to remote-source/" 83 | jq '.[].path' -r < "$config_json" | while IFS= read -r path; do 84 | # Show the path indented by 4 spaces 85 | echo " $path" 86 | mkdir -p "$(dirname "$output_dir/remote-source/$path")" 87 | 88 | jq '.[] | select(.path == "'"$path"'") | .content' -r < "$config_json" | 89 | base64 --decode > "$output_dir/remote-source/$path" 90 | done 91 | } 92 | 93 | generate_cachito_env () { 94 | local request_url=$1 95 | local output_dir=$2 96 | 97 | local env_json="$output_dir/environment-variables.json" 98 | local cachito_env="$output_dir/remote-source/cachito.env" 99 | 100 | echo "Getting environment variables" 101 | curl -fsS "$request_url/environment-variables" > "$env_json" 102 | 103 | echo "Injecting cachito.env to remote-source/" 104 | echo "#/bin/bash" > "$cachito_env" 105 | 106 | jq 'to_entries[] | "\(.value.kind) \(.key) \(.value.value)"' -r < "$env_json" | 107 | while read -r kind key value; do 108 | if [ "$kind" = "path" ]; then 109 | value="$(realpath "$output_dir")/remote-source/$value" 110 | fi 111 | printf "export %s=%q\n" "$key" "$value" 112 | done >> "$cachito_env" 113 | 114 | # Show the generated env file indented by 4 spaces 115 | sed 's/^/ /' "$cachito_env" 116 | } 117 | 118 | check_dependencies 119 | prepare_output_dir "$output_dir" 120 | download_and_extract "$request_url" "$output_dir" 121 | inject_config_files "$request_url" "$output_dir" 122 | generate_cachito_env "$request_url" "$output_dir" 123 | -------------------------------------------------------------------------------- /cachito/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/cachito/__init__.py -------------------------------------------------------------------------------- /cachito/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/cachito/common/__init__.py -------------------------------------------------------------------------------- /cachito/common/checksum.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | import hashlib 4 | from pathlib import Path 5 | from typing import Union 6 | 7 | from cachito.errors import UnknownHashAlgorithm 8 | 9 | 10 | def hash_file(file_path: Union[str, Path], chunk_size: int = 10240, algorithm: str = "sha256"): 11 | """Hash a file. 12 | 13 | :param file_path: compute checksum for this file. 14 | :type file_path: str, pathlib.Path 15 | :param int chunk_size: the optional chunk size passed to file object ``read`` method. 16 | :param str algorithm: the algorithm name used to hash the file. By default, sha256 is used. 17 | :return: a hash object containing the data to generate digest. 18 | :rtype: Hasher 19 | :raise UnknownHashAlgorithm: if the algorithm cannot be found. 20 | """ 21 | try: 22 | hasher = hashlib.new(algorithm) 23 | except ValueError: 24 | raise UnknownHashAlgorithm(f"Hash algorithm {algorithm} is unknown.") 25 | with open(file_path, "rb") as f: 26 | while chunk := f.read(chunk_size): 27 | hasher.update(chunk) 28 | return hasher 29 | -------------------------------------------------------------------------------- /cachito/common/paths.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | import os 4 | import shutil 5 | from pathlib import Path 6 | from typing import Any 7 | 8 | # Subclassing from type(Path()) is a workaround because pathlib does not 9 | # support subclass from Path directly. This base type will be the correct type 10 | # for Linux or Windows individually. 11 | base_path: Any = type(Path()) 12 | 13 | 14 | class RequestBundleDir(base_path): 15 | """ 16 | Represents a directory tree for a request. 17 | 18 | :param int request_id: the request ID. 19 | :param str root: the root directory. A request bundle directory will be 20 | created under ``root/temp/``. 21 | """ 22 | 23 | go_mod_cache_download_part = Path("pkg", "mod", "cache", "download") 24 | 25 | def __new__(cls, request_id, root, app_subpath=os.curdir): 26 | """ 27 | Create a new Path object. 28 | 29 | :param int request_id: the ID of the request this bundle is for. 30 | :param str root: the root directory to the bundles. 31 | :param str app_subpath: an optional relative path to where the application resides in the 32 | source directory. This sets ``self.source_dir`` and all other related paths to 33 | start from that directory. If this is not set, it is assumed the application lives in 34 | the root of the source directory. 35 | """ 36 | self = super().__new__(cls, root, "temp", str(request_id)) 37 | self._request_id = request_id 38 | self._path_root = root 39 | 40 | self.source_root_dir = self.joinpath("app") 41 | self.source_dir = self.source_root_dir.joinpath(app_subpath) 42 | self.go_mod_file = self.source_dir.joinpath("go.mod") 43 | 44 | self.deps_dir = self.joinpath("deps") 45 | self.gomod_download_dir = self.joinpath("deps", "gomod", cls.go_mod_cache_download_part) 46 | 47 | self.node_modules = self.source_dir.joinpath("node_modules") 48 | self.npm_deps_dir = self.joinpath("deps", "npm") 49 | self.npm_package_file = self.source_dir.joinpath("package.json") 50 | self.npm_package_lock_file = self.source_dir.joinpath("package-lock.json") 51 | self.npm_shrinkwrap_file = self.source_dir.joinpath("npm-shrinkwrap.json") 52 | 53 | self.pip_deps_dir = self.joinpath("deps", "pip") 54 | 55 | self.rubygems_deps_dir = self.joinpath("deps", "rubygems") 56 | 57 | self.yarn_deps_dir = self.joinpath("deps", "yarn") 58 | 59 | self.bundle_archive_file = Path(root, f"{request_id}.tar.gz") 60 | self.bundle_archive_checksum = Path(root, f"{request_id}.checksum.sha256") 61 | 62 | self.packages_data = Path(root, f"{request_id}-packages.json") 63 | self.gomod_packages_data = self.joinpath("gomod_packages.json") 64 | self.npm_packages_data = self.joinpath("npm_packages.json") 65 | self.pip_packages_data = self.joinpath("pip_packages.json") 66 | self.yarn_packages_data = self.joinpath("yarn_packages.json") 67 | self.rubygems_packages_data = self.joinpath("rubygems_packages.json") 68 | self.git_submodule_packages_data = self.joinpath("git_submodule_packages.json") 69 | 70 | return self 71 | 72 | def app_subpath(self, subpath): 73 | """Create a new ``RequestBundleDir`` object with the sources pointed to the subpath.""" 74 | return RequestBundleDir(self._request_id, self._path_root, subpath) 75 | 76 | def relpath(self, path): 77 | """Get the relative path of a path from the root of the source directory.""" 78 | return os.path.relpath(path, start=self.source_root_dir) 79 | 80 | def rmtree(self): 81 | """Remove this directory tree entirely.""" 82 | shutil.rmtree(str(self)) 83 | -------------------------------------------------------------------------------- /cachito/common/utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import base64 3 | import urllib 4 | 5 | 6 | def b64encode(s: bytes) -> str: 7 | """Encode a bytes string in base64.""" 8 | return base64.b64encode(s).decode("utf-8") 9 | 10 | 11 | def get_repo_name(url): 12 | """Get the repo name from the URL.""" 13 | parsed_url = urllib.parse.urlparse(url) 14 | repo = parsed_url.path.strip("/") 15 | if repo.endswith(".git"): 16 | repo = repo[: -len(".git")] 17 | return repo 18 | -------------------------------------------------------------------------------- /cachito/errors.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | from enum import Enum 3 | 4 | 5 | class RequestErrorOrigin(str, Enum): 6 | """An Enum that represents the request error origin.""" 7 | 8 | client = "client" 9 | server = "server" 10 | 11 | 12 | class CachitoError(RuntimeError): 13 | """An error was encountered in Cachito.""" 14 | 15 | 16 | class ValidationError(CachitoError, ValueError): 17 | """An error was encountered during validation.""" 18 | 19 | 20 | class ConfigError(CachitoError): 21 | """An error was encountered during configuration validation.""" 22 | 23 | 24 | class ContentManifestError(CachitoError, ValueError): 25 | """An error was encountered during content manifest generation.""" 26 | 27 | 28 | class CachitoNotImplementedError(CachitoError, ValueError): 29 | """An error was encountered during request validation.""" 30 | 31 | 32 | class UnknownHashAlgorithm(CachitoError): 33 | """The hash algorithm is unknown by Cachito.""" 34 | 35 | 36 | class GitError(CachitoError): 37 | """An error was encountered during manipulation with a Git repository.""" 38 | 39 | 40 | # Request error classifiers 41 | class ClientError(Exception): 42 | """Client Error.""" 43 | 44 | origin = RequestErrorOrigin.client 45 | 46 | 47 | class ServerError(Exception): 48 | """Server Error.""" 49 | 50 | origin = RequestErrorOrigin.server 51 | 52 | 53 | # Web errors 54 | class InvalidQueryParameters(ClientError): 55 | """Invalid query parameters.""" 56 | 57 | pass 58 | 59 | 60 | class InvalidRequestData(ClientError): 61 | """Invalid request data.""" 62 | 63 | pass 64 | 65 | 66 | # Repository errors 67 | class InvalidRepoStructure(ClientError): 68 | """Invalid repository structure. The provided repository has a missing file or directory.""" 69 | 70 | pass 71 | 72 | 73 | class InvalidFileFormat(ClientError): 74 | """Invalid file format.""" 75 | 76 | pass 77 | 78 | 79 | class InvalidChecksum(ClientError): 80 | """Checksum verification failed.""" 81 | 82 | pass 83 | 84 | 85 | class UnsupportedFeature(ClientError): 86 | """Unsupported feature.""" 87 | 88 | pass 89 | 90 | 91 | # Deployment errors 92 | class WebConfigError(ServerError): 93 | """Invalid API configuration.""" 94 | 95 | pass 96 | 97 | 98 | class WorkerConfigError(ServerError): 99 | """Invalid worker configuration.""" 100 | 101 | pass 102 | 103 | 104 | class NexusConfigError(ServerError): 105 | """Invalid Nexus configuration.""" 106 | 107 | pass 108 | 109 | 110 | class NoWorkers(ServerError): 111 | """No available workers found.""" 112 | 113 | pass 114 | 115 | 116 | # Low-level errors 117 | class FileAccessError(ServerError): 118 | """File not found.""" 119 | 120 | pass 121 | 122 | 123 | class FilePermissionError(ServerError): 124 | """No permissions to open file.""" 125 | 126 | pass 127 | 128 | 129 | class SubprocessCallError(ServerError): 130 | """Error calling subprocess.""" 131 | 132 | pass 133 | 134 | 135 | class NetworkError(ServerError): 136 | """Network connection error.""" 137 | 138 | pass 139 | 140 | 141 | class DatabaseError(ServerError): 142 | """DB connection error.""" 143 | 144 | pass 145 | 146 | 147 | class MessageBrokerError(ServerError): 148 | """Message broker connection error.""" 149 | 150 | pass 151 | 152 | 153 | # Third-party service errors 154 | class RepositoryAccessError(ServerError): 155 | """Repository is not accessible and can't be cloned.""" 156 | 157 | pass 158 | 159 | 160 | class GoModError(ServerError): 161 | """Go mod related error. A module can't be downloaded by go mod download command.""" 162 | 163 | pass 164 | 165 | 166 | class NexusError(ServerError): 167 | """Nexus related error.""" 168 | 169 | pass 170 | -------------------------------------------------------------------------------- /cachito/web/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | from flask_sqlalchemy import SQLAlchemy 4 | 5 | db = SQLAlchemy() 6 | -------------------------------------------------------------------------------- /cachito/web/auth.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | from flask import current_app 3 | 4 | from cachito.web import db 5 | from cachito.web.models import User 6 | 7 | 8 | def user_loader(username): 9 | """ 10 | Get the user by their username from the database. 11 | 12 | This is used by the Flask-Login library. 13 | 14 | :param str username: the username of the user 15 | :return: the User object associated with the username or None 16 | :rtype: cachito.web.models.User 17 | """ 18 | return User.query.filter_by(username=username).first() 19 | 20 | 21 | def _get_kerberos_principal(request): 22 | """ 23 | Get the Kerberos principal from the current request. 24 | 25 | This relies on the "REMOTE_USER" environment variable being set. This is usually set by the 26 | mod_auth_gssapi Apache authentication module. 27 | 28 | :param flask.Request request: the Flask request 29 | :return: the user's Kerberos principal or None 30 | :rtype: str 31 | """ 32 | return request.environ.get("REMOTE_USER") 33 | 34 | 35 | def _get_cert_dn(request): 36 | """ 37 | Get the client certificate's subject's distinguished name. 38 | 39 | This relies on the "SSL_CLIENT_S_DN" environment variable being set. This is set by the mod_ssl 40 | Apache module. If Apache is unable to verify the client certificate, no user will be returned. 41 | 42 | :param flask.Request request: the Flask request 43 | :return: the client certificate's subject's distinguished name or None 44 | :rtype: str 45 | """ 46 | ssl_client_verify = request.environ.get("SSL_CLIENT_VERIFY") 47 | if ssl_client_verify != "SUCCESS": 48 | current_app.logger.debug( 49 | "The SSL_CLIENT_VERIFY environment variable was set to %s", ssl_client_verify 50 | ) 51 | return None 52 | 53 | return request.environ.get("SSL_CLIENT_S_DN") 54 | 55 | 56 | def load_user_from_request(request): 57 | """ 58 | Load the user that authenticated from the current request. 59 | 60 | This is used by the Flask-Login library. If the user does not exist in the database, an entry 61 | will be created. 62 | 63 | If None is returned, then Flask-Login will set `flask_login.current_user` to an 64 | `AnonymousUserMixin` object, which has the `is_authenticated` property set to `False`. 65 | Additionally, any route decorated with `@login_required` will raise an `Unauthorized` exception. 66 | 67 | :param flask.Request request: the Flask request 68 | :return: the User object associated with the username or None 69 | :rtype: cachito.web.models.User 70 | """ 71 | username = _get_kerberos_principal(request) or _get_cert_dn(request) 72 | if not username: 73 | if current_app.config.get("LOGIN_DISABLED", False) is True: 74 | current_app.logger.info( 75 | "The REMOTE_USER environment variable wasn't set on the request, but the " 76 | "LOGIN_DISABLED configuration is set to True." 77 | ) 78 | return None 79 | 80 | current_app.logger.info(f'The user "{username}" was authenticated successfully by httpd') 81 | user = User.get_or_create(username) 82 | if not user.id: 83 | db.session.commit() 84 | 85 | return user 86 | -------------------------------------------------------------------------------- /cachito/web/docs.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | from flask import Blueprint, send_from_directory 3 | 4 | docs = Blueprint("docs", __name__) 5 | 6 | 7 | @docs.route("/", methods=["GET"]) 8 | def index(): 9 | """Return the OpenAPI documentation presented by redoc.""" 10 | return send_from_directory("static", "docs.html") 11 | -------------------------------------------------------------------------------- /cachito/web/errors.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | from flask import jsonify 3 | from werkzeug.exceptions import HTTPException 4 | 5 | from cachito.errors import ( 6 | CachitoError, 7 | CachitoNotImplementedError, 8 | ClientError, 9 | ContentManifestError, 10 | ValidationError, 11 | ) 12 | 13 | 14 | def json_error(error): 15 | """ 16 | Convert exceptions to JSON responses. 17 | 18 | :param Exception error: an Exception to convert to JSON 19 | :return: a Flask JSON response 20 | :rtype: flask.Response 21 | """ 22 | if isinstance(error, HTTPException): 23 | if error.code == 404: 24 | msg = "The requested resource was not found" 25 | else: 26 | msg = error.description 27 | response = jsonify({"error": msg}) 28 | response.status_code = error.code 29 | else: 30 | # Default status code is for "server" request errors 31 | status_code = 500 32 | msg = str(error) 33 | if isinstance(error, (ClientError, ValidationError)): 34 | status_code = 400 35 | elif isinstance(error, ContentManifestError): 36 | # If the request was completed and a ICM cannot be generated, 37 | # some package type or corner case is not yet implemented 38 | status_code = 501 39 | elif isinstance(error, CachitoNotImplementedError): 40 | # If the request asks for not implemented functionality 41 | status_code = 501 42 | elif isinstance(error, CachitoError): 43 | # If a generic exception is raised, assume the service is unavailable 44 | status_code = 503 45 | 46 | response = jsonify({"error": msg}) 47 | response.status_code = status_code 48 | return response 49 | 50 | 51 | def validation_error(error): 52 | """ 53 | Handle pydandic ValidationError. 54 | 55 | Prepare JSON response in the following format: 56 | { 57 | "errors": { 58 | "field1": "error message", 59 | ... 60 | } 61 | } 62 | 63 | :param Exception error: validation error 64 | :return: a Flask JSON response 65 | :rtype: flask.Response 66 | """ 67 | errors = {".".join(error["loc"]): error["msg"] for error in error.errors()} 68 | response = jsonify({"errors": errors}) 69 | response.status_code = 400 70 | return response 71 | -------------------------------------------------------------------------------- /cachito/web/manage.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import time 3 | 4 | import click 5 | from flask.cli import FlaskGroup 6 | from sqlalchemy.exc import OperationalError 7 | 8 | from cachito.web.app import create_cli_app 9 | from cachito.web.models import db 10 | 11 | 12 | @click.group(cls=FlaskGroup, create_app=create_cli_app) 13 | def cli(): 14 | """Manage the Cachito Flask application.""" 15 | 16 | 17 | @cli.command(name="wait-for-db") 18 | def wait_for_db(): 19 | """Wait until database server is reachable.""" 20 | # The polling interval in seconds 21 | poll_interval = 10 22 | while True: 23 | try: 24 | db.engine.connect() 25 | except OperationalError as e: 26 | click.echo("Failed to connect to database: {}".format(e), err=True) 27 | click.echo("Sleeping for {} seconds...".format(poll_interval)) 28 | time.sleep(poll_interval) 29 | click.echo("Retrying...") 30 | else: 31 | break 32 | 33 | 34 | if __name__ == "__main__": 35 | cli() 36 | -------------------------------------------------------------------------------- /cachito/web/metrics.py: -------------------------------------------------------------------------------- 1 | import os 2 | import socket 3 | 4 | from prometheus_client import Gauge, Summary, multiprocess 5 | from prometheus_client.core import CollectorRegistry 6 | from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics 7 | 8 | cachito_metrics = {} 9 | 10 | 11 | def init_metrics(app): 12 | """ 13 | Initialize the Prometheus Flask Exporter. 14 | 15 | :return: a Prometheus Flash Metrics object 16 | :rtype: PrometheusMetrics 17 | """ 18 | registry = CollectorRegistry() 19 | multiproc_temp_dir = app.config["PROMETHEUS_METRICS_TEMP_DIR"] 20 | hostname = socket.gethostname() 21 | 22 | if not os.path.isdir(multiproc_temp_dir): 23 | os.makedirs(multiproc_temp_dir) 24 | multiprocess.MultiProcessCollector(registry, path=multiproc_temp_dir) 25 | metrics = GunicornInternalPrometheusMetrics.for_app_factory( 26 | default_labels={"host": hostname}, group_by="endpoint", defaults_prefix="cachito_flask" 27 | ) 28 | metrics.init_app(app) 29 | gauge_state = Gauge( 30 | "cachito_requests_count", "Requests in each state", ["state"], multiprocess_mode="livesum" 31 | ) 32 | request_duration = Summary( 33 | "cachito_request_duration_seconds", "Time spent in in_progress state" 34 | ) 35 | cachito_metrics["gauge_state"] = gauge_state 36 | cachito_metrics["request_duration"] = request_duration 37 | -------------------------------------------------------------------------------- /cachito/web/migrations/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # template used to generate migration files 5 | # file_template = %%(rev)s_%%(slug)s 6 | 7 | # set to 'true' to run the environment during 8 | # the 'revision' command, regardless of autogenerate 9 | # revision_environment = false 10 | 11 | 12 | # Logging configuration 13 | [loggers] 14 | keys = root,sqlalchemy,alembic 15 | 16 | [handlers] 17 | keys = console 18 | 19 | [formatters] 20 | keys = generic 21 | 22 | [logger_root] 23 | level = WARN 24 | handlers = console 25 | qualname = 26 | 27 | [logger_sqlalchemy] 28 | level = WARN 29 | handlers = 30 | qualname = sqlalchemy.engine 31 | 32 | [logger_alembic] 33 | level = INFO 34 | handlers = 35 | qualname = alembic 36 | 37 | [handler_console] 38 | class = StreamHandler 39 | args = (sys.stderr,) 40 | level = NOTSET 41 | formatter = generic 42 | 43 | [formatter_generic] 44 | format = %(levelname)-5.5s [%(name)s] %(message)s 45 | datefmt = %H:%M:%S 46 | -------------------------------------------------------------------------------- /cachito/web/migrations/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | 3 | import logging 4 | from logging.config import fileConfig 5 | 6 | from sqlalchemy import engine_from_config 7 | from sqlalchemy import pool 8 | 9 | from alembic import context 10 | 11 | # this is the Alembic Config object, which provides 12 | # access to the values within the .ini file in use. 13 | config = context.config 14 | 15 | # Interpret the config file for Python logging. 16 | # This line sets up loggers basically. 17 | if config.config_file_name is not None: 18 | fileConfig(config.config_file_name) 19 | logger = logging.getLogger("alembic.env") 20 | 21 | # add your model's MetaData object here 22 | # for 'autogenerate' support 23 | # from myapp import mymodel 24 | # target_metadata = mymodel.Base.metadata 25 | from flask import current_app 26 | 27 | config.set_main_option("sqlalchemy.url", current_app.config["SQLALCHEMY_DATABASE_URI"]) 28 | target_metadata = current_app.extensions["migrate"].db.metadata 29 | 30 | # other values from the config, defined by the needs of env.py, 31 | # can be acquired: 32 | # my_important_option = config.get_main_option("my_important_option") 33 | # ... etc. 34 | 35 | 36 | def run_migrations_offline(): 37 | """Run migrations in 'offline' mode. 38 | 39 | This configures the context with just a URL 40 | and not an Engine, though an Engine is acceptable 41 | here as well. By skipping the Engine creation 42 | we don't even need a DBAPI to be available. 43 | 44 | Calls to context.execute() here emit the given string to the 45 | script output. 46 | 47 | """ 48 | url = config.get_main_option("sqlalchemy.url") 49 | context.configure( 50 | url=url, 51 | target_metadata=target_metadata, 52 | literal_binds=True, 53 | ) 54 | 55 | with context.begin_transaction(): 56 | context.run_migrations() 57 | 58 | 59 | def run_migrations_online(): 60 | """Run migrations in 'online' mode. 61 | 62 | In this scenario we need to create an Engine 63 | and associate a connection with the context. 64 | 65 | """ 66 | 67 | # this callback is used to prevent an auto-migration from being generated 68 | # when there are no changes to the schema 69 | # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html 70 | def process_revision_directives(context, revision, directives): 71 | if getattr(config.cmd_opts, "autogenerate", False): 72 | script = directives[0] 73 | if script.upgrade_ops.is_empty(): 74 | directives[:] = [] 75 | logger.info("No changes in schema detected.") 76 | 77 | connectable = engine_from_config( 78 | config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool 79 | ) 80 | 81 | with connectable.connect() as connection: 82 | context.configure( 83 | connection=connection, 84 | target_metadata=target_metadata, 85 | process_revision_directives=process_revision_directives, 86 | **current_app.extensions["migrate"].configure_args, 87 | ) 88 | 89 | with context.begin_transaction(): 90 | context.run_migrations() 91 | 92 | 93 | if context.is_offline_mode(): 94 | run_migrations_offline() 95 | else: 96 | run_migrations_online() 97 | -------------------------------------------------------------------------------- /cachito/web/migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/01bb0873ddcb_add_the_remove_unsafe_symlinks_flag.py: -------------------------------------------------------------------------------- 1 | """Add the remove-unsafe-symlinks flag 2 | 3 | Revision ID: 01bb0873ddcb 4 | Revises: 02229e089b24 5 | Create Date: 2022-03-15 15:31:11.301867 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "01bb0873ddcb" 14 | down_revision = "02229e089b24" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | flag_table = sa.Table( 20 | "flag", 21 | sa.MetaData(), 22 | sa.Column("id", sa.Integer(), primary_key=True), 23 | sa.Column("name", sa.String(), nullable=False), 24 | sa.Column("active", sa.Boolean(), nullable=False, default=True), 25 | ) 26 | 27 | 28 | def upgrade(): 29 | connection = op.get_bind() 30 | res = connection.execute( 31 | flag_table.select().where(flag_table.c.name == "remove-unsafe-symlinks") 32 | ).fetchone() 33 | if res is None: 34 | connection.execute(flag_table.insert().values(name="remove-unsafe-symlinks", active=True)) 35 | else: 36 | connection.execute( 37 | flag_table.update() 38 | .where(flag_table.c.name == "remove-unsafe-symlinks") 39 | .values(active=True) 40 | ) 41 | 42 | 43 | def downgrade(): 44 | connection = op.get_bind() 45 | connection.execute(flag_table.update().values(name="remove-unsafe-symlinks", active=False)) 46 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/02229e089b24_delete_user_username_index.py: -------------------------------------------------------------------------------- 1 | """Remove User.username unique index 2 | 3 | Revision ID: 02229e089b24 4 | Revises: 7d979987402d 5 | Create Date: 2021-12-15 21:37:30.582812 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "02229e089b24" 14 | down_revision = "7d979987402d" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | with op.batch_alter_table("user", schema=None) as batch_op: 21 | batch_op.drop_index("ix_user_username") 22 | 23 | 24 | def downgrade(): 25 | with op.batch_alter_table("user", schema=None) as batch_op: 26 | batch_op.create_index("ix_user_username", ["username"], unique=True) 27 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/044548f3f83a_add_pip_dev_preview_flag.py: -------------------------------------------------------------------------------- 1 | """Add the pip-dev-preview flag 2 | 3 | Revision ID: 044548f3f83a 4 | Revises: cb6bdbe533cc 5 | Create Date: 2020-09-03 02:16:57.554550 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "044548f3f83a" 14 | down_revision = "cb6bdbe533cc" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | flag_table = sa.Table( 19 | "flag", 20 | sa.MetaData(), 21 | sa.Column("id", sa.Integer(), primary_key=True), 22 | sa.Column("name", sa.String(), nullable=False), 23 | sa.Column("active", sa.Boolean(), nullable=False, default=True), 24 | ) 25 | 26 | 27 | def upgrade(): 28 | connection = op.get_bind() 29 | res = connection.execute( 30 | flag_table.select().where(flag_table.c.name == "pip-dev-preview") 31 | ).fetchone() 32 | if res is None: 33 | connection.execute(flag_table.insert().values(name="pip-dev-preview", active=True)) 34 | else: 35 | connection.execute( 36 | flag_table.update().where(flag_table.c.name == "pip-dev-preview").values(active=True) 37 | ) 38 | 39 | 40 | def downgrade(): 41 | connection = op.get_bind() 42 | connection.execute(flag_table.update().values(name="pip-dev-preview", active=False)) 43 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/07d89c5778f2_add_cgo_disable_flag.py: -------------------------------------------------------------------------------- 1 | """Add the cgo-disable flag 2 | 3 | Revision ID: 07d89c5778f2 4 | Revises: 97d5df7fca86 5 | Create Date: 2021-02-12 10:04:40.989666 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "07d89c5778f2" 14 | down_revision = "97d5df7fca86" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | flag_table = sa.Table( 20 | "flag", 21 | sa.MetaData(), 22 | sa.Column("id", sa.Integer(), primary_key=True), 23 | sa.Column("name", sa.String(), nullable=False), 24 | sa.Column("active", sa.Boolean(), nullable=False, default=True), 25 | ) 26 | 27 | 28 | def upgrade(): 29 | connection = op.get_bind() 30 | res = connection.execute( 31 | flag_table.select().where(flag_table.c.name == "cgo-disable") 32 | ).fetchone() 33 | if res is None: 34 | connection.execute(flag_table.insert().values(name="cgo-disable", active=True)) 35 | else: 36 | connection.execute( 37 | flag_table.update().where(flag_table.c.name == "cgo-disable").values(active=True) 38 | ) 39 | 40 | 41 | def downgrade(): 42 | connection = op.get_bind() 43 | connection.execute(flag_table.update().values(name="cgo-disable", active=False)) 44 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/15c4aa0c4144_rename_dependency_table.py: -------------------------------------------------------------------------------- 1 | """Rename the dependency table to package 2 | 3 | Revision ID: 15c4aa0c4144 4 | Revises: 71909d479045 5 | Create Date: 2019-11-11 16:51:50.917910 6 | 7 | """ 8 | from alembic import op 9 | 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "15c4aa0c4144" 13 | down_revision = "71909d479045" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | with op.batch_alter_table("dependency", schema=None) as batch_op: 20 | batch_op.drop_index("ix_dependency_name") 21 | batch_op.drop_index("ix_dependency_type") 22 | batch_op.drop_index("ix_dependency_version") 23 | 24 | with op.batch_alter_table("request_dependency", schema=None) as batch_op: 25 | batch_op.drop_constraint("request_dependency_request_id_fkey", type_="foreignkey") 26 | batch_op.drop_constraint("request_dependency_dependency_id_fkey", type_="foreignkey") 27 | batch_op.drop_constraint("fk_replaced_dependency_id", type_="foreignkey") 28 | 29 | op.rename_table("dependency", "package") 30 | 31 | with op.batch_alter_table("request_dependency", schema=None) as batch_op: 32 | batch_op.create_foreign_key( 33 | "request_dependency_request_id_fkey", "request", ["request_id"], ["id"] 34 | ) 35 | batch_op.create_foreign_key( 36 | "request_dependency_dependency_id_fkey", "package", ["dependency_id"], ["id"] 37 | ) 38 | batch_op.create_foreign_key( 39 | "fk_replaced_dependency_id", "package", ["replaced_dependency_id"], ["id"] 40 | ) 41 | 42 | with op.batch_alter_table("package", schema=None) as batch_op: 43 | batch_op.create_index(op.f("ix_package_name"), ["name"], unique=False) 44 | batch_op.create_index(op.f("ix_package_type"), ["type"], unique=False) 45 | batch_op.create_index(op.f("ix_package_version"), ["version"], unique=False) 46 | 47 | 48 | def downgrade(): 49 | with op.batch_alter_table("package", schema=None) as batch_op: 50 | batch_op.drop_index(op.f("ix_package_version")) 51 | batch_op.drop_index(op.f("ix_package_type")) 52 | batch_op.drop_index(op.f("ix_package_name")) 53 | 54 | with op.batch_alter_table("request_dependency", schema=None) as batch_op: 55 | batch_op.drop_constraint("request_dependency_request_id_fkey", type_="foreignkey") 56 | batch_op.drop_constraint("request_dependency_dependency_id_fkey", type_="foreignkey") 57 | batch_op.drop_constraint("fk_replaced_dependency_id", type_="foreignkey") 58 | 59 | op.rename_table("package", "dependency") 60 | 61 | with op.batch_alter_table("request_dependency", schema=None) as batch_op: 62 | batch_op.create_foreign_key( 63 | "request_dependency_request_id_fkey", "request", ["request_id"], ["id"] 64 | ) 65 | batch_op.create_foreign_key( 66 | "request_dependency_dependency_id_fkey", "dependency", ["dependency_id"], ["id"] 67 | ) 68 | batch_op.create_foreign_key( 69 | "fk_replaced_dependency_id", "dependency", ["replaced_dependency_id"], ["id"] 70 | ) 71 | 72 | with op.batch_alter_table("dependency", schema=None) as batch_op: 73 | batch_op.create_index("ix_dependency_version", ["version"], unique=False) 74 | batch_op.create_index("ix_dependency_type", ["type"], unique=False) 75 | batch_op.create_index("ix_dependency_name", ["name"], unique=False) 76 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/193baf9d7cbf_config_file_base64.py: -------------------------------------------------------------------------------- 1 | """Add the config_file_base64 table 2 | 3 | Revision ID: 193baf9d7cbf 4 | Revises: 3c208b05d703 5 | Create Date: 2020-04-03 19:16:34.581217 6 | """ 7 | from alembic import op 8 | import sqlalchemy as sa 9 | 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "193baf9d7cbf" 13 | down_revision = "3c208b05d703" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | op.create_table( 20 | "config_file_base64", 21 | sa.Column("id", sa.Integer(), nullable=False), 22 | sa.Column("path", sa.String(), nullable=False), 23 | sa.Column("content", sa.String(), nullable=False), 24 | sa.PrimaryKeyConstraint("id"), 25 | ) 26 | op.create_index( 27 | op.f("ix_config_file_base64_path"), "config_file_base64", ["path"], unique=False 28 | ) 29 | 30 | op.create_table( 31 | "request_config_file_base64", 32 | sa.Column("request_id", sa.Integer(), nullable=False), 33 | sa.Column("config_file_base64_id", sa.Integer(), nullable=False), 34 | sa.ForeignKeyConstraint(["config_file_base64_id"], ["config_file_base64.id"]), 35 | sa.ForeignKeyConstraint(["request_id"], ["request.id"]), 36 | sa.UniqueConstraint("request_id", "config_file_base64_id"), 37 | ) 38 | op.create_index( 39 | op.f("ix_request_config_file_base64_config_file_base64_id"), 40 | "request_config_file_base64", 41 | ["config_file_base64_id"], 42 | unique=False, 43 | ) 44 | op.create_index( 45 | op.f("ix_request_config_file_base64_request_id"), 46 | "request_config_file_base64", 47 | ["request_id"], 48 | unique=False, 49 | ) 50 | 51 | 52 | def downgrade(): 53 | with op.batch_alter_table("request_config_file_base64") as batch_op: 54 | batch_op.drop_index(batch_op.f("ix_request_config_file_base64_request_id")) 55 | batch_op.drop_index(batch_op.f("ix_request_config_file_base64_config_file_base64_id")) 56 | 57 | op.drop_table("request_config_file_base64") 58 | 59 | with op.batch_alter_table("config_file_base64") as batch_op: 60 | batch_op.drop_index(batch_op.f("ix_config_file_base64_path")) 61 | 62 | op.drop_table("config_file_base64") 63 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/2f83b3e4c5cc_add_request_package_subpath.py: -------------------------------------------------------------------------------- 1 | """Add the subpath column to the request_package table 2 | 3 | Revision ID: 2f83b3e4c5cc 4 | Revises: f133002ffdb4 5 | Create Date: 2020-10-13 11:43:25.052014 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "2f83b3e4c5cc" 14 | down_revision = "f133002ffdb4" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | with op.batch_alter_table("request_package") as batch_op: 21 | batch_op.add_column(sa.Column("subpath", sa.String(), nullable=True)) 22 | 23 | 24 | def downgrade(): 25 | with op.batch_alter_table("request_package") as batch_op: 26 | batch_op.drop_column("subpath") 27 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/3c208b05d703_request_state_fk.py: -------------------------------------------------------------------------------- 1 | """Add the request_state_id foreign key. 2 | 3 | Revision ID: 3c208b05d703 4 | Revises: fdd6d6978386 5 | Create Date: 2019-12-19 14:57:01.313098 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy import desc as sa_desc 11 | 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = "3c208b05d703" 15 | down_revision = "fdd6d6978386" 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | request_table = sa.Table( 21 | "request", 22 | sa.MetaData(), 23 | sa.Column("id", sa.Integer(), primary_key=True), 24 | sa.Column("request_state_id", sa.Integer(), sa.ForeignKey("request_state.id")), 25 | ) 26 | 27 | 28 | request_state_table = sa.Table( 29 | "request_state", 30 | sa.MetaData(), 31 | sa.Column("id", sa.Integer(), primary_key=True), 32 | sa.Column("updated", sa.DateTime()), 33 | sa.Column("request_id", sa.Integer(), sa.ForeignKey("request.id")), 34 | ) 35 | 36 | 37 | def upgrade(): 38 | with op.batch_alter_table("request") as batch_op: 39 | batch_op.add_column(sa.Column("request_state_id", sa.Integer(), nullable=True)) 40 | batch_op.create_index( 41 | batch_op.f("ix_request_request_state_id"), ["request_state_id"], unique=True 42 | ) 43 | batch_op.create_foreign_key( 44 | "fk_request_state_id", "request_state", ["request_state_id"], ["id"] 45 | ) 46 | 47 | connection = op.get_bind() 48 | for request in connection.execute(request_table.select()): 49 | request_id = request[0] 50 | last_state = connection.execute( 51 | request_state_table.select() 52 | .where(request_state_table.c.request_id == request_id) 53 | .order_by(sa_desc(request_state_table.c.updated)) 54 | .limit(1) 55 | ).fetchone() 56 | if not last_state: 57 | continue 58 | 59 | last_state_id = last_state[0] 60 | connection.execute( 61 | request_table.update() 62 | .where(request_table.c.id == request_id) 63 | .values(request_state_id=last_state_id) 64 | ) 65 | 66 | 67 | def downgrade(): 68 | with op.batch_alter_table("request") as batch_op: 69 | batch_op.drop_constraint("fk_request_state_id", type_="foreignkey") 70 | batch_op.drop_index(batch_op.f("ix_request_request_state_id")) 71 | batch_op.drop_column("request_state_id") 72 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/418241dba06c_add_force_gomod_tidy_flag.py: -------------------------------------------------------------------------------- 1 | """Add the force-gomod-tidy flag 2 | 3 | Revision ID: 418241dba06c 4 | Revises: 01bb0873ddcb 5 | Create Date: 2022-01-18 07:24:31.927867 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "418241dba06c" 14 | down_revision = "01bb0873ddcb" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | flag_table = sa.Table( 20 | "flag", 21 | sa.MetaData(), 22 | sa.Column("id", sa.Integer(), primary_key=True), 23 | sa.Column("name", sa.String(), nullable=False), 24 | sa.Column("active", sa.Boolean(), nullable=False, default=True), 25 | ) 26 | 27 | 28 | def upgrade(): 29 | connection = op.get_bind() 30 | res = connection.execute( 31 | flag_table.select().where(flag_table.c.name == "force-gomod-tidy") 32 | ).fetchone() 33 | if res is None: 34 | connection.execute(flag_table.insert().values(name="force-gomod-tidy", active=True)) 35 | else: 36 | connection.execute( 37 | flag_table.update().where(flag_table.c.name == "force-gomod-tidy").values(active=True) 38 | ) 39 | 40 | 41 | def downgrade(): 42 | connection = op.get_bind() 43 | connection.execute( 44 | flag_table.update().where(flag_table.c.name == "force-gomod-tidy").values(active=False) 45 | ) 46 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/491454f79a8b_add_packages_and_deps_count.py: -------------------------------------------------------------------------------- 1 | """Add packages_count and dependencies_count to requests 2 | 3 | Revision ID: 491454f79a8b 4 | Revises: 4d17dec0cfc3 5 | Create Date: 2021-06-03 18:01:55.976908 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "491454f79a8b" 14 | down_revision = "4d17dec0cfc3" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | with op.batch_alter_table("request") as batch_op: 21 | batch_op.add_column(sa.Column("packages_count", sa.Integer())) 22 | batch_op.add_column(sa.Column("dependencies_count", sa.Integer())) 23 | 24 | 25 | def downgrade(): 26 | with op.batch_alter_table("request") as batch_op: 27 | batch_op.drop_column("packages_count") 28 | batch_op.drop_column("dependencies_count") 29 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/4a64656ba27f_add_git_submodule.py: -------------------------------------------------------------------------------- 1 | """Add the git-submodule package manager 2 | 3 | Revision ID: 4a64656ba27f 4 | Revises: cfbbf7675e3b 5 | Create Date: 2020-09-21 21:40:36.901272 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "4a64656ba27f" 14 | down_revision = "cfbbf7675e3b" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | package_manager_table = sa.Table( 20 | "package_manager", 21 | sa.MetaData(), 22 | sa.Column("id", sa.Integer(), primary_key=True), 23 | sa.Column("name", sa.String(), nullable=False), 24 | ) 25 | 26 | 27 | def upgrade(): 28 | connection = op.get_bind() 29 | connection.execute(package_manager_table.insert().values(name="git-submodule")) 30 | 31 | 32 | def downgrade(): 33 | connection = op.get_bind() 34 | connection.execute( 35 | package_manager_table.delete().where(package_manager_table.c.name == "git-submodule") 36 | ) 37 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/4d17dec0cfc3_adjust_flag_unique_constraint.py: -------------------------------------------------------------------------------- 1 | """Adjust unique constraint on Flag table 2 | 3 | Revision ID: 4d17dec0cfc3 4 | Revises: c6ac095d8e9f 5 | Create Date: 2021-05-25 13:21:48.298960 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "4d17dec0cfc3" 14 | down_revision = "c6ac095d8e9f" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | with op.batch_alter_table("flag") as batch_op: 21 | batch_op.create_unique_constraint("flag_name_key", ["name"]) 22 | batch_op.drop_constraint("flag_id_name_key", type_="unique") 23 | 24 | 25 | def downgrade(): 26 | with op.batch_alter_table("flag") as batch_op: 27 | batch_op.drop_constraint("flag_name_key", type_="unique") 28 | batch_op.create_unique_constraint("flag_id_name_key", ["id", "name"]) 29 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/5854e700a35e_dependency_replacements.py: -------------------------------------------------------------------------------- 1 | """Add the replaced_dependency_id column and indexes on the request_dependency table 2 | 3 | Revision ID: 5854e700a35e 4 | Revises: a655a299e967 5 | Create Date: 2019-10-14 16:33:01.601651 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "5854e700a35e" 14 | down_revision = "a655a299e967" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # Must use batch_alter_table to support SQLite 21 | with op.batch_alter_table("request_dependency") as b: 22 | b.add_column(sa.Column("replaced_dependency_id", sa.Integer(), nullable=True)) 23 | b.create_foreign_key( 24 | "fk_replaced_dependency_id", "dependency", ["replaced_dependency_id"], ["id"] 25 | ) 26 | 27 | op.create_index( 28 | op.f("ix_request_dependency_dependency_id"), 29 | "request_dependency", 30 | ["dependency_id"], 31 | unique=False, 32 | ) 33 | op.create_index( 34 | op.f("ix_request_dependency_replaced_dependency_id"), 35 | "request_dependency", 36 | ["replaced_dependency_id"], 37 | unique=False, 38 | ) 39 | op.create_index( 40 | op.f("ix_request_dependency_request_id"), "request_dependency", ["request_id"], unique=False 41 | ) 42 | 43 | 44 | def downgrade(): 45 | op.drop_index(op.f("ix_request_dependency_request_id"), table_name="request_dependency") 46 | op.drop_index( 47 | op.f("ix_request_dependency_replaced_dependency_id"), table_name="request_dependency" 48 | ) 49 | op.drop_index(op.f("ix_request_dependency_dependency_id"), table_name="request_dependency") 50 | # Must use batch_alter_table to support SQLite 51 | with op.batch_alter_table("request_dependency") as b: 52 | b.drop_constraint("fk_replaced_dependency_id", type_="foreignkey") 53 | b.drop_column("replaced_dependency_id") 54 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/615c19a1cee1_add_npm.py: -------------------------------------------------------------------------------- 1 | """Add the npm package manager 2 | 3 | Revision ID: 615c19a1cee1 4 | Revises: 193baf9d7cbf 5 | Create Date: 2020-04-06 19:50:06.577126 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "615c19a1cee1" 14 | down_revision = "193baf9d7cbf" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | package_manager_table = sa.Table( 20 | "package_manager", 21 | sa.MetaData(), 22 | sa.Column("id", sa.Integer(), primary_key=True), 23 | sa.Column("name", sa.String(), nullable=False), 24 | ) 25 | 26 | 27 | def upgrade(): 28 | connection = op.get_bind() 29 | connection.execute(package_manager_table.insert().values(name="npm")) 30 | 31 | with op.batch_alter_table("package") as batch_op: 32 | batch_op.add_column( 33 | sa.Column("dev", sa.Boolean(), server_default=sa.text("false"), nullable=False) 34 | ) 35 | batch_op.create_index(batch_op.f("ix_package_dev"), ["dev"], unique=False) 36 | batch_op.drop_constraint("dependency_name_type_version_key", type_="unique") 37 | batch_op.create_unique_constraint( 38 | "dependency_dev_name_type_version_key", ["dev", "name", "type", "version"] 39 | ) 40 | 41 | 42 | def downgrade(): 43 | connection = op.get_bind() 44 | connection.execute(package_manager_table.delete().where(package_manager_table.c.name == "npm")) 45 | 46 | with op.batch_alter_table("package") as batch_op: 47 | batch_op.drop_constraint("dependency_dev_name_type_version_key", type_="unique") 48 | batch_op.drop_column("dev") 49 | batch_op.create_unique_constraint( 50 | "dependency_name_type_version_key", ["name", "type", "version"] 51 | ) 52 | batch_op.drop_index(batch_op.f("ix_package_dev")) 53 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/71909d479045_add_submitted_by.py: -------------------------------------------------------------------------------- 1 | """Add the submitted_by_id column 2 | 3 | Revision ID: 71909d479045 4 | Revises: 9118b23629ef 5 | Create Date: 2019-10-21 13:38:48.372486 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "71909d479045" 14 | down_revision = "9118b23629ef" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # Must use batch_alter_table to support SQLite 21 | with op.batch_alter_table("request") as b: 22 | b.add_column(sa.Column("submitted_by_id", sa.Integer(), nullable=True)) 23 | b.create_foreign_key("fk_submitted_by_id", "user", ["submitted_by_id"], ["id"]) 24 | 25 | 26 | def downgrade(): 27 | # Must use batch_alter_table to support SQLite 28 | with op.batch_alter_table("request") as b: 29 | b.drop_constraint("fk_submitted_by_id", type_="foreignkey") 30 | b.drop_column("submitted_by_id") 31 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/9118b23629ef_add_indexes.py: -------------------------------------------------------------------------------- 1 | """Add indexes to commonly used columns 2 | 3 | Revision ID: 9118b23629ef 4 | Revises: 5854e700a35e 5 | Create Date: 2019-10-15 16:56:37.362817 6 | 7 | """ 8 | from alembic import op 9 | 10 | # revision identifiers, used by Alembic. 11 | revision = "9118b23629ef" 12 | down_revision = "5854e700a35e" 13 | branch_labels = None 14 | depends_on = None 15 | 16 | 17 | def upgrade(): 18 | op.create_index(op.f("ix_dependency_name"), "dependency", ["name"], unique=False) 19 | op.create_index(op.f("ix_dependency_type"), "dependency", ["type"], unique=False) 20 | op.create_index(op.f("ix_dependency_version"), "dependency", ["version"], unique=False) 21 | op.create_index( 22 | op.f("ix_request_environment_variable_env_var_id"), 23 | "request_environment_variable", 24 | ["env_var_id"], 25 | unique=False, 26 | ) 27 | op.create_index( 28 | op.f("ix_request_environment_variable_request_id"), 29 | "request_environment_variable", 30 | ["request_id"], 31 | unique=False, 32 | ) 33 | op.create_index(op.f("ix_request_flag_flag_id"), "request_flag", ["flag_id"], unique=False) 34 | op.create_index( 35 | op.f("ix_request_flag_request_id"), "request_flag", ["request_id"], unique=False 36 | ) 37 | op.create_index( 38 | op.f("ix_request_pkg_manager_pkg_manager_id"), 39 | "request_pkg_manager", 40 | ["pkg_manager_id"], 41 | unique=False, 42 | ) 43 | op.create_index( 44 | op.f("ix_request_pkg_manager_request_id"), 45 | "request_pkg_manager", 46 | ["request_id"], 47 | unique=False, 48 | ) 49 | op.create_index( 50 | op.f("ix_request_state_request_id"), "request_state", ["request_id"], unique=False 51 | ) 52 | op.create_index(op.f("ix_user_username"), "user", ["username"], unique=True) 53 | 54 | 55 | def downgrade(): 56 | op.drop_index(op.f("ix_user_username"), table_name="user") 57 | op.drop_index(op.f("ix_request_state_request_id"), table_name="request_state") 58 | op.drop_index(op.f("ix_request_pkg_manager_request_id"), table_name="request_pkg_manager") 59 | op.drop_index(op.f("ix_request_pkg_manager_pkg_manager_id"), table_name="request_pkg_manager") 60 | op.drop_index(op.f("ix_request_flag_request_id"), table_name="request_flag") 61 | op.drop_index(op.f("ix_request_flag_flag_id"), table_name="request_flag") 62 | op.drop_index( 63 | op.f("ix_request_environment_variable_request_id"), 64 | table_name="request_environment_variable", 65 | ) 66 | op.drop_index( 67 | op.f("ix_request_environment_variable_env_var_id"), 68 | table_name="request_environment_variable", 69 | ) 70 | op.drop_index(op.f("ix_dependency_version"), table_name="dependency") 71 | op.drop_index(op.f("ix_dependency_type"), table_name="dependency") 72 | op.drop_index(op.f("ix_dependency_name"), table_name="dependency") 73 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/92f0d370ba4d_add_requesterror_table.py: -------------------------------------------------------------------------------- 1 | """Add RequestError table 2 | Revision ID: 92f0d370ba4d 3 | Revises: 418241dba06c 4 | Create Date: 2022-07-15 16:42:12.752972 5 | """ 6 | from alembic import op 7 | import sqlalchemy as sa 8 | 9 | 10 | # revision identifiers, used by Alembic. 11 | revision = "92f0d370ba4d" 12 | down_revision = "418241dba06c" 13 | branch_labels = None 14 | depends_on = None 15 | 16 | 17 | def upgrade(): 18 | # ### commands auto generated by Alembic - please adjust! ### 19 | op.create_table( 20 | "request_error", 21 | sa.Column("id", sa.Integer(), nullable=False), 22 | sa.Column("request_id", sa.Integer(), nullable=False), 23 | sa.Column("origin", sa.Enum("client", "server", name="requesterrororigin"), nullable=False), 24 | sa.Column("error_type", sa.String(), nullable=False), 25 | sa.Column("message", sa.String(), nullable=False), 26 | sa.Column("occurred", sa.DateTime(), nullable=True), 27 | sa.ForeignKeyConstraint( 28 | ["request_id"], 29 | ["request.id"], 30 | ), 31 | sa.PrimaryKeyConstraint("id"), 32 | sa.UniqueConstraint("id"), 33 | sa.UniqueConstraint("request_id"), 34 | ) 35 | with op.batch_alter_table("request_error", schema=None) as batch_op: 36 | batch_op.create_index( 37 | batch_op.f("ix_request_error_error_type"), ["error_type"], unique=False 38 | ) 39 | batch_op.create_index(batch_op.f("ix_request_error_occurred"), ["occurred"], unique=False) 40 | batch_op.create_index(batch_op.f("ix_request_error_origin"), ["origin"], unique=False) 41 | # ### end Alembic commands ### 42 | 43 | 44 | def downgrade(): 45 | # ### commands auto generated by Alembic - please adjust! ### 46 | with op.batch_alter_table("request_error", schema=None) as batch_op: 47 | batch_op.drop_index(batch_op.f("ix_request_error_origin")) 48 | batch_op.drop_index(batch_op.f("ix_request_error_occurred")) 49 | batch_op.drop_index(batch_op.f("ix_request_error_error_type")) 50 | 51 | op.drop_table("request_error") 52 | # ### end Alembic commands ### 53 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/976b7ef3ec86_add_created_field.py: -------------------------------------------------------------------------------- 1 | """Add created field to Request table 2 | 3 | Revision ID: 976b7ef3ec86 4 | Revises: 491454f79a8b 5 | Create Date: 2021-10-01 05:17:37.099131 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "976b7ef3ec86" 14 | down_revision = "491454f79a8b" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | with op.batch_alter_table("request", schema=None) as batch_op: 21 | batch_op.add_column(sa.Column("created", sa.DateTime(), nullable=True)) 22 | batch_op.create_index(batch_op.f("ix_request_created"), ["created"], unique=False) 23 | 24 | 25 | def downgrade(): 26 | with op.batch_alter_table("request", schema=None) as batch_op: 27 | batch_op.drop_index(batch_op.f("ix_request_created")) 28 | batch_op.drop_column("created") 29 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/97d5df7fca86_add_include_git_dir_flag.py: -------------------------------------------------------------------------------- 1 | """Add the include-git-dir flag 2 | 3 | Revision ID: 97d5df7fca86 4 | Revises: eff9db96576e 5 | Create Date: 2020-12-02 12:24:05.149962 6 | """ 7 | from alembic import op 8 | import sqlalchemy as sa 9 | 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "97d5df7fca86" 13 | down_revision = "eff9db96576e" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | flag_table = sa.Table( 18 | "flag", 19 | sa.MetaData(), 20 | sa.Column("id", sa.Integer(), primary_key=True), 21 | sa.Column("name", sa.String(), nullable=False), 22 | sa.Column("active", sa.Boolean(), nullable=False, default=True), 23 | ) 24 | 25 | flag_name = "include-git-dir" 26 | 27 | 28 | def upgrade(): 29 | connection = op.get_bind() 30 | res = connection.execute(flag_table.select().where(flag_table.c.name == flag_name)).fetchone() 31 | if res is None: 32 | connection.execute(flag_table.insert().values(name=flag_name, active=True)) 33 | else: 34 | connection.execute( 35 | flag_table.update().where(flag_table.c.name == flag_name).values(active=True) 36 | ) 37 | 38 | 39 | def downgrade(): 40 | connection = op.get_bind() 41 | connection.execute(flag_table.update().values(name=flag_name, active=False)) 42 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/a655a299e967_request_flags.py: -------------------------------------------------------------------------------- 1 | """Add table for flags 2 | 3 | Revision ID: a655a299e967 4 | Revises: cdf17fad3edb 5 | Create Date: 2019-09-17 11:28:08.090670 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "a655a299e967" 14 | down_revision = "cdf17fad3edb" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.create_table( 21 | "flag", 22 | sa.Column("id", sa.Integer(), nullable=False), 23 | sa.Column("name", sa.String(), nullable=False), 24 | sa.Column("active", sa.Boolean(), nullable=False), 25 | sa.PrimaryKeyConstraint("id"), 26 | sa.UniqueConstraint("id", "name", name="flag_id_name_key"), 27 | ) 28 | 29 | op.create_table( 30 | "request_flag", 31 | sa.Column("request_id", sa.Integer(), nullable=False), 32 | sa.Column("flag_id", sa.Integer(), nullable=False), 33 | sa.ForeignKeyConstraint(["flag_id"], ["flag.id"]), 34 | sa.ForeignKeyConstraint(["request_id"], ["request.id"]), 35 | sa.UniqueConstraint("request_id", "flag_id"), 36 | ) 37 | 38 | 39 | def downgrade(): 40 | op.drop_table("request_flag") 41 | op.drop_table("flag") 42 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/b46cf36806d7_add_gomod_vendor_flag.py: -------------------------------------------------------------------------------- 1 | """Add the gomod-vendor flag 2 | 3 | Revision ID: b46cf36806d7 4 | Revises: 615c19a1cee1 5 | Create Date: 2020-05-19 10:33:19.638354 6 | """ 7 | from alembic import op 8 | import sqlalchemy as sa 9 | 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "b46cf36806d7" 13 | down_revision = "615c19a1cee1" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | flag_table = sa.Table( 18 | "flag", 19 | sa.MetaData(), 20 | sa.Column("id", sa.Integer(), primary_key=True), 21 | sa.Column("name", sa.String(), nullable=False), 22 | sa.Column("active", sa.Boolean(), nullable=False, default=True), 23 | ) 24 | 25 | 26 | def upgrade(): 27 | connection = op.get_bind() 28 | res = connection.execute( 29 | flag_table.select().where(flag_table.c.name == "gomod-vendor") 30 | ).fetchone() 31 | if res is None: 32 | connection.execute(flag_table.insert().values(name="gomod-vendor", active=True)) 33 | else: 34 | connection.execute( 35 | flag_table.update().where(flag_table.c.name == "gomod-vendor").values(active=True) 36 | ) 37 | 38 | 39 | def downgrade(): 40 | connection = op.get_bind() 41 | connection.execute(flag_table.update().values(name="gomod-vendor", active=False)) 42 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/c6ac095d8e9f_add_gomod_vendor_check_flag.py: -------------------------------------------------------------------------------- 1 | """Add the gomod-vendor-check flag 2 | 3 | Revision ID: c6ac095d8e9f 4 | Revises: f201f05a95a7 5 | Create Date: 2021-04-15 13:45:44.761284 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "c6ac095d8e9f" 14 | down_revision = "f201f05a95a7" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | flag_table = sa.Table( 20 | "flag", 21 | sa.MetaData(), 22 | sa.Column("id", sa.Integer(), primary_key=True), 23 | sa.Column("name", sa.String(), nullable=False), 24 | sa.Column("active", sa.Boolean(), nullable=False, default=True), 25 | ) 26 | 27 | 28 | def upgrade(): 29 | connection = op.get_bind() 30 | res = connection.execute( 31 | flag_table.select().where(flag_table.c.name == "gomod-vendor-check") 32 | ).fetchone() 33 | if res is None: 34 | connection.execute(flag_table.insert().values(name="gomod-vendor-check", active=True)) 35 | else: 36 | connection.execute( 37 | flag_table.update().where(flag_table.c.name == "gomod-vendor-check").values(active=True) 38 | ) 39 | 40 | 41 | def downgrade(): 42 | connection = op.get_bind() 43 | connection.execute(flag_table.update().values(name="gomod-vendor-check", active=False)) 44 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/c8b2a3a26191_initial_migration.py: -------------------------------------------------------------------------------- 1 | """The initial migration 2 | 3 | Revision ID: c8b2a3a26191 4 | Create Date: 2019-04-08 17:30:01.062645 5 | 6 | """ 7 | from alembic import op 8 | import sqlalchemy as sa 9 | 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "c8b2a3a26191" 13 | down_revision = None 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | pkg_manager_table = op.create_table( 20 | "package_manager", 21 | sa.Column("id", sa.Integer(), nullable=False), 22 | sa.Column("name", sa.String(), nullable=False), 23 | sa.PrimaryKeyConstraint("id"), 24 | ) 25 | 26 | op.create_table( 27 | "user", 28 | sa.Column("id", sa.Integer(), nullable=False), 29 | sa.Column("username", sa.String(), nullable=False), 30 | sa.PrimaryKeyConstraint("id"), 31 | sa.UniqueConstraint("username"), 32 | ) 33 | 34 | op.create_table( 35 | "request", 36 | sa.Column("id", sa.Integer(), nullable=False), 37 | sa.Column("repo", sa.String(), nullable=False), 38 | sa.Column("ref", sa.String(), nullable=False), 39 | sa.Column("user_id", sa.Integer(), nullable=True), 40 | sa.PrimaryKeyConstraint("id"), 41 | sa.ForeignKeyConstraint(["user_id"], ["user.id"]), 42 | ) 43 | 44 | op.create_table( 45 | "request_pkg_manager", 46 | sa.Column("request_id", sa.Integer(), nullable=False), 47 | sa.Column("pkg_manager_id", sa.Integer(), nullable=False), 48 | sa.ForeignKeyConstraint(["pkg_manager_id"], ["package_manager.id"]), 49 | sa.ForeignKeyConstraint(["request_id"], ["request.id"]), 50 | sa.UniqueConstraint("request_id", "pkg_manager_id"), 51 | ) 52 | 53 | # Insert supported pkg managers 54 | op.bulk_insert(pkg_manager_table, [{"name": "gomod"}]) 55 | 56 | op.create_table( 57 | "request_state", 58 | sa.Column("id", sa.Integer(), nullable=False), 59 | sa.Column("state", sa.Integer(), nullable=False), 60 | sa.Column("state_reason", sa.String(), nullable=False), 61 | sa.Column("updated", sa.DateTime(), nullable=False), 62 | sa.Column("request_id", sa.Integer(), nullable=False), 63 | sa.ForeignKeyConstraint(["request_id"], ["request.id"]), 64 | sa.PrimaryKeyConstraint("id"), 65 | ) 66 | 67 | op.create_table( 68 | "dependency", 69 | sa.Column("id", sa.Integer(), nullable=False), 70 | sa.Column("name", sa.String(), nullable=False), 71 | sa.Column("type", sa.String(), nullable=False), 72 | sa.Column("version", sa.String(), nullable=False), 73 | sa.PrimaryKeyConstraint("id"), 74 | sa.UniqueConstraint("name", "type", "version", name="dependency_name_type_version_key"), 75 | ) 76 | 77 | op.create_table( 78 | "request_dependency", 79 | sa.Column("request_id", sa.Integer(), nullable=False), 80 | sa.Column("dependency_id", sa.Integer(), nullable=False), 81 | sa.ForeignKeyConstraint( 82 | ["dependency_id"], ["dependency.id"], "request_dependency_request_id_fkey" 83 | ), 84 | sa.ForeignKeyConstraint( 85 | ["request_id"], ["request.id"], "request_dependency_dependency_id_fkey" 86 | ), 87 | sa.UniqueConstraint( 88 | "request_id", "dependency_id", name="request_dependency_request_id_dependency_id_key" 89 | ), 90 | ) 91 | 92 | 93 | def downgrade(): 94 | op.drop_table("request_pkg_manager") 95 | op.drop_table("package_manager") 96 | op.drop_table("request_state") 97 | op.drop_table("request_dependency") 98 | op.drop_table("dependency") 99 | op.drop_table("request") 100 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/cb6bdbe533cc_add_env_var_types.py: -------------------------------------------------------------------------------- 1 | """Add environment variables types 2 | 3 | Revision ID: cb6bdbe533cc 4 | Revises: 615c19a1cee1 5 | Create Date: 2020-06-22 14:56:53.584293 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "cb6bdbe533cc" 14 | down_revision = "1714d8e3002b" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | env_var_table = sa.Table( 20 | "environment_variable", 21 | sa.MetaData(), 22 | sa.Column("id", sa.Integer(), primary_key=True), 23 | sa.Column("name", sa.String, nullable=False), 24 | sa.Column("value", sa.String, nullable=False), 25 | sa.Column("kind", sa.String, nullable=False), 26 | ) 27 | 28 | 29 | def upgrade(): 30 | with op.batch_alter_table("environment_variable") as batch_op: 31 | batch_op.drop_constraint("environment_variable_name_value_key", type_="unique") 32 | # Make this nullable initially, so we can adjust the data first 33 | batch_op.add_column(sa.Column("kind", sa.String(), nullable=True)) 34 | 35 | connection = op.get_bind() 36 | connection.execute( 37 | env_var_table.update().where(env_var_table.c.kind == sa.null()).values(kind="path") 38 | ) 39 | 40 | with op.batch_alter_table("environment_variable") as batch_op: 41 | batch_op.create_unique_constraint( 42 | "environment_variable_name_value_kind_key", ["name", "value", "kind"] 43 | ) 44 | batch_op.alter_column("kind", nullable=False) 45 | 46 | 47 | def downgrade(): 48 | with op.batch_alter_table("environment_variable") as batch_op: 49 | batch_op.drop_constraint("environment_variable_name_value_kind_key", type_="unique") 50 | batch_op.drop_column("kind") 51 | batch_op.create_unique_constraint("environment_variable_name_value_key", ["name", "value"]) 52 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/cdf17fad3edb_request_env_vars.py: -------------------------------------------------------------------------------- 1 | """Add support for environment variables in request 2 | 3 | Revision ID: cdf17fad3edb 4 | Revises: c8b2a3a26191 5 | Create Date: 2019-09-04 21:07:16.631196 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "cdf17fad3edb" 14 | down_revision = "c8b2a3a26191" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.create_table( 21 | "environment_variable", 22 | sa.Column("id", sa.Integer(), nullable=False), 23 | sa.Column("name", sa.String(), nullable=False), 24 | sa.Column("value", sa.String(), nullable=False), 25 | sa.PrimaryKeyConstraint("id"), 26 | sa.UniqueConstraint("name", "value", name="environment_variable_name_value_key"), 27 | ) 28 | op.create_table( 29 | "request_environment_variable", 30 | sa.Column("request_id", sa.Integer(), nullable=False), 31 | sa.Column("env_var_id", sa.Integer(), nullable=False), 32 | sa.ForeignKeyConstraint(["env_var_id"], ["environment_variable.id"]), 33 | sa.ForeignKeyConstraint(["request_id"], ["request.id"]), 34 | sa.UniqueConstraint("request_id", "env_var_id"), 35 | ) 36 | 37 | 38 | def downgrade(): 39 | op.drop_table("request_environment_variable") 40 | op.drop_table("environment_variable") 41 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/cfbbf7675e3b_add_pip.py: -------------------------------------------------------------------------------- 1 | """Add the pip package manager 2 | 3 | Revision ID: cfbbf7675e3b 4 | Revises: 044548f3f83a 5 | Create Date: 2020-09-04 18:38:07.060924 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "cfbbf7675e3b" 14 | down_revision = "044548f3f83a" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | package_manager_table = sa.Table( 20 | "package_manager", 21 | sa.MetaData(), 22 | sa.Column("id", sa.Integer(), primary_key=True), 23 | sa.Column("name", sa.String(), nullable=False), 24 | ) 25 | 26 | 27 | def upgrade(): 28 | connection = op.get_bind() 29 | connection.execute(package_manager_table.insert().values(name="pip")) 30 | 31 | 32 | def downgrade(): 33 | connection = op.get_bind() 34 | connection.execute(package_manager_table.delete().where(package_manager_table.c.name == "pip")) 35 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/e16de598d00d_add_rubygems.py: -------------------------------------------------------------------------------- 1 | """Add RubyGems package manager 2 | 3 | Revision ID: e16de598d00d 4 | Revises: 92f0d370ba4d 5 | Create Date: 2022-08-03 15:15:49.434025 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "e16de598d00d" 14 | down_revision = "92f0d370ba4d" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | package_manager_table = sa.Table( 19 | "package_manager", 20 | sa.MetaData(), 21 | sa.Column("id", sa.Integer(), primary_key=True), 22 | sa.Column("name", sa.String(), nullable=False), 23 | ) 24 | 25 | 26 | def upgrade(): 27 | connection = op.get_bind() 28 | connection.execute(package_manager_table.insert().values(name="rubygems")) 29 | 30 | 31 | def downgrade(): 32 | connection = op.get_bind() 33 | connection.execute( 34 | package_manager_table.delete().where(package_manager_table.c.name == "rubygems") 35 | ) 36 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/eff9db96576e_add_yarn.py: -------------------------------------------------------------------------------- 1 | """Add the yarn package manager 2 | 3 | Revision ID: eff9db96576e 4 | Revises: 2f83b3e4c5cc 5 | Create Date: 2020-11-10 23:18:10.607458 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "eff9db96576e" 14 | down_revision = "2f83b3e4c5cc" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | package_manager_table = sa.Table( 20 | "package_manager", 21 | sa.MetaData(), 22 | sa.Column("id", sa.Integer(), primary_key=True), 23 | sa.Column("name", sa.String(), nullable=False), 24 | ) 25 | 26 | 27 | def upgrade(): 28 | connection = op.get_bind() 29 | connection.execute(package_manager_table.insert().values(name="yarn")) 30 | 31 | 32 | def downgrade(): 33 | connection = op.get_bind() 34 | connection.execute(package_manager_table.delete().where(package_manager_table.c.name == "yarn")) 35 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/f133002ffdb4_disable_pip_dev_review.py: -------------------------------------------------------------------------------- 1 | """Disable pip-dev-preview flag" 2 | 3 | Revision ID: f133002ffdb4 4 | Revises: 4a64656ba27f 5 | Create Date: 2020-09-29 06:37:57.811885 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "f133002ffdb4" 14 | down_revision = "4a64656ba27f" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | flag_table = sa.Table( 20 | "flag", 21 | sa.MetaData(), 22 | sa.Column("id", sa.Integer(), primary_key=True), 23 | sa.Column("name", sa.String(), nullable=False), 24 | sa.Column("active", sa.Boolean(), nullable=False, default=True), 25 | ) 26 | 27 | 28 | def upgrade(): 29 | connection = op.get_bind() 30 | connection.execute( 31 | flag_table.update().where(flag_table.c.name == "pip-dev-preview").values(active=False) 32 | ) 33 | 34 | 35 | def downgrade(): 36 | connection = op.get_bind() 37 | res = connection.execute( 38 | flag_table.select().where(flag_table.c.name == "pip-dev-preview") 39 | ).fetchone() 40 | if res is None: 41 | connection.execute(flag_table.insert().values(name="pip-dev-preview", active=True)) 42 | else: 43 | connection.execute( 44 | flag_table.update().where(flag_table.c.name == "pip-dev-preview").values(active=True) 45 | ) 46 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/f201f05a95a7_add_index_to_request_repo_and_request_ref.py: -------------------------------------------------------------------------------- 1 | """Add index to Request.repo and Request.ref 2 | 3 | Revision ID: f201f05a95a7 4 | Revises: 07d89c5778f2 5 | Create Date: 2021-04-21 22:12:36.880084 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | revision = "f201f05a95a7" 13 | down_revision = "07d89c5778f2" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | with op.batch_alter_table("request", schema=None) as batch_op: 20 | batch_op.create_index(batch_op.f("ix_request_repo"), ["repo"], unique=False) 21 | batch_op.create_index(batch_op.f("ix_request_ref"), ["ref"], unique=False) 22 | 23 | 24 | def downgrade(): 25 | with op.batch_alter_table("request", schema=None) as batch_op: 26 | batch_op.drop_index(batch_op.f("ix_request_ref")) 27 | batch_op.drop_index(batch_op.f("ix_request_repo")) 28 | -------------------------------------------------------------------------------- /cachito/web/migrations/versions/fdd6d6978386_add_request_package_table.py: -------------------------------------------------------------------------------- 1 | """Add the request_package table 2 | 3 | Revision ID: fdd6d6978386 4 | Revises: 15c4aa0c4144 5 | Create Date: 2019-11-12 11:54:18.760937 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "fdd6d6978386" 14 | down_revision = "15c4aa0c4144" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.create_table( 21 | "request_package", 22 | sa.Column("request_id", sa.Integer(), autoincrement=False, nullable=False), 23 | sa.Column("package_id", sa.Integer(), autoincrement=False, nullable=False), 24 | sa.ForeignKeyConstraint(["package_id"], ["package.id"]), 25 | sa.ForeignKeyConstraint(["request_id"], ["request.id"]), 26 | sa.PrimaryKeyConstraint("request_id", "package_id"), 27 | sa.UniqueConstraint("request_id", "package_id"), 28 | ) 29 | with op.batch_alter_table("request_package", schema=None) as batch_op: 30 | batch_op.create_index( 31 | batch_op.f("ix_request_package_package_id"), ["package_id"], unique=False 32 | ) 33 | batch_op.create_index( 34 | batch_op.f("ix_request_package_request_id"), ["request_id"], unique=False 35 | ) 36 | 37 | 38 | def downgrade(): 39 | with op.batch_alter_table("request_package", schema=None) as batch_op: 40 | batch_op.drop_index(batch_op.f("ix_request_package_request_id")) 41 | batch_op.drop_index(batch_op.f("ix_request_package_package_id")) 42 | 43 | op.drop_table("request_package") 44 | -------------------------------------------------------------------------------- /cachito/web/static/docs.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Cachito API Documentation 6 | 7 | 8 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /cachito/web/utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | from datetime import date, datetime, time 4 | from operator import itemgetter 5 | from typing import Union 6 | 7 | from flask import request, url_for 8 | 9 | CONTAINER_TYPES = (dict, list) 10 | SORT_KEY_BY_PURL = itemgetter("purl") 11 | 12 | 13 | def deep_sort_icm(orig_item): 14 | """ 15 | Return a new element recursively sorted in ascending order. 16 | 17 | The function for sorting image content manifests 18 | 19 | All lists of dicts with a "purl" key will be sorted alphabetically by the 20 | "purl" value. Any other objects will be left as is. 21 | 22 | :param orig_item: Original content manifest to be sorted 23 | :return: Recursively sorted dict or list according to orig_item 24 | :rtype: Any 25 | """ 26 | if orig_item and isinstance(orig_item, dict): 27 | keys = orig_item.keys() 28 | for key in keys: 29 | val = orig_item[key] 30 | if val and isinstance(val, CONTAINER_TYPES): 31 | deep_sort_icm(val) 32 | elif orig_item and isinstance(orig_item, list): 33 | for item in orig_item: 34 | deep_sort_icm(item) 35 | # If item is a list of dicts with the "purl" key, sort by the "purl" value 36 | if isinstance(orig_item[0], dict) and "purl" in orig_item[0]: 37 | orig_item.sort(key=SORT_KEY_BY_PURL) 38 | else: 39 | raise TypeError("Unknown type is included in the content manifest.") 40 | 41 | 42 | def pagination_metadata(pagination_query, **kwargs): 43 | """ 44 | Return a dictionary containing metadata about the paginated query. 45 | 46 | This must be run as part of a Flask request. 47 | 48 | :param pagination_query: flask_sqlalchemy.Pagination object 49 | :param dict kwargs: the query parameters to add to the URLs 50 | :return: a dictionary containing metadata about the paginated query 51 | """ 52 | pagination_data = { 53 | "first": url_for( 54 | request.endpoint, page=1, per_page=pagination_query.per_page, _external=True, **kwargs 55 | ), 56 | "last": url_for( 57 | request.endpoint, 58 | page=pagination_query.pages, 59 | per_page=pagination_query.per_page, 60 | _external=True, 61 | **kwargs, 62 | ), 63 | "next": None, 64 | "page": pagination_query.page, 65 | "pages": pagination_query.pages, 66 | "per_page": pagination_query.per_page, 67 | "previous": None, 68 | "total": pagination_query.total, 69 | } 70 | 71 | if pagination_query.has_prev: 72 | pagination_data["previous"] = url_for( 73 | request.endpoint, 74 | page=pagination_query.prev_num, 75 | per_page=pagination_query.per_page, 76 | _external=True, 77 | **kwargs, 78 | ) 79 | if pagination_query.has_next: 80 | pagination_data["next"] = url_for( 81 | request.endpoint, 82 | page=pagination_query.next_num, 83 | per_page=pagination_query.per_page, 84 | _external=True, 85 | **kwargs, 86 | ) 87 | 88 | return pagination_data 89 | 90 | 91 | def str_to_bool(item): 92 | """ 93 | Convert a string to a boolean. 94 | 95 | :param str item: string to parse 96 | :return: a boolean equivalent 97 | :rtype: boolean 98 | """ 99 | if isinstance(item, str): 100 | return item.lower() in ("true", "1") 101 | else: 102 | return False 103 | 104 | 105 | def normalize_end_date(value: Union[datetime, date, None]): 106 | """ 107 | Convert date value to the end of the day datetime. 108 | 109 | The function doesn't touch values of any other input types. 110 | Example: 111 | Input value: date(2021, 10, 21) 112 | Output value: datetime(2021, 10, 21, 23, 59, 59, 999999) 113 | """ 114 | if isinstance(value, date) and not isinstance(value, datetime): 115 | return datetime.combine(value, time.max) 116 | return value 117 | -------------------------------------------------------------------------------- /cachito/web/validation.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | import datetime 4 | from typing import Any, Dict, List 5 | 6 | import jsonschema 7 | from connexion import decorators 8 | from connexion.exceptions import BadRequestProblem, ExtraParameterProblem 9 | 10 | from cachito.errors import ValidationError 11 | 12 | 13 | def validate_replacement(replacement: Dict[str, Any]) -> None: 14 | """ 15 | Validate the JSON representation of a dependency replacement. 16 | 17 | :param replacement: the JSON representation of a dependency replacement 18 | :type replacement: dict[str, any] 19 | :raise ValidationError: if the JSON does not match the required schema 20 | """ 21 | required = {"name", "type", "version"} 22 | optional = {"new_name"} 23 | 24 | if not isinstance(replacement, dict) or (replacement.keys() - required - optional): 25 | raise ValidationError( 26 | "A dependency replacement must be a JSON object with the following " 27 | f'keys: {", ".join(sorted(required))}. It may also contain the following optional ' 28 | f'keys: {", ".join(sorted(optional))}.' 29 | ) 30 | 31 | for key in required | optional: 32 | # Skip the validation of optional keys that are not set 33 | if key not in replacement and key in optional: 34 | continue 35 | 36 | if not isinstance(replacement[key], str): 37 | raise ValidationError( 38 | 'The "{}" key of the dependency replacement must be a string'.format(key) 39 | ) 40 | 41 | 42 | def validate_dependency_replacements(replacements: List[Dict[str, Any]]) -> None: 43 | """ 44 | Validate the JSON representation of dependency replacements. 45 | 46 | :param replacement: a list of JSON representation of dependency replacements. 47 | :type replacement: list[dict[str, any]] 48 | :raise ValidationError: if the JSON does not match the required schema. 49 | """ 50 | if not isinstance(replacements, list): 51 | raise ValidationError('"dependency_replacements" must be an array') 52 | for replacement in replacements: 53 | validate_replacement(replacement) 54 | 55 | 56 | class RequestBodyValidator(decorators.validation.RequestBodyValidator): 57 | """ 58 | Changes the default Connexion exception to Cachito's ValidationError. 59 | 60 | For more information about custom validation error handling: 61 | - https://github.com/zalando/connexion/issues/558 62 | - https://connexion.readthedocs.io/en/latest/request.html 63 | """ 64 | 65 | def validate_schema(self, data, url): 66 | """Raise cachito.ValidationError.""" 67 | if self.is_null_value_valid and jsonschema.is_null(data): 68 | return None 69 | try: 70 | self.validator.validate(data) 71 | except jsonschema.ValidationError as exception: 72 | raise ValidationError(exception.message) 73 | 74 | return None 75 | 76 | 77 | class ParameterValidator(decorators.validation.ParameterValidator): 78 | """ 79 | Changes the default Connexion exception to Cachito's ValidationError. 80 | 81 | For more information about custom validation error handling: 82 | - https://github.com/zalando/connexion/issues/558 83 | - https://connexion.readthedocs.io/en/latest/request.html 84 | """ 85 | 86 | def __call__(self, function): 87 | """Throw cachito.ValidationError.""" 88 | wrapper = super().__call__(function) 89 | 90 | def handle_wrapper(request): 91 | """Handle original wrapper.""" 92 | try: 93 | return wrapper(request) 94 | except (BadRequestProblem, ExtraParameterProblem) as exception: 95 | raise ValidationError(exception.detail) 96 | 97 | return handle_wrapper 98 | 99 | 100 | @jsonschema.draft4_format_checker.checks("datetime") 101 | def datetime_validator(val: Any) -> bool: 102 | """Validate that datetime fields have the correct format.""" 103 | if not isinstance(val, str): 104 | raise ValidationError( 105 | f"'{val}' is not string type to be evaluated as datetime(ISO 8601 format)." 106 | ) 107 | 108 | try: 109 | datetime.datetime.fromisoformat(val) 110 | except ValueError: 111 | raise ValidationError(f"'{val}' is not a valid datetime(ISO 8601 format).") 112 | 113 | return True 114 | -------------------------------------------------------------------------------- /cachito/web/wsgi.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | from cachito.web.app import create_app 3 | from cachito.web.config import validate_cachito_config 4 | 5 | app = create_app() 6 | validate_cachito_config(app.config) 7 | -------------------------------------------------------------------------------- /cachito/workers/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | import json 4 | import logging 5 | import re 6 | import subprocess # nosec 7 | from pathlib import Path 8 | from tarfile import ExtractError, TarFile 9 | from typing import Iterator 10 | 11 | from opentelemetry import trace 12 | 13 | from cachito.errors import SubprocessCallError 14 | from cachito.workers.config import get_worker_config 15 | from cachito.workers.errors import CachitoCalledProcessError 16 | 17 | log = logging.getLogger(__name__) 18 | tracer = trace.get_tracer(__name__) 19 | 20 | 21 | def run_cmd(cmd, params, exc_msg=None): 22 | """ 23 | Run the given command with provided parameters. 24 | 25 | :param iter cmd: iterable representing command to be executed 26 | :param dict params: keyword parameters for command execution 27 | :param str exc_msg: an optional exception message when the command fails 28 | :returns: the command output 29 | :rtype: str 30 | :raises SubprocessCallError: if the command fails 31 | """ 32 | params.setdefault("capture_output", True) 33 | params.setdefault("universal_newlines", True) 34 | params.setdefault("encoding", "utf-8") 35 | with tracer.start_as_current_span("running cmd " + " ".join(cmd)): 36 | conf = get_worker_config() 37 | params.setdefault("timeout", conf.cachito_subprocess_timeout) 38 | 39 | try: 40 | response = subprocess.run(cmd, **params) # nosec 41 | except subprocess.TimeoutExpired as e: 42 | raise SubprocessCallError(str(e)) 43 | 44 | if response.returncode != 0: 45 | log.error('The command "%s" failed with: %s', " ".join(cmd), response.stderr) 46 | raise CachitoCalledProcessError( 47 | exc_msg or "An unexpected error occurred", response.returncode 48 | ) 49 | 50 | return response.stdout 51 | 52 | 53 | def load_json_stream(s: str) -> Iterator: 54 | """ 55 | Load all JSON objects from input string. 56 | 57 | The objects can be separated by one or more whitespace characters. The return value is 58 | a generator that will yield the parsed objects one by one. 59 | """ 60 | decoder = json.JSONDecoder() 61 | non_whitespace = re.compile(r"\S") 62 | i = 0 63 | 64 | while match := non_whitespace.search(s, i): 65 | obj, i = decoder.raw_decode(s, match.start()) 66 | yield obj 67 | 68 | 69 | def safe_extract(tar: TarFile, path: str = ".", *, numeric_owner: bool = False): 70 | """ 71 | CVE-2007-4559 replacement for extract() or extractall(). 72 | 73 | By using extract() or extractall() on a tarfile object without sanitizing input, 74 | a maliciously crafted .tar file could perform a directory path traversal attack. 75 | The patch essentially checks to see if all tarfile members will be 76 | extracted safely and throws an exception otherwise. 77 | 78 | :param tarfile tar: the tarfile to be extracted. 79 | :param str path: specifies a different directory to extract to. 80 | :param numeric_owner: if True, only the numbers for user/group names are used and not the names. 81 | :raise ExtractError: if there is a Traversal Path Attempt in the Tar File. 82 | """ 83 | abs_path = Path(path).resolve() 84 | for member in tar.getmembers(): 85 | member_path = Path(path).joinpath(member.name) 86 | abs_member_path = member_path.resolve() 87 | 88 | if not abs_member_path.is_relative_to(abs_path): 89 | raise ExtractError("Attempted Path Traversal in Tar File") 90 | 91 | tar.extractall(path, numeric_owner=numeric_owner) 92 | -------------------------------------------------------------------------------- /cachito/workers/cleanup_job.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import logging 3 | from datetime import datetime, timedelta 4 | 5 | import requests 6 | 7 | from cachito.errors import NetworkError 8 | from cachito.workers.config import get_worker_config 9 | from cachito.workers.requests import get_requests_session 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | auth_session = get_requests_session(auth=True) 14 | session = get_requests_session() 15 | config = get_worker_config() 16 | 17 | # state, state_reason and payload values for marking a request in Cachito API 18 | state = "stale" 19 | state_reason = "The request has expired" 20 | payload = {"state": state, "state_reason": state_reason} 21 | 22 | 23 | def main(): 24 | """Mark all end of life requests as stale using the REST API.""" 25 | logging.basicConfig(level=logging.INFO) 26 | for state in ("complete", "in_progress", "failed"): 27 | stale_candidate_requests = find_all_requests_in_state(state) 28 | identify_and_mark_stale_requests(stale_candidate_requests) 29 | 30 | 31 | def find_all_requests_in_state(state): 32 | """ 33 | Find all requests in specified state. 34 | 35 | :param str state: state of request, e.g. 'complete', 'in_progress' 36 | :return: list of Cachito requests (as JSON data) in specified state 37 | :raises NetworkError: if connection fails 38 | """ 39 | found_requests = [] 40 | 41 | url = f"{config.cachito_api_url.rstrip('/')}/requests" 42 | while url: 43 | try: 44 | response = session.get(url, params={"state": state}, timeout=config.cachito_api_timeout) 45 | except requests.RequestException: 46 | msg = f"The connection failed when querying {url}" 47 | log.exception(msg) 48 | raise NetworkError(msg) 49 | 50 | if not response.ok: 51 | log.error( 52 | "The request to %s failed with the status code %d and the following text: %s", 53 | url, 54 | response.status_code, 55 | response.text, 56 | ) 57 | raise NetworkError( 58 | "Could not reach the Cachito API to find the requests to be marked as stale" 59 | ) 60 | 61 | json_response = response.json() 62 | found_requests.extend(json_response["items"]) 63 | url = json_response["meta"]["next"] 64 | 65 | # Remove potential duplicates found due to the dynamic behaviour of pagination 66 | deduplicated = {request["id"]: request for request in found_requests} 67 | return list(deduplicated.values()) 68 | 69 | 70 | def identify_and_mark_stale_requests(requests_json): 71 | """ 72 | Identify Cachito requests which have reached end of life and mark them as stale. 73 | 74 | :param list requests_json: list of Cachito requests (as JSON data) 75 | """ 76 | current_time = datetime.utcnow() 77 | for request in requests_json: 78 | if request["state"] not in ("complete", "in_progress", "failed"): 79 | continue 80 | 81 | lifetime = config.cachito_request_lifetime 82 | if request["state"] == "failed": 83 | lifetime = config.cachito_request_lifetime_failed 84 | date_time_obj = datetime.strptime(request["updated"], "%Y-%m-%dT%H:%M:%S.%f") 85 | if current_time - date_time_obj > timedelta(lifetime): 86 | mark_as_stale(request["id"]) 87 | 88 | 89 | def mark_as_stale(request_id): 90 | """ 91 | Mark the identified stale request ID as `stale` in Cachito. 92 | 93 | :param int request_id: request ID identified as stale 94 | :raise NetworkError: if the request to the Cachito API fails 95 | """ 96 | try: 97 | log.info("Setting state of request %d to `stale`", request_id) 98 | request_rv = auth_session.patch( 99 | f'{config.cachito_api_url.rstrip("/")}/requests/{request_id}', 100 | json=payload, 101 | timeout=config.cachito_api_timeout, 102 | ) 103 | except requests.RequestException: 104 | msg = f"The connection failed when setting the `stale` state on request {request_id}" 105 | log.exception(msg) 106 | raise NetworkError(msg) 107 | 108 | if not request_rv.ok: 109 | log.error( 110 | "Failed to set the `stale` state on request %d. The status was %d. " 111 | "The text was:\n%s", 112 | request_id, 113 | request_rv.status_code, 114 | request_rv.text, 115 | ) 116 | 117 | 118 | if __name__ == "__main__": 119 | main() 120 | -------------------------------------------------------------------------------- /cachito/workers/errors.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | from cachito.errors import CachitoError 3 | 4 | 5 | class NexusScriptError(CachitoError): 6 | """An error was encountered while executing a Nexus script.""" 7 | 8 | 9 | class CachitoCalledProcessError(CachitoError): 10 | """Command executed with subprocess.run() returned non-zero value.""" 11 | 12 | def __init__(self, err_msg: str, retcode: int): 13 | """Initialize the error with a message and the return code of the failing command.""" 14 | super().__init__(err_msg) 15 | self.retcode = retcode 16 | 17 | 18 | class UploadError(CachitoError): 19 | """Uploading of dependency to a temporary Nexus repo failure.""" 20 | -------------------------------------------------------------------------------- /cachito/workers/nexus_scripts/js_after_content_staged.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | This script configures Nexus so that the NPM proxy repository for the Cachito request is blocked from getting additional 3 | content. 4 | 5 | In addition, a temporary user for the request is created and is given permission to access this NPM proxy repository. 6 | It'd be preferable to give access to the Nexus anonymous user instead, but there is no way to add a role to a user. 7 | You can only set the entire set of roles at once. This is an issue since if more than one Cachito request is in progress 8 | and modifies the set of roles at the same time, one of the additions will be lost. 9 | */ 10 | import com.google.common.collect.Sets 11 | import groovy.json.JsonSlurper 12 | import groovy.transform.Field 13 | import org.slf4j.Logger 14 | import org.slf4j.LoggerFactory 15 | import org.sonatype.nexus.security.role.NoSuchRoleException 16 | import org.sonatype.nexus.security.user.UserStatus 17 | import org.sonatype.nexus.repository.config.Configuration 18 | import org.sonatype.nexus.security.authz.AuthorizationManager 19 | import org.sonatype.nexus.security.role.Role 20 | import static org.sonatype.nexus.security.user.UserManager.DEFAULT_SOURCE 21 | import org.sonatype.nexus.security.user.UserNotFoundException 22 | 23 | 24 | // Scope logger to the script using @Field 25 | @Field final Logger logger = LoggerFactory.getLogger('cachito'); 26 | 27 | 28 | void createUser(String username, String password, List roles) { 29 | try { 30 | // security is an object that is injected by Nexus when the script is executed 31 | def user = security.securitySystem.getUser(username) 32 | logger.info("Modifying the existing user ${username}") 33 | user.setFirstName(username) 34 | user.setLastName(username) 35 | user.setEmailAddress('noreply@domain.local') 36 | user.setStatus(UserStatus.active) 37 | security.securitySystem.updateUser(user) 38 | security.setUserRoles(username, roles) 39 | security.securitySystem.changePassword(username, password) 40 | } catch (UserNotFoundException e) { 41 | logger.info("Creating the user ${username}") 42 | String firstName = username 43 | String lastName = username 44 | String email = 'noreply@domain.local' 45 | Boolean active = true 46 | // security is an object that is injected by Nexus when the script is executed 47 | security.addUser(username, firstName, lastName, email, active, password, roles) 48 | } 49 | } 50 | 51 | 52 | void createRole(String name, String description, List privileges) { 53 | // security is an object that is injected by Nexus when the script is executed 54 | AuthorizationManager authorizationManager = security.securitySystem.getAuthorizationManager(DEFAULT_SOURCE) 55 | 56 | String roleID = name 57 | try { 58 | Role role = authorizationManager.getRole(roleID) 59 | logger.info("Modifying the role ${name}") 60 | role.privileges = Sets.newHashSet(privileges) 61 | authorizationManager.updateRole(role) 62 | } catch (NoSuchRoleException e) { 63 | logger.info("Creating the role ${name}") 64 | List roles = [] 65 | security.addRole(roleID, name, description, privileges, roles) 66 | } 67 | } 68 | 69 | 70 | void blockOutboundConnections(String repositoryName) { 71 | logger.info("Blocking outbound connections from the NPM proxy repository ${repositoryName}") 72 | // repository is an object that is injected by Nexus when the script is executed 73 | Configuration repoConfig = repository.repositoryManager.get(repositoryName).configuration 74 | repoConfig.attributes('httpclient').set('blocked', true) 75 | repository.repositoryManager.update(repoConfig) 76 | } 77 | 78 | 79 | // Main execution starts here 80 | request = new JsonSlurper().parseText(args) 81 | ['repository_name', 'password', 'username'].each { param -> 82 | assert request.get(param): "The ${param} parameter is required" 83 | } 84 | 85 | // Block outbound connections so that the proxy cannot fetch additional content 86 | blockOutboundConnections(request.repository_name) 87 | // Just name the role the same as the username for convenience 88 | String roleName = request.username 89 | // toString is needed to convert the GString to the Java String 90 | List privileges = ["nx-repository-view-npm-${request.repository_name}-read".toString()] 91 | // Create a role that has read access on the new repository. This will allow a user with this role to utilize the 92 | // the NPM proxy for this Cachito request. 93 | createRole(roleName, "Read access for ${request.repository_name}".toString(), privileges) 94 | List roles = [roleName] 95 | // Create a user with the role above 96 | createUser(request.username, request.password, roles) 97 | 98 | return 'The repository was configured successfully' 99 | -------------------------------------------------------------------------------- /cachito/workers/nexus_scripts/js_cleanup.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | This script deletes the NPM proxy repository, user, and role for the Cachito request. 3 | 4 | This script should get executed when the Cachito request is set to the stale state. 5 | */ 6 | import groovy.json.JsonSlurper 7 | import org.slf4j.Logger 8 | import org.slf4j.LoggerFactory 9 | import org.sonatype.nexus.security.authz.AuthorizationManager 10 | import org.sonatype.nexus.security.role.NoSuchRoleException 11 | import org.sonatype.nexus.security.user.UserNotFoundException 12 | 13 | import static org.sonatype.nexus.security.user.UserManager.DEFAULT_SOURCE 14 | 15 | 16 | request = new JsonSlurper().parseText(args) 17 | ['repository_name', 'username'].each { param -> 18 | assert request.get(param): "The ${param} parameter is required" 19 | } 20 | 21 | final Logger logger = LoggerFactory.getLogger('cachito'); 22 | try { 23 | logger.info("Deleting the user ${request.username}") 24 | // security is an object that is injected by Nexus when the script is executed 25 | security.securitySystem.deleteUser(request.username, DEFAULT_SOURCE) 26 | } catch(UserNotFoundException e) { 27 | logger.warn("The user ${request.username} was not found") 28 | } 29 | 30 | // security is an object that is injected by Nexus when the script is executed 31 | AuthorizationManager authorizationManager = security.securitySystem.getAuthorizationManager(DEFAULT_SOURCE) 32 | // The role is named the same as the username 33 | String roleName = request.username 34 | try { 35 | logger.info("Deleting the role ${roleName}") 36 | authorizationManager.deleteRole(roleName) 37 | } catch(NoSuchRoleException e) { 38 | logger.warn("The role ${roleName} was not found") 39 | } 40 | 41 | // repository is an object that is injected by Nexus when the script is executed 42 | logger.info("Deleting the repository ${request.repository_name}") 43 | if (repository.repositoryManager.exists(request.repository_name)) { 44 | repository.repositoryManager.delete(request.repository_name) 45 | } else { 46 | logger.warn("The repository ${request.repository_name} was not found") 47 | } 48 | 49 | return "The NPM proxy repository ${request.repository_name} and the user and role ${request.username} are removed" 50 | -------------------------------------------------------------------------------- /cachito/workers/nexus_scripts/pip_after_content_staged.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | This script configures Nexus so that a temporary user for the request is created and is given 3 | permission to access the PyPI hosted repository and the raw hosted repository. It'd be preferable 4 | to give access to the Nexus anonymous user instead, but there is no way to add a role to a user. 5 | You can only set the entire set of roles at once. This is an issue since if more than one Cachito 6 | request is in progress and modifies the set of roles at the same time, one of the additions will be 7 | lost. 8 | 9 | Differently from its JS counterpart, this script does not block outbound connections for the 10 | temporary request repositories. This is not needed because both Python repositories used here 11 | are hosted repositories, and not PyPI proxies. In other words, they would not automatically pull 12 | unavailable contents from PyPI, as the npm repository would. 13 | */ 14 | import com.google.common.collect.Sets 15 | import groovy.json.JsonSlurper 16 | import groovy.transform.Field 17 | import org.slf4j.Logger 18 | import org.slf4j.LoggerFactory 19 | import org.sonatype.nexus.security.role.NoSuchRoleException 20 | import org.sonatype.nexus.security.user.UserStatus 21 | import org.sonatype.nexus.security.authz.AuthorizationManager 22 | import org.sonatype.nexus.security.role.Role 23 | import static org.sonatype.nexus.security.user.UserManager.DEFAULT_SOURCE 24 | import org.sonatype.nexus.security.user.UserNotFoundException 25 | 26 | 27 | // Scope logger to the script using @Field 28 | @Field final Logger logger = LoggerFactory.getLogger('cachito'); 29 | 30 | 31 | void createUser(String username, String password, List roles) { 32 | try { 33 | // security is an object that is injected by Nexus when the script is executed 34 | def user = security.securitySystem.getUser(username) 35 | logger.info("Modifying the existing user ${username}") 36 | user.setFirstName(username) 37 | user.setLastName(username) 38 | user.setEmailAddress('noreply@domain.local') 39 | user.setStatus(UserStatus.active) 40 | security.securitySystem.updateUser(user) 41 | security.setUserRoles(username, roles) 42 | security.securitySystem.changePassword(username, password) 43 | } catch (UserNotFoundException e) { 44 | logger.info("Creating the user ${username}") 45 | String firstName = username 46 | String lastName = username 47 | String email = 'noreply@domain.local' 48 | Boolean active = true 49 | // security is an object that is injected by Nexus when the script is executed 50 | security.addUser(username, firstName, lastName, email, active, password, roles) 51 | } 52 | } 53 | 54 | 55 | void createRole(String name, String description, List privileges) { 56 | // security is an object that is injected by Nexus when the script is executed 57 | AuthorizationManager authorizationManager = security.securitySystem.getAuthorizationManager(DEFAULT_SOURCE) 58 | 59 | String roleID = name 60 | try { 61 | Role role = authorizationManager.getRole(roleID) 62 | logger.info("Modifying the role ${name}") 63 | role.privileges = Sets.newHashSet(privileges) 64 | authorizationManager.updateRole(role) 65 | } catch (NoSuchRoleException e) { 66 | logger.info("Creating the role ${name}") 67 | List roles = [] 68 | security.addRole(roleID, name, description, privileges, roles) 69 | } 70 | } 71 | 72 | 73 | // Main execution starts here 74 | request = new JsonSlurper().parseText(args) 75 | ['pip_repository_name', 'raw_repository_name', 'password', 'username'].each { param -> 76 | assert request.get(param): "The ${param} parameter is required" 77 | } 78 | 79 | // Just name the role the same as the username for convenience 80 | String roleName = request.username 81 | // toString is needed to convert the GString to the Java String 82 | String pypiHostedPrivilege = "nx-repository-view-pypi-${request.pip_repository_name}-read".toString() 83 | String rawHostedPrivilege = "nx-repository-view-raw-${request.raw_repository_name}-read".toString() 84 | List privileges = [pypiHostedPrivilege, rawHostedPrivilege] 85 | // Create a role that has read access on the new repositories. 86 | // This will allow a user with this role to utilize the the Python repos for this Cachito request. 87 | String desc = "Read access for ${request.pip_repository_name} and ${request.raw_repository_name}".toString() 88 | createRole(roleName, desc, privileges) 89 | List roles = [roleName] 90 | // Create a user with the role above 91 | createUser(request.username, request.password, roles) 92 | 93 | return 'The repositories, user, and role were configured successfully' 94 | -------------------------------------------------------------------------------- /cachito/workers/nexus_scripts/pip_before_content_staged.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | This script configures Nexus so that Cachito can stage Python content for the Cachito request. 3 | 4 | This script creates a PyPI hosted repository and a raw repository to be used by a Cachito request 5 | to fetch Python content 6 | 7 | No permissions are configured since it is expected that Cachito's Nexus service account has access 8 | to use all Python related repositories managed by the Nexus instance. 9 | */ 10 | import groovy.json.JsonSlurper 11 | import groovy.transform.Field 12 | import org.slf4j.Logger 13 | import org.slf4j.LoggerFactory 14 | import org.sonatype.nexus.repository.config.Configuration 15 | import org.sonatype.nexus.repository.config.WritePolicy 16 | 17 | 18 | // Scope logger to the script using @Field 19 | @Field final Logger logger = LoggerFactory.getLogger('cachito'); 20 | 21 | 22 | def createHostedRepo(String name, String repoType) { 23 | WritePolicy writePolicy = WritePolicy.ALLOW_ONCE 24 | Boolean strictContentValidation = true 25 | String blobStoreName = "cachito-pip" 26 | // repository is an object that is injected by Nexus when the script is executed 27 | if(repository.repositoryManager.exists(name)) { 28 | logger.info("Modifying the hosted repository ${name}") 29 | Configuration hostedRepoConfig = repository.repositoryManager.get(name).configuration 30 | def storage = hostedRepoConfig.attributes('storage') 31 | storage.set('strictContentTypeValidation', strictContentValidation) 32 | storage.set('writePolicy', writePolicy) 33 | repository.repositoryManager.update(hostedRepoConfig) 34 | } 35 | else { 36 | logger.info("Creating the hosted ${repoType} repository ${name}") 37 | switch(repoType) { 38 | case "raw": 39 | repository.createRawHosted(name, blobStoreName, strictContentValidation, writePolicy) 40 | break; 41 | case "pypi": 42 | repository.createPyPiHosted(name, blobStoreName, strictContentValidation, writePolicy) 43 | break; 44 | default: 45 | logger.warn("Type ${repoType} not supported. repository ${name} not created.") 46 | break; 47 | } 48 | } 49 | } 50 | 51 | 52 | request = new JsonSlurper().parseText(args) 53 | ['pip_repository_name', 'raw_repository_name'].each { param -> 54 | assert request.get(param): "The ${param} parameter is required" 55 | } 56 | 57 | createHostedRepo(request.pip_repository_name, "pypi") 58 | createHostedRepo(request.raw_repository_name, "raw") 59 | 60 | return 'The repositories were created successfully' 61 | -------------------------------------------------------------------------------- /cachito/workers/nexus_scripts/pip_cleanup.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | This script deletes the PyPI hosted repositories (PyPI and raw), user, and role for the Cachito request. 3 | 4 | This script should get executed when the Cachito request is set to the stale state. 5 | */ 6 | import groovy.json.JsonSlurper 7 | import org.slf4j.Logger 8 | import org.slf4j.LoggerFactory 9 | import org.sonatype.nexus.security.authz.AuthorizationManager 10 | import org.sonatype.nexus.security.role.NoSuchRoleException 11 | import org.sonatype.nexus.security.user.UserNotFoundException 12 | 13 | import static org.sonatype.nexus.security.user.UserManager.DEFAULT_SOURCE 14 | 15 | 16 | request = new JsonSlurper().parseText(args) 17 | ['pip_repository_name', 'raw_repository_name', 'username'].each { param -> 18 | assert request.get(param): "The ${param} parameter is required" 19 | } 20 | 21 | final Logger logger = LoggerFactory.getLogger('cachito'); 22 | try { 23 | logger.info("Deleting the user ${request.username}") 24 | // security is an object that is injected by Nexus when the script is executed 25 | security.securitySystem.deleteUser(request.username, DEFAULT_SOURCE) 26 | } catch(UserNotFoundException e) { 27 | logger.warn("The user ${request.username} was not found") 28 | } 29 | 30 | // security is an object that is injected by Nexus when the script is executed 31 | AuthorizationManager authorizationManager = security.securitySystem.getAuthorizationManager(DEFAULT_SOURCE) 32 | // The role is named the same as the username 33 | String roleName = request.username 34 | try { 35 | logger.info("Deleting the role ${roleName}") 36 | authorizationManager.deleteRole(roleName) 37 | } catch(NoSuchRoleException e) { 38 | logger.warn("The role ${roleName} was not found") 39 | } 40 | 41 | // repository is an object that is injected by Nexus when the script is executed 42 | logger.info("Deleting the repository ${request.pip_repository_name}") 43 | if (repository.repositoryManager.exists(request.pip_repository_name)) { 44 | repository.repositoryManager.delete(request.pip_repository_name) 45 | } else { 46 | logger.warn("The repository ${request.pip_repository_name} was not found") 47 | } 48 | logger.info("Deleting the repository ${request.raw_repository_name}") 49 | if (repository.repositoryManager.exists(request.raw_repository_name)) { 50 | repository.repositoryManager.delete(request.raw_repository_name) 51 | } else { 52 | logger.warn("The repository ${request.raw_repository_name} was not found") 53 | } 54 | 55 | return "The Python repositories ${request.pip_repository_name}, ${request.raw_repository_name}, and the user and role ${request.username} are removed" 56 | -------------------------------------------------------------------------------- /cachito/workers/nexus_scripts/rubygems_after_content_staged.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | This script configures Nexus so that a temporary user for the request is created and is given 3 | permission to access the Rubygems hosted repository and the raw hosted repository. It'd be 4 | preferable to give access to the Nexus anonymous user instead, but there is no way to add a role to 5 | a user. You can only set the entire set of roles at once. This is an issue since if more than one 6 | Cachito request is in progress and modifies the set of roles at the same time, one of the additions 7 | will be lost. 8 | 9 | Differently from its JS counterpart, this script does not block outbound connections for the 10 | temporary request repositories. This is not needed because both Rubygems repositories used here 11 | are hosted repositories, and not rubygems.org proxies. In other words, they would not automatically 12 | pull unavailable contents from rubygems.org, as the npm repository would. 13 | */ 14 | import com.google.common.collect.Sets 15 | import groovy.json.JsonSlurper 16 | import groovy.transform.Field 17 | import org.slf4j.Logger 18 | import org.slf4j.LoggerFactory 19 | import org.sonatype.nexus.security.role.NoSuchRoleException 20 | import org.sonatype.nexus.security.user.UserStatus 21 | import org.sonatype.nexus.security.authz.AuthorizationManager 22 | import org.sonatype.nexus.security.role.Role 23 | import static org.sonatype.nexus.security.user.UserManager.DEFAULT_SOURCE 24 | import org.sonatype.nexus.security.user.UserNotFoundException 25 | 26 | 27 | // Scope logger to the script using @Field 28 | @Field final Logger logger = LoggerFactory.getLogger('cachito'); 29 | 30 | 31 | void createUser(String username, String password, List roles) { 32 | try { 33 | // security is an object that is injected by Nexus when the script is executed 34 | def user = security.securitySystem.getUser(username) 35 | logger.info("Modifying the existing user ${username}") 36 | user.setFirstName(username) 37 | user.setLastName(username) 38 | user.setEmailAddress('noreply@domain.local') 39 | user.setStatus(UserStatus.active) 40 | security.securitySystem.updateUser(user) 41 | security.setUserRoles(username, roles) 42 | security.securitySystem.changePassword(username, password) 43 | } catch (UserNotFoundException e) { 44 | logger.info("Creating the user ${username}") 45 | String firstName = username 46 | String lastName = username 47 | String email = 'noreply@domain.local' 48 | Boolean active = true 49 | // security is an object that is injected by Nexus when the script is executed 50 | security.addUser(username, firstName, lastName, email, active, password, roles) 51 | } 52 | } 53 | 54 | 55 | void createRole(String name, String description, List privileges) { 56 | // security is an object that is injected by Nexus when the script is executed 57 | AuthorizationManager authorizationManager = security.securitySystem.getAuthorizationManager(DEFAULT_SOURCE) 58 | 59 | String roleID = name 60 | try { 61 | Role role = authorizationManager.getRole(roleID) 62 | logger.info("Modifying the role ${name}") 63 | role.privileges = Sets.newHashSet(privileges) 64 | authorizationManager.updateRole(role) 65 | } catch (NoSuchRoleException e) { 66 | logger.info("Creating the role ${name}") 67 | List roles = [] 68 | security.addRole(roleID, name, description, privileges, roles) 69 | } 70 | } 71 | 72 | 73 | // Main execution starts here 74 | request = new JsonSlurper().parseText(args) 75 | ['rubygems_repository_name', 'password', 'username'].each { param -> 76 | assert request.get(param): "The ${param} parameter is required" 77 | } 78 | 79 | // Just name the role the same as the username for convenience 80 | String roleName = request.username 81 | // toString is needed to convert the GString to the Java String 82 | String rubygemsHostedPrivilege = "nx-repository-view-rubygems-${request.rubygems_repository_name}-read".toString() 83 | List privileges = [rubygemsHostedPrivilege] 84 | // Create a role that has read access on the new repositories. 85 | // This will allow a user with this role to utilize the the Rubygems repos for this Cachito request. 86 | String desc = "Read access for ${request.rubygems_repository_name} and ${request.raw_repository_name}".toString() 87 | createRole(roleName, desc, privileges) 88 | List roles = [roleName] 89 | // Create a user with the role above 90 | createUser(request.username, request.password, roles) 91 | 92 | return 'The repositories, user, and role were configured successfully' 93 | -------------------------------------------------------------------------------- /cachito/workers/nexus_scripts/rubygems_before_content_staged.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | This script configures Nexus so that Cachito can stage Rubygems content for the Cachito request. 3 | 4 | This script creates a Rubygems hosted repository and a raw repository to be used by a Cachito 5 | request to fetch Ruby content 6 | 7 | No permissions are configured since it is expected that Cachito's Nexus service account has access 8 | to use all Ruby related repositories managed by the Nexus instance. 9 | */ 10 | import groovy.json.JsonSlurper 11 | import groovy.transform.Field 12 | import org.slf4j.Logger 13 | import org.slf4j.LoggerFactory 14 | import org.sonatype.nexus.repository.config.Configuration 15 | import org.sonatype.nexus.repository.config.WritePolicy 16 | 17 | 18 | // Scope logger to the script using @Field 19 | @Field final Logger logger = LoggerFactory.getLogger('cachito'); 20 | 21 | 22 | def createHostedRubyGemsRepo(String name) { 23 | WritePolicy writePolicy = WritePolicy.ALLOW_ONCE 24 | Boolean strictContentValidation = true 25 | String blobStoreName = "cachito-rubygems" 26 | // repository is an object that is injected by Nexus when the script is executed 27 | if(repository.repositoryManager.exists(name)) { 28 | logger.info("Modifying the hosted repository ${name}") 29 | Configuration hostedRepoConfig = repository.repositoryManager.get(name).configuration 30 | def storage = hostedRepoConfig.attributes('storage') 31 | storage.set('strictContentTypeValidation', strictContentValidation) 32 | storage.set('writePolicy', writePolicy) 33 | repository.repositoryManager.update(hostedRepoConfig) 34 | } 35 | else { 36 | logger.info("Creating the hosted Rubygems repository ${name}") 37 | repository.createRubygemsHosted(name, blobStoreName, strictContentValidation, writePolicy) 38 | } 39 | } 40 | 41 | 42 | request = new JsonSlurper().parseText(args) 43 | ['rubygems_repository_name'].each { param -> 44 | assert request.get(param): "The ${param} parameter is required" 45 | } 46 | 47 | createHostedRubyGemsRepo(request.rubygems_repository_name) 48 | 49 | return 'The repositories were created successfully' 50 | -------------------------------------------------------------------------------- /cachito/workers/nexus_scripts/rubygems_cleanup.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | This script deletes the RubyGems hosted repository, user, and role for the Cachito request. 3 | 4 | This script should get executed when the Cachito request is set to the stale state. 5 | */ 6 | import groovy.json.JsonSlurper 7 | import org.slf4j.Logger 8 | import org.slf4j.LoggerFactory 9 | import org.sonatype.nexus.security.authz.AuthorizationManager 10 | import org.sonatype.nexus.security.role.NoSuchRoleException 11 | import org.sonatype.nexus.security.user.UserNotFoundException 12 | 13 | import static org.sonatype.nexus.security.user.UserManager.DEFAULT_SOURCE 14 | 15 | 16 | request = new JsonSlurper().parseText(args) 17 | ['rubygems_repository_name', 'username'].each { param -> 18 | assert request.get(param): "The ${param} parameter is required" 19 | } 20 | 21 | final Logger logger = LoggerFactory.getLogger('cachito'); 22 | try { 23 | logger.info("Deleting the user ${request.username}") 24 | // security is an object that is injected by Nexus when the script is executed 25 | security.securitySystem.deleteUser(request.username, DEFAULT_SOURCE) 26 | } catch(UserNotFoundException e) { 27 | logger.warn("The user ${request.username} was not found") 28 | } 29 | 30 | // security is an object that is injected by Nexus when the script is executed 31 | AuthorizationManager authorizationManager = security.securitySystem.getAuthorizationManager(DEFAULT_SOURCE) 32 | // The role is named the same as the username 33 | String roleName = request.username 34 | try { 35 | logger.info("Deleting the role ${roleName}") 36 | authorizationManager.deleteRole(roleName) 37 | } catch(NoSuchRoleException e) { 38 | logger.warn("The role ${roleName} was not found") 39 | } 40 | 41 | // repository is an object that is injected by Nexus when the script is executed 42 | logger.info("Deleting the repository ${request.rubygems_repository_name}") 43 | if (repository.repositoryManager.exists(request.rubygems_repository_name)) { 44 | repository.repositoryManager.delete(request.rubygems_repository_name) 45 | } else { 46 | logger.warn("The repository ${request.rubygems_repository_name} was not found") 47 | } 48 | 49 | return "The RubyGems repository ${request.rubygems_repository_name} and the user and role ${request.username} are removed" 50 | -------------------------------------------------------------------------------- /cachito/workers/paths.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | import logging 4 | import pathlib 5 | from typing import Any 6 | 7 | from cachito.common import paths 8 | from cachito.workers.config import get_worker_config 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | class RequestBundleDir(paths.RequestBundleDir): 14 | """ 15 | Represents a concrete request bundle directory used on the worker. 16 | 17 | The root directory is set to the ``cachito_bundles_dir`` configuration. 18 | 19 | By default, this request bundle directory and its dependency directory will 20 | be created when this object is instantiated. 21 | 22 | :param int request_id: the request ID. 23 | """ 24 | 25 | def __new__(cls, request_id): 26 | """Create a new Path object.""" 27 | root_dir = get_worker_config().cachito_bundles_dir 28 | self = super().__new__(cls, request_id, root_dir) 29 | 30 | log.debug("Ensure directory %s exists.", self) 31 | log.debug("Ensure directory %s exists.", self.deps_dir) 32 | self.deps_dir.mkdir(parents=True, exist_ok=True) 33 | 34 | return self 35 | 36 | 37 | # Similar with cachito.common.paths.RequestBundleDir, this base type will be the 38 | # correct type for Linux or Windows individually. 39 | base_path: Any = type(pathlib.Path()) 40 | 41 | 42 | class SourcesDir(base_path): 43 | """ 44 | Represents a sources directory tree for a package. 45 | 46 | The directory will be created automatically when this object is instantiated. 47 | 48 | :param str repo_name: a namespaced repository name of package. For example, 49 | ``release-engineering/retrodep``. 50 | :param str ref: the revision reference used to construct archive filename. 51 | """ 52 | 53 | def __new__(cls, repo_name, ref): 54 | """Create a new Path object.""" 55 | self = super().__new__(cls, get_worker_config().cachito_sources_dir) 56 | 57 | repo_relative_dir = pathlib.Path(*repo_name.split("/")) 58 | self.package_dir = self.joinpath(repo_relative_dir) 59 | self.archive_path = self.joinpath(repo_relative_dir, f"{ref}.tar.gz") 60 | 61 | log.debug("Ensure directory %s exists.", self.package_dir) 62 | self.package_dir.mkdir(parents=True, exist_ok=True) 63 | 64 | return self 65 | -------------------------------------------------------------------------------- /cachito/workers/pkg_managers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/cachito/workers/pkg_managers/__init__.py -------------------------------------------------------------------------------- /cachito/workers/requests.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import logging 3 | 4 | import requests 5 | import requests_kerberos 6 | from urllib3.util.retry import Retry 7 | 8 | from cachito.workers.config import get_worker_config 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | # The set is extended version of constant Retry.DEFAULT_ALLOWED_METHODS 13 | # with PATCH and POST methods included. 14 | ALL_REQUEST_METHODS = frozenset( 15 | {"GET", "POST", "PATCH", "PUT", "DELETE", "HEAD", "OPTIONS", "TRACE"} 16 | ) 17 | # The set includes only methods which don't modify state of the service. 18 | SAFE_REQUEST_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "TRACE"}) 19 | DEFAULT_RETRY_OPTIONS = { 20 | "total": 5, 21 | "read": 5, 22 | "connect": 5, 23 | "backoff_factor": 1.3, 24 | "status_forcelist": (500, 502, 503, 504), 25 | } 26 | 27 | 28 | def get_requests_session(auth=False, retry_options={}): 29 | """ 30 | Create a requests session with authentication (when enabled). 31 | 32 | :param bool auth: configure authentication on the session 33 | :param dict retry_options: overwrite options for initialization of Retry instance 34 | :return: the configured requests session 35 | :rtype: requests.Session 36 | """ 37 | config = get_worker_config() 38 | session = requests.Session() 39 | if auth: 40 | if config.cachito_auth_type == "kerberos": 41 | session.auth = requests_kerberos.HTTPKerberosAuth( 42 | mutual_authentication=requests_kerberos.OPTIONAL 43 | ) 44 | elif config.cachito_auth_type == "cert": 45 | session.cert = config.cachito_auth_cert 46 | 47 | retry_options = {**DEFAULT_RETRY_OPTIONS, **retry_options} 48 | adapter = requests.adapters.HTTPAdapter(max_retries=Retry(**retry_options)) 49 | session.mount("http://", adapter) 50 | session.mount("https://", adapter) 51 | return session 52 | 53 | 54 | # These sessions are only for connecting to the internal Cachito API 55 | requests_auth_session = get_requests_session( 56 | auth=True, retry_options={"allowed_methods": ALL_REQUEST_METHODS} 57 | ) 58 | requests_session = get_requests_session(retry_options={"allowed_methods": ALL_REQUEST_METHODS}) 59 | -------------------------------------------------------------------------------- /cachito/workers/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | from cachito.workers.tasks.general import * # noqa: F401, F403 3 | from cachito.workers.tasks.gitsubmodule import * # noqa: F401, F403 4 | from cachito.workers.tasks.gomod import * # noqa: F401, F403 5 | from cachito.workers.tasks.npm import * # noqa: F401, F403 6 | from cachito.workers.tasks.pip import * # noqa: F401, F403 7 | from cachito.workers.tasks.rubygems import * # noqa: F401, F403 8 | from cachito.workers.tasks.yarn import * # noqa: F401, F403 9 | -------------------------------------------------------------------------------- /cachito/workers/tasks/celery.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import sys 3 | 4 | import celery 5 | from celery.signals import celeryd_init, task_postrun, task_prerun, worker_process_init 6 | from opentelemetry import trace 7 | from opentelemetry.exporter.jaeger.thrift import JaegerExporter 8 | from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter 9 | from opentelemetry.instrumentation.celery import CeleryInstrumentor 10 | from opentelemetry.sdk.resources import SERVICE_NAME, Resource 11 | from opentelemetry.sdk.trace import TracerProvider 12 | from opentelemetry.sdk.trace.export import BatchSpanProcessor 13 | 14 | from cachito.workers.celery_logging import ( 15 | cleanup_task_logging, 16 | cleanup_task_logging_customization, 17 | setup_task_logging, 18 | setup_task_logging_customization, 19 | ) 20 | from cachito.workers.config import app, get_worker_config, validate_celery_config # noqa: F401 21 | 22 | 23 | def _init_celery_tracing(*args, **kwargs): # pragma: no cover 24 | """Initialize OTLP tracing, set the processor & endpoint.""" 25 | CeleryInstrumentor().instrument() 26 | config = get_worker_config() 27 | if config.cachito_jaeger_exporter_endpoint: 28 | jaeger_exporter = JaegerExporter( 29 | agent_host_name=config.cachito_jaeger_exporter_endpoint, 30 | agent_port=int(config.cachito_jaeger_exporter_port), 31 | ) 32 | processor = BatchSpanProcessor(jaeger_exporter) 33 | elif config.cachito_otlp_exporter_endpoint: 34 | otlp_exporter = OTLPSpanExporter(endpoint=config.cachito_otlp_exporter_endpoint) 35 | processor = BatchSpanProcessor(otlp_exporter) 36 | if config.cachito_otlp_exporter_endpoint or config.cachito_jaeger_exporter_endpoint: 37 | resource = Resource(attributes={SERVICE_NAME: "cachito-worker"}) 38 | provider = TracerProvider(resource=resource) 39 | # Useful for debugging trace issues... 40 | # processor = BatchSpanProcessor(ConsoleSpanExporter()) 41 | provider.add_span_processor(processor) 42 | trace.set_tracer_provider(provider) 43 | 44 | 45 | # Workaround https://github.com/celery/celery/issues/5416 46 | if celery.version_info < (4, 3) and sys.version_info >= (3, 7): # pragma: no cover 47 | from re import Pattern 48 | 49 | from celery.app.routes import re as routes_re 50 | 51 | routes_re._pattern_type = Pattern 52 | 53 | celeryd_init.connect(validate_celery_config) 54 | task_prerun.connect(setup_task_logging_customization) 55 | task_prerun.connect(setup_task_logging) 56 | task_postrun.connect(cleanup_task_logging_customization) 57 | task_postrun.connect(cleanup_task_logging) 58 | worker_process_init.connect(_init_celery_tracing) 59 | -------------------------------------------------------------------------------- /cachito/workers/tasks/gitsubmodule.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import logging 3 | 4 | import git 5 | 6 | from cachito.common.packages_data import PackagesData 7 | from cachito.workers.paths import RequestBundleDir 8 | from cachito.workers.tasks.celery import app 9 | from cachito.workers.tasks.utils import runs_if_request_in_progress 10 | 11 | __all__ = ["add_git_submodules_as_package"] 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | @app.task 16 | @runs_if_request_in_progress 17 | def add_git_submodules_as_package(request_id): 18 | """ 19 | Add git submodules as package to the Cachtio request. 20 | 21 | :param int request_id: the Cachito request ID this is for 22 | :raises InvalidRequestData: if adding submodules as a package fail. 23 | """ 24 | bundle_dir = RequestBundleDir(request_id) 25 | repo = git.Repo(str(bundle_dir.source_root_dir)) 26 | packages_json_data = PackagesData() 27 | for sm in repo.submodules: 28 | # Save package to db 29 | package = { 30 | "type": "git-submodule", 31 | "name": sm.name, 32 | "version": f"{sm.url}#{sm.hexsha}", 33 | } 34 | log.debug("Adding submodule '%s' as a package for Cachito request", sm.name) 35 | packages_json_data.add_package(package, sm.path, []) 36 | packages_json_data.write_to_file(bundle_dir.git_submodule_packages_data) 37 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | db: 3 | image: docker.io/postgres:9.6 4 | environment: 5 | POSTGRES_USER: cachito 6 | POSTGRES_PASSWORD: cachito 7 | POSTGRES_DB: cachito 8 | POSTGRES_INITDB_ARGS: "--auth='ident' --auth='trust'" 9 | 10 | rabbitmq: 11 | image: docker.io/rabbitmq:3.11-management 12 | volumes: 13 | - ./docker/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:z 14 | ports: 15 | # The RabbitMQ management console 16 | - 8081:15672 17 | 18 | athens: 19 | image: docker.io/gomods/athens:v0.12.1 20 | environment: 21 | ATHENS_DISK_STORAGE_ROOT: /var/lib/athens 22 | ATHENS_STORAGE_TYPE: disk 23 | ATHENS_GO_BINARY_ENV_VARS: GOSUMDB=off 24 | ATHENS_CLOUD_RUNTIME: JSON 25 | volumes: 26 | - athens-storage:/var/lib/athens:z 27 | ports: 28 | - 3000:3000 29 | 30 | nexus: 31 | image: docker.io/sonatype/nexus3:3.74.0 32 | environment: 33 | # Enable the script API. This is disabled by default in 3.21.2+. 34 | INSTALL4J_ADD_VM_PARAMS: > 35 | -Dnexus.scripts.allowCreation=true 36 | -Dstorage.diskCache.diskFreeSpaceLimit=512 37 | -Djava.util.prefs.userRoot=/nexus-data/javaprefs 38 | volumes: 39 | # Use a local volume for nexus data (podman compatibility) 40 | - ./tmp/nexus-data:/nexus-data:z 41 | ports: 42 | - 8082:8081 43 | # Nexus needs more open files, else it will throw errors. The Cachito integration tests expose the problem, 44 | # a java.io.exception of too many open files will be thrown. 45 | ulimits: 46 | nofile: 47 | soft: 65536 48 | hard: 65536 49 | 50 | cachito-api: 51 | build: 52 | context: . 53 | dockerfile: ./docker/Dockerfile-api 54 | command: 55 | - /bin/sh 56 | - -c 57 | - >- 58 | pip3 uninstall -y cachito && 59 | python3 setup.py develop --no-deps && 60 | cachito wait-for-db && 61 | cachito db upgrade -x delete_data=True && 62 | flask run --reload --host 0.0.0.0 --port 8080 63 | environment: 64 | FLASK_ENV: development 65 | FLASK_APP: cachito/web/wsgi.py 66 | CACHITO_DEV: 'true' 67 | volumes: 68 | - ./:/src:z 69 | - cachito-archives:/tmp/cachito-archives:z 70 | - cachito-request-logs:/var/log/cachito/requests:z 71 | # This is needed in order to allow cleaning up a temporary tarball 72 | # which is populated by the worker container. In an OpenShift environment 73 | # this is not needed. 74 | privileged: true 75 | depends_on: 76 | - db 77 | ports: 78 | - 8080:8080 79 | 80 | cachito-worker: 81 | build: 82 | context: . 83 | dockerfile: ./docker/Dockerfile-workers 84 | # Override the default command so that Celery auto-reloads on code changes 85 | # Instead of using cachito-update-nexus-scripts directly, we must perform this workaround 86 | # because the console scripts are not available in this container. This is because the 87 | # API container performs the `setup.py develop` and not this container and the source is 88 | # shared. 89 | command: 90 | - /bin/bash 91 | - -c 92 | - >- 93 | pip3 install watchdog[watchmedo] && 94 | /src/docker/configure-nexus.py && 95 | python3 -c 'from cachito.workers.nexus import create_or_update_scripts; create_or_update_scripts()' && 96 | watchmedo auto-restart -d ./cachito/workers -p '*.py' --recursive \ 97 | -- celery -A cachito.workers.tasks worker --loglevel=info 98 | environment: 99 | CACHITO_DEV: 'true' 100 | # This is needed in order to allow volume share with the cachito-api service. 101 | # In an OpenShift environment this is not needed. 102 | privileged: true 103 | volumes: 104 | - ./:/src:z 105 | - cachito-archives:/tmp/cachito-archives:z 106 | # This is mounted in this container so that this container can view the admin.password file 107 | # generated by Nexus and delete it once the configure-nexus.py script changes the admin 108 | # password 109 | - ./tmp/nexus-data:/nexus-data:z 110 | - cachito-request-logs:/var/log/cachito/requests:z 111 | depends_on: 112 | - cachito-api 113 | - nexus 114 | - rabbitmq 115 | 116 | jaeger: 117 | image: docker.io/jaegertracing/all-in-one:latest 118 | ports: 119 | # OTLP/gprc - used by pytest to forward integration test traces to jaeger. 120 | - "4317:4317" 121 | # Development environment is hard-coded to send to port 6831. 122 | - "6831:6831/udp" 123 | # Web UI is available at port 16686 124 | - "16686:16686" 125 | 126 | volumes: 127 | cachito-archives: 128 | athens-storage: 129 | cachito-request-logs: 130 | -------------------------------------------------------------------------------- /docker/Dockerfile-api: -------------------------------------------------------------------------------- 1 | FROM registry.fedoraproject.org/fedora:38 2 | LABEL maintainer="Red Hat" 3 | 4 | WORKDIR /src 5 | RUN dnf -y install \ 6 | --setopt=deltarpm=0 \ 7 | --setopt=install_weak_deps=false \ 8 | --setopt=tsflags=nodocs \ 9 | httpd \ 10 | gcc \ 11 | git-core \ 12 | libffi-devel \ 13 | libpq-devel \ 14 | mod_auth_gssapi \ 15 | mod_ssl \ 16 | mod_wsgi \ 17 | krb5-devel \ 18 | python3-pip \ 19 | python3-setuptools \ 20 | python-devel \ 21 | redhat-rpm-config \ 22 | && dnf clean all 23 | COPY . . 24 | COPY ./docker/cachito-httpd.conf /etc/httpd/conf/httpd.conf 25 | 26 | RUN pip3 install -r requirements.txt --no-deps --no-cache-dir --require-hashes \ 27 | && pip3 install -r requirements-web.txt --no-deps --no-cache-dir --require-hashes \ 28 | && pip3 install . --no-deps --no-cache-dir \ 29 | && rm -rf .git 30 | 31 | # Use the system CA bundle for the requests library 32 | ENV REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/pem/directory-hash/ca-bundle.crt 33 | # Use the system CA bundle for native SSL calls from celery (python) 34 | ENV SSL_CERT_FILE=/etc/pki/ca-trust/extracted/pem/directory-hash/ca-bundle.crt 35 | 36 | # Disable gitpython check for the git executable, cachito-api doesn't use git 37 | ENV GIT_PYTHON_REFRESH=quiet 38 | 39 | # Environment variable used by the Prometheus Flask exporter. 40 | ENV PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_multiproc_dir 41 | ENV DEBUG_METRICS=false 42 | 43 | EXPOSE 8080 44 | CMD ["/usr/sbin/httpd", "-DFOREGROUND"] 45 | -------------------------------------------------------------------------------- /docker/Dockerfile-workers: -------------------------------------------------------------------------------- 1 | FROM registry.fedoraproject.org/fedora:38 2 | LABEL maintainer="Red Hat" 3 | 4 | WORKDIR /src 5 | RUN dnf -y install \ 6 | --setopt=deltarpm=0 \ 7 | --setopt=install_weak_deps=false \ 8 | --setopt=tsflags=nodocs \ 9 | golang \ 10 | gcc \ 11 | git-core \ 12 | krb5-devel \ 13 | libffi-devel \ 14 | mercurial \ 15 | nodejs-npm-9.5.0 \ 16 | procps \ 17 | python3-devel \ 18 | python3-pip \ 19 | python3-setuptools \ 20 | strace \ 21 | && dnf clean all 22 | 23 | COPY . . 24 | 25 | # All the requirements except pyarn should already be installed 26 | RUN pip3 install -r requirements.txt --no-deps --no-cache-dir --require-hashes \ 27 | && pip3 install . --no-deps --no-cache-dir \ 28 | && rm -rf .git 29 | 30 | # Install an older version of Go fixed at 1.20 (along with the base >= 1.21): 31 | # - install Go's official shim 32 | # - let the shim download the actual Go SDK (the download forces the output parent dir to $HOME) 33 | # - move the SDK to a host local install system-wide location 34 | # - remove the shim as it forces and expects the SDK to be used from $HOME 35 | # - clean any build artifacts Go creates as part of the process. 36 | RUN for go_ver in "go1.20" "go1.21.0"; do \ 37 | go install "golang.org/dl/${go_ver}@latest" && \ 38 | "$HOME/go/bin/$go_ver" download && \ 39 | mkdir -p /usr/local/go && \ 40 | mv "$HOME/sdk/$go_ver" /usr/local/go && \ 41 | rm -rf "$HOME/go" "$HOME/.cache/go-build/"; \ 42 | done 43 | 44 | # Use the system CA bundle for the requests library 45 | ENV REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/pem/directory-hash/ca-bundle.crt 46 | # Use the system CA bundle for native SSL calls from celery (python) 47 | ENV SSL_CERT_FILE=/etc/pki/ca-trust/extracted/pem/directory-hash/ca-bundle.crt 48 | 49 | # Set git user configuration for GitPython 50 | ENV GIT_COMMITTER_NAME=cachito \ 51 | GIT_COMMITTER_EMAIL=cachito@localhost \ 52 | GIT_AUTHOR_NAME=cachito \ 53 | GIT_AUTHOR_EMAIL=cachito@localhost 54 | 55 | EXPOSE 8080 56 | CMD ["celery", "-A", "cachito.workers.tasks", "worker", "--loglevel=info"] 57 | -------------------------------------------------------------------------------- /docker/cachito-httpd.conf: -------------------------------------------------------------------------------- 1 | ServerRoot "/etc/httpd" 2 | PidFile /tmp/httpd.pid 3 | Listen 0.0.0.0:8080 http 4 | User apache 5 | Group apache 6 | DocumentRoot "/var/www/html" 7 | ErrorLog /dev/stderr 8 | TransferLog /dev/stdout 9 | LogLevel warn 10 | TypesConfig /etc/mime.types 11 | DefaultRuntimeDir /tmp 12 | Include conf.modules.d/*.conf 13 | # ServerName cachito.domain.local 14 | 15 | WSGISocketPrefix /tmp/wsgi 16 | WSGIDaemonProcess cachito threads=5 home=/tmp 17 | WSGIScriptAlias / /src/cachito/web/wsgi.py 18 | WSGICallableObject app 19 | 20 | 21 | AllowOverride None 22 | 23 | 24 | 25 | WSGIProcessGroup cachito 26 | WSGIApplicationGroup %{GLOBAL} 27 | 28 | Require all granted 29 | 30 | -------------------------------------------------------------------------------- /docker/configure-nexus.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python3 2 | import os 3 | import sys 4 | import time 5 | 6 | import requests 7 | import requests.auth 8 | 9 | base_url = 'http://nexus:8081' 10 | while True: 11 | print('Waiting for the Nexus server to be up...') 12 | try: 13 | rv = requests.get(base_url, timeout=5) 14 | except requests.ConnectionError: 15 | time.sleep(3) 16 | continue 17 | 18 | if rv.ok: 19 | print('The Nexus server is now up') 20 | break 21 | else: 22 | print(f'The request to the Nexus server failed with the status code: {rv.status_code}') 23 | 24 | 25 | admin_password_path = '/nexus-data/admin.password' 26 | if not os.path.exists(admin_password_path): 27 | print(f'{admin_password_path} is not present. Will skip the running of the script.') 28 | sys.exit(0) 29 | 30 | with open(admin_password_path, 'r') as admin_password_file: 31 | admin_password = admin_password_file.read() 32 | 33 | 34 | auth = requests.auth.HTTPBasicAuth('admin', admin_password) 35 | headers = {'Content-Type': 'application/json'} 36 | name = 'configure_nexus' 37 | print(f'Adding the {name} script...') 38 | with open('/src/docker/configure-nexus.groovy', 'r') as script: 39 | payload = {'name': name, 'type': 'groovy', 'content': script.read()} 40 | 41 | 42 | rv_script = requests.post( 43 | f'{base_url}/service/rest/v1/script', 44 | headers=headers, 45 | auth=auth, 46 | json=payload, 47 | timeout=15, 48 | ) 49 | if not rv_script.ok: 50 | print( 51 | f'The request to create the {name} script failed with the status ' 52 | f'code: {rv_script.status_code}' 53 | ) 54 | sys.exit(1) 55 | 56 | 57 | print(f'Running the {name} script...') 58 | rv_script_run = requests.post( 59 | f'{base_url}/service/rest/v1/script/{name}/run', 60 | timeout=15, 61 | headers=headers, 62 | auth=auth, 63 | json={ 64 | 'base_url': 'http://localhost:8082', 65 | 'cachito_password': 'cachito', 66 | 'cachito_unprivileged_password': 'cachito_unprivileged', 67 | 'new_admin_password': 'admin', 68 | }, 69 | ) 70 | 71 | if not rv_script_run.ok: 72 | print( 73 | f'Running the {name} script failed with the status code: {rv_script_run.status_code}\n' 74 | f'The output was: {rv_script_run.text}', 75 | file=sys.stderr, 76 | ) 77 | sys.exit(1) 78 | elif os.path.exists(admin_password_path): 79 | os.remove(admin_password_path) 80 | -------------------------------------------------------------------------------- /docker/rabbitmq.conf: -------------------------------------------------------------------------------- 1 | # Only intended to be used in the local dev environment (docker-compose) 2 | default_user = cachito 3 | default_pass = cachito 4 | -------------------------------------------------------------------------------- /docs/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Maintainers will complete the following section 2 | 3 | - [ ] Commit messages are descriptive enough 4 | - [ ] Code coverage from testing does not decrease and new code is covered 5 | - [ ] New code has type annotations 6 | - [ ] OpenAPI schema is updated (if applicable) 7 | - [ ] DB schema change has corresponding DB migration (if applicable) 8 | - [ ] README updated (if worker configuration changed, or if applicable) 9 | - [ ] Draft release notes are updated before merging 10 | -------------------------------------------------------------------------------- /docs/tracing.md: -------------------------------------------------------------------------------- 1 | # Tracing 2 | 3 | Cachito supports [OpenTelemetry tracing][1]. Internally, various python libraries used have automated tracing available, including [Requests][2], [Flask][3], [SQLAlchemy][4], and [Celery][5]. Spans are created automatically in each component, and the overall trace ID is passed appropriately. 4 | 5 | ### Development 6 | 7 | The docker-compose.yml file includes the configuration of a jaeger container to collect traces in the development environment. The local instance of Jaeger is available at [http://localhost:16686][6] 8 | 9 | ### Deployment configuration 10 | 11 | Cachito's tracing is configured via configuration variables in the /etc/cachito/settings.py (CACHITO_OTLP_EXPORTER_ENDPOINT) and /etc/cachito/celery.py (cachito_otlp_exporter_endpoint). 12 | This should be set to a valid URL that includes the URL and port of a listening OTLP-compatible service. 13 | If the configuration variable is not defined, trace information will be printed in the log. 14 | 15 | 16 | 17 | [1]: https://opentelemetry.io/docs/concepts/signals/traces/ 18 | [2]: https://pypi.org/project/opentelemetry-instrumentation-requests/ 19 | [3]: https://pypi.org/project/opentelemetry-instrumentation-flask/ 20 | [4]: https://pypi.org/project/opentelemetry-instrumentation-sqlalchemy/ 21 | [5]: https://pypi.org/project/opentelemetry-instrumentation-celery/ 22 | [6]: https://localhost:16686/ 23 | -------------------------------------------------------------------------------- /docs/using_requests_locally.md: -------------------------------------------------------------------------------- 1 | # Using Cachito requests locally 2 | 3 | The typical mode of interaction with the output of a Cachito request is through 4 | [OSBS][osbs-cachito]. Installing the content from a finished request without the help of 5 | OSBS may seem like a daunting task, but is in fact not that difficult. Doing so may be 6 | especially useful when trying to debug a build failure without going through the entire 7 | process again. 8 | 9 | ## Get the relevant files 10 | 11 | A Cachito request has three main parts: the **archive**, the **environment variables** 12 | and the **configuration files**. Getting these manually is a bit of a pain, which is 13 | why we use the [cachito-download.sh](../bin/cachito-download.sh) script to do it. You 14 | will need `jq` and the typical unix utils to run this script. 15 | 16 | ```shell 17 | cachito-download.sh https://cachito.example.org/api/v1/requests/1 /tmp/cachito-1 18 | ``` 19 | 20 | ## Use the right environment 21 | 22 | If you are debugging a build, make sure to match the target environment as closely as 23 | possible. For example, if you are building a container based on `python:3.9`, then you 24 | may want to try out the build in that base image. 25 | 26 | ```shell 27 | cd /tmp/cachito-1/remote-source 28 | # mount the remote-source/ directory, later you will run the build from there 29 | podman run --rm -ti -v "$PWD:$PWD:z" python:3.9 bash 30 | ``` 31 | 32 | If using a container is not an option/not relevant, you should still consider using some 33 | kind of virtual environment (such as the Python venv) and at least making sure that the 34 | versions of your build tools match what you will be using in the real build. 35 | 36 | ## Run the build 37 | 38 | In the container/virtual environment/your own system (not recommended), set the 39 | environment variables provided by Cachito and run the build. 40 | 41 | ```shell 42 | cd /tmp/cachito-1/remote-source 43 | # source the env vars from the generated file 44 | source cachito.env 45 | # cd to the app/ directory, some package managers will only work properly from there 46 | cd app 47 | # run your package manager commands 48 | pip install -r requirements.txt 49 | npm install 50 | go build 51 | # etc. 52 | ``` 53 | 54 | --- 55 | 56 | # Debugging failures 57 | 58 | If you *have* resorted to trying out a build locally, it is probably because that build 59 | is failing. The possible reasons are endless, but here you will find some of the most 60 | useful concepts for debugging. 61 | 62 | ## Pip 63 | 64 | Python's packaging system has its oddities, even more so when used through Cachito. The 65 | [packaging glossary][packaging-glossary] may be useful to you throughout this section. 66 | 67 | ### Per project index 68 | 69 | Your build uses a per project index, specified by the `PIP_INDEX_URL` environment 70 | variable. This index contains only the source distributions (sdists) for your packages, 71 | not wheels. That means all packages need to be built from source. 72 | 73 | ### Building from source 74 | 75 | When building a package from source, the most important files are typically 76 | **pyproject.toml** and, if the build backend is setuptools, **setup.cfg** or 77 | **setup.py**. If you want to know more, check out [PEP 517][pep-517] and 78 | [PEP 518][pep-518]. 79 | 80 | > :warning: PEPs 518 and 517 were implemented in pip versions 10.0 and 19.0, 81 | > respectively. Additionally, only pip>=18.0 supports sdists for PEP 518. Older 82 | > versions of pip will fail to install packages that rely on these PEPs. 83 | 84 | If you just want to try installing packages from sdists, you do not need to go through 85 | Cachito. You can simply pass the `--no-binary :all:` option to pip. 86 | 87 | ```shell 88 | pip install --no-binary :all: -r requirements.txt 89 | ``` 90 | 91 | ### Building extension modules 92 | 93 | For pure Python modules, building from source is usually not a problem if you have all 94 | the build dependencies. For extension modules written in C or other languages, this is 95 | a bit trickier. 96 | 97 | The build will still work if you have the build dependencies, but you cannot fetch them 98 | through Cachito, not to mention that it is nearly impossible to determine what they are. 99 | As always, `pip install --no-binary :all:` is your friend. Just keep trying until you 100 | make it work. 101 | 102 | [osbs-cachito]: https://osbs.readthedocs.io/en/latest/users.html#fetching-source-code-from-external-source-using-cachito 103 | [packaging-glossary]: https://packaging.python.org/glossary/ 104 | [pep-517]: https://www.python.org/dev/peps/pep-0517/ 105 | [pep-518]: https://www.python.org/dev/peps/pep-0518/ 106 | -------------------------------------------------------------------------------- /hack/mock-unittest-data/gomod.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -o errexit -o nounset -o pipefail 3 | 4 | cat << banner-end 5 | -------------------------------------------------------------------------------- 6 | Generating mock data for gomod unit tests 7 | -------------------------------------------------------------------------------- 8 | banner-end 9 | 10 | mocked_data_dir=${1:-tests/test_workers/test_pkg_managers/data/gomod-mocks} 11 | mkdir -p "$mocked_data_dir/non-vendored" 12 | mkdir -p "$mocked_data_dir/vendored" 13 | mocked_data_dir_abspath=$(realpath "$mocked_data_dir") 14 | 15 | tmpdir=$(dirname "$(mktemp --dry-run)") 16 | 17 | git clone https://github.com/cachito-testing/gomod-pandemonium \ 18 | "$tmpdir/gomod-pandemonium" 19 | trap 'rm -rf "$tmpdir/gomod-pandemonium"' EXIT 20 | 21 | cat << banner-end 22 | -------------------------------------------------------------------------------- 23 | $( 24 | # cd in a subshell, doesn't change the $PWD of the main process 25 | cd "$tmpdir/gomod-pandemonium" 26 | export GOMODCACHE="$tmpdir/cachito-mock-gomodcache" 27 | 28 | echo "generating $mocked_data_dir/non-vendored/go_mod_download.json" 29 | go mod download -json > \ 30 | "$mocked_data_dir_abspath/non-vendored/go_mod_download.json" 31 | 32 | echo "generating $mocked_data_dir/non-vendored/go_list_deps_all.json" 33 | go list -deps -json=ImportPath,Module,Standard,Deps all > \ 34 | "$mocked_data_dir_abspath/non-vendored/go_list_deps_all.json" 35 | 36 | echo "generating $mocked_data_dir/non-vendored/go_list_deps_threedot.json" 37 | go list -deps -json=ImportPath,Module,Standard,Deps ./... > \ 38 | "$mocked_data_dir_abspath/non-vendored/go_list_deps_threedot.json" 39 | 40 | echo "generating $mocked_data_dir/vendored/modules.txt" 41 | go mod vendor 42 | cp vendor/modules.txt "$mocked_data_dir_abspath/vendored/modules.txt" 43 | 44 | echo "generating $mocked_data_dir/vendored/go_list_deps_all.json" 45 | go list -deps -json=ImportPath,Module,Standard,Deps all > \ 46 | "$mocked_data_dir_abspath/vendored/go_list_deps_all.json" 47 | 48 | echo "generating $mocked_data_dir/vendored/go_list_deps_threedot.json" 49 | go list -deps -json=ImportPath,Module,Standard,Deps ./... > \ 50 | "$mocked_data_dir_abspath/vendored/go_list_deps_threedot.json" 51 | ) 52 | -------------------------------------------------------------------------------- 53 | banner-end 54 | 55 | find "$mocked_data_dir/non-vendored" "$mocked_data_dir/vendored" -type f | 56 | while read -r f; do 57 | sed "s|$tmpdir.cachito-mock-gomodcache|{gomodcache_dir}|" -i "$f" 58 | sed "s|$tmpdir.gomod-pandemonium|{repo_dir}|" -i "$f" 59 | done 60 | 61 | nonvendor_changed=$(git diff -- "$mocked_data_dir/non-vendored") 62 | vendor_changed=$(git diff -- "$mocked_data_dir/vendored") 63 | 64 | if [[ -n "$vendor_changed" || -n "$nonvendor_changed" ]]; then 65 | cat << banner-end 66 | The mock data changed => the expected unit test results may change. 67 | The following files may need to be adjusted manually: 68 | $( 69 | if [[ -n "$nonvendor_changed" ]]; then 70 | echo " $mocked_data_dir/expected-results/resolve_gomod.json" 71 | fi 72 | if [[ -n "$vendor_changed" ]]; then 73 | echo " $mocked_data_dir/expected-results/resolve_gomod_vendored.json" 74 | fi 75 | ) 76 | -------------------------------------------------------------------------------- 77 | banner-end 78 | fi 79 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 100 3 | target-version = ['py311'] 4 | 5 | [tool.isort] 6 | profile = "black" 7 | line_length = 100 8 | extend_skip_glob = ["cachito/web/migrations/*"] 9 | -------------------------------------------------------------------------------- /requirements-test.in: -------------------------------------------------------------------------------- 1 | jsonschema 2 | pluggy>=1 3 | pytest 4 | pytest-asyncio 5 | pytest-cov 6 | pyyaml 7 | opentelemetry-sdk 8 | opentelemetry-instrumentation-celery 9 | opentelemetry-instrumentation-requests 10 | opentelemetry-instrumentation-sqlalchemy 11 | opentelemetry-exporter-jaeger 12 | opentelemetry-exporter-otlp-proto-http 13 | -------------------------------------------------------------------------------- /requirements-web.in: -------------------------------------------------------------------------------- 1 | connexion<3 2 | Flask 3 | flask-login 4 | Flask-Migrate 5 | Flask-SQLAlchemy 6 | greenlet>=1.1.0 7 | psycopg2-binary 8 | SQLAlchemy<2.1 9 | prometheus-flask-exporter 10 | opentelemetry-instrumentation-flask 11 | opentelemetry-instrumentation-requests 12 | opentelemetry-instrumentation-psycopg2 13 | opentelemetry-instrumentation-sqlalchemy 14 | opentelemetry-exporter-jaeger 15 | opentelemetry-exporter-otlp-proto-http 16 | opentelemetry-sdk 17 | opentelemetry-api 18 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | # FIXME: there was a regression in py-amqp. It was supposed to be fixed in 2 | # https://github.com/celery/py-amqp/pull/350 3 | # but cachito is still being affected by the regression 4 | aiohttp 5 | aiohttp-retry 6 | amqp 7 | backoff 8 | celery>=5.2.2 9 | defusedxml 10 | gitpython 11 | gemlock_parser @ https://github.com/containerbuildsystem/gemlock-parser/archive/a2f0f4020e07b1e87813a3254eb3e40047d8e981.zip 12 | kombu>=5 # A celery dependency but it's directly imported 13 | packaging 14 | pyarn 15 | pydantic<2 16 | pyyaml 17 | ratelimit 18 | requests_kerberos>=0.13.0 19 | requests 20 | semver 21 | setuptools 22 | opentelemetry-sdk 23 | opentelemetry-instrumentation-celery 24 | opentelemetry-instrumentation-requests 25 | opentelemetry-instrumentation-sqlalchemy 26 | opentelemetry-exporter-jaeger 27 | opentelemetry-exporter-otlp-proto-http 28 | typer-slim 29 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | from setuptools import find_packages, setup 3 | 4 | GEMLOCK_PARSER_REPO_URL = "https://github.com/containerbuildsystem/gemlock-parser.git" 5 | GEMLOCK_PARSER_PIP_REF = f"git+{GEMLOCK_PARSER_REPO_URL}@master#egg=gemlock_parser" 6 | 7 | setup( 8 | name="cachito", 9 | long_description=__doc__, 10 | packages=find_packages(), 11 | include_package_data=True, 12 | zip_safe=False, 13 | install_requires=[ 14 | "backoff", 15 | "celery>=5", 16 | f"gemlock_parser @ {GEMLOCK_PARSER_PIP_REF}", 17 | "gitpython", 18 | "kombu>=5", # A celery dependency but it's directly imported 19 | "packaging", 20 | "pyarn", 21 | "pydantic", 22 | "requests_kerberos", 23 | "requests", 24 | "semver", 25 | "setuptools", 26 | "opentelemetry-sdk", 27 | "opentelemetry-instrumentation-celery", 28 | "opentelemetry-instrumentation-requests", 29 | "opentelemetry-exporter-jaeger", 30 | "opentelemetry-exporter-otlp-proto-http", 31 | ], 32 | extras_require={ 33 | "web": [ 34 | "Flask", 35 | "flask-login", 36 | "Flask-Migrate", 37 | "Flask-SQLAlchemy", 38 | "psycopg2-binary", 39 | "prometheus-flask-exporter", 40 | "opentelemetry-instrumentation-sqlalchemy", 41 | ], 42 | }, 43 | entry_points={ 44 | "console_scripts": [ 45 | "cachito=cachito.web.manage:cli", 46 | "cachito-cleanup=cachito.workers.cleanup_job:main", 47 | "cachito-prune-archives=cachito.workers.prune_archives:app", 48 | "cachito-update-nexus-scripts=cachito.workers.nexus:create_or_update_scripts", 49 | ] 50 | }, 51 | classifiers=[ 52 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 53 | "Programming Language :: Python :: 3 :: Only", 54 | "Programming Language :: Python :: 3.11", 55 | ], 56 | license="GPLv3+", 57 | python_requires=">=3.11", 58 | use_scm_version={ 59 | "version_scheme": "post-release", 60 | }, 61 | setup_requires=["setuptools_scm"], 62 | scripts=["bin/pip_find_builddeps.py"], 63 | ) 64 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | -------------------------------------------------------------------------------- /tests/helper_utils/__init__.py: -------------------------------------------------------------------------------- 1 | import filecmp 2 | import os 3 | from pathlib import Path 4 | from typing import Union 5 | 6 | 7 | def assert_directories_equal(dir_a, dir_b, ignore_files=[]): 8 | """ 9 | Check recursively directories have equal content. 10 | 11 | :param dir_a: first directory to check 12 | :param dir_b: second directory to check 13 | """ 14 | ignore_files = list(set(filecmp.DEFAULT_IGNORES).union(ignore_files)) 15 | dirs_cmp = filecmp.dircmp(dir_a, dir_b, ignore=ignore_files) 16 | 17 | assert ( 18 | len(dirs_cmp.left_only) == 0 19 | ), f"Found files: {dirs_cmp.left_only} in {dir_a}, but not {dir_b}." 20 | assert ( 21 | len(dirs_cmp.right_only) == 0 22 | ), f"Found files: {dirs_cmp.right_only} in {dir_b}, but not {dir_a}." 23 | assert ( 24 | len(dirs_cmp.funny_files) == 0 25 | ), f"Found files: {dirs_cmp.funny_files} in {dir_a}, {dir_b} which could not be compared." 26 | (_, mismatch, errors) = filecmp.cmpfiles(dir_a, dir_b, dirs_cmp.common_files, shallow=False) 27 | assert len(mismatch) == 0, f"Found mismatches: {mismatch} between {dir_a} {dir_b}." 28 | assert len(errors) == 0, f"Found errors: {errors} between {dir_a} {dir_b}." 29 | 30 | for common_dir in dirs_cmp.common_dirs: 31 | inner_a = os.path.join(dir_a, common_dir) 32 | inner_b = os.path.join(dir_b, common_dir) 33 | assert_directories_equal(inner_a, inner_b, ignore_files) 34 | 35 | 36 | class Symlink(str): 37 | """ 38 | Use this to create symlinks via write_file_tree(). 39 | 40 | The value of a Symlink instance is the target path (path to make a symlink to). 41 | """ 42 | 43 | 44 | def write_file_tree(tree_def: dict, rooted_at: Union[str, Path], exist_ok: bool = False): 45 | """ 46 | Write a file tree to disk. 47 | 48 | :param tree_def: Definition of file tree, see usage for intuitive examples 49 | :param rooted_at: Root of file tree, must be an existing directory 50 | :param exist_ok: If True, existing directories will not cause this function to fail 51 | """ 52 | root = Path(rooted_at) 53 | for entry, value in tree_def.items(): 54 | entry_path = root / entry 55 | if isinstance(value, Symlink): 56 | os.symlink(value, entry_path) 57 | elif isinstance(value, str): 58 | entry_path.write_text(value) 59 | else: 60 | entry_path.mkdir(exist_ok=exist_ok) 61 | write_file_tree(value, entry_path) 62 | -------------------------------------------------------------------------------- /tests/integration/README.md: -------------------------------------------------------------------------------- 1 | # Integration tests for Cachito 2 | 3 | This directory stores the integration tests for Cachito. 4 | 5 | ## Configuration 6 | 7 | Input data for tests should be configured in the file `test_env_vars.yaml`. The file should be 8 | placed in top-level directory of this repository. The path can be changed by setting 9 | `CACHITO_TEST_CONFIG` to a different path. 10 | 11 | See `test_env_vars.yaml` for a complete list of configuration options and examples at the top-level 12 | directory of this repo. 13 | 14 | ## Running the tests 15 | 16 | Tests can be triggered from the top-level directory of this repository with: 17 | 18 | ```bash 19 | make test-integration 20 | ``` 21 | 22 | The integration environment is not part of the default `tox` envlist. 23 | 24 | `REQUESTS_CA_BUNDLE` can be passed in `tox.ini` for the `integration` 25 | environment in order to enable running the tests against Cachito instances which 26 | have certificates issued by a custom root certificate authority. Example usage: 27 | 28 | REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt tox -e integration 29 | 30 | `KRB5CCNAME` can be passed in `tox.ini` for the `integration` 31 | environment when using Kerberos authentication in requests. 32 | 33 | To use certificate authentication, set `api_auth_type` to `cert` in the integration tests 34 | yaml configuration file. You must also set the environment variables `CACHITO_TEST_CERT` 35 | and `CACHITO_TEST_KEY` to reference the certificate and key files respectively. 36 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | -------------------------------------------------------------------------------- /tests/integration/conftest.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | import os 4 | from collections import namedtuple 5 | 6 | import pytest 7 | import yaml 8 | 9 | from . import utils 10 | 11 | DefaultRequest = namedtuple("DefaultRequest", "initial_response complete_response") 12 | 13 | 14 | @pytest.fixture(scope="session") 15 | def test_env(): 16 | """ 17 | Load the test environment configuration. 18 | 19 | :return: Test environment configuration. 20 | :rtype: dict 21 | """ 22 | config_file = os.getenv("CACHITO_TEST_CONFIG", "test_env_vars.yaml") 23 | with open(config_file) as f: 24 | env = yaml.safe_load(f) 25 | return env 26 | 27 | 28 | @pytest.fixture(scope="session") 29 | def api_client(test_env): 30 | return utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 31 | 32 | 33 | @pytest.fixture(scope="session") 34 | def default_requests(test_env): 35 | """ 36 | Create a new request for every package manager in Cachito. 37 | 38 | :param test_env: Test environment configuration 39 | :return: a dict of packages with initial and completed responses from the Cachito API 40 | :rtype: dict 41 | """ 42 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 43 | result_requests = {} 44 | packages = test_env["packages"] 45 | for package_name in packages: 46 | initial_response = client.create_new_request( 47 | payload={ 48 | "repo": packages[package_name]["repo"], 49 | "ref": packages[package_name]["ref"], 50 | "pkg_managers": packages[package_name]["pkg_managers"], 51 | }, 52 | ) 53 | completed_response = client.wait_for_complete_request(initial_response) 54 | result_requests[package_name] = DefaultRequest(initial_response, completed_response) 55 | 56 | return result_requests 57 | -------------------------------------------------------------------------------- /tests/integration/test_content_manifest.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | from typing import Any, Dict 4 | 5 | import pytest 6 | import requests 7 | 8 | from . import utils 9 | from .conftest import DefaultRequest 10 | 11 | 12 | def test_invalid_content_manifest_request(test_env): 13 | """ 14 | Send an invalid content-manifest request to the Cachito API. 15 | 16 | Checks: 17 | * Check that the response code is 404 18 | """ 19 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 20 | 21 | with pytest.raises(requests.HTTPError) as e: 22 | client.fetch_content_manifest(request_id=0) 23 | assert e.value.response.status_code == 404 24 | assert e.value.response.json() == {"error": "The requested resource was not found"} 25 | 26 | 27 | def test_valid_content_manifest_request(test_env, default_requests): 28 | """ 29 | Send a valid content-manifest request to the Cachito API. 30 | 31 | Checks: 32 | * Check that the response code is 200 33 | * Check validation of the response data with content manifest JSON schema 34 | """ 35 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 36 | 37 | pkg_managers = test_env["content_manifest"]["pkg_managers"] 38 | for pkg_manager in pkg_managers: 39 | initial_response = default_requests[pkg_manager].initial_response 40 | content_manifest_response = client.fetch_content_manifest(initial_response.id) 41 | assert content_manifest_response.status == 200 42 | 43 | response_data = content_manifest_response.data 44 | utils.assert_content_manifest_schema(response_data) 45 | 46 | 47 | def test_invalid_sbom_request(test_env: Dict[str, Any]) -> None: 48 | """ 49 | Send an invalid sbom request to the Cachito API. 50 | 51 | Checks: 52 | * Check that the response code is 400 53 | """ 54 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 55 | 56 | with pytest.raises(requests.HTTPError) as e: 57 | client.fetch_sbom(request_ids=0) 58 | assert e.value.response.status_code == 400 59 | assert e.value.response.json() == {"error": "Cannot find request(s) 0."} 60 | 61 | 62 | def test_valid_sbom_request( 63 | test_env: Dict[str, Any], default_requests: Dict[str, DefaultRequest] 64 | ) -> None: 65 | """ 66 | Send a valid sbom request to the Cachito API. 67 | 68 | Checks: 69 | * Check that the response code is 200 70 | * Check validation of the response data with content manifest JSON schema 71 | """ 72 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 73 | 74 | pkg_managers = test_env["content_manifest"]["pkg_managers"] 75 | 76 | request_ids = [] 77 | for pkg_manager in pkg_managers: 78 | initial_response = default_requests[pkg_manager].initial_response 79 | request_ids.append(initial_response.id) 80 | 81 | sbom_response = client.fetch_sbom(",".join(map(str, request_ids))) 82 | assert sbom_response.status == 200 83 | 84 | response_data = sbom_response.data 85 | utils.assert_sbom_schema(response_data) 86 | -------------------------------------------------------------------------------- /tests/integration/test_creating_new_request.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | from . import utils 4 | 5 | 6 | def test_creating_new_request(test_env, default_requests): 7 | """ 8 | Send a new request to the Cachito API. 9 | 10 | Checks: 11 | * Check that response code is 201 12 | * Check that response contains id number, same ref and repo as in request, 13 | state_reason is: The request was initiated 14 | """ 15 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 16 | response_created_req = default_requests["gomod"].initial_response 17 | assert response_created_req.status == 201 18 | 19 | assert "id" in response_created_req.data 20 | assert response_created_req.id > 0 21 | 22 | response_specific_req = client.fetch_request(response_created_req.id) 23 | assert response_created_req.id == response_specific_req.id 24 | 25 | response_pkg_managers = set(response_created_req.data["pkg_managers"]) 26 | assert set(test_env["packages"]["gomod"]["pkg_managers"]) == response_pkg_managers 27 | assert test_env["packages"]["gomod"]["ref"] == response_created_req.data["ref"] 28 | assert test_env["packages"]["gomod"]["repo"] == response_created_req.data["repo"] 29 | assert test_env["packages"]["gomod"]["ref"] == response_specific_req.data["ref"] 30 | assert test_env["packages"]["gomod"]["repo"] == response_specific_req.data["repo"] 31 | 32 | assert response_created_req.data["state_reason"] == "The request was initiated" 33 | -------------------------------------------------------------------------------- /tests/integration/test_data/git_submodule_packages.yaml: -------------------------------------------------------------------------------- 1 | # Test data for git submodules 2 | # repo: The URL for the upstream git repository 3 | # ref: A git reference at the given git repository 4 | # expected_files: Expected source files : 5 | # expected_deps_files: Expected dependencies files (empty) 6 | # response_expectations: Parts of the Cachito response to check 7 | # content_manifest: PURLs for image contents part 8 | # With git-submodule 9 | git_submodule_no_master_branch: 10 | repo: https://github.com/cachito-testing/git-submodule-no-master 11 | ref: 3351cc868284974bf8f232551d045f4b6ec926a0 12 | pkg_managers: ["git-submodule"] 13 | expected_files: 14 | app: https://github.com/cachito-testing/git-submodule-no-master-tarball/tarball/4d608aa801bc30753499c9910c72aa98abe9194f 15 | deps: null 16 | response_expectations: 17 | dependencies: [] 18 | packages: 19 | - dependencies: [] 20 | name: repo-no-master-branch 21 | path: repo-no-master-branch 22 | type: git-submodule 23 | version: https://github.com/cachito-testing/repo-no-master-branch.git#309749e2ef8755319cb4f7dd5faa8f2fbdacda70 24 | content_manifest: 25 | - purl: "pkg:github/cachito-testing/repo-no-master-branch@309749e2ef8755319cb4f7dd5faa8f2fbdacda70" 26 | sbom: 27 | - name: repo-no-master-branch 28 | type: library 29 | version: https://github.com/cachito-testing/repo-no-master-branch.git#309749e2ef8755319cb4f7dd5faa8f2fbdacda70 30 | purl: pkg:github/cachito-testing/repo-no-master-branch@309749e2ef8755319cb4f7dd5faa8f2fbdacda70 31 | -------------------------------------------------------------------------------- /tests/integration/test_dependency_replacement.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | from os import path 4 | 5 | from . import utils 6 | 7 | 8 | def test_dependency_replacement(test_env, tmpdir): 9 | """ 10 | Check that proper versions of dependencies were used. 11 | 12 | Process: 13 | * Send new request to Cachito API to fetch retrodep with another version of dependency package 14 | * Download a bundle archive 15 | 16 | Checks: 17 | * Check that the state of request is complete 18 | * Check that in the response there is a key "replaces" with dict values which was replaced 19 | * Check that dir deps/gomod/pkg/mod/cache/download/github.com/pkg/errors/@v/… contains 20 | only the required version 21 | * Check that app/go.mod file has replace directive for the specified package 22 | """ 23 | dependency_replacements = test_env["dep_replacement"]["dependency_replacements"] 24 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 25 | response_created_req = client.create_new_request( 26 | payload={ 27 | "repo": test_env["packages"]["gomod"]["repo"], 28 | "ref": test_env["packages"]["gomod"]["ref"], 29 | "pkg_managers": test_env["packages"]["gomod"]["pkg_managers"], 30 | "dependency_replacements": dependency_replacements, 31 | }, 32 | ) 33 | response = client.wait_for_complete_request(response_created_req) 34 | utils.assert_properly_completed_response(response) 35 | 36 | names_replaced_dependencies = { 37 | i["replaces"]["name"] for i in response.data["dependencies"] if i["replaces"] is not None 38 | } 39 | supposed_replaced_dependencies = set(i["name"] for i in dependency_replacements) 40 | assert names_replaced_dependencies == supposed_replaced_dependencies 41 | 42 | bundle_dir_name = tmpdir.join(f"download_{str(response.id)}") 43 | client.download_and_extract_archive(response.id, tmpdir) 44 | 45 | for dependency in dependency_replacements: 46 | dep_name = utils.escape_path_go(dependency["name"]) 47 | dependency_version_file = path.join( 48 | bundle_dir_name, 49 | "deps", 50 | "gomod", 51 | "pkg", 52 | "mod", 53 | "cache", 54 | "download", 55 | dep_name, 56 | "@v", 57 | "list", 58 | ) 59 | assert path.exists(dependency_version_file), ( 60 | f"#{response.id}: Path for version of dependency " 61 | f"{dep_name} does not exist: {dependency_version_file}" 62 | ) 63 | with open(dependency_version_file, "r") as file: 64 | lines = {line.rstrip() for line in file.readlines()} 65 | assert dependency["version"] in lines, ( 66 | f"#{response.id}: File {dependency_version_file} does not contain" 67 | f" version {dependency['version']} that should have replaced the original one." 68 | ) 69 | 70 | go_mod_path = path.join(bundle_dir_name, "app", "go.mod") 71 | assert path.exists( 72 | go_mod_path 73 | ), f"#{response.id}: File go.mod does not exist in location: {go_mod_path}" 74 | with open(go_mod_path, "r") as file: 75 | go_mod_replace = [] 76 | for line in file: 77 | if line.startswith("replace "): 78 | go_mod_replace.append( 79 | {"name": line.split()[-2], "type": "gomod", "version": line.split()[-1]} 80 | ) 81 | sorted_dep_replacements = utils.make_list_of_packages_hashable(dependency_replacements) 82 | sorted_go_mod_replace = utils.make_list_of_packages_hashable(go_mod_replace) 83 | assert sorted_go_mod_replace == sorted_dep_replacements 84 | -------------------------------------------------------------------------------- /tests/integration/test_get_latest_request.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | from typing import Any 4 | 5 | from cachito.common.utils import get_repo_name 6 | 7 | from . import utils 8 | 9 | 10 | def test_get_latest_request(api_client: utils.Client, test_env: dict[str, Any]) -> None: 11 | """ 12 | Generates requests and ensures that the requests/latest endpoint returns the most recent one. 13 | 14 | For each package in "various_packages", a request and a duplicate request are generated. 15 | After all requests have been completed for all packages, the requests/latest endpoint is 16 | queried for each repo_name/ref combination. The request_id of the final duplicate request 17 | for each package should match what is returned by the latest endpoint. 18 | """ 19 | latest_created_request_ids = {} 20 | repeat_count = 2 21 | 22 | # Generate the requests 23 | for pkg_manager, package in test_env["various_packages"].items(): 24 | repo_name = get_repo_name(package["repo"]) 25 | for _ in range(repeat_count): 26 | initial_response = api_client.create_new_request( 27 | payload={ 28 | "repo": package["repo"], 29 | "ref": package["ref"], 30 | "pkg_managers": [pkg_manager], 31 | }, 32 | ) 33 | completed_response = api_client.wait_for_complete_request(initial_response) 34 | utils.assert_properly_completed_response(completed_response) 35 | latest_created_request_ids[(repo_name, package["ref"])] = completed_response.data["id"] 36 | 37 | # Check that the latest is the latest 38 | for package in test_env["various_packages"].values(): 39 | repo_name = get_repo_name(package["repo"]) 40 | latest_request = api_client.fetch_latest_request( 41 | repo_name=repo_name, ref=package["ref"] 42 | ).json() 43 | 44 | assert { 45 | "id", 46 | "repo", 47 | "ref", 48 | "updated", 49 | }.issubset(latest_request), "Required fields missing from returned Request" 50 | assert "configuration_files" not in latest_request, "A verbose Request was returned" 51 | assert ( 52 | latest_created_request_ids[(repo_name, package["ref"])] == latest_request["id"] 53 | ), f"id={latest_request['id']} is not the latest request for {(repo_name, package['ref'])}" 54 | -------------------------------------------------------------------------------- /tests/integration/test_pip_packages.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | from . import utils 4 | 5 | 6 | def test_failing_pip_local_path(test_env): 7 | """ 8 | Validate failing of the pip package request with local dependencies. 9 | 10 | Process: 11 | Send new request to the Cachito API 12 | Send request to check status of existing request 13 | 14 | Checks: 15 | * Check that the request fails with expected error 16 | """ 17 | env_data = utils.load_test_data("pip_packages.yaml")["local_path"] 18 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 19 | initial_response = client.create_new_request( 20 | payload={"repo": env_data["repo"], "ref": env_data["ref"], "pkg_managers": ["pip"]} 21 | ) 22 | completed_response = client.wait_for_complete_request(initial_response) 23 | assert completed_response.status == 200 24 | assert completed_response.data["state"] == "failed" 25 | error_msg = "Direct references with 'file' scheme are not supported" 26 | assert error_msg in completed_response.data["state_reason"], ( 27 | f"#{completed_response.id}: Request failed correctly, but with unexpected message: " 28 | f"{completed_response.data['state_reason']}. Expected message was: {error_msg}" 29 | ) 30 | 31 | 32 | def test_failing_pip_invalid_req_path(test_env): 33 | """ 34 | Validate failing of the pip package request with a non-existent requirement file path. 35 | 36 | Process: 37 | Send new request to the Cachito API 38 | Send request to check status of existing request 39 | 40 | Checks: 41 | * Check that the request fails with expected error 42 | """ 43 | env_data = utils.load_test_data("pip_packages.yaml")["local_path"] 44 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 45 | invalid_path = "foo.txt" 46 | initial_response = client.create_new_request( 47 | payload={ 48 | "repo": env_data["repo"], 49 | "ref": env_data["ref"], 50 | "pkg_managers": ["pip"], 51 | "packages": {"pip": [{"requirements_files": [invalid_path]}]}, 52 | } 53 | ) 54 | completed_response = client.wait_for_complete_request(initial_response) 55 | assert completed_response.status == 200 56 | assert completed_response.data["state"] == "failed" 57 | error_msg = "Following requirement file has an invalid path: " 58 | assert all(msg in completed_response.data["state_reason"] for msg in [error_msg, invalid_path]) 59 | -------------------------------------------------------------------------------- /tests/integration/test_private_repos.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | import os 4 | import random 5 | import string 6 | from pathlib import Path 7 | from typing import Any 8 | 9 | import pytest 10 | from git import Repo 11 | 12 | from . import utils 13 | 14 | 15 | @pytest.mark.parametrize("env_package", ["private_repo_https", "private_repo_ssh"]) 16 | def test_private_repos(env_package: str, test_env: dict[str, Any], tmp_path: Path) -> None: 17 | """ 18 | Validate a cachito request with no package managers to a private repo. 19 | 20 | Process: 21 | Create new commit at "cachito-no-package-manager-private" repo 22 | (To prevent cachito from caching private source code and serving 23 | it without trying to access the repository, more info: STONEBLD-661) 24 | Send new request to the Cachito API 25 | Send request to check status of existing request 26 | 27 | Checks: 28 | * Check that the request completes successfully 29 | * Check that no packages are identified in response 30 | * Check that no dependencies are identified in response 31 | * Check that the source tarball includes the application source code. Verify the expected files 32 | by checking both the ref and diff because we don't have a clone of the private source repo. 33 | * Check that the source tarball includes empty deps directory 34 | * Check that the content manifest is successfully generated and empty 35 | """ 36 | test_data = utils.load_test_data("private_repo_packages.yaml") 37 | private_repo_test_envs = test_data["private_repo_test_envs"] 38 | env_data = test_data[env_package] 39 | job_name = str(os.environ.get("JOB_NAME")) 40 | is_supported_env = any(x in job_name for x in private_repo_test_envs) 41 | if not is_supported_env: 42 | pytest.skip( 43 | ( 44 | "This test is only executed in environments that " 45 | "have been configured with the credentials needed " 46 | "to access private repositories." 47 | ) 48 | ) 49 | 50 | repo = Repo.clone_from(test_data["private_repo_ssh"]["repo"], tmp_path) 51 | 52 | repo.config_writer().set_value("user", "name", test_env["git_user"]).release() 53 | repo.config_writer().set_value("user", "email", test_env["git_email"]).release() 54 | 55 | generated_suffix = "".join( 56 | random.choice(string.ascii_letters + string.digits) for x in range(10) 57 | ) 58 | branch_name = f"tmp-branch-{generated_suffix}" 59 | 60 | try: 61 | repo.create_head(branch_name).checkout() 62 | 63 | message = "Committed by Cachito integration test (test_private_repos)" 64 | repo.git.commit("--allow-empty", m=message) 65 | repo.git.push("-u", "origin", branch_name) 66 | 67 | ref = repo.head.commit.hexsha 68 | 69 | client = utils.Client( 70 | test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout") 71 | ) 72 | payload = { 73 | "repo": env_data["repo"], 74 | "ref": ref, 75 | "pkg_managers": [], 76 | "flags": ["include-git-dir"], 77 | } 78 | 79 | initial_response = client.create_new_request(payload=payload) 80 | completed_response = client.wait_for_complete_request(initial_response) 81 | 82 | utils.assert_properly_completed_response(completed_response) 83 | assert completed_response.data["packages"] == [] 84 | assert completed_response.data["dependencies"] == [] 85 | 86 | client.download_and_extract_archive(completed_response.id, tmp_path) 87 | source_path = tmp_path / f"download_{str(completed_response.id)}" 88 | downloaded_repo = Repo(source_path / "app") 89 | assert downloaded_repo.head.commit.hexsha == ref 90 | assert not downloaded_repo.git.diff() 91 | assert not os.listdir(source_path / "deps") 92 | 93 | utils.assert_content_manifest(client, completed_response.id, []) 94 | 95 | finally: 96 | repo.git.push("--delete", "origin", branch_name) 97 | -------------------------------------------------------------------------------- /tests/integration/test_request_error.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | from . import utils 4 | 5 | 6 | def test_complete_request_no_error_info(test_env): 7 | """ 8 | Check that the response of a complete request does not include error information. 9 | 10 | Process: 11 | Send new request to the Cachito API 12 | Send request to check status of existing request 13 | 14 | Checks: 15 | * Check that the request error origin and type are not present in the response data 16 | """ 17 | env_data = utils.load_test_data("pip_packages.yaml")["local_path"] 18 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 19 | initial_response = client.create_new_request( 20 | payload={"repo": env_data["repo"], "ref": env_data["ref"]} 21 | ) 22 | completed_response = client.wait_for_complete_request(initial_response) 23 | assert completed_response.status == 200 24 | assert completed_response.data["state"] == "complete" 25 | assert ( 26 | "error_origin" not in completed_response.data 27 | and "error_type" not in completed_response.data 28 | ) 29 | 30 | 31 | def test_failed_request_error_info(test_env): 32 | """ 33 | Check that the response of a failed request includes appropriate error information. 34 | 35 | Process: 36 | Send new request to the Cachito API 37 | Send request to check status of existing request 38 | 39 | Checks: 40 | * Validate the request error origin and type in the response data 41 | """ 42 | env_data = utils.load_test_data("pip_packages.yaml")["local_path"] 43 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 44 | initial_response = client.create_new_request( 45 | payload={ 46 | "repo": env_data["repo"], 47 | "ref": env_data["ref"], 48 | "pkg_managers": ["npm"], # Wrong package manager 49 | } 50 | ) 51 | completed_response = client.wait_for_complete_request(initial_response) 52 | assert completed_response.status == 200 53 | assert completed_response.data["state"] == "failed" 54 | assert completed_response.data["state_reason"] == ( 55 | "The npm-shrinkwrap.json or package-lock.json file " 56 | "must be present for the npm package manager" 57 | ) 58 | assert "error_origin" in completed_response.data and "error_type" in completed_response.data 59 | assert completed_response.data["error_origin"] == "client" 60 | assert completed_response.data["error_type"] == "InvalidRepoStructure" 61 | -------------------------------------------------------------------------------- /tests/integration/test_request_metrics.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import os 3 | from datetime import datetime 4 | 5 | import pytest 6 | 7 | from . import utils 8 | 9 | 10 | def test_get_request_metrics(api_client): 11 | finished_from = datetime.utcnow().isoformat() 12 | env_data = utils.load_test_data("pip_packages.yaml")["without_deps"] 13 | request = api_client.create_new_request( 14 | payload={ 15 | "repo": env_data["repo"], 16 | "ref": env_data["ref"], 17 | "pkg_managers": env_data["pkg_managers"], 18 | }, 19 | ) 20 | api_client.wait_for_complete_request(request) 21 | 22 | resp = api_client.fetch_request_metrics(finished_from=finished_from) 23 | assert resp.status_code == 200 24 | request_metrics = resp.json()["items"][0] 25 | assert request_metrics["id"] == request.id 26 | assert request_metrics["duration"] > 0 27 | assert request_metrics["time_in_queue"] > 0 28 | 29 | 30 | @pytest.mark.skipif( 31 | "cachito-prod" in str(os.environ.get("JOB_NAME")), 32 | reason="Test is skipped in production environment", 33 | ) 34 | def test_get_request_metrics_summary(api_client): 35 | finished_from = datetime.utcnow().isoformat() 36 | env_data = utils.load_test_data("pip_packages.yaml")["without_deps"] 37 | total = 3 38 | for _ in range(total): 39 | request = api_client.create_new_request( 40 | payload={ 41 | "repo": env_data["repo"], 42 | "ref": env_data["ref"], 43 | "pkg_managers": env_data["pkg_managers"], 44 | }, 45 | ) 46 | api_client.wait_for_complete_request(request) 47 | 48 | resp = api_client.fetch_request_metrics_summary( 49 | finished_from=finished_from, 50 | finished_to=datetime.utcnow().isoformat(), 51 | ) 52 | assert resp.status_code == 200 53 | summary = resp.json() 54 | assert not { 55 | "duration_avg", 56 | "duration_50", 57 | "duration_95", 58 | "time_in_queue_avg", 59 | "time_in_queue_95", 60 | "client_errors", 61 | "server_errors", 62 | "total", 63 | }.difference(summary) 64 | assert summary["total"] >= total 65 | assert 0 < summary["duration_50"] <= summary["duration_95"] 66 | assert summary["duration_avg"] > 0 67 | assert summary["time_in_queue_avg"] > 0 68 | assert summary["time_in_queue_95"] > 0 69 | assert summary["client_errors"] == 0 70 | assert summary["server_errors"] == 0 71 | 72 | # Check empty datetime range 73 | resp = api_client.fetch_request_metrics_summary( 74 | finished_from=finished_from, 75 | finished_to=finished_from, 76 | ) 77 | assert resp.status_code == 200 78 | summary = resp.json() 79 | assert summary["total"] == 0 80 | -------------------------------------------------------------------------------- /tests/integration/test_run_app_from_bundle.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | import shutil 4 | import subprocess 5 | from os import path 6 | 7 | import pytest 8 | 9 | from . import utils 10 | 11 | 12 | @pytest.mark.skipif(not shutil.which("go"), reason="requires go to be installed") 13 | def test_run_app_from_bundle(test_env, default_requests, tmpdir): 14 | """ 15 | Check that downloaded bundle could be used to run the application. 16 | 17 | Process: 18 | * Send new request to Cachito API 19 | * Download a bundle from the request 20 | * Run go build 21 | * Run the application 22 | 23 | Checks: 24 | * Check that the state of request is complete 25 | * Check that the bundle is properly downloaded 26 | * Check that the application runs successfully 27 | """ 28 | response = default_requests["gomod"].complete_response 29 | utils.assert_properly_completed_response(response) 30 | 31 | client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout")) 32 | client.download_and_extract_archive(response.id, tmpdir) 33 | bundle_dir = tmpdir.join(f"download_{str(response.id)}") 34 | app_name = test_env["run_app"]["app_name"] 35 | app_binary_file = str(tmpdir.join(app_name)) 36 | subprocess.run( 37 | ["go", "build", "-o", app_binary_file, str(bundle_dir.join("app", "main.go"))], 38 | env={ 39 | "GOPATH": str(bundle_dir.join("deps", "gomod")), 40 | "GOCACHE": str(bundle_dir.join("deps", "gomod")), 41 | "GOMODCACHE": "{}/pkg/mod".format(str(bundle_dir.join("deps", "gomod"))), 42 | }, 43 | cwd=str(bundle_dir.join("app")), 44 | check=True, 45 | ) 46 | 47 | assert path.exists( 48 | app_binary_file 49 | ), f"#{response.id}: Path for application binary file {app_binary_file} does not exist" 50 | sp = subprocess.run([app_binary_file, "--help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) 51 | assert sp.returncode == 0 52 | -------------------------------------------------------------------------------- /tests/test_cachito_config.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import os 3 | import tempfile 4 | from unittest.mock import patch 5 | 6 | import pytest 7 | 8 | from cachito.errors import ConfigError 9 | from cachito.web.config import validate_cachito_config 10 | 11 | 12 | @patch("os.path.isdir", return_value=True) 13 | def test_validate_cachito_config_success(mock_isdir, app): 14 | validate_cachito_config(app.config) 15 | mock_isdir.assert_any_call(os.path.join(tempfile.gettempdir(), "cachito-archives/bundles")) 16 | 17 | 18 | @patch("os.path.isdir", return_value=True) 19 | @pytest.mark.parametrize( 20 | "variable_name", 21 | ( 22 | "CACHITO_BUNDLES_DIR", 23 | "CACHITO_DEFAULT_PACKAGE_MANAGERS", 24 | "CACHITO_LOG_LEVEL", 25 | "CACHITO_MAX_PER_PAGE", 26 | "CACHITO_MUTUALLY_EXCLUSIVE_PACKAGE_MANAGERS", 27 | "CACHITO_LOG_FORMAT", 28 | "SQLALCHEMY_DATABASE_URI", 29 | ), 30 | ) 31 | def test_validate_cachito_config_failure(mock_isdir, app, variable_name): 32 | expected = f'The configuration "{variable_name}" must be set' 33 | if variable_name == "CACHITO_BUNDLES_DIR": 34 | expected += " to an existing directory" 35 | with patch.dict(app.config, {variable_name: None}): 36 | with pytest.raises(ConfigError, match=expected): 37 | validate_cachito_config(app.config) 38 | 39 | 40 | @patch("os.path.isdir") 41 | def test_validate_cachito_config_cli(mock_isdir, app): 42 | validate_cachito_config(app.config, cli=True) 43 | mock_isdir.assert_not_called() 44 | 45 | 46 | @pytest.mark.parametrize( 47 | "value, is_valid", 48 | [ 49 | ([], True), 50 | (["gomod"], False), 51 | ([("gomod", "git-submodule")], True), 52 | ([["gomod", "git-submodule"]], True), 53 | ([("gomod",)], False), 54 | ([["gomod"]], False), 55 | ], 56 | ) 57 | def test_validate_mutually_exclusive_package_managers(app, value, is_valid, tmpdir): 58 | config = app.config.copy() 59 | config["CACHITO_MUTUALLY_EXCLUSIVE_PACKAGE_MANAGERS"] = value 60 | 61 | if is_valid: 62 | # Successful validation requires this dir exists. 63 | config["CACHITO_BUNDLES_DIR"] = str(tmpdir) 64 | validate_cachito_config(config) 65 | else: 66 | expected = ( 67 | r'All values in "CACHITO_MUTUALLY_EXCLUSIVE_PACKAGE_MANAGERS" ' 68 | r"must be pairs \(2-tuples or 2-item lists\)" 69 | ) 70 | with pytest.raises(ConfigError, match=expected): 71 | validate_cachito_config(config) 72 | -------------------------------------------------------------------------------- /tests/test_checksum.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import hashlib 3 | 4 | import pytest 5 | 6 | from cachito.common.checksum import hash_file 7 | from cachito.errors import UnknownHashAlgorithm 8 | 9 | 10 | def test_get_unknown_hash_algorithm(): 11 | with pytest.raises(UnknownHashAlgorithm): 12 | hash_file("some_file.tar", algorithm="xxx") 13 | 14 | 15 | @pytest.mark.parametrize("file_content", ["", "abc123" * 100]) 16 | @pytest.mark.parametrize("algorithm", [None, "sha512"]) 17 | def test_hash_file(file_content, algorithm, tmpdir): 18 | data_file = tmpdir.join("file.data") 19 | data_file.write(file_content) 20 | 21 | if algorithm is None: 22 | hasher = hash_file(str(data_file)) 23 | assert hashlib.sha256(file_content.encode()).digest() == hasher.digest() 24 | else: 25 | hasher = hash_file(str(data_file), algorithm=algorithm) 26 | h = hashlib.new(algorithm) 27 | h.update(file_content.encode()) 28 | assert h.digest() == hasher.digest() 29 | -------------------------------------------------------------------------------- /tests/test_common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_common/__init__.py -------------------------------------------------------------------------------- /tests/test_common/test_utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import pytest 3 | 4 | from cachito.common import utils 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "url, expected_repo_name", 9 | [ 10 | ("https://github.com/containerbuildsystem/cachito/", "containerbuildsystem/cachito"), 11 | ("https://github.com/containerbuildsystem/cachito.git/", "containerbuildsystem/cachito"), 12 | ("https://github.com/containerbuildsystem/cachito.git", "containerbuildsystem/cachito"), 13 | ], 14 | ) 15 | def test_get_repo_name(url, expected_repo_name): 16 | repo_name = utils.get_repo_name(url) 17 | assert repo_name == expected_repo_name 18 | -------------------------------------------------------------------------------- /tests/test_docs.py: -------------------------------------------------------------------------------- 1 | # # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | 4 | def test_docs(client): 5 | rv = client.get("/") 6 | assert "Cachito API Documentation" in rv.data.decode("utf-8") 7 | -------------------------------------------------------------------------------- /tests/test_healthcheck.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | from unittest.mock import patch 3 | 4 | from sqlalchemy.exc import NoSuchTableError 5 | 6 | 7 | def test_health_check(client, db): 8 | rv = client.get("/healthcheck") 9 | assert rv.status_code == 200 10 | assert rv.data == b"OK" 11 | 12 | 13 | def test_health_check_failed(client, db): 14 | with patch("cachito.web.app.db.session.execute") as mock_execute: 15 | mock_execute.side_effect = NoSuchTableError() 16 | rv = client.get("/healthcheck") 17 | assert rv.status_code == 500 18 | assert rv.json.keys() == {"error"} 19 | -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import datetime 3 | from typing import Optional 4 | from unittest import mock 5 | 6 | import pytest 7 | 8 | from cachito.web.models import PackageManager, Request, RequestStateMapping 9 | 10 | 11 | @pytest.mark.parametrize( 12 | "name,expected", 13 | [["gomod", "gomod"], ["", None], [None, None], ["unknown", None]], 14 | ) 15 | def test_package_manager_get_by_name(name, expected, app, db, auth_env): 16 | pkg_manager: Optional[PackageManager] = PackageManager.get_by_name(name) 17 | if expected is None: 18 | assert expected == pkg_manager 19 | else: 20 | assert expected == pkg_manager.name 21 | 22 | 23 | class TestRequest: 24 | def _create_request_object(self): 25 | request = Request() 26 | request.repo = "a_repo" 27 | request.ref = "a_ref" 28 | request.user_id = 1 29 | request.submitted_by_id = 1 30 | request.packages_count = 1 31 | request.dependencies_count = 1 32 | 33 | return request 34 | 35 | @pytest.mark.parametrize( 36 | "state, call_count", 37 | [ 38 | [RequestStateMapping.in_progress.name, 0], 39 | [RequestStateMapping.complete.name, 2], 40 | [RequestStateMapping.failed.name, 0], 41 | [RequestStateMapping.stale.name, 0], 42 | ], 43 | ) 44 | @mock.patch("cachito.common.packages_data.PackagesData.load") 45 | def test_package_data_is_only_accessed_when_request_is_complete( 46 | self, load_mock, state, call_count, app, auth_env 47 | ): 48 | request = self._create_request_object() 49 | request.add_state(state, "Reason") 50 | 51 | with app.test_request_context(environ_base=auth_env): 52 | request.to_json() 53 | request.content_manifest.to_json() 54 | 55 | assert load_mock.call_count == call_count 56 | 57 | def test_utcnow(self, app, auth_env): 58 | request = self._create_request_object() 59 | request.add_state(RequestStateMapping.complete.name, "Complete") 60 | current_utc_datetime = datetime.datetime.utcnow() 61 | request_created_datetime = request.created 62 | diff_in_secs = (current_utc_datetime - request_created_datetime).total_seconds() 63 | 64 | # Difference between "created" and current UTC datetimes is within 10 seconds 65 | assert abs(diff_in_secs) <= 10 66 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | 3 | import pytest 4 | 5 | from cachito.web.utils import deep_sort_icm 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "orig_items", 10 | [ 11 | [ 12 | { 13 | "metadata": {"image_layer_index": -1, "icm_spec": "sample-URL", "icm_version": 1}, 14 | "image_contents": [ 15 | { 16 | "dependencies": [ 17 | {"purl": "5sample-URL"}, 18 | {"purl": "4sample-URL"}, 19 | {"purl": "3sample-URL"}, 20 | {"purl": "2sample-URL"}, 21 | {"purl": "1sample-URL"}, 22 | {"purl": "0sample-URL"}, 23 | ], 24 | "purl": "1sample-URL", 25 | "sources": [], 26 | }, 27 | { 28 | "dependencies": [], 29 | "purl": "0sample-URL", 30 | "sources": [{"purl": "1sample-URL"}, {"purl": "0sample-URL"}], 31 | }, 32 | ], 33 | }, 34 | ], 35 | ], 36 | ) 37 | def test_deep_sort_icm(orig_items): 38 | expected = [ 39 | { 40 | "image_contents": [ 41 | { 42 | "dependencies": [], 43 | "purl": "0sample-URL", 44 | "sources": [{"purl": "0sample-URL"}, {"purl": "1sample-URL"}], 45 | }, 46 | { 47 | "dependencies": [ 48 | {"purl": "0sample-URL"}, 49 | {"purl": "1sample-URL"}, 50 | {"purl": "2sample-URL"}, 51 | {"purl": "3sample-URL"}, 52 | {"purl": "4sample-URL"}, 53 | {"purl": "5sample-URL"}, 54 | ], 55 | "purl": "1sample-URL", 56 | "sources": [], 57 | }, 58 | ], 59 | "metadata": {"icm_spec": "sample-URL", "icm_version": 1, "image_layer_index": -1}, 60 | } 61 | ] 62 | deep_sort_icm(orig_items) 63 | assert orig_items == expected 64 | 65 | 66 | @pytest.mark.parametrize( 67 | "error_icm", 68 | [ 69 | "image content manifest", 70 | { 71 | "image_contents": [ 72 | { 73 | "dependencies": [], 74 | "purl": "0sample-URL", 75 | "sources": [("purl", "0sample-URL"), ("purl", "1sample-URL")], 76 | }, 77 | ], 78 | "metadata": {"icm_spec": "sample-URL", "icm_version": 1, "image_layer_index": -1}, 79 | }, 80 | ], 81 | ) 82 | def test_deep_sort_icm_raises_error_when_unknown_type_included(error_icm): 83 | with pytest.raises(TypeError, match="Unknown type is included in the content manifest"): 84 | deep_sort_icm(error_icm) 85 | -------------------------------------------------------------------------------- /tests/test_workers/test_pkg_managers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_workers/test_pkg_managers/__init__.py -------------------------------------------------------------------------------- /tests/test_workers/test_pkg_managers/data/gomod-mocks/non-vendored/go_list_modules.json: -------------------------------------------------------------------------------- 1 | { 2 | "Path": "github.com/cachito-testing/gomod-pandemonium", 3 | "Main": true, 4 | "Dir": "{repo_dir}", 5 | "GoMod": "{repo_dir}/go.mod", 6 | "GoVersion": "1.19" 7 | } 8 | -------------------------------------------------------------------------------- /tests/test_workers/test_pkg_managers/data/myapp-0.1.tar.Z: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_workers/test_pkg_managers/data/myapp-0.1.tar.Z -------------------------------------------------------------------------------- /tests/test_workers/test_pkg_managers/data/myapp-0.1.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_workers/test_pkg_managers/data/myapp-0.1.tar.bz2 -------------------------------------------------------------------------------- /tests/test_workers/test_pkg_managers/data/myapp-0.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_workers/test_pkg_managers/data/myapp-0.1.tar.gz -------------------------------------------------------------------------------- /tests/test_workers/test_pkg_managers/data/myapp-0.1.tar.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_workers/test_pkg_managers/data/myapp-0.1.tar.xz -------------------------------------------------------------------------------- /tests/test_workers/test_pkg_managers/data/myapp-0.1.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_workers/test_pkg_managers/data/myapp-0.1.zip -------------------------------------------------------------------------------- /tests/test_workers/test_pkg_managers/data/myapp-0.1.zip.fake.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_workers/test_pkg_managers/data/myapp-0.1.zip.fake.tar -------------------------------------------------------------------------------- /tests/test_workers/test_pkg_managers/data/myapp-without-pkg-info.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_workers/test_pkg_managers/data/myapp-without-pkg-info.tar.gz -------------------------------------------------------------------------------- /tests/test_workers/test_pkg_managers/golang_git_repo.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_workers/test_pkg_managers/golang_git_repo.tar.gz -------------------------------------------------------------------------------- /tests/test_workers/test_tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerbuildsystem/cachito/01a944ebe81c58e294aa79bda43b475634306828/tests/test_workers/test_tasks/__init__.py -------------------------------------------------------------------------------- /tests/test_workers/test_tasks/test_gitsubmodule.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest import mock 3 | 4 | from cachito.workers.paths import RequestBundleDir 5 | from cachito.workers.tasks import gitsubmodule 6 | 7 | url = "https://github.com/release-engineering/retrodep.git" 8 | ref = "c50b93a32df1c9d700e3e80996845bc2e13be848" 9 | archive_path = f"/tmp/cachito-archives/release-engineering/retrodep/{ref}.tar.gz" 10 | 11 | 12 | @mock.patch("git.Repo") 13 | @mock.patch("cachito.workers.paths.get_worker_config") 14 | def test_add_git_submodules_as_package( 15 | get_worker_config, mock_repo, task_passes_state_check, tmpdir 16 | ): 17 | get_worker_config.return_value = mock.Mock(cachito_bundles_dir=tmpdir) 18 | submodule = mock.Mock() 19 | submodule.name = "tour" 20 | submodule.hexsha = "522fb816eec295ad58bc488c74b2b46748d471b2" 21 | submodule.url = "https://github.com/user/tour.git" 22 | submodule.path = "tour" 23 | mock_repo.return_value.submodules = [submodule] 24 | package = { 25 | "type": "git-submodule", 26 | "name": "tour", 27 | "version": "https://github.com/user/tour.git#522fb816eec295ad58bc488c74b2b46748d471b2", 28 | } 29 | gitsubmodule.add_git_submodules_as_package(3) 30 | 31 | bundle_dir = RequestBundleDir(3) 32 | expected = package.copy() 33 | expected["path"] = "tour" 34 | expected["dependencies"] = [] 35 | assert {"packages": [expected]} == json.loads( 36 | bundle_dir.git_submodule_packages_data.read_bytes() 37 | ) 38 | -------------------------------------------------------------------------------- /tests/test_workers/test_worker_utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-3.0-or-later 2 | import json 3 | import tarfile 4 | from typing import Any, Dict 5 | from unittest.mock import patch 6 | 7 | import pytest 8 | 9 | from cachito.workers import load_json_stream, run_cmd, safe_extract 10 | 11 | 12 | @pytest.mark.parametrize( 13 | "input_params,expected_run_params", 14 | [ 15 | [ 16 | {"timeout": 300}, 17 | { 18 | "timeout": 300, 19 | "capture_output": True, 20 | "universal_newlines": True, 21 | "encoding": "utf-8", 22 | }, 23 | ], 24 | # No timeout is passed in, use the default in config 25 | [ 26 | {}, 27 | { 28 | "timeout": 3600, 29 | "capture_output": True, 30 | "universal_newlines": True, 31 | "encoding": "utf-8", 32 | }, 33 | ], 34 | ], 35 | ) 36 | @patch("subprocess.run") 37 | def test_run_cmd_with_timeout( 38 | mock_run, input_params: Dict[str, Any], expected_run_params: Dict[str, Any] 39 | ): 40 | mock_run.return_value.returncode = 0 41 | cmd = ["git", "fcsk"] 42 | run_cmd(cmd, input_params) 43 | mock_run.assert_called_once_with(cmd, **expected_run_params) 44 | 45 | 46 | @pytest.mark.parametrize( 47 | "test_input, expected_output", 48 | [ 49 | ("\n", []), 50 | ("1 2 3 4", [1, 2, 3, 4]), 51 | ("[1, 2][3, 4]", [[1, 2], [3, 4]]), 52 | ('\n{"a": 1}\n\n{"b": 2}\n', [{"a": 1}, {"b": 2}]), 53 | ], 54 | ) 55 | def test_load_json_stream(test_input, expected_output): 56 | assert list(load_json_stream(test_input)) == expected_output 57 | 58 | 59 | def test_load_json_stream_invalid(): 60 | invalid_input = "1 2 invalid" 61 | data = load_json_stream(invalid_input) 62 | assert next(data) == 1 63 | assert next(data) == 2 64 | with pytest.raises(json.JSONDecodeError, match="Expecting value: line 1 column 5"): 65 | next(data) 66 | 67 | 68 | def test_safe_extract(tmp_path): 69 | 70 | with tarfile.open(tmp_path / "fake.tar", "w") as tf: 71 | tf.add("README.md", "../README.md") 72 | 73 | with tarfile.open(tmp_path / "fake.tar") as tar: 74 | with pytest.raises(tarfile.ExtractError): 75 | safe_extract(tar, tmp_path) 76 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = bandit,black,isort,flake8,mypy,python3.11 3 | 4 | [gh-actions] 5 | python = 6 | 3.11: python3.11 7 | 8 | [testenv] 9 | deps = 10 | -rrequirements.txt 11 | -rrequirements-web.txt 12 | -rrequirements-test.txt 13 | allowlist_externals = 14 | make 15 | mkdir 16 | rm 17 | passenv = TOX_ENV_DIR 18 | setenv = 19 | CACHITO_TESTING=true 20 | PROMETHEUS_MULTIPROC_DIR={envtmpdir}/prometheus_metrics 21 | 22 | usedevelop = true 23 | 24 | commands = 25 | py.test \ 26 | --ignore tests/integration \ 27 | --cov-config .coveragerc --cov=cachito --cov-report term \ 28 | --cov-report xml --cov-report html {posargs} 29 | 30 | commands_post = 31 | rm -rf {envtmpdir}/prometheus_metrics 32 | 33 | [testenv:black] 34 | description = black checks [Mandatory] 35 | skip_install = true 36 | deps = 37 | # Pin the version of black and click to avoid a newer version causing tox to fail 38 | black==22.6.0 39 | click==8.0.3 40 | commands = 41 | black --check --diff cachito tests 42 | # Use shorter line length for scripts 43 | black --check --diff bin --line-length=88 44 | 45 | [testenv:isort] 46 | skip_install = true 47 | # Remove colorama after https://github.com/PyCQA/isort/issues/2211 is released 48 | deps = 49 | isort[colors] 50 | colorama 51 | commands = 52 | isort --check --diff --color cachito tests 53 | 54 | [testenv:flake8] 55 | skip_install = true 56 | deps = 57 | flake8==3.9.2 58 | flake8-docstrings==1.6.0 59 | commands = 60 | flake8 61 | 62 | [flake8] 63 | show-source = True 64 | max-line-length = 100 65 | exclude = venv,.git,.tox,dist,*egg,cachito/web/migrations,.env 66 | # W503 line break before binary operator 67 | # E203 whitespace before ':' ("black" will catch the valid cases) 68 | ignore = D100,D104,D105,W503,E203 69 | per-file-ignores = 70 | # Ignore missing docstrings in the tests and migrations 71 | tests/*:D101,D102,D103 72 | cachito/web/migrations/*:D103 73 | 74 | [testenv:bandit] 75 | skip_install = true 76 | deps = 77 | bandit 78 | commands = 79 | bandit -r cachito 80 | 81 | [testenv:mypy] 82 | commands = 83 | pip install mypy # cannot be in deps due requirement of hashes 84 | mypy -p cachito --install-types --non-interactive --ignore-missing-imports 85 | 86 | [testenv:integration] 87 | basepython = python3 88 | skipsdist = true 89 | skip_install = true 90 | commands = 91 | pytest -rA -vvvv \ 92 | --confcutdir=tests/integration \ 93 | {posargs:tests/integration} 94 | passenv = KRB5CCNAME,REQUESTS_CA_BUNDLE,KRB5_CLIENT_KTNAME,CACHITO_TEST_CERT\ 95 | CACHITO_TEST_KEY,JOB_NAME 96 | --------------------------------------------------------------------------------