├── quetz ├── jobs │ ├── __init__.py │ ├── handlers.py │ └── dao.py ├── tasks │ ├── __init__.py │ ├── cleanup.py │ └── assertions.py ├── tests │ ├── __init__.py │ ├── api │ │ ├── __init__.py │ │ └── conftest.py │ ├── data │ │ ├── dummy-plugin │ │ │ ├── quetz_dummyplugin-1.0-py3.egg-info │ │ │ │ ├── top_level.txt │ │ │ │ └── entry_points.txt │ │ │ └── quetz_dummyplugin │ │ │ │ └── jobs.py │ │ ├── other-package-0.2-0.conda │ │ ├── test-package-0.1-0.tar.bz2 │ │ ├── test-package-0.2-0.tar.bz2 │ │ ├── other-package-0.1-0.tar.bz2 │ │ ├── other-package-0.2-0.tar.bz2 │ │ └── test-package-0.1-0_copy.tar.bz2 │ ├── test_profiling.py │ ├── test_health.py │ ├── test_docs.py │ ├── test_db_models.py │ ├── test_database.py │ └── authentification │ │ ├── test_base.py │ │ └── test_pam.py ├── testing │ ├── __init__.py │ ├── mockups.py │ └── utils.py ├── metrics │ ├── __init__.py │ ├── view.py │ ├── rest_models.py │ ├── api.py │ ├── tasks.py │ └── db_models.py ├── basic_frontend │ ├── avatar.jpg │ └── favicon.ico ├── _version.py ├── authentication │ ├── __init__.py │ ├── registry.py │ ├── github.py │ └── google.py ├── __init__.py ├── config.toml ├── exceptions.py ├── errors.py ├── migrations │ ├── versions │ │ ├── ea6eba9a9ffc_merge_ebe550f9fbbe_and_b9886d9cadb0.py │ │ ├── 0a0ab48887ab_adding_function_args_to_job_spec.py │ │ ├── cd404ed93cc0_add_per_channel_ttl.py │ │ ├── 3c3288034362_add_channel_metadata.py │ │ ├── 303ff70c27fc_configure_mirror_endpoints.py │ │ ├── db1c56bf4d57_add_channel_size_limit.py │ │ ├── cddba8e6e639_scheduling_spec_for_jobs.py │ │ ├── 98c04a65df4a_register_mirrors.py │ │ ├── ebe550f9fbbe_added_create_at_and_expire_at_date_to_.py │ │ ├── a3ffa287d074_case_insensitive_channel_names.py │ │ ├── 30241b33d849_add_task_pending_state.py │ │ ├── d212023a8e0b_add_useremail_table_for_email_addresses.py │ │ ├── 0653794b6252_adding_url_and_platforms_dirs.py │ │ ├── 8d1e9a9e0b1f_adding_download_metrics.py │ │ ├── b9886d9cadb0_create_indexes_for_download_count_.py │ │ ├── 53f81aba78ce_use_biginteger_for_size.py │ │ ├── 3ba25f23fb7d_update_scoped_api_key_uploader_id.py │ │ ├── 794249a0b1bd_adding_jobs_tables.py │ │ └── 8dfb7c4bfbd7_new_package_versions.py │ └── script.py.mako ├── repo_data.py ├── templates │ ├── subdir-index.html.j2 │ └── channeldata-index.html.j2 └── channel_data.py ├── plugins ├── quetz_tos │ ├── quetz_tos │ │ ├── __init__.py │ │ ├── migrations │ │ │ ├── __init__.py │ │ │ └── versions │ │ │ │ ├── __init__.py │ │ │ │ ├── 44ec522465e5_add_tables_for_terms_of_service_and_.py │ │ │ │ └── c3a635971280_tos_languages.py │ │ ├── db_models.py │ │ └── main.py │ ├── setup.py │ ├── README.md │ └── tests │ │ └── conftest.py ├── quetz_googleiap │ ├── quetz_googleiap │ │ └── __init__.py │ ├── setup.py │ └── tests │ │ ├── conftest.py │ │ └── test_googleiap.py ├── quetz_harvester │ ├── quetz_harvester │ │ ├── __init__.py │ │ └── jobs.py │ ├── conda-requirements.txt │ ├── tests │ │ ├── data │ │ │ └── xtensor-io-0.10.3-hb585cf6_0.tar.bz2 │ │ └── test_main.py │ ├── setup.py │ └── README.md ├── quetz_runexports │ ├── quetz_runexports │ │ ├── __init__.py │ │ ├── migrations │ │ │ ├── __init__.py │ │ │ └── versions │ │ │ │ ├── __init__.py │ │ │ │ └── cb095bcf3bb4_initial_revision.py │ │ ├── db_models.py │ │ ├── main.py │ │ └── api.py │ ├── setup.py │ ├── README.md │ ├── alembic.ini │ └── tests │ │ ├── conftest.py │ │ └── test_quetz_runexports.py ├── quetz_mamba_solve │ ├── quetz_mamba_solve │ │ ├── __init__.py │ │ ├── rest_models.py │ │ ├── main.py │ │ ├── utils.py │ │ └── api.py │ ├── conda-requirements.txt │ ├── tests │ │ ├── conftest.py │ │ └── test_mamba_solve.py │ ├── setup.py │ └── README.md ├── quetz_conda_suggest │ ├── quetz_conda_suggest │ │ ├── __init__.py │ │ ├── migrations │ │ │ ├── __init__.py │ │ │ └── versions │ │ │ │ ├── __init__.py │ │ │ │ └── c726f33caeeb_initial_revision.py │ │ ├── db_models.py │ │ └── api.py │ ├── setup.py │ ├── README.md │ └── tests │ │ └── conftest.py ├── quetz_content_trust │ ├── quetz_content_trust │ │ ├── __init__.py │ │ ├── migrations │ │ │ ├── __init__.py │ │ │ └── versions │ │ │ │ ├── __init__.py │ │ │ │ └── dadfc30be670_add_repo_signing_key.py │ │ ├── rest_models.py │ │ ├── repo_signer.py │ │ ├── main.py │ │ └── db_models.py │ ├── conda-requirements.txt │ ├── docs │ │ ├── test_corrupted_key_mgr_metadata.py │ │ ├── test_corrupted_key_mgr_sigs.py │ │ └── testing.rst │ ├── setup.py │ ├── tests │ │ └── data │ │ │ ├── key_mgr.json │ │ │ └── root.json │ └── README.md ├── quetz_repodata_zchunk │ ├── conda-requirements.txt │ ├── quetz_repodata_zchunk │ │ ├── __init__.py │ │ └── main.py │ ├── tests │ │ └── conftest.py │ ├── setup.py │ └── README.md ├── quetz_transmutation │ ├── quetz_transmutation │ │ ├── __init__.py │ │ ├── rest_models.py │ │ └── jobs.py │ ├── conda-requirements.txt │ ├── setup.py │ └── README.md ├── quetz_current_repodata │ ├── quetz_current_repodata │ │ ├── __init__.py │ │ └── main.py │ ├── setup.py │ ├── README.md │ └── tests │ │ ├── conftest.py │ │ └── test_current_repodata.py ├── quetz_repodata_patching │ ├── quetz_repodata_patching │ │ └── __init__.py │ ├── tests │ │ └── conftest.py │ ├── setup.py │ └── README.md ├── README.md └── quetz_dictauthenticator │ ├── tests │ ├── conftest.py │ └── test_authenticator.py │ ├── setup.py │ ├── README.md │ └── quetz_dictauthenticator │ └── __init__.py ├── setup.py ├── set_env_dev.sh ├── .flake8 ├── docker ├── grafana.env ├── graphana_datasources.yml ├── postgres.env ├── Dockerfile.jupyterhub ├── wait-for-postgres.sh ├── prometheus.yml ├── docker_config.toml ├── Dockerfile └── nginx.conf ├── docs ├── assets │ └── quetz_header.png ├── source │ ├── _static │ │ ├── quetz_logo.png │ │ └── quetz_favicon.ico │ ├── using │ │ ├── index.rst │ │ └── basics.rst │ ├── qeps │ │ └── index.rst │ ├── deploying │ │ ├── index.rst │ │ ├── frontend.rst │ │ ├── workers.rst │ │ ├── database.rst │ │ └── migrations.rst │ ├── index.rst │ └── conf.py ├── Makefile └── make.bat ├── _typos.toml ├── .readthedocs.yml ├── test-cli-client.sh ├── .gitignore ├── .github └── workflows │ ├── enforce-label.yml │ ├── docker-build.yml │ ├── lint.yml │ ├── sphinx.yml │ └── docker-build-push.yml ├── quetz_db_ext ├── Dockerfile ├── conda.h ├── CMakeLists.txt ├── quetz_pg.c ├── quetz_sqlite.c └── LIBSOLV_LICENSE.txt ├── download-test-package.sh ├── dev_config.toml ├── Dockerfile ├── .pre-commit-config.yaml ├── MANIFEST.in ├── environment.yml ├── CONTRIBUTING.md ├── LICENSE ├── setup.cfg ├── pyproject.toml ├── utils └── repodata_compare.py ├── RELEASE.md ├── docker-compose.yml └── alembic.ini /quetz/jobs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /quetz/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /quetz/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /quetz/testing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /quetz/tests/api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_tos/quetz_tos/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_googleiap/quetz_googleiap/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_harvester/quetz_harvester/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/quetz_runexports/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_tos/quetz_tos/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_mamba_solve/quetz_mamba_solve/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup() 4 | -------------------------------------------------------------------------------- /plugins/quetz_conda_suggest/quetz_conda_suggest/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/quetz_content_trust/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_harvester/conda-requirements.txt: -------------------------------------------------------------------------------- 1 | ruamel.yaml 2 | -------------------------------------------------------------------------------- /plugins/quetz_repodata_zchunk/conda-requirements.txt: -------------------------------------------------------------------------------- 1 | zchunk 2 | -------------------------------------------------------------------------------- /plugins/quetz_repodata_zchunk/quetz_repodata_zchunk/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_tos/quetz_tos/migrations/versions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_transmutation/quetz_transmutation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /quetz/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .view import init # noqa 2 | -------------------------------------------------------------------------------- /plugins/quetz_current_repodata/quetz_current_repodata/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_repodata_patching/quetz_repodata_patching/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/quetz_runexports/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /set_env_dev.sh: -------------------------------------------------------------------------------- 1 | export QUETZ_CONFIG_FILE="${PWD}/dev_config.toml" 2 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | extend-ignore = E203 4 | -------------------------------------------------------------------------------- /plugins/quetz_conda_suggest/quetz_conda_suggest/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/quetz_content_trust/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/quetz_runexports/migrations/versions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_transmutation/conda-requirements.txt: -------------------------------------------------------------------------------- 1 | conda-package-handling 2 | -------------------------------------------------------------------------------- /quetz/tests/data/dummy-plugin/quetz_dummyplugin-1.0-py3.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_conda_suggest/quetz_conda_suggest/migrations/versions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/quetz_content_trust/migrations/versions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/quetz_mamba_solve/conda-requirements.txt: -------------------------------------------------------------------------------- 1 | mamba 2 | conda-build >=3.20 3 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/conda-requirements.txt: -------------------------------------------------------------------------------- 1 | conda-content-trust 2 | libmambapy 3 | -------------------------------------------------------------------------------- /plugins/quetz_mamba_solve/tests/conftest.py: -------------------------------------------------------------------------------- 1 | pytest_plugins = "quetz.testing.fixtures" 2 | -------------------------------------------------------------------------------- /docker/grafana.env: -------------------------------------------------------------------------------- 1 | GF_SECURITY_ADMIN_USER=admin 2 | GF_SECURITY_ADMIN_PASSWORD=mysecretpassword -------------------------------------------------------------------------------- /docs/assets/quetz_header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/docs/assets/quetz_header.png -------------------------------------------------------------------------------- /quetz/tests/data/dummy-plugin/quetz_dummyplugin/jobs.py: -------------------------------------------------------------------------------- 1 | def dummy_job(package_version): 2 | pass 3 | -------------------------------------------------------------------------------- /quetz/basic_frontend/avatar.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/quetz/basic_frontend/avatar.jpg -------------------------------------------------------------------------------- /quetz/basic_frontend/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/quetz/basic_frontend/favicon.ico -------------------------------------------------------------------------------- /docs/source/_static/quetz_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/docs/source/_static/quetz_logo.png -------------------------------------------------------------------------------- /docker/graphana_datasources.yml: -------------------------------------------------------------------------------- 1 | datasources: 2 | - name: Prometheus 3 | type: prometheus 4 | url: prometheus:9090 5 | -------------------------------------------------------------------------------- /docs/source/_static/quetz_favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/docs/source/_static/quetz_favicon.ico -------------------------------------------------------------------------------- /quetz/_version.py: -------------------------------------------------------------------------------- 1 | version_info = (0, 10, 4, "", "") 2 | __version__ = ".".join(filter(lambda s: len(s) > 0, map(str, version_info))) 3 | -------------------------------------------------------------------------------- /quetz/tests/data/other-package-0.2-0.conda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/quetz/tests/data/other-package-0.2-0.conda -------------------------------------------------------------------------------- /quetz/tests/data/test-package-0.1-0.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/quetz/tests/data/test-package-0.1-0.tar.bz2 -------------------------------------------------------------------------------- /quetz/tests/data/test-package-0.2-0.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/quetz/tests/data/test-package-0.2-0.tar.bz2 -------------------------------------------------------------------------------- /quetz/tests/data/other-package-0.1-0.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/quetz/tests/data/other-package-0.1-0.tar.bz2 -------------------------------------------------------------------------------- /quetz/tests/data/other-package-0.2-0.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/quetz/tests/data/other-package-0.2-0.tar.bz2 -------------------------------------------------------------------------------- /docs/source/using/index.rst: -------------------------------------------------------------------------------- 1 | Using 2 | ===== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | basics 8 | mirroring 9 | 10 | 11 | -------------------------------------------------------------------------------- /quetz/tests/data/test-package-0.1-0_copy.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/quetz/tests/data/test-package-0.1-0_copy.tar.bz2 -------------------------------------------------------------------------------- /quetz/tests/data/dummy-plugin/quetz_dummyplugin-1.0-py3.egg-info/entry_points.txt: -------------------------------------------------------------------------------- 1 | [quetz.jobs] 2 | 3 | quetz-dummyplugin = quetz_dummyplugin.jobs 4 | -------------------------------------------------------------------------------- /_typos.toml: -------------------------------------------------------------------------------- 1 | [default.extend-words] 2 | fo = "fo" 3 | 4 | [files] 5 | extend-exclude = ["quetz/migrations/*.po", "quetz/tests/authentification/test_oauth.py"] 6 | -------------------------------------------------------------------------------- /quetz/authentication/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import BaseAuthenticationHandlers, BaseAuthenticator # noqa 2 | from .registry import AuthenticatorRegistry # noqa 3 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: "ubuntu-22.04" 5 | tools: 6 | python: "mambaforge-4.10" 7 | 8 | conda: 9 | environment: environment.yml 10 | -------------------------------------------------------------------------------- /docs/source/qeps/index.rst: -------------------------------------------------------------------------------- 1 | Quetz enhancement proposals (QEPs) 2 | ================================== 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | qep-001-user-permissions 8 | 9 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/quetz_content_trust/rest_models.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class SigningKey(BaseModel): 5 | channel: str 6 | private_key: str 7 | -------------------------------------------------------------------------------- /plugins/quetz_harvester/tests/data/xtensor-io-0.10.3-hb585cf6_0.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/quetz/HEAD/plugins/quetz_harvester/tests/data/xtensor-io-0.10.3-hb585cf6_0.tar.bz2 -------------------------------------------------------------------------------- /docker/postgres.env: -------------------------------------------------------------------------------- 1 | POSTGRES_DB=postgres 2 | POSTGRES_USER=postgres 3 | POSTGRES_PASSWORD=mysecretpassword 4 | QUETZ_SQLALCHEMY_DATABASE_URL="postgresql://postgres:mysecretpassword@database:5432/postgres" -------------------------------------------------------------------------------- /plugins/quetz_repodata_patching/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from pytest import fixture 2 | 3 | pytest_plugins = "quetz.testing.fixtures" 4 | 5 | 6 | @fixture 7 | def plugins(): 8 | return ["quetz-repodata_patching"] 9 | -------------------------------------------------------------------------------- /docker/Dockerfile.jupyterhub: -------------------------------------------------------------------------------- 1 | FROM jupyterhub/jupyterhub 2 | 3 | # openssl passwd -1 test 4 | RUN useradd testuser --no-log-init -u 1000 -p "$(openssl passwd -1 test)" 5 | COPY jupyterhub_config.py /srv/jupyterhub/jupyterhub_config.py 6 | -------------------------------------------------------------------------------- /quetz/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 QuantStack 2 | # Distributed under the terms of the Modified BSD License. 3 | try: 4 | import pluggy 5 | 6 | hookimpl = pluggy.HookimplMarker("quetz") 7 | except ImportError: 8 | pass 9 | -------------------------------------------------------------------------------- /quetz/tests/test_profiling.py: -------------------------------------------------------------------------------- 1 | def test_endpoint_profiling(client): 2 | response = client.get("/health/ready/?profile=true") 3 | assert response.status_code == 200 4 | assert "text/html" in response.headers["Content-Type"] 5 | -------------------------------------------------------------------------------- /plugins/quetz_mamba_solve/quetz_mamba_solve/rest_models.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from pydantic import BaseModel 4 | 5 | 6 | class SolveTask(BaseModel): 7 | channels: List[str] 8 | subdir: str 9 | spec: List[str] 10 | -------------------------------------------------------------------------------- /plugins/README.md: -------------------------------------------------------------------------------- 1 | # Quetz plugins 2 | 3 | Quetz plugins extend the functionality of quetz server by implementing hook functions that are called when requests are handled. 4 | 5 | A list of possible hooks is in the [`hooks.py`](../quetz/hooks.py) file. 6 | -------------------------------------------------------------------------------- /plugins/quetz_dictauthenticator/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | pytest_plugins = "quetz.testing.fixtures" 4 | 5 | 6 | @pytest.fixture 7 | def plugins(): 8 | # defines plugins to enable for testing 9 | return ["quetz-dictauthenticator"] 10 | -------------------------------------------------------------------------------- /plugins/quetz_repodata_zchunk/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | pytest_plugins = "quetz.testing.fixtures" 4 | 5 | 6 | @pytest.fixture 7 | def plugins(): 8 | # defines plugins to enable for testing 9 | return ["quetz-repodata_zchunk"] 10 | -------------------------------------------------------------------------------- /quetz/config.toml: -------------------------------------------------------------------------------- 1 | [github] 2 | # Register the app here: https://github.com/settings/applications/new 3 | client_id = "{}" 4 | client_secret = "{}" 5 | 6 | [sqlalchemy] 7 | database_url = "{}" 8 | 9 | [session] 10 | secret = "{}" 11 | https_only = {} 12 | -------------------------------------------------------------------------------- /plugins/quetz_transmutation/quetz_transmutation/rest_models.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from pydantic import BaseModel, Field 4 | 5 | 6 | class PackageSpec(BaseModel): 7 | package_spec: Optional[str] = Field(None, title="package version specification") 8 | -------------------------------------------------------------------------------- /quetz/exceptions.py: -------------------------------------------------------------------------------- 1 | class PackageError(Exception): 2 | def __init__(self, detail: str) -> None: 3 | self.detail = detail 4 | 5 | def __repr__(self) -> str: 6 | class_name = self.__class__.__name__ 7 | return f"{class_name}(detail={self.detail!r})" 8 | -------------------------------------------------------------------------------- /plugins/quetz_mamba_solve/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="quetz-mamba_solve", 5 | install_requires=["conda-build>=3.20"], 6 | entry_points={"quetz": ["quetz-mamba_solve = quetz_mamba_solve.main"]}, 7 | packages=["quetz_mamba_solve"], 8 | ) 9 | -------------------------------------------------------------------------------- /plugins/quetz_repodata_zchunk/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="quetz-repodata_zchunk", 5 | install_requires=[], 6 | entry_points={"quetz": ["quetz-repodata_zchunk = quetz_repodata_zchunk.main"]}, 7 | packages=["quetz_repodata_zchunk"], 8 | ) 9 | -------------------------------------------------------------------------------- /plugins/quetz_current_repodata/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="quetz-current_repodata", 5 | install_requires=[], 6 | entry_points={"quetz": ["quetz-current_repodata = quetz_current_repodata.main"]}, 7 | packages=["quetz_current_repodata"], 8 | ) 9 | -------------------------------------------------------------------------------- /plugins/quetz_repodata_patching/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="quetz-repodata_patching", 5 | install_requires=[], 6 | entry_points={"quetz": ["quetz-repodata_patching = quetz_repodata_patching.main"]}, 7 | packages=["quetz_repodata_patching"], 8 | ) 9 | -------------------------------------------------------------------------------- /test-cli-client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | curl \ 3 | -v \ 4 | -F 'files=@xtensor/linux-64/xtensor-0.16.1-0.tar.bz2' \ 5 | -F 'files=@xtensor/osx-64/xtensor-0.16.1-0.tar.bz2' \ 6 | -H 'X-API-Key: E_KaBFstCKI9hTdPM7DQq56GglRHf2HW7tQtq6si370' \ 7 | http://localhost:8000/api/channels/channel0/packages/xtensor/files/ 8 | -------------------------------------------------------------------------------- /plugins/quetz_mamba_solve/quetz_mamba_solve/main.py: -------------------------------------------------------------------------------- 1 | import quetz 2 | 3 | from .api import router 4 | 5 | 6 | @quetz.hookimpl 7 | def register_router(): 8 | return router 9 | 10 | 11 | @quetz.hookimpl 12 | def post_add_package_version(version, condainfo): 13 | # Implement your logic 14 | pass 15 | -------------------------------------------------------------------------------- /plugins/quetz_transmutation/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="quetz-transmutation", 5 | install_requires=[], 6 | version="0.1.0", 7 | entry_points={ 8 | "quetz.jobs": ["quetz-transmutation = quetz_transmutation.jobs"], 9 | }, 10 | packages=["quetz_transmutation"], 11 | ) 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | files 3 | quetz.sqlite 4 | /channels/ 5 | xtensor/ 6 | build 7 | dist/ 8 | .jupyter_releaser_checkout/ 9 | 10 | # Build files 11 | *.egg-info 12 | 13 | # Local setup 14 | test_quetz 15 | 16 | # Logs 17 | *.log 18 | 19 | # IDE 20 | .idea/ 21 | .vscode/ 22 | 23 | .env 24 | .envrc 25 | .venv 26 | 27 | # OS 28 | .DS_Store 29 | -------------------------------------------------------------------------------- /.github/workflows/enforce-label.yml: -------------------------------------------------------------------------------- 1 | name: Enforce PR label 2 | 3 | on: 4 | pull_request: 5 | types: [labeled, unlabeled, opened, edited, synchronize] 6 | 7 | jobs: 8 | enforce-label: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: enforce-triage-label 12 | uses: jupyterlab/maintainer-tools/.github/actions/enforce-label@v1 13 | -------------------------------------------------------------------------------- /plugins/quetz_googleiap/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | plugin_name = "quetz-googleiap" 4 | 5 | setup( 6 | name=plugin_name, 7 | install_requires=[], 8 | entry_points={ 9 | "quetz.middlewares": [f"{plugin_name} = quetz_googleiap.middleware"], 10 | }, 11 | packages=[ 12 | "quetz_googleiap", 13 | ], 14 | ) 15 | -------------------------------------------------------------------------------- /docker/wait-for-postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # wait-for-postgres.sh 3 | 4 | set -e 5 | 6 | host="$1" 7 | shift 8 | cmd="$@" 9 | 10 | until PGPASSWORD=$POSTGRES_PASSWORD psql -h "$host" -U "postgres" -c '\q'; do 11 | >&2 echo "Postgres is unavailable - sleeping" 12 | sleep 1 13 | done 14 | 15 | >&2 echo "Postgres is up - executing command" 16 | exec $cmd 17 | -------------------------------------------------------------------------------- /plugins/quetz_dictauthenticator/tests/test_authenticator.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.parametrize( 5 | "config_extra", ['[dictauthenticator]\nusers=["user:password"]'] 6 | ) 7 | def test_authenticator(client): 8 | response = client.get("/auth/dict/login") 9 | assert "password" in response.text 10 | assert "username" in response.text 11 | -------------------------------------------------------------------------------- /quetz/tests/test_health.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | 3 | 4 | def test_get_health_ready(client: TestClient): 5 | response = client.get("/health/ready") 6 | assert response.status_code == 200 7 | 8 | 9 | def test_get_health_live(client: TestClient): 10 | response = client.get("/health/live") 11 | assert response.status_code == 200 12 | -------------------------------------------------------------------------------- /plugins/quetz_dictauthenticator/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="quetz-dictauthenticator", 5 | install_requires=[], 6 | entry_points={ 7 | "quetz.authenticator": [ 8 | "dictauthenticator = quetz_dictauthenticator:DictionaryAuthenticator" 9 | ] 10 | }, 11 | packages=["quetz_dictauthenticator"], 12 | ) 13 | -------------------------------------------------------------------------------- /quetz/tasks/cleanup.py: -------------------------------------------------------------------------------- 1 | from quetz.dao import Dao 2 | from quetz.pkgstores import PackageStore 3 | 4 | 5 | def cleanup_channel_db(dao: Dao, channel_name: str, dry_run: bool): 6 | dao.cleanup_channel_db(channel_name, None, dry_run) 7 | 8 | 9 | def cleanup_temp_files(pkgstore: PackageStore, channel_name: str, dry_run: bool): 10 | pkgstore.cleanup_temp_files(channel_name, dry_run) 11 | -------------------------------------------------------------------------------- /plugins/quetz_googleiap/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | pytest_plugins = "quetz.testing.fixtures" 4 | 5 | 6 | @pytest.fixture 7 | def plugins(): 8 | # defines plugins to enable for testing 9 | return ["quetz-googleiap"] 10 | 11 | 12 | @pytest.fixture 13 | def sqlite_in_memory(): 14 | # use sqlite on disk so that we can modify it in a different process 15 | return False 16 | -------------------------------------------------------------------------------- /quetz_db_ext/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM postgres:13 2 | 3 | RUN apt-get update && \ 4 | apt-get install postgresql-server-dev-13 -y 5 | 6 | RUN apt-get install build-essential -y 7 | ADD . /quetz_db_ext/ 8 | 9 | RUN cd /quetz_db_ext/ && \ 10 | /usr/bin/cc -fPIC -c conda.c && \ 11 | /usr/bin/cc -fPIC -I /usr/include/postgresql/13/server/ -c quetz_pg.c && \ 12 | /usr/bin/cc -shared -o quetz_pg.so conda.o quetz_pg.o -------------------------------------------------------------------------------- /plugins/quetz_harvester/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="quetz-harvester", 5 | # you should install libcflib using 6 | # $ pip install git+https://git@github.com/regro/libcflib@master --no-deps 7 | install_requires=["libcflib"], 8 | entry_points={ 9 | "quetz.jobs": ["quetz-harvester=quetz_harvester.jobs"], 10 | }, 11 | packages=["quetz_harvester"], 12 | ) 13 | -------------------------------------------------------------------------------- /download-test-package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p xtensor/osx-64 4 | mkdir -p xtensor/linux-64 5 | mkdir -p xtensor/osx-arm64 6 | wget https://conda.anaconda.org/conda-forge/osx-64/xtensor-0.24.3-h1b54a9f_1.tar.bz2 -P xtensor/osx-64/ 7 | wget https://conda.anaconda.org/conda-forge/linux-64/xtensor-0.24.3-h924138e_1.tar.bz2 -P xtensor/linux-64/ 8 | wget https://conda.anaconda.org/conda-forge/osx-arm64/xtensor-0.24.3-hf86a087_1.tar.bz2 -P xtensor/osx-arm64/ 9 | -------------------------------------------------------------------------------- /quetz/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Codethink Ltd 2 | # Distributed under the terms of the Modified BSD License. 3 | 4 | 5 | class QuetzError(Exception): 6 | pass 7 | 8 | 9 | class DBError(QuetzError): 10 | pass 11 | 12 | 13 | class ConfigError(QuetzError): 14 | pass 15 | 16 | 17 | class ValidationError(QuetzError): 18 | pass 19 | 20 | 21 | class QuotaError(ValidationError): 22 | pass 23 | 24 | 25 | class TaskError(QuetzError): 26 | pass 27 | -------------------------------------------------------------------------------- /docs/source/deploying/index.rst: -------------------------------------------------------------------------------- 1 | Deploying 2 | ========= 3 | 4 | There are many options for deploying a package repository with Quetz. You can deploy Quetz either locally for 5 | testing, on a production server or in the cloud. The process can be tailored to your needs and existing 6 | infrastructure. 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | configuration 12 | authenticators 13 | database 14 | migrations 15 | frontend 16 | workers 17 | nginx 18 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/docs/test_corrupted_key_mgr_metadata.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pathlib import Path 3 | 4 | if __name__ == "__main__": 5 | key_mgr_file = Path.cwd() / "test_quetz/channels/channel0/key_mgr.json" 6 | 7 | j = dict() 8 | with open(key_mgr_file, "r") as f: 9 | j = json.load(f) 10 | 11 | j["signed"]["delegations"]["pkg_mgr"]["pubkeys"] = ["some_new_key"] 12 | with open(key_mgr_file, "w") as f: 13 | json.dump(j, f) 14 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/docs/test_corrupted_key_mgr_sigs.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pathlib import Path 3 | 4 | if __name__ == "__main__": 5 | key_mgr_file = Path.cwd() / "test_quetz/channels/channel0/key_mgr.json" 6 | 7 | j = dict() 8 | with open(key_mgr_file, "r") as f: 9 | j = json.load(f) 10 | 11 | j["signatures"] = {"not_trusted_key_id": {"signature": "not_trusted_signature"}} 12 | with open(key_mgr_file, "w") as f: 13 | json.dump(j, f) 14 | -------------------------------------------------------------------------------- /plugins/quetz_tos/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="quetz-tos", 5 | install_requires="quetz-server", 6 | entry_points={ 7 | "quetz": ["quetz-tos = quetz_tos.main"], 8 | "quetz.models": ["quetz-tos = quetz_tos.db_models"], 9 | "quetz.migrations": ["quetz-tos = quetz_tos.migrations"], 10 | }, 11 | packages=[ 12 | "quetz_tos", 13 | "quetz_tos.migrations", 14 | "quetz_tos.migrations.versions", 15 | ], 16 | ) 17 | -------------------------------------------------------------------------------- /.github/workflows/docker-build.yml: -------------------------------------------------------------------------------- 1 | name: test docker build 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | docker: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v2 12 | 13 | - name: Set up Docker Buildx 14 | uses: docker/setup-buildx-action@v1 15 | 16 | - name: Build 17 | uses: docker/build-push-action@v2 18 | with: 19 | context: . 20 | load: true 21 | tags: mambaorg/quetz 22 | -------------------------------------------------------------------------------- /quetz/migrations/versions/ea6eba9a9ffc_merge_ebe550f9fbbe_and_b9886d9cadb0.py: -------------------------------------------------------------------------------- 1 | """merge ebe550f9fbbe and b9886d9cadb0 2 | 3 | Revision ID: ea6eba9a9ffc 4 | Revises: ebe550f9fbbe, b9886d9cadb0 5 | Create Date: 2021-02-09 16:40:52.637115 6 | 7 | """ 8 | 9 | # revision identifiers, used by Alembic. 10 | revision = 'ea6eba9a9ffc' 11 | down_revision = ('ebe550f9fbbe', 'b9886d9cadb0') 12 | branch_labels = None 13 | depends_on = None 14 | 15 | 16 | def upgrade(): 17 | pass 18 | 19 | 20 | def downgrade(): 21 | pass 22 | -------------------------------------------------------------------------------- /docs/source/deploying/frontend.rst: -------------------------------------------------------------------------------- 1 | Frontend 2 | ######## 3 | 4 | Quetz comes with a initial frontend implementation. It can be found in quetz_frontend. 5 | To build it, one needs to install: 6 | 7 | .. code:: bash 8 | 9 | mamba install 'nodejs>=14' 10 | cd quetz_frontend 11 | npm install 12 | npm run build 13 | # for development 14 | npm run watch 15 | 16 | This will build the javascript files and place them in ``/quetz_frontend/dist/`` from where they are automatically picked up by the quetz server. 17 | 18 | 19 | -------------------------------------------------------------------------------- /quetz/tasks/assertions.py: -------------------------------------------------------------------------------- 1 | def can_channel_synchronize(channel): 2 | return channel.mirror_channel_url and (channel.mirror_mode == "mirror") 3 | 4 | 5 | def can_channel_synchronize_metrics(channel): 6 | return not channel.mirror_channel_url 7 | 8 | 9 | def can_channel_generate_indexes(channel): 10 | return True 11 | 12 | 13 | def can_channel_validate_package_cache(channel): 14 | return True 15 | 16 | 17 | def can_channel_reindex(channel): 18 | return True 19 | 20 | 21 | def can_cleanup(channel): 22 | return True 23 | -------------------------------------------------------------------------------- /docker/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s # By default, scrape targets every 15 seconds. 3 | # A scrape configuration containing exactly one endpoint to scrape: 4 | # Here it's Prometheus itself. 5 | scrape_configs: 6 | # The job name is added as a label `job=` to any timeseries scraped from this config. 7 | - job_name: "quetz" 8 | metrics_path: /metricsp 9 | 10 | # Override the global default and scrape targets from this job every 5 seconds. 11 | scrape_interval: 5s 12 | 13 | static_configs: 14 | - targets: ["web:8000"] 15 | -------------------------------------------------------------------------------- /quetz_db_ext/conda.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019, SUSE LLC 3 | * 4 | * This program is licensed under the BSD license, read LIBSOLV_LICENSE.txt 5 | * for further information 6 | */ 7 | 8 | int pool_evrcmp_conda_int(const char *evr1, const char *evr1e, const char *evr2, const char *evr2e, int startswith); 9 | int solvable_conda_matchversion_single(const char* evr, const char* version, size_t versionlen); 10 | int solvable_conda_matchversion_rec(const char* evr, const char** versionpp, char* versionend); 11 | int solvable_conda_matchversion(const char* evr, const char* version); 12 | -------------------------------------------------------------------------------- /plugins/quetz_transmutation/README.md: -------------------------------------------------------------------------------- 1 | # quetz_transmutation plugin 2 | 3 | This is a plugin to use with the [quetz](https://github.com/mamba-org/quetz) package server. 4 | 5 | ## Installing 6 | 7 | To install use: 8 | 9 | ``` 10 | pip install . 11 | ``` 12 | 13 | ## Using 14 | 15 | Run the transumtation job on all packages (server wide) using the following RESTful request: 16 | 17 | ``` 18 | QUETZ_API_KEY=... 19 | curl -X POST localhost:8000/api/jobs \ 20 | -H "X-api-key: ${QUETZ_API_KEY}" \ 21 | -d '{"items_spec": "*", "manifest": "quetz-transmutation:transmutation"}' 22 | ``` 23 | -------------------------------------------------------------------------------- /plugins/quetz_conda_suggest/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="quetz-conda_suggest", 5 | install_requires=[], 6 | entry_points={ 7 | "quetz": ["quetz-conda_suggest = quetz_conda_suggest.main"], 8 | "quetz.migrations": ["quetz-conda_suggest = quetz_conda_suggest.migrations"], 9 | "quetz.models": ["quetz-conda_suggest = quetz_conda_suggest.db_models"], 10 | }, 11 | packages=[ 12 | "quetz_conda_suggest", 13 | "quetz_conda_suggest.migrations", 14 | "quetz_conda_suggest.migrations.versions", 15 | ], 16 | ) 17 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | plugin_name = "quetz-runexports" 4 | 5 | setup( 6 | name=plugin_name, 7 | install_requires=[], 8 | entry_points={ 9 | "quetz": [f"{plugin_name} = quetz_runexports.main"], 10 | "quetz.migrations": [f"{plugin_name} = quetz_runexports.migrations"], 11 | "quetz.models": [f"{plugin_name} = quetz_runexports.db_models"], 12 | }, 13 | packages=[ 14 | "quetz_runexports", 15 | "quetz_runexports.migrations", 16 | "quetz_runexports.migrations.versions", 17 | ], 18 | ) 19 | -------------------------------------------------------------------------------- /quetz/tests/test_docs.py: -------------------------------------------------------------------------------- 1 | def test_docs_endpoint(client): 2 | response = client.get("/docs") 3 | assert response.status_code == 200 4 | assert "text/html" in response.headers["Content-Type"] 5 | 6 | 7 | def test_openapi_endpoint(client): 8 | response = client.get("/openapi.json") 9 | assert response.status_code == 200 10 | assert "application/json" in response.headers["Content-Type"] 11 | 12 | 13 | def test_redoc_endpoint(client): 14 | response = client.get("/redoc") 15 | assert response.status_code == 200 16 | assert "text/html" in response.headers["Content-Type"] 17 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="quetz-content_trust", 5 | install_requires=["conda_content_trust"], 6 | entry_points={ 7 | "quetz": ["quetz-content_trust = quetz_content_trust.main"], 8 | "quetz.migrations": ["quetz-content_trust = quetz_content_trust.migrations"], 9 | "quetz.models": ["quetz-content_trust = quetz_content_trust.db_models"], 10 | }, 11 | packages=[ 12 | "quetz_content_trust", 13 | "quetz_content_trust.migrations", 14 | "quetz_content_trust.migrations.versions", 15 | ], 16 | ) 17 | -------------------------------------------------------------------------------- /quetz/tests/test_db_models.py: -------------------------------------------------------------------------------- 1 | """Tests for the database models""" 2 | # Copyright 2020 QuantStack 3 | # Distributed under the terms of the Modified BSD License. 4 | 5 | import uuid 6 | 7 | from quetz.db_models import User 8 | 9 | 10 | def test_user(db): 11 | user = User(id=uuid.uuid4().bytes, username="paul") 12 | db.add(user) 13 | db.commit() 14 | 15 | assert len(db.query(User).all()) == 1 16 | 17 | found = User.find(db, "paul") 18 | assert found.username == user.username 19 | found = User.find(db, "dave") 20 | assert found is None 21 | 22 | db.delete(user) 23 | db.commit() 24 | -------------------------------------------------------------------------------- /quetz/migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/quetz_content_trust/repo_signer.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | import conda_content_trust.signing as cct_signing 5 | 6 | 7 | class RepoSigner: 8 | def sign_repodata(self, repodata_fn, private_key): 9 | final_fn = self.in_folder / "repodata.json" 10 | 11 | cct_signing.sign_all_in_repodata(str(final_fn), private_key) 12 | 13 | def __init__(self, in_folder, private_key): 14 | self.in_folder = Path(in_folder).resolve() 15 | 16 | f = os.path.join(self.in_folder, "repodata.json") 17 | if os.path.isfile(f): 18 | self.sign_repodata(Path(f), private_key) 19 | -------------------------------------------------------------------------------- /plugins/quetz_conda_suggest/quetz_conda_suggest/db_models.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, ForeignKey, String 2 | from sqlalchemy.orm import backref, relationship 3 | 4 | from quetz.db_models import UUID, Base 5 | 6 | 7 | class CondaSuggestMetadata(Base): 8 | __tablename__ = "quetz_conda_suggest_metadata" 9 | 10 | version_id = Column(UUID, ForeignKey("package_versions.id"), primary_key=True) 11 | package_version = relationship( 12 | "PackageVersion", 13 | backref=backref( 14 | "binfiles", 15 | uselist=False, 16 | cascade="delete,all", 17 | ), 18 | ) 19 | data = Column(String) 20 | -------------------------------------------------------------------------------- /dev_config.toml: -------------------------------------------------------------------------------- 1 | [github] 2 | # Register the app here: https://github.com/settings/applications/new 3 | # Use a callback URL like `http://localhost:8000/auth/github/authorize` 4 | client_id = "37cdbb80c7733e2a1831" 5 | client_secret = "75e648e981545902ab7802de94e2f2707c8e0ff8" 6 | 7 | [sqlalchemy] 8 | database_url = "sqlite:///./quetz.sqlite" 9 | 10 | [session] 11 | # openssl rand -hex 32 12 | secret = "b72376b88e6f249cb0921052ea8a092381ca17fd8bb0caf4d847e337b3d34cf8" 13 | https_only = false 14 | 15 | [logging] 16 | level = "DEBUG" 17 | file = "quetz.log" 18 | 19 | [users] 20 | admins = ["dummy:alice"] 21 | 22 | [profiling] 23 | enable_sampling = false 24 | 25 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/quetz_runexports/db_models.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, ForeignKey, String 2 | from sqlalchemy.orm import backref, relationship 3 | 4 | from quetz.db_models import UUID, Base 5 | 6 | 7 | class PackageVersionMetadata(Base): 8 | __tablename__ = "quetz_runexports_package_version_metadata" 9 | 10 | version_id = Column(UUID, ForeignKey("package_versions.id"), primary_key=True) 11 | package_version = relationship( 12 | "PackageVersion", 13 | backref=backref( 14 | "runexports", 15 | uselist=False, 16 | cascade="delete,all", 17 | ), 18 | ) 19 | data = Column(String) 20 | -------------------------------------------------------------------------------- /plugins/quetz_current_repodata/README.md: -------------------------------------------------------------------------------- 1 | # quetz_current_repodata plugin 2 | 3 | This is a plugin to use with the [quetz](https://github.com/mamba-org/quetz) package server. 4 | 5 | It generates `current_repodata.json` file specific to a particular channel (such as `conda-forge`) and a platform (such as `linux-64`). It is a trimmed version of `repodata.json` which contains the latest versions of each package. For more information, refer https://docs.conda.io/projects/conda-build/en/latest/concepts/generating-index.html#trimming-to-current-repodata 6 | 7 | ## Installing 8 | 9 | To install use: 10 | 11 | ```bash 12 | quetz plugin install plugins/quetz_current_repodata 13 | ``` 14 | -------------------------------------------------------------------------------- /quetz_db_ext/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.2) 2 | 3 | project(quetz-db-ext) 4 | 5 | find_package(SQLite3) 6 | find_package(PostgreSQL) 7 | 8 | option(POSTGRES_13_DOCKER OFF) 9 | 10 | if (SQLite3_FOUND) 11 | add_library(quetz_sqlite SHARED conda.c quetz_sqlite.c) 12 | target_link_libraries(quetz_sqlite SQLite::SQLite3) 13 | endif() 14 | 15 | if (PostgreSQL_FOUND) 16 | add_library(quetz_pg SHARED conda.c quetz_pg.c) 17 | target_link_libraries(quetz_pg PostgreSQL::PostgreSQL) 18 | endif() 19 | 20 | if (POSTGRES_13_DOCKER) 21 | include_directories(/usr/include/postgresql/13/server/) 22 | add_library(quetz_pg SHARED conda.c quetz_pg.c) 23 | target_link_libraries(quetz_pg pq) 24 | endif() -------------------------------------------------------------------------------- /plugins/quetz_mamba_solve/README.md: -------------------------------------------------------------------------------- 1 | # quetz_mamba_solve plugin 2 | 3 | This is a plugin to use with the [quetz](https://github.com/mamba-org/quetz) package server. 4 | 5 | It takes `a list of channels`, a `subdir` and a `spec` as input and generates a specification file (similar to what `conda list --explicit` would produce). The contents of this file can be used to create an identical environment on a machine with the same platform. This can be done using the command `conda create --name myenv --file spec-file.txt` where `spec-file.txt` is the file containing the response from this API endpoint. 6 | 7 | ## Installing 8 | 9 | To install use: 10 | 11 | ```bash 12 | quetz plugin install plugins/quetz_mamba_solve 13 | ``` 14 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | 3 | on: [push, pull_request] 4 | 5 | defaults: 6 | run: 7 | shell: bash -el {0} 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: mamba-org/setup-micromamba@v1 15 | with: 16 | environment-file: environment.yml 17 | - name: Add micromamba to GITHUB_PATH 18 | run: echo "${HOME}/micromamba-bin" >> "$GITHUB_PATH" 19 | - run: ln -s "${CONDA_PREFIX}" .venv # Necessary for pyright. 20 | - uses: pre-commit/action@v3.0.0 21 | with: 22 | extra_args: --all-files --show-diff-on-failure 23 | env: 24 | PRE_COMMIT_USE_MICROMAMBA: 1 25 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /plugins/quetz_mamba_solve/tests/test_mamba_solve.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture 5 | def plugins(): 6 | return ["quetz-mamba_solve"] 7 | 8 | 9 | def test_mamba_solve_endpoint(client): 10 | response = client.post( 11 | "/api/mamba/solve", 12 | json={ 13 | "channels": ["conda-forge"], 14 | "subdir": "osx-64", 15 | "spec": ["xtensor>=0.21.0"], 16 | }, 17 | ) 18 | assert response.status_code == 200 19 | assert b"@EXPLICIT" in response.content 20 | assert response.content.startswith(b"# platform: osx-64") 21 | 22 | assert b"conda-forge/osx-64/libcxx-" in response.content 23 | assert b"conda-forge/osx-64/xtensor-" in response.content 24 | -------------------------------------------------------------------------------- /quetz/jobs/handlers.py: -------------------------------------------------------------------------------- 1 | from quetz.metrics import tasks as metrics_tasks 2 | from quetz.tasks import cleanup, indexing, mirror, reindexing 3 | 4 | JOB_HANDLERS = { 5 | "synchronize": mirror.synchronize_packages, 6 | "synchronize_repodata": mirror.synchronize_packages, 7 | "validate_packages": indexing.validate_packages, 8 | "generate_indexes": indexing.update_indexes, 9 | "reindex": reindexing.reindex_packages_from_store, 10 | "synchronize_metrics": metrics_tasks.synchronize_metrics_from_mirrors, 11 | "pkgstore_cleanup": cleanup.cleanup_channel_db, 12 | "db_cleanup": cleanup.cleanup_temp_files, 13 | "pkgstore_cleanup_dry_run": cleanup.cleanup_channel_db, 14 | "db_cleanup_dry_run": cleanup.cleanup_temp_files, 15 | } 16 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/tests/data/key_mgr.json: -------------------------------------------------------------------------------- 1 | { 2 | "signatures": { 3 | "013ddd714962866d12ba5bae273f14d48c89cf0773dee2dbf6d4561e521c83f7": { 4 | "signature": "20d8728ae8ba212e6229f9a69b3de14cd747fcd20cfaa1c5d39111cc6aad7a94036187a6c49e13a531d08c282a0d11b07c276d0f0773dc5344f54a14fb0d7700" 5 | } 6 | }, 7 | "signed": { 8 | "delegations": { 9 | "pkg_mgr": { 10 | "pubkeys": [ 11 | "f46b5a7caa43640744186564c098955147daa8bac4443887bc64d8bfee3d3569" 12 | ], 13 | "threshold": 1 14 | } 15 | }, 16 | "expiration": "2022-06-01T19:16:49Z", 17 | "metadata_spec_version": "0.6.0", 18 | "timestamp": "2021-06-01T19:16:49Z", 19 | "type": "key_mgr", 20 | "version": 1 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /plugins/quetz_mamba_solve/quetz_mamba_solve/utils.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from functools import lru_cache, wraps 3 | 4 | 5 | def timed_lru_cache(hours: int, maxsize: int = 128): 6 | def wrapper_cache(func): 7 | func = lru_cache(maxsize=maxsize)(func) 8 | func.lifetime = timedelta(hours=hours) 9 | func.expiration = datetime.utcnow() + func.lifetime 10 | 11 | @wraps(func) 12 | def wrapped_func(*args, **kwargs): 13 | if datetime.utcnow() >= func.expiration: 14 | func.cache_clear() 15 | func.expiration = datetime.utcnow() + func.lifetime 16 | return func(*args, **kwargs) 17 | 18 | return wrapped_func 19 | 20 | return wrapper_cache 21 | -------------------------------------------------------------------------------- /quetz/migrations/versions/0a0ab48887ab_adding_function_args_to_job_spec.py: -------------------------------------------------------------------------------- 1 | """adding function args to job spec 2 | 3 | Revision ID: 0a0ab48887ab 4 | Revises: 3c3288034362 5 | Create Date: 2021-02-24 16:58:56.886842 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = '0a0ab48887ab' 13 | down_revision = '3c3288034362' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | with op.batch_alter_table('jobs', schema=None) as batch_op: 20 | batch_op.add_column(sa.Column('extra_args', sa.String(), nullable=True)) 21 | 22 | 23 | def downgrade(): 24 | with op.batch_alter_table('jobs', schema=None) as batch_op: 25 | batch_op.drop_column('extra_args') 26 | -------------------------------------------------------------------------------- /docker/docker_config.toml: -------------------------------------------------------------------------------- 1 | [sqlalchemy] 2 | database_url = "" 3 | 4 | [session] 5 | # openssl rand -hex 32 6 | secret = "b72376b88e6f249cb0921052ea8a092381ca17fd8bb0caf4d847e337b3d34cf8" 7 | https_only = false 8 | 9 | [logging] 10 | level = "DEBUG" 11 | file = "quetz.log" 12 | 13 | [users] 14 | admins = [] 15 | maintainers = [] 16 | users = [] 17 | 18 | [pamauthenticator] 19 | provider = "pam" 20 | admin_groups = ["root", "quetz"] 21 | 22 | [jupyterhubauthenticator] 23 | client_id = "quetz_client" 24 | client_secret = "super-secret" 25 | access_token_url = "http://jupyterhub:8000/hub/api/oauth2/token" 26 | authorize_url = "http://localhost:8001/hub/api/oauth2/authorize" 27 | api_base_url = "http://jupyterhub:8000/hub/api/" 28 | 29 | [local_store] 30 | redirect_enabled = true 31 | -------------------------------------------------------------------------------- /quetz/tests/test_database.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from quetz.database import sanitize_db_url 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "input_url,expected_output_url", 8 | ( 9 | ( 10 | "sqlite:///./quetz.sqlite", 11 | "sqlite:///./quetz.sqlite", 12 | ), # No password, no effect 13 | ( 14 | "postgresql+psycopg2://postgres_user:postgres_password@localhost:5432/postgres", # noqa: E501 15 | "postgresql+psycopg2://postgres_user:***@localhost:5432/postgres", 16 | ), 17 | ("A:B@C:1111/DB", "A:***@C:1111/DB"), 18 | ("THISISNOTAURL", "THISISNOTAURL"), 19 | ), 20 | ) 21 | def test_sanitize_db_url(input_url, expected_output_url): 22 | assert sanitize_db_url(input_url) == expected_output_url 23 | -------------------------------------------------------------------------------- /.github/workflows/sphinx.yml: -------------------------------------------------------------------------------- 1 | name: "sphinx docs check" 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | docs: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | fail-fast: false 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: "install dependencies" 19 | uses: mamba-org/setup-micromamba@v1 20 | with: 21 | environment-file: environment.yml 22 | - name: "build docs" 23 | shell: bash -l {0} 24 | run: | 25 | cd docs 26 | make html SPHINXOPTS="-W" 27 | - uses: actions/upload-artifact@v1 28 | name: "upload artifacts" 29 | with: 30 | name: DocumentationHTML 31 | path: docs/build/html 32 | -------------------------------------------------------------------------------- /quetz_db_ext/quetz_pg.c: -------------------------------------------------------------------------------- 1 | #include "postgres.h" 2 | #include "utils/builtins.h" 3 | #include "conda.h" 4 | 5 | #ifdef PG_MODULE_MAGIC 6 | PG_MODULE_MAGIC; 7 | #endif 8 | 9 | /* 10 | 11 | Insert into postgres with 12 | CREATE FUNCTION version_compare(varchar, varchar) 13 | RETURNS boolean 14 | AS '/home/wolfv/Programs/quetz/quetz_db_ext/build/libquetz_pg.so' 15 | LANGUAGE 'c' 16 | \g 17 | 18 | Delete with 19 | 20 | DROP FUNCTION version_compare(varchar, varchar) 21 | 22 | */ 23 | PG_FUNCTION_INFO_V1(version_compare); 24 | 25 | Datum version_compare(PG_FUNCTION_ARGS) 26 | { 27 | const char* lhs = text_to_cstring(PG_GETARG_VARCHAR_P(0)); 28 | const char* rhs = text_to_cstring(PG_GETARG_VARCHAR_P(1)); 29 | PG_RETURN_BOOL(solvable_conda_matchversion(lhs, rhs)); 30 | } 31 | 32 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build conda environment 2 | FROM condaforge/mambaforge:4.9.2-7 as conda 3 | 4 | COPY environment.yml /tmp/environment.yml 5 | RUN CONDA_COPY_ALWAYS=true mamba env create -p /env -f /tmp/environment.yml \ 6 | && conda clean -afy 7 | 8 | COPY . /code 9 | RUN conda run -p /env python -m pip install --no-deps /code 10 | 11 | # Create image 12 | FROM debian:buster-slim 13 | 14 | ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 15 | 16 | COPY --from=conda /env /env 17 | 18 | # Set WORKDIR to /tmp because quetz always creates a quetz.log file 19 | # in the current directory 20 | WORKDIR /tmp 21 | ENV PATH /env/bin:$PATH 22 | EXPOSE 8000 23 | 24 | # The following command assumes that a deployment has been initialized 25 | # in the /quetz-deployment volume 26 | CMD ["quetz", "start", "/quetz-deployment", "--host", "0.0.0.0", "--port", "8000"] 27 | -------------------------------------------------------------------------------- /plugins/quetz_googleiap/tests/test_googleiap.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.parametrize( 5 | "config_extra", ['[googleiam]\nserver_admin_emails=["test@tester.com"]'] 6 | ) 7 | def test_authentication(client, db): 8 | response = client.get("/api/me") 9 | assert response.status_code == 401 10 | 11 | # add headers 12 | headers = { 13 | "X-Goog-Authenticated-User-Email": "accounts.google.com:someone@tester.com", 14 | "X-Goog-Authenticated-User-Id": "accounts.google.com:someone@tester.com", 15 | } 16 | 17 | response = client.get("/api/me", headers=headers) 18 | assert response.status_code == 200 19 | 20 | # # check if channel was created 21 | # response = client.get("/api/channels", headers=headers) 22 | # assert response.status_code == 200 23 | # assert response.json()['channels'][0]['name'] == 'someone' 24 | -------------------------------------------------------------------------------- /quetz/migrations/versions/cd404ed93cc0_add_per_channel_ttl.py: -------------------------------------------------------------------------------- 1 | """add per channel TTL 2 | 3 | Revision ID: cd404ed93cc0 4 | Revises: 303ff70c27fc 5 | Create Date: 2021-01-07 20:12:23.527311 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'cd404ed93cc0' 13 | down_revision = '303ff70c27fc' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | op.add_column( 21 | 'channels', 22 | sa.Column('ttl', sa.Integer(), server_default='36000', nullable=False), 23 | ) 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade(): 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | op.drop_column('channels', 'ttl') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /quetz/metrics/view.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from prometheus_client import ( 4 | CONTENT_TYPE_LATEST, 5 | REGISTRY, 6 | CollectorRegistry, 7 | generate_latest, 8 | ) 9 | from prometheus_client.multiprocess import MultiProcessCollector 10 | from starlette.requests import Request 11 | from starlette.responses import Response 12 | from starlette.types import ASGIApp 13 | 14 | from .middleware import PrometheusMiddleware 15 | 16 | 17 | def metrics(request: Request) -> Response: 18 | if "prometheus_multiproc_dir" in os.environ: 19 | registry = CollectorRegistry() 20 | MultiProcessCollector(registry) 21 | else: 22 | registry = REGISTRY 23 | 24 | return Response(generate_latest(registry), media_type=CONTENT_TYPE_LATEST) 25 | 26 | 27 | def init(app: ASGIApp): 28 | app.add_middleware(PrometheusMiddleware) 29 | app.add_route("/metricsp", metrics) 30 | -------------------------------------------------------------------------------- /plugins/quetz_dictauthenticator/README.md: -------------------------------------------------------------------------------- 1 | ## DictionaryAuthenticator 2 | 3 | This is a **demo** of creating new authenticator classes. 4 | 5 | NOT MEANT FOR USE IN PRODUCTION! 6 | 7 | Sample authenticator inspired by an example from [JupyterHub docs.](https://jupyterhub.readthedocs.io/en/stable/reference/authenticators.html#authenticator-authenticate-method). 8 | 9 | ### Installation 10 | 11 | ```bash 12 | quetz plugin install plugins/quetz_dictauthenticator 13 | ``` 14 | 15 | ### Configure 16 | 17 | add the following section to your `config.toml`: 18 | 19 | ```ini 20 | [dictauthenticator] 21 | users = ["happyuser:happy", "unhappyuser:unhappy"] 22 | ``` 23 | 24 | where users is a list of pairs of username and passwords (in plain text, sic!) separated by colon `:`. 25 | 26 | ### Usage 27 | 28 | The authenticator should be active now, you can login by going to the URL `http://HOST/auth/dict/login` 29 | -------------------------------------------------------------------------------- /plugins/quetz_repodata_zchunk/README.md: -------------------------------------------------------------------------------- 1 | # quetz_repodata_zchunk plugin 2 | 3 | This is a plugin to use with the [quetz](https://github.com/mamba-org/quetz) package server that allows to serve/download the repodata using [zchunk](https://github.com/zchunk/zchunk), so that not all the repodata is downloaded every time it changes, but only the changed parts. This dramatically reduces the download time, and is more scalable in the long run (as repodata grows with time). 4 | 5 | ## Installing 6 | 7 | To install use: 8 | 9 | ```bash 10 | quetz plugin install plugins/quetz_repodata_zchunk 11 | ``` 12 | 13 | ## Using 14 | 15 | After installing the package, the `repodata.json` from any channel will also be available in the zchunk format, and if you have a recent enough version of `mamba`, the repodata will be downloaded using `zckdl` (the zchunk download utility), providing all the benefits described above. 16 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: (quetz/migrations) 2 | repos: 3 | - repo: https://github.com/astral-sh/ruff-pre-commit 4 | # Ruff version. 5 | rev: v0.4.1 6 | hooks: 7 | # Run the linter. 8 | - id: ruff 9 | # Run the formatter. 10 | - id: ruff-format 11 | - repo: https://github.com/pre-commit/mirrors-mypy 12 | # Note: updating to v1.0.0 a bit more work 13 | rev: v0.991 14 | hooks: 15 | - id: mypy 16 | files: ^quetz/ 17 | additional_dependencies: 18 | - sqlalchemy-stubs 19 | - types-click 20 | - types-Jinja2 21 | - types-mock 22 | - types-orjson 23 | - types-pkg-resources 24 | - types-redis 25 | - types-requests 26 | - types-six 27 | - types-toml 28 | - types-ujson 29 | - types-aiofiles 30 | args: [--show-error-codes, --implicit-optional] -------------------------------------------------------------------------------- /quetz/migrations/versions/3c3288034362_add_channel_metadata.py: -------------------------------------------------------------------------------- 1 | """add channel metadata 2 | 3 | Revision ID: 3c3288034362 4 | Revises: ea6eba9a9ffc 5 | Create Date: 2021-02-09 18:45:13.198051 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = '3c3288034362' 13 | down_revision = 'ea6eba9a9ffc' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | op.add_column( 21 | 'channels', 22 | sa.Column('channel_metadata', sa.String(), server_default='{}', nullable=False), 23 | ) 24 | 25 | # ### end Alembic commands ### 26 | 27 | 28 | def downgrade(): 29 | # ### commands auto generated by Alembic - please adjust! ### 30 | 31 | op.drop_column('channels', 'channel_metadata') 32 | 33 | # ### end Alembic commands ### 34 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | RUN apt-get update && apt-get install -y \ 4 | bzip2 \ 5 | ca-certificates \ 6 | curl \ 7 | postgresql-client \ 8 | && rm -rf /var/lib/{apt,dpkg,cache,log} 9 | RUN curl -L https://micro.mamba.pm/api/micromamba/linux-64/latest | \ 10 | tar -xj -C /tmp bin/micromamba 11 | RUN cp /tmp/bin/micromamba /bin/micromamba 12 | 13 | ENV MAMBA_ROOT_PREFIX=/opt/conda 14 | 15 | 16 | RUN mkdir -p $(dirname $MAMBA_ROOT_PREFIX) && \ 17 | /bin/micromamba shell init -s bash -p $MAMBA_ROOT_PREFIX && \ 18 | echo "micromamba activate base" >> ~/.bashrc 19 | 20 | ENV PATH="/opt/conda/bin:${PATH}" 21 | 22 | RUN mkdir /code 23 | WORKDIR /code 24 | COPY environment.yml /code/ 25 | 26 | RUN micromamba install -v -y -n base -c conda-forge -f environment.yml 27 | 28 | COPY . /code/ 29 | RUN pip install -e . 30 | 31 | RUN useradd quetz --no-log-init -u 1000 -p "$(openssl passwd -1 mamba)" 32 | -------------------------------------------------------------------------------- /plugins/quetz_conda_suggest/README.md: -------------------------------------------------------------------------------- 1 | # quetz_conda_suggest plugin 2 | 3 | This is a plugin to use with the [quetz](https://github.com/mamba-org/quetz) package server. 4 | 5 | It generates `.map` files specific to a particular channel (such as `conda-forge`) and a platform (such as `linux-64`). These map files facilitate the functioning of `conda-suggest`. See https://github.com/conda-incubator/conda-suggest for additional information. 6 | 7 | ## Installing 8 | 9 | To install use: 10 | 11 | ```bash 12 | quetz plugin install plugins/quetz_conda_suggest 13 | ``` 14 | 15 | ## Usage 16 | 17 | After installation, simply create channels and upload packages to them. Then, to get the `.map` file, make a GET request to the following endpoint: 18 | 19 | `GET /api/channels/{channel_name}/{subdir}/conda-suggest` 20 | 21 | where `{channel_name}` is the name of the channel (such as `conda-forge`) and `{subdir}` is the platform (such as `linux-64`). 22 | -------------------------------------------------------------------------------- /plugins/quetz_current_repodata/quetz_current_repodata/main.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pathlib import Path 3 | 4 | from conda_index.index import _build_current_repodata 5 | 6 | import quetz 7 | from quetz.utils import add_temp_static_file 8 | 9 | 10 | @quetz.hookimpl 11 | def post_package_indexing(tempdir: Path, channel_name, subdirs, files, packages): 12 | pins = {} 13 | for subdir in subdirs: 14 | with open(tempdir / channel_name / subdir / "repodata.json") as f: 15 | repodata = json.load(f) 16 | 17 | current_repodata = _build_current_repodata(subdir, repodata, pins) 18 | 19 | current_repodata_string = json.dumps(current_repodata, indent=2, sort_keys=True) 20 | 21 | add_temp_static_file( 22 | current_repodata_string, 23 | channel_name, 24 | subdir, 25 | "current_repodata.json", 26 | tempdir, 27 | files, 28 | ) 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md 2 | include alembic.ini 3 | include dev_config.toml 4 | include environment.yml 5 | include init_db.py 6 | include .flake8 7 | include .pre-commit-config.yaml 8 | include .readthedocs.yml 9 | include download-test-package.sh 10 | 11 | recursive-include quetz/migrations *.* 12 | recursive-include quetz/tests *.* 13 | recursive-include quetz_db_ext * 14 | recursive-include quetz_frontend *.* 15 | 16 | graft quetz/basic_frontend 17 | include quetz/config.toml 18 | graft quetz/templates 19 | 20 | recursive-exclude plugins *.* 21 | recursive-exclude quetz_client *.* 22 | 23 | exclude set_env_dev.sh 24 | exclude test-cli-client.sh 25 | exclude utils/repodata_compare.py 26 | 27 | include Dockerfile 28 | include docker-compose.yml 29 | recursive-include docker * 30 | 31 | recursive-include docs/source *.* 32 | recursive-include docs/assets *.* 33 | include docs/make.bat 34 | include docs/Makefile 35 | 36 | global-exclude *.py[cod] 37 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/quetz_runexports/main.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import quetz 4 | from quetz.database import get_db_manager 5 | 6 | from . import db_models 7 | from .api import router 8 | 9 | 10 | @quetz.hookimpl 11 | def register_router(): 12 | return router 13 | 14 | 15 | @quetz.hookimpl 16 | def post_add_package_version(version, condainfo): 17 | run_exports = json.dumps(condainfo.run_exports) 18 | 19 | with get_db_manager() as db: 20 | if not version.runexports: 21 | metadata = db_models.PackageVersionMetadata( 22 | version_id=version.id, data=run_exports 23 | ) 24 | db.add(metadata) 25 | else: 26 | metadata = db.get(db_models.PackageVersionMetadata, version.id) 27 | if not metadata: 28 | raise KeyError(f"No metadata found for version '{version.id}'.") 29 | metadata.data = run_exports 30 | db.commit() 31 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/README.md: -------------------------------------------------------------------------------- 1 | # quetz_content_trust plugin 2 | 3 | This is a plugin to use with the [quetz](https://github.com/mamba-org/quetz) package server. 4 | 5 | It generates signed repodata files for different `subdirs` of a particular `channel`. 6 | However, this is only done when a `private key` for that particular `channel` exists in the database. 7 | See `usage` section below for more details. 8 | 9 | ## Installing 10 | 11 | To install use: 12 | 13 | ```bash 14 | quetz plugin install plugins/quetz_content_trust 15 | ``` 16 | 17 | ## Usage 18 | 19 | - `POST /api/content-trust/upload-root` endpoint is used to upload `root.json` files. 20 | - `POST /api/content-trust/upload-key-mgr` endpoint is used to upload `key_mgr.json` files. 21 | - `POST /api/content-trust/private-key` endpoint is used to add a `private key` for a particular `channel` in the database. The endpoint expects a JSON with two fields: `{channel: abc, private_key: xyz}` 22 | -------------------------------------------------------------------------------- /plugins/quetz_tos/README.md: -------------------------------------------------------------------------------- 1 | # quetz_tos plugin 2 | 3 | This is a plugin to use with the [quetz](https://github.com/mamba-org/quetz) package server. 4 | 5 | It checks if a user has signed terms of services document or not. An `owner` need not sign the TOS. 6 | However, `maintainer` or `members` need to sign terms of services to make sure they have the relevant access 7 | to their resources. In case of not signing, their permissions will be restricted -- for eg: they might not be able to upload a package to a channel even if their status permits them to. 8 | 9 | ## Installing 10 | 11 | To install use: 12 | 13 | ``` 14 | pip install . 15 | ``` 16 | 17 | ## Usage 18 | 19 | - `GET /api/tos` endpoint is used to get information about latest terms of service document 20 | - `POST /api/tos/sign` endpoint is used to sign a particular (or latest) terms of service document 21 | - `POST /api/tos/upload` endpoint is used to upload a terms of service document. Only `owners` can do it. 22 | -------------------------------------------------------------------------------- /quetz/testing/mockups.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Union 2 | 3 | import requests 4 | 5 | from quetz.config import Config 6 | from quetz.dao import Dao 7 | from quetz.tasks.workers import job_wrapper 8 | 9 | 10 | class MockWorker: 11 | "synchronous worker for testing" 12 | 13 | def __init__( 14 | self, 15 | config: Config, 16 | db, 17 | dao: Dao, 18 | session: Optional[requests.Session] = None, 19 | ): 20 | self.db = db 21 | self.dao = dao 22 | self.session = session 23 | self.config = config 24 | 25 | def execute(self, func: Union[Callable, bytes], *args, **kwargs): 26 | resources = { 27 | "db": self.db, 28 | "dao": self.dao, 29 | "pkgstore": self.config.get_package_store(), 30 | } 31 | 32 | if self.session: 33 | resources["session"] = self.session 34 | 35 | kwargs.update(resources) 36 | job_wrapper(func, self.config, *args, **kwargs) 37 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/tests/data/root.json: -------------------------------------------------------------------------------- 1 | { 2 | "signatures": { 3 | "2b920f88531576643ada0a632915d1dcdd377557647093f29cbe251ba8c33724": { 4 | "other_headers": "04001608001d1621040673d781a8b80bcb7b002040ac7bc8bcf821360d050260b687a1", 5 | "signature": "8eecc8f58df848f7af0188fbb47f99a0f2622f8a32ab8ede6340507fc48b8785c96a217c17889d39154c290d99ac0bb6ca75c971f913778598dbab368b49040e" 6 | } 7 | }, 8 | "signed": { 9 | "delegations": { 10 | "key_mgr": { 11 | "pubkeys": [ 12 | "013ddd714962866d12ba5bae273f14d48c89cf0773dee2dbf6d4561e521c83f7" 13 | ], 14 | "threshold": 1 15 | }, 16 | "root": { 17 | "pubkeys": [ 18 | "2b920f88531576643ada0a632915d1dcdd377557647093f29cbe251ba8c33724" 19 | ], 20 | "threshold": 1 21 | } 22 | }, 23 | "expiration": "2022-06-01T19:16:49Z", 24 | "metadata_spec_version": "0.6.0", 25 | "timestamp": "2021-06-01T19:16:49Z", 26 | "type": "root", 27 | "version": 1 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /quetz/jobs/dao.py: -------------------------------------------------------------------------------- 1 | import json 2 | from datetime import datetime 3 | from typing import Optional 4 | 5 | from quetz.jobs.models import Job, JobStatus 6 | 7 | 8 | class JobsDao: 9 | def __init__(self, db): 10 | self.db = db 11 | 12 | def create_job( 13 | self, 14 | job_manifest, 15 | user_id, 16 | extra_args={}, 17 | start_at: Optional[datetime] = None, 18 | repeat_every_seconds: Optional[int] = None, 19 | ): 20 | extra_args_json: Optional[str] 21 | if extra_args: 22 | extra_args_json = json.dumps(extra_args) 23 | else: 24 | extra_args_json = None 25 | job = Job( 26 | manifest=job_manifest, 27 | owner_id=user_id, 28 | extra_args=extra_args_json, 29 | status=JobStatus.pending, 30 | start_at=start_at, 31 | repeat_every_seconds=repeat_every_seconds, 32 | ) 33 | self.db.add(job) 34 | self.db.commit() 35 | return job 36 | -------------------------------------------------------------------------------- /quetz/repo_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Codethink Ltd 2 | # Distributed under the terms of the Modified BSD License. 3 | 4 | import json 5 | 6 | from quetz import db_models 7 | 8 | 9 | def export(dao, channel_name, subdir): 10 | repodata = { 11 | "info": {"subdir": subdir}, 12 | "packages": {}, 13 | "packages.conda": {}, 14 | "repodata_version": 1, 15 | } 16 | if dao.is_active_platform(channel_name, subdir): 17 | packages = repodata["packages"] 18 | packages_conda = repodata["packages.conda"] 19 | 20 | for filename, info, format, time_modified in dao.get_package_infos( 21 | channel_name, subdir 22 | ): 23 | data = json.loads(info) 24 | data["time_modified"] = int(time_modified.timestamp()) 25 | if format == db_models.PackageFormatEnum.conda: 26 | packages_conda[filename] = data 27 | else: 28 | packages[filename] = data 29 | 30 | return repodata 31 | else: 32 | return repodata 33 | -------------------------------------------------------------------------------- /plugins/quetz_repodata_zchunk/quetz_repodata_zchunk/main.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from pathlib import Path 3 | 4 | import quetz 5 | from quetz.utils import add_entry_for_index 6 | 7 | 8 | @quetz.hookimpl 9 | def post_package_indexing(tempdir: Path, channel_name, subdirs, files, packages): 10 | for subdir in subdirs: 11 | path1 = tempdir / channel_name / subdir / "repodata.json" 12 | path2 = tempdir / channel_name / subdir / "repodata.json.zck" 13 | 14 | try: 15 | subprocess.check_call(["zck", path1, "-o", path2]) 16 | except FileNotFoundError: 17 | raise RuntimeError( 18 | "zchunk does not seem to be installed, " 19 | "you can install it with:\n" 20 | "mamba install zchunk -c conda-forge" 21 | ) 22 | except subprocess.CalledProcessError: 23 | raise RuntimeError("Error calling zck on repodata.") 24 | 25 | with open(path2, "rb") as fi: 26 | add_entry_for_index(files, subdir, "repodata.json.zck", fi.read()) 27 | -------------------------------------------------------------------------------- /quetz/migrations/versions/303ff70c27fc_configure_mirror_endpoints.py: -------------------------------------------------------------------------------- 1 | """configure mirror endpoints 2 | 3 | Revision ID: 303ff70c27fc 4 | Revises: 8dfb7c4bfbd7 5 | Create Date: 2020-12-22 17:35:45.164946 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = '303ff70c27fc' 13 | down_revision = '8dfb7c4bfbd7' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | op.add_column( 21 | 'channel_mirrors', sa.Column('api_endpoint', sa.String(), nullable=True) 22 | ) 23 | op.add_column( 24 | 'channel_mirrors', sa.Column('metrics_endpoint', sa.String(), nullable=True) 25 | ) 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.drop_column('channel_mirrors', 'metrics_endpoint') 32 | op.drop_column('channel_mirrors', 'api_endpoint') 33 | # ### end Alembic commands ### 34 | -------------------------------------------------------------------------------- /quetz/migrations/versions/db1c56bf4d57_add_channel_size_limit.py: -------------------------------------------------------------------------------- 1 | """add channel size limit 2 | 3 | Revision ID: db1c56bf4d57 4 | Revises: a3ffa287d074 5 | Create Date: 2020-12-08 18:44:14.215836 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'db1c56bf4d57' 13 | down_revision = 'a3ffa287d074' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | op.add_column('channels', sa.Column('size', sa.Integer(), nullable=True)) 21 | op.add_column('channels', sa.Column('size_limit', sa.Integer(), nullable=True)) 22 | op.add_column('package_versions', sa.Column('size', sa.Integer(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('package_versions', 'size') 29 | op.drop_column('channels', 'size_limit') 30 | op.drop_column('channels', 'size') 31 | # ### end Alembic commands ### 32 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: quetz 2 | channels: 3 | - https://repo.mamba.pm/conda-forge 4 | dependencies: 5 | - python>=3.7 6 | - pip 7 | - fastapi 8 | - typer >=0.9,<1.0 9 | - authlib=0.15.5 10 | - psycopg2 11 | - httpx>=0.22.0 12 | - sqlalchemy 13 | - sqlalchemy-utils 14 | - sqlite 15 | - python-multipart 16 | - uvicorn 17 | - zstandard 18 | - conda-build 19 | - appdirs 20 | - toml 21 | - fsspec 22 | - requests 23 | - h2 24 | - pluggy 25 | - jinja2 26 | - itsdangerous 27 | - alembic 28 | - zchunk 29 | - s3fs 30 | - gcsfs >=2022.03 31 | - sphinx 32 | - sphinx-book-theme>=0.3 33 | - tenacity 34 | - xattr 35 | - aiofiles 36 | - aioshutil 37 | - pyyaml 38 | - ujson 39 | - prometheus_client 40 | - pamela 41 | - typing_extensions 42 | - adlfs 43 | - importlib_metadata 44 | - pre-commit 45 | - pytest 7.* 46 | - pytest-mock 47 | - rq 48 | - libcflib 49 | - mamba 50 | - conda-content-trust 51 | - pyinstrument 52 | - pytest-asyncio 53 | - pytest-timeout 54 | - pydantic >=2 55 | - pip: 56 | - git+https://github.com/jupyter-server/jupyter_releaser.git@v2 57 | -------------------------------------------------------------------------------- /quetz/metrics/rest_models.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import Dict, List 3 | 4 | from pydantic import BaseModel, ConfigDict, Field 5 | 6 | from quetz.metrics.db_models import IntervalType 7 | 8 | 9 | class PackageVersionMetricItem(BaseModel): 10 | timestamp: datetime 11 | count: int 12 | model_config = ConfigDict(from_attributes=True) 13 | 14 | 15 | class PackageVersionMetricSeries(BaseModel): 16 | series: List[PackageVersionMetricItem] 17 | 18 | 19 | class PackageVersionMetricResponse(PackageVersionMetricSeries): 20 | server_timestamp: datetime = Field( 21 | default_factory=datetime.utcnow, 22 | title="server timestamp at which the response was generated", 23 | ) 24 | period: IntervalType 25 | metric_name: str 26 | total: int 27 | 28 | 29 | class ChannelMetricResponse(BaseModel): 30 | server_timestamp: datetime = Field( 31 | default_factory=datetime.utcnow, 32 | title="server timestamp at which the response was generated", 33 | ) 34 | period: IntervalType 35 | metric_name: str 36 | packages: Dict[str, PackageVersionMetricSeries] 37 | -------------------------------------------------------------------------------- /quetz/migrations/versions/cddba8e6e639_scheduling_spec_for_jobs.py: -------------------------------------------------------------------------------- 1 | """scheduling spec for jobs 2 | 3 | Revision ID: cddba8e6e639 4 | Revises: 0a0ab48887ab 5 | Create Date: 2021-03-02 11:26:04.330580 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'cddba8e6e639' 13 | down_revision = '0a0ab48887ab' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | with op.batch_alter_table('jobs', schema=None) as batch_op: 21 | batch_op.add_column( 22 | sa.Column('repeat_every_seconds', sa.Integer(), nullable=True) 23 | ) 24 | batch_op.add_column(sa.Column('start_at', sa.DateTime(), nullable=True)) 25 | 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | with op.batch_alter_table('jobs', schema=None) as batch_op: 32 | batch_op.drop_column('start_at') 33 | batch_op.drop_column('repeat_every_seconds') 34 | 35 | # ### end Alembic commands ### 36 | -------------------------------------------------------------------------------- /plugins/quetz_conda_suggest/quetz_conda_suggest/migrations/versions/c726f33caeeb_initial_revision.py: -------------------------------------------------------------------------------- 1 | """initial revision 2 | 3 | Revision ID: c726f33caeeb 4 | Revises: 5 | Create Date: 2020-11-26 00:15:03.617759 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "c726f33caeeb" 14 | down_revision = None 15 | branch_labels = ("quetz-conda_suggest",) 16 | depends_on = "quetz" 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "quetz_conda_suggest_metadata", 23 | sa.Column("version_id", sa.LargeBinary(length=16), nullable=False), 24 | sa.Column("data", sa.String(), nullable=True), 25 | sa.ForeignKeyConstraint( 26 | ["version_id"], 27 | ["package_versions.id"], 28 | ), 29 | sa.PrimaryKeyConstraint("version_id"), 30 | ) 31 | # ### end Alembic commands ### 32 | 33 | 34 | def downgrade(): 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | op.drop_table("quetz_conda_suggest_metadata") 37 | # ### end Alembic commands ### 38 | -------------------------------------------------------------------------------- /plugins/quetz_repodata_patching/README.md: -------------------------------------------------------------------------------- 1 | # quetz_repodata_patching plugin 2 | 3 | A plugin for [quetz](https://github.com/mamba-org/quetz) package server that implements the repodata patching. Repodata patching allow for hotfixing package index by changing some metadata in `repodata.json` files on the fly. For more information, see [conda-build](https://docs.conda.io/projects/conda-build/en/latest/concepts/generating-index.html#repodata-patching) docs. 4 | 5 | ## Installing 6 | 7 | To install use: 8 | 9 | ```bash 10 | quetz plugin install plugins/quetz_repodata_patching 11 | ``` 12 | 13 | ## Using 14 | 15 | After installing the package, the `repodata.json` from any channel will be patched with the files found in the `{channel_name}-repodata-patches` package in the same channel. This package should contain one `patch_instructions.json` file per subdir (for example, `linux-64/patch_instructions.json`). For details, follow the structure of the [package](https://anaconda.org/conda-forge/conda-forge-repodata-patches) from `conda-forge` channel or checkout the scripts to generate it in the conda-forge [feedstock](https://github.com/conda-forge/conda-forge-repodata-patches-feedstock/tree/master/recipe). 16 | -------------------------------------------------------------------------------- /plugins/quetz_mamba_solve/quetz_mamba_solve/api.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from fastapi import APIRouter 4 | from fastapi.responses import PlainTextResponse 5 | 6 | from .rest_models import SolveTask 7 | from .solver import MambaSolver 8 | from .utils import timed_lru_cache 9 | 10 | router = APIRouter() 11 | 12 | 13 | @timed_lru_cache(hours=3, maxsize=128) 14 | def get_solver(channels, subdir): 15 | s = MambaSolver(list(channels), subdir) 16 | return s 17 | 18 | 19 | @router.post("/api/mamba/solve", response_class=PlainTextResponse) 20 | def mamba_solve(solve_task: SolveTask): 21 | channels = tuple(solve_task.channels) 22 | subdir = solve_task.subdir 23 | spec = solve_task.spec 24 | 25 | s = get_solver(channels, subdir) 26 | _, link, _ = s.solve(spec).to_conda() 27 | 28 | data = [] 29 | data_bytes = f"# platform: {subdir}\n\n" 30 | data_bytes += "@EXPLICIT\n\n" 31 | for c, pkg, jsn_s in link: 32 | jsn_content = json.loads(jsn_s) 33 | url = jsn_content["url"] 34 | md5 = jsn_content["md5"] 35 | each_pkg = f"{url}#{md5}" 36 | data.append(each_pkg) 37 | 38 | data_bytes += "\n".join(data) 39 | 40 | return data_bytes 41 | -------------------------------------------------------------------------------- /plugins/quetz_harvester/README.md: -------------------------------------------------------------------------------- 1 | # quetz_harvester plugin 2 | 3 | This is a plugin to use with the [quetz](https://github.com/mamba-org/quetz) package server that allows to extract additional metadata from the packages, using the [libcflib](https://github.com/regro/libcflib) harvester. 4 | 5 | ## Installing 6 | 7 | To install use: 8 | 9 | ```bash 10 | # no other libcflib deps necessary for the harvester itself 11 | pip install git+https://git@github.com/regro/libcflib@master --no-deps 12 | quetz plugin install plugins/quetz_harvester 13 | ``` 14 | 15 | ## Using 16 | 17 | After installing the package run the `harvest` job using the standard /jobs endpoint in quetz: 18 | 19 | ```bash 20 | QUETZ_API_KEY=... # setup you api key retrieved through the quetz fronted 21 | curl -X POST localhost:8000/api/jobs -d '{"items_spec": "*", "manifest": "quetz-harvester:harvest"}' -H "X-api-key: ${QUETZ_API_KEY}" 22 | ``` 23 | 24 | it will run the `harvest` job on ALL package files on the server. 25 | 26 | Each package will have an additional file added to the packagestore (`channel/metadata/subdir/package-name.json`). 27 | You can access that file from the URL: `http://quetz/channels/channel/metadata/subdir/package-name.json`. 28 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | When contributing to this repository, it is always a good idea to first 4 | discuss the change you wish to make via issue, email, or any other method with 5 | the owners of this repository before making a change. 6 | 7 | We welcome all kinds of contribution -- code or non-code -- and value them 8 | highly. We pledge to treat everyones contribution fairly and with respect and 9 | we are here to help awesome pull requests over the finish line. 10 | 11 | Please note we have a code of conduct, and follow it in all your interactions with the project. 12 | 13 | We follow the [NumFOCUS code of conduct](https://numfocus.org/code-of-conduct). 14 | 15 | # Developing 16 | 17 | You can install all development dependencies via: 18 | 19 | pip install -e .[dev] 20 | pre-commit install 21 | 22 | This will also setup hooks in your local repository to run linting and code 23 | formatting checks prior to making a commit. 24 | 25 | # Pull requests 26 | 27 | Make sure to include the following with your pull requests: 28 | 29 | - description, 30 | - links to relevant issues, 31 | - any necessary code changes, 32 | - unit tests for these changes, 33 | - changes to relevant documentation sections 34 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/quetz_runexports/migrations/versions/cb095bcf3bb4_initial_revision.py: -------------------------------------------------------------------------------- 1 | """initial revision 2 | 3 | Revision ID: cb095bcf3bb4 4 | Revises: a80fb051a659 5 | Create Date: 2020-11-25 00:14:48.269216 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "cb095bcf3bb4" 14 | down_revision = None 15 | branch_labels = ("quetz-runexports",) 16 | depends_on = "a80fb051a659" 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "quetz_runexports_package_version_metadata", 23 | sa.Column("version_id", sa.LargeBinary(length=16), nullable=False), 24 | sa.Column("data", sa.String(), nullable=True), 25 | sa.ForeignKeyConstraint( 26 | ["version_id"], 27 | ["package_versions.id"], 28 | ), 29 | sa.PrimaryKeyConstraint("version_id"), 30 | ) 31 | # ### end Alembic commands ### 32 | 33 | 34 | def downgrade(): 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | op.drop_table("quetz_runexports_package_version_metadata") 37 | # ### end Alembic commands ### 38 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/docs/testing.rst: -------------------------------------------------------------------------------- 1 | # Quetz Content Trust 2 | 3 | ## Testing 4 | 5 | Few scripts are provided to perform a manual integration test of package signing. 6 | 7 | This is intended to be used by channels owners/maintainers to get familiar with the workflow of generating TUF roles (delegations, etc.) and push them on a Quetz server. 8 | 9 | For that, simply run the 2 following commands from your quetz source directory: 10 | - `quetz run test_quetz --copy-conf ./dev_config.toml --dev --reload --delete` to get an up and running server 11 | - `python plugins/quetz_content_trust/docs/test_script.py` to generate TUF roles, push them on the server but also push an empty `test-package` package 12 | 13 | then just test from client side: `micromamba create -n foo -c http://127.0.0.1:8000/get/channel0 test-package --no-rc -y -vvv --repodata-ttl=0 --experimental --verify-artifacts` 14 | 15 | You can also simulate invalid signatures/role metadata running: 16 | - `python plugins/quetz_content_trust/docs/test_corrupted_key_mgr_metadata.py`: overwrite `KeyMgr` role metadata with an invalid delegation 17 | - `python plugins/quetz_content_trust/docs/test_corrupted_key_mgr_sigs.py`: overwrite `KeyMgr` role keys with an invalid one 18 | -------------------------------------------------------------------------------- /quetz_db_ext/quetz_sqlite.c: -------------------------------------------------------------------------------- 1 | #include "sqlite3ext.h" 2 | SQLITE_EXTENSION_INIT1 3 | #include 4 | #include 5 | 6 | #include "conda.h" 7 | 8 | static void version_match_func( 9 | sqlite3_context* context, 10 | int argc, 11 | sqlite3_value** argv) 12 | { 13 | const unsigned char* zIn1; 14 | const unsigned char* zIn2; 15 | assert(argc == 2); 16 | if (sqlite3_value_type(argv[0]) == SQLITE_NULL || sqlite3_value_type(argv[1]) == SQLITE_NULL) 17 | return; 18 | 19 | zIn1 = (const unsigned char*)sqlite3_value_text(argv[0]); 20 | zIn2 = (const unsigned char*)sqlite3_value_text(argv[1]); 21 | int res = solvable_conda_matchversion(zIn1, zIn2); 22 | sqlite3_result_int(context, res); 23 | } 24 | 25 | #ifdef _WIN32 26 | __declspec(dllexport) 27 | #endif 28 | int sqlite3_quetzsqlite_init( 29 | sqlite3* db, 30 | char** pzErrMsg, 31 | const sqlite3_api_routines* pApi) 32 | { 33 | int rc = SQLITE_OK; 34 | SQLITE_EXTENSION_INIT2(pApi); 35 | (void)pzErrMsg; /* Unused parameter */ 36 | rc = sqlite3_create_function(db, "version_match", 2, 37 | SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, 38 | 0, version_match_func, 0, 0); 39 | return rc; 40 | } -------------------------------------------------------------------------------- /quetz/tests/authentification/test_base.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from fastapi import Request, Response 3 | from starlette.testclient import TestClient 4 | 5 | from quetz.authentication.base import BaseAuthenticationHandlers, BaseAuthenticator 6 | 7 | 8 | class DummyHandlers(BaseAuthenticationHandlers): 9 | authorize_methods = ["POST"] 10 | 11 | async def login(self, request: Request): 12 | return Response("success") 13 | 14 | 15 | class DummyAuthenticator(BaseAuthenticator): 16 | handler_cls = DummyHandlers 17 | provider = "testprovider" 18 | 19 | def configure(self, config): 20 | self.is_enabled = True 21 | 22 | async def authenticate(self, request, data, **kwargs): 23 | return "dummy-user" 24 | 25 | 26 | @pytest.fixture 27 | def dummy_authenticator(app, config): 28 | from quetz.main import auth_registry 29 | 30 | authenticator = DummyAuthenticator(config) 31 | 32 | auth_registry.register(authenticator) 33 | 34 | return authenticator 35 | 36 | 37 | def test_login_endpoint(app, dummy_authenticator): 38 | client = TestClient(app) 39 | 40 | response = client.get(f"/auth/{dummy_authenticator.provider}/login") 41 | 42 | assert response.status_code == 200 43 | assert response.text == "success" 44 | -------------------------------------------------------------------------------- /plugins/quetz_conda_suggest/quetz_conda_suggest/api.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from fastapi import APIRouter, Depends, HTTPException, status 4 | from fastapi.responses import FileResponse, RedirectResponse 5 | from sqlalchemy.orm.session import Session 6 | 7 | from quetz.config import Config 8 | from quetz.deps import get_db 9 | 10 | config = Config() 11 | pkgstore = config.get_package_store() 12 | 13 | router = APIRouter() 14 | 15 | 16 | @router.get("/api/channels/{channel_name}/{subdir}/conda-suggest") 17 | def get_conda_suggest(channel_name, subdir, db: Session = Depends(get_db)): 18 | map_filename = f"{channel_name}.{subdir}.map" 19 | map_filepath = pkgstore.url(channel_name, f"{subdir}/{map_filename}") 20 | try: 21 | if pkgstore.support_redirect: 22 | return RedirectResponse(map_filepath) 23 | elif os.path.isfile(map_filepath): 24 | return FileResponse( 25 | map_filepath, 26 | media_type="application/octet-stream", 27 | filename=map_filename, 28 | ) 29 | except Exception: 30 | raise HTTPException( 31 | status_code=status.HTTP_404_NOT_FOUND, 32 | detail=f"conda-suggest map file for {channel_name}.{subdir} not found", 33 | ) 34 | -------------------------------------------------------------------------------- /quetz/migrations/versions/98c04a65df4a_register_mirrors.py: -------------------------------------------------------------------------------- 1 | """register mirrors 2 | 3 | Revision ID: 98c04a65df4a 4 | Revises: 8d1e9a9e0b1f 5 | Create Date: 2020-12-21 11:10:15.107635 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = '98c04a65df4a' 13 | down_revision = '8d1e9a9e0b1f' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | op.create_table( 21 | 'channel_mirrors', 22 | sa.Column('id', sa.LargeBinary(length=16), nullable=False), 23 | sa.Column('channel_name', sa.String(), nullable=True), 24 | sa.Column('url', sa.String(), nullable=False), 25 | sa.Column('last_synchronised', sa.DateTime(), nullable=True), 26 | sa.ForeignKeyConstraint( 27 | ['channel_name'], 28 | ['channels.name'], 29 | ), 30 | sa.PrimaryKeyConstraint('id'), 31 | sa.UniqueConstraint('url'), 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade(): 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_table('channel_mirrors') 39 | # ### end Alembic commands ### 40 | -------------------------------------------------------------------------------- /plugins/quetz_dictauthenticator/quetz_dictauthenticator/__init__.py: -------------------------------------------------------------------------------- 1 | from quetz.authentication.base import SimpleAuthenticator 2 | from quetz.config import Config, ConfigEntry, ConfigSection 3 | 4 | 5 | class DictionaryAuthenticator(SimpleAuthenticator): 6 | passwords: dict 7 | provider = "dict" 8 | 9 | def configure(self, config: Config): 10 | config.register( 11 | [ 12 | ConfigSection( 13 | "dictauthenticator", 14 | [ 15 | ConfigEntry("users", list, default=list), 16 | ], 17 | ) 18 | ] 19 | ) 20 | 21 | if config.configured_section("dictauthenticator"): 22 | self.passwords = dict( 23 | user_pass.split(":") for user_pass in config.dictauthenticator_users 24 | ) 25 | self.is_enabled = True 26 | else: 27 | self.passwords = {} 28 | 29 | # call the config of base class to configure default roles and 30 | # channels 31 | super().configure(config) 32 | 33 | async def authenticate(self, request, data, **kwargs): 34 | if self.passwords.get(data["username"]) == data["password"]: 35 | return data["username"] 36 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Quetz documentation master file, created by 2 | sphinx-quickstart on Mon Nov 2 11:02:31 2020. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Quetz: conda package server 7 | =========================== 8 | 9 | The quetz project is an open source server for conda packages. It is built upon `FastAPI`_ with an API-first approach. A quetz server can have many users, channels and packages. Quetz allows for setting fine-grained permissions on channel and package-name level. 10 | 11 | The development of quetz is taking place on `github`_. 12 | 13 | You can also contact the community of quetz developers and users on our `gitter`_ channel. 14 | 15 | Quetz project is supported by `QuantStack`_. 16 | 17 | .. _github: https://github.com/mamba-org/quetz 18 | .. _gitter: https://gitter.im/QuantStack/Lobby 19 | .. _QuantStack: https://twitter.com/QuantStack 20 | .. _FastAPI: https://fastapi.tiangolo.com/ 21 | 22 | Contents 23 | -------- 24 | 25 | .. toctree:: 26 | :maxdepth: 2 27 | 28 | deploying/index 29 | using/index 30 | plugins 31 | qeps/index 32 | 33 | Indices and tables 34 | ------------------ 35 | 36 | * :ref:`genindex` 37 | * :ref:`modindex` 38 | * :ref:`search` 39 | -------------------------------------------------------------------------------- /plugins/quetz_current_repodata/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from pytest import fixture 4 | 5 | from quetz import rest_models 6 | from quetz.dao import Dao 7 | from quetz.db_models import Profile, User 8 | 9 | pytest_plugins = "quetz.testing.fixtures" 10 | 11 | 12 | @fixture 13 | def dao(db) -> Dao: 14 | return Dao(db) 15 | 16 | 17 | @fixture 18 | def user(db): 19 | user = User(id=uuid.uuid4().bytes, username="madhurt") 20 | db.add(user) 21 | db.commit() 22 | yield user 23 | 24 | 25 | @fixture 26 | def profile(db, user): 27 | user_profile = Profile( 28 | name="madhur", avatar_url="madhur-tandon", user_id=user.id, user=user 29 | ) 30 | db.add(user_profile) 31 | db.commit() 32 | yield user 33 | 34 | 35 | @fixture 36 | def channel(dao, user, db): 37 | channel_data = rest_models.Channel( 38 | name="test-channel", 39 | private=False, 40 | ) 41 | 42 | channel = dao.create_channel(channel_data, user.id, "owner") 43 | 44 | yield channel 45 | 46 | db.delete(channel) 47 | db.commit() 48 | 49 | 50 | @fixture 51 | def subdirs(): 52 | return ["linux-64"] 53 | 54 | 55 | @fixture 56 | def files(): 57 | return {"linux-64": []} 58 | 59 | 60 | @fixture 61 | def packages(): 62 | return {"linux-64": []} 63 | -------------------------------------------------------------------------------- /quetz/migrations/versions/ebe550f9fbbe_added_create_at_and_expire_at_date_to_.py: -------------------------------------------------------------------------------- 1 | """Added create_at and expire_at date to API key 2 | 3 | Revision ID: ebe550f9fbbe 4 | Revises: 0653794b6252 5 | Create Date: 2021-01-22 22:38:05.693595 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'ebe550f9fbbe' 13 | down_revision = '0653794b6252' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | with op.batch_alter_table('api_keys', schema=None) as batch_op: 21 | batch_op.add_column( 22 | sa.Column( 23 | 'time_created', 24 | sa.Date(), 25 | nullable=False, 26 | server_default='2020-01-01', 27 | ) 28 | ) 29 | batch_op.add_column(sa.Column('expire_at', sa.Date(), nullable=True)) 30 | # ### end Alembic commands ### 31 | 32 | 33 | def downgrade(): 34 | # ### commands auto generated by Alembic - please adjust! ### 35 | 36 | with op.batch_alter_table('api_keys', schema=None) as batch_op: 37 | batch_op.drop_column('expire_at') 38 | batch_op.drop_column('time_created') 39 | 40 | # ### end Alembic commands ### 41 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/quetz_runexports/api.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from fastapi import APIRouter, Depends, HTTPException, status 4 | from sqlalchemy.orm.session import Session 5 | 6 | from quetz.db_models import PackageVersion 7 | from quetz.deps import get_db 8 | 9 | router = APIRouter() 10 | 11 | 12 | @router.get( 13 | "/api/channels/{channel_name}/packages/{package_name}/versions/" 14 | "{platform}/{filename}/run_exports" 15 | ) 16 | def get_run_exports( 17 | channel_name: str, 18 | package_name: str, 19 | platform: str, 20 | filename: str, 21 | db: Session = Depends(get_db), 22 | ): 23 | package_version = ( 24 | db.query(PackageVersion) 25 | .filter(PackageVersion.channel_name == channel_name) 26 | .filter(PackageVersion.platform == platform) 27 | .filter(PackageVersion.filename == filename) 28 | .first() 29 | ) 30 | 31 | if package_version is None or not package_version.runexports: 32 | raise HTTPException( 33 | status_code=status.HTTP_404_NOT_FOUND, 34 | detail=( 35 | f"run_exports for package {channel_name}/{platform}/{filename}" 36 | "not found" 37 | ), 38 | ) 39 | run_exports = json.loads(package_version.runexports.data) 40 | return run_exports 41 | -------------------------------------------------------------------------------- /.github/workflows/docker-build-push.yml: -------------------------------------------------------------------------------- 1 | name: docker build and push 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | tags: 8 | - "**" 9 | 10 | jobs: 11 | docker: 12 | runs-on: ubuntu-latest 13 | env: 14 | IMAGE_NAME: mambaorg/quetz 15 | steps: 16 | - name: Set Docker image for main branch 17 | if: ${{ github.ref == 'refs/heads/main' }} 18 | run: echo "DOCKER_IMAGES=${IMAGE_NAME}:latest" >> $GITHUB_ENV 19 | 20 | - name: Set Docker image for tag 21 | if: ${{ startsWith(github.ref, 'refs/tags') }} 22 | run: echo "DOCKER_IMAGES=${IMAGE_NAME}:latest,${IMAGE_NAME}:${GITHUB_REF#refs/*/}" >> $GITHUB_ENV 23 | 24 | - name: Show docker images 25 | run: echo $DOCKER_IMAGES 26 | 27 | - name: Set up Docker Buildx 28 | uses: docker/setup-buildx-action@v1 29 | 30 | - name: Login to DockerHub 31 | uses: docker/login-action@v1 32 | with: 33 | username: ${{ secrets.DOCKERHUB_USERNAME }} 34 | password: ${{ secrets.DOCKERHUB_TOKEN }} 35 | 36 | - name: Build and push 37 | id: docker_build 38 | uses: docker/build-push-action@v2 39 | with: 40 | push: true 41 | tags: ${{ env.DOCKER_IMAGES }} 42 | 43 | - name: Image digest 44 | run: echo ${{ steps.docker_build.outputs.digest }} 45 | -------------------------------------------------------------------------------- /quetz/migrations/versions/a3ffa287d074_case_insensitive_channel_names.py: -------------------------------------------------------------------------------- 1 | """case insensitive channel names 2 | 3 | Revision ID: a3ffa287d074 4 | Revises: a80fb051a659 5 | Create Date: 2020-12-07 14:10:27.374580 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'a3ffa287d074' 13 | down_revision = 'a80fb051a659' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | if op.get_context().dialect.name == 'postgresql': 21 | op.execute( 22 | "CREATE COLLATION IF NOT EXISTS nocase (provider = icu, " 23 | "locale = 'und-u-ks-level2', " 24 | "deterministic = false)" 25 | ) 26 | with op.batch_alter_table("channels") as batch_op: 27 | batch_op.alter_column("name", type_=sa.types.String(100, collation="nocase")) 28 | # ### end Alembic commands ### 29 | 30 | 31 | def downgrade(): 32 | # ### commands auto generated by Alembic - please adjust! ### 33 | with op.batch_alter_table("channels") as batch_op: 34 | batch_op.alter_column("name", type_=sa.types.String(None, collation=None)) 35 | if op.get_context().dialect.name == 'postgresql': 36 | op.execute("DROP COLLATION nocase;") 37 | # ### end Alembic commands ### 38 | -------------------------------------------------------------------------------- /plugins/quetz_harvester/tests/test_main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from quetz.db_models import PackageVersion 7 | from quetz.jobs.models import Job, Task, TaskStatus 8 | 9 | 10 | @pytest.mark.skipif( 11 | sys.version_info >= (3, 10), 12 | reason="xonsh pinning used by libcflib not compatible with python 3.10", 13 | ) 14 | def test_harvest_endpoint_and_job( 15 | api_key, auth_client, db, config, supervisor, package_version, app, channel_name 16 | ): 17 | response = auth_client.post( 18 | "/api/jobs", json={"items_spec": "*", "manifest": "quetz-harvester:harvest"} 19 | ) 20 | 21 | assert response.status_code == 201 22 | supervisor.run_jobs() 23 | supervisor.run_tasks() 24 | 25 | supervisor.check_status() 26 | 27 | task = db.query(Task).one() 28 | db.refresh(task) 29 | assert task.status == TaskStatus.success 30 | 31 | pkgstore = config.get_package_store() 32 | fileh = pkgstore.serve_path( 33 | channel_name, 34 | Path("metadata") 35 | / package_version.platform 36 | / package_version.filename.replace(".tar.bz2", ".json"), 37 | ) 38 | ok_ = fileh.read(10) 39 | assert ok_ 40 | 41 | # cleanup 42 | try: 43 | db.query(PackageVersion).delete() 44 | db.query(Job).delete() 45 | db.query(Task).delete() 46 | finally: 47 | db.commit() 48 | -------------------------------------------------------------------------------- /quetz/migrations/versions/30241b33d849_add_task_pending_state.py: -------------------------------------------------------------------------------- 1 | """add task pending state 2 | 3 | Revision ID: 30241b33d849 4 | Revises: cd404ed93cc0 5 | Create Date: 2021-01-07 14:39:43.251123 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = '30241b33d849' 13 | down_revision = 'cd404ed93cc0' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # manually entered 20 | 21 | if op.get_context().dialect.name == 'postgresql': 22 | # https://alembic.sqlalchemy.org/en/latest/api/runtime.html#alembic.runtime.migration.MigrationContext.autocommit_block 23 | with op.get_context().autocommit_block(): 24 | op.execute("ALTER TYPE taskstatus ADD VALUE 'created'") 25 | else: 26 | # sqlite uses varchar + constraint for enum types 27 | taskstatus_enum = sa.Enum( 28 | 'created', 29 | 'pending', 30 | 'running', 31 | 'success', 32 | 'failed', 33 | 'skipped', 34 | name='taskstatus', 35 | ) 36 | 37 | with op.batch_alter_table("tasks") as batch_op: 38 | batch_op.alter_column("status", type_=taskstatus_enum) 39 | 40 | 41 | def downgrade(): 42 | # ### commands auto generated by Alembic - please adjust! ### 43 | pass 44 | # ### end Alembic commands ### 45 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/README.md: -------------------------------------------------------------------------------- 1 | # quetz_runexports 2 | 3 | Quetz plugin to extract and expose `run_exports` from package files. 4 | 5 | ## Installation 6 | 7 | Install quetz and then from this plugin directory do: 8 | 9 | ``` 10 | pip install -e . 11 | ``` 12 | 13 | The plugin should be automatically integrated with quetz server, when you start it. 14 | 15 | ## Usage 16 | 17 | To retrieve the `run_exports` make a GET requests on the following endpoint: 18 | 19 | `GET /api/channels/{channel_name}/packages/{package_name}/versions/{version_id}/run_exports` 20 | 21 | where `{version_id}` is the version number and the build hash with the minus sign in between (`version_number-build_hash`). For example: 22 | 23 | ``` 24 | # download zeromq package 25 | curl -OL https://anaconda.org/conda-forge/zeromq/4.3.3/download/linux-64/zeromq-4.3.3-he1b5a44_2.tar.bz2 26 | 27 | # export an api key 28 | export QUETZ_API_KEY=... 29 | 30 | # create a new channel 31 | curl -X POST http://localhost:8000/api/channels -d '{"name": "test-channel", "private": false}' -H "X-API-Key: ${QUETZ_API_KEY}" 32 | 33 | # upload a package 34 | quetz-client http://localhost:8000/api/channels/test-channel zeromq-4.3.3-he1b5a44_2.tar.bz2 35 | 36 | # get run_exports 37 | curl -X GET http://localhost:8000/api/channels/test-channel/packages/zeromq/versions/4.3.3-he1b5a44_2/run_exports 38 | ``` 39 | 40 | This should return: 41 | 42 | ``` 43 | {"weak":["zeromq >=4.3.3,<4.4.0a0"]} 44 | ``` 45 | -------------------------------------------------------------------------------- /plugins/quetz_tos/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from pytest import fixture 4 | from quetz_tos import db_models 5 | 6 | from quetz.db_models import User 7 | 8 | pytest_plugins = "quetz.testing.fixtures" 9 | 10 | 11 | @fixture 12 | def owner_user(db): 13 | user = User(id=uuid.uuid4().bytes, username="madhurt", role="owner") 14 | db.add(user) 15 | db.commit() 16 | 17 | yield user 18 | 19 | 20 | @fixture 21 | def member_user(db): 22 | user = User(id=uuid.uuid4().bytes, username="alice", role="member") 23 | db.add(user) 24 | db.commit() 25 | 26 | yield user 27 | 28 | 29 | @fixture 30 | def tos(db, owner_user): 31 | tos_en = db_models.TermsOfServiceFile(filename="tos_en.txt", language="EN") 32 | tos_fr = db_models.TermsOfServiceFile(filename="tos_fr.txt", language="FR") 33 | 34 | tos = db_models.TermsOfService(uploader_id=owner_user.id, files=[tos_en, tos_fr]) 35 | 36 | db.add(tos) 37 | db.commit() 38 | 39 | yield tos 40 | 41 | 42 | @fixture 43 | def tos_file(config): 44 | pkgstore = config.get_package_store() 45 | pkgstore.add_file(b"demo tos en", "root", "tos_en.txt") 46 | pkgstore.add_file(b"demo tos fr", "root", "tos_fr.txt") 47 | 48 | 49 | @fixture 50 | def tos_sign(db, tos, member_user): 51 | tos_sign = db_models.TermsOfServiceSignatures( 52 | tos_id=tos.id, 53 | user_id=member_user.id, 54 | ) 55 | 56 | db.add(tos_sign) 57 | db.commit() 58 | 59 | yield tos_sign 60 | -------------------------------------------------------------------------------- /quetz_db_ext/LIBSOLV_LICENSE.txt: -------------------------------------------------------------------------------- 1 | Redistribution and use in source and binary forms, with or without 2 | modification, are permitted provided that the following conditions 3 | are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright 6 | notice, this list of conditions and the following disclaimer. 7 | 8 | 2. Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | 12 | 3. Neither the name of Novell nor the names of its contributors may 13 | be used to endorse or promote products derived from this software 14 | without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 20 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 | STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25 | IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /plugins/quetz_tos/quetz_tos/db_models.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from sqlalchemy import Column, DateTime, ForeignKey, String, UniqueConstraint, func 4 | from sqlalchemy.orm import backref, relationship 5 | 6 | from quetz.db_models import UUID, Base 7 | 8 | 9 | class TermsOfServiceFile(Base): 10 | __tablename__ = "quetz_tos_file" 11 | 12 | id = Column(UUID, primary_key=True, default=lambda: uuid.uuid4().bytes) 13 | filename = Column(String) 14 | language = Column(String) 15 | tos_id = Column(UUID, ForeignKey("quetz_tos.id"), primary_key=True) 16 | 17 | 18 | class TermsOfService(Base): 19 | __tablename__ = "quetz_tos" 20 | 21 | id = Column(UUID, primary_key=True, default=lambda: uuid.uuid4().bytes) 22 | uploader_id = Column(UUID) 23 | files = relationship("TermsOfServiceFile") 24 | time_created = Column(DateTime, nullable=False, server_default=func.now()) 25 | 26 | 27 | class TermsOfServiceSignatures(Base): 28 | __tablename__ = "quetz_tos_signatures" 29 | __table_args__ = (UniqueConstraint("tos_id", "user_id"),) 30 | 31 | tos_id = Column(UUID, ForeignKey("quetz_tos.id"), primary_key=True) 32 | user_id = Column(UUID, ForeignKey("users.id"), primary_key=True) 33 | time_created = Column(DateTime, nullable=False, server_default=func.now()) 34 | tos = relationship( 35 | "TermsOfService", 36 | backref=backref( 37 | "tos", 38 | uselist=False, 39 | cascade="delete,all", 40 | ), 41 | ) 42 | -------------------------------------------------------------------------------- /plugins/quetz_harvester/quetz_harvester/jobs.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from pathlib import Path 4 | from typing import Optional 5 | 6 | from libcflib.harvester import harvest as libc_harvest 7 | from pydantic import BaseModel, Field 8 | 9 | from quetz.dao import Dao 10 | from quetz.pkgstores import PackageStore 11 | 12 | logger = logging.getLogger("quetz.plugins") 13 | 14 | 15 | class PackageSpec(BaseModel): 16 | package_spec: Optional[str] = Field(None, title="package version specification") 17 | 18 | 19 | def harvest(package_version: dict, config, pkgstore: PackageStore, dao: Dao): 20 | filename: str = package_version["filename"] 21 | channel: str = package_version["channel_name"] 22 | platform = package_version["platform"] 23 | 24 | logger.debug(f"Harvesting: {filename}, {channel}, {platform}") 25 | # TODO figure out how to handle properly either .conda or .tar.bz2 26 | if not filename.endswith(".tar.bz2"): 27 | return 28 | 29 | fh = pkgstore.serve_path(channel, Path(platform) / filename) 30 | 31 | try: 32 | result = libc_harvest(fh) 33 | except Exception as e: 34 | logger.exception(f"Exception caught in harvesting {filename}: {str(e)}") 35 | return 36 | 37 | logger.debug(f"Uploading harvest result for {channel}/{platform}/{filename}") 38 | 39 | pkgstore.add_file( 40 | json.dumps(result, indent=4, sort_keys=True), 41 | channel, 42 | Path("metadata") / platform / filename.replace(".tar.bz2", ".json"), 43 | ) 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, QuantStack 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /docs/source/using/basics.rst: -------------------------------------------------------------------------------- 1 | Basics 2 | ###### 3 | 4 | 5 | Channels 6 | ^^^^^^^^ 7 | 8 | Create a channel 9 | """""""""""""""" 10 | 11 | First, make sure you're logged in to the web app. 12 | 13 | Then, using the swagger docs at ``:/docs``, POST to ``/api/channels`` with the name and description of your new channel: 14 | 15 | .. code:: json 16 | 17 | { 18 | "name": "my-channel", 19 | "description": "Description for my-channel", 20 | "private": false 21 | } 22 | 23 | This will create a new channel called ``my-channel`` and your user will be the Owner of that channel. 24 | 25 | API keys 26 | ^^^^^^^^ 27 | 28 | .. _generate-an-api-key: 29 | 30 | Generate an API key 31 | """"""""""""""""""" 32 | 33 | API keys are scoped per channel, per user and optionally per package. 34 | In order to generate an API key the following must be true: 35 | 36 | 1. First, make sure you're logged in to the web app. 37 | 2. The user must be part of the target channel (you might need to create a channel first, see the previous section on how to create a channel via the swagger docs) 38 | 3. Go to the swagger docs at ``:/docs`` and POST to ``/api/api-keys``: 39 | 40 | .. code:: json 41 | 42 | { 43 | "description": "my-test-token", 44 | "roles": [ 45 | { 46 | "role": "owner", 47 | "channel": "my-channel" 48 | } 49 | ] 50 | } 51 | 52 | 4. Then, GET on ``/api/api-keys`` to retrieve your token 53 | 5. Finally, set this value to QUETZ_API_KEY so you can use quetz-client to interact with the server. 54 | 55 | 56 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/quetz_content_trust/migrations/versions/dadfc30be670_add_repo_signing_key.py: -------------------------------------------------------------------------------- 1 | """add_repo_signing_key 2 | 3 | Revision ID: dadfc30be670 4 | Revises: 5 | Create Date: 2021-05-28 14:20:01.479359 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "dadfc30be670" 14 | down_revision = None 15 | branch_labels = ("quetz-content_trust",) 16 | depends_on = "quetz" 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "repodata_signing_keys", 23 | sa.Column("id", sa.LargeBinary(length=16), nullable=False), 24 | sa.Column("private_key", sa.String(), nullable=True), 25 | sa.Column( 26 | "time_created", 27 | sa.Date(), 28 | server_default=sa.text("(CURRENT_TIMESTAMP)"), 29 | nullable=False, 30 | ), 31 | sa.Column("user_id", sa.LargeBinary(length=16), nullable=True), 32 | sa.Column("channel_name", sa.String(), nullable=True), 33 | sa.ForeignKeyConstraint( 34 | ["channel_name"], 35 | ["channels.name"], 36 | ), 37 | sa.ForeignKeyConstraint( 38 | ["user_id"], 39 | ["users.id"], 40 | ), 41 | sa.PrimaryKeyConstraint("id"), 42 | ) 43 | # ### end Alembic commands ### 44 | 45 | 46 | def downgrade(): 47 | # ### commands auto generated by Alembic - please adjust! ### 48 | op.drop_table("repodata_signing_keys") 49 | # ### end Alembic commands ### 50 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = quetz-server 3 | version = attr: quetz._version.__version__ 4 | description = The mamba-org server for conda packages 5 | long_description = file: README.md 6 | long_description_content_type = text/markdown 7 | license_file = LICENSE 8 | author = QuantStack & Quetz contributors 9 | author_email = wolf.vollprecht@quantstack.net 10 | url = https://github.com/mamba-org/quetz 11 | platforms = Linux, Mac OS X 12 | keywords = conda, mamba, server 13 | 14 | [bdist_wheel] 15 | universal = 1 16 | 17 | [options] 18 | include_package_data = True 19 | packages = find: 20 | python_requires = >=3.7 21 | 22 | install_requires = 23 | alembic 24 | aiofiles 25 | appdirs 26 | authlib<1.0.0 27 | fastapi 28 | fsspec 29 | h2 30 | httpx>=0.22.0 31 | importlib-metadata 32 | itsdangerous 33 | jinja2 34 | pluggy 35 | prometheus_client 36 | python-multipart 37 | pydantic>=2.0.0 38 | pyyaml 39 | requests 40 | sqlalchemy 41 | sqlalchemy-utils 42 | tenacity 43 | toml 44 | typer >=0.9,<1.0 45 | typing_extensions 46 | ujson 47 | uvicorn 48 | zstandard 49 | aioshutil 50 | 51 | [options.entry_points] 52 | console_scripts = 53 | quetz = quetz.cli:app 54 | 55 | [options.extras_require] 56 | azure = 57 | adlfs 58 | gcs = 59 | gcsfs >=2022.02 60 | pam = 61 | pamela 62 | postgre = 63 | psycopg2 64 | s3 = 65 | s3fs 66 | all = 67 | %(azure)s 68 | %(gcs)s 69 | %(pam)s 70 | %(postgre)s 71 | %(s3)s 72 | client = 73 | quetz-client 74 | dev = 75 | black 76 | flake8 77 | isort 78 | pre-commit 79 | pytest >=7,<8 80 | pytest-asyncio 81 | pytest-mock 82 | pytest-cov 83 | pytest-timeout 84 | tbump 85 | test = 86 | %(dev)s 87 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=41", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.pytest.ini_options] 6 | asyncio_mode="strict" 7 | 8 | [tool.black] 9 | skip-string-normalization = true 10 | 11 | [tool.isort] 12 | line_length = 88 13 | multi_line_output = 3 14 | include_trailing_comma = true 15 | 16 | [tool.jupyter-releaser.options] 17 | check-imports = ["quetz"] 18 | 19 | [tool.check-wheel-contents] 20 | ignore = ["W004"] 21 | 22 | [tool.tbump.version] 23 | current = "0.10.4" 24 | regex = ''' 25 | (?P\d+)\.(?P\d+)\.(?P\d+) 26 | ((?Pa|b|rc|.dev)(?P\d+))? 27 | ''' 28 | 29 | [[tool.tbump.field]] 30 | name = "channel" 31 | default = "" 32 | 33 | [[tool.tbump.field]] 34 | name = "release" 35 | default = "" 36 | 37 | [tool.tbump.git] 38 | message_template = "Bump to {new_version}" 39 | tag_template = "v{new_version}" 40 | 41 | [[tool.tbump.file]] 42 | src = "quetz/_version.py" 43 | version_template = '({major}, {minor}, {patch}, "{channel}", "{release}")' 44 | 45 | [tool.pyright] 46 | include = ["quetz"] 47 | reportGeneralTypeIssues = false 48 | reportMissingImports = true 49 | reportMissingModuleSource = true 50 | reportMissingTypeStubs = false 51 | reportOptionalMemberAccess = true 52 | reportOptionalOperand = true 53 | reportOptionalSubscript = true 54 | reportPrivateImportUsage = true 55 | reportUnboundVariable = true 56 | reportUndefinedVariable = false 57 | venv = ".venv" 58 | venvPath= "." 59 | 60 | [tool.mypy] 61 | ignore_missing_imports = true 62 | plugins = [ 63 | "sqlmypy" 64 | ] 65 | disable_error_code = [ 66 | "misc" 67 | ] 68 | 69 | [tool.coverage.run] 70 | omit = [ 71 | "quetz/tests/*", 72 | ] 73 | -------------------------------------------------------------------------------- /quetz/authentication/registry.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Dict 3 | 4 | from . import BaseAuthenticator 5 | 6 | logger = logging.getLogger("quetz") 7 | 8 | 9 | class AuthenticatorRegistry: 10 | _instance = None 11 | _router = None 12 | enabled_authenticators: Dict[str, BaseAuthenticator] = {} 13 | 14 | def __new__(cls): 15 | if cls._instance is None: 16 | cls._instance = super().__new__(cls) 17 | return cls._instance 18 | 19 | @classmethod 20 | def set_router(cls, router): 21 | cls._router = router 22 | 23 | def register(self, auth: BaseAuthenticator): 24 | if auth.provider in self.enabled_authenticators: 25 | logger.warning(f"authenticator '{auth.provider}' already registered") 26 | return 27 | 28 | if not self._router: 29 | raise Exception( 30 | "AuthenticationRegistry not completely configure, you need to set the" 31 | "root router using set_router method" 32 | ) 33 | self._router.include_router(auth.router) 34 | self.enabled_authenticators[auth.provider] = auth 35 | logger.info( 36 | f"authentication provider '{auth.provider}' " 37 | f"of class {auth.__class__.__name__} registered" 38 | ) 39 | 40 | if len(self.enabled_authenticators) > 1: 41 | logger.warning( 42 | "You have registered multiple authentication providers." 43 | "Please note that this is currently discouraged in production setups!" 44 | ) 45 | 46 | def is_registered(self, provider_name: str): 47 | return provider_name in self.enabled_authenticators 48 | -------------------------------------------------------------------------------- /quetz/migrations/versions/d212023a8e0b_add_useremail_table_for_email_addresses.py: -------------------------------------------------------------------------------- 1 | """add Email table for email addresses 2 | 3 | Revision ID: d212023a8e0b 4 | Revises: cddba8e6e639 5 | Create Date: 2021-09-07 18:14:30.387156 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'd212023a8e0b' 13 | down_revision = 'cddba8e6e639' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | op.create_table( 20 | 'emails', 21 | sa.Column('provider', sa.String(), nullable=False), 22 | sa.Column('identity_id', sa.String(), nullable=False), 23 | sa.Column('email', sa.String(), nullable=False), 24 | sa.Column('user_id', sa.LargeBinary(length=16), nullable=True), 25 | sa.Column('verified', sa.Boolean(), nullable=True), 26 | sa.Column('primary', sa.Boolean(), nullable=True), 27 | sa.ForeignKeyConstraint( 28 | ['provider', 'identity_id'], 29 | ['identities.provider', 'identities.identity_id'], 30 | ), 31 | sa.ForeignKeyConstraint( 32 | ['user_id'], 33 | ['users.id'], 34 | ), 35 | sa.PrimaryKeyConstraint('provider', 'identity_id', 'email'), 36 | sa.UniqueConstraint('email'), 37 | ) 38 | with op.batch_alter_table('emails', schema=None) as batch_op: 39 | batch_op.create_index( 40 | 'email_index', ['provider', 'identity_id', 'email'], unique=True 41 | ) 42 | 43 | 44 | def downgrade(): 45 | with op.batch_alter_table('emails', schema=None) as batch_op: 46 | batch_op.drop_index('email_index') 47 | 48 | op.drop_table('emails') 49 | -------------------------------------------------------------------------------- /quetz/testing/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import signal 3 | from multiprocessing import active_children 4 | 5 | from starlette.requests import Request as ASGIRequest 6 | from starlette.responses import Response as ASGIResponse 7 | 8 | 9 | class AsyncPathMapDispatch: 10 | # dummy server, copied from authlib tests 11 | def __init__(self, path_maps): 12 | self.path_maps = path_maps 13 | 14 | async def __call__(self, scope, receive, send): 15 | request = ASGIRequest(scope, receive=receive) 16 | 17 | rv = self.path_maps[request.url.path] 18 | status_code = rv.get("status_code", 200) 19 | body = rv.get("body") 20 | headers = rv.get("headers", {}) 21 | if isinstance(body, dict): 22 | body = json.dumps(body).encode() 23 | headers["Content-Type"] = "application/json" 24 | else: 25 | if isinstance(body, str): 26 | body = body.encode() 27 | headers["Content-Type"] = "application/x-www-form-urlencoded" 28 | 29 | response = ASGIResponse( 30 | status_code=status_code, 31 | content=body, 32 | headers=headers, 33 | ) 34 | await response(scope, receive, send) 35 | 36 | 37 | class Interrupt: 38 | # Interrupt child when SIGALRM is received. 39 | # Useful to kill the server when it is correctly launched, using a timeout. 40 | def _handle_interrupt(self, signum, frame): 41 | for p in active_children(): 42 | p.terminate() 43 | p.join() 44 | 45 | def __enter__(self): 46 | signal.signal(signal.SIGALRM, self._handle_interrupt) 47 | 48 | def __exit__(self, exc_type, exc_val, exc_tb): 49 | pass 50 | -------------------------------------------------------------------------------- /utils/repodata_compare.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | my_headers = {"accept-encoding": "gzip"} 4 | 5 | anaconda = requests.get( 6 | "https://conda.anaconda.org/conda-forge/linux-64/repodata.json", headers=my_headers 7 | ) 8 | print("got anaconda") 9 | quetz = requests.get( 10 | "https://repo.mamba.pm/conda-forge/linux-64/repodata.json", headers=my_headers 11 | ) 12 | print("got quetz") 13 | 14 | anaconda_json = anaconda.json() 15 | quetz_json = quetz.json() 16 | 17 | anaconda_keys = anaconda_json["packages"].keys() 18 | quetz_keys = quetz_json["packages"].keys() 19 | 20 | anaconda_keys = set(anaconda_keys) 21 | quetz_keys = set(quetz_keys) 22 | 23 | diff = anaconda_keys - quetz_keys 24 | # print(anaconda_keys) 25 | print(diff) 26 | 27 | deep_compare_keys = anaconda_keys & quetz_keys 28 | 29 | 30 | def compare_pkg_rec(a, b): 31 | keys = a.keys() 32 | for key in keys: 33 | ax = a[key] 34 | if key not in b: 35 | print( 36 | f"Quetz does not have key {key} for" 37 | f"{a['name']} {a['version']} {a['build']}" 38 | ) 39 | continue 40 | 41 | bx = b[key] 42 | if isinstance(ax, list): 43 | ax = sorted(ax) 44 | bx = sorted(bx) 45 | 46 | if a[key] != b[key]: 47 | print( 48 | "Quetz has difference for " f"{a['name']} {a['version']} {a['build']}:" 49 | ) 50 | print(a[key]) 51 | print(b[key]) 52 | 53 | 54 | for k in deep_compare_keys: 55 | anaconda_dict = anaconda_json["packages"][k] 56 | quetz_dict = quetz_json["packages"][k] 57 | 58 | compare_pkg_rec(anaconda_dict, quetz_dict) 59 | print(f"Checked {len(deep_compare_keys)}") 60 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Making a new release of quetz server 2 | 3 | The quetz server can be published to `PyPI` manually or using the [Jupyter Releaser](https://github.com/jupyter-server/jupyter_releaser). 4 | 5 | ## Manual release 6 | 7 | ### Python package 8 | 9 | This extension can be distributed as Python 10 | packages. All of the Python 11 | packaging instructions in the `pyproject.toml` file to wrap your extension in a 12 | Python package. Before generating a package, we first need to install `build`. 13 | 14 | ```bash 15 | pip install build twine 16 | ``` 17 | 18 | To create a Python source package (`.tar.gz`) and the binary package (`.whl`) in the `dist/` directory, do: 19 | 20 | ```bash 21 | python -m build 22 | ``` 23 | 24 | > `python setup.py sdist bdist_wheel` is deprecated and will not work for this package. 25 | 26 | Then to upload the package to PyPI, do: 27 | 28 | ```bash 29 | twine upload dist/* 30 | ``` 31 | 32 | ## Automated releases with the Jupyter Releaser 33 | 34 | The extension repository should already be compatible with the Jupyter Releaser. 35 | 36 | Check out the [workflow documentation](https://github.com/jupyter-server/jupyter_releaser#typical-workflow) for more information. 37 | 38 | Here is a summary of the steps to cut a new release: 39 | 40 | - Fork the [`jupyter-releaser` repo](https://github.com/jupyter-server/jupyter_releaser) 41 | - Add `ADMIN_GITHUB_TOKEN` and `PYPI_TOKEN` to the Github Secrets in the fork 42 | - Go to the Actions panel 43 | - Run the "Draft Changelog" workflow 44 | - Merge the Changelog PR 45 | - Run the "Draft Release" workflow 46 | - Run the "Publish Release" workflow 47 | 48 | ## Publishing to `conda-forge` 49 | 50 | A bot should pick up the new version publish to PyPI, and open a new PR on the feedstock repository automatically. 51 | -------------------------------------------------------------------------------- /plugins/quetz_tos/quetz_tos/main.py: -------------------------------------------------------------------------------- 1 | from fastapi import HTTPException, status 2 | 3 | import quetz 4 | from quetz.authorization import OWNER 5 | from quetz.dao import Dao 6 | 7 | from .api import router 8 | from .db_models import TermsOfService, TermsOfServiceSignatures 9 | 10 | 11 | @quetz.hookimpl 12 | def register_router(): 13 | return router 14 | 15 | 16 | def check_for_signed_tos(db, user_id, user_role): 17 | dao = Dao(db) 18 | user = dao.get_user(user_id) 19 | if user: 20 | if user_role == OWNER: 21 | return True 22 | else: 23 | selected_tos = ( 24 | db.query(TermsOfService) 25 | .order_by(TermsOfService.time_created.desc()) 26 | .first() 27 | ) 28 | if selected_tos: 29 | signature = ( 30 | db.query(TermsOfServiceSignatures) 31 | .filter(TermsOfServiceSignatures.user_id == user_id) 32 | .filter(TermsOfServiceSignatures.tos_id == selected_tos.id) 33 | .one_or_none() 34 | ) 35 | if signature: 36 | return True 37 | else: 38 | detail = f"terms of service is not signed for {user.username}" 39 | raise HTTPException( 40 | status_code=status.HTTP_403_FORBIDDEN, detail=detail 41 | ) 42 | else: 43 | return True 44 | else: 45 | detail = f"user with id {user_id} not found" 46 | raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=detail) 47 | 48 | 49 | @quetz.hookimpl 50 | def check_additional_permissions(db, user_id, user_role): 51 | return check_for_signed_tos(db, user_id, user_role) 52 | -------------------------------------------------------------------------------- /quetz/migrations/versions/0653794b6252_adding_url_and_platforms_dirs.py: -------------------------------------------------------------------------------- 1 | """adding url and platforms dirs 2 | 3 | Revision ID: 0653794b6252 4 | Revises: 53f81aba78ce 5 | Create Date: 2021-01-11 18:18:44.530101 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '0653794b6252' 14 | down_revision = '53f81aba78ce' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | with op.batch_alter_table('aggregated_metrics', schema=None) as batch_op: 22 | batch_op.alter_column( 23 | 'period', 24 | existing_type=postgresql.INTERVAL(), 25 | type_=sa.Enum('hour', 'day', 'month', 'year', name='intervaltype'), 26 | existing_nullable=True, 27 | ) 28 | 29 | with op.batch_alter_table('packages', schema=None) as batch_op: 30 | batch_op.add_column(sa.Column('platforms', sa.String(), nullable=True)) 31 | batch_op.add_column(sa.Column('url', sa.String(), nullable=True)) 32 | 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade(): 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | with op.batch_alter_table('packages', schema=None) as batch_op: 39 | batch_op.drop_column('url') 40 | batch_op.drop_column('platforms') 41 | 42 | with op.batch_alter_table('aggregated_metrics', schema=None) as batch_op: 43 | batch_op.alter_column( 44 | 'period', 45 | existing_type=sa.Enum('hour', 'day', 'month', 'year', name='intervaltype'), 46 | type_=postgresql.INTERVAL(), 47 | existing_nullable=True, 48 | ) 49 | 50 | # ### end Alembic commands ### 51 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/quetz_content_trust/main.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pathlib import Path 3 | 4 | from sqlalchemy import desc 5 | 6 | import quetz 7 | from quetz.database import get_db_manager 8 | 9 | from . import db_models 10 | from .api import router 11 | 12 | logger = logging.getLogger("quetz") 13 | 14 | 15 | @quetz.hookimpl 16 | def register_router(): 17 | return router 18 | 19 | 20 | @quetz.hookimpl 21 | def post_index_creation(raw_repodata: dict, channel_name, subdir): 22 | """Use available online keys to sign packages""" 23 | 24 | with get_db_manager() as db: 25 | query = ( 26 | db.query(db_models.SigningKey) 27 | .join(db_models.RoleDelegation.keys) 28 | .filter( 29 | db_models.RoleDelegation.channel == channel_name, 30 | db_models.RoleDelegation.type == "pkg_mgr", 31 | db_models.SigningKey.private_key is not None, 32 | ) 33 | .order_by(desc("time_created")) 34 | .all() 35 | ) 36 | 37 | signatures = {} 38 | if query: 39 | import json 40 | 41 | from libmambapy import bindings as libmamba_api 42 | 43 | packages = raw_repodata.get("packages", {}) | raw_repodata.get( 44 | "packages.conda", {} 45 | ) 46 | for name, metadata in packages.items(): 47 | sig = libmamba_api.sign( 48 | json.dumps(metadata, indent=2, sort_keys=True), query[0].private_key 49 | ) 50 | if name not in signatures: 51 | signatures[name] = {} 52 | 53 | signatures[name][query[0].public_key] = dict(signature=sig) 54 | 55 | logger.info(f"Signed {Path(channel_name) / subdir}") 56 | raw_repodata["signatures"] = signatures 57 | -------------------------------------------------------------------------------- /quetz/authentication/github.py: -------------------------------------------------------------------------------- 1 | from .oauth2 import OAuthAuthenticator 2 | 3 | 4 | class GithubAuthenticator(OAuthAuthenticator): 5 | """Use Github account to authenticate users with Quetz. 6 | 7 | To enable add the following to the configuration file: 8 | 9 | .. code:: 10 | 11 | [github] 12 | client_id = "fde330aef1fbe39991" 13 | client_secret = "03728444a12abff17e9444fd231b4379d58f0b" 14 | 15 | You can obtain ``client_id`` and ``client_secret`` by registering your 16 | application with Github at this URL: 17 | ``_. 18 | """ 19 | 20 | provider = "github" 21 | collect_emails = False 22 | 23 | # oauth client params 24 | access_token_url = "https://github.com/login/oauth/access_token" 25 | authorize_url = "https://github.com/login/oauth/authorize" 26 | api_base_url = "https://api.github.com/" 27 | scope = "user:email" 28 | 29 | # endpoint urls 30 | validate_token_url = "user" 31 | revoke_url = "https://github.com/settings/connections/applications/{client_id}" 32 | 33 | async def userinfo(self, request, token): 34 | resp = await self.client.get("user", token=token) 35 | profile = resp.json() 36 | 37 | if self.collect_emails: 38 | emails = await self.client.get("user/emails", token=token) 39 | profile["emails"] = emails.json() 40 | 41 | return profile 42 | 43 | def configure(self, config): 44 | if config.configured_section("github"): 45 | self.client_id = config.github_client_id 46 | self.client_secret = config.github_client_secret 47 | self.is_enabled = True 48 | if config.configured_section("users"): 49 | self.collect_emails = config.users_collect_emails 50 | 51 | else: 52 | self.is_enabled = False 53 | 54 | # call the configure of base class to set default_channel and default role 55 | super().configure(config) 56 | -------------------------------------------------------------------------------- /plugins/quetz_tos/quetz_tos/migrations/versions/44ec522465e5_add_tables_for_terms_of_service_and_.py: -------------------------------------------------------------------------------- 1 | """add tables for terms of service and signatures 2 | 3 | Revision ID: 44ec522465e5 4 | Revises: 5 | Create Date: 2021-10-07 15:12:14.309053 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "44ec522465e5" 14 | down_revision = None 15 | branch_labels = ("quetz-tos",) 16 | depends_on = "quetz" 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "quetz_tos", 23 | sa.Column("id", sa.LargeBinary(length=16), nullable=False), 24 | sa.Column("uploader_id", sa.LargeBinary(length=16), nullable=True), 25 | sa.Column("filename", sa.String(), nullable=True), 26 | sa.Column( 27 | "time_created", 28 | sa.DateTime(), 29 | server_default=sa.text("(CURRENT_TIMESTAMP)"), 30 | nullable=False, 31 | ), 32 | sa.PrimaryKeyConstraint("id"), 33 | ) 34 | op.create_table( 35 | "quetz_tos_signatures", 36 | sa.Column("tos_id", sa.LargeBinary(length=16), nullable=False), 37 | sa.Column("user_id", sa.LargeBinary(length=16), nullable=False), 38 | sa.Column( 39 | "time_created", 40 | sa.DateTime(), 41 | server_default=sa.text("(CURRENT_TIMESTAMP)"), 42 | nullable=False, 43 | ), 44 | sa.ForeignKeyConstraint( 45 | ["tos_id"], 46 | ["quetz_tos.id"], 47 | ), 48 | sa.ForeignKeyConstraint( 49 | ["user_id"], 50 | ["users.id"], 51 | ), 52 | sa.PrimaryKeyConstraint("tos_id", "user_id"), 53 | sa.UniqueConstraint("tos_id", "user_id"), 54 | ) 55 | # ### end Alembic commands ### 56 | 57 | 58 | def downgrade(): 59 | # ### commands auto generated by Alembic - please adjust! ### 60 | op.drop_table("quetz_tos_signatures") 61 | op.drop_table("quetz_tos") 62 | # ### end Alembic commands ### 63 | -------------------------------------------------------------------------------- /quetz/migrations/versions/8d1e9a9e0b1f_adding_download_metrics.py: -------------------------------------------------------------------------------- 1 | """adding download metrics 2 | 3 | Revision ID: 8d1e9a9e0b1f 4 | Revises: 794249a0b1bd 5 | Create Date: 2020-12-18 12:29:29.994522 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = '8d1e9a9e0b1f' 13 | down_revision = '794249a0b1bd' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | 21 | enum_values = ('hour', 'day', 'month', 'year') 22 | if op.get_context().dialect.name == 'postgresql': 23 | intervaltype = sa.dialects.postgresql.ENUM( 24 | *enum_values, name='intervaltype', create_type=True 25 | ) 26 | else: 27 | intervaltype = sa.Enum(*enum_values, name='intervaltype') 28 | op.create_table( 29 | 'package_version_metrics', 30 | sa.Column('package_version_id', sa.LargeBinary(length=16), nullable=False), 31 | sa.Column('metric_name', sa.String(length=255), nullable=False), 32 | sa.Column( 33 | 'period', 34 | intervaltype, 35 | nullable=False, 36 | ), 37 | sa.Column('count', sa.Integer(), nullable=False, server_default=sa.text("0")), 38 | sa.Column('timestamp', sa.DateTime(), nullable=False), 39 | sa.ForeignKeyConstraint( 40 | ['package_version_id'], 41 | ['package_versions.id'], 42 | ), 43 | sa.PrimaryKeyConstraint( 44 | 'package_version_id', 'metric_name', 'period', 'timestamp' 45 | ), 46 | ) 47 | op.add_column( 48 | 'package_versions', sa.Column('download_count', sa.Integer(), nullable=True) 49 | ) 50 | # ### end Alembic commands ### 51 | 52 | 53 | def downgrade(): 54 | # ### commands auto generated by Alembic - please adjust! ### 55 | op.drop_column('package_versions', 'download_count') 56 | op.drop_table('package_version_metrics') 57 | if op.get_context().dialect.name == 'postgresql': 58 | op.execute("DROP TYPE intervaltype;") 59 | # ### end Alembic commands ### 60 | -------------------------------------------------------------------------------- /quetz/migrations/versions/b9886d9cadb0_create_indexes_for_download_count_.py: -------------------------------------------------------------------------------- 1 | """create indexes for download count queries 2 | 3 | Revision ID: b9886d9cadb0 4 | Revises: 0653794b6252 5 | Create Date: 2021-01-28 13:11:27.197409 6 | 7 | """ 8 | from alembic import op 9 | 10 | # revision identifiers, used by Alembic. 11 | revision = 'b9886d9cadb0' 12 | down_revision = '0653794b6252' 13 | branch_labels = None 14 | depends_on = None 15 | 16 | 17 | def upgrade(): 18 | # ### commands auto generated by Alembic - please adjust! ### 19 | with op.batch_alter_table('aggregated_metrics', schema=None) as batch_op: 20 | batch_op.create_index( 21 | 'package_version_metric_index', 22 | [ 23 | 'channel_name', 24 | 'platform', 25 | 'filename', 26 | 'metric_name', 27 | 'period', 28 | 'timestamp', 29 | ], 30 | unique=False, 31 | ) 32 | batch_op.create_unique_constraint( 33 | "package_version_metric_constraint", 34 | [ 35 | 'channel_name', 36 | 'platform', 37 | 'filename', 38 | 'metric_name', 39 | 'period', 40 | 'timestamp', 41 | ], 42 | ) 43 | 44 | with op.batch_alter_table('package_versions', schema=None) as batch_op: 45 | batch_op.create_index( 46 | 'package_version_filename_index', 47 | ['channel_name', 'filename', 'platform'], 48 | unique=True, 49 | ) 50 | 51 | # ### end Alembic commands ### 52 | 53 | 54 | def downgrade(): 55 | # ### commands auto generated by Alembic - please adjust! ### 56 | with op.batch_alter_table('package_versions', schema=None) as batch_op: 57 | batch_op.drop_index('package_version_filename_index') 58 | 59 | with op.batch_alter_table('aggregated_metrics', schema=None) as batch_op: 60 | batch_op.drop_constraint("package_version_metric_constraint", type_='unique') 61 | batch_op.drop_index('package_version_metric_index') 62 | 63 | # ### end Alembic commands ### 64 | -------------------------------------------------------------------------------- /quetz/templates/subdir-index.html.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | {{ title }} 4 | 44 | 45 | 46 |

{{ title }}

47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | {% for path in add_files %} 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | {%- endfor %} 64 | {% for fn, record in packages.items() %} 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | {%- endfor %} 73 |
FilenameSizeLast ModifiedSHA256MD5
{{ path.name | opt_href(path.name) }}{{ path.size | iec_bytes }}{% if path.timestamp %}{{ path.timestamp|strftime("%Y-%m-%d %H:%M:%S %z") }}{% endif %}{{ path.sha256 }}{{ path.md5 }}
{{ fn | opt_href(fn) }}{{ record.size | iec_bytes }}{% if record.timestamp %}{{ record.timestamp|strftime("%Y-%m-%d %H:%M:%S %z") }}{% endif %}{{ record.sha256 }}{{ record.md5 }}
74 |
Updated: {{ current_time|strftime("%Y-%m-%d %H:%M:%S %z") }} - Files: {{ packages|length }}
75 | 76 | 77 | -------------------------------------------------------------------------------- /quetz/migrations/versions/53f81aba78ce_use_biginteger_for_size.py: -------------------------------------------------------------------------------- 1 | """use BigInteger for size 2 | 3 | Revision ID: 53f81aba78ce 4 | Revises: 30241b33d849 5 | Create Date: 2021-01-08 22:09:48.268270 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = '53f81aba78ce' 13 | down_revision = '30241b33d849' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | if op.get_context().dialect.name == 'postgresql': 21 | op.alter_column( 22 | 'channels', 23 | 'size', 24 | existing_type=sa.INTEGER(), 25 | type_=sa.BigInteger(), 26 | existing_nullable=True, 27 | ) 28 | op.alter_column( 29 | 'channels', 30 | 'size_limit', 31 | existing_type=sa.INTEGER(), 32 | type_=sa.BigInteger(), 33 | existing_nullable=True, 34 | ) 35 | 36 | op.alter_column( 37 | 'package_versions', 38 | 'size', 39 | existing_type=sa.INTEGER(), 40 | type_=sa.BigInteger(), 41 | existing_nullable=True, 42 | ) 43 | 44 | # ### end Alembic commands ### 45 | 46 | 47 | def downgrade(): 48 | # ### commands auto generated by Alembic - please adjust! ### 49 | if op.get_context().dialect.name == 'postgresql': 50 | op.alter_column( 51 | 'package_versions', 52 | 'size', 53 | existing_type=sa.BigInteger(), 54 | type_=sa.INTEGER(), 55 | existing_nullable=True, 56 | ) 57 | 58 | op.alter_column( 59 | 'channels', 60 | 'size_limit', 61 | existing_type=sa.BigInteger(), 62 | type_=sa.INTEGER(), 63 | existing_nullable=True, 64 | ) 65 | op.alter_column( 66 | 'channels', 67 | 'size', 68 | existing_type=sa.BigInteger(), 69 | type_=sa.INTEGER(), 70 | existing_nullable=True, 71 | ) 72 | 73 | # ### end Alembic commands ### 74 | -------------------------------------------------------------------------------- /quetz/migrations/versions/3ba25f23fb7d_update_scoped_api_key_uploader_id.py: -------------------------------------------------------------------------------- 1 | """Update scoped API key uploader id 2 | 3 | Revision ID: 3ba25f23fb7d 4 | Revises: d212023a8e0b 5 | Create Date: 2023-08-02 08:03:09.961559 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = '3ba25f23fb7d' 13 | down_revision = 'd212023a8e0b' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | package_versions = sa.sql.table("package_versions", sa.sql.column("uploader_id")) 20 | conn = op.get_bind() 21 | # Get all user_id/owner_id from channel scoped API keys 22 | # (user is anonymous - username is null) 23 | res = conn.execute( 24 | sa.text( 25 | """SELECT api_keys.user_id, api_keys.owner_id FROM api_keys 26 | INNER JOIN users ON users.id = api_keys.user_id 27 | WHERE users.username is NULL; 28 | """ 29 | ) 30 | ) 31 | results = res.fetchall() 32 | # Replace the uploader with the key owner (real user instead of the anonymous one) 33 | for result in results: 34 | op.execute( 35 | package_versions.update() 36 | .where(package_versions.c.uploader_id == result[0]) 37 | .values(uploader_id=result[1]) 38 | ) 39 | 40 | 41 | def downgrade(): 42 | package_versions = sa.sql.table("package_versions", sa.sql.column("uploader_id")) 43 | conn = op.get_bind() 44 | # Get all user_id/owner_id from channel scoped API keys 45 | # (user is anonymous - username is null) 46 | res = conn.execute( 47 | sa.text( 48 | """SELECT api_keys.user_id, api_keys.owner_id FROM api_keys 49 | INNER JOIN users ON users.id = api_keys.user_id 50 | WHERE users.username is NULL; 51 | """ 52 | ) 53 | ) 54 | results = res.fetchall() 55 | # Replace the uploader with the key anonymous user 56 | for result in results: 57 | op.execute( 58 | package_versions.update() 59 | .where(package_versions.c.uploader_id == result[1]) 60 | .values(uploader_id=result[0]) 61 | ) 62 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | 16 | sys.path.insert(0, os.path.abspath("../..")) 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = "Quetz" 22 | copyright = "2020, QuantStack" 23 | author = "QuantStack" 24 | 25 | # The full version, including alpha/beta/rc tags 26 | release = "0.1" 27 | 28 | 29 | # -- General configuration --------------------------------------------------- 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ["sphinx.ext.autodoc"] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ["_templates"] 38 | 39 | # List of patterns, relative to source directory, that match files and 40 | # directories to ignore when looking for source files. 41 | # This pattern also affects html_static_path and html_extra_path. 42 | exclude_patterns = [] 43 | 44 | 45 | # -- Options for HTML output ------------------------------------------------- 46 | 47 | # The theme to use for HTML and HTML Help pages. See the documentation for 48 | # a list of builtin themes. 49 | # 50 | html_theme = "sphinx_book_theme" 51 | html_logo = "_static/quetz_logo.png" 52 | html_title = "documentation" 53 | html_favicon = "_static/quetz_favicon.ico" 54 | 55 | # Add any paths that contain custom static files (such as style sheets) here, 56 | # relative to this directory. They are copied after the builtin static files, 57 | # so a file named "default.css" will overwrite the builtin "default.css". 58 | html_static_path = ["_static"] 59 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | web: 5 | image: quetz_dev_image 6 | container_name: quetz-web 7 | ports: 8 | - "8000:8000" 9 | build: 10 | context: . 11 | dockerfile: ./docker/Dockerfile 12 | environment: 13 | - HTTPX_LOG_LEVEL=TRACE 14 | command: 15 | [ 16 | "sh", 17 | "-c", 18 | "./docker/wait-for-postgres.sh database && quetz start /quetz-deployment --host 0.0.0.0 --port 8000 --reload", 19 | ] 20 | depends_on: 21 | - database 22 | - init-db 23 | volumes: 24 | - .:/code 25 | - quetz_deployment:/quetz-deployment 26 | env_file: 27 | - docker/postgres.env 28 | init-db: 29 | image: quetz_dev_image 30 | command: 31 | [ 32 | "sh", 33 | "-c", 34 | "./docker/wait-for-postgres.sh database && quetz create /quetz-deployment --copy-conf /code/docker/docker_config.toml --exists-ok && quetz init-db /quetz-deployment", 35 | ] 36 | depends_on: 37 | - database 38 | volumes: 39 | - .:/code 40 | - quetz_deployment:/quetz-deployment 41 | env_file: 42 | - docker/postgres.env 43 | database: 44 | image: postgres 45 | volumes: 46 | - ./docker/postgres.conf:/etc/postgresql/postgresql.conf 47 | env_file: 48 | - docker/postgres.env 49 | prometheus: 50 | image: prom/prometheus 51 | volumes: 52 | - ./docker/prometheus.yml:/etc/prometheus/prometheus.yml 53 | grafana: 54 | image: grafana/grafana 55 | ports: 56 | - 3000:3000 57 | volumes: 58 | - ./docker/graphana_datasources.yml:/etc/grafana/provisioning/datasources/datasource.yaml 59 | env_file: 60 | - docker/grafana.env 61 | jupyterhub: 62 | build: 63 | context: docker 64 | dockerfile: Dockerfile.jupyterhub 65 | command: jupyterhub --debug 66 | ports: 67 | - 8001:8000 68 | nginx: 69 | image: nginx:stable 70 | container_name: quetz-nginx 71 | entrypoint: ["nginx", "-g", "daemon off;"] 72 | ports: 73 | - "8080:8080" 74 | depends_on: 75 | - web 76 | volumes: 77 | - ./docker/nginx.conf:/etc/nginx/nginx.conf 78 | - quetz_deployment:/quetz-deployment 79 | 80 | volumes: 81 | quetz_deployment: 82 | -------------------------------------------------------------------------------- /docker/nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes 1; 2 | # Nginx shall have read access to files created by Quetz 3 | # Same user should be used to run both applications 4 | # Using root for development even if it's not recommended 5 | user root; 6 | pid /tmp/nginx.pid; 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | http { 13 | map $cache $control { 14 | 1 "max-age=1200"; 15 | } 16 | map $uri $cache { 17 | ~*\.(json)$ 1; 18 | } 19 | proxy_temp_path /tmp/proxy_temp; 20 | client_body_temp_path /tmp/client_temp; 21 | fastcgi_temp_path /tmp/fastcgi_temp; 22 | uwsgi_temp_path /tmp/uwsgi_temp; 23 | scgi_temp_path /tmp/scgi_temp; 24 | 25 | include mime.types; 26 | default_type application/octet-stream; 27 | 28 | sendfile on; 29 | tcp_nopush on; 30 | tcp_nodelay on; 31 | 32 | keepalive_timeout 65; 33 | 34 | gzip on; 35 | gzip_types application/json; 36 | 37 | client_max_body_size 100m; 38 | 39 | upstream quetz { 40 | server quetz-web:8000; 41 | } 42 | 43 | map $uri $file_name { 44 | default none; 45 | "~*/files/channels/(?.*)" channels/$name; 46 | } 47 | 48 | server { 49 | listen 8080; 50 | add_header Cache-Control $control; 51 | 52 | server_name localhost; 53 | 54 | location / { 55 | proxy_set_header Host $http_host; 56 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 57 | proxy_set_header X-Forwarded-Proto $scheme; 58 | proxy_set_header Upgrade $http_upgrade; 59 | proxy_set_header Connection $connection_upgrade; 60 | proxy_redirect off; 61 | proxy_buffering off; 62 | proxy_pass http://quetz; 63 | } 64 | 65 | # path for channels 66 | location /files/channels/ { 67 | # secure_link $arg_md5,$arg_expires; 68 | # secure_link_md5 "$secure_link_expires$file_name mysecrettoken"; 69 | 70 | # if ($secure_link = "") { return 403; } 71 | # if ($secure_link = "0") { return 410; } 72 | 73 | alias /quetz-deployment/channels/; 74 | } 75 | } 76 | 77 | map $http_upgrade $connection_upgrade { 78 | default upgrade; 79 | '' close; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /quetz/tests/authentification/test_pam.py: -------------------------------------------------------------------------------- 1 | import getpass 2 | from unittest import mock 3 | 4 | import pytest 5 | from fastapi import Request 6 | 7 | from quetz.authentication.pam import PAMAuthenticator 8 | 9 | 10 | @pytest.fixture 11 | def groups(): 12 | return {} 13 | 14 | 15 | @pytest.fixture 16 | def config_extra(groups): 17 | return f""" 18 | [pamauthenticator] 19 | admin_groups = {groups.get('admins', [])} 20 | maintainer_groups = {groups.get('maintainers', [])} 21 | member_groups = {groups.get('members', [])}""" 22 | 23 | 24 | @pytest.mark.asyncio 25 | @pytest.mark.parametrize( 26 | "groups,expected_role", 27 | [ 28 | ( 29 | {}, 30 | None, 31 | ), 32 | ( 33 | {"admins": ["usergroup"]}, 34 | "owner", 35 | ), 36 | ( 37 | {"admins": ["usergroup"], "maintainers": ["usergroup"]}, 38 | "owner", 39 | ), 40 | ( 41 | {"maintainers": ["usergroup"]}, 42 | "maintainer", 43 | ), 44 | ( 45 | {"members": ["usergroup"]}, 46 | "member", 47 | ), 48 | pytest.param( 49 | {"members": ["missinggroup"]}, 50 | None, 51 | id="missing-group", 52 | ), 53 | ], 54 | ) 55 | async def test_user_role(config, expected_role): 56 | auth = PAMAuthenticator(config) 57 | request = Request(scope={"type": "http"}) 58 | 59 | _group_ids = {"usergroup": 1001} 60 | _user_group_ids = {"quetzuser": [1001, 1002]} 61 | 62 | with mock.patch.multiple( 63 | auth, 64 | _get_group_id_by_name=lambda k: _group_ids[k], 65 | _get_user_group_ids=lambda k: _user_group_ids[k], 66 | ): 67 | role = await auth.user_role(request, {"login": "quetzuser"}) 68 | 69 | if expected_role is None: 70 | assert role is None 71 | else: 72 | assert role == expected_role 73 | 74 | 75 | @pytest.mark.asyncio 76 | async def test_authenticate(config): 77 | auth = PAMAuthenticator(config) 78 | request = Request(scope={"type": "http"}) 79 | current_user = getpass.getuser() 80 | 81 | result = await auth.authenticate( 82 | request, {"username": current_user, "password": "test"} 83 | ) 84 | 85 | assert result is None # authentication failed due to incorrect password 86 | -------------------------------------------------------------------------------- /alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | script_location = quetz:migrations 6 | 7 | # template used to generate migration files 8 | # file_template = %%(rev)s_%%(slug)s 9 | 10 | # timezone to use when rendering the date 11 | # within the migration file as well as the filename. 12 | # string value is passed to dateutil.tz.gettz() 13 | # leave blank for localtime 14 | # timezone = 15 | 16 | # max length of characters to apply to the 17 | # "slug" field 18 | # truncate_slug_length = 40 19 | 20 | # set to 'true' to run the environment during 21 | # the 'revision' command, regardless of autogenerate 22 | # revision_environment = false 23 | 24 | # set to 'true' to allow .pyc and .pyo files without 25 | # a source .py file to be detected as revisions in the 26 | # versions/ directory 27 | # sourceless = false 28 | 29 | # version location specification; this defaults 30 | # to migrations/versions. When using multiple version 31 | # directories, initial revisions must be specified with --version-path 32 | version_locations = quetz:migrations/versions 33 | 34 | # the output encoding used when revision files 35 | # are written from script.py.mako 36 | # output_encoding = utf-8 37 | 38 | #sqlalchemy.url = driver://user:pass@localhost/dbname 39 | 40 | 41 | [post_write_hooks] 42 | # post_write_hooks defines scripts or Python functions that are run 43 | # on newly generated revision scripts. See the documentation for further 44 | # detail and examples 45 | 46 | # format using "black" - use the console_scripts runner, against the "black" entrypoint 47 | # hooks=black 48 | # black.type=console_scripts 49 | # black.entrypoint=black 50 | # black.options=-l 79 51 | 52 | # Logging configuration 53 | [loggers] 54 | keys = root,sqlalchemy,alembic 55 | 56 | [handlers] 57 | keys = console 58 | 59 | [formatters] 60 | keys = generic 61 | 62 | [logger_root] 63 | level = WARN 64 | handlers = console 65 | qualname = 66 | 67 | [logger_sqlalchemy] 68 | level = WARN 69 | handlers = 70 | qualname = sqlalchemy.engine 71 | 72 | [logger_alembic] 73 | level = INFO 74 | handlers = 75 | qualname = alembic 76 | 77 | [handler_console] 78 | class = StreamHandler 79 | args = (sys.stderr,) 80 | level = NOTSET 81 | formatter = generic 82 | 83 | [formatter_generic] 84 | format = %(levelname)-5.5s [%(name)s] %(message)s 85 | datefmt = %H:%M:%S 86 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | script_location = quetz_runexports:migrations 6 | 7 | # template used to generate migration files 8 | # file_template = %%(rev)s_%%(slug)s 9 | 10 | # timezone to use when rendering the date 11 | # within the migration file as well as the filename. 12 | # string value is passed to dateutil.tz.gettz() 13 | # leave blank for localtime 14 | # timezone = 15 | 16 | # max length of characters to apply to the 17 | # "slug" field 18 | # truncate_slug_length = 40 19 | 20 | # set to 'true' to run the environment during 21 | # the 'revision' command, regardless of autogenerate 22 | # revision_environment = false 23 | 24 | # set to 'true' to allow .pyc and .pyo files without 25 | # a source .py file to be detected as revisions in the 26 | # versions/ directory 27 | # sourceless = false 28 | 29 | # version location specification; this defaults 30 | # to migrations/versions. When using multiple version 31 | # directories, initial revisions must be specified with --version-path 32 | # version_locations = %(here)s/bar %(here)s/bat migrations/versions 33 | 34 | # the output encoding used when revision files 35 | # are written from script.py.mako 36 | # output_encoding = utf-8 37 | 38 | #sqlalchemy.url = driver://user:pass@localhost/dbname 39 | 40 | 41 | [post_write_hooks] 42 | # post_write_hooks defines scripts or Python functions that are run 43 | # on newly generated revision scripts. See the documentation for further 44 | # detail and examples 45 | 46 | # format using "black" - use the console_scripts runner, against the "black" entrypoint 47 | # hooks=black 48 | # black.type=console_scripts 49 | # black.entrypoint=black 50 | # black.options=-l 79 51 | 52 | # Logging configuration 53 | [loggers] 54 | keys = root,sqlalchemy,alembic 55 | 56 | [handlers] 57 | keys = console 58 | 59 | [formatters] 60 | keys = generic 61 | 62 | [logger_root] 63 | level = WARN 64 | handlers = console 65 | qualname = 66 | 67 | [logger_sqlalchemy] 68 | level = WARN 69 | handlers = 70 | qualname = sqlalchemy.engine 71 | 72 | [logger_alembic] 73 | level = INFO 74 | handlers = 75 | qualname = alembic 76 | 77 | [handler_console] 78 | class = StreamHandler 79 | args = (sys.stderr,) 80 | level = NOTSET 81 | formatter = generic 82 | 83 | [formatter_generic] 84 | format = %(levelname)-5.5s [%(name)s] %(message)s 85 | datefmt = %H:%M:%S 86 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import json 2 | import uuid 3 | 4 | from pytest import fixture 5 | from quetz_runexports import db_models 6 | 7 | from quetz import rest_models 8 | from quetz.dao import Dao 9 | from quetz.db_models import User 10 | 11 | pytest_plugins = "quetz.testing.fixtures" 12 | 13 | 14 | @fixture 15 | def dao(db) -> Dao: 16 | return Dao(db) 17 | 18 | 19 | @fixture 20 | def user(db): 21 | user = User(id=uuid.uuid4().bytes, username="bartosz") 22 | db.add(user) 23 | db.commit() 24 | yield user 25 | 26 | 27 | @fixture 28 | def channel(dao, user, db): 29 | channel_data = rest_models.Channel( 30 | name="test-mirror-channel", 31 | private=False, 32 | mirror_channel_url="http://host", 33 | mirror_mode="mirror", 34 | ) 35 | 36 | channel = dao.create_channel(channel_data, user.id, "owner") 37 | 38 | yield channel 39 | 40 | db.delete(channel) 41 | db.commit() 42 | 43 | 44 | @fixture 45 | def package(dao, user, channel, db): 46 | new_package_data = rest_models.Package(name="test-package") 47 | 48 | package = dao.create_package( 49 | channel.name, 50 | new_package_data, 51 | user_id=user.id, 52 | role="owner", 53 | ) 54 | 55 | yield package 56 | 57 | db.delete(package) 58 | db.commit() 59 | 60 | 61 | @fixture 62 | def package_version(user, channel, db, dao, package): 63 | # create package version that will added to local repodata 64 | package_format = "tarbz2" 65 | package_info = '{"size": 5000, "subdirs":["noarch"]}' 66 | 67 | version = dao.create_version( 68 | channel.name, 69 | package.name, 70 | package_format, 71 | "noarch", 72 | "0.1", 73 | "0", 74 | "0", 75 | "test-package-0.1-0.tar.bz2", 76 | package_info, 77 | user.id, 78 | size=5000, 79 | ) 80 | 81 | yield version 82 | 83 | db.delete(version) 84 | db.commit() 85 | 86 | 87 | @fixture 88 | def package_runexports(package_version, db): 89 | meta = db_models.PackageVersionMetadata( 90 | version_id=package_version.id, 91 | data=json.dumps({"weak": ["somepackage > 3.0"]}), 92 | ) 93 | 94 | db.add(meta) 95 | db.commit() 96 | 97 | yield meta 98 | 99 | db.delete(meta) 100 | db.commit() 101 | -------------------------------------------------------------------------------- /quetz/authentication/google.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 QuantStack, Codethink Ltd 2 | # Distributed under the terms of the Modified BSD License. 3 | 4 | from quetz.config import Config 5 | 6 | from .oauth2 import OAuthAuthenticator 7 | 8 | 9 | class GoogleAuthenticator(OAuthAuthenticator): 10 | """Use Google account to authenticate users with Quetz. 11 | 12 | To enable add the following to the configuration file: 13 | 14 | .. code:: 15 | 16 | [google] 17 | client_id = "1111111111-dha39auqzp92110sdf.apps.googleusercontent.com" 18 | client_secret = "03728444a12abff17e9444fd231b4379d58f0b" 19 | 20 | You can obtain ``client_id`` and ``client_secret`` by registering your 21 | application with Google platform at this URL: 22 | ``_. 23 | """ 24 | 25 | provider = "google" 26 | server_metadata_url = "https://accounts.google.com/.well-known/openid-configuration" 27 | scope = "openid email profile" 28 | prompt = "select_account" 29 | 30 | revoke_url = "https://myaccount.google.com/permissions" 31 | validate_token_url = "https://openidconnect.googleapis.com/v1/userinfo" 32 | 33 | collect_emails = False 34 | 35 | async def userinfo(self, request, token): 36 | profile = await self.client.parse_id_token(request, token) 37 | 38 | github_profile = { 39 | "id": profile["sub"], 40 | "name": profile["name"], 41 | "avatar_url": profile["picture"], 42 | "email": profile["email"], 43 | "login": profile["email"], 44 | } 45 | 46 | if self.collect_emails: 47 | github_profile["emails"] = [ 48 | { 49 | "email": profile["email"], 50 | "primary": True, 51 | "verified": profile["email_verified"], 52 | } 53 | ] 54 | 55 | return github_profile 56 | 57 | def configure(self, config: Config): 58 | if config.configured_section("google"): 59 | self.client_id = config.google_client_id 60 | self.client_secret = config.google_client_secret 61 | self.is_enabled = True 62 | if config.configured_section("users"): 63 | self.collect_emails = config.users_collect_emails 64 | 65 | else: 66 | self.is_enabled = False 67 | 68 | # call the configure of base class to set default_channel and default role 69 | super().configure(config) 70 | -------------------------------------------------------------------------------- /plugins/quetz_runexports/tests/test_quetz_runexports.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import tempfile 3 | from contextlib import contextmanager 4 | from unittest import mock 5 | 6 | import pytest 7 | from quetz_runexports import db_models 8 | 9 | from quetz.condainfo import CondaInfo 10 | 11 | pytest_plugins = "quetz.testing.fixtures" 12 | 13 | 14 | @pytest.fixture 15 | def plugins(): 16 | return ["quetz-runexports"] 17 | 18 | 19 | def test_run_exports_endpoint( 20 | client, 21 | channel, 22 | package, 23 | package_version, 24 | package_runexports, 25 | db, 26 | session_maker, 27 | ): 28 | filename = package_version.filename 29 | platform = package_version.platform 30 | 31 | response = client.get( 32 | f"/api/channels/{channel.name}/packages/{package.name}/versions/{platform}/{filename}/run_exports" # noqa 33 | ) 34 | assert response.status_code == 200 35 | assert response.json() == {"weak": ["somepackage > 3.0"]} 36 | 37 | 38 | def test_endpoint_without_metadata( 39 | client, channel, package, package_version, db, session_maker 40 | ): 41 | filename = package_version.filename 42 | platform = package_version.platform 43 | 44 | response = client.get( 45 | f"/api/channels/{channel.name}/packages/{package.name}/versions/{platform}/{filename}/run_exports" # noqa 46 | ) 47 | assert response.status_code == 404 48 | 49 | 50 | def test_post_add_package_version(package_version, config, db, session_maker): 51 | filename = "test-package-0.1-0.tar.bz2" 52 | 53 | with tempfile.SpooledTemporaryFile(mode="wb") as target: 54 | with open(filename, "rb") as fid: 55 | shutil.copyfileobj(fid, target) 56 | target.seek(0) 57 | condainfo = CondaInfo(target, filename) 58 | 59 | @contextmanager 60 | def get_db(): 61 | yield db 62 | 63 | from quetz_runexports import main 64 | 65 | with mock.patch("quetz_runexports.main.get_db_manager", get_db): 66 | main.post_add_package_version(package_version, condainfo) 67 | 68 | meta = db.query(db_models.PackageVersionMetadata).first() 69 | 70 | assert meta.data == "{}" 71 | 72 | # modify runexport and re-save 73 | condainfo.run_exports = {"weak": ["somepackage < 0.3"]} 74 | with mock.patch("quetz_runexports.main.get_db_manager", get_db): 75 | main.post_add_package_version(package_version, condainfo) 76 | 77 | meta = db.query(db_models.PackageVersionMetadata).all() 78 | 79 | assert len(meta) == 1 80 | 81 | assert meta[0].data == '{"weak": ["somepackage < 0.3"]}' 82 | -------------------------------------------------------------------------------- /plugins/quetz_transmutation/quetz_transmutation/jobs.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import shutil 5 | from pathlib import Path 6 | from tempfile import TemporaryDirectory 7 | 8 | from conda_package_handling.api import _convert 9 | 10 | from quetz.condainfo import calculate_file_hashes_and_size 11 | from quetz.dao import Dao 12 | from quetz.pkgstores import PackageStore 13 | 14 | logger = logging.getLogger("quetz.plugins") 15 | 16 | 17 | def transmutation(package_version: dict, config, pkgstore: PackageStore, dao: Dao): 18 | filename: str = package_version["filename"] 19 | channel: str = package_version["channel_name"] 20 | package_format: str = package_version["package_format"] 21 | package_name: str = package_version["package_name"] 22 | platform = package_version["platform"] 23 | version = package_version["version"] 24 | build_number = package_version["build_number"] 25 | build_string = package_version["build_string"] 26 | uploader_id = package_version["uploader_id"] 27 | info = json.loads(package_version["info"]) 28 | 29 | if package_format == "tarbz2" or not filename.endswith(".tar.bz2"): 30 | return 31 | 32 | fh = pkgstore.serve_path(channel, Path(platform) / filename) 33 | 34 | with TemporaryDirectory() as tmpdirname: 35 | local_file_name = os.path.join(tmpdirname, filename) 36 | with open(local_file_name, "wb") as local_file: 37 | # chunk size 10MB 38 | shutil.copyfileobj(fh, local_file, 10 * 1024 * 1024) 39 | 40 | fn, out_fn, errors = _convert(local_file_name, ".conda", tmpdirname, force=True) 41 | 42 | if errors: 43 | logger.error(f"transmutation errors --> {errors}") 44 | return 45 | 46 | filename_conda = os.path.basename(filename).replace(".tar.bz2", ".conda") 47 | 48 | logger.info(f"Adding file to package store: {Path(platform) / filename_conda}") 49 | 50 | with open(out_fn, "rb") as f: 51 | calculate_file_hashes_and_size(info, f) 52 | f.seek(0) 53 | pkgstore.add_package(f, channel, str(Path(platform) / filename_conda)) 54 | 55 | version = dao.create_version( 56 | channel, 57 | package_name, 58 | "conda", 59 | platform, 60 | version, 61 | build_number, 62 | build_string, 63 | filename_conda, 64 | json.dumps(info), 65 | uploader_id, 66 | info["size"], 67 | upsert=True, 68 | ) 69 | 70 | if os.path.exists(out_fn): 71 | os.remove(out_fn) 72 | -------------------------------------------------------------------------------- /plugins/quetz_conda_suggest/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import json 2 | import uuid 3 | 4 | from pytest import fixture 5 | from quetz_conda_suggest import db_models 6 | 7 | from quetz import rest_models 8 | from quetz.dao import Dao 9 | from quetz.db_models import Profile, User 10 | 11 | pytest_plugins = "quetz.testing.fixtures" 12 | 13 | 14 | @fixture 15 | def dao(db) -> Dao: 16 | return Dao(db) 17 | 18 | 19 | @fixture 20 | def user(db): 21 | user = User(id=uuid.uuid4().bytes, username="madhurt") 22 | db.add(user) 23 | db.commit() 24 | yield user 25 | 26 | 27 | @fixture 28 | def profile(db, user): 29 | user_profile = Profile( 30 | name="madhur", avatar_url="madhur-tandon", user_id=user.id, user=user 31 | ) 32 | db.add(user_profile) 33 | db.commit() 34 | yield user 35 | 36 | 37 | @fixture 38 | def channel(dao, user, db): 39 | channel_data = rest_models.Channel( 40 | name="test-channel", 41 | private=False, 42 | ) 43 | 44 | channel = dao.create_channel(channel_data, user.id, "owner") 45 | 46 | yield channel 47 | 48 | db.delete(channel) 49 | db.commit() 50 | 51 | 52 | @fixture 53 | def package(dao, user, channel, db): 54 | new_package_data = rest_models.Package(name="test-package") 55 | 56 | package = dao.create_package( 57 | channel.name, 58 | new_package_data, 59 | user_id=user.id, 60 | role="owner", 61 | ) 62 | 63 | yield package 64 | 65 | db.delete(package) 66 | db.commit() 67 | 68 | 69 | @fixture 70 | def package_version(user, channel, db, dao, package): 71 | package_format = "tarbz2" 72 | package_info = '{"size": 5000, "subdir": "linux-64"}' 73 | 74 | version = dao.create_version( 75 | channel.name, 76 | package.name, 77 | package_format, 78 | "linux-64", 79 | "0.1", 80 | "0", 81 | "0", 82 | "test-package-0.1-0.tar.bz2", 83 | package_info, 84 | user.id, 85 | size=0, 86 | ) 87 | 88 | yield version 89 | 90 | db.delete(version) 91 | db.commit() 92 | 93 | 94 | @fixture 95 | def subdir(): 96 | return "linux-64" 97 | 98 | 99 | @fixture 100 | def package_conda_suggest(package_version, db): 101 | meta = db_models.CondaSuggestMetadata( 102 | version_id=package_version.id, 103 | data=json.dumps({"test-bin": "test-package"}), 104 | ) 105 | 106 | db.add(meta) 107 | db.commit() 108 | 109 | yield meta 110 | 111 | db.delete(meta) 112 | db.commit() 113 | -------------------------------------------------------------------------------- /quetz/tests/api/conftest.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from quetz.db_models import Identity, Profile, User 7 | from quetz.rest_models import Channel, Package 8 | 9 | 10 | @pytest.fixture 11 | def private_channel(dao, other_user): 12 | channel_name = "private-channel" 13 | 14 | channel_data = Channel(name=channel_name, private=True) 15 | channel = dao.create_channel(channel_data, other_user.id, "owner") 16 | 17 | return channel 18 | 19 | 20 | @pytest.fixture 21 | def private_package(dao, other_user, private_channel): 22 | package_name = "private-package" 23 | package_data = Package(name=package_name) 24 | package = dao.create_package( 25 | private_channel.name, package_data, other_user.id, "owner" 26 | ) 27 | 28 | return package 29 | 30 | 31 | @pytest.fixture 32 | def private_package_version( 33 | dao, private_channel, private_package, other_user, config, package_name 34 | ): 35 | package_format = "tarbz2" 36 | package_info = "{}" 37 | channel_name = private_channel.name 38 | filename = Path("test-package-0.1-0.tar.bz2") 39 | 40 | pkgstore = config.get_package_store() 41 | with open(filename, "rb") as fid: 42 | pkgstore.add_file(fid.read(), channel_name, "linux-64" / filename) 43 | 44 | platform = "linux-64" 45 | version = dao.create_version( 46 | private_channel.name, 47 | private_package.name, 48 | package_format, 49 | platform, 50 | "0.1", 51 | "0", 52 | "", 53 | str(filename), 54 | package_info, 55 | other_user.id, 56 | size=0, 57 | ) 58 | 59 | dao.update_package_channeldata( 60 | private_channel.name, 61 | private_package.name, 62 | {"name": package_name, "subdirs": [platform]}, 63 | ) 64 | 65 | return version 66 | 67 | 68 | @pytest.fixture() 69 | def other_user_without_profile(db): 70 | user = User(id=uuid.uuid4().bytes, username="other") 71 | db.add(user) 72 | return user 73 | 74 | 75 | @pytest.fixture 76 | def other_user(other_user_without_profile, db): 77 | profile = Profile( 78 | name="Other", avatar_url="http:///avatar", user=other_user_without_profile 79 | ) 80 | identity = Identity( 81 | provider="github", 82 | identity_id="github", 83 | username="btel", 84 | user=other_user_without_profile, 85 | ) 86 | db.add(profile) 87 | db.add(identity) 88 | db.commit() 89 | yield other_user_without_profile 90 | 91 | 92 | @pytest.fixture 93 | def pkgstore(config): 94 | return config.get_package_store() 95 | -------------------------------------------------------------------------------- /quetz/metrics/api.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import Optional 3 | 4 | from fastapi import APIRouter, Depends, HTTPException, status 5 | 6 | from quetz import db_models 7 | from quetz.dao import Dao 8 | from quetz.deps import get_dao, get_package_or_fail 9 | from quetz.metrics import rest_models 10 | from quetz.metrics.db_models import IntervalType 11 | 12 | api_router = APIRouter(prefix="/metrics") 13 | 14 | 15 | @api_router.get( 16 | "/channels/{channel_name}/packages/{package_name}/versions/{platform}/{filename}", # noqa 17 | response_model=rest_models.PackageVersionMetricResponse, 18 | tags=["metrics"], 19 | ) 20 | def get_package_version_metrics( 21 | platform: str, 22 | filename: str, 23 | package_name: str, 24 | channel_name: str, 25 | fill_zeros: bool = False, 26 | period: IntervalType = IntervalType.day, 27 | metric_name: str = "download", 28 | start: Optional[datetime] = None, 29 | end: Optional[datetime] = None, 30 | package: db_models.Package = Depends(get_package_or_fail), 31 | dao: Dao = Depends(get_dao), 32 | ): 33 | version = dao.get_package_version_by_filename( 34 | channel_name, package_name, filename, platform 35 | ) 36 | 37 | if not version: 38 | raise HTTPException( 39 | status_code=status.HTTP_404_NOT_FOUND, 40 | detail=f"package version {platform}/{filename} not found", 41 | ) 42 | 43 | series = dao.get_package_version_metrics( 44 | version.id, period, metric_name, start=start, end=end, fill_zeros=fill_zeros 45 | ) 46 | 47 | total = sum(s.count for s in series) 48 | 49 | return { 50 | "server_timestamp": datetime.utcnow(), 51 | "period": period, 52 | "metric_name": metric_name, 53 | "total": total, 54 | "series": series, 55 | } 56 | 57 | 58 | @api_router.get( 59 | "/channels/{channel_name}", 60 | response_model=rest_models.ChannelMetricResponse, 61 | tags=["metrics"], 62 | ) 63 | def get_channel_metrics( 64 | channel_name: str, 65 | period: IntervalType = IntervalType.day, 66 | metric_name: str = "download", 67 | platform: Optional[str] = None, 68 | start: Optional[datetime] = None, 69 | end: Optional[datetime] = None, 70 | dao: Dao = Depends(get_dao), 71 | ): 72 | metrics = dao.get_channel_metrics( 73 | channel_name, period, metric_name, platform=platform, start=start, end=end 74 | ) 75 | 76 | return { 77 | "server_timestamp": datetime.utcnow(), 78 | "period": period, 79 | "metric_name": metric_name, 80 | "packages": metrics, 81 | } 82 | 83 | 84 | def get_router(): 85 | return api_router 86 | -------------------------------------------------------------------------------- /plugins/quetz_current_repodata/tests/test_current_repodata.py: -------------------------------------------------------------------------------- 1 | import json 2 | import shutil 3 | import tempfile 4 | from pathlib import Path 5 | 6 | 7 | def test_current_repodata_hook( 8 | client, 9 | channel, 10 | subdirs, 11 | files, 12 | packages, 13 | config, 14 | profile, 15 | ): 16 | response = client.get("/api/dummylogin/madhurt") 17 | assert response.status_code == 200 18 | 19 | pkgstore = config.get_package_store() 20 | 21 | old_package_filename = "test-package-0.1-0.tar.bz2" 22 | url = f"/api/channels/{channel.name}/files/" 23 | files_to_upload = { 24 | "files": (old_package_filename, open(old_package_filename, "rb")) 25 | } 26 | response = client.post(url, files=files_to_upload) 27 | assert response.status_code == 201 28 | 29 | new_package_filename = "test-package-0.2-0.tar.bz2" 30 | url = f"/api/channels/{channel.name}/files/" 31 | files_to_upload = { 32 | "files": (new_package_filename, open(new_package_filename, "rb")) 33 | } 34 | response = client.post(url, files=files_to_upload) 35 | assert response.status_code == 201 36 | 37 | f = pkgstore.serve_path(channel.name, "linux-64/repodata.json") 38 | repodata = json.load(f) 39 | assert len(repodata["packages"]) == 2 40 | assert old_package_filename in repodata["packages"] 41 | assert new_package_filename in repodata["packages"] 42 | assert repodata["packages"][old_package_filename]["version"] == "0.1" 43 | assert repodata["packages"][new_package_filename]["version"] == "0.2" 44 | 45 | from quetz_current_repodata import main 46 | 47 | # this is emulating how we move repodata to the filestore now 48 | with tempfile.TemporaryDirectory() as tempdir: 49 | tempdir_path = Path(tempdir) 50 | 51 | f = pkgstore.serve_path(channel.name, "linux-64/repodata.json") 52 | 53 | stubdir = tempdir_path / channel.name / "linux-64" 54 | stubdir.mkdir(parents=True) 55 | 56 | with open(stubdir / "repodata.json", "wb") as ftemp: 57 | shutil.copyfileobj(f, ftemp) 58 | 59 | main.post_package_indexing(tempdir_path, channel.name, subdirs, files, packages) 60 | 61 | with open(stubdir / "current_repodata.json", "rb") as fo: 62 | pkgstore.add_file(fo.read(), channel.name, "linux-64/current_repodata.json") 63 | 64 | f = pkgstore.serve_path(channel.name, "linux-64/current_repodata.json") 65 | current_repodata = json.load(f) 66 | assert len(current_repodata["packages"]) == 1 67 | assert old_package_filename not in current_repodata["packages"] 68 | assert new_package_filename in current_repodata["packages"] 69 | assert current_repodata["packages"][new_package_filename]["version"] == "0.2" 70 | -------------------------------------------------------------------------------- /quetz/metrics/tasks.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime 3 | from typing import Optional 4 | 5 | import requests 6 | 7 | from quetz.dao import Dao 8 | 9 | 10 | def synchronize_metrics_from_mirrors( 11 | channel_name: str, 12 | dao: Dao, 13 | session: requests.Session, 14 | now: datetime = datetime.utcnow(), 15 | ): 16 | logger = logging.getLogger("quetz") 17 | channel = dao.get_channel(channel_name) 18 | if not channel: 19 | return 20 | for m in channel.mirrors: 21 | if not m.metrics_endpoint: 22 | logger.warning( 23 | f"metrics endpoint not configured for mirror {m.url}." 24 | "Skipping metrics synchronisation" 25 | ) 26 | continue 27 | query_str = ["period=H"] 28 | start_time: Optional[datetime] 29 | if m.last_synchronised: 30 | start_time = m.last_synchronised.replace(minute=0, second=0, microsecond=0) 31 | else: 32 | start_time = None 33 | if isinstance(start_time, datetime): 34 | query_str.append(f"start={start_time.isoformat()}") 35 | 36 | # exclude incomplete intervals (the current hour) 37 | end_time = now.replace(minute=0, second=0, microsecond=0) 38 | 39 | if start_time == end_time: 40 | logger.debug(f"metrics data for mirror {m.url} are up-to-date") 41 | continue 42 | 43 | query_str.append(f"end={end_time.isoformat()}") 44 | 45 | metrics_url = m.metrics_endpoint + "?" + "&".join(query_str) 46 | response = session.get(metrics_url) 47 | 48 | if response.status_code != 200: 49 | logger.error( 50 | f"mirror server {metrics_url} returned bad response with code " 51 | f"{response.status_code} and message {response.text}" 52 | ) 53 | continue 54 | 55 | response_data = response.json() 56 | try: 57 | packages = response_data["packages"] 58 | except KeyError: 59 | logger.error( 60 | f"malfromated response received from {metrics_url}: " 61 | "missing 'packages' key" 62 | ) 63 | continue 64 | 65 | for platform_filename, data in packages.items(): 66 | platform, filename = platform_filename.split("/") 67 | for s in data["series"]: 68 | timestamp = datetime.fromisoformat(s["timestamp"]) 69 | count = s["count"] 70 | dao.incr_download_count( 71 | channel_name, filename, platform, timestamp, count 72 | ) 73 | logger.debug(f"synchronized metrics from {metrics_url}") 74 | m.last_synchronised = end_time 75 | dao.db.commit() 76 | -------------------------------------------------------------------------------- /docs/source/deploying/workers.rst: -------------------------------------------------------------------------------- 1 | .. _task_workers: 2 | 3 | Task Workers 4 | ============= 5 | 6 | Quetz offers 3 types of backends for task workers. Each of them is explained below. 7 | 8 | Thread 9 | ----------- 10 | 11 | Thread workers process tasks in a separate thread. This functionality is in-built into FastAPI using 12 | `BackgroundTasks`_. 13 | 14 | .. _BackgroundTasks: https://fastapi.tiangolo.com/tutorial/background-tasks/ 15 | 16 | 17 | Subprocess 18 | ----------- 19 | 20 | Subprocess workers start in a separate process and are implemented through ``ProcessPoolExecutor`` of the 21 | ``concurrent.futures`` module. Once again, this is shipped as a part of Quetz. 22 | 23 | 24 | Redis 25 | ----------- 26 | 27 | For advanced use-cases, Quetz also offers the ability to use `redis-queue`_ to manage jobs and run them on 28 | multiple processes or even multiple servers. 29 | 30 | To use this backend, one needs to setup ``redis`` and ``redis-queue``. 31 | 32 | .. _redis-queue: https://python-rq.org/ 33 | 34 | Setting up ``redis`` 35 | ^^^^^^^^^^^^^^^^^^^^ 36 | 37 | Make sure that ``redis`` is installed. There are multiple ways to do this. One can compile it from source, 38 | use a package manager for your distribution (such as ``brew`` for MacOS, ``apt-get`` for Debian/Ubuntu) or use a 39 | Docker `redis image`_. 40 | (``docker pull redis`` if you have docker installed). 41 | 42 | .. _redis image: https://hub.docker.com/_/redis/ 43 | 44 | Once ``redis`` is installed, it needs to be started. This is as simple as executing the command ``redis-server`` on a 45 | terminal. 46 | (or can be run in a container through ``docker run -p 6379:6379 redis``) 47 | 48 | We also need to install `redis-py`_ - the python client for Redis. 49 | 50 | .. _redis-py: https://github.com/andymccurdy/redis-py 51 | 52 | Installing ``redis-queue`` 53 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 54 | 55 | ``redis-queue`` is a python library that facilitates using Redis for queueing jobs and processing them in the background with 56 | workers. The installation can be done by following the appropriate `instructions`_. 57 | 58 | .. _instructions: https://python-rq.org/#installation 59 | 60 | Once this has been done, a new worker needs to be spawned (which will continuously listen for jobs to execute). This can be done by 61 | running ``rq worker`` in a separate terminal. 62 | 63 | Edit ``config.toml`` 64 | ^^^^^^^^^^^^^^^^^^^^ 65 | Make sure to add a ``[worker]`` section with the ``type`` parameter set to ``redis``. This tells Quetz to use this backend. 66 | 67 | .. note:: 68 | 69 | The IP address of the machine running the ``redis-server``, along with the port and the DB Index should be 70 | present in the ``config.toml`` file. The default values (corresponding to running the server locally) will be picked up 71 | if they are not explicitly supplied. 72 | 73 | See :ref:`worker_config` for more details. 74 | -------------------------------------------------------------------------------- /quetz/channel_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Codethink Ltd 2 | # Distributed under the terms of the Modified BSD License. 3 | 4 | import json 5 | 6 | from quetz.versionorder import VersionOrder 7 | 8 | CHANNELDATA_OPTIONAL_FIELDS = ( 9 | "description", 10 | "dev_url", 11 | "doc_source_url", 12 | "doc_url", 13 | "home", 14 | "icon_hash", 15 | "icon_url", 16 | "license", 17 | "license_family", 18 | "spdx_license", 19 | "source_git_url", 20 | "reference_package", 21 | "source_url", 22 | "summary", 23 | "version", 24 | "plugin_metadata", 25 | ) 26 | CHANNELDATA_BINARY_FIELDS = ( 27 | "activate.d", 28 | "deactivate.d", 29 | "post_link", 30 | "pre_link", 31 | "pre_unlink", 32 | "binary_prefix", 33 | "text_prefix", 34 | ) 35 | 36 | 37 | def combine(old_data, new_data): 38 | if old_data is None: 39 | data = new_data 40 | else: 41 | data = {} 42 | newer = VersionOrder(old_data.get("version", "0")) < VersionOrder( 43 | new_data.get("version", "0") 44 | ) 45 | for field in CHANNELDATA_BINARY_FIELDS: 46 | data[field] = any((new_data.get(field, False), old_data.get(field, False))) 47 | 48 | for field in ("keywords", "identifiers", "tags"): 49 | if newer and new_data.get(field): 50 | data[field] = new_data[field] 51 | else: 52 | data[field] = old_data.get(field, {}) 53 | 54 | for field in CHANNELDATA_OPTIONAL_FIELDS: 55 | if newer and field in new_data: 56 | data[field] = new_data[field] 57 | elif field in old_data: 58 | data[field] = old_data[field] 59 | 60 | run_exports = old_data.get("run_exports", {}) 61 | if "run_exports" in new_data: 62 | if new_data["run_exports"]: 63 | run_exports[new_data["version"]] = new_data["run_exports"] 64 | data["run_exports"] = run_exports 65 | 66 | data["timestamp"] = max( 67 | old_data.get("timestamp", 0), new_data.get("timestamp", 0) 68 | ) 69 | 70 | data["subdirs"] = sorted( 71 | list(set(new_data.get("subdirs", [])) | set(old_data.get("subdirs", []))) 72 | ) 73 | 74 | data = dict(sorted(data.items(), key=lambda item: item[0])) 75 | 76 | return data 77 | 78 | 79 | def export(dao, channel_name): 80 | channeldata = {"channeldata_version": 1, "packages": {}, "subdirs": {}} 81 | packages = channeldata["packages"] 82 | subdirs = set(["noarch"]) 83 | 84 | for name, info in dao.get_channel_data(channel_name): 85 | if info is not None: 86 | data = json.loads(info) 87 | packages[name] = data 88 | subdirs = set(data.get("subdirs", [])) | subdirs 89 | 90 | channeldata["subdirs"] = list(subdirs) 91 | 92 | return channeldata 93 | -------------------------------------------------------------------------------- /plugins/quetz_content_trust/quetz_content_trust/db_models.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from sqlalchemy import BigInteger, Column, Date, ForeignKey, String, Table, func 4 | from sqlalchemy.orm import relationship 5 | 6 | from quetz.db_models import UUID, Base 7 | 8 | association_table = Table( 9 | "delegations_keys", 10 | Base.metadata, 11 | Column("role_delegations_id", ForeignKey("role_delegations.id"), primary_key=True), 12 | Column( 13 | "signing_keys_public_key", 14 | ForeignKey("signing_keys.public_key"), 15 | primary_key=True, 16 | ), 17 | ) 18 | 19 | 20 | class ContentTrustRole(Base): 21 | __tablename__ = "content_trust_roles" 22 | 23 | id = Column( 24 | UUID, primary_key=False, unique=True, default=lambda: uuid.uuid4().bytes 25 | ) 26 | type = Column(String, nullable=False, primary_key=True) 27 | channel = Column(String, nullable=False, primary_key=True) 28 | version = Column(BigInteger, nullable=False, primary_key=True) 29 | 30 | timestamp = Column(String, nullable=False) 31 | expiration = Column(String, nullable=False) 32 | 33 | delegator_id = Column(UUID, ForeignKey("role_delegations.id"), nullable=True) 34 | delegations = relationship( 35 | "RoleDelegation", 36 | backref="issuer", 37 | foreign_keys="RoleDelegation.issuer_id", 38 | cascade="all, delete-orphan", 39 | ) 40 | 41 | # delegator created by 'role_delegations.consumers' relationship backref 42 | 43 | time_created = Column(Date, nullable=False, server_default=func.current_date()) 44 | 45 | 46 | class RoleDelegation(Base): 47 | __tablename__ = "role_delegations" 48 | 49 | id = Column(UUID, primary_key=True, default=lambda: uuid.uuid4().bytes) 50 | 51 | issuer_id = Column(UUID, ForeignKey("content_trust_roles.id"), nullable=False) 52 | consumers = relationship( 53 | "ContentTrustRole", 54 | backref="delegator", 55 | foreign_keys=ContentTrustRole.delegator_id, 56 | post_update=True, 57 | cascade="all, delete-orphan", 58 | ) 59 | 60 | # issuer created by 'content_trust_roles.delegations' relationship backref 61 | 62 | type = Column(String, nullable=False) 63 | channel = Column(String, nullable=False) 64 | threshold = Column(BigInteger, nullable=False) 65 | keys = relationship( 66 | "SigningKey", secondary=association_table, backref="delegations" 67 | ) 68 | time_created = Column(Date, nullable=False, server_default=func.current_date()) 69 | 70 | 71 | class SigningKey(Base): 72 | __tablename__ = "signing_keys" 73 | 74 | public_key = Column(String, primary_key=True) 75 | private_key = Column(String) 76 | time_created = Column(Date, nullable=False, server_default=func.current_date()) 77 | user_id = Column(UUID, ForeignKey("users.id")) 78 | channel_name = Column(String, ForeignKey("channels.name")) 79 | -------------------------------------------------------------------------------- /docs/source/deploying/database.rst: -------------------------------------------------------------------------------- 1 | Database 2 | ======== 3 | 4 | 5 | PostgreSQL 6 | ^^^^^^^^^^ 7 | 8 | By default, quetz will run with sqlite database, which works well for local tests and small local instances. However, if you plan to run quetz in production, we recommend to configure it with the PostgreSQL database. There are several options to install PostgreSQL server on your local machine or production server, one of them being the official PostgreSQL docker image. 9 | 10 | 11 | Running PostgreSQL server with docker 12 | """"""""""""""""""""""""""""""""""""" 13 | 14 | You can the PostgresSQL image from the docker hub and start the server with the commands: 15 | 16 | .. code:: 17 | 18 | docker pull postgres 19 | docker run --name some-postgres -p 5432:5432 -e POSTGRES_PASSWORD=mysecretpassword -d postgres 20 | 21 | This will start the server with the user ``postgres`` and the password ``mysecretpassword`` that will be listening for connection on the port 5432 of localhost. 22 | 23 | You can then create a database in PostgreSQL for quetz tables: 24 | 25 | .. code:: 26 | 27 | sudo -u postgres psql -h localhost -c 'CREATE DATABASE quetz OWNER postgres;' 28 | 29 | Deploying Quetz with PostgreSQL backend 30 | """"""""""""""""""""""""""""""""""""""" 31 | 32 | Then in your configuration file (such as `dev_config.toml`) replace the `[sqlalchemy]` section with: 33 | 34 | .. code:: 35 | 36 | [sqlalchemy] 37 | database_url = "postgresql://postgres:mysecretpassword@localhost:5432/quetz" 38 | 39 | Finally, you can create and run a new quetz deployment based on this configuration (we assume that you saved it in file `config_postgres.toml`): 40 | 41 | 42 | .. code:: 43 | 44 | quetz run postgres_quetz --copy-conf config_postgres.toml 45 | 46 | Note that this recipe will create an ephemeral PostgreSQL database and it will delete all data after the `some-postgres` container is stopped and removed. To make the data persistent, please check the documentation of the `postgres` [image](https://hub.docker.com/_/postgres/) or your container orchestration system (Kubernetes or similar). 47 | 48 | Running tests with PostgreSQL backend 49 | """"""""""""""""""""""""""""""""""""" 50 | 51 | To run the tests with the PostgreSQL database instead of the default SQLite, follow the steps [above](#running-postgresql-server-with-docker) to start the PG server. Then create an new database: 52 | 53 | .. code:: 54 | 55 | psql -U postgres -h localhost -c 'CREATE DATABASE test_quetz OWNER postgres;' 56 | 57 | You will be asked to type the password to the DB, which you defined when starting your PG server. In the docker-based instructions above, we set it to `mysecretpassword`. 58 | 59 | To run the tests with this database you need to configure the `QUETZ_TEST_DATABASE` environment variable: 60 | 61 | .. code:: 62 | 63 | QUETZ_TEST_DATABASE="postgresql://postgres:mysecretpassword@localhost:5432/test_quetz" pytest -v ./quetz/tests 64 | 65 | 66 | -------------------------------------------------------------------------------- /quetz/migrations/versions/794249a0b1bd_adding_jobs_tables.py: -------------------------------------------------------------------------------- 1 | """adding jobs tables 2 | 3 | Revision ID: 794249a0b1bd 4 | Revises: db1c56bf4d57 5 | Create Date: 2020-12-14 16:11:19.819492 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = '794249a0b1bd' 13 | down_revision = 'db1c56bf4d57' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | op.create_table( 21 | 'jobs', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('created', sa.DateTime(), nullable=False), 24 | sa.Column('updated', sa.DateTime(), nullable=False), 25 | sa.Column('manifest', sa.LargeBinary(), nullable=False), 26 | sa.Column('owner_id', sa.LargeBinary(length=16), nullable=True), 27 | sa.Column("items_spec", sa.String(), nullable=True), 28 | sa.Column( 29 | 'items', 30 | sa.Enum('watch', 'watch_for', 'all', 'list', name='itemsselection'), 31 | nullable=False, 32 | ), 33 | sa.Column( 34 | 'status', 35 | sa.Enum( 36 | 'pending', 37 | 'queued', 38 | 'running', 39 | 'success', 40 | 'failed', 41 | 'timeout', 42 | 'cancelled', 43 | name='jobstatus', 44 | ), 45 | nullable=False, 46 | ), 47 | sa.ForeignKeyConstraint( 48 | ['owner_id'], 49 | ['users.id'], 50 | ondelete="cascade", 51 | ), 52 | sa.PrimaryKeyConstraint('id'), 53 | ) 54 | op.create_table( 55 | 'tasks', 56 | sa.Column('id', sa.Integer(), nullable=False), 57 | sa.Column('created', sa.DateTime(), nullable=False), 58 | sa.Column('updated', sa.DateTime(), nullable=False), 59 | sa.Column( 60 | 'status', 61 | sa.Enum( 62 | 'pending', 'running', 'success', 'failed', 'skipped', name='taskstatus' 63 | ), 64 | nullable=False, 65 | ), 66 | sa.Column('job_id', sa.Integer(), nullable=False), 67 | sa.Column('package_version_id', sa.LargeBinary(length=16), nullable=True), 68 | sa.ForeignKeyConstraint( 69 | ['job_id'], 70 | ['jobs.id'], 71 | ondelete="cascade", 72 | ), 73 | sa.ForeignKeyConstraint( 74 | ['package_version_id'], 75 | ['package_versions.id'], 76 | ondelete="cascade", 77 | ), 78 | sa.PrimaryKeyConstraint('id'), 79 | ) 80 | # ### end Alembic commands ### 81 | 82 | 83 | def downgrade(): 84 | # ### commands auto generated by Alembic - please adjust! ### 85 | op.drop_table('tasks') 86 | op.drop_table('jobs') 87 | # ### end Alembic commands ### 88 | -------------------------------------------------------------------------------- /plugins/quetz_tos/quetz_tos/migrations/versions/c3a635971280_tos_languages.py: -------------------------------------------------------------------------------- 1 | """tos_languages 2 | 3 | Revision ID: c3a635971280 4 | Revises: 44ec522465e5 5 | Create Date: 2022-07-27 15:15:22.341630 6 | 7 | """ 8 | 9 | import sqlalchemy as sa 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "c3a635971280" 14 | down_revision = "44ec522465e5" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "quetz_tos_file", 23 | sa.Column("id", sa.LargeBinary(length=16), nullable=False), 24 | sa.Column("filename", sa.String(), nullable=True), 25 | sa.Column("language", sa.String(), nullable=True), 26 | sa.Column("tos_id", sa.LargeBinary(length=16), nullable=False), 27 | sa.ForeignKeyConstraint( 28 | ["tos_id"], 29 | ["quetz_tos.id"], 30 | ), 31 | sa.PrimaryKeyConstraint("id", "tos_id"), 32 | ) 33 | with op.batch_alter_table("channels", schema=None) as batch_op: 34 | batch_op.alter_column( 35 | "size", 36 | existing_type=sa.INTEGER(), 37 | type_=sa.BigInteger(), 38 | existing_nullable=True, 39 | ) 40 | batch_op.alter_column( 41 | "size_limit", 42 | existing_type=sa.INTEGER(), 43 | type_=sa.BigInteger(), 44 | existing_nullable=True, 45 | ) 46 | 47 | with op.batch_alter_table("package_versions", schema=None) as batch_op: 48 | batch_op.alter_column( 49 | "size", 50 | existing_type=sa.INTEGER(), 51 | type_=sa.BigInteger(), 52 | existing_nullable=True, 53 | ) 54 | 55 | with op.batch_alter_table("quetz_tos", schema=None) as batch_op: 56 | batch_op.drop_column("filename") 57 | 58 | # ### end Alembic commands ### 59 | 60 | 61 | def downgrade(): 62 | # ### commands auto generated by Alembic - please adjust! ### 63 | with op.batch_alter_table("quetz_tos", schema=None) as batch_op: 64 | batch_op.add_column(sa.Column("filename", sa.VARCHAR(), nullable=True)) 65 | 66 | with op.batch_alter_table("package_versions", schema=None) as batch_op: 67 | batch_op.alter_column( 68 | "size", 69 | existing_type=sa.BigInteger(), 70 | type_=sa.INTEGER(), 71 | existing_nullable=True, 72 | ) 73 | 74 | with op.batch_alter_table("channels", schema=None) as batch_op: 75 | batch_op.alter_column( 76 | "size_limit", 77 | existing_type=sa.BigInteger(), 78 | type_=sa.INTEGER(), 79 | existing_nullable=True, 80 | ) 81 | batch_op.alter_column( 82 | "size", 83 | existing_type=sa.BigInteger(), 84 | type_=sa.INTEGER(), 85 | existing_nullable=True, 86 | ) 87 | 88 | op.drop_table("quetz_tos_file") 89 | # ### end Alembic commands ### 90 | -------------------------------------------------------------------------------- /quetz/templates/channeldata-index.html.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | {{ title }} 4 | 67 | 68 | 69 |

{{ title }}

70 |

channeldata.json

71 | {% for subdir in subdirs %}{{ subdir }}   {% endfor %} 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | {% for subdir in subdirs %} 80 | 81 | {%- endfor %} 82 | 83 | 84 | {% for name, record in packages.items() %} 85 | 86 | 87 | 88 | 89 | 90 | 91 | {% for subdir in subdirs %} 92 | 93 | {%- endfor %} 94 | 95 | 96 | {%- endfor %} 97 |
PackageLatest VersionDocDevLicense{{ subdir }}Summary
{{ name | opt_href(record.home) }}{{ record.version | truncate(14, True, '') }}{% if record.doc_url %}doc{% endif %}{% if record.dev_url %}dev{% endif %}{% if record.license %}{{ record.license.split(' ')[0] | truncate(15, True, '') }}{% endif %}{% if 'subdirs' in record and subdir in record.subdirs %}X{% endif %}{{ record.summary | escape | truncate(75) }}
98 |
Updated: {{ current_time|strftime("%Y-%m-%d %H:%M:%S %z") }} - Files: {{ packages|length }}
99 | 100 | 101 | -------------------------------------------------------------------------------- /quetz/metrics/db_models.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from datetime import datetime, timedelta 3 | from enum import Enum 4 | 5 | import sqlalchemy as sa 6 | 7 | from quetz.db_models import UUID, Base 8 | 9 | 10 | class IntervalType(Enum): 11 | hour = "H" 12 | day = "D" 13 | month = "M" 14 | year = "Y" 15 | 16 | @property 17 | def timedelta(self): 18 | if self == IntervalType.hour: 19 | return timedelta(hours=1) 20 | if self == IntervalType.day: 21 | return timedelta(days=1) 22 | raise ValueError(f"can not create timedelta for interval '{self.name}'") 23 | 24 | 25 | def round_timestamp(timestamp, period): 26 | """round timestamp to nearest period""" 27 | now_interval = timestamp.replace(minute=0, second=0, microsecond=0) 28 | if period in [IntervalType.day, IntervalType.month, IntervalType.year]: 29 | now_interval = now_interval.replace(hour=0) 30 | if period in [IntervalType.month, IntervalType.year]: 31 | now_interval = now_interval.replace(day=1) 32 | if period == IntervalType.year: 33 | now_interval = now_interval.replace(month=1) 34 | return now_interval 35 | 36 | 37 | def next_timestamp(timestamp: datetime, interval: IntervalType): 38 | """next timestamp advanced by interval time""" 39 | if interval in [IntervalType.day, IntervalType.hour]: 40 | return timestamp + interval.timedelta 41 | if interval == IntervalType.month: 42 | if timestamp.month == 12: 43 | return timestamp.replace(year=timestamp.year + 1, month=1) 44 | else: 45 | return timestamp.replace(month=timestamp.month + 1) 46 | if interval == IntervalType.year: 47 | return timestamp.replace(year=timestamp.year + 1) 48 | raise ValueError(f"interval {interval.name} not supported") 49 | 50 | 51 | class PackageVersionMetric(Base): 52 | __tablename__ = "aggregated_metrics" 53 | 54 | id = sa.Column(UUID, default=lambda: uuid.uuid4().bytes, primary_key=True) 55 | 56 | channel_name = sa.Column(sa.String) 57 | platform = sa.Column(sa.String) 58 | 59 | filename = sa.Column(sa.String) 60 | 61 | metric_name = sa.Column(sa.String(255), nullable=False) 62 | period = sa.Column(sa.Enum(IntervalType)) 63 | count = sa.Column(sa.Integer, server_default=sa.text("0"), nullable=False) 64 | timestamp = sa.Column(sa.DateTime(), nullable=False) 65 | 66 | __table_args__ = ( 67 | sa.Index( 68 | "package_version_metric_index", 69 | channel_name, 70 | platform, 71 | filename, 72 | metric_name, 73 | period, 74 | timestamp, 75 | ), 76 | sa.UniqueConstraint( 77 | channel_name, 78 | platform, 79 | filename, 80 | metric_name, 81 | period, 82 | timestamp, 83 | name="package_version_metric_constraint", 84 | ), 85 | ) 86 | 87 | def __repr__(self): 88 | return ( 89 | f"PackageVersionMetric(metric_name={self.metric_name}, " 90 | f"period={self.period.value}, " 91 | f"timestamp={self.timestamp},count={self.count})" 92 | ) 93 | -------------------------------------------------------------------------------- /docs/source/deploying/migrations.rst: -------------------------------------------------------------------------------- 1 | Migrations 2 | ========== 3 | 4 | When the data classes in Quetz are modified (for example, a column is added, remove or updated), the database need to be upgraded while keeping the stored data. This cane be done automatically, using migrations. Quetz uses `alembic`_ to handle migrations. 5 | 6 | .. note:: 7 | 8 | Before running any of the commands below, you need to make sure that your database is backed up. The backup process depends on the infrastructure, but in the simplest case it may involve the dump of the whole database (using ``pg_dump`` for example). 9 | 10 | Migrating database 11 | ------------------ 12 | 13 | When you install a new version of Quetz or Quetz plugin, you should apply the provided migrations using the ``quetz init-db`` command. For example, assuming that your deployment is in ``deployment_dir`` folder: 14 | 15 | .. code:: 16 | 17 | quetz init-db deployment_dir 18 | 19 | 20 | .. _alembic : https://alembic.sqlalchemy.org 21 | 22 | 23 | Adding new migrations 24 | --------------------- 25 | 26 | Once you modified your data models in Quetz, you can autogenerate the appropriate migrations using ``make-migrations`` command: 27 | 28 | .. code:: 29 | 30 | quetz init-db deployment_dir # to make sure that the db is up-to-date 31 | quetz make-migrations deployment_dir --message "my revision message" 32 | 33 | This should create a new file in ``quetz/migrations/versions`` directory, which you can then add to the git repository. Then you can apply the migrations the standard way: 34 | 35 | .. code:: 36 | 37 | quetz init-db deployment_dir # to make sure that the db is up-to-date 38 | 39 | .. note:: 40 | 41 | For running unit test there is no need to create the migrations, the testing framework 42 | will create all tables automatically for you. However, if you want to run the tests 43 | with the migrations, you can configure it with env variable: :code:`QUETZ_TEST_DBINIT=use-migrations pytest quetz` 44 | 45 | Initializing migrations for plugins 46 | ----------------------------------- 47 | 48 | To use migrations in plugins, you need to initialize them. Our cookiecutter template will create the necessary backbone for you, you will just need to define your models in the ``db_models.py`` file of plugin directory, and then run the command: 49 | 50 | .. code:: 51 | 52 | quetz make-migrations deployment_dir --message "initial revision" --initialize --plugin quetz-plugin_name 53 | 54 | This should create a new migration script in `PLUGIN_DIR/migrations/versions`. 55 | 56 | .. note:: 57 | 58 | If you want to add the migration script to you working directory, you need to install the plugin using the development mode: ``pip install -e PATH_TO_PLUGIN`` 59 | 60 | As always the ``quetz init-db`` will upgrade automatically your database to reflect the data models defined in the plugin. 61 | 62 | Adding migrations for plugins 63 | ----------------------------- 64 | 65 | When you change the data model in the plugin, you can create the required migrations using the same ``quetz make-migrations`` command, but without ``--initialize``: 66 | 67 | .. code:: 68 | 69 | quetz make-migrations deployment_dir --message "second revision" --plugin quetz-plugin_name 70 | -------------------------------------------------------------------------------- /quetz/migrations/versions/8dfb7c4bfbd7_new_package_versions.py: -------------------------------------------------------------------------------- 1 | """new package versions 2 | 3 | Revision ID: 8dfb7c4bfbd7 4 | Revises: 98c04a65df4a 5 | Create Date: 2020-12-21 16:35:33.940460 6 | 7 | """ 8 | import sqlalchemy as sa 9 | from alembic import op 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '8dfb7c4bfbd7' 14 | down_revision = '98c04a65df4a' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | enum_values = ('hour', 'day', 'month', 'year') 19 | 20 | 21 | def get_intervaltype(): 22 | if op.get_context().dialect.name == 'postgresql': 23 | intervaltype = sa.dialects.postgresql.ENUM( 24 | *enum_values, name='intervaltype', create_type=False 25 | ) 26 | else: 27 | intervaltype = sa.Enum(*enum_values, name='intervaltype') # type: ignore 28 | return intervaltype 29 | 30 | 31 | def upgrade(): 32 | # ### commands auto generated by Alembic - please adjust! ### 33 | intervaltype = get_intervaltype() 34 | 35 | op.drop_table('package_version_metrics') 36 | op.create_table( 37 | 'aggregated_metrics', 38 | sa.Column('id', sa.LargeBinary(length=16), nullable=False), 39 | sa.Column('channel_name', sa.String(), nullable=True), 40 | sa.Column('platform', sa.String(), nullable=True), 41 | sa.Column('filename', sa.String(), nullable=True), 42 | sa.Column('metric_name', sa.String(length=255), nullable=False), 43 | sa.Column('period', intervaltype, nullable=True), 44 | sa.Column('count', sa.Integer(), server_default=sa.text('0'), nullable=False), 45 | sa.Column('timestamp', sa.DateTime(), nullable=False), 46 | sa.PrimaryKeyConstraint('id'), 47 | ) 48 | # ### end Alembic commands ### 49 | 50 | 51 | def downgrade(): 52 | # ### commands auto generated by Alembic - please adjust! ### 53 | intervaltype = get_intervaltype() 54 | 55 | op.create_table( 56 | 'package_version_metrics', 57 | sa.Column( 58 | 'package_version_id', 59 | postgresql.BYTEA(), 60 | autoincrement=False, 61 | nullable=False, 62 | ), 63 | sa.Column( 64 | 'metric_name', sa.VARCHAR(length=255), autoincrement=False, nullable=False 65 | ), 66 | sa.Column('period', intervaltype, autoincrement=False, nullable=False), 67 | sa.Column( 68 | 'count', 69 | sa.INTEGER(), 70 | server_default=sa.text('0'), 71 | autoincrement=False, 72 | nullable=False, 73 | ), 74 | sa.Column( 75 | 'timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=False 76 | ), 77 | sa.ForeignKeyConstraint( 78 | ['package_version_id'], 79 | ['package_versions.id'], 80 | name='package_version_metrics_package_version_id_fkey', 81 | ), 82 | sa.PrimaryKeyConstraint( 83 | 'package_version_id', 84 | 'metric_name', 85 | 'period', 86 | 'timestamp', 87 | name='package_version_metrics_pkey', 88 | ), 89 | ) 90 | op.drop_table('aggregated_metrics') 91 | # ### end Alembic commands ### 92 | --------------------------------------------------------------------------------