├── .github └── workflows │ ├── python-app.yml │ └── python-publish.yml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── pyproject.toml └── simple_repository_server ├── __init__.py ├── __main__.py ├── _http_response_iterator.py ├── py.typed ├── routers ├── __init__.py └── simple.py ├── tests ├── __init__.py ├── api │ ├── __init__.py │ └── test_simple_router.py ├── integration │ ├── __init__.py │ └── test_repo_dependency_injection.py ├── test__http_response_iterator.py └── unit │ ├── __init__.py │ └── test_utils.py └── utils.py /.github/workflows/python-app.yml: -------------------------------------------------------------------------------- 1 | name: Python application 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Set up Python 3.11 20 | uses: actions/setup-python@v3 21 | with: 22 | python-version: "3.11" 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install .[dev] mypy 26 | - name: Type check 27 | run: | 28 | python -m mypy ./simple_repository_server 29 | - name: Test with pytest 30 | run: | 31 | python -m pytest ./simple_repository_server 32 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package to PyPI when a release is created 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | release: 13 | types: [published] 14 | 15 | permissions: 16 | contents: read 17 | 18 | jobs: 19 | release-build: 20 | runs-on: ubuntu-latest 21 | 22 | steps: 23 | - uses: actions/checkout@v4 24 | 25 | - uses: actions/setup-python@v5 26 | with: 27 | python-version: "3.x" 28 | 29 | - name: Build release distributions 30 | run: | 31 | # NOTE: put your own distribution build steps here. 32 | python -m pip install build 33 | python -m build 34 | 35 | - name: Upload distributions 36 | uses: actions/upload-artifact@v4 37 | with: 38 | name: release-dists 39 | path: dist/ 40 | 41 | pypi-publish: 42 | runs-on: ubuntu-latest 43 | needs: 44 | - release-build 45 | permissions: 46 | # IMPORTANT: this permission is mandatory for trusted publishing 47 | id-token: write 48 | 49 | # Dedicated environments with protections for publishing are strongly recommended. 50 | # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules 51 | environment: 52 | name: pypi 53 | url: https://pypi.org/p/simple-repository-server 54 | 55 | steps: 56 | - name: Retrieve release distributions 57 | uses: actions/download-artifact@v4 58 | with: 59 | name: release-dists 60 | path: dist/ 61 | 62 | - name: Publish release distributions to PyPI 63 | uses: pypa/gh-action-pypi-publish@release/v1 64 | with: 65 | packages-dir: dist/ 66 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | simple_repository_server/_version.py 2 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | 8 | repos: 9 | - repo: https://github.com/pre-commit/pre-commit-hooks 10 | rev: "v4.4.0" 11 | hooks: 12 | - id: trailing-whitespace 13 | - id: end-of-file-fixer 14 | - id: check-yaml 15 | - id: check-toml 16 | 17 | - repo: https://github.com/asottile/add-trailing-comma 18 | rev: "v2.4.0" 19 | hooks: 20 | - id: add-trailing-comma 21 | args: 22 | - "--py36-plus" 23 | 24 | - repo: https://github.com/PyCQA/isort 25 | rev: "5.12.0" 26 | hooks: 27 | - id: isort 28 | 29 | - repo: https://github.com/pycqa/flake8 30 | rev: "6.0.0" 31 | hooks: 32 | - id: flake8 33 | args: 34 | - "--max-line-length=120" 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 simple-repository 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # simple-repository-server 2 | 3 | A tool for running a PEP-503 simple Python package repository, including features such as dist metadata (PEP-658) and JSON API (PEP-691) 4 | 5 | ## Installation 6 | 7 | ```bash 8 | python -m pip install simple-repository-server 9 | ``` 10 | 11 | ## Usage 12 | 13 | The ``simple-repository-server`` is designed to be used as a library, but also includes a convenient command line interface for running 14 | a simple repository service: 15 | 16 | ```bash 17 | $ simple-repository-server --help 18 | usage: simple-repository-server [-h] [--port PORT] repository-url [repository-url ...] 19 | 20 | Run a Simple Repository Server 21 | 22 | positional arguments: 23 | repository-url 24 | 25 | options: 26 | -h, --help show this help message and exit 27 | --port PORT 28 | ``` 29 | 30 | The simplest example of this is to simply mirror the Python Package Index: 31 | 32 | ```bash 33 | python -m simple_repository_server https://pypi.org/simple/ 34 | ``` 35 | 36 | However, if multiple repositories are provided, the ``PrioritySelectedProjectsRepository`` component will be used to 37 | combine them together in a way that mitigates the [dependency confusion attack](https://medium.com/@alex.birsan/dependency-confusion-4a5d60fec610). 38 | 39 | The server handles PEP-691 content negotiation to serve either HTML or JSON formats. 40 | Per PEP-691, the default (fallback) content type is HTML, but a JSON response can 41 | be previewed in the browser by adding the ``?format=application/vnd.pypi.simple.v1+json`` 42 | querystring to any of the repository URLs. 43 | 44 | The server has been configured to include PEP-658 metadata, even if the upstream repository does 45 | not include such metadata. This is done on the fly, and as a result the distribution will be 46 | temporarily downloaded to the server in order to extract and serve the metadata. 47 | 48 | It is possible to use the resulting repository as input for the 49 | [``simple-repository-browser``](https://github.com/simple-repository/simple-repository-browser), which 50 | offers a web interface to browse and search packages in any simple package repository (PEP-503), 51 | inspired by PyPI / warehouse. 52 | 53 | It is expected that as new features appear in the underlying ``simple-repository`` library, those 54 | which make general sense to enable by default will be introduced into the CLI without providing a 55 | mechanism to disable those features. For more control, please see the "Non CLI usage" section. 56 | 57 | ## Non CLI usage 58 | 59 | This project provides a number of tools in order to build a repository service using FastAPI. 60 | For cases when control of the repository configuration is required, and where details of the 61 | ASGI environment need more precise control, it is expected that ``simple-repository-server`` is used 62 | as a library instead of a CLI. 63 | 64 | Currently, the API for this functionality is under development, and will certainly change in the 65 | future. 66 | 67 | ## License and Support 68 | 69 | This code has been released under the MIT license. 70 | It is an initial prototype which is developed in-house, and _not_ currently openly developed. 71 | 72 | It is hoped that the release of this prototype will trigger interest from other parties that have similar needs. 73 | With sufficient collaborative interest there is the potential for the project to be openly 74 | developed, and to power Python package repositories across many domains. 75 | 76 | Please get in touch at https://github.com/orgs/simple-repository/discussions to share how 77 | this project may be useful to you. This will help us to gauge the level of interest and 78 | provide valuable insight when deciding whether to commit future resources to the project. 79 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | 8 | [build-system] 9 | requires = ["setuptools>=61", "setuptools_scm>=8"] 10 | build-backend = "setuptools.build_meta" 11 | 12 | [project] 13 | name = "simple-repository-server" 14 | dynamic = ["version"] 15 | requires-python = ">=3.11" 16 | classifiers = [ 17 | "Development Status :: 4 - Beta", 18 | "Programming Language :: Python :: 3", 19 | "Framework :: FastAPI", 20 | "Operating System :: OS Independent", 21 | "Typing :: Typed", 22 | ] 23 | authors = [ 24 | {name = "Phil Elson"}, 25 | {name = "Ivan Sinkarenko"}, 26 | {name = "Francesco Iannaccone"}, 27 | {name = "Wouter Koorn"}, 28 | ] 29 | dependencies = [ 30 | "httpx", 31 | "fastapi>=0.100.0", 32 | "packaging", 33 | "uvicorn[standard]", 34 | "simple-repository>=0.6.0", 35 | ] 36 | readme = "README.md" 37 | description = "A tool for running a PEP-503 simple Python package repository, including features such as dist metadata (PEP-658) and JSON API (PEP-691)" 38 | 39 | [project.urls] 40 | Homepage = "https://github.com/simple-repository/simple-repository-server" 41 | 42 | [project.optional-dependencies] 43 | test = [ 44 | "pytest", 45 | "pytest_asyncio", 46 | "pytest_httpx", 47 | "starlette>=0.26.1", 48 | "pytest_httpserver", 49 | ] 50 | dev = [ 51 | "simple-repository-server[test]", 52 | ] 53 | 54 | [project.scripts] 55 | simple-repository-server = "simple_repository_server.__main__:main" 56 | [tool.setuptools_scm] 57 | version_file = "simple_repository_server/_version.py" 58 | 59 | [tool.isort] 60 | py_version = 39 61 | line_length = 100 62 | multi_line_output = 3 63 | include_trailing_comma = true 64 | force_grid_wrap = 0 65 | use_parentheses = true 66 | ensure_newline_before_comments = true 67 | force_sort_within_sections = true 68 | 69 | [tool.mypy] 70 | python_version = "3.11" 71 | exclude = "simple_repository_server/tests" 72 | ignore_missing_imports = false 73 | strict = true 74 | 75 | [tool.setuptools.packages.find] 76 | namespaces = false 77 | -------------------------------------------------------------------------------- /simple_repository_server/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | 8 | from ._version import version as __version__ # noqa 9 | 10 | __all__ = ['__version__'] 11 | -------------------------------------------------------------------------------- /simple_repository_server/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | 8 | import argparse 9 | from contextlib import asynccontextmanager 10 | import logging 11 | from pathlib import Path 12 | import typing 13 | from urllib.parse import urlparse 14 | 15 | import fastapi 16 | from fastapi import FastAPI 17 | import httpx 18 | from simple_repository.components.core import SimpleRepository 19 | from simple_repository.components.http import HttpRepository 20 | from simple_repository.components.local import LocalRepository 21 | from simple_repository.components.metadata_injector import MetadataInjectorRepository 22 | from simple_repository.components.priority_selected import PrioritySelectedProjectsRepository 23 | import uvicorn 24 | 25 | from simple_repository_server.routers import simple 26 | 27 | 28 | def is_url(url: str) -> bool: 29 | return urlparse(url).scheme in ("http", "https") 30 | 31 | 32 | def configure_parser(parser: argparse.ArgumentParser) -> None: 33 | parser.description = "Run a Python Package Index" 34 | 35 | parser.add_argument("--host", default="0.0.0.0") 36 | parser.add_argument("--port", type=int, default=8000) 37 | parser.add_argument("repository_url", metavar="repository-url", type=str, nargs="+") 38 | 39 | 40 | def create_repository( 41 | repository_urls: list[str], 42 | http_client: httpx.AsyncClient, 43 | ) -> SimpleRepository: 44 | base_repos: list[SimpleRepository] = [] 45 | repo: SimpleRepository 46 | for repo_url in repository_urls: 47 | if is_url(repo_url): 48 | repo = HttpRepository( 49 | url=repo_url, 50 | http_client=http_client, 51 | ) 52 | else: 53 | repo = LocalRepository( 54 | index_path=Path(repo_url), 55 | ) 56 | base_repos.append(repo) 57 | 58 | if len(base_repos) > 1: 59 | repo = PrioritySelectedProjectsRepository(base_repos) 60 | else: 61 | repo = base_repos[0] 62 | return MetadataInjectorRepository(repo, http_client) 63 | 64 | 65 | def create_app(repository_urls: list[str]) -> fastapi.FastAPI: 66 | @asynccontextmanager 67 | async def lifespan(app: FastAPI) -> typing.AsyncIterator[None]: 68 | async with httpx.AsyncClient() as http_client: 69 | repo = create_repository(repository_urls, http_client) 70 | app.include_router(simple.build_router(repo, http_client=http_client)) 71 | yield 72 | 73 | app = FastAPI( 74 | openapi_url=None, # Disables automatic OpenAPI documentation (Swagger & Redoc) 75 | lifespan=lifespan, 76 | ) 77 | return app 78 | 79 | 80 | def handler(args: typing.Any) -> None: 81 | host: str = args.host 82 | port: int = args.port 83 | repository_urls: list[str] = args.repository_url 84 | uvicorn.run( 85 | app=create_app(repository_urls), 86 | host=host, 87 | port=port, 88 | ) 89 | 90 | 91 | def main() -> None: 92 | logging.basicConfig(level=logging.INFO) 93 | parser = argparse.ArgumentParser() 94 | configure_parser(parser) 95 | args = parser.parse_args() 96 | handler(args) 97 | 98 | 99 | if __name__ == '__main__': 100 | main() 101 | -------------------------------------------------------------------------------- /simple_repository_server/_http_response_iterator.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | 8 | from __future__ import annotations 9 | 10 | import typing 11 | 12 | import httpx 13 | 14 | 15 | class HttpResponseIterator: 16 | """ 17 | A class providing a generator to iterate over response body bytes from an httpx request. 18 | 19 | This class creates an iterator that allows you to iterate over the bytes of a response body 20 | obtained from an httpx request. Additionally, it provides access to the response status code 21 | and headers before the streaming response is constructed. It is particularly designed to be 22 | used with Starlette's streaming responses, enabling access to headers and status code before 23 | the response is returned by an API endpoint. The class will keep the httpx session alive 24 | until the entire response content is accessed. 25 | """ 26 | 27 | PROXIED_REQUEST_HEADERS = { 28 | 'accept', 29 | 'user-agent', 30 | 'accept-encoding', 31 | 'if-unmodified-since', 32 | 'if-range', 'if-none-match', 33 | 'if-modified-since', 34 | 'if-match', 35 | 'range', 36 | 'referer', 37 | } 38 | 39 | def __init__(self, http_client: httpx.AsyncClient, url: str): 40 | """ 41 | Do not call the constructor of this class directly. 42 | Use StreamResponseIterator.create_iterator. 43 | """ 44 | self.http_client = http_client 45 | self.url: str = url 46 | self.status_code: int 47 | self.headers: typing.Mapping[str, str] 48 | self._agen: typing.AsyncGenerator[bytes, None] 49 | 50 | def __aiter__(self) -> HttpResponseIterator: 51 | return self 52 | 53 | async def __anext__(self) -> bytes: 54 | return await self._agen.__anext__() 55 | 56 | @classmethod 57 | async def create_iterator( 58 | cls, 59 | http_client: httpx.AsyncClient, 60 | url: str, 61 | *, 62 | request_headers: typing.Mapping[str, str] | None = None, 63 | ) -> HttpResponseIterator: 64 | iterator = HttpResponseIterator( 65 | http_client=http_client, 66 | url=url, 67 | ) 68 | request_headers = request_headers or {} 69 | headers = { 70 | header_name: header_value for header_name, header_value in request_headers.items() 71 | if header_name.lower() in cls.PROXIED_REQUEST_HEADERS 72 | } 73 | 74 | async def agenerator() -> typing.AsyncGenerator[bytes, None]: 75 | async with iterator.http_client.stream( 76 | method="GET", 77 | url=iterator.url, 78 | headers=headers, 79 | ) as resp: 80 | # Expose the response status and headers. 81 | iterator.status_code, iterator.headers = resp.status_code, resp.headers 82 | 83 | # The first time that anext is called, set status_code and 84 | # headers, without yielding the first byte of the stream. 85 | yield b"" 86 | async for chunk in resp.aiter_raw(1024 * 1024): 87 | yield chunk 88 | 89 | iterator._agen = agenerator() 90 | # Call anext to set the values of status_code and headers. 91 | await iterator.__anext__() 92 | 93 | return iterator 94 | -------------------------------------------------------------------------------- /simple_repository_server/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simple-repository/simple-repository-server/d9a861d82761cd6ae1b40ceb06ffbdbcc1697780/simple_repository_server/py.typed -------------------------------------------------------------------------------- /simple_repository_server/routers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | -------------------------------------------------------------------------------- /simple_repository_server/routers/simple.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | 8 | import functools 9 | import hashlib 10 | import typing 11 | 12 | from fastapi import APIRouter, Depends, HTTPException 13 | import fastapi.params 14 | from fastapi.responses import ( 15 | FileResponse, 16 | HTMLResponse, 17 | PlainTextResponse, 18 | RedirectResponse, 19 | Response, 20 | StreamingResponse, 21 | ) 22 | import httpx 23 | import packaging.utils 24 | import packaging.version 25 | from simple_repository import SimpleRepository, content_negotiation, errors, model, serializer 26 | 27 | from .. import utils 28 | from .._http_response_iterator import HttpResponseIterator 29 | 30 | 31 | def get_response_format( 32 | request: fastapi.Request, 33 | format: str | None = None, 34 | ) -> content_negotiation.Format: 35 | """ 36 | A fastapi dependent which can optionally enable a PEP-691 format querystring, 37 | for example: 38 | 39 | /simple/some-project/?format=application/vnd.pypi.simple.v1+json 40 | 41 | """ 42 | if format: 43 | # Allow the consumer to request a format as a query string such as 44 | # {URL}?format=application/vnd.pypi.simple.v1+json 45 | # Note: + in urls are interpreted as spaces by 46 | # urllib.parse.parse_qsl, used by FastAPI. 47 | requested_format = format.replace(" ", "+") 48 | else: 49 | requested_format = request.headers.get("Accept", "") 50 | 51 | try: 52 | response_format = content_negotiation.select_response_format( 53 | content_type=requested_format, 54 | ) 55 | except errors.UnsupportedSerialization as e: 56 | raise HTTPException(status_code=406, detail=str(e)) 57 | 58 | return response_format 59 | 60 | 61 | def build_router( 62 | resource_repository: SimpleRepository, 63 | *, 64 | http_client: httpx.AsyncClient, 65 | prefix: str = "/simple/", 66 | repo_factory: typing.Optional[typing.Callable[..., SimpleRepository]] = None, 67 | ) -> APIRouter: 68 | """ 69 | Build a FastAPI router for the given repository and http_client. 70 | 71 | Note that for the simple end-points, the repository is an injected 72 | dependency, meaning that you can add your own dependencies into the repository 73 | (see the test_repo_dependency_injection for an example of this). 74 | 75 | """ 76 | if not prefix.endswith("/"): 77 | raise ValueError("Prefix must end in '/'") 78 | 79 | if repo_factory is None: 80 | # If no repo factory is provided, use the same repository that we want to 81 | # use for resource handling. 82 | def repo_factory() -> SimpleRepository: 83 | return resource_repository 84 | 85 | simple_router = APIRouter( 86 | tags=["simple"], 87 | default_response_class=HTMLResponse, 88 | ) 89 | #: To be fixed by https://github.com/tiangolo/fastapi/pull/2763 90 | get = functools.partial(simple_router.api_route, methods=["HEAD", "GET"]) 91 | 92 | @get(prefix) 93 | async def project_list( 94 | response_format: typing.Annotated[content_negotiation.Format, Depends(get_response_format)], 95 | repository: typing.Annotated[SimpleRepository, Depends(repo_factory)], 96 | ) -> Response: 97 | project_list = await repository.get_project_list() 98 | 99 | serialized_project_list = serializer.serialize( 100 | page=project_list, 101 | format=response_format, 102 | ) 103 | 104 | return Response( 105 | serialized_project_list, 106 | media_type=response_format.value, 107 | ) 108 | 109 | @get(prefix + "{project_name}/") 110 | async def simple_project_page( 111 | request: fastapi.Request, 112 | project_name: str, 113 | repository: typing.Annotated[SimpleRepository, Depends(repo_factory)], 114 | response_format: typing.Annotated[content_negotiation.Format, Depends(get_response_format)], 115 | ) -> Response: 116 | normed_prj_name = packaging.utils.canonicalize_name(project_name) 117 | if normed_prj_name != project_name: 118 | # Update the original path params with the normed name. 119 | path_params = request.path_params | {'project_name': normed_prj_name} 120 | correct_url = utils.relative_url_for( 121 | request=request, 122 | name="simple_project_page", 123 | **path_params, 124 | ) 125 | if request.url.query: 126 | correct_url = correct_url + "?" + request.url.query 127 | return RedirectResponse( 128 | url=correct_url, 129 | status_code=301, 130 | ) 131 | 132 | try: 133 | package_releases = await repository.get_project_page(project_name) 134 | except errors.PackageNotFoundError as e: 135 | raise HTTPException(404, str(e)) 136 | 137 | # Point all resource URLs to this router. The router may choose to redirect these 138 | # back to the original source, but this means that all resource requests go through 139 | # this server (it may be desirable to be able to disable this behaviour in the 140 | # future, though it would mean that there is the potential for a SimpleRepository 141 | # to have implemented a resource handler, yet it never sees the request). 142 | project_releases = utils.replace_urls(package_releases, project_name, request) 143 | 144 | serialized_project_page = serializer.serialize( 145 | page=project_releases, 146 | format=response_format, 147 | ) 148 | return Response( 149 | serialized_project_page, 150 | media_type=response_format.value, 151 | ) 152 | 153 | @get("/resources/{project_name}/{resource_name}") 154 | async def resources( 155 | request: fastapi.Request, 156 | resource_name: str, 157 | project_name: str, 158 | ) -> fastapi.Response: 159 | 160 | req_ctx = model.RequestContext( 161 | resource_repository, 162 | context=dict(request.headers.items()), 163 | ) 164 | 165 | try: 166 | resource = await resource_repository.get_resource( 167 | project_name, 168 | resource_name, 169 | request_context=req_ctx, 170 | ) 171 | except errors.ResourceUnavailable as e: 172 | raise HTTPException(status_code=404, detail=str(e)) from e 173 | except errors.InvalidPackageError as e: 174 | raise HTTPException(status_code=502, detail=str(e)) from e 175 | 176 | if isinstance(resource, model.TextResource): 177 | # Use the first 12 characters of the metadata digest as ETag 178 | text_hash = hashlib.sha256(resource.text.encode('UTF-8')).hexdigest()[:12] 179 | etag = f'"{text_hash}"' 180 | response_headers = {'ETag': etag} 181 | if etag == request.headers.get("if-none-match"): 182 | raise HTTPException( 183 | 304, 184 | headers=response_headers, 185 | ) 186 | return PlainTextResponse( 187 | content=resource.text, 188 | headers=response_headers, 189 | ) 190 | 191 | if isinstance(resource, model.HttpResource): 192 | response_iterator = await HttpResponseIterator.create_iterator( 193 | http_client=http_client, 194 | url=resource.url, 195 | request_headers=request.headers, 196 | ) 197 | return StreamingResponse( 198 | content=response_iterator, 199 | status_code=response_iterator.status_code, 200 | headers=response_iterator.headers, 201 | ) 202 | 203 | if isinstance(resource, model.LocalResource): 204 | ctx_etag = resource.context.get("etag") 205 | response_headers = {"ETag": ctx_etag} if ctx_etag else {} 206 | if client_etag := request.headers.get("if-none-match"): 207 | if client_etag == ctx_etag: 208 | raise HTTPException( 209 | 304, 210 | headers=response_headers, 211 | ) 212 | return FileResponse( 213 | path=resource.path, 214 | media_type="application/octet-stream", 215 | headers=response_headers, 216 | ) 217 | 218 | raise ValueError("Unsupported resource type") 219 | 220 | return simple_router 221 | -------------------------------------------------------------------------------- /simple_repository_server/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | -------------------------------------------------------------------------------- /simple_repository_server/tests/api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | -------------------------------------------------------------------------------- /simple_repository_server/tests/api/test_simple_router.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | 8 | import pathlib 9 | import typing 10 | from unittest import mock 11 | 12 | from fastapi import FastAPI 13 | import httpx 14 | import pytest 15 | from pytest_httpx import HTTPXMock 16 | from simple_repository import errors, model 17 | from simple_repository.components.core import SimpleRepository 18 | from starlette.testclient import TestClient 19 | 20 | import simple_repository_server.routers.simple as simple_router 21 | 22 | 23 | @pytest.fixture 24 | def mock_repo() -> mock.AsyncMock: 25 | mock_repo = mock.AsyncMock(spec=SimpleRepository) 26 | return mock_repo 27 | 28 | 29 | @pytest.fixture 30 | def client(tmp_path: pathlib.PosixPath, mock_repo: mock.AsyncMock) -> typing.Generator[TestClient, None, None]: 31 | app = FastAPI() 32 | http_client = httpx.AsyncClient() 33 | app.include_router(simple_router.build_router(mock_repo, http_client=http_client)) 34 | 35 | with TestClient(app) as test_client: 36 | yield test_client 37 | 38 | 39 | @pytest.mark.parametrize( 40 | "headers", [{}, {"Accept": "text/html"}, {"Accept": "*/*"}], 41 | ) 42 | def test_simple_project_list(client: TestClient, headers: dict[str, str], mock_repo: mock.AsyncMock) -> None: 43 | assert isinstance(client.app, FastAPI) 44 | mock_repo.get_project_list.return_value = model.ProjectList( 45 | meta=model.Meta("1.0"), 46 | projects=frozenset([ 47 | model.ProjectListElement("a"), 48 | ]), 49 | ) 50 | 51 | expected = """ 52 | 53 | 54 | 55 | Simple index 56 | 57 | 58 | a
59 | 60 | """ 61 | 62 | response = client.get("/simple/", headers=headers) 63 | assert response.status_code == 200 64 | assert response.text == expected 65 | 66 | 67 | @pytest.mark.parametrize( 68 | "headers", [{}, {"Accept": "text/html"}, {"Accept": "*/*"}], 69 | ) 70 | def test_simple_project_page(client: TestClient, headers: dict[str, str], mock_repo: mock.AsyncMock) -> None: 71 | assert isinstance(client.app, FastAPI) 72 | mock_repo.get_project_page.return_value = model.ProjectDetail( 73 | meta=model.Meta("1.0"), 74 | name="name", 75 | files=(model.File("name.whl", "original_url", {}),), 76 | ) 77 | 78 | expected = """ 79 | 80 | 81 | 82 | Links for name 83 | 84 | 85 |

Links for name

86 | name.whl
87 | 88 | """ 89 | 90 | response = client.get("/simple/name/", headers=headers) 91 | assert response.status_code == 200 92 | assert response.text == expected 93 | 94 | 95 | def test_simple_package_releases__not_normalized(client: TestClient, mock_repo: mock.AsyncMock) -> None: 96 | assert isinstance(client.app, FastAPI) 97 | response = client.get("/simple/not_Normalized/", follow_redirects=False) 98 | assert response.status_code == 301 99 | assert response.headers['location'] == '../not-normalized/' 100 | 101 | 102 | def test_simple_package_releases__no_trailing_slash(client: TestClient, mock_repo: mock.AsyncMock) -> None: 103 | assert isinstance(client.app, FastAPI) 104 | response = client.get("/simple/some-project", follow_redirects=False) 105 | assert response.status_code == 307 # Provided by FastAPI itself 106 | 107 | 108 | @pytest.mark.asyncio 109 | async def test_simple_package_releases__package_not_found(client: TestClient, mock_repo: mock.AsyncMock) -> None: 110 | assert isinstance(client.app, FastAPI) 111 | mock_repo.get_project_page.side_effect = errors.PackageNotFoundError( 112 | package_name="ghost", 113 | ) 114 | 115 | response = client.get("/simple/ghost") 116 | assert response.status_code == 404 117 | assert response.json() == { 118 | 'detail': "Package 'ghost' was not found in " 119 | "the configured source", 120 | } 121 | 122 | 123 | def test_get_resource__remote(mock_repo: mock.AsyncMock, httpx_mock: HTTPXMock) -> None: 124 | mock_repo.get_resource.return_value = model.HttpResource( 125 | url="http://my_url", 126 | ) 127 | 128 | httpx_mock.add_response( 129 | status_code=201, 130 | headers={"my_header": "header"}, 131 | text="b1b2b3", 132 | ) 133 | http_client = httpx.AsyncClient() 134 | app = FastAPI() 135 | app.include_router(simple_router.build_router(mock_repo, http_client=http_client)) 136 | client = TestClient(app) 137 | 138 | response = client.get("/resources/numpy/numpy-1.0-ciao.whl", follow_redirects=False) 139 | assert response.status_code == 201 140 | assert response.headers.get("my_header") == "header" 141 | assert response.text == "b1b2b3" 142 | 143 | 144 | def test_get_resource_not_found(client: TestClient, mock_repo: mock.AsyncMock) -> None: 145 | assert isinstance(client.app, FastAPI) 146 | mock_repo.get_resource.side_effect = errors.ResourceUnavailable("resource_name") 147 | response = client.get("/resources/numpy/numpy1.0.whl") 148 | assert response.status_code == 404 149 | 150 | 151 | def test_unsupported_serialization(client: TestClient) -> None: 152 | response = client.get("/simple/", headers={"accept": "pizza/margherita"}) 153 | assert response.status_code == 406 154 | 155 | response = client.get("/simple/numpy/", headers={"accept": "application/vnd.pypi.simple.v2+html"}) 156 | assert response.status_code == 406 157 | 158 | 159 | def test_simple_project_page_json(client: TestClient, mock_repo: mock.AsyncMock) -> None: 160 | assert isinstance(client.app, FastAPI) 161 | mock_repo.get_project_page.return_value = model.ProjectDetail( 162 | meta=model.Meta("1.0"), 163 | name="name", 164 | files=(model.File("name.whl", "url", {}),), 165 | ) 166 | 167 | expected = ( 168 | '{"meta": {"api-version": "1.0"}, "name": "name",' 169 | ' "files": [{"filename": "name.whl", "url": ' 170 | '"../../resources/name/name.whl", "hashes": {}}]}' 171 | ) 172 | 173 | response = client.get("/simple/name/", headers={"accept": "application/vnd.pypi.simple.v1+json"}) 174 | assert response.status_code == 200 175 | assert response.text == expected 176 | assert response.headers["Content-Type"] == "application/vnd.pypi.simple.v1+json" 177 | 178 | 179 | def test_simple_project_list_json(client: TestClient, mock_repo: mock.AsyncMock) -> None: 180 | assert isinstance(client.app, FastAPI) 181 | mock_repo.get_project_list.return_value = model.ProjectList( 182 | meta=model.Meta("1.0"), 183 | projects=frozenset([ 184 | model.ProjectListElement("a"), 185 | ]), 186 | ) 187 | 188 | expected = '{"meta": {"api-version": "1.0"}, "projects": [{"name": "a"}]}' 189 | 190 | response = client.get("/simple/", headers={"accept": "application/vnd.pypi.simple.v1+json"}) 191 | assert response.status_code == 200 192 | assert response.text == expected 193 | 194 | 195 | @pytest.mark.parametrize( 196 | "url_format", [ 197 | "application/vnd.pypi.simple.v1+json", 198 | "application/vnd.pypi.simple.v1+html", 199 | ], 200 | ) 201 | def test_simple_project_page__json_url_params( 202 | client: TestClient, 203 | url_format: str, 204 | mock_repo: mock.AsyncMock, 205 | ) -> None: 206 | assert isinstance(client.app, FastAPI) 207 | mock_repo.get_project_page.return_value = model.ProjectDetail( 208 | meta=model.Meta("1.0"), 209 | name="name", 210 | files=(model.File("name.whl", "url", {}),), 211 | ) 212 | 213 | response = client.get(f"/simple/name/?format={url_format}") 214 | assert response.headers.get("content-type") == url_format 215 | 216 | 217 | @pytest.mark.parametrize( 218 | "url_format", [ 219 | "application/vnd.pypi.simple.v1+json", 220 | "application/vnd.pypi.simple.v1+html", 221 | ], 222 | ) 223 | def test_simple_project_list__json_url_params( 224 | client: TestClient, 225 | url_format: str, 226 | mock_repo: mock.AsyncMock, 227 | ) -> None: 228 | assert isinstance(client.app, FastAPI) 229 | mock_repo.get_project_list.return_value = model.ProjectList( 230 | meta=model.Meta("1.0"), 231 | projects=frozenset([ 232 | model.ProjectListElement("a"), 233 | ]), 234 | ) 235 | 236 | response = client.get(f"/simple/?format={url_format}") 237 | assert response.headers.get("content-type") == url_format 238 | 239 | 240 | @pytest.mark.parametrize( 241 | ['headers', 'expected_return_code'], 242 | [ 243 | [{}, 200], 244 | [{"If-None-Match": '"45447b7afbd5"'}, 304], 245 | [{"If-None-Match": '"not-the-etag"'}, 200], 246 | ], 247 | ) 248 | def test_get_resource__metadata( 249 | client: TestClient, 250 | mock_repo: mock.AsyncMock, 251 | headers: dict[str, str], 252 | expected_return_code: int, 253 | ) -> None: 254 | assert isinstance(client.app, FastAPI) 255 | mock_repo.get_resource.return_value = model.TextResource( 256 | text="metadata", 257 | ) 258 | expected_etag = '"45447b7afbd5"' 259 | 260 | response = client.get("/resources/numpy/numpy-1.0-ciao.whl.metadata", headers=headers) 261 | assert response.status_code == expected_return_code 262 | # The etag must always be returned, see the following for details: 263 | # https://github.com/simple-repository/simple-repository-server/issues/6#issue-2317360891 264 | assert response.headers.get("etag") == expected_etag 265 | 266 | 267 | @pytest.mark.parametrize( 268 | ['headers', 'expected_return_code'], 269 | [ 270 | [{}, 200], 271 | [{"If-None-Match": '"430fddbf0a7ab4aebc1389262dbe2404"'}, 304], 272 | [{"If-None-Match": '"not-the-etag"'}, 200], 273 | ], 274 | ) 275 | def test_get_resource__local( 276 | client: TestClient, 277 | mock_repo: mock.AsyncMock, 278 | tmp_path: pathlib.Path, 279 | headers: dict[str, str], 280 | expected_return_code: int, 281 | ) -> None: 282 | local_resource = tmp_path / "my_file" 283 | local_resource.write_text("hello!") 284 | expected_tag = '"430fddbf0a7ab4aebc1389262dbe2404"' 285 | 286 | assert isinstance(client.app, FastAPI) 287 | mock_repo.get_resource.return_value = model.LocalResource( 288 | path=local_resource, 289 | context=model.Context(etag=expected_tag), 290 | ) 291 | 292 | response = client.get("/resources/numpy/numpy-1.0-ciao.whl", headers=headers) 293 | assert response.status_code == expected_return_code 294 | # The etag must always be returned, see the following for details: 295 | # https://github.com/simple-repository/simple-repository-server/issues/6#issue-2317360891 296 | assert response.headers.get("etag") == expected_tag 297 | -------------------------------------------------------------------------------- /simple_repository_server/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simple-repository/simple-repository-server/d9a861d82761cd6ae1b40ceb06ffbdbcc1697780/simple_repository_server/tests/integration/__init__.py -------------------------------------------------------------------------------- /simple_repository_server/tests/integration/test_repo_dependency_injection.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | import fastapi 4 | from fastapi.testclient import TestClient 5 | import httpx 6 | import pytest 7 | from simple_repository import model 8 | from simple_repository.components.core import SimpleRepository 9 | from simple_repository.tests.components.fake_repository import FakeRepository 10 | 11 | from simple_repository_server.routers import simple 12 | 13 | 14 | def create_app(repo: SimpleRepository, repo_factory: typing.Callable[..., SimpleRepository]) -> fastapi.FastAPI: 15 | app = fastapi.FastAPI(openapi_url=None) 16 | 17 | http_client = httpx.AsyncClient() 18 | app.include_router( 19 | simple.build_router( 20 | repo, 21 | http_client=http_client, 22 | prefix="/snapshot/{cutoff_date}/", 23 | repo_factory=repo_factory, 24 | ), 25 | ) 26 | 27 | return app 28 | 29 | 30 | @pytest.fixture 31 | def fake_repo() -> SimpleRepository: 32 | return FakeRepository( 33 | project_list=model.ProjectList(model.Meta("1.0"), [model.ProjectListElement("foo-bar")]), 34 | project_pages=[ 35 | model.ProjectDetail( 36 | model.Meta('1.1'), 37 | "foo-bar", 38 | files=( 39 | model.File("foo_bar-2.0-any.whl", "", {}, size=1), 40 | model.File("foo_bar-3.0-any.whl", "", {}, size=1), 41 | ), 42 | ), 43 | ], 44 | ) 45 | 46 | 47 | @pytest.fixture 48 | def empty_repo() -> SimpleRepository: 49 | return FakeRepository() 50 | 51 | 52 | class SimpleFactoryWithParams: 53 | def __init__(self, repo: SimpleRepository): 54 | self.cutoff_date = None 55 | self.repo = repo 56 | 57 | def __call__(self, cutoff_date: str) -> SimpleRepository: 58 | self.cutoff_date = cutoff_date 59 | # In this factory, just return the original repo, but we return a 60 | # more specific repo here. 61 | return self.repo 62 | 63 | 64 | @pytest.fixture 65 | def repo_factory(fake_repo: SimpleRepository) -> SimpleFactoryWithParams: 66 | return SimpleFactoryWithParams(repo=fake_repo) 67 | 68 | 69 | @pytest.mark.asyncio 70 | async def test_repo_with_dependency_injection__projects_list( 71 | empty_repo: SimpleRepository, 72 | repo_factory: SimpleFactoryWithParams, 73 | ): 74 | app = create_app(empty_repo, repo_factory=repo_factory) 75 | client = TestClient(app) 76 | response = client.get("/snapshot/2020-10-12/?format=application/vnd.pypi.simple.v1+json") 77 | 78 | # Check that the factory was called with the expected args. 79 | assert repo_factory.cutoff_date == "2020-10-12" 80 | 81 | # And that the response is not for the empty repo, but the factory one. 82 | assert response.status_code == 200 83 | assert response.headers['content-type'] == 'application/vnd.pypi.simple.v1+json' 84 | assert response.json() == { 85 | "meta": { 86 | "api-version": "1.0", 87 | }, 88 | "projects": [ 89 | { 90 | "name": "foo-bar", 91 | }, 92 | ], 93 | } 94 | 95 | 96 | @pytest.mark.asyncio 97 | async def test_repo_with_dependency_injection__project_page( 98 | empty_repo: SimpleRepository, 99 | repo_factory: SimpleFactoryWithParams, 100 | ): 101 | app = create_app(empty_repo, repo_factory=repo_factory) 102 | client = TestClient(app) 103 | response = client.get("/snapshot/2020-10-12/foo-bar/?format=application/vnd.pypi.simple.v1+json") 104 | 105 | # Check that the factory was called with the expected args. 106 | assert repo_factory.cutoff_date == "2020-10-12" 107 | 108 | assert response.status_code == 200 109 | assert response.headers['content-type'] == 'application/vnd.pypi.simple.v1+json' 110 | assert response.json() == { 111 | "meta": { 112 | "api-version": "1.1", 113 | }, 114 | "name": "foo-bar", 115 | "files": [ 116 | { 117 | "filename": "foo_bar-2.0-any.whl", 118 | "url": "../../../resources/foo-bar/foo_bar-2.0-any.whl", 119 | "hashes": {}, 120 | "size": 1, 121 | }, 122 | { 123 | "filename": "foo_bar-3.0-any.whl", 124 | "url": "../../../resources/foo-bar/foo_bar-3.0-any.whl", 125 | "hashes": {}, 126 | "size": 1, 127 | }, 128 | ], 129 | "versions": [ 130 | "2.0", 131 | "3.0", 132 | ], 133 | } 134 | 135 | 136 | @pytest.mark.asyncio 137 | async def test_repo_with_dependency_injection__project_page__redirect( 138 | empty_repo: SimpleRepository, 139 | repo_factory: SimpleFactoryWithParams, 140 | ): 141 | app = create_app(empty_repo, repo_factory=repo_factory) 142 | client = TestClient(app) 143 | response = client.get( 144 | "/snapshot/2020-10-12/foo_Bar/?format=application/vnd.pypi.simple.v1+json", 145 | follow_redirects=False, 146 | ) 147 | 148 | # Check that the factory was called with the expected args. 149 | assert repo_factory.cutoff_date == "2020-10-12" 150 | 151 | assert response.status_code == 301 152 | # Ensure that we maintain the querystring. 153 | assert response.headers['location'] == '../foo-bar/?format=application/vnd.pypi.simple.v1+json' 154 | -------------------------------------------------------------------------------- /simple_repository_server/tests/test__http_response_iterator.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | 8 | from random import randbytes 9 | import typing 10 | import zlib 11 | 12 | import httpx 13 | import pytest 14 | from pytest_httpserver import HTTPServer 15 | from pytest_httpx import HTTPXMock 16 | 17 | from simple_repository_server._http_response_iterator import HttpResponseIterator 18 | 19 | 20 | @pytest.mark.asyncio 21 | async def test_http_response_iterator__request_headers_passed_through( 22 | httpx_mock: HTTPXMock, 23 | ) -> None: 24 | # Check that we can pass headers through to the proxied request. 25 | httpx_mock.add_response() 26 | 27 | http_client = httpx.AsyncClient() 28 | _ = await HttpResponseIterator.create_iterator( 29 | http_client, 30 | 'https://example.com/some/path', 31 | request_headers={'foo': 'bar', 'accept-encoding': 'wibble-wobble'}, 32 | ) 33 | 34 | request = httpx_mock.get_request() 35 | assert request is not None 36 | assert request.headers['accept-encoding'] == 'wibble-wobble' 37 | assert 'foo' not in request.headers 38 | 39 | 40 | _DEFLATE = zlib.compressobj(4, zlib.DEFLATED, -zlib.MAX_WBITS) 41 | 42 | 43 | @pytest.mark.parametrize( 44 | ['input_content'], 45 | [ 46 | ["This is the response content".encode('utf-8')], 47 | [randbytes(1024 * 1024 * 3)], # 3 pages of chunked content 48 | ], 49 | ids=['utf8_encoded_bytes', 'multi_page_bytestring'], 50 | ) 51 | @pytest.mark.parametrize( 52 | ['encoding_name', 'encoder', 'decoder'], 53 | [ 54 | ['gzip', zlib.compress, zlib.decompress], 55 | # See https://stackoverflow.com/a/22311297/741316 56 | [ 57 | 'deflate', 58 | lambda data: _DEFLATE.compress(data) + _DEFLATE.flush(), 59 | lambda data: zlib.decompress(data, -zlib.MAX_WBITS), 60 | ], 61 | ['never-seen-before', lambda data: data + b'abc', lambda data: data[:-3]], 62 | ], 63 | ids=['gzip', 'deflate', 'never-seen-before'], 64 | ) 65 | @pytest.mark.asyncio 66 | async def test_http_response_iterator__response_remains_gzipped( 67 | httpserver: HTTPServer, 68 | input_content: bytes, 69 | encoding_name: str, 70 | encoder: typing.Callable[[bytes], bytes], 71 | decoder: typing.Callable[[bytes], bytes], 72 | ) -> typing.Any: 73 | # Serve some content as compressed bytes, and ensure that we can stream it 74 | # through the iterator (with the correct headers etc.). 75 | # We use a real test http server, to ensure that we are genuinely handling 76 | # gzipped responses correctly. 77 | try: 78 | compressed = encoder(input_content) 79 | except zlib.error: 80 | return pytest.xfail(reason='Known zlib error') 81 | httpserver.expect_request('/path').respond_with_data( 82 | compressed, 83 | headers={ 84 | 'content-type': 'application/octet-stream', 85 | 'content-encoding': encoding_name, 86 | }, 87 | ) 88 | 89 | http_client = httpx.AsyncClient() 90 | response_it = await HttpResponseIterator.create_iterator( 91 | http_client, 92 | httpserver.url_for('/path'), 93 | request_headers={'foo': 'bar', 'accept-encoding': 'gzip'}, 94 | ) 95 | 96 | assert response_it.headers['content-type'] == 'application/octet-stream' 97 | assert response_it.headers['content-encoding'] == encoding_name 98 | assert int(response_it.headers['content-length']) == len(compressed) 99 | content = b''.join([chunk async for chunk in response_it]) 100 | assert len(content) == len(compressed) 101 | assert decoder(content) == input_content 102 | -------------------------------------------------------------------------------- /simple_repository_server/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | -------------------------------------------------------------------------------- /simple_repository_server/tests/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | 8 | from unittest import mock 9 | 10 | from packaging.version import Version 11 | import pytest 12 | from simple_repository import model 13 | from starlette.datastructures import URL 14 | 15 | from simple_repository_server import utils 16 | 17 | 18 | def test_replace_urls() -> None: 19 | page = model.ProjectDetail( 20 | meta=model.Meta("1.0"), 21 | name="numpy", 22 | files=(model.File("numpy-1.0-any.whl", "old_url", {}),), 23 | ) 24 | 25 | with mock.patch("simple_repository_server.utils.relative_url_for", return_value="new_url"): 26 | page = utils.replace_urls(page, "numpy", mock.Mock()) 27 | 28 | assert page.files == ( 29 | model.File("numpy-1.0-any.whl", "new_url", {}), 30 | ) 31 | 32 | 33 | @pytest.mark.parametrize( 34 | "origin, destination, result", [ 35 | ( 36 | "https://simple-repository/simple/numpy/", 37 | "https://simple-repository/resources/numpy/numpy-1.0.whl", 38 | "../../resources/numpy/numpy-1.0.whl", 39 | ), ( 40 | "https://simple-repository/simple/Numpy/", 41 | "https://simple-repository/simple/numpy/", 42 | "../numpy/", 43 | ), ( 44 | "https://simple-repository/simple/Numpy", 45 | "https://simple-repository/simple/numpy", 46 | "numpy", 47 | ), ( 48 | "https://simple-repository/simple/", 49 | "https://simple-repository/simple/numpy/", 50 | "numpy/", 51 | ), ( 52 | "https://simple-repository/simple/", 53 | "https://simple-repository/simple/", 54 | "", 55 | ), ( 56 | "https://simple-repository/simple", 57 | "https://simple-repository/simple", 58 | "simple", 59 | ), ( 60 | "https://simple-repository/simple", 61 | "https://simple-repository/simple/", 62 | "simple/", 63 | ), ( 64 | "https://simple-repository/simple/", 65 | "https://simple-repository/simple", 66 | "../simple", 67 | ), ( 68 | "https://simple-repository/simple/project/numpy", 69 | "https://simple-repository/simple/", 70 | "../", 71 | ), 72 | ], 73 | ) 74 | def test_url_as_relative(destination: str, origin: str, result: str) -> None: 75 | assert utils.url_as_relative( 76 | destination_absolute_url=destination, 77 | origin_absolute_url=origin, 78 | ) == result 79 | 80 | 81 | @pytest.mark.parametrize( 82 | "origin, destination", [ 83 | ( 84 | "http://simple-repository/simple/numpy/", 85 | "https://simple-repository/resources/numpy/numpy-1.0.whl", 86 | ), ( 87 | "https://simple-repository/simple/Numpy/", 88 | "https://simple-repository2/simple/numpy/", 89 | ), ( 90 | "https://simple-repository:81/simple/Numpy", 91 | "https://simple-repository:80/simple/numpy", 92 | ), ( 93 | "https://simple-repository/simple/numpy/", 94 | "../tensorflow", 95 | ), ( 96 | "../tensorflow", 97 | "https://simple-repository/simple/numpy/", 98 | ), 99 | ], 100 | ) 101 | def test_url_as_relative__invalid(origin: str, destination: str) -> None: 102 | with pytest.raises( 103 | ValueError, 104 | match=f"Cannot create a relative url from {origin} to {destination}", 105 | ): 106 | utils.url_as_relative( 107 | destination_absolute_url=destination, 108 | origin_absolute_url=origin, 109 | ) 110 | 111 | 112 | def test_relative_url_for() -> None: 113 | request_mock = mock.Mock( 114 | url=URL("https://url/number/one"), 115 | url_for=mock.Mock(return_value=URL("https://url/number/one")), 116 | ) 117 | url_as_relative_mock = mock.Mock() 118 | 119 | with mock.patch("simple_repository_server.utils.url_as_relative", url_as_relative_mock): 120 | utils.relative_url_for(request=request_mock, name="name") 121 | 122 | url_as_relative_mock.assert_called_once_with( 123 | origin_absolute_url="https://url/number/one", 124 | destination_absolute_url="https://url/number/one", 125 | ) 126 | 127 | 128 | @pytest.mark.parametrize( 129 | "header, version", [ 130 | ('pip/23.0.1 {"installer":{"name":"pip","version":"23.0.1"}}', Version("23.0.1")), 131 | ('{"installer":{"name":"pip","version":"23.0.1"}}', Version("23.0.1")), 132 | ('', None), 133 | ('*/*', None), 134 | ('pip/23.0.1 {"installer":{"name":"pip","version":"AAA"}}', None), 135 | ], 136 | ) 137 | def test_get_pip_version(header: str, version: Version | None) -> None: 138 | mock_request = mock.Mock(headers={"user-agent": header}) 139 | utils.get_pip_version(mock_request) == version 140 | -------------------------------------------------------------------------------- /simple_repository_server/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023, CERN 2 | # This software is distributed under the terms of the MIT 3 | # licence, copied verbatim in the file "LICENSE". 4 | # In applying this license, CERN does not waive the privileges and immunities 5 | # granted to it by virtue of its status as Intergovernmental Organization 6 | # or submit itself to any jurisdiction. 7 | 8 | from dataclasses import replace 9 | import json 10 | import re 11 | import typing 12 | from urllib.parse import urlparse 13 | 14 | import fastapi 15 | from packaging.version import InvalidVersion, Version 16 | from simple_repository import model 17 | 18 | 19 | def url_as_relative( 20 | destination_absolute_url: str, 21 | origin_absolute_url: str, 22 | ) -> str: 23 | """Converts, if possible, the destination_absolute_url to a relative to origin_absolute_url""" 24 | parsed_destination_url = urlparse(destination_absolute_url) 25 | parsed_origin_url = urlparse(origin_absolute_url) 26 | 27 | if ( 28 | parsed_origin_url.scheme != parsed_destination_url.scheme or 29 | parsed_origin_url.scheme not in ["http", "https"] or 30 | parsed_origin_url.netloc != parsed_destination_url.netloc 31 | ): 32 | raise ValueError( 33 | "Cannot create a relative url from " 34 | f"{origin_absolute_url} to {destination_absolute_url}", 35 | ) 36 | 37 | destination_absolute_path = parsed_destination_url.path 38 | origin_absolute_path = parsed_origin_url.path 39 | 40 | # Extract all the segments in the url contained between two "/" 41 | destination_path_tokens = destination_absolute_path.split("/")[1:-1] 42 | origin_path_tokens = origin_absolute_path.split("/")[1:-1] 43 | # Calculate the depth of the origin path. It will be the initial 44 | # number of dirs to delete from the url to get the relative path. 45 | dirs_up = len(origin_path_tokens) 46 | 47 | common_prefix = "/" 48 | for destination_path_token, origin_path_token in zip( 49 | destination_path_tokens, origin_path_tokens, 50 | ): 51 | if destination_path_token == origin_path_token: 52 | # If the two urls share a parent dir, reduce the number of dirs to delete 53 | dirs_up -= 1 54 | common_prefix += destination_path_token + "/" 55 | else: 56 | break 57 | 58 | return "../" * dirs_up + destination_absolute_path.removeprefix(common_prefix) 59 | 60 | 61 | def relative_url_for( 62 | request: fastapi.Request, 63 | name: str, 64 | **kwargs: typing.Any, 65 | ) -> str: 66 | origin_url = str(request.url) 67 | destination_url = str(request.url_for(name, **kwargs)) 68 | 69 | return url_as_relative( 70 | origin_absolute_url=origin_url, 71 | destination_absolute_url=destination_url, 72 | ) 73 | 74 | 75 | def replace_urls( 76 | project_page: model.ProjectDetail, 77 | project_name: str, 78 | request: fastapi.Request, 79 | ) -> model.ProjectDetail: 80 | files = tuple( 81 | replace( 82 | file, 83 | url=relative_url_for( 84 | request=request, 85 | name="resources", 86 | project_name=project_name, 87 | resource_name=file.filename, 88 | ), 89 | ) for file in project_page.files 90 | ) 91 | return replace(project_page, files=files) 92 | 93 | 94 | PIP_HEADER_REGEX = re.compile(r'^.*?{') 95 | 96 | 97 | def get_pip_version( 98 | request: fastapi.Request, 99 | ) -> Version | None: 100 | if not (pip_header_string := request.headers.get('user-agent', '')): 101 | return None 102 | pip_header = PIP_HEADER_REGEX.sub("{", pip_header_string) 103 | try: 104 | pip_info = json.loads(pip_header) 105 | except json.decoder.JSONDecodeError: 106 | return None 107 | if not isinstance(pip_info, dict): 108 | return None 109 | 110 | if implementation := pip_info.get('installer'): 111 | if isinstance(implementation, dict): 112 | version_string = implementation.get('version', '') 113 | try: 114 | return Version(version_string) 115 | except InvalidVersion: 116 | return None 117 | return None 118 | --------------------------------------------------------------------------------