├── .bumpv.cfg ├── .coveragerc ├── .envrc ├── .gitignore ├── .readthedocs.yml ├── .travis.yml ├── MANIFEST.in ├── Makefile ├── README.rst ├── conftest.py ├── constraints.txt ├── default.nix ├── integrationtests ├── __init__.py ├── aiohttp │ └── requirements_override.nix ├── appdirs │ └── requirements_override.nix ├── awscli_and_requests │ └── requirements_override.nix ├── connexion │ └── requirements_override.nix ├── dependency_graph │ └── requirements_override.nix ├── dependency_graph_input │ └── requirements_override.nix ├── empy │ └── requirements_override.nix ├── fava │ └── requirements_override.nix ├── flake8-mercurial │ └── requirements_override.nix ├── flake8 │ └── requirements_override.nix ├── flake8_include_default_overrides │ └── requirements_override.nix ├── flit │ └── requirements_override.nix ├── framework.py ├── ldap │ └── requirements_override.nix ├── lektor │ └── requirements_override.nix ├── local_path │ ├── egg │ │ ├── setup.cfg │ │ └── setup.py │ └── requirements_override.nix ├── pillow │ └── requirements_override.nix ├── pynacl │ └── requirements_override.nix ├── relative_paths │ ├── requirements_override.nix │ └── test_package │ │ ├── setup.cfg │ │ └── setup.py ├── rss2email │ └── requirements_override.nix ├── scipy │ └── requirements_override.nix ├── serpy │ └── requirements_override.nix ├── setuptools │ └── requirements_override.nix ├── test_aiohttp.py ├── test_appdirs.py ├── test_awscli_and_requests.py ├── test_connexion.py ├── test_dependency_graph.py ├── test_empy.py ├── test_fava.py ├── test_flake8.py ├── test_flake8_mercurial.py ├── test_flit.py ├── test_ldap.py ├── test_lektor.py ├── test_local_path.py ├── test_pillow.py ├── test_pynacl.py ├── test_relative_paths.py ├── test_rss2email.py ├── test_scipy.py ├── test_serpy_0_1_1.py ├── test_setuptools.py ├── test_tornado.py └── tornado │ └── requirements_override.nix ├── mypy ├── jsonschema.pyi ├── nix_prefetch_github │ └── __init__.pyi ├── packaging │ ├── __init__.pyi │ ├── markers.pyi │ ├── utils.pyi │ └── version.pyi ├── parsley.pyi ├── setuptools │ ├── __init__.py │ └── config.pyi ├── setuptools_scm.pyi └── venv.pyi ├── pytest.ini ├── requirements-dev.txt ├── requirements.nix ├── requirements.txt ├── requirements_frozen.txt ├── requirements_override.nix ├── scripts ├── __init__.py ├── build-pip.nix ├── build_wheel.py ├── deploy_to_pypi.py ├── format_sources.py ├── install_test.py ├── package_source.py ├── prepare_test_data.py ├── repository.py ├── run_integration_tests.py ├── update_dependencies.py └── update_python_packages.py ├── setup.cfg ├── setup.py ├── shell.nix ├── source ├── conf.py ├── development.rst ├── external-dependencies.rst ├── index.rst └── modules.rst ├── src └── pypi2nix │ ├── VERSION │ ├── __init__.py │ ├── archive.py │ ├── cli.py │ ├── configuration.py │ ├── dependency_graph.py │ ├── environment_marker.py │ ├── exceptions.py │ ├── expression_renderer.py │ ├── external_dependencies │ ├── __init__.py │ └── external_dependency.py │ ├── external_dependency_collector │ ├── __init__.py │ ├── collector.py │ └── lookup.py │ ├── license.py │ ├── logger.py │ ├── main.py │ ├── memoize.py │ ├── metadata_fetcher.py │ ├── network_file.py │ ├── nix.py │ ├── nix_language.py │ ├── overrides.py │ ├── package │ ├── __init__.py │ ├── exceptions.py │ ├── interfaces.py │ ├── metadata.py │ ├── pyproject.py │ └── setupcfg.py │ ├── package_source.py │ ├── path.py │ ├── pip │ ├── __init__.py │ ├── base.nix │ ├── bootstrap.nix │ ├── download.nix │ ├── exceptions.py │ ├── implementation.py │ ├── install.nix │ ├── interface.py │ ├── virtualenv.py │ └── wheel.nix │ ├── project_directory.py │ ├── pypi.py │ ├── pypi_package.py │ ├── pypi_release.py │ ├── python_version.py │ ├── requirement_parser.py │ ├── requirement_parser_grammar.py │ ├── requirement_set.py │ ├── requirements.py │ ├── requirements_collector.py │ ├── requirements_file.py │ ├── source_distribution.py │ ├── sources.py │ ├── target_platform.py │ ├── templates │ ├── generated.nix.j2 │ ├── overrides.nix.j2 │ ├── prefetch-github.nix.j2 │ └── requirements.nix.j2 │ ├── utils.py │ ├── version.py │ ├── wheel.py │ ├── wheel_builder.py │ └── wheels │ ├── __init__.py │ ├── index.json │ └── schema.py └── unittests ├── __init__.py ├── data ├── flit-1.3-py3-none-any.whl ├── flit-1.3.tar.gz ├── jsonschema-3.0.1.tar.gz ├── package1-1.0-py3-none-any.whl ├── package1-1.0.tar.gz ├── package1 │ ├── package1.tar.gz │ ├── setup.cfg │ └── setup.py ├── package2-1.0-py3-none-any.whl ├── package2-1.0.tar.gz ├── package2 │ ├── package2.tar.gz │ ├── setup.cfg │ └── setup.py ├── package3-1.0-py3-none-any.whl ├── package3-1.0.tar.gz ├── package3 │ ├── package3.tar.gz │ ├── setup.cfg │ └── setup.py ├── package4-1.0-py3-none-any.whl ├── package4-1.0.tar.gz ├── package4 │ ├── package4.tar.gz │ ├── setup.cfg │ └── setup.py ├── setupcfg-package-1.0.tar.gz ├── setupcfg-package │ ├── setup.cfg │ ├── setup.py │ └── setupcfg-package.tar.gz ├── setupcfg_package-1.0-py3-none-any.whl ├── setuptools-41.2.0.zip ├── shell_environment.nix ├── six-1.12.0.tar.gz ├── spacy-2.1.0.tar.gz ├── test.tar.bz2 ├── test.tar.gz ├── test.txt ├── test.zip └── wheel-0.33.6-py2.py3-none-any.whl ├── logger.py ├── package_generator.py ├── pip ├── __init__.py ├── conftest.py ├── test_download.py ├── test_freeze.py ├── test_install.py ├── test_virtualenv_pip.py └── test_wheel.py ├── regression ├── test_issue_363.py └── test_issue_394.py ├── switches.py ├── templates.py ├── templates ├── setup.cfg └── setup.py ├── test_archive.py ├── test_dependency_graph.py ├── test_dependency_graph_serialization.py ├── test_environment_marker.py ├── test_license.py ├── test_logger.py ├── test_memoize.py ├── test_network_file.py ├── test_nix.py ├── test_package_generator.py ├── test_package_source.py ├── test_prefetch_url.py ├── test_project_directory.py ├── test_pypi.py ├── test_python_version.py ├── test_requirement.py ├── test_requirement_collector.py ├── test_requirement_dependency_retriever.py ├── test_requirement_parser.py ├── test_requirement_set.py ├── test_requirements_file.py ├── test_source_distribution.py ├── test_sources.py ├── test_target_platform.py ├── test_util_cmd.py ├── test_wheel.py └── test_wheel_builder.py /.bumpv.cfg: -------------------------------------------------------------------------------- 1 | [bumpv] 2 | current_version = 2.0.4 3 | commit = True 4 | tag = True 5 | parse = (?P\d+)\.(?P\d+)\.(?P\d+) 6 | serialize = {major}.{minor}.{patch} 7 | search = {current_version} 8 | replace = {new_version} 9 | tag_name = v{new_version} 10 | message = Bump version: {current_version} → {new_version} 11 | 12 | [bumpv:file:src/pypi2nix/VERSION] 13 | search = {current_version} 14 | replace = {new_version} 15 | 16 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_lines = 3 | pragma: no cover 4 | @abstract -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | eval "$(lorri direnv)" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ 3 | /.coverage 4 | /build 5 | /deploy_rsa 6 | /out/ 7 | /result* 8 | /src/pypi2nix.egg-info/ 9 | /integrationtests/*/requirements.nix 10 | /integrationtests/*/constraints.txt 11 | /integrationtests/*/requirements.txt 12 | /integrationtests/*/requirements_frozen.txt 13 | /integrationtests/*/dependency-graph.yml 14 | /integrationtests/*/dependency-input.yml 15 | /integrationtests/*/result 16 | /integrationtests/*/build 17 | /integrationtests/pypi2nix 18 | /integrationtests/*/src 19 | [#]*[#] 20 | .[#]* 21 | /htmlcov 22 | .pytest_cache 23 | /dist/ 24 | *.egg-info 25 | **/.eggs/** 26 | .mypy_cache 27 | src/pypi2nix/_version.py 28 | /.hypothesis -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sphinx: 3 | configuration: source/conf.py 4 | 5 | python: 6 | version: 3.7 7 | install: 8 | - requirements: requirements.txt 9 | - requirements: requirements-dev.txt 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: nix 2 | sudo: required 3 | os: 4 | - linux 5 | env: 6 | global: 7 | - NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz 8 | matrix: 9 | - TEST_FILE=aiohttp 10 | - TEST_FILE=dependency_graph 11 | - TEST_FILE=appdirs 12 | - TEST_FILE=awscli_and_requests 13 | - TEST_FILE=connexion 14 | - TEST_FILE=empy 15 | - TEST_FILE=fava 16 | - TEST_FILE=flake8 17 | - TEST_FILE=flake8_mercurial 18 | - TEST_FILE=flit 19 | - TEST_FILE=ldap 20 | - TEST_FILE=lektor 21 | - TEST_FILE=local_path 22 | - TEST_FILE=pillow 23 | - TEST_FILE=pynacl 24 | - TEST_FILE=relative_paths 25 | - TEST_FILE=rss2email 26 | - TEST_FILE=scipy 27 | - TEST_FILE=serpy_0_1_1 28 | - TEST_FILE=setuptools 29 | - TEST_FILE=tornado 30 | install: 31 | - sudo mount -o remount,exec,size=4G,mode=755 /run/user || true 32 | - nix-env -iA nixpkgs.nix-prefetch-git nixpkgs.nix-prefetch-hg 33 | before_script: 34 | - sudo mkdir -p /etc/nix && echo 'sandbox = true' | sudo tee /etc/nix/nix.conf 35 | script: 36 | - nix-shell --command 'run_integration_tests.py --file integrationtests/test_${TEST_FILE}.py' 37 | stages: 38 | - unittests 39 | jobs: 40 | include: 41 | - stage: unittests 42 | name: nixos-19.09 43 | os: linux 44 | env: 45 | - NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09.tar.gz 46 | before_script: 47 | - sudo mkdir -p /etc/nix && echo 'sandbox = true' | sudo tee /etc/nix/nix.conf 48 | script: 49 | - set -e 50 | - nix-build 51 | - result/bin/pypi2nix --version 52 | - nix-shell --command 'flake8 && pytest unittests/' 53 | - name: nixos-unstable 54 | os: linux 55 | env: 56 | - NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz 57 | before_script: 58 | - sudo mkdir -p /etc/nix && echo 'sandbox = true' | sudo tee /etc/nix/nix.conf 59 | script: 60 | - set -e 61 | - nix build 62 | - result/bin/pypi2nix --version 63 | - nix-shell --command 'pytest unittests/' 64 | - name: nixos-20.03 65 | os: linux 66 | env: 67 | - NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03.tar.gz 68 | before_script: 69 | - sudo mkdir -p /etc/nix && echo 'sandbox = true' | sudo tee /etc/nix/nix.conf 70 | script: 71 | - set -e 72 | - nix build 73 | - result/bin/pypi2nix --version 74 | - nix-shell --command 'pytest unittests/' 75 | - stage: package tests 76 | name: package installation 77 | os: linux 78 | env: 79 | - NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz 80 | before_script: 81 | - sudo mkdir -p /etc/nix && echo 'sandbox = true' | sudo tee /etc/nix/nix.conf 82 | script: 83 | - nix-shell --command 'install_test.py' 84 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include src/pypi2nix/pip/*.nix 2 | include src/pypi2nix/templates/*.j2 3 | include src/pypi2nix/wheels/index.json 4 | include src/pypi2nix/VERSION -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /constraints.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import { }, include_nixfmt ? false }: 2 | 3 | let 4 | pythonPackages = import ./requirements.nix { inherit pkgs; }; 5 | 6 | version = "dev"; 7 | 8 | sourceFilter = with pkgs.lib; 9 | with builtins; 10 | path: type: 11 | let 12 | name = baseNameOf (toString path); 13 | ignoreDirectories = directories: 14 | !(any (directory: directory == name && type == "directory") 15 | directories); 16 | ignoreFileTypes = types: 17 | !(any (type: hasSuffix ("." + type) name && type == "regular") types); 18 | ignoreEmacsFiles = !(hasPrefix ".#" name); 19 | in ignoreDirectories [ 20 | "__pycache__" 21 | "build" 22 | "dist" 23 | ".mypy_cache" 24 | ".git" 25 | "integrationtests" 26 | ] && ignoreFileTypes [ "pyc" ] && ignoreEmacsFiles; 27 | 28 | source = pkgs.lib.cleanSourceWith { 29 | src = ./.; 30 | filter = sourceFilter; 31 | }; 32 | 33 | pypi2nixFunction = { mkDerivation, lib, attrs, click, jinja2 34 | , nix-prefetch-github, packaging, parsley, setuptools, setuptools-scm, toml 35 | , jsonschema, hypothesis, pyyaml, }: 36 | mkDerivation { 37 | name = "pypi2nix-${version}"; 38 | src = source; 39 | buildInputs = [ ]; 40 | doCheck = false; 41 | propagatedBuildInputs = [ 42 | attrs 43 | click 44 | jinja2 45 | jsonschema 46 | nix-prefetch-github 47 | packaging 48 | parsley 49 | pyyaml 50 | setuptools 51 | setuptools-scm 52 | toml 53 | ]; 54 | meta = { 55 | homepage = "https://github.com/nix-community/pypi2nix"; 56 | description = 57 | "A tool that generates nix expressions for your python packages, so you don't have to."; 58 | maintainers = with lib.maintainers; [ seppeljordan ]; 59 | }; 60 | }; 61 | 62 | callPackage = pkgs.lib.callPackageWith ({ 63 | mkDerivation = pythonPackages.mkDerivation; 64 | lib = pkgs.lib; 65 | git = pkgs.git; 66 | nix-prefetch-hg = pkgs.nix-prefetch-hg; 67 | } // pythonPackages.packages); 68 | 69 | in callPackage pypi2nixFunction { } 70 | -------------------------------------------------------------------------------- /integrationtests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/integrationtests/__init__.py -------------------------------------------------------------------------------- /integrationtests/aiohttp/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/appdirs/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/awscli_and_requests/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: { 4 | 5 | "awscli" = python.overrideDerivation super."awscli" (old: { 6 | propagatedBuildInputs = old.propagatedBuildInputs 7 | ++ [ pkgs.groff pkgs.less ]; 8 | }); 9 | 10 | } 11 | -------------------------------------------------------------------------------- /integrationtests/connexion/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: { 4 | "jsonschema" = python.overrideDerivation super."jsonschema" 5 | (old: { buildInputs = old.buildInputs ++ [ self."vcversioner" ]; }); 6 | 7 | "pytest-runner" = super."pytest-runner".overrideDerivation (old: { 8 | nativeBuildInputs = old.nativeBuildInputs ++ [ self."setuptools-scm" ]; 9 | }); 10 | 11 | "mccabe" = super."mccabe".overrideDerivation (old: { 12 | nativeBuildInputs = old.nativeBuildInputs ++ [ self."pytest-runner" ]; 13 | }); 14 | 15 | "clickclick" = super."clickclick".overrideDerivation (old: { 16 | nativeBuildInputs = old.nativeBuildInputs ++ [ self."flake8" self."six" ]; 17 | }); 18 | 19 | "connexion" = super."connexion".overrideDerivation 20 | (old: { nativeBuildInputs = old.nativeBuildInputs ++ [ self."flake8" ]; }); 21 | } 22 | -------------------------------------------------------------------------------- /integrationtests/dependency_graph/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/dependency_graph_input/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/empy/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/fava/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: { 4 | "setuptools-scm-git-archive" = 5 | super."setuptools-scm-git-archive".overrideDerivation 6 | (old: { buildInputs = old.buildInputs ++ [ self."setuptools-scm" ]; }); 7 | } 8 | -------------------------------------------------------------------------------- /integrationtests/flake8-mercurial/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: { 4 | "pytest-runner" = super."pytest-runner".overrideDerivation 5 | (old: { buildInputs = old.buildInputs ++ [ self."setuptools-scm" ]; }); 6 | 7 | "mccabe" = super.mccabe.overrideDerivation 8 | (old: { buildInputs = old.buildInputs ++ [ self."pytest-runner" ]; }); 9 | } 10 | -------------------------------------------------------------------------------- /integrationtests/flake8/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: { 4 | "pytest-runner" = super."pytest-runner".overrideDerivation (old: { 5 | nativeBuildInputs = old.nativeBuildInputs ++ [ self."setuptools-scm" ]; 6 | }); 7 | 8 | "mccabe" = super."mccabe".overrideDerivation (old: { 9 | nativeBuildInputs = old.nativeBuildInputs ++ [ self."pytest-runner" ]; 10 | }); 11 | } 12 | -------------------------------------------------------------------------------- /integrationtests/flake8_include_default_overrides/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: { 4 | "pytest-runner" = super."pytest-runner".overrideDerivation (old: { 5 | nativeBuildInputs = old.nativeBuildInputs ++ [ self."setuptools-scm" ]; 6 | }); 7 | } 8 | -------------------------------------------------------------------------------- /integrationtests/flit/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/ldap/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: { 4 | 5 | "python-ldap" = python.overrideDerivation super."python-ldap" 6 | (old: { NIX_CFLAGS_COMPILE = "-I${pkgs.cyrus_sasl.dev}/include/sasl"; }); 7 | } 8 | -------------------------------------------------------------------------------- /integrationtests/lektor/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: { 4 | "lektor" = python.overrideDerivation super."lektor" (old: { 5 | patchPhase = '' 6 | sed -i -e "s|requests\[security\]|requests|" setup.py 7 | ''; 8 | }); 9 | 10 | "pip" = super."pip".overrideDerivation 11 | (old: { pipInstallFlags = [ "--ignore-installed" ]; }); 12 | } 13 | -------------------------------------------------------------------------------- /integrationtests/local_path/egg/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = local-path 3 | version = 1.0 4 | -------------------------------------------------------------------------------- /integrationtests/local_path/egg/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /integrationtests/local_path/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/pillow/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/pynacl/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/relative_paths/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/relative_paths/test_package/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = test_package 3 | version = "1.0" 4 | -------------------------------------------------------------------------------- /integrationtests/relative_paths/test_package/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /integrationtests/rss2email/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/scipy/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: { 4 | 5 | "scipy" = python.overrideDerivation super."scipy" (old: { 6 | prePatch = '' 7 | rm scipy/linalg/tests/test_lapack.py 8 | ''; 9 | preConfigure = '' 10 | sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py 11 | ''; 12 | preBuild = '' 13 | echo "Creating site.cfg file..." 14 | cat << EOF > site.cfg 15 | [openblas] 16 | include_dirs = ${pkgs.openblasCompat}/include 17 | library_dirs = ${pkgs.openblasCompat}/lib 18 | EOF 19 | ''; 20 | setupPyBuildFlags = [ "--fcompiler='gnu95'" ]; 21 | passthru = { blas = pkgs.openblasCompat; }; 22 | }); 23 | 24 | "numpy" = python.overrideDerivation super."numpy" (old: { 25 | preConfigure = '' 26 | sed -i 's/-faltivec//' numpy/distutils/system_info.py 27 | ''; 28 | preBuild = '' 29 | echo "Creating site.cfg file..." 30 | cat << EOF > site.cfg 31 | [openblas] 32 | include_dirs = ${pkgs.openblasCompat}/include 33 | library_dirs = ${pkgs.openblasCompat}/lib 34 | EOF 35 | ''; 36 | passthru = { blas = pkgs.openblasCompat; }; 37 | }); 38 | 39 | } 40 | -------------------------------------------------------------------------------- /integrationtests/serpy/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/setuptools/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /integrationtests/test_aiohttp.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class AiohttpTestCase(IntegrationTest): 5 | name_of_testcase = "aiohttp" 6 | code_for_testing = ["import aiohttp"] 7 | requirements = ["aiohttp==2.0.6.post1"] 8 | python_version = "python35" 9 | -------------------------------------------------------------------------------- /integrationtests/test_appdirs.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class AppDirsTestCase(IntegrationTest): 5 | """This test checks if we handle quote characters '"' in package descriptions. 6 | 7 | The appdirs package has a description that includes a '"'. This 8 | description gets rendered into the "meta" attribute of the result 9 | nix derivation. We evaluate this attribute to make sure that 10 | everything is escaped fine. 11 | """ 12 | 13 | name_of_testcase = "appdirs" 14 | requirements = ["appdirs==1.4.3"] 15 | additional_paths_to_build = ["packages.appdirs.meta"] 16 | -------------------------------------------------------------------------------- /integrationtests/test_awscli_and_requests.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | from .framework import TestCommand 3 | 4 | 5 | class AwscliAndRequestsTestCase(IntegrationTest): 6 | name_of_testcase = "awscli_and_requests" 7 | requirements = ["awscli", "requests"] 8 | code_for_testing = ["import awscli", "import requests"] 9 | 10 | def executables_for_testing(self): 11 | return [TestCommand(command=["aws", "help"], env={"PAGER": "none"})] 12 | -------------------------------------------------------------------------------- /integrationtests/test_connexion.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class ConnexionTestCase(IntegrationTest): 5 | name_of_testcase = "connexion" 6 | requirements = ["connexion"] 7 | code_for_testing = ["import connexion"] 8 | constraints = ["clickclick == 1.2.1", "flake8 == 3.7.7"] 9 | 10 | def setup_requires(self): 11 | return ["flit", "pytest-runner", "setuptools-scm", "vcversioner", "flake8"] 12 | -------------------------------------------------------------------------------- /integrationtests/test_dependency_graph.py: -------------------------------------------------------------------------------- 1 | from pypi2nix.dependency_graph import DependencyGraph 2 | from pypi2nix.requirement_parser import RequirementParser 3 | 4 | from .framework import IntegrationTest 5 | 6 | 7 | class DependencyGraphOutputTestCase(IntegrationTest): 8 | name_of_testcase = "dependency_graph" 9 | requirements = ["django == 3.0.5"] 10 | 11 | def check_dependency_graph( 12 | self, dependency_graph: DependencyGraph, requirement_parser: RequirementParser 13 | ): 14 | self.assertTrue( 15 | dependency_graph.is_runtime_dependency( 16 | requirement_parser.parse("django"), requirement_parser.parse("pytz"), 17 | ) 18 | ) 19 | 20 | 21 | class DependencyGraphInputTestCase(IntegrationTest): 22 | """This class checks behavior if the user supplies a dependency graph 23 | when running pypi2nix. 24 | 25 | Normally requests should not come with django. In this test case 26 | we tell pypi2nix that requests is a dependecy of django. After 27 | running pypi2nix nix we check if requests was also installed. 28 | """ 29 | 30 | name_of_testcase = "dependency_graph_input" 31 | requirements = ["django == 3.0.5"] 32 | dependency_graph = {"django": {"runtimeDependencies": ["requests"]}} 33 | code_for_testing = ["import requests"] 34 | -------------------------------------------------------------------------------- /integrationtests/test_empy.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class EmpyTestCase(IntegrationTest): 5 | name_of_testcase = "empy" 6 | code_for_testing = ["import em"] 7 | requirements = ["empy"] 8 | -------------------------------------------------------------------------------- /integrationtests/test_fava.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class FavaTestCase(IntegrationTest): 5 | name_of_testcase = "fava" 6 | requirements = ["fava==1.13"] 7 | external_dependencies = ["libxml2", "libxslt"] 8 | constraints = ["jaraco-functools == 2.0"] 9 | -------------------------------------------------------------------------------- /integrationtests/test_flake8.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class Flake8TestCase(IntegrationTest): 5 | name_of_testcase = "flake8" 6 | code_for_testing = ["import flake8"] 7 | requirements = ["flake8 == 3.7.7"] 8 | 9 | def setup_requires(self): 10 | return ["intreehooks", "pytest-runner", "setuptools-scm", "flit"] 11 | -------------------------------------------------------------------------------- /integrationtests/test_flake8_mercurial.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | REVISION = "a209fb6" 4 | 5 | 6 | class Flake8MercurialTestCase(IntegrationTest): 7 | name_of_testcase = "flake8-mercurial" 8 | code_for_testing = ["import flake8"] 9 | requirements = [ 10 | "-e hg+https://bitbucket.org/tarek/flake8@{revision}#egg=flake8".format( 11 | revision=REVISION 12 | ) 13 | ] 14 | 15 | def setup_requires(self): 16 | return ["setuptools-scm", "pytest-runner"] 17 | 18 | def requirements_file_check(self, content): 19 | self.assertIn(REVISION, content) 20 | -------------------------------------------------------------------------------- /integrationtests/test_flit.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class FlitTestCase(IntegrationTest): 5 | name_of_testcase = "flit" 6 | requirements = ["flit"] 7 | -------------------------------------------------------------------------------- /integrationtests/test_ldap.py: -------------------------------------------------------------------------------- 1 | from unittest import expectedFailure 2 | 3 | from .framework import IntegrationTest 4 | 5 | 6 | @expectedFailure 7 | class LdapTestCase(IntegrationTest): 8 | name_of_testcase = "ldap" 9 | python_version = "python27" 10 | code_for_testing = ["import ldap"] 11 | requirements = ["python-ldap"] 12 | external_dependencies = ["openldap", "cyrus_sasl", "openssl"] 13 | 14 | def extra_environment(self): 15 | return { 16 | "NIX_CFLAGS_COMPILE": '"-I${pkgs.cyrus_sasl.dev}/include/sasl $NIX_CFLAGS_COMPILE"' 17 | } 18 | -------------------------------------------------------------------------------- /integrationtests/test_lektor.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | from .framework import TestCommand 3 | 4 | 5 | class LektorTestCase(IntegrationTest): 6 | name_of_testcase = "lektor" 7 | code_for_testing = ["import lektor"] 8 | requirements = ["Lektor"] 9 | external_dependencies = ["libffi", "openssl", "unzip"] 10 | 11 | def executables_for_testing(self): 12 | return [TestCommand(command=["lektor", "--help"])] 13 | -------------------------------------------------------------------------------- /integrationtests/test_local_path.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class LocalPathTestCase(IntegrationTest): 5 | name_of_testcase = "local_path" 6 | requirements = ["-e egg#egg=local_path"] 7 | -------------------------------------------------------------------------------- /integrationtests/test_pillow.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class PillowTestCase(IntegrationTest): 5 | name_of_testcase = "pillow" 6 | code_for_testing = ["import PIL"] 7 | requirements = ["Pillow==7.1.2"] 8 | 9 | external_dependencies = [ 10 | "pkgconfig", 11 | "zlib", 12 | "libjpeg", 13 | "openjpeg", 14 | "libtiff", 15 | "freetype", 16 | "lcms2", 17 | "libwebp", 18 | "tcl", 19 | "xorg.libxcb", 20 | ] 21 | -------------------------------------------------------------------------------- /integrationtests/test_pynacl.py: -------------------------------------------------------------------------------- 1 | from unittest import expectedFailure 2 | 3 | from .framework import IntegrationTest 4 | 5 | 6 | @expectedFailure 7 | class PynaclTestCase(IntegrationTest): 8 | name_of_testcase = "pynacl" 9 | requirements = ["pynacl"] 10 | external_dependencies = ["libffi"] 11 | explicit_build_directory = True 12 | -------------------------------------------------------------------------------- /integrationtests/test_relative_paths.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class RelativePathsTestCase(IntegrationTest): 5 | name_of_testcase = "relative_paths" 6 | requirements = ["test_package/.#egg=test_package"] 7 | 8 | def requirements_file_check(self, content): 9 | self.assertIn("src = test_package/.", content) 10 | -------------------------------------------------------------------------------- /integrationtests/test_rss2email.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | from .framework import TestCommand 3 | 4 | 5 | class Rss2EmailTestCase(IntegrationTest): 6 | name_of_testcase = "rss2email" 7 | code_for_testing = ["import rss2email"] 8 | requirements = [ 9 | "https://github.com/wking/rss2email/archive/master.zip#egg=rss2email" 10 | ] 11 | 12 | def executables_for_testing(self): 13 | return [TestCommand(command=["r2e", "--help"])] 14 | -------------------------------------------------------------------------------- /integrationtests/test_scipy.py: -------------------------------------------------------------------------------- 1 | from unittest import expectedFailure 2 | 3 | from .framework import IntegrationTest 4 | 5 | 6 | @expectedFailure 7 | class ScipyTestCase(IntegrationTest): 8 | name_of_testcase = "scipy" 9 | code_for_testing = ["import scipy"] 10 | requirements = ["scipy", "numpy"] 11 | external_dependencies = ["gfortran", "blas"] 12 | 13 | def setup_requires(self): 14 | return ["numpy"] 15 | -------------------------------------------------------------------------------- /integrationtests/test_serpy_0_1_1.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class SerpyTestCase(IntegrationTest): 5 | name_of_testcase = "serpy" 6 | requirements = ["serpy==0.1.1"] 7 | 8 | def setup_requires(self): 9 | return ["six==1.12.0"] 10 | -------------------------------------------------------------------------------- /integrationtests/test_setuptools.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | 4 | class SetuptoolsTestCase(IntegrationTest): 5 | name_of_testcase = "setuptools" 6 | code_to_test = ["import setuptools"] 7 | requirements = ["setuptools"] 8 | 9 | def requirements_file_check(self, content): 10 | self.assertIn('"setuptools" =', content) 11 | -------------------------------------------------------------------------------- /integrationtests/test_tornado.py: -------------------------------------------------------------------------------- 1 | from .framework import IntegrationTest 2 | 3 | REVISION = "69253c820df473407c562a227d0ba36df25018ab" 4 | 5 | 6 | class TornadoTestCase(IntegrationTest): 7 | name_of_testcase = "tornado" 8 | code_for_testing = ["import tornado"] 9 | requirements = [ 10 | "-e git+git://github.com/tornadoweb/tornado.git@69253c820df473407c562a227d0ba36df25018ab#egg=tornado" 11 | ] 12 | 13 | def requirements_file_check(self, content): 14 | self.assertIn(REVISION, content) 15 | -------------------------------------------------------------------------------- /integrationtests/tornado/requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /mypy/jsonschema.pyi: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | def validate(instance: Any, schema: Any) -> None: ... 4 | 5 | class ValidationError(Exception): ... 6 | -------------------------------------------------------------------------------- /mypy/nix_prefetch_github/__init__.pyi: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | from typing import Optional 3 | 4 | def nix_prefetch_github( 5 | owner: str, repo: str, prefetch: bool = ..., rev: Optional[str] = ... 6 | ) -> Dict[str, str]: ... 7 | -------------------------------------------------------------------------------- /mypy/packaging/__init__.pyi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/mypy/packaging/__init__.pyi -------------------------------------------------------------------------------- /mypy/packaging/markers.pyi: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | from typing import Optional 3 | 4 | def default_environment() -> Dict[str, str]: ... 5 | 6 | class Marker: 7 | def __init__(self, marker: str) -> None: ... 8 | def evaluate(self, environment: Optional[Dict[str, str]] = ...) -> bool: ... 9 | 10 | class InvalidMarker(ValueError): ... 11 | class UndefinedComparison(ValueError): ... 12 | class UndefinedEnvironmentName(ValueError): ... 13 | -------------------------------------------------------------------------------- /mypy/packaging/utils.pyi: -------------------------------------------------------------------------------- 1 | def canonicalize_name(name: str) -> str: ... 2 | -------------------------------------------------------------------------------- /mypy/packaging/version.pyi: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | class InvalidVersion(ValueError): ... 4 | 5 | class Version: 6 | def __init__(self, version: str) -> None: ... 7 | 8 | class LegacyVersion: ... 9 | 10 | def parse(version: str) -> Union[Version, LegacyVersion]: ... 11 | -------------------------------------------------------------------------------- /mypy/parsley.pyi: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from typing import Callable 3 | from typing import Dict 4 | from typing import overload 5 | 6 | class ParseError(Exception): 7 | def formatError(self) -> str: ... 8 | 9 | @overload 10 | def makeGrammar(source: str, bindings: Dict[str, Callable[..., Any]]) -> Any: ... 11 | @overload 12 | def makeGrammar( 13 | source: str, bindings: Dict[str, Callable[..., Any]], name: str 14 | ) -> Any: ... 15 | @overload 16 | def makeGrammar( 17 | source: str, bindings: Dict[str, Callable[..., Any]], unwrap: bool 18 | ) -> Any: ... 19 | @overload 20 | def makeGrammar( 21 | source: str, bindings: Dict[str, Callable[..., Any]], name: str, unwrap: bool 22 | ) -> Any: ... 23 | -------------------------------------------------------------------------------- /mypy/setuptools/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | 4 | def setup(*args, **kwargs) -> Any: # type: ignore 5 | ... 6 | -------------------------------------------------------------------------------- /mypy/setuptools/config.pyi: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from typing import Dict 3 | 4 | def read_configuration( 5 | filepath: str, find_others: bool = ..., ignore_option_errors: bool = ... 6 | ) -> Dict[str, Any]: ... 7 | -------------------------------------------------------------------------------- /mypy/setuptools_scm.pyi: -------------------------------------------------------------------------------- 1 | def get_version() -> str: ... 2 | -------------------------------------------------------------------------------- /mypy/venv.pyi: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | class EnvBuilder: 4 | def __init__( 5 | self, 6 | system_site_packages: bool = ..., 7 | clear: bool = ..., 8 | symlinks: bool = ..., 9 | upgrade: bool = ..., 10 | with_pip: bool = ..., 11 | prompt: Optional[str] = None, 12 | ) -> None: ... 13 | def create(self, env_dir: str) -> None: ... 14 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | nix: marks tests that that cannot run inside a nix build sandbox -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | # linting 2 | black 3 | flake8 4 | flake8-debugger 5 | flake8-unused-arguments 6 | mypy 7 | isort 8 | 9 | # testing 10 | pytest 11 | pytest-cov 12 | 13 | # develop 14 | pdbpp 15 | 16 | # packaging 17 | twine 18 | bumpv 19 | 20 | # documentation 21 | sphinx 22 | 23 | -c constraints.txt 24 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | attrs 2 | click 3 | jinja2 4 | nix-prefetch-github 5 | parsley 6 | toml 7 | packaging 8 | jsonschema 9 | hypothesis 10 | pyyaml 11 | 12 | -c constraints.txt 13 | -------------------------------------------------------------------------------- /requirements_frozen.txt: -------------------------------------------------------------------------------- 1 | alabaster==0.7.12 2 | appdirs==1.4.3 3 | attrs==19.3.0 4 | Babel==2.8.0 5 | black==19.10b0 6 | bleach==3.1.1 7 | bumpv==0.3.0 8 | certifi==2019.11.28 9 | cffi==1.14.0 10 | chardet==3.0.4 11 | Click==7.0 12 | coverage==5.0.3 13 | cryptography==2.8 14 | docutils==0.16 15 | effect==1.1.0 16 | entrypoints==0.3 17 | fancycompleter==0.9.1 18 | flake8==3.7.9 19 | flake8-debugger==3.2.1 20 | flake8-unused-arguments==0.0.3 21 | flit==2.2.0 22 | flit-core==2.2.0 23 | hypothesis==5.6.0 24 | idna==2.9 25 | imagesize==1.2.0 26 | importlib-metadata==1.5.0 27 | intreehooks==1.0 28 | isort==4.3.21 29 | jeepney==0.4.3 30 | Jinja2==2.11.1 31 | jsonschema==3.2.0 32 | keyring==21.1.1 33 | MarkupSafe==1.1.1 34 | mccabe==0.6.1 35 | more-itertools==8.2.0 36 | mypy==0.761 37 | mypy-extensions==0.4.3 38 | nix-prefetch-github==2.3.2 39 | packaging==20.3 40 | Parsley==1.3 41 | pathspec==0.7.0 42 | pdbpp==0.10.2 43 | pkginfo==1.5.0.1 44 | pluggy==0.13.1 45 | py==1.8.1 46 | pyaml==19.4.1 47 | pycodestyle==2.5.0 48 | pycparser==2.20 49 | pyflakes==2.1.1 50 | Pygments==2.6.1 51 | pyparsing==2.4.6 52 | pyrepl==0.9.0 53 | pyrsistent==0.15.7 54 | pytest==5.3.5 55 | pytest-cov==2.8.1 56 | pytest-runner==5.2 57 | pytoml==0.1.21 58 | pytz==2019.3 59 | PyYAML==5.3 60 | readme-renderer==24.0 61 | regex==2020.2.20 62 | requests==2.23.0 63 | requests-toolbelt==0.9.1 64 | SecretStorage==3.1.2 65 | setupmeta==2.6.20 66 | setuptools-scm==3.5.0 67 | six==1.14.0 68 | snowballstemmer==2.0.0 69 | sortedcontainers==2.1.0 70 | Sphinx==2.4.4 71 | sphinxcontrib-applehelp==1.0.2 72 | sphinxcontrib-devhelp==1.0.2 73 | sphinxcontrib-htmlhelp==1.0.3 74 | sphinxcontrib-jsmath==1.0.1 75 | sphinxcontrib-qthelp==1.0.3 76 | sphinxcontrib-serializinghtml==1.1.4 77 | toml==0.10.0 78 | tqdm==4.43.0 79 | twine==3.1.1 80 | typed-ast==1.4.1 81 | typing-extensions==3.7.4.1 82 | urllib3==1.25.8 83 | wcwidth==0.1.8 84 | webencodings==0.5.1 85 | wmctrl==0.3 86 | zipp==3.1.0 87 | -------------------------------------------------------------------------------- /requirements_override.nix: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: 4 | let 5 | addBuildInputs = packages: old: { 6 | buildInputs = old.buildInputs ++ packages; 7 | }; 8 | pipInstallIgnoresInstalled = old: { 9 | pipInstallFlags = ["--ignore-installed"]; 10 | }; 11 | addSingleBuildInput = package: addBuildInputs [package]; 12 | overridePythonPackage = name: overrides: 13 | let 14 | combinedOverrides = old: pkgs.lib.fold 15 | (override: previous: previous // override previous) 16 | old 17 | overrides; 18 | in python.overrideDerivation super."${name}" combinedOverrides; 19 | in { 20 | "fancycompleter" = overridePythonPackage "fancycompleter" 21 | [ 22 | (addBuildInputs [self."setuptools-scm" self."setupmeta"]) 23 | ]; 24 | 25 | "flake8-debugger" = overridePythonPackage "flake8-debugger" 26 | [ 27 | (addBuildInputs [self."pytest-runner"]) 28 | ]; 29 | 30 | "jsonschema" = overridePythonPackage "jsonschema" 31 | [ 32 | (addBuildInputs [self."setuptools-scm"]) 33 | ]; 34 | 35 | "keyring" = overridePythonPackage "keyring" 36 | [ 37 | (addBuildInputs [self."toml"]) 38 | ]; 39 | 40 | "mccabe" = overridePythonPackage "mccabe" 41 | [ 42 | (addBuildInputs [self."pytest-runner"]) 43 | ]; 44 | 45 | "pdbpp" = overridePythonPackage "pdbpp" 46 | [ 47 | (addBuildInputs [self."setuptools-scm"]) 48 | ]; 49 | 50 | "py" = overridePythonPackage "py" 51 | [ 52 | (addBuildInputs [self."setuptools-scm"]) 53 | ]; 54 | 55 | "setuptools" = overridePythonPackage "setuptools" 56 | [ 57 | pipInstallIgnoresInstalled 58 | ]; 59 | 60 | "wheel" = overridePythonPackage "wheel" 61 | [ 62 | pipInstallIgnoresInstalled 63 | ]; 64 | 65 | "zipp" = overridePythonPackage "zipp" 66 | [ 67 | (addBuildInputs [self."toml"]) 68 | ]; 69 | } 70 | -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/scripts/__init__.py -------------------------------------------------------------------------------- /scripts/build-pip.nix: -------------------------------------------------------------------------------- 1 | with import {}; 2 | with python3Packages; 3 | 4 | stdenv.mkDerivation { 5 | name = "impure-environment"; 6 | buildInputs = [ 7 | python3Packages.pip 8 | python3Packages.wheel 9 | python3Packages.setuptools 10 | ]; 11 | shellHook = '' 12 | # set SOURCE_DATE_EPOCH so that we can use python wheels 13 | SOURCE_DATE_EPOCH=$(date +%s) 14 | ''; 15 | } 16 | -------------------------------------------------------------------------------- /scripts/build_wheel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import shlex 5 | import shutil 6 | import sys 7 | import tempfile 8 | 9 | from pypi2nix.logger import StreamLogger 10 | from pypi2nix.nix import Nix 11 | from pypi2nix.requirement_parser import ParsingFailed 12 | from pypi2nix.requirement_parser import RequirementParser 13 | from repository import ROOT 14 | 15 | HERE = os.path.abspath(os.path.dirname(__file__)) 16 | DERIVATION_PATH = os.path.join(HERE, "build-pip.nix") 17 | 18 | 19 | def build_wheel(target_directory: str, requirement: str) -> str: 20 | logger = StreamLogger(sys.stdout) 21 | requirement_parser = RequirementParser(logger=logger) 22 | package_directory = os.path.join(ROOT, "unittests", "data") 23 | escaped_requirement = shlex.quote(requirement) 24 | target_directory = os.path.abspath(target_directory) 25 | with tempfile.TemporaryDirectory() as build_directory: 26 | os.chdir(build_directory) 27 | nix = Nix(logger=logger) 28 | nix.shell( 29 | command=f"pip wheel {escaped_requirement} --find-links {package_directory} --no-deps", 30 | derivation_path=DERIVATION_PATH, 31 | nix_arguments=dict(), 32 | ) 33 | try: 34 | parsed_requirement = requirement_parser.parse(requirement) 35 | except ParsingFailed: 36 | for path in os.listdir("."): 37 | if path.endswith(".whl"): 38 | wheel_path = path 39 | break 40 | else: 41 | raise Exception("Build process did not produce .whl file") 42 | else: 43 | for path in os.listdir("."): 44 | if path.endswith(".whl") and parsed_requirement.name() in path: 45 | wheel_path = path 46 | break 47 | else: 48 | raise Exception("Build process did not produce .whl file") 49 | 50 | target_file_name = os.path.basename(wheel_path) 51 | target_path = os.path.join(target_directory, target_file_name) 52 | shutil.move(wheel_path, target_path) 53 | return target_file_name 54 | -------------------------------------------------------------------------------- /scripts/deploy_to_pypi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import os 4 | import shutil 5 | import subprocess 6 | 7 | from pypi2nix.version import pypi2nix_version 8 | 9 | 10 | def main(): 11 | set_up_environment() 12 | args = parse_args() 13 | pypi_name = get_pypi_name_from_args(args) 14 | remove_old_build_artifacts() 15 | deploy_to(pypi_name) 16 | 17 | 18 | def set_up_environment(): 19 | os.putenv("SOURCE_DATE_EPOCH", "315532800") 20 | 21 | 22 | def parse_args(): 23 | parser = argparse.ArgumentParser(description="Deploy pypi2nix to pypi") 24 | parser.add_argument("--production", action="store_true", default=False) 25 | return parser.parse_args() 26 | 27 | 28 | def get_pypi_name_from_args(args): 29 | return "pypi" if args.production else "test-pypi" 30 | 31 | 32 | def remove_old_build_artifacts(): 33 | shutil.rmtree("src/pypi2nix.egg-info", ignore_errors=True) 34 | 35 | 36 | def deploy_to(pypi_name): 37 | subprocess.run(["python", "setup.py", "sdist", "bdist_wheel"], check=True) 38 | distribution_paths = [ 39 | f"dist/pypi2nix-{pypi2nix_version}.tar.gz", 40 | f"dist/pypi2nix-{pypi2nix_version}-py3-none-any.whl", 41 | ] 42 | subprocess.run( 43 | ["twine", "upload", "-r", pypi_name] + distribution_paths, check=True 44 | ) 45 | 46 | 47 | if __name__ == "__main__": 48 | main() 49 | -------------------------------------------------------------------------------- /scripts/format_sources.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import os.path 5 | import subprocess 6 | import sys 7 | from typing import List 8 | 9 | from pypi2nix.logger import Logger 10 | from pypi2nix.logger import StreamLogger 11 | from repository import ROOT 12 | 13 | 14 | class CodeFormatter: 15 | def __init__(self): 16 | self._logger = initialize_logger() 17 | 18 | def main(self): 19 | relative_paths = [ 20 | "src", 21 | "unittests", 22 | "integrationtests", 23 | "conftest.py", 24 | "setup.py", 25 | "mypy", 26 | "scripts", 27 | ] 28 | self.format_nix_files() 29 | absolute_paths = [os.path.join(ROOT, relative) for relative in relative_paths] 30 | self._logger.info("Running isort") 31 | subprocess.run(["isort", "-rc", "."], check=True) 32 | self._logger.info("Running black") 33 | subprocess.run(["black"] + absolute_paths, check=True) 34 | self.run_check_process("flake8") 35 | self.run_check_process("mypy") 36 | 37 | def run_check_process(self, executable, arguments: List[str] = []): 38 | self._logger.info(f"Running {executable}") 39 | try: 40 | subprocess.run([executable] + arguments, check=True) 41 | except subprocess.CalledProcessError: 42 | self._logger.error(f"{executable} failed, see errors above") 43 | exit(1) 44 | 45 | def format_nix_files(self) -> None: 46 | if is_nixfmt_installed(): 47 | self._logger.info("Formatting nix files") 48 | integration_test_nix_files = find_nix_files_in_integration_tests() 49 | subprocess.run( 50 | ["nixfmt", "default.nix", "src/pypi2nix/pip/bootstrap.nix"] 51 | + integration_test_nix_files, 52 | check=True, 53 | ) 54 | else: 55 | self._logger.warning( 56 | "Could not find `nixfmt` executable. Cannot format .nix files" 57 | ) 58 | 59 | 60 | def find_nix_files_in_integration_tests() -> List[str]: 61 | found_files: List[str] = [] 62 | for root, _, files in os.walk("integrationtests"): 63 | found_files += [ 64 | os.path.join(root, file) for file in files if file.endswith(".nix") 65 | ] 66 | return found_files 67 | 68 | 69 | def initialize_logger() -> Logger: 70 | return StreamLogger(output=sys.stdout) 71 | 72 | 73 | def is_nixfmt_installed() -> bool: 74 | process_result = subprocess.run("nixfmt --version", shell=True, capture_output=True) 75 | return process_result.returncode == 0 76 | 77 | 78 | if __name__ == "__main__": 79 | CodeFormatter().main() 80 | -------------------------------------------------------------------------------- /scripts/install_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import os.path 4 | import shutil 5 | import subprocess 6 | 7 | from pypi2nix.version import pypi2nix_version 8 | 9 | 10 | def main(): 11 | set_up_environment() 12 | create_virtual_env() 13 | create_sdist() 14 | install_sdist() 15 | run_help_command() 16 | create_virtual_env() 17 | create_wheel() 18 | install_wheel() 19 | run_help_command() 20 | 21 | 22 | def set_up_environment(): 23 | os.putenv("SOURCE_DATE_EPOCH", "315532800") 24 | os.unsetenv("PYTHONPATH") 25 | 26 | 27 | def create_sdist(): 28 | shutil.rmtree(os.path.join("src", "pypi2nix.egg-info"), ignore_errors=True) 29 | subprocess.run(["build/venv/bin/python", "setup.py", "sdist"], check=True) 30 | 31 | 32 | def create_virtual_env(): 33 | os.makedirs("build", exist_ok=True) 34 | try: 35 | shutil.rmtree("build/venv") 36 | except FileNotFoundError: 37 | pass 38 | subprocess.run(["python", "-m", "venv", "build/venv"], check=True) 39 | 40 | 41 | def create_wheel(): 42 | shutil.rmtree(os.path.join("src", "pypi2nix.egg-info"), ignore_errors=True) 43 | subprocess.run( 44 | ["build/venv/bin/python", "-m", "pip", "install", "wheel"], check=True 45 | ) 46 | subprocess.run(["build/venv/bin/python", "setup.py", "bdist_wheel"], check=True) 47 | 48 | 49 | def install_sdist(): 50 | subprocess.run( 51 | [ 52 | "build/venv/bin/python", 53 | "-m", 54 | "pip", 55 | "install", 56 | f"dist/pypi2nix-{pypi2nix_version}.tar.gz", 57 | ], 58 | check=True, 59 | ) 60 | 61 | 62 | def install_wheel(): 63 | subprocess.run( 64 | [ 65 | "build/venv/bin/python", 66 | "-m", 67 | "pip", 68 | "install", 69 | f"dist/pypi2nix-{pypi2nix_version}-py3-none-any.whl", 70 | ], 71 | check=True, 72 | ) 73 | 74 | 75 | def run_help_command(): 76 | subprocess.run(["build/venv/bin/pypi2nix", "--help"], check=True) 77 | 78 | 79 | if __name__ == "__main__": 80 | main() 81 | -------------------------------------------------------------------------------- /scripts/package_source.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from pypi2nix.logger import Logger 4 | from pypi2nix.pypi import Pypi 5 | from pypi2nix.utils import prefetch_git 6 | from pypi2nix.wheels import Index 7 | 8 | 9 | class PackageSource: 10 | def __init__(self, index: Index, pypi: Pypi, logger: Logger): 11 | self.index = index 12 | self.pypi = pypi 13 | self.logger = logger 14 | 15 | def update_package_from_master(self, package_name: str) -> None: 16 | url = self._get_url_for_package(package_name) 17 | if url is None: 18 | self._log_no_update_warning(package_name) 19 | return 20 | repo_data = prefetch_git(url) 21 | self.index[package_name] = Index.GitEntry( 22 | url=repo_data["url"], rev=repo_data["rev"], sha256=repo_data["sha256"], 23 | ) 24 | self._log_update_success(package_name) 25 | 26 | def update_package_from_pip(self, package_name: str) -> None: 27 | package = self.pypi.get_package(package_name) 28 | source_release = self.pypi.get_source_release( 29 | name=package_name, version=package.version 30 | ) 31 | if source_release is None: 32 | self._log_no_update_warning(package_name) 33 | return 34 | self.index[package_name] = Index.UrlEntry( 35 | url=source_release.url, sha256=source_release.sha256_digest 36 | ) 37 | self._log_update_success(package_name) 38 | 39 | def _get_url_for_package(self, package_name: str) -> Optional[str]: 40 | return SOURCE_BY_PACKAGE_NAME.get(package_name) 41 | 42 | def _log_no_update_warning(self, package_name: str) -> None: 43 | self.logger.warning(f"Could not update source for package `{package_name}`") 44 | 45 | def _log_update_success(self, package_name: str) -> None: 46 | self.logger.info(f"Successfully updated package `{package_name}`") 47 | 48 | 49 | SOURCE_BY_PACKAGE_NAME = { 50 | "pip": "https://github.com/pypa/pip.git", 51 | "setuptools": "https://github.com/pypa/setuptools.git", 52 | "wheel": "https://github.com/pypa/wheel.git", 53 | } 54 | -------------------------------------------------------------------------------- /scripts/prepare_test_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | "This script prepares the test fixtures for the unittests of this package" 3 | 4 | import os 5 | import os.path 6 | import shutil 7 | import subprocess 8 | 9 | from build_wheel import build_wheel 10 | from repository import ROOT 11 | 12 | wheel_target_directory = os.path.join(ROOT, "unittests", "data") 13 | TEST_PACKAGES = ["setupcfg-package", "package1", "package2", "package3", "package4"] 14 | 15 | 16 | def build_test_package(package_name): 17 | package_name_with_underscores = package_name.replace("-", "_") 18 | package_dir = os.path.join(ROOT, "unittests", "data", package_name) 19 | paths_to_delete = [ 20 | f"{package_name_with_underscores}.egg-info", 21 | "dist", 22 | f"{package_name}.tar.gz", 23 | ] 24 | for path in paths_to_delete: 25 | shutil.rmtree(os.path.join(package_dir, "path"), ignore_errors=True) 26 | subprocess.run(["python", "setup.py", "sdist"], cwd=package_dir, check=True) 27 | shutil.copy( 28 | os.path.join(package_dir, "dist", f"{package_name}-1.0.tar.gz"), 29 | wheel_target_directory, 30 | ) 31 | shutil.move( 32 | os.path.join(package_dir, "dist", f"{package_name}-1.0.tar.gz"), 33 | os.path.join(package_dir, f"{package_name}.tar.gz"), 34 | ) 35 | build_wheel(wheel_target_directory, package_dir) 36 | 37 | 38 | def download_flit_wheel(): 39 | build_wheel(wheel_target_directory, "flit==1.3") 40 | 41 | 42 | if __name__ == "__main__": 43 | for test_package in TEST_PACKAGES: 44 | build_test_package(test_package) 45 | download_flit_wheel() 46 | -------------------------------------------------------------------------------- /scripts/repository.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | 4 | def find_root(start: Path = Path(".")) -> Path: 5 | absolute_location = start.resolve() 6 | if (absolute_location / ".git").is_dir(): 7 | return absolute_location 8 | else: 9 | return find_root(absolute_location / "..") 10 | 11 | 12 | ROOT = find_root() 13 | -------------------------------------------------------------------------------- /scripts/run_integration_tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import os 5 | import shlex 6 | import subprocess 7 | 8 | from repository import ROOT 9 | 10 | 11 | def generator(iterable): 12 | yield from iterable 13 | 14 | 15 | def parse_args(): 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument("--file", default=None) 18 | args = parser.parse_args() 19 | return args.file 20 | 21 | 22 | def run_tests_from_file(path: str) -> None: 23 | command = ["python", "-m", "unittest", path, "-k", "TestCase"] 24 | print("Executing test: ", " ".join(map(shlex.quote, command))) 25 | subprocess.run(command, check=True) 26 | 27 | 28 | def main(): 29 | file = parse_args() 30 | if file: 31 | files = generator([file]) 32 | else: 33 | files = ( 34 | os.path.join(ROOT, "integrationtests", name) 35 | for name in os.listdir(os.path.join(ROOT, "integrationtests")) 36 | if name.startswith("test_") and name.endswith(".py") 37 | ) 38 | for path in files: 39 | run_tests_from_file(path) 40 | 41 | 42 | if __name__ == "__main__": 43 | main() 44 | -------------------------------------------------------------------------------- /scripts/update_dependencies.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import subprocess 5 | 6 | from repository import ROOT 7 | 8 | 9 | def main(): 10 | arguments = parse_arguments() 11 | subprocess.run( 12 | [ 13 | "pypi2nix", 14 | "-r", 15 | "requirements.txt", 16 | "-r", 17 | "requirements-dev.txt", 18 | "-s", 19 | "pytest-runner", 20 | "-s", 21 | "setupmeta", 22 | "--no-default-overrides", 23 | "-E", 24 | "openssl libffi", 25 | ] 26 | + (["-v"] if arguments.verbose else []), 27 | cwd=ROOT, 28 | check=True, 29 | ) 30 | 31 | 32 | def parse_arguments(): 33 | argument_parser = argparse.ArgumentParser( 34 | description="Update development dependencies of pypi2nix" 35 | ) 36 | argument_parser.add_argument( 37 | "--verbose", 38 | "-v", 39 | help="Print debugging output", 40 | default=False, 41 | action="store_true", 42 | ) 43 | args = argument_parser.parse_args() 44 | return args 45 | 46 | 47 | if __name__ == "__main__": 48 | main() 49 | -------------------------------------------------------------------------------- /scripts/update_python_packages.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | from typing import List 5 | 6 | from package_source import PackageSource 7 | from pypi2nix.logger import StreamLogger 8 | from pypi2nix.pypi import Pypi 9 | from pypi2nix.wheels import Index 10 | 11 | 12 | def main(): 13 | logger = StreamLogger(sys.stdout) 14 | pypi = Pypi(logger=logger) 15 | pip_requirements: List[str] = ["setuptools", "wheel", "pip"] 16 | git_requirements: List[str] = [] 17 | index = Index(logger=logger) 18 | package_source = PackageSource(index=index, pypi=pypi, logger=logger) 19 | for requirement in pip_requirements: 20 | package_source.update_package_from_pip(requirement) 21 | for requirement in git_requirements: 22 | package_source.update_package_from_master(requirement) 23 | 24 | 25 | if __name__ == "__main__": 26 | main() 27 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = pypi2nix 3 | version = file: src/pypi2nix/VERSION 4 | author = Rok Garbas, Cillian de Róiste, Jaka Hudoklin 5 | description = A tool that generates nix expressions for your python packages, so you do not have to. 6 | long_description = file: README.rst 7 | license = BSD 8 | keywords = nixos, nix, packaging 9 | url = https://github.com/nix-community/pypi2nix 10 | 11 | [options] 12 | classifiers = 13 | Intended Audience :: Developers 14 | License :: OSI Approved :: BSD License 15 | packages = 16 | pypi2nix 17 | pypi2nix.pip 18 | pypi2nix.wheels 19 | pypi2nix.package 20 | pypi2nix.external_dependencies 21 | pypi2nix.external_dependency_collector 22 | package_dir = 23 | = src 24 | pypi2nix = src/pypi2nix 25 | pypi2nix.pip = src/pypi2nix/pip 26 | pypi2nix.wheels = src/pypi2nix/wheels 27 | pypi2nix.package = src/pypi2nix/package 28 | pypi2nix.external_dependencies = src/pypi2nix/external_dependencies 29 | pypi2nix.external_dependency_collector = src/pypi2nix/external_dependency_collector 30 | install_requires = 31 | attrs 32 | click 33 | jinja2 34 | jsonschema 35 | nix-prefetch-github 36 | packaging 37 | Parsley 38 | pyyaml 39 | setuptools 40 | setuptools-scm 41 | toml 42 | include_package_data = true 43 | 44 | [options.entry_points] 45 | console_scripts = 46 | pypi2nix = pypi2nix.cli:main 47 | 48 | [flake8] 49 | ignore = E203, E266, E501, W503, F403, E231 50 | max-line-length = 79 51 | max-complexity = 8 52 | select = B,C,E,F,W,T4,B9 53 | enabled_extensions = U10,T100,C90 54 | per-file-ignores = 55 | **/__init__.py:F401 56 | exclude = 57 | build/** 58 | 59 | [isort] 60 | line_length = 159 61 | force_single_line = True 62 | default_section=FIRSTPARTY 63 | sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER 64 | known_first_party = 65 | pypi2nix 66 | known_third_party = 67 | setuptools 68 | skip_glob = 69 | /integrationtests/pypi2nix/** 70 | /unittests/templates/** 71 | result/** 72 | filter_files = True 73 | 74 | [mypy] 75 | show_column_numbers=True 76 | show_error_context=True 77 | ignore_missing_imports=False 78 | allow_untyped_calls=True 79 | warn_return_any=True 80 | strict_optional=True 81 | warn_no_return=True 82 | warn_redundant_casts=True 83 | warn_unused_ignores=True 84 | allow_untyped_defs=False 85 | check_untyped_defs=True 86 | mypy_path = mypy:src:unittests:integrationtests:scripts 87 | no_implicit_optional=True 88 | files=src,unittests,integrationtests,scripts,mypy,setup.py 89 | scripts_are_modules=True 90 | 91 | [mypy-integrationtests.*,scripts.*,unittests.*,conftest] 92 | allow_untyped_defs=True 93 | 94 | [mypy-pytest,hypothesis.*] 95 | ignore_missing_imports=True 96 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from setuptools import setup 4 | 5 | setup() 6 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | let 2 | nixpkgs = import {}; 3 | pythonPackages = import ./requirements.nix { pkgs = nixpkgs; }; 4 | in 5 | nixpkgs.mkShell { 6 | buildInputs = with nixpkgs; [ 7 | pythonPackages.interpreter 8 | nixfmt 9 | git 10 | nix-prefetch-hg 11 | ]; 12 | shellHook = '' 13 | export PATH=${./scripts}:$PATH 14 | export PYTHONPATH=${./src}:$PYTHONPATH 15 | ''; 16 | } 17 | -------------------------------------------------------------------------------- /source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | import os 18 | import sys 19 | 20 | sys.path.insert(0, os.path.abspath( 21 | os.path.join( 22 | '..', 23 | 'src', 24 | ) 25 | )) 26 | 27 | # -- Project information ----------------------------------------------------- 28 | 29 | project = 'pypi2nix' 30 | copyright = '2020, Rok Garbas, Cillian de Róiste, Jaka Hudoklin' 31 | author = 'Rok Garbas, Cillian de Róiste, Jaka Hudoklin' 32 | 33 | 34 | # -- General configuration --------------------------------------------------- 35 | 36 | master_doc = 'index' 37 | 38 | # Add any Sphinx extension module names here, as strings. They can be 39 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 40 | # ones. 41 | extensions = [ 42 | 'sphinx.ext.autodoc' 43 | ] 44 | 45 | # Add any paths that contain templates here, relative to this directory. 46 | templates_path = ['_templates'] 47 | 48 | # List of patterns, relative to source directory, that match files and 49 | # directories to ignore when looking for source files. 50 | # This pattern also affects html_static_path and html_extra_path. 51 | exclude_patterns = [] 52 | 53 | 54 | # -- Options for HTML output ------------------------------------------------- 55 | 56 | # The theme to use for HTML and HTML Help pages. See the documentation for 57 | # a list of builtin themes. 58 | # 59 | html_theme = 'alabaster' 60 | 61 | # Add any paths that contain custom static files (such as style sheets) here, 62 | # relative to this directory. They are copied after the builtin static files, 63 | # so a file named "default.css" will overwrite the builtin "default.css". 64 | html_static_path = ['_static'] 65 | html_sidebars = { 66 | '**': [ 67 | 'about.html', 68 | 'navigation.html', 69 | 'relations.html', 70 | 'searchbox.html', 71 | 'donate.html', 72 | ] 73 | } 74 | -------------------------------------------------------------------------------- /source/development.rst: -------------------------------------------------------------------------------- 1 | Help developing pypi2nix 2 | ------------------------ 3 | 4 | Clone `pypi2nix repository`_ and using the ``nix-shell`` command enter 5 | development environment.:: 6 | 7 | % git clone https://github.com/nix-community/pypi2nix 8 | % cd pypi2nix 9 | % nix-shell 10 | 11 | Code is located in ``src/pypi2nix``. 12 | 13 | Testing 14 | ^^^^^^^ 15 | 16 | Pypi2nix comes with two kinds of tests: unit tests and integration 17 | tests. They can be found in the folders ``/unittests`` and 18 | ``/integrationtests`` respectively. 19 | 20 | Unit tests are straight forward. They are run via `pytest`_ and (try 21 | to) follow `pytest`_ best practices. Idealy all of pypi2nix's code 22 | should be covered by unittests. If possible unittests should not go 23 | online and fetch data from the internet. If this cannot be avoided 24 | use the ``@nix`` decorator, found in ``unittests.switches`` to mark 25 | tests that require network access. 26 | 27 | Integration tests 28 | """"""""""""""""" 29 | 30 | Integration tests are a little bit more involved. We implemented a 31 | small framework to write new tests and maintain old ones. Check out 32 | ``integrationtests.framework`` for information on how to write custom 33 | integration tests. To run all integration tests run 34 | ``run_integration_tests.py`` from the ``scripts`` directory. If you 35 | use ``nix-shell`` to create your development environment then the 36 | ``scripts`` directory should be in you ``PATH`` variable. 37 | 38 | Please note that all integration test cases are classes deriving from 39 | ``integrationtests.framework.IntegrationTest``. Also all these tests 40 | must end with ``TestCase``, e.g. ``MyCustomTestCase``. 41 | 42 | Maintainance scripts 43 | ^^^^^^^^^^^^^^^^^^^^ 44 | 45 | The ``scripts`` folder contains programs that help to maintain the 46 | repository. We expect the user to have all the packages from the 47 | build environment of pypi2nix installed. We register the ``scripts`` 48 | directory in the users ``PATH`` if they choose to enter ``nix-shell`` 49 | in the top level directory of this project. All maintainance scripts 50 | should offer a list of legal command line arguments via the ``--help`` 51 | flag. 52 | 53 | Version bumping 54 | ^^^^^^^^^^^^^^^ 55 | 56 | We use ``bumpv`` to manage the current version of this project. This 57 | program should be part of the development environment. 58 | 59 | Code formatting 60 | ^^^^^^^^^^^^^^^ 61 | 62 | We try to automate as much code formatting as possible. For python 63 | source code we use ``black`` and ``isort``. For nix source code we 64 | use ``nixfmt``. Both tools are available in development environment 65 | provided by ``nix-shell``. The continous integration system will 66 | complain if the code is not formatted properly and the package won't 67 | build. You can automatically format all code via the 68 | ``format_sources.py`` program. You can run it like any other 69 | maintainance script from any working directory you like as long as you 70 | are inside the provided ``nix-shell`` environment. Example:: 71 | 72 | [nix-shell:~/src/pypi2nix]$ format_sources.py 73 | Skipped 2 files 74 | All done! ✨ 🍰 ✨ 75 | 131 files left unchanged. 76 | Success: no issues found in 47 source files 77 | Success: no issues found in 122 source files 78 | 79 | 80 | .. _`pytest`: https://pytest.org 81 | .. _`pypi2nix repository`: https://github.com/nix-community/pypi2nix 82 | -------------------------------------------------------------------------------- /source/external-dependencies.rst: -------------------------------------------------------------------------------- 1 | External Dependencies 2 | ===================== 3 | 4 | This chapter is based on a `PR for pypi2nix`_. Got there if you want 5 | to learn about the intial implementation of external dependency 6 | detection. 7 | 8 | Goal 9 | ---- 10 | 11 | The goal is to have a system that helps the user to deal with external 12 | dependencies. Currently the user has to know by heart (or trial and 13 | error) that e.g. \ ``lxml`` needs ``libxml2`` and ``libxslt``. We want 14 | to automate that for the user, at least for the most common packages. 15 | 16 | The idea is to have a system that you pass your initial requirements and 17 | as a result you get a list of the necessary external dependencies. This 18 | will have synergy with implementing automatic setup requirement 19 | detection. 20 | 21 | Mechanism 22 | --------- 23 | 24 | Every ``pypi2nix`` invocation has with it associated a set of 25 | requirements. This is usually the set of requirements that the user has 26 | for their project. To find out what kind of external dependencies are 27 | necessary to build the requested packages we need to solve two problems: 28 | 29 | 1) Find out all the dependencies of the specified/requested packages 30 | from the user without building them. 31 | 2) For all of these dependencies find out if and what external 32 | dependencies are required without building all the packages. 33 | 34 | We have to know all required external dependencies in advance since 35 | restarting the whole build just because a dependency was detected us 36 | unacceptable. For some users with slower or older hardware even one 37 | single build might take more than 10 minutes for larger package sets. If 38 | the build for restart several times it would render ``pypi2nix`` 39 | unusable to those users. 40 | 41 | That means that we have to have a place where dependency information can 42 | be collected and used by pypi2nix. The information about the 43 | dependencies is basically a directed acyclic graph since this 44 | implementation will not support circular dependencies (for now). 45 | 46 | For collecting information about external dependencies we will rely on 47 | users reporting such external dependencies for know. Developing a tool 48 | to detect external dependencies from build output is out of scope of 49 | this PR. 50 | 51 | The dependency graph for python dependencies can be generated by 52 | ``pypi2nix`` automatically. This PR will include a mechanism to 53 | generated dependency graphs in the right data format for ingestion by 54 | ``pypi2nix``. 55 | 56 | Infrastructure 57 | -------------- 58 | 59 | We need way to distribute a dependency tree to users. This will make the 60 | detection mechanism much more useful since it frees users from 61 | maintaining fresh set of dependencies. We want to implement a similar 62 | mechanism as with 63 | `pypi2nix-overrides `__. 64 | This means that we a have a semi-central git repository that contains 65 | all the detected dependencies. Since git is content addressable we can 66 | ensure reproducible builds. Our security model with that approach is 67 | **Trust on First Use**. 68 | 69 | Data format 70 | ----------- 71 | 72 | In order to minimize necessary labor we have to automate the generation 73 | of dependencies as much as possible. This means that we need to have 74 | data format that allows seamless merging of generated and curated 75 | dependency trees. Also we should use a data format that is easy to edit 76 | by humans and machines alike. A suitable candidate would be the `yaml 77 | format `__. This would allow us to provide `json 78 | schemas `__ for the data format to allow for 79 | effective reuse of the data. A concern might be that the volume of the 80 | data makes a compact data file to large to download. If in the future we 81 | run into traffic or performance problems we might consider implementing 82 | a web API. Already using `json schemas `__ 83 | would make that transition easy as we could leverage OpenAPI 3.0 to make 84 | the data format and the API accessible to many people. 85 | 86 | .. _`PR for pypi2nix`: https://github.com/nix-community/pypi2nix/pull/426 87 | -------------------------------------------------------------------------------- /source/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to pypi2nix's documentation! 2 | ==================================== 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | :caption: Contents: 7 | 8 | modules 9 | external-dependencies 10 | development 11 | 12 | 13 | Introduction 14 | ============ 15 | .. include:: ../README.rst 16 | :start-after: inclusion-marker 17 | 18 | Indices and tables 19 | ================== 20 | 21 | * :ref:`genindex` 22 | * :ref:`modindex` 23 | * :ref:`search` 24 | -------------------------------------------------------------------------------- /src/pypi2nix/VERSION: -------------------------------------------------------------------------------- 1 | 2.0.4 -------------------------------------------------------------------------------- /src/pypi2nix/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /src/pypi2nix/archive.py: -------------------------------------------------------------------------------- 1 | import tarfile 2 | import tempfile 3 | import zipfile 4 | from contextlib import contextmanager 5 | from typing import Iterator 6 | 7 | 8 | class UnpackingFailed(Exception): 9 | pass 10 | 11 | 12 | class Archive: 13 | def __init__(self, path: str) -> None: 14 | self.path = path 15 | 16 | @contextmanager 17 | def extracted_files(self) -> Iterator[str]: 18 | with tempfile.TemporaryDirectory() as directory: 19 | self.unpack_archive(directory) 20 | yield directory 21 | 22 | def unpack_archive(self, target_directory: str) -> None: 23 | if self.path.endswith(".tar.gz"): 24 | with tarfile.open(self.path, "r:gz") as tar: 25 | tar.extractall(path=target_directory) 26 | elif self.path.endswith(".zip") or self.path.endswith(".whl"): 27 | with zipfile.ZipFile(self.path) as archive: 28 | archive.extractall(path=target_directory) 29 | elif self.path.endswith(".tar.bz2"): 30 | with tarfile.open(self.path, "r:bz2") as tar: 31 | tar.extractall(path=target_directory) 32 | else: 33 | raise UnpackingFailed( 34 | "Could not detect archive format for file {}".format(self.path) 35 | ) 36 | 37 | def __str__(self) -> str: 38 | return f"Archive" 39 | -------------------------------------------------------------------------------- /src/pypi2nix/configuration.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from typing import Optional 3 | 4 | from attr import attrib 5 | from attr import attrs 6 | 7 | from pypi2nix.dependency_graph import DependencyGraph 8 | from pypi2nix.logger import Verbosity 9 | from pypi2nix.overrides import Overrides 10 | from pypi2nix.path import Path 11 | from pypi2nix.python_version import PythonVersion 12 | 13 | 14 | @attrs 15 | class ApplicationConfiguration: 16 | verbosity: Verbosity = attrib() 17 | nix_executable_directory: Optional[str] = attrib() 18 | nix_path: List[str] = attrib() 19 | extra_build_inputs: List[str] = attrib() 20 | emit_extra_build_inputs: bool = attrib() 21 | extra_environment: str = attrib() 22 | enable_tests: bool = attrib() 23 | python_version: PythonVersion = attrib() 24 | requirement_files: List[str] = attrib() 25 | requirements: List[str] = attrib() 26 | setup_requirements: List[str] = attrib() 27 | overrides: List[Overrides] = attrib() 28 | wheels_caches: List[str] = attrib() 29 | output_basename: str = attrib() 30 | project_directory: Path = attrib() 31 | target_directory: str = attrib() 32 | dependency_graph_output_location: Optional[Path] = attrib() 33 | dependency_graph_input: DependencyGraph = attrib() 34 | -------------------------------------------------------------------------------- /src/pypi2nix/environment_marker.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from attr import attrib 4 | from attr import attrs 5 | from packaging.markers import InvalidMarker 6 | from packaging.markers import Marker 7 | from packaging.markers import UndefinedComparison 8 | from packaging.markers import UndefinedEnvironmentName 9 | 10 | from pypi2nix.target_platform import TargetPlatform 11 | 12 | 13 | class MarkerEvaluationFailed(Exception): 14 | pass 15 | 16 | 17 | @attrs 18 | class EnvironmentMarker: 19 | _marker_string: str = attrib() 20 | 21 | def applies_to_platform( 22 | self, target_platform: TargetPlatform, extras: List[str] = [] 23 | ) -> bool: 24 | def _applies_to_platform_with_extra(extra: str) -> bool: 25 | environment = target_platform.environment_dictionary() 26 | environment["extra"] = extra 27 | try: 28 | return Marker(self._marker_string).evaluate(environment) 29 | except (InvalidMarker, UndefinedComparison, UndefinedEnvironmentName): 30 | raise MarkerEvaluationFailed( 31 | f"Failed to evaluate marker {self._marker_string}" 32 | ) 33 | 34 | if not extras: 35 | extras = [""] 36 | for extra in extras: 37 | if _applies_to_platform_with_extra(extra): 38 | return True 39 | return False 40 | -------------------------------------------------------------------------------- /src/pypi2nix/exceptions.py: -------------------------------------------------------------------------------- 1 | class UnknownTargetPlatform(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /src/pypi2nix/external_dependencies/__init__.py: -------------------------------------------------------------------------------- 1 | from .external_dependency import ExternalDependency 2 | -------------------------------------------------------------------------------- /src/pypi2nix/external_dependencies/external_dependency.py: -------------------------------------------------------------------------------- 1 | from attr import attrib 2 | from attr import attrs 3 | 4 | 5 | @attrs(frozen=True) 6 | class ExternalDependency: 7 | _attribute_name: str = attrib() 8 | 9 | def attribute_name(self) -> str: 10 | return self._attribute_name 11 | -------------------------------------------------------------------------------- /src/pypi2nix/external_dependency_collector/__init__.py: -------------------------------------------------------------------------------- 1 | from .collector import ExternalDependencyCollector 2 | from .lookup import RequirementDependencyRetriever 3 | -------------------------------------------------------------------------------- /src/pypi2nix/external_dependency_collector/collector.py: -------------------------------------------------------------------------------- 1 | from copy import copy 2 | from typing import Set 3 | 4 | from pypi2nix.external_dependencies import ExternalDependency 5 | from pypi2nix.requirement_set import RequirementSet 6 | 7 | from .lookup import RequirementDependencyRetriever 8 | 9 | 10 | class ExternalDependencyCollector: 11 | def __init__( 12 | self, requirement_dependency_retriever: RequirementDependencyRetriever 13 | ) -> None: 14 | self._external_dependencies: Set[ExternalDependency] = set() 15 | self._requirement_dependency_retriever = requirement_dependency_retriever 16 | 17 | def collect_explicit(self, attribute_name: str) -> None: 18 | self._external_dependencies.add(ExternalDependency(attribute_name)) 19 | 20 | def collect_from_requirements(self, requirements: RequirementSet) -> None: 21 | for requirement in requirements: 22 | self._external_dependencies.update( 23 | self._requirement_dependency_retriever.get_external_dependency_for_requirement( 24 | requirement 25 | ) 26 | ) 27 | 28 | def get_collected(self) -> Set[ExternalDependency]: 29 | return copy(self._external_dependencies) 30 | -------------------------------------------------------------------------------- /src/pypi2nix/external_dependency_collector/lookup.py: -------------------------------------------------------------------------------- 1 | from typing import Set 2 | 3 | from pypi2nix.dependency_graph import DependencyGraph 4 | from pypi2nix.external_dependencies import ExternalDependency 5 | from pypi2nix.requirements import Requirement 6 | 7 | 8 | class RequirementDependencyRetriever: 9 | def __init__(self, dependency_graph: DependencyGraph = DependencyGraph()): 10 | self._dependency_graph = dependency_graph 11 | 12 | def get_external_dependency_for_requirement( 13 | self, requirement: Requirement 14 | ) -> Set[ExternalDependency]: 15 | return self._dependency_graph.get_all_external_dependencies(requirement) 16 | -------------------------------------------------------------------------------- /src/pypi2nix/logger.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | from abc import ABCMeta 5 | from abc import abstractmethod 6 | from enum import Enum 7 | from enum import unique 8 | from typing import Optional 9 | from typing import TextIO 10 | 11 | 12 | class LoggerNotConnected(Exception): 13 | pass 14 | 15 | 16 | @unique 17 | class Verbosity(Enum): 18 | ERROR = -1 19 | WARNING = 0 20 | INFO = 1 21 | DEBUG = 2 22 | 23 | 24 | VERBOSITY_MIN: int = min(*map(lambda v: v.value, Verbosity)) # type: ignore 25 | VERBOSITY_MAX: int = max(*map(lambda v: v.value, Verbosity)) # type: ignore 26 | 27 | 28 | def verbosity_from_int(n: int) -> Verbosity: 29 | for verbosity_level in Verbosity: 30 | if verbosity_level.value == n: 31 | return verbosity_level 32 | if n < VERBOSITY_MIN: 33 | return Verbosity.ERROR 34 | else: 35 | return Verbosity.DEBUG 36 | 37 | 38 | class Logger(metaclass=ABCMeta): 39 | @abstractmethod 40 | def error(self, text: str) -> None: 41 | pass 42 | 43 | @abstractmethod 44 | def warning(self, text: str) -> None: 45 | pass 46 | 47 | @abstractmethod 48 | def info(self, text: str) -> None: 49 | pass 50 | 51 | @abstractmethod 52 | def debug(self, text: str) -> None: 53 | pass 54 | 55 | @abstractmethod 56 | def set_verbosity(self, level: Verbosity) -> None: 57 | pass 58 | 59 | 60 | class StreamLogger(Logger): 61 | def __init__(self, output: TextIO): 62 | self.output = output 63 | self.verbosity_level: Verbosity = Verbosity.DEBUG 64 | 65 | def warning(self, text: str) -> None: 66 | if self.verbosity_level.value >= Verbosity.WARNING.value: 67 | for line in text.splitlines(): 68 | print("WARNING:", line, file=self.output) 69 | 70 | def error(self, text: str) -> None: 71 | for line in text.splitlines(): 72 | print("ERROR:", line, file=self.output) 73 | 74 | def info(self, text: str) -> None: 75 | if self.verbosity_level.value >= Verbosity.INFO.value: 76 | for line in text.splitlines(): 77 | print("INFO:", line, file=self.output) 78 | 79 | def debug(self, text: str) -> None: 80 | if self.verbosity_level.value >= Verbosity.DEBUG.value: 81 | for line in text.splitlines(): 82 | print("DEBUG:", line, file=self.output) 83 | 84 | def set_verbosity(self, level: Verbosity) -> None: 85 | self.verbosity_level = level 86 | 87 | @classmethod 88 | def stdout_logger(constructor) -> StreamLogger: 89 | return constructor(sys.stdout) 90 | 91 | 92 | class ProxyLogger(Logger): 93 | def __init__(self) -> None: 94 | self._target_logger: Optional[Logger] = None 95 | 96 | def info(self, text: str) -> None: 97 | if self._target_logger is not None: 98 | self._target_logger.info(text) 99 | else: 100 | raise LoggerNotConnected("Logger not connected") 101 | 102 | def debug(self, text: str) -> None: 103 | if self._target_logger is not None: 104 | self._target_logger.debug(text) 105 | else: 106 | raise LoggerNotConnected("Logger not connected") 107 | 108 | def warning(self, text: str) -> None: 109 | if self._target_logger is not None: 110 | self._target_logger.warning(text) 111 | else: 112 | raise LoggerNotConnected("Logger not connected") 113 | 114 | def error(self, text: str) -> None: 115 | if self._target_logger is not None: 116 | self._target_logger.error(text) 117 | else: 118 | raise LoggerNotConnected("Logger not connected") 119 | 120 | def set_verbosity(self, level: Verbosity) -> None: 121 | if self._target_logger is not None: 122 | self._target_logger.set_verbosity(level) 123 | else: 124 | raise LoggerNotConnected("Logger not connected") 125 | 126 | def set_target_logger(self, target: Logger) -> None: 127 | self._target_logger = target 128 | 129 | def get_target_logger(self) -> Optional[Logger]: 130 | return self._target_logger 131 | -------------------------------------------------------------------------------- /src/pypi2nix/memoize.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from typing import Callable 3 | from typing import TypeVar 4 | 5 | S = TypeVar("S") 6 | T = TypeVar("T") 7 | 8 | 9 | def memoize(method: Callable[[S], T]) -> Callable[[S], T]: 10 | @wraps(method) 11 | def wrapped_method(self: S) -> T: 12 | attribute_name = "_memoize_attribute_" + method.__name__ 13 | if not hasattr(self, attribute_name): 14 | setattr(self, attribute_name, method(self)) 15 | return getattr(self, attribute_name) # type: ignore 16 | 17 | return wrapped_method 18 | -------------------------------------------------------------------------------- /src/pypi2nix/metadata_fetcher.py: -------------------------------------------------------------------------------- 1 | """Parse metadata from .dist-info directories in a wheelhouse.""" 2 | # flake8: noqa: E501 3 | 4 | import email 5 | import hashlib 6 | import json 7 | import os.path 8 | import tempfile 9 | from typing import Any 10 | from typing import Dict 11 | from typing import Iterable 12 | from typing import List 13 | from urllib.request import urlopen 14 | 15 | import click 16 | 17 | from pypi2nix.logger import Logger 18 | from pypi2nix.package_source import UrlSource 19 | from pypi2nix.pypi import Pypi 20 | from pypi2nix.requirement_parser import RequirementParser 21 | from pypi2nix.requirement_set import RequirementSet 22 | from pypi2nix.requirements import Requirement 23 | from pypi2nix.source_distribution import SourceDistribution 24 | from pypi2nix.sources import Sources 25 | from pypi2nix.target_platform import TargetPlatform 26 | from pypi2nix.utils import cmd 27 | from pypi2nix.utils import prefetch_git 28 | from pypi2nix.wheel import Wheel 29 | 30 | 31 | class MetadataFetcher: 32 | def __init__( 33 | self, 34 | sources: Sources, 35 | logger: Logger, 36 | requirement_parser: RequirementParser, 37 | pypi: Pypi, 38 | ) -> None: 39 | self.sources = sources 40 | self.logger = logger 41 | self.requirement_parser = requirement_parser 42 | self.pypi = pypi 43 | 44 | def main( 45 | self, 46 | wheel_paths: Iterable[str], 47 | target_platform: TargetPlatform, 48 | source_distributions: Dict[str, SourceDistribution], 49 | ) -> List[Wheel]: 50 | """Extract packages metadata from wheels dist-info folders. 51 | """ 52 | output = "" 53 | metadata: List[Wheel] = [] 54 | 55 | self.logger.info( 56 | "-- sources ---------------------------------------------------------------" 57 | ) 58 | for name, source in self.sources.items(): 59 | self.logger.info("{name}, {source}".format(name=name, source=name)) 60 | self.logger.info( 61 | "--------------------------------------------------------------------------" 62 | ) 63 | 64 | wheels = [] 65 | for wheel_path in wheel_paths: 66 | 67 | self.logger.debug("|-> from %s" % os.path.basename(wheel_path)) 68 | 69 | wheel_metadata = Wheel.from_wheel_directory_path( 70 | wheel_path, target_platform, self.logger, self.requirement_parser 71 | ) 72 | if not wheel_metadata: 73 | continue 74 | 75 | if wheel_metadata.name in source_distributions: 76 | source_distribution = source_distributions[wheel_metadata.name] 77 | wheel_metadata.add_build_dependencies( 78 | source_distribution.build_dependencies(target_platform) 79 | ) 80 | wheel_metadata.package_format = source_distribution.package_format 81 | 82 | wheels.append(wheel_metadata) 83 | 84 | self.logger.debug( 85 | "-- wheel_metadata --------------------------------------------------------" 86 | ) 87 | self.logger.debug( 88 | json.dumps(wheel_metadata.to_dict(), sort_keys=True, indent=4) 89 | ) 90 | self.logger.debug( 91 | "--------------------------------------------------------------------------" 92 | ) 93 | 94 | self.process_wheel(wheel_metadata) 95 | return wheels 96 | 97 | def process_wheel(self, wheel: Wheel) -> None: 98 | if wheel.name not in self.sources: 99 | release = self.pypi.get_source_release(wheel.name, wheel.version) 100 | if release: 101 | source = UrlSource( 102 | url=release.url, 103 | logger=self.logger, 104 | hash_value=release.sha256_digest, 105 | ) 106 | self.sources.add(wheel.name, source) 107 | else: 108 | self.logger.error( 109 | f"Failed to query pypi for release name=`{wheel.name}`, version=`{wheel.version}`" 110 | ) 111 | raise MetadataFetchingFailed() 112 | 113 | 114 | class MetadataFetchingFailed(Exception): 115 | pass 116 | -------------------------------------------------------------------------------- /src/pypi2nix/network_file.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import tempfile 3 | from abc import ABCMeta 4 | from abc import abstractmethod 5 | from typing import Dict 6 | from typing import Optional 7 | from urllib.request import urlopen 8 | 9 | from pypi2nix.logger import Logger 10 | from pypi2nix.memoize import memoize 11 | from pypi2nix.utils import cmd 12 | from pypi2nix.utils import prefetch_git 13 | from pypi2nix.utils import prefetch_url 14 | 15 | 16 | class NetworkFile(metaclass=ABCMeta): 17 | @abstractmethod 18 | def nix_expression(self) -> str: 19 | pass 20 | 21 | @abstractmethod 22 | def fetch(self) -> str: 23 | pass 24 | 25 | 26 | class UrlTextFile(NetworkFile): 27 | def __init__( 28 | self, 29 | url: str, 30 | logger: Logger, 31 | sha256: Optional[str] = None, 32 | name: Optional[str] = None, 33 | ) -> None: 34 | self.url = url 35 | self._sha256 = sha256 36 | self._logger = logger 37 | self._name = name 38 | 39 | def nix_expression(self) -> str: 40 | fetchurl_arguments = f'url = "{self.url}";' 41 | fetchurl_arguments += f'sha256 = "{self.sha256}";' 42 | if self._name: 43 | fetchurl_arguments += f'name = "{self._name}";' 44 | return f"pkgs.fetchurl {{ {fetchurl_arguments} }}" 45 | 46 | @property # type: ignore 47 | @memoize 48 | def sha256(self) -> str: 49 | if self._sha256: 50 | return self._sha256 51 | else: 52 | return prefetch_url(self.url, self._logger, name=self._name) 53 | 54 | def fetch(self) -> str: 55 | with urlopen(self.url) as content: 56 | return content.read().decode("utf-8") # type: ignore 57 | 58 | 59 | class GitTextFile(NetworkFile): 60 | def __init__( 61 | self, repository_url: str, revision_name: str, path: str, logger: Logger 62 | ) -> None: 63 | self.repository_url = repository_url 64 | self._revision_name = revision_name 65 | self.path = path 66 | self._logger = logger 67 | 68 | def nix_expression(self) -> str: 69 | fetchgit_arguments = f'url = "{self.repository_url}";' 70 | fetchgit_arguments += f'sha256 = "{self.sha256}";' 71 | fetchgit_arguments += f'rev = "{self.revision}";' 72 | fetchgit_expression = f"pkgs.fetchgit {{ {fetchgit_arguments} }}" 73 | return f'"${{ {fetchgit_expression } }}/{self.path}"' 74 | 75 | @property 76 | def revision(self) -> str: 77 | return self._prefetch_data["rev"] 78 | 79 | @property 80 | def sha256(self) -> str: 81 | return self._prefetch_data["sha256"] 82 | 83 | @property # type: ignore 84 | @memoize 85 | def _prefetch_data(self) -> Dict[str, str]: 86 | return prefetch_git(self.repository_url, self._revision_name) 87 | 88 | @memoize 89 | def fetch(self) -> str: 90 | with tempfile.TemporaryDirectory() as target_directory: 91 | cmd( 92 | ["git", "clone", self.repository_url, target_directory], 93 | logger=self._logger, 94 | ) 95 | cmd( 96 | ["git", "checkout", self._revision_name], 97 | logger=self._logger, 98 | cwd=target_directory, 99 | ) 100 | with open(os.path.join(target_directory, self.path)) as f: 101 | return f.read() 102 | 103 | 104 | class DiskTextFile(NetworkFile): 105 | def __init__(self, path: str): 106 | self._path = path 107 | 108 | def nix_expression(self) -> str: 109 | return self._path 110 | 111 | def fetch(self) -> str: 112 | with open(self._path) as f: 113 | return f.read() 114 | -------------------------------------------------------------------------------- /src/pypi2nix/nix.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | from typing import Dict 3 | from typing import List 4 | from typing import Optional 5 | 6 | from pypi2nix.logger import Logger 7 | from pypi2nix.utils import NixOption 8 | from pypi2nix.utils import cmd 9 | from pypi2nix.utils import create_command_options 10 | 11 | 12 | class ExecutableNotFound(FileNotFoundError): 13 | pass 14 | 15 | 16 | class EvaluationFailed(Exception): 17 | def __init__(self, *args, output: Optional[str] = None, **kwargs): # type: ignore 18 | super().__init__(self, *args, **kwargs) # type: ignore 19 | self.output: Optional[str] = output 20 | 21 | 22 | class Nix: 23 | def __init__( 24 | self, 25 | logger: Logger, 26 | nix_path: List[str] = [], 27 | executable_directory: Optional[str] = None, 28 | ): 29 | self.nix_path = nix_path 30 | self.executable_directory = executable_directory 31 | self.logger = logger 32 | 33 | def evaluate_expression(self, expression: str) -> str: 34 | output = self.run_nix_command( 35 | "nix-instantiate", ["--eval", "--expr", expression] 36 | ) 37 | # cut off the last newline character append to the output 38 | return output[:-1] 39 | 40 | def shell( 41 | self, 42 | command: str, 43 | derivation_path: str, 44 | nix_arguments: Dict[str, NixOption] = {}, 45 | pure: bool = True, 46 | ) -> str: 47 | output = self.run_nix_command( 48 | "nix-shell", 49 | create_command_options(nix_arguments) 50 | + (["--pure"] if pure else []) 51 | + [derivation_path, "--run", command], 52 | ) 53 | return output 54 | 55 | def build( 56 | self, 57 | source_file: str, 58 | attribute: Optional[str] = None, 59 | out_link: Optional[str] = None, 60 | arguments: Dict[str, NixOption] = dict(), 61 | ) -> None: 62 | self.run_nix_command( 63 | "nix-build", 64 | [source_file] 65 | + (["-o", out_link] if out_link else []) 66 | + (["-A", attribute] if attribute else []) 67 | + create_command_options(arguments), 68 | ) 69 | 70 | def evaluate_file(self, source_file: str, attribute: Optional[str] = None) -> None: 71 | absolute_source_file = os.path.abspath(source_file) 72 | self.evaluate_expression( 73 | f"let file_path = {absolute_source_file}; " 74 | + "file_expression = import file_path {}; " 75 | + "in " 76 | + f"file_expression.{attribute}" 77 | if attribute 78 | else "file_expression", 79 | ) 80 | 81 | def build_expression( 82 | self, 83 | expression: str, 84 | out_link: Optional[str] = None, 85 | arguments: Dict[str, NixOption] = dict(), 86 | ) -> None: 87 | self.run_nix_command( 88 | "nix-build", 89 | ["--expr", expression] 90 | + (["-o", out_link] if out_link else ["--no-out-link"]) 91 | + create_command_options(arguments), 92 | ) 93 | 94 | def run_nix_command(self, binary_name: str, command: List[str]) -> str: 95 | final_command = ( 96 | [self.executable_path(binary_name)] + self.nix_path_arguments() + command 97 | ) 98 | returncode: int 99 | output: str 100 | try: 101 | returncode, output = cmd(final_command, self.logger) 102 | except FileNotFoundError: 103 | raise ExecutableNotFound( 104 | "Could not find executable '{program}'".format(program=binary_name) 105 | ) 106 | if returncode != 0: 107 | raise EvaluationFailed( 108 | "'{program}' exited with non-zero exit code ({code}).".format( 109 | program=binary_name, code=returncode 110 | ), 111 | output=output, 112 | ) 113 | return output 114 | 115 | def nix_path_arguments(self) -> List[str]: 116 | path_arguments = [] 117 | for path in self.nix_path: 118 | path_arguments.append("-I") 119 | path_arguments.append(path) 120 | return path_arguments 121 | 122 | def executable_path(self, program_name: str) -> str: 123 | if self.executable_directory is None: 124 | return program_name 125 | else: 126 | return os.path.join(self.executable_directory, program_name) 127 | -------------------------------------------------------------------------------- /src/pypi2nix/nix_language.py: -------------------------------------------------------------------------------- 1 | def escape_string(string: str) -> str: 2 | return string.replace('"', '\\"') 3 | -------------------------------------------------------------------------------- /src/pypi2nix/overrides.py: -------------------------------------------------------------------------------- 1 | import urllib 2 | from abc import ABCMeta 3 | from abc import abstractmethod 4 | from typing import Dict 5 | from typing import Optional 6 | from typing import no_type_check 7 | from urllib.parse import urldefrag 8 | from urllib.parse import urlparse 9 | 10 | import click 11 | 12 | from pypi2nix.logger import Logger 13 | from pypi2nix.logger import StreamLogger 14 | from pypi2nix.network_file import DiskTextFile 15 | from pypi2nix.network_file import GitTextFile 16 | from pypi2nix.network_file import NetworkFile 17 | from pypi2nix.network_file import UrlTextFile 18 | 19 | from .utils import prefetch_github 20 | 21 | 22 | class UnsupportedUrlError(Exception): 23 | pass 24 | 25 | 26 | class Overrides(metaclass=ABCMeta): 27 | @abstractmethod 28 | def nix_expression(self, logger: Logger) -> str: 29 | pass 30 | 31 | 32 | class OverridesNetworkFile(Overrides): 33 | def __init__(self, network_file: NetworkFile) -> None: 34 | self._network_file = network_file 35 | 36 | def nix_expression(self, logger: Logger) -> str: 37 | return f"import ({self._network_file.nix_expression()}) {{ inherit pkgs python ; }}" 38 | 39 | 40 | class OverridesGithub(Overrides): 41 | def __init__( 42 | self, owner: str, repo: str, path: str, rev: Optional[str] = None 43 | ) -> None: 44 | self.owner = owner 45 | self.repo = repo 46 | self.path = path 47 | self.rev = rev 48 | 49 | def nix_expression(self, logger: Logger) -> str: # noqa: U100 50 | prefetch_data = prefetch_github(self.owner, self.repo, self.rev) 51 | template = " ".join( 52 | [ 53 | "let src = pkgs.fetchFromGitHub {{", 54 | 'owner = "{owner}";', 55 | 'repo = "{repo}";', 56 | 'rev = "{rev}";', 57 | 'sha256 = "{sha256}";', 58 | "}} ;", 59 | 'in import "${{src}}/{path}" {{', 60 | "inherit pkgs python;", 61 | "}}", 62 | ] 63 | ) 64 | return template.format( 65 | owner=self.owner, 66 | repo=self.repo, 67 | rev=prefetch_data["rev"], 68 | sha256=prefetch_data["sha256"], 69 | path=self.path, 70 | ) 71 | 72 | 73 | class NetworkFileParameter(click.ParamType): 74 | name = "url" 75 | 76 | @no_type_check 77 | def convert(self, value, param, ctx): 78 | try: 79 | return self._url_to_network_file(value) 80 | except UnsupportedUrlError as e: 81 | self.fail(str(e), param, ctx) 82 | 83 | def _url_to_network_file(self, url_string: str) -> NetworkFile: 84 | url = urlparse(url_string) 85 | if url.scheme == "": 86 | return DiskTextFile(url.path) 87 | elif url.scheme == "file": 88 | return DiskTextFile(url.path) 89 | elif url.scheme == "http" or url.scheme == "https": 90 | return UrlTextFile(url.geturl(), StreamLogger.stdout_logger()) 91 | elif url.scheme.startswith("git+"): 92 | return self._handle_git_override_url(url, url_string) 93 | else: 94 | raise UnsupportedUrlError( 95 | "Cannot handle common overrides url %s" % url_string 96 | ) 97 | 98 | def _handle_git_override_url( 99 | self, url: urllib.parse.ParseResult, url_string: str 100 | ) -> GitTextFile: 101 | if not url.fragment: 102 | raise UnsupportedUrlError( 103 | ( 104 | "Cannot handle overrides with no path given, offending url was" 105 | " {url}." 106 | ).format(url=url_string) 107 | ) 108 | fragments: Dict[str, str] = dict() 109 | for fragment_item in url.fragment.split("&"): 110 | try: 111 | fragment_name, fragment_value = fragment_item.split() 112 | except ValueError: 113 | raise UnsupportedUrlError( 114 | f"Encountered deformed URL fragment `{fragment_item}` " 115 | f"in url `{url_string}`" 116 | ) 117 | else: 118 | fragments[fragment_name] = fragment_value 119 | return GitTextFile( 120 | repository_url=urldefrag(url.geturl()[4:])[0], 121 | path=fragments["path"], 122 | revision_name=fragments.get("rev", "master"), 123 | logger=StreamLogger.stdout_logger(), 124 | ) 125 | 126 | 127 | FILE_URL = NetworkFileParameter() 128 | -------------------------------------------------------------------------------- /src/pypi2nix/package/__init__.py: -------------------------------------------------------------------------------- 1 | from .exceptions import DistributionNotDetected 2 | from .interfaces import HasBuildDependencies 3 | from .interfaces import HasPackageName 4 | from .interfaces import HasRuntimeDependencies 5 | from .metadata import PackageMetadata 6 | from .pyproject import PyprojectToml 7 | from .setupcfg import SetupCfg 8 | -------------------------------------------------------------------------------- /src/pypi2nix/package/exceptions.py: -------------------------------------------------------------------------------- 1 | class DistributionNotDetected(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /src/pypi2nix/package/interfaces.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | 4 | from pypi2nix.requirement_set import RequirementSet 5 | from pypi2nix.target_platform import TargetPlatform 6 | 7 | 8 | class HasBuildDependencies(metaclass=ABCMeta): 9 | @abstractmethod 10 | def build_dependencies(self, target_platform: TargetPlatform) -> RequirementSet: 11 | pass 12 | 13 | 14 | class HasRuntimeDependencies(metaclass=ABCMeta): 15 | @abstractmethod 16 | def runtime_dependencies(self, target_platform: TargetPlatform) -> RequirementSet: 17 | pass 18 | 19 | 20 | class HasPackageName(metaclass=ABCMeta): 21 | @abstractmethod 22 | def package_name(self) -> str: 23 | pass 24 | -------------------------------------------------------------------------------- /src/pypi2nix/package/metadata.py: -------------------------------------------------------------------------------- 1 | import os 2 | from email.header import Header 3 | from email.parser import Parser as EmailParser 4 | 5 | from attr import attrib 6 | from attr import attrs 7 | 8 | from .exceptions import DistributionNotDetected 9 | 10 | 11 | @attrs 12 | class PackageMetadata: 13 | name: str = attrib() 14 | 15 | @classmethod 16 | def from_package_directory(package_metadata, path: str) -> "PackageMetadata": 17 | builder = _PackageMetadataBuilder(path) 18 | return package_metadata(name=builder.name) 19 | 20 | 21 | class _PackageMetadataBuilder: 22 | def __init__(self, path_to_directory: str) -> None: 23 | self.path_to_directory = path_to_directory 24 | self._name: str = "" 25 | 26 | self.build() 27 | 28 | @property 29 | def name(self) -> str: 30 | return self._name 31 | 32 | def build(self) -> None: 33 | pkg_info_file = os.path.join(self.path_to_directory, "PKG-INFO") 34 | try: 35 | with open(pkg_info_file) as f: 36 | parser = EmailParser() 37 | metadata = parser.parse(f) 38 | except FileNotFoundError: 39 | raise DistributionNotDetected( 40 | f"Could not find PKG-INFO file in {self.path_to_directory}" 41 | ) 42 | self._name = metadata.get("name") 43 | if isinstance(self._name, Header): 44 | raise DistributionNotDetected( 45 | "Could not parse source distribution metadata, name detection failed" 46 | ) 47 | -------------------------------------------------------------------------------- /src/pypi2nix/package/pyproject.py: -------------------------------------------------------------------------------- 1 | import toml 2 | 3 | from pypi2nix.logger import Logger 4 | from pypi2nix.requirement_parser import ParsingFailed 5 | from pypi2nix.requirement_parser import RequirementParser 6 | from pypi2nix.requirement_set import RequirementSet 7 | from pypi2nix.target_platform import TargetPlatform 8 | 9 | from .interfaces import HasBuildDependencies 10 | 11 | 12 | class PyprojectToml(HasBuildDependencies): 13 | def __init__( 14 | self, 15 | name: str, 16 | file_content: str, 17 | logger: Logger, 18 | requirement_parser: RequirementParser, 19 | ) -> None: 20 | self.pyproject_toml = toml.loads(file_content) 21 | self.logger = logger 22 | self.requirement_parser = requirement_parser 23 | self.name = name 24 | 25 | def build_dependencies(self, target_platform: TargetPlatform) -> RequirementSet: 26 | requirement_set = RequirementSet(target_platform) 27 | if self.pyproject_toml is not None: 28 | for build_input in self.pyproject_toml.get("build-system", {}).get( 29 | "requires", [] 30 | ): 31 | try: 32 | requirement = self.requirement_parser.parse(build_input) 33 | except ParsingFailed as e: 34 | self.logger.warning( 35 | "Failed to parse build dependency of `{name}`".format( 36 | name=self.name 37 | ) 38 | ) 39 | self.logger.warning( 40 | "Possible reason: `{reason}`".format(reason=e.reason) 41 | ) 42 | else: 43 | if requirement.applies_to_target(target_platform): 44 | requirement_set.add(requirement) 45 | return requirement_set 46 | -------------------------------------------------------------------------------- /src/pypi2nix/package/setupcfg.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from setuptools.config import read_configuration 4 | 5 | from pypi2nix.logger import Logger 6 | from pypi2nix.requirement_parser import ParsingFailed 7 | from pypi2nix.requirement_parser import RequirementParser 8 | from pypi2nix.requirement_set import RequirementSet 9 | from pypi2nix.target_platform import TargetPlatform 10 | 11 | from .interfaces import HasBuildDependencies 12 | 13 | 14 | class SetupCfg(HasBuildDependencies): 15 | def __init__( 16 | self, 17 | setup_cfg_path: str, 18 | logger: Logger, 19 | requirement_parser: RequirementParser, 20 | ): 21 | self.setup_cfg_path = setup_cfg_path 22 | self.setup_cfg = read_configuration(setup_cfg_path) 23 | self.logger = logger 24 | self.requirement_parser = requirement_parser 25 | 26 | def build_dependencies(self, target_platform: TargetPlatform) -> RequirementSet: 27 | setup_requires = self.setup_cfg.get("options", {}).get("setup_requires") 28 | requirements = RequirementSet(target_platform) 29 | if isinstance(setup_requires, str): 30 | requirements.add(self.requirement_parser.parse(setup_requires)) 31 | elif isinstance(setup_requires, list): 32 | for requirement_string in setup_requires: 33 | try: 34 | requirement = self.requirement_parser.parse(requirement_string) 35 | except ParsingFailed as e: 36 | self.logger.warning( 37 | f"Failed to parse build dependency of `{self.setup_cfg_path}`" 38 | ) 39 | self.logger.warning(f"Possible reason: `{e.reason}`") 40 | else: 41 | if requirement.applies_to_target(target_platform): 42 | requirements.add(requirement) 43 | return requirements 44 | 45 | @property 46 | def name(self) -> Optional[str]: 47 | return self.setup_cfg.get("metadata", {}).get("name", None) # type: ignore 48 | -------------------------------------------------------------------------------- /src/pypi2nix/package_source.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | from typing import Dict 3 | from typing import Optional 4 | from typing import Union 5 | 6 | from pypi2nix.logger import Logger 7 | from pypi2nix.utils import prefetch_git 8 | from pypi2nix.utils import prefetch_hg 9 | from pypi2nix.utils import prefetch_url 10 | 11 | PackageSource = Union["GitSource", "HgSource", "PathSource", "UrlSource"] 12 | 13 | 14 | class GitSource: 15 | def __init__(self, url: str, revision: Optional[str] = None): 16 | self.url = url 17 | self._prefetch_data: Optional[Dict[str, str]] = None 18 | self._revision = revision 19 | 20 | def nix_expression(self) -> str: 21 | return "\n".join( 22 | [ 23 | "pkgs.fetchgit {", 24 | ' url = "%(url)s";', 25 | ' %(hash_type)s = "%(hash_value)s";', 26 | ' rev = "%(rev)s";', 27 | " }", 28 | ] 29 | ) % dict( 30 | url=self.url, 31 | hash_type="sha256", 32 | hash_value=self.hash_value(), 33 | rev=self.revision(), 34 | ) 35 | 36 | def hash_value(self) -> str: 37 | return self.prefetch_data()["sha256"] 38 | 39 | def revision(self) -> str: 40 | return self.prefetch_data()["rev"] 41 | 42 | def prefetch_data(self) -> Dict[str, str]: 43 | if self._prefetch_data is None: 44 | self._prefetch_data = prefetch_git(self.url, self._revision) 45 | return self._prefetch_data 46 | 47 | 48 | class HgSource: 49 | def __init__( 50 | self, url: str, logger: Logger, revision: Optional[str] = None 51 | ) -> None: 52 | self.url = url 53 | self._revision = revision 54 | self._prefetch_data: Optional[Dict[str, str]] = None 55 | self.logger = logger 56 | 57 | def nix_expression(self) -> str: 58 | return "\n".join( 59 | [ 60 | "pkgs.fetchhg {{", 61 | ' url = "{url}";', 62 | ' sha256 = "{hash_value}";', 63 | ' rev = "{revision}";', 64 | " }}", 65 | ] 66 | ).format(url=self.url, hash_value=self.hash_value(), revision=self.revision()) 67 | 68 | def hash_value(self) -> str: 69 | return self.prefetch_data()["sha256"] 70 | 71 | def revision(self) -> str: 72 | return self.prefetch_data()["revision"] 73 | 74 | def prefetch_data(self) -> Dict[str, str]: 75 | if self._prefetch_data is None: 76 | self._prefetch_data = prefetch_hg(self.url, self.logger, self._revision) 77 | return self._prefetch_data 78 | 79 | 80 | class UrlSource: 81 | def __init__( 82 | self, url: str, logger: Logger, hash_value: Optional[str] = None 83 | ) -> None: 84 | self.url = url 85 | self._hash_value = hash_value 86 | self.chunk_size = 2048 87 | self.logger = logger 88 | 89 | def nix_expression(self) -> str: 90 | return "\n".join( 91 | [ 92 | "pkgs.fetchurl {{", 93 | ' url = "{url}";', 94 | ' sha256 = "{hash_value}";', 95 | "}}", 96 | ] 97 | ).format(url=self.url, hash_value=self.hash_value()) 98 | 99 | def hash_value(self) -> str: 100 | if self._hash_value is None: 101 | self._hash_value = self.calculate_hash_value() 102 | return self._hash_value 103 | 104 | def calculate_hash_value(self) -> str: 105 | return prefetch_url(self.url, self.logger) 106 | 107 | 108 | class PathSource: 109 | def __init__(self, path: str) -> None: 110 | self.path = path 111 | 112 | @property 113 | def _normalized_path(self) -> str: 114 | if os.path.isabs(self.path): 115 | return self.path 116 | else: 117 | head, tail = os.path.split(self.path) 118 | if head: 119 | return self.path 120 | else: 121 | return os.path.join(self.path, ".") 122 | 123 | def nix_expression(self) -> str: 124 | return self._normalized_path 125 | -------------------------------------------------------------------------------- /src/pypi2nix/path.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | import os.path 5 | import pathlib 6 | from typing import List 7 | from typing import Union 8 | 9 | 10 | class Path: 11 | def __init__(self, path: Union[pathlib.Path, str, Path]) -> None: 12 | self._path: pathlib.Path 13 | if isinstance(path, str): 14 | self._path = pathlib.Path(path) 15 | elif isinstance(path, pathlib.Path): 16 | self._path = path 17 | else: 18 | self._path = path._path 19 | 20 | def list_files(self) -> List[Path]: 21 | return list(map(lambda f: self / f, os.listdir(str(self)))) 22 | 23 | def ensure_directory(self) -> None: 24 | return os.makedirs(self._path, exist_ok=True) 25 | 26 | def write_text(self, text: str) -> None: 27 | self._path.write_text(text) 28 | 29 | def endswith(self, suffix: str) -> bool: 30 | return str(self).endswith(suffix) 31 | 32 | def is_file(self) -> bool: 33 | return os.path.isfile(self._path) 34 | 35 | def __truediv__(self, other: Union[str, Path]) -> Path: 36 | if isinstance(other, str): 37 | return Path(self._path / other) 38 | else: 39 | return Path(self._path / other._path) 40 | 41 | def __str__(self) -> str: 42 | return str(self._path) 43 | 44 | def __hash__(self) -> int: 45 | return hash(self._path) 46 | 47 | def __eq__(self, other: object) -> bool: 48 | if isinstance(other, Path): 49 | return self._path == other._path 50 | else: 51 | return False 52 | -------------------------------------------------------------------------------- /src/pypi2nix/pip/__init__.py: -------------------------------------------------------------------------------- 1 | from .exceptions import PipFailed 2 | from .implementation import NixPip 3 | from .interface import Pip 4 | from .virtualenv import VirtualenvPip 5 | -------------------------------------------------------------------------------- /src/pypi2nix/pip/base.nix: -------------------------------------------------------------------------------- 1 | { project_dir 2 | , download_cache_dir 3 | , python_version 4 | , extra_build_inputs 5 | }: 6 | 7 | let 8 | pkgs = import {}; 9 | python = builtins.getAttr python_version pkgs; 10 | pypi2nix_bootstrap = pkgs.callPackage ./bootstrap.nix {inherit python;}; 11 | nixpkg_from_name = name: 12 | pkgs.lib.getAttrFromPath (pkgs.lib.splitString "." name) pkgs; 13 | extra_build_inputs_derivations = map nixpkg_from_name extra_build_inputs; 14 | locales = pkgs.lib.optional pkgs.stdenv.isLinux pkgs.glibcLocales; 15 | in pkgs.lib.makeOverridable pkgs.stdenv.mkDerivation rec { 16 | name = "pypi2nix-pip"; 17 | buildInputs = with pkgs; [ 18 | pypi2nix_bootstrap 19 | unzip 20 | gitAndTools.git 21 | mercurial 22 | ] ++ extra_build_inputs_derivations ++ locales; 23 | shellHook = '' 24 | set -e 25 | export TMPDIR=${project_dir} 26 | export GIT_SSL_CAINFO="${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" 27 | export SSL_CERT_FILE="${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" 28 | export PYTHONPATH=${project_dir}/lib 29 | export LANG=en_US.UTF-8 30 | export HOME=${project_dir} 31 | export SOURCE_DATE_EPOCH=315532800 32 | export PYPI2NIX_BOOTSTRAP="${pypi2nix_bootstrap}" 33 | export PIP_CACHE_DIR=${download_cache_dir} 34 | export PIP_EXISTS_ACTION="s" 35 | ''; 36 | } 37 | -------------------------------------------------------------------------------- /src/pypi2nix/pip/bootstrap.nix: -------------------------------------------------------------------------------- 1 | { stdenv, python, fetchurl, fetchgit }: 2 | 3 | let 4 | _fetchSource = jsonValue: 5 | let cleanedJsonValue = builtins.removeAttrs jsonValue [ "__type__" ]; 6 | in (if jsonValue."__type__" == "fetchurl" then 7 | fetchurl cleanedJsonValue 8 | else 9 | fetchgit cleanedJsonValue); 10 | wrappedPip = '' 11 | #!${stdenv.shell} 12 | 13 | exec python -m pip "$@" 14 | ''; 15 | indexJson = with builtins; fromJSON (readFile ../wheels/index.json); 16 | packageOverrides = self: super: { 17 | pip = super.pip.overridePythonAttrs 18 | (old: { src = _fetchSource indexJson.pip; }); 19 | setuptools = super.setuptools.overridePythonAttrs 20 | (old: { src = _fetchSource indexJson.setuptools; }); 21 | wheel = super.wheel.overridePythonAttrs 22 | (old: { src = _fetchSource indexJson.wheel; }); 23 | }; 24 | overriddenPython = python.override { inherit packageOverrides; }; 25 | pythonWithPackages = 26 | overriddenPython.withPackages (pkgs: with pkgs; [ pip setuptools wheel ]); 27 | in pythonWithPackages 28 | -------------------------------------------------------------------------------- /src/pypi2nix/pip/download.nix: -------------------------------------------------------------------------------- 1 | { download_cache_dir 2 | , extra_build_inputs 3 | , project_dir 4 | , python_version 5 | , extra_env 6 | , requirements_files 7 | , destination_directory 8 | , editable_sources_directory 9 | , build_directory 10 | }: 11 | let 12 | pip_base = import ./base.nix { 13 | inherit project_dir download_cache_dir python_version extra_build_inputs; 14 | }; 15 | requirements_files_option = 16 | builtins.concatStringsSep " " (map (x: "-r ${x} ") requirements_files); 17 | in 18 | pip_base.override( old: { 19 | shellHook = old.shellHook + '' 20 | ${extra_env} pip download \ 21 | ${requirements_files_option} \ 22 | --dest ${destination_directory} \ 23 | --src ${editable_sources_directory} \ 24 | --no-binary :all: 25 | ''; 26 | }) 27 | -------------------------------------------------------------------------------- /src/pypi2nix/pip/exceptions.py: -------------------------------------------------------------------------------- 1 | class PipFailed(Exception): 2 | def __init__(self, output: str) -> None: 3 | self.output = output 4 | super().__init__() 5 | -------------------------------------------------------------------------------- /src/pypi2nix/pip/install.nix: -------------------------------------------------------------------------------- 1 | { project_dir 2 | , download_cache_dir 3 | , python_version 4 | , extra_build_inputs 5 | , requirements_files 6 | , target_directory 7 | , sources_directories 8 | }: 9 | let 10 | pip_base = import ./base.nix { 11 | inherit project_dir download_cache_dir python_version extra_build_inputs; 12 | }; 13 | sources_directories_links = 14 | with builtins; 15 | concatStringsSep " " (map (x: "--find-links file://${x}") sources_directories); 16 | in 17 | pip_base.override( old: { 18 | shellHook = old.shellHook + '' 19 | pip install \ 20 | ${builtins.concatStringsSep " " (map (x: "-r ${x} ") requirements_files)} \ 21 | --target=${target_directory} \ 22 | ${sources_directories_links} \ 23 | --find-links file://${project_dir}/wheel \ 24 | --no-index 25 | ''; 26 | }) 27 | -------------------------------------------------------------------------------- /src/pypi2nix/pip/interface.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | from typing import List 4 | 5 | from pypi2nix.path import Path 6 | from pypi2nix.requirement_set import RequirementSet 7 | 8 | 9 | class Pip(metaclass=ABCMeta): 10 | @abstractmethod 11 | def download_sources( 12 | self, requirements: RequirementSet, target_directory: Path 13 | ) -> None: 14 | pass 15 | 16 | @abstractmethod 17 | def build_wheels( 18 | self, 19 | requirements: RequirementSet, 20 | target_directory: Path, 21 | source_directories: List[Path], 22 | ) -> None: 23 | pass 24 | 25 | @abstractmethod 26 | def install( 27 | self, 28 | requirements: RequirementSet, 29 | source_directories: List[Path], 30 | target_directory: Path, 31 | ) -> None: 32 | pass 33 | 34 | @abstractmethod 35 | def freeze(self, python_path: List[Path]) -> str: 36 | pass 37 | -------------------------------------------------------------------------------- /src/pypi2nix/pip/wheel.nix: -------------------------------------------------------------------------------- 1 | { project_dir 2 | , download_cache_dir 3 | , python_version 4 | , extra_build_inputs 5 | , extra_env 6 | , wheels_cache 7 | , requirements_files 8 | , editable_sources_directory 9 | , wheels_dir 10 | , build_directory 11 | , sources 12 | }: 13 | 14 | with builtins; 15 | 16 | let 17 | pip_base = import ./base.nix { 18 | inherit project_dir download_cache_dir python_version extra_build_inputs; 19 | }; 20 | sources_directories_links = 21 | concatStringsSep " " (map (x: "--find-links file://${x}") sources); 22 | find_directory_link = directory: "--find-links ${directory}"; 23 | links = sources ++ wheels_cache ++ [wheels_dir]; 24 | in 25 | pip_base.override( old: { 26 | shellHook = old.shellHook + '' 27 | ${extra_env} pip wheel \ 28 | ${concatStringsSep " " (map find_directory_link links)} \ 29 | ${concatStringsSep " " (map (x: "-r ${x} ") requirements_files)} \ 30 | --src ${editable_sources_directory} \ 31 | --wheel-dir ${wheels_dir} \ 32 | --no-index \ 33 | --prefer-binary 34 | ''; 35 | }) 36 | -------------------------------------------------------------------------------- /src/pypi2nix/project_directory.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | from abc import ABCMeta 3 | from abc import abstractmethod 4 | from sys import stderr 5 | from types import TracebackType 6 | from typing import Optional 7 | from typing import Type 8 | 9 | 10 | class ProjectDirectory(metaclass=ABCMeta): 11 | @abstractmethod 12 | def __enter__(self) -> str: 13 | pass 14 | 15 | @abstractmethod 16 | def __exit__( 17 | self, 18 | exc_type: Optional[Type], 19 | exc_value: Optional[Exception], 20 | traceback: Optional[TracebackType], 21 | ) -> None: 22 | pass 23 | 24 | 25 | class TemporaryProjectDirectory(ProjectDirectory): 26 | def __init__(self) -> None: 27 | self.temporary_directory = tempfile.TemporaryDirectory() 28 | 29 | def __enter__(self) -> str: 30 | return self.temporary_directory.__enter__() 31 | 32 | def __exit__( 33 | self, 34 | exc_type: Optional[Type], 35 | exc_value: Optional[Exception], 36 | traceback: Optional[TracebackType], 37 | ) -> None: 38 | return self.temporary_directory.__exit__(exc_type, exc_value, traceback) 39 | 40 | 41 | class PersistentProjectDirectory(ProjectDirectory): 42 | def __init__(self, path: str) -> None: 43 | self.path = path 44 | 45 | def __enter__(self) -> str: 46 | print( 47 | "WARNING: You have specified the `--build-directory OPTION`.", file=stderr 48 | ) 49 | print("WARNING: It is recommended to not use this flag.", file=stderr) 50 | return self.path 51 | 52 | def __exit__( 53 | self, 54 | exc_type: Optional[Type], 55 | exc_value: Optional[Exception], 56 | traceback: Optional[TracebackType], 57 | ) -> None: 58 | pass 59 | -------------------------------------------------------------------------------- /src/pypi2nix/pypi.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | from functools import lru_cache 4 | from http.client import HTTPException 5 | from typing import Optional 6 | from typing import Union 7 | from urllib.request import urlopen 8 | 9 | from attr import attrib 10 | from attr import attrs 11 | from packaging.version import LegacyVersion 12 | from packaging.version import Version 13 | from packaging.version import parse as parse_version 14 | 15 | from pypi2nix.logger import Logger 16 | from pypi2nix.pypi_package import PypiPackage 17 | from pypi2nix.pypi_release import PypiRelease 18 | from pypi2nix.pypi_release import ReleaseType 19 | from pypi2nix.pypi_release import get_release_type_by_packagetype 20 | 21 | 22 | @attrs(frozen=True) 23 | class Pypi: 24 | _logger: Logger = attrib() 25 | _index: str = attrib(default="https://pypi.org/pypi") 26 | 27 | @lru_cache(maxsize=None) 28 | def get_package(self, name: str) -> PypiPackage: 29 | def get_release_type(package_type: str) -> ReleaseType: 30 | release_type = get_release_type_by_packagetype(package_type) 31 | if release_type is None: 32 | self._logger.warning( 33 | f"Found unexpected `packagetype` entry {package_type} for package `{name}`" 34 | ) 35 | return ReleaseType.UNKNOWN 36 | else: 37 | return release_type 38 | 39 | url = f"{self._index}/{name}/json" 40 | try: 41 | with urlopen(url) as response_buffer: 42 | metadata = json.loads(response_buffer.read().decode("utf-8")) 43 | except HTTPException: 44 | raise PypiFailed( 45 | f"Failed to download metadata information from `{url}`, given package name `{name}`" 46 | ) 47 | releases = { 48 | PypiRelease( 49 | url=data["url"], 50 | sha256_digest=data["digests"]["sha256"], 51 | version=version, 52 | type=get_release_type(data["packagetype"]), 53 | filename=data["filename"], 54 | ) 55 | for version, release_list in metadata["releases"].items() 56 | for data in release_list 57 | } 58 | 59 | return PypiPackage( 60 | name=metadata["info"]["name"], 61 | releases=releases, 62 | version=metadata["info"]["version"], 63 | ) 64 | 65 | def get_source_release(self, name: str, version: str) -> Optional[PypiRelease]: 66 | def version_tag_from_filename(filename: str) -> Union[Version, LegacyVersion]: 67 | extension = "|".join( 68 | map(re.escape, [".tar.gz", ".tar.bz2", ".tar", ".zip", ".tgz"]) 69 | ) 70 | regular_expression = r"{name}-(?P.*)(?P{extension})$".format( 71 | name=re.escape(name), extension=extension 72 | ) 73 | result = re.match(regular_expression, filename) 74 | if result: 75 | return parse_version(result.group("version")) 76 | else: 77 | message = f"Could not guess version of package from url `{filename}`" 78 | self._logger.error(message) 79 | raise PypiFailed(message) 80 | 81 | package = self.get_package(name) 82 | source_releases = [ 83 | release 84 | for release in package.releases 85 | if release.type == ReleaseType.SOURCE 86 | ] 87 | releases_for_version = ( 88 | release 89 | for release in source_releases 90 | if parse_version(release.version) == parse_version(version) 91 | ) 92 | 93 | for release in releases_for_version: 94 | return release 95 | else: 96 | releases_for_version_by_filename = ( 97 | release 98 | for release in source_releases 99 | if version_tag_from_filename(release.filename) == parse_version(version) 100 | ) 101 | for release in releases_for_version_by_filename: 102 | return release 103 | else: 104 | return None 105 | 106 | 107 | class PypiFailed(Exception): 108 | pass 109 | -------------------------------------------------------------------------------- /src/pypi2nix/pypi_package.py: -------------------------------------------------------------------------------- 1 | from typing import Set 2 | 3 | from attr import attrib 4 | from attr import attrs 5 | 6 | from pypi2nix.pypi_release import PypiRelease 7 | 8 | 9 | @attrs 10 | class PypiPackage: 11 | name: str = attrib() 12 | releases: Set[PypiRelease] = attrib() 13 | version: str = attrib() 14 | -------------------------------------------------------------------------------- /src/pypi2nix/pypi_release.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from enum import unique 3 | from typing import Optional 4 | 5 | from attr import attrib 6 | from attr import attrs 7 | 8 | 9 | @unique 10 | class ReleaseType(Enum): 11 | UNKNOWN = 0 12 | SOURCE = 1 13 | WHEEL = 2 14 | EGG = 3 15 | WIN_INSTALLER = 4 16 | RPM = 5 17 | MSI = 6 18 | 19 | 20 | _release_type_mapping = { 21 | "sdist": ReleaseType.SOURCE, 22 | "bdist_wheel": ReleaseType.WHEEL, 23 | "bdist_egg": ReleaseType.EGG, 24 | "bdist_wininst": ReleaseType.WIN_INSTALLER, 25 | "bdist_rpm": ReleaseType.RPM, 26 | "bdist_msi": ReleaseType.MSI, 27 | } 28 | 29 | 30 | def get_release_type_by_packagetype(packagetype: str) -> Optional[ReleaseType]: 31 | return _release_type_mapping.get(packagetype) 32 | 33 | 34 | @attrs(frozen=True) 35 | class PypiRelease: 36 | url: str = attrib() 37 | sha256_digest: str = attrib() 38 | version: str = attrib() 39 | type: ReleaseType = attrib() 40 | filename: str = attrib() 41 | -------------------------------------------------------------------------------- /src/pypi2nix/python_version.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from enum import unique 3 | from typing import Dict 4 | from typing import List 5 | from typing import Optional 6 | 7 | 8 | @unique 9 | class PythonVersion(Enum): 10 | python35 = "python35" 11 | python36 = "python36" 12 | python37 = "python37" 13 | python38 = "python38" 14 | python3 = "python3" 15 | 16 | def nixpkgs_attribute(self) -> str: 17 | return self.value # type: ignore 18 | 19 | def derivation_name(self) -> str: 20 | return self.value # type: ignore 21 | 22 | def major_version(self) -> str: 23 | return self.derivation_name().replace("python", "")[0] 24 | 25 | 26 | _PYTHON_VERSIONS: Dict[str, PythonVersion] = { 27 | "3.5": PythonVersion.python35, 28 | "3.6": PythonVersion.python36, 29 | "3.7": PythonVersion.python37, 30 | "3.8": PythonVersion.python38, 31 | } 32 | 33 | 34 | def python_version_from_version_string(version_string: str) -> Optional[PythonVersion]: 35 | return _PYTHON_VERSIONS.get(version_string) 36 | 37 | 38 | available_python_versions: List[str] = [version.name for version in PythonVersion] 39 | -------------------------------------------------------------------------------- /src/pypi2nix/requirement_parser.py: -------------------------------------------------------------------------------- 1 | from typing import no_type_check 2 | 3 | import parsley 4 | 5 | from pypi2nix.requirement_parser_grammar import requirement_parser_grammar 6 | from pypi2nix.requirements import Logger 7 | from pypi2nix.requirements import Requirement 8 | 9 | 10 | class ParsingFailed(Exception): 11 | def __init__(self, reason: str) -> None: 12 | self.reason = reason 13 | 14 | def __str__(self) -> str: 15 | return self.reason 16 | 17 | 18 | class RequirementParser: 19 | def __init__(self, logger: Logger) -> None: 20 | self._compiled_grammar = None 21 | self.logger = logger 22 | 23 | @no_type_check 24 | def compiled_grammar(self): 25 | with requirement_parser_grammar(self.logger) as grammar: 26 | return grammar 27 | 28 | def parse(self, line: str) -> Requirement: 29 | line = line.strip() 30 | if "\n" in line: 31 | raise ParsingFailed( 32 | "Failed to parse requirement from string `{}`".format(line) 33 | ) 34 | try: 35 | return self.compiled_grammar()(line).specification() # type: ignore 36 | except parsley.ParseError as e: 37 | raise ParsingFailed("{message}".format(message=e.formatError())) 38 | -------------------------------------------------------------------------------- /src/pypi2nix/requirements_collector.py: -------------------------------------------------------------------------------- 1 | """This module implements a class to collect requirements from command line arguments 2 | given to pypi2nix 3 | """ 4 | 5 | import os.path 6 | 7 | from pypi2nix.dependency_graph import DependencyGraph 8 | from pypi2nix.logger import Logger 9 | from pypi2nix.package_source import PathSource 10 | from pypi2nix.requirement_parser import RequirementParser 11 | from pypi2nix.requirement_set import RequirementSet 12 | from pypi2nix.requirements import PathRequirement 13 | from pypi2nix.requirements_file import RequirementsFile 14 | from pypi2nix.sources import Sources 15 | from pypi2nix.target_platform import TargetPlatform 16 | 17 | 18 | class RequirementsCollector: 19 | def __init__( 20 | self, 21 | platform: TargetPlatform, 22 | requirement_parser: RequirementParser, 23 | logger: Logger, 24 | project_directory: str, 25 | base_dependency_graph: DependencyGraph, 26 | ): 27 | self.platform = platform 28 | self.requirement_set = RequirementSet(platform) 29 | self.requirement_parser = requirement_parser 30 | self.logger = logger 31 | self._project_directory = project_directory 32 | self._sources = Sources() 33 | self._base_dependency_graph = base_dependency_graph 34 | 35 | def requirements(self) -> RequirementSet: 36 | return self.requirement_set 37 | 38 | def add_line(self, line: str) -> None: 39 | original_dependency = self.requirement_parser.parse(line) 40 | transitive_requirements = self._base_dependency_graph.get_all_build_dependency_names( 41 | original_dependency 42 | ) 43 | self._add_line_without_dependency_check(line) 44 | for requirement in transitive_requirements: 45 | self._add_line_without_dependency_check(requirement) 46 | 47 | def add_file(self, file_path: str) -> None: 48 | requirements_file = RequirementsFile( 49 | file_path, self._project_directory, self.requirement_parser, self.logger 50 | ) 51 | requirements_file.process() 52 | self._sources.update(requirements_file.sources()) 53 | added_requirements = RequirementSet.from_file( 54 | requirements_file, self.platform, self.requirement_parser, self.logger 55 | ) 56 | transitive_requirements = set() 57 | for requirement in added_requirements: 58 | transitive_requirements.update( 59 | self._base_dependency_graph.get_all_build_dependency_names(requirement) 60 | ) 61 | for line in transitive_requirements: 62 | self._add_line_without_dependency_check(line) 63 | self.requirement_set += added_requirements 64 | 65 | def sources(self) -> Sources: 66 | sources = Sources() 67 | sources.update(self.requirement_set.sources()) 68 | sources.update(self._sources) 69 | return sources 70 | 71 | def _add_line_without_dependency_check(self, line: str) -> None: 72 | requirement = self.requirement_parser.parse(line) 73 | if isinstance(requirement, PathRequirement): 74 | requirement = requirement.change_path( 75 | lambda path: self._handle_requirements_path( 76 | name=requirement.name(), path=path 77 | ) 78 | ) 79 | self.requirement_set.add(requirement) 80 | 81 | def _handle_requirements_path(self, name: str, path: str) -> str: 82 | self._sources.add(name, PathSource(path)) 83 | return os.path.abspath(path) 84 | -------------------------------------------------------------------------------- /src/pypi2nix/sources.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | from typing import List 3 | from typing import Tuple 4 | 5 | from pypi2nix.package_source import PackageSource 6 | 7 | 8 | class Sources: 9 | def __init__(self) -> None: 10 | self.sources: Dict[str, PackageSource] = dict() 11 | 12 | def add(self, name: str, source: PackageSource) -> None: 13 | self.sources[name] = source 14 | 15 | def __contains__(self, item: str) -> bool: 16 | return item in self.sources 17 | 18 | def __getitem__(self, item_name: str) -> PackageSource: 19 | return self.sources[item_name] 20 | 21 | def update(self, other_sources: "Sources") -> None: 22 | self.sources = dict(self.sources, **other_sources.sources) 23 | 24 | def items(self) -> List[Tuple[str, PackageSource]]: 25 | return list(self.sources.items()) 26 | 27 | def __len__(self) -> int: 28 | return len(self.sources) 29 | -------------------------------------------------------------------------------- /src/pypi2nix/templates/generated.nix.j2: -------------------------------------------------------------------------------- 1 | "{{ name }}" = python.mkDerivation { 2 | name = "{{ name }}-{{ version }}"; 3 | src = {{ fetch_expression }}; 4 | doCheck = commonDoCheck; 5 | format = "{{ package_format }}"; 6 | buildInputs = commonBuildInputs ++ {{ buildInputs }}; 7 | propagatedBuildInputs = {{ propagatedBuildInputs }}; 8 | meta = with pkgs.stdenv.lib; { 9 | homepage = "{{ homepage }}"; 10 | license = {{ license }}; 11 | description = "{{ description }}"; 12 | }; 13 | }; 14 | -------------------------------------------------------------------------------- /src/pypi2nix/templates/overrides.nix.j2: -------------------------------------------------------------------------------- 1 | { pkgs, python }: 2 | 3 | self: super: { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /src/pypi2nix/templates/prefetch-github.nix.j2: -------------------------------------------------------------------------------- 1 | let 2 | pkgs = import {}; 3 | src = pkgs.fetchFromGitHub { 4 | owner = "{{ owner }}"; 5 | repo = "{{ repo }}"; 6 | rev = "{{ rev }}"; 7 | sha256 = "{{ fake_hash }}"; 8 | }; 9 | in 10 | import "${src}/overrides.nix" { inherit pkgs; python = pkgs.python3; } 11 | -------------------------------------------------------------------------------- /src/pypi2nix/templates/requirements.nix.j2: -------------------------------------------------------------------------------- 1 | # generated using pypi2nix tool (version: {{ version }}) 2 | # See more at: https://github.com/nix-community/pypi2nix 3 | # 4 | # COMMAND: 5 | # pypi2nix {{ command_arguments }} 6 | # 7 | 8 | { pkgs ? import {}, 9 | overrides ? ({ pkgs, python }: self: super: {}) 10 | }: 11 | 12 | let 13 | 14 | inherit (pkgs) makeWrapper; 15 | inherit (pkgs.stdenv.lib) fix' extends inNixShell; 16 | 17 | pythonPackages = 18 | import "${toString pkgs.path}/pkgs/top-level/python-packages.nix" { 19 | inherit pkgs; 20 | inherit (pkgs) stdenv; 21 | python = pkgs.{{ python_version }}; 22 | }; 23 | 24 | commonBuildInputs = {{ extra_build_inputs }}; 25 | commonDoCheck = {{ enable_tests }}; 26 | 27 | withPackages = pkgs': 28 | let 29 | pkgs = builtins.removeAttrs pkgs' ["__unfix__"]; 30 | interpreterWithPackages = selectPkgsFn: pythonPackages.buildPythonPackage { 31 | name = "{{ python_version }}-interpreter"; 32 | buildInputs = [ makeWrapper ] ++ (selectPkgsFn pkgs); 33 | buildCommand = '' 34 | mkdir -p $out/bin 35 | ln -s ${pythonPackages.python.interpreter} \ 36 | $out/bin/${pythonPackages.python.executable} 37 | for dep in ${builtins.concatStringsSep " " 38 | (selectPkgsFn pkgs)}; do 39 | if [ -d "$dep/bin" ]; then 40 | for prog in "$dep/bin/"*; do 41 | if [ -x "$prog" ] && [ -f "$prog" ]; then 42 | ln -s $prog $out/bin/`basename $prog` 43 | fi 44 | done 45 | fi 46 | done 47 | for prog in "$out/bin/"*; do 48 | wrapProgram "$prog" --prefix PYTHONPATH : "$PYTHONPATH" 49 | done 50 | pushd $out/bin 51 | ln -s ${pythonPackages.python.executable} python 52 | ln -s ${pythonPackages.python.executable} \ 53 | python{{ python_major_version }} 54 | popd 55 | ''; 56 | passthru.interpreter = pythonPackages.python; 57 | }; 58 | 59 | interpreter = interpreterWithPackages builtins.attrValues; 60 | in { 61 | __old = pythonPackages; 62 | inherit interpreter; 63 | inherit interpreterWithPackages; 64 | mkDerivation = args: pythonPackages.buildPythonPackage (args // { 65 | nativeBuildInputs = (args.nativeBuildInputs or []) ++ args.buildInputs; 66 | }); 67 | packages = pkgs; 68 | overrideDerivation = drv: f: 69 | pythonPackages.buildPythonPackage ( 70 | drv.drvAttrs // f drv.drvAttrs // { meta = drv.meta; } 71 | ); 72 | withPackages = pkgs'': 73 | withPackages (pkgs // pkgs''); 74 | }; 75 | 76 | python = withPackages {}; 77 | 78 | generated = self: { 79 | {{ generated_package_nix }} 80 | }; 81 | localOverridesFile = {{ overrides_file }}; 82 | localOverrides = import localOverridesFile { inherit pkgs python; }; 83 | commonOverrides = [ 84 | {{ common_overrides }} 85 | ]; 86 | paramOverrides = [ 87 | (overrides { inherit pkgs python; }) 88 | ]; 89 | allOverrides = 90 | (if (builtins.pathExists localOverridesFile) 91 | then [localOverrides] else [] ) ++ commonOverrides ++ paramOverrides; 92 | 93 | in python.withPackages 94 | (fix' (pkgs.lib.fold 95 | extends 96 | generated 97 | allOverrides 98 | ) 99 | ) 100 | -------------------------------------------------------------------------------- /src/pypi2nix/version.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | HERE = os.path.dirname(__file__) 4 | VERSION_FILE = os.path.join(HERE, "VERSION") 5 | with open(VERSION_FILE) as handle: 6 | pypi2nix_version = handle.read().strip() 7 | -------------------------------------------------------------------------------- /src/pypi2nix/wheels/__init__.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os.path 3 | from collections import namedtuple 4 | from contextlib import contextmanager 5 | from typing import Any 6 | from typing import Dict 7 | from typing import Iterator 8 | from typing import Union 9 | 10 | from attr import attrib 11 | from attr import attrs 12 | from jsonschema import ValidationError 13 | from jsonschema import validate 14 | 15 | from pypi2nix.logger import Logger 16 | 17 | from .schema import GIT_SCHEMA 18 | from .schema import INDEX_SCHEMA 19 | from .schema import URL_SCHEMA 20 | 21 | 22 | @attrs 23 | class Index: 24 | UrlEntry = namedtuple("UrlEntry", ["url", "sha256"]) 25 | GitEntry = namedtuple("GitEntry", ["url", "sha256", "rev"]) 26 | Entry = Union[UrlEntry, GitEntry] 27 | 28 | logger: Logger = attrib() 29 | path: str = attrib(default=os.path.join(os.path.dirname(__file__), "index.json")) 30 | 31 | def __getitem__(self, key: str) -> "Index.Entry": 32 | with self._index_json() as index: 33 | entry = index[key] 34 | if self._is_schema_valid(entry, URL_SCHEMA): 35 | return Index.UrlEntry(url=entry["url"], sha256=entry["sha256"]) 36 | elif self._is_schema_valid(entry, GIT_SCHEMA): 37 | return Index.GitEntry( 38 | url=entry["url"], sha256=entry["sha256"], rev=entry["rev"] 39 | ) 40 | else: 41 | raise Exception() 42 | 43 | def __setitem__(self, key: str, value: "Index.Entry") -> None: 44 | with self._index_json(write=True) as index: 45 | if isinstance(value, self.UrlEntry): 46 | index[key] = { 47 | "url": value.url, 48 | "sha256": value.sha256, 49 | "__type__": "fetchurl", 50 | } 51 | if isinstance(value, self.GitEntry): 52 | index[key] = { 53 | "url": value.url, 54 | "sha256": value.sha256, 55 | "rev": value.rev, 56 | "__type__": "fetchgit", 57 | } 58 | 59 | def is_valid(self) -> bool: 60 | with self._index_json() as index: 61 | return self._is_schema_valid(index, INDEX_SCHEMA) 62 | 63 | @contextmanager 64 | def _index_json(self, write: bool = False) -> Iterator[Dict[str, Dict[str, str]]]: 65 | with open(self.path) as f: 66 | index = json.load(f) 67 | yield index 68 | if write: 69 | with open(self.path, "w") as f: 70 | json.dump(index, f, sort_keys=True, indent=4) 71 | 72 | def _is_schema_valid(self, json_value: Any, schema: Any) -> bool: 73 | try: 74 | validate(json_value, schema) 75 | except ValidationError as e: 76 | self.logger.error(str(e)) 77 | return False 78 | else: 79 | return True 80 | -------------------------------------------------------------------------------- /src/pypi2nix/wheels/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "pip": { 3 | "__type__": "fetchurl", 4 | "sha256": "7db0c8ea4c7ea51c8049640e8e6e7fde949de672bfa4949920675563a5a6967f", 5 | "url": "https://files.pythonhosted.org/packages/8e/76/66066b7bc71817238924c7e4b448abdb17eb0c92d645769c223f9ace478f/pip-20.0.2.tar.gz" 6 | }, 7 | "setuptools": { 8 | "__type__": "fetchurl", 9 | "sha256": "89c6e6011ec2f6d57d43a3f9296c4ef022c2cbf49bab26b407fe67992ae3397f", 10 | "url": "https://files.pythonhosted.org/packages/68/75/d1d7b7340b9eb6e0388bf95729e63c410b381eb71fe8875cdfd949d8f9ce/setuptools-45.2.0.zip" 11 | }, 12 | "wheel": { 13 | "__type__": "fetchurl", 14 | "sha256": "8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96", 15 | "url": "https://files.pythonhosted.org/packages/75/28/521c6dc7fef23a68368efefdcd682f5b3d1d58c2b90b06dc1d0b805b51ae/wheel-0.34.2.tar.gz" 16 | } 17 | } -------------------------------------------------------------------------------- /src/pypi2nix/wheels/schema.py: -------------------------------------------------------------------------------- 1 | URL_SCHEMA = { 2 | "type": "object", 3 | "properties": { 4 | "sha256": {"type": "string"}, 5 | "url": {"type": "string"}, 6 | "__type__": {"const": "fetchurl",}, 7 | }, 8 | "required": ["sha256", "url", "__type__"], 9 | } 10 | 11 | GIT_SCHEMA = { 12 | "type": "object", 13 | "properties": { 14 | "sha256": {"type": "string"}, 15 | "url": {"type": "string"}, 16 | "rev": {"type": "string"}, 17 | "__type__": {"const": "fetchgit"}, 18 | }, 19 | "required": ["sha256", "url", "rev", "__type__"], 20 | } 21 | 22 | INDEX_ITEM_SCHEMA = {"anyOf": [URL_SCHEMA, GIT_SCHEMA,]} 23 | 24 | INDEX_SCHEMA = { 25 | "type": "object", 26 | "additionalProperties": INDEX_ITEM_SCHEMA, 27 | } 28 | -------------------------------------------------------------------------------- /unittests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/__init__.py -------------------------------------------------------------------------------- /unittests/data/flit-1.3-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/flit-1.3-py3-none-any.whl -------------------------------------------------------------------------------- /unittests/data/flit-1.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/flit-1.3.tar.gz -------------------------------------------------------------------------------- /unittests/data/jsonschema-3.0.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/jsonschema-3.0.1.tar.gz -------------------------------------------------------------------------------- /unittests/data/package1-1.0-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package1-1.0-py3-none-any.whl -------------------------------------------------------------------------------- /unittests/data/package1-1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package1-1.0.tar.gz -------------------------------------------------------------------------------- /unittests/data/package1/package1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package1/package1.tar.gz -------------------------------------------------------------------------------- /unittests/data/package1/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | 3 | name = package1 4 | version = 1.0 5 | -------------------------------------------------------------------------------- /unittests/data/package1/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /unittests/data/package2-1.0-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package2-1.0-py3-none-any.whl -------------------------------------------------------------------------------- /unittests/data/package2-1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package2-1.0.tar.gz -------------------------------------------------------------------------------- /unittests/data/package2/package2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package2/package2.tar.gz -------------------------------------------------------------------------------- /unittests/data/package2/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | 3 | name = package2 4 | version = 1.0 5 | 6 | [options] 7 | install_requires = package1 -------------------------------------------------------------------------------- /unittests/data/package2/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /unittests/data/package3-1.0-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package3-1.0-py3-none-any.whl -------------------------------------------------------------------------------- /unittests/data/package3-1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package3-1.0.tar.gz -------------------------------------------------------------------------------- /unittests/data/package3/package3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package3/package3.tar.gz -------------------------------------------------------------------------------- /unittests/data/package3/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | 3 | name = package3 4 | version = 1.0 5 | 6 | [options.extras_require] 7 | myextra = 8 | package1 9 | other_platform = 10 | package2; python_version == "1.0" -------------------------------------------------------------------------------- /unittests/data/package3/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /unittests/data/package4-1.0-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package4-1.0-py3-none-any.whl -------------------------------------------------------------------------------- /unittests/data/package4-1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package4-1.0.tar.gz -------------------------------------------------------------------------------- /unittests/data/package4/package4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/package4/package4.tar.gz -------------------------------------------------------------------------------- /unittests/data/package4/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | 3 | name = package4 4 | version = 1.0 5 | 6 | [options] 7 | install_requires = package3[myextra] 8 | -------------------------------------------------------------------------------- /unittests/data/package4/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /unittests/data/setupcfg-package-1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/setupcfg-package-1.0.tar.gz -------------------------------------------------------------------------------- /unittests/data/setupcfg-package/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | 3 | name = setupcfg-package 4 | version = 1.0 5 | 6 | [options] 7 | 8 | setup_requires = 9 | requests <=1.3.2.>2.3 10 | install_requires = 11 | requests 12 | 13 | [options.extras_require] 14 | testing = 15 | pytest -------------------------------------------------------------------------------- /unittests/data/setupcfg-package/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /unittests/data/setupcfg-package/setupcfg-package.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/setupcfg-package/setupcfg-package.tar.gz -------------------------------------------------------------------------------- /unittests/data/setupcfg_package-1.0-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/setupcfg_package-1.0-py3-none-any.whl -------------------------------------------------------------------------------- /unittests/data/setuptools-41.2.0.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/setuptools-41.2.0.zip -------------------------------------------------------------------------------- /unittests/data/shell_environment.nix: -------------------------------------------------------------------------------- 1 | { dummy_argument ? "hello" }: 2 | 3 | let 4 | pkgs = import {}; 5 | in 6 | pkgs."${dummy_argument}" 7 | -------------------------------------------------------------------------------- /unittests/data/six-1.12.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/six-1.12.0.tar.gz -------------------------------------------------------------------------------- /unittests/data/spacy-2.1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/spacy-2.1.0.tar.gz -------------------------------------------------------------------------------- /unittests/data/test.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/test.tar.bz2 -------------------------------------------------------------------------------- /unittests/data/test.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/test.tar.gz -------------------------------------------------------------------------------- /unittests/data/test.txt: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /unittests/data/test.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/test.zip -------------------------------------------------------------------------------- /unittests/data/wheel-0.33.6-py2.py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/data/wheel-0.33.6-py2.py3-none-any.whl -------------------------------------------------------------------------------- /unittests/logger.py: -------------------------------------------------------------------------------- 1 | from pypi2nix.logger import Logger 2 | from pypi2nix.logger import ProxyLogger 3 | from pypi2nix.logger import StreamLogger 4 | 5 | 6 | def get_logger_output(logger: Logger) -> str: 7 | def get_inner_logger(logger: Logger) -> StreamLogger: 8 | if isinstance(logger, StreamLogger): 9 | return logger 10 | elif isinstance(logger, ProxyLogger): 11 | inner_logger = logger.get_target_logger() 12 | if inner_logger is None: 13 | raise Exception("ProxyLogger is not connected, cannot get output") 14 | else: 15 | return get_inner_logger(inner_logger) 16 | else: 17 | raise Exception("Unhandled Logger implementation", type(logger)) 18 | 19 | logger = get_inner_logger(logger) 20 | logger.output.seek(0) 21 | output = logger.output.read() 22 | logger.output.seek(0, 2) 23 | return output 24 | -------------------------------------------------------------------------------- /unittests/package_generator.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import subprocess 3 | from tempfile import TemporaryDirectory 4 | from typing import List 5 | 6 | from attr import attrib 7 | from attr import attrs 8 | 9 | from pypi2nix.archive import Archive 10 | from pypi2nix.logger import Logger 11 | from pypi2nix.path import Path 12 | from pypi2nix.requirement_parser import RequirementParser 13 | from pypi2nix.source_distribution import SourceDistribution 14 | 15 | from .templates import render_template 16 | 17 | 18 | @attrs 19 | class PackageGenerator: 20 | """Generate source distributions on for testing 21 | 22 | This class aims to provide an easy to use way of generating test 23 | data. Since pypi2nix deals a lot with python packages it is 24 | necessary have python packages available for testing. 25 | """ 26 | 27 | _target_directory: Path = attrib() 28 | _requirement_parser: RequirementParser = attrib() 29 | _logger: Logger = attrib() 30 | 31 | def generate_setuptools_package( 32 | self, name: str, version: str = "1.0", install_requires: List[str] = [] 33 | ) -> SourceDistribution: 34 | with TemporaryDirectory() as directory_path_string: 35 | build_directory: Path = Path(directory_path_string) 36 | self._generate_setup_py(build_directory, name=name, version=version) 37 | self._generate_setup_cfg( 38 | build_directory, 39 | name=name, 40 | version=version, 41 | install_requires=install_requires, 42 | ) 43 | built_distribution_archive = self._build_package( 44 | build_directory=build_directory, name=name, version=version 45 | ) 46 | source_distribution = SourceDistribution.from_archive( 47 | built_distribution_archive, 48 | logger=self._logger, 49 | requirement_parser=self._requirement_parser, 50 | ) 51 | self._move_package_target_directory(built_distribution_archive) 52 | return source_distribution 53 | 54 | def _generate_setup_py( 55 | self, target_directory: Path, name: str, version: str 56 | ) -> None: 57 | content = render_template(Path("setup.py"), context={},) 58 | (target_directory / "setup.py").write_text(content) 59 | 60 | def _generate_setup_cfg( 61 | self, 62 | target_directory: Path, 63 | name: str, 64 | version: str, 65 | install_requires: List[str], 66 | ) -> None: 67 | content = render_template( 68 | Path("setup.cfg"), 69 | context={ 70 | "name": name, 71 | "version": version, 72 | "install_requires": install_requires, 73 | }, 74 | ) 75 | (target_directory / "setup.cfg").write_text(content) 76 | 77 | def _build_package(self, build_directory: Path, name: str, version: str) -> Archive: 78 | subprocess.run( 79 | ["python", "setup.py", "sdist"], cwd=str(build_directory), check=True 80 | ) 81 | tar_gz_path = build_directory / "dist" / f"{name}-{version}.tar.gz" 82 | return Archive(path=str(tar_gz_path)) 83 | 84 | def _move_package_target_directory(self, distribution_archive: Archive) -> None: 85 | shutil.copy(distribution_archive.path, str(self._target_directory)) 86 | -------------------------------------------------------------------------------- /unittests/pip/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nix-community/pypi2nix/4a5a9d399e960d85b3e37b6a564bcbe655287e3c/unittests/pip/__init__.py -------------------------------------------------------------------------------- /unittests/pip/conftest.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import venv 3 | 4 | import pytest 5 | 6 | from pypi2nix.logger import Logger 7 | from pypi2nix.nix import Nix 8 | from pypi2nix.path import Path 9 | from pypi2nix.pip import NixPip 10 | from pypi2nix.pip import VirtualenvPip 11 | from pypi2nix.requirement_parser import RequirementParser 12 | from pypi2nix.target_platform import TargetPlatform 13 | 14 | 15 | @pytest.fixture(params=("nix", "venv")) 16 | def pip( 17 | request, 18 | nix: Nix, 19 | project_dir: str, 20 | current_platform: TargetPlatform, 21 | logger: Logger, 22 | requirement_parser: RequirementParser, 23 | ): 24 | if request.param == "nix": 25 | return NixPip( 26 | nix=nix, 27 | project_directory=Path(project_dir), 28 | extra_build_inputs=[], 29 | extra_env="", 30 | wheels_cache=[], 31 | target_platform=current_platform, 32 | logger=logger, 33 | requirement_parser=requirement_parser, 34 | ) 35 | else: 36 | pip = VirtualenvPip( 37 | logger=logger, 38 | target_platform=current_platform, 39 | target_directory=os.path.join(project_dir, "venv-pip"), 40 | env_builder=venv.EnvBuilder(with_pip=True), 41 | requirement_parser=requirement_parser, 42 | ) 43 | pip.prepare_virtualenv() 44 | return pip 45 | -------------------------------------------------------------------------------- /unittests/pip/test_download.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | from typing import List 4 | 5 | from pypi2nix.path import Path 6 | from pypi2nix.pip import Pip 7 | from pypi2nix.requirement_parser import RequirementParser 8 | from pypi2nix.requirement_set import RequirementSet 9 | from pypi2nix.target_platform import TargetPlatform 10 | 11 | from ..switches import nix 12 | 13 | 14 | def list_files(dirname: str) -> List[str]: 15 | return [ 16 | candidate 17 | for candidate in os.listdir(dirname) 18 | if os.path.isfile(os.path.join(dirname, candidate)) 19 | ] 20 | 21 | 22 | @nix 23 | def test_pip_downloads_sources_to_target_directory( 24 | pip: Pip, 25 | project_dir: str, 26 | current_platform: TargetPlatform, 27 | requirement_parser: RequirementParser, 28 | ): 29 | download_path = Path(project_dir) / "download" 30 | requirements = RequirementSet(current_platform) 31 | requirements.add(requirement_parser.parse("six")) 32 | pip.download_sources(requirements=requirements, target_directory=download_path) 33 | assert download_path.list_files() 34 | 35 | 36 | @nix 37 | def test_pip_downloads_nothing_when_no_requirements_are_given( 38 | pip: Pip, download_dir: Path, current_platform: TargetPlatform 39 | ): 40 | pip.download_sources( 41 | requirements=RequirementSet(current_platform), target_directory=download_dir 42 | ) 43 | assert not download_dir.list_files() 44 | -------------------------------------------------------------------------------- /unittests/pip/test_freeze.py: -------------------------------------------------------------------------------- 1 | from pypi2nix.path import Path 2 | from pypi2nix.pip import Pip 3 | from pypi2nix.requirement_parser import RequirementParser 4 | from pypi2nix.requirement_set import RequirementSet 5 | from pypi2nix.target_platform import TargetPlatform 6 | 7 | from ..switches import nix 8 | 9 | 10 | @nix 11 | def test_freeze_on_empty_environment_yields_empty_file(pip: Pip): 12 | frozen_requirements = pip.freeze([]) 13 | assert not frozen_requirements.strip() 14 | 15 | 16 | @nix 17 | def test_freeze_respects_additional_python_path( 18 | pip: Pip, 19 | project_dir: str, 20 | current_platform: TargetPlatform, 21 | requirement_parser: RequirementParser, 22 | download_dir: Path, 23 | ): 24 | prefix = Path(project_dir) / "custom-prefix" 25 | requirements = RequirementSet(current_platform) 26 | requirements.add(requirement_parser.parse("six")) 27 | pip.download_sources(requirements, download_dir) 28 | pip.install( 29 | requirements, target_directory=prefix, source_directories=[download_dir] 30 | ) 31 | freeze_without_six = pip.freeze([]) 32 | freeze_with_six = pip.freeze(python_path=[prefix]) 33 | assert len(freeze_without_six) < len(freeze_with_six) 34 | -------------------------------------------------------------------------------- /unittests/pip/test_install.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | 4 | from pypi2nix.path import Path 5 | from pypi2nix.pip import Pip 6 | from pypi2nix.requirement_parser import RequirementParser 7 | from pypi2nix.requirement_set import RequirementSet 8 | from pypi2nix.target_platform import TargetPlatform 9 | 10 | from ..switches import nix 11 | 12 | 13 | @nix 14 | def test_install_six_yields_non_empty_freeze_output( 15 | pip: Pip, 16 | project_dir: str, 17 | download_dir: Path, 18 | current_platform: TargetPlatform, 19 | requirement_parser, 20 | ): 21 | lib_dir = Path(os.path.join(project_dir, "lib")) 22 | requirements = RequirementSet(current_platform) 23 | requirements.add(requirement_parser.parse("six")) 24 | pip.download_sources(requirements, download_dir) 25 | pip.install( 26 | requirements, source_directories=[download_dir], target_directory=lib_dir 27 | ) 28 | assert pip.freeze([lib_dir]) 29 | 30 | 31 | @nix 32 | def test_install_to_target_directory_does_not_install_to_default_directory( 33 | pip: Pip, 34 | project_dir: str, 35 | download_dir: Path, 36 | current_platform: TargetPlatform, 37 | requirement_parser: RequirementParser, 38 | ): 39 | requirements = RequirementSet(current_platform) 40 | requirements.add(requirement_parser.parse("six")) 41 | target_directory = Path(project_dir) / "target-directory" 42 | target_directory.ensure_directory() 43 | pip.download_sources(requirements, download_dir) 44 | 45 | assert not target_directory.list_files() 46 | 47 | pip.install( 48 | requirements, 49 | source_directories=[download_dir], 50 | target_directory=target_directory, 51 | ) 52 | 53 | assert target_directory.list_files() 54 | 55 | 56 | @nix 57 | def test_install_does_not_install_anything_with_empty_requirements( 58 | pip: Pip, project_dir: str, current_platform: TargetPlatform 59 | ): 60 | target_directory = Path(project_dir) / "target_dir" 61 | target_directory.ensure_directory() 62 | pip.install(RequirementSet(current_platform), [], target_directory) 63 | assert not target_directory.list_files() 64 | -------------------------------------------------------------------------------- /unittests/pip/test_virtualenv_pip.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | import venv 4 | 5 | import pytest 6 | 7 | from pypi2nix.logger import Logger 8 | from pypi2nix.path import Path 9 | from pypi2nix.pip import PipFailed 10 | from pypi2nix.pip import VirtualenvPip 11 | from pypi2nix.requirement_parser import RequirementParser 12 | from pypi2nix.requirement_set import RequirementSet 13 | from pypi2nix.target_platform import TargetPlatform 14 | 15 | 16 | @pytest.fixture 17 | def pip_without_index( 18 | logger: Logger, 19 | current_platform: TargetPlatform, 20 | project_dir: str, 21 | wheel_distribution_archive_path: str, 22 | requirement_parser: RequirementParser, 23 | ) -> VirtualenvPip: 24 | pip = VirtualenvPip( 25 | logger=logger, 26 | target_platform=current_platform, 27 | target_directory=os.path.join(project_dir, "pip-without-index-venv"), 28 | env_builder=venv.EnvBuilder(with_pip=True), 29 | no_index=True, 30 | wheel_distribution_path=wheel_distribution_archive_path, 31 | requirement_parser=requirement_parser, 32 | ) 33 | pip.prepare_virtualenv() 34 | return pip 35 | 36 | 37 | @pytest.fixture 38 | def pip_from_data_directory( 39 | logger: Logger, 40 | current_platform: TargetPlatform, 41 | project_dir: str, 42 | wheel_distribution_archive_path: str, 43 | data_directory: str, 44 | requirement_parser: RequirementParser, 45 | ) -> VirtualenvPip: 46 | pip = VirtualenvPip( 47 | logger=logger, 48 | target_platform=current_platform, 49 | target_directory=os.path.join(project_dir, "pip-without-index-venv"), 50 | env_builder=venv.EnvBuilder(with_pip=True), 51 | no_index=True, 52 | wheel_distribution_path=wheel_distribution_archive_path, 53 | find_links=[data_directory], 54 | requirement_parser=requirement_parser, 55 | ) 56 | pip.prepare_virtualenv() 57 | return pip 58 | 59 | 60 | def test_pip_without_index_cannot_download_six( 61 | pip_without_index: VirtualenvPip, 62 | download_dir: Path, 63 | requirement_parser: RequirementParser, 64 | current_platform: TargetPlatform, 65 | ) -> None: 66 | requirements = RequirementSet(current_platform) 67 | requirements.add(requirement_parser.parse("six")) 68 | with pytest.raises(PipFailed): 69 | pip_without_index.download_sources(requirements, download_dir) 70 | 71 | 72 | def test_pip_without_index_cannot_be_prepared_without_wheel_supplied( 73 | logger: Logger, 74 | current_platform: TargetPlatform, 75 | project_dir: str, 76 | requirement_parser: RequirementParser, 77 | ) -> None: 78 | pip = VirtualenvPip( 79 | logger=logger, 80 | target_platform=current_platform, 81 | target_directory=os.path.join(project_dir, "pip-without-index-venv"), 82 | env_builder=venv.EnvBuilder(with_pip=True), 83 | no_index=True, 84 | requirement_parser=requirement_parser, 85 | ) 86 | with pytest.raises(PipFailed): 87 | pip.prepare_virtualenv() 88 | 89 | 90 | def test_pip_with_data_directory_index_can_download_six( 91 | pip_from_data_directory: VirtualenvPip, 92 | download_dir: Path, 93 | requirement_parser: RequirementParser, 94 | current_platform: TargetPlatform, 95 | ) -> None: 96 | requirements = RequirementSet(current_platform) 97 | requirements.add(requirement_parser.parse("six")) 98 | pip_from_data_directory.download_sources(requirements, download_dir) 99 | 100 | 101 | def test_that_set_environment_variable_undoes_changes_when_exiting( 102 | pip_without_index: VirtualenvPip, 103 | ): 104 | old_environment = dict(os.environ) 105 | with pip_without_index._set_environment_variable({"test": "definitly_unset"}): 106 | pass 107 | assert dict(os.environ) == old_environment 108 | -------------------------------------------------------------------------------- /unittests/pip/test_wheel.py: -------------------------------------------------------------------------------- 1 | from pypi2nix.path import Path 2 | from pypi2nix.pip import Pip 3 | from pypi2nix.requirement_parser import RequirementParser 4 | from pypi2nix.requirement_set import RequirementSet 5 | from pypi2nix.target_platform import TargetPlatform 6 | 7 | from ..switches import nix 8 | 9 | 10 | @nix 11 | def test_pip_can_install_wheels_previously_downloaded( 12 | pip: Pip, 13 | project_dir: str, 14 | current_platform: TargetPlatform, 15 | requirement_parser: RequirementParser, 16 | download_dir: Path, 17 | wheels_dir: Path, 18 | ): 19 | requirements = RequirementSet(current_platform) 20 | requirements.add(requirement_parser.parse("six")) 21 | pip.download_sources(requirements, download_dir) 22 | pip.build_wheels( 23 | requirements=requirements, 24 | source_directories=[download_dir], 25 | target_directory=wheels_dir, 26 | ) 27 | assert wheels_dir.list_files() 28 | assert any(map(lambda x: x.endswith(".whl"), wheels_dir.list_files())) 29 | 30 | 31 | @nix 32 | def test_pip_wheel_does_not_build_wheels_if_requirements_are_empty( 33 | pip: Pip, wheels_dir: Path, download_dir: Path, current_platform: TargetPlatform 34 | ): 35 | pip.build_wheels( 36 | requirements=RequirementSet(current_platform), 37 | target_directory=wheels_dir, 38 | source_directories=[download_dir], 39 | ) 40 | assert not wheels_dir.list_files() 41 | -------------------------------------------------------------------------------- /unittests/regression/test_issue_363.py: -------------------------------------------------------------------------------- 1 | """Regression test for https://github.com/nix-community/pypi2nix/issues/363""" 2 | from pypi2nix.requirement_parser import RequirementParser 3 | 4 | 5 | def test_can_parse_enum_requirement_from_issue_363( 6 | requirement_parser: RequirementParser, 7 | ): 8 | requirement = requirement_parser.parse( 9 | "enum34 (>=1.0.4) ; (python_version=='2.7' or python_version=='2.6' or python_version=='3.3')" 10 | ) 11 | assert requirement.name() == "enum34" 12 | 13 | 14 | def test_can_parse_pyinotify_requirement_from_issue_363( 15 | requirement_parser: RequirementParser, 16 | ): 17 | requirement = requirement_parser.parse( 18 | "pyinotify (>=0.9.6) ; (sys_platform!='win32' and sys_platform!='darwin' and sys_platform!='sunos5')" 19 | ) 20 | assert requirement.name() == "pyinotify" 21 | -------------------------------------------------------------------------------- /unittests/regression/test_issue_394.py: -------------------------------------------------------------------------------- 1 | # https://github.com/nix-community/pypi2nix/issues/394 2 | from pypi2nix.requirement_parser import RequirementParser 3 | 4 | 5 | def test_can_parse_requirements_with_comments(requirement_parser: RequirementParser): 6 | requirement = requirement_parser.parse("requirement # comment") 7 | assert requirement.name() == "requirement" 8 | 9 | 10 | def test_can_parse_given_test_case_from_issue(requirement_parser: RequirementParser): 11 | requirement = requirement_parser.parse("aioredis # my favourite package") 12 | assert requirement.name() == "aioredis" 13 | -------------------------------------------------------------------------------- /unittests/switches.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | nix = pytest.mark.nix 4 | -------------------------------------------------------------------------------- /unittests/templates.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Dict 3 | 4 | import jinja2 5 | 6 | from pypi2nix.path import Path 7 | 8 | HERE = Path(os.path.dirname(__file__)) 9 | 10 | _templates = jinja2.Environment(loader=jinja2.FileSystemLoader(str(HERE / "templates"))) 11 | 12 | 13 | def render_template(template_path: Path, context=Dict[str, str]) -> str: 14 | template = _templates.get_template(str(template_path)) 15 | return template.render(**context) 16 | -------------------------------------------------------------------------------- /unittests/templates/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = {{ name }} 3 | version = {{ version }} 4 | url = https://example.test 5 | author = "Test Author" 6 | author_email = "testauthor@example.test" 7 | 8 | [options] 9 | {% if install_requires %} 10 | install_requires = {% for requirement in install_requires %} 11 | {{ requirement }} 12 | {% endfor %} 13 | {% endif %} -------------------------------------------------------------------------------- /unittests/templates/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /unittests/test_archive.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | 4 | import pytest 5 | 6 | from pypi2nix.archive import Archive 7 | 8 | 9 | @pytest.fixture(params=("tar", "zip", "bz2")) 10 | def archive( 11 | request, test_zip_path: str, test_tar_gz_path: str, test_tar_bz2_path: str 12 | ) -> Archive: 13 | if request.param == "tar": 14 | return Archive(path=test_tar_gz_path) 15 | elif request.param == "bz2": 16 | return Archive(path=test_tar_bz2_path) 17 | else: 18 | return Archive(path=test_zip_path) 19 | 20 | 21 | def test_that_we_can_inspect_the_content_of_an_archive(archive: Archive): 22 | with archive.extracted_files() as directory: 23 | files = tuple(os.listdir(directory)) 24 | assert files == ("test.txt",) 25 | 26 | 27 | def test_that_we_can_inspect_the_content_of_a_wheel(setupcfg_package_wheel_path: str): 28 | archive = Archive(path=setupcfg_package_wheel_path) 29 | with archive.extracted_files() as directory: 30 | assert "setupcfg_package-1.0.dist-info" in os.listdir(directory) 31 | -------------------------------------------------------------------------------- /unittests/test_dependency_graph_serialization.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | 3 | from hypothesis import given 4 | from hypothesis.strategies import booleans 5 | from hypothesis.strategies import composite 6 | from hypothesis.strategies import integers 7 | from hypothesis.strategies import lists 8 | from hypothesis.strategies import text 9 | 10 | from pypi2nix.dependency_graph import CyclicDependencyOccured 11 | from pypi2nix.dependency_graph import DependencyGraph 12 | from pypi2nix.external_dependencies import ExternalDependency 13 | from pypi2nix.logger import StreamLogger 14 | from pypi2nix.requirements import VersionRequirement 15 | 16 | logger = StreamLogger(output=StringIO()) 17 | 18 | 19 | @composite 20 | def requirement(draw, name=text(min_size=1)): 21 | return VersionRequirement( 22 | name=draw(name), 23 | logger=logger, 24 | versions=[], 25 | extras=set(), 26 | environment_markers=None, 27 | ) 28 | 29 | 30 | @composite 31 | def external_dependency(draw, attribute_name=text(min_size=1)): 32 | return ExternalDependency(attribute_name=draw(attribute_name)) 33 | 34 | 35 | @composite 36 | def dependency_graph( 37 | draw, 38 | python_packages=lists(requirement(), unique_by=lambda x: x.name()), 39 | external_dependencies=lists(external_dependency()), 40 | is_runtime_dependency=booleans(), 41 | selections=integers(), 42 | ): 43 | graph = DependencyGraph() 44 | packages = draw(python_packages) 45 | if not packages: 46 | return graph 47 | for package in packages: 48 | index = draw(selections) % len(packages) 49 | try: 50 | if draw(is_runtime_dependency): 51 | graph.set_runtime_dependency( 52 | dependent=package, dependency=packages[index] 53 | ) 54 | else: 55 | graph.set_buildtime_dependency( 56 | dependent=package, dependency=packages[index] 57 | ) 58 | except CyclicDependencyOccured: 59 | continue 60 | for dependency in draw(external_dependencies): 61 | graph.set_external_dependency( 62 | dependent=packages[draw(selections) % len(packages)], dependency=dependency 63 | ) 64 | return graph 65 | 66 | 67 | @given(dependency_graph=dependency_graph()) 68 | def test_equality_to_self(dependency_graph): 69 | assert dependency_graph == dependency_graph 70 | 71 | 72 | def test_equality_of_empty_graphs(): 73 | assert DependencyGraph() == DependencyGraph() 74 | 75 | 76 | @given(dependency_graph=dependency_graph()) 77 | def test_serialization_and_deserialization_leads_to_identity( 78 | dependency_graph: DependencyGraph, 79 | ): 80 | assert DependencyGraph.deserialize(dependency_graph.serialize()) == dependency_graph 81 | -------------------------------------------------------------------------------- /unittests/test_environment_marker.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pypi2nix.environment_marker import EnvironmentMarker 4 | 5 | 6 | @pytest.mark.parametrize("operator", ("<", "<=", "==", "!=", ">", ">=")) 7 | def test_that_version_comparisons_do_not_throw(operator, current_platform): 8 | marker = EnvironmentMarker(f"python_version {operator} '1.0'") 9 | marker.applies_to_platform(current_platform) 10 | -------------------------------------------------------------------------------- /unittests/test_license.py: -------------------------------------------------------------------------------- 1 | from pypi2nix.license import license_from_string 2 | from pypi2nix.logger import Logger 3 | from pypi2nix.wheel import Wheel 4 | 5 | from .logger import get_logger_output 6 | 7 | 8 | def test_license_from_string_detects_apache_2_0() -> None: 9 | assert license_from_string("Apache 2.0") == "licenses.asl20" 10 | 11 | 12 | def test_license_from_string_detects_bsd_dash_licenses() -> None: 13 | assert license_from_string("BSD - whatever") == "licenses.bsdOriginal" 14 | 15 | 16 | def test_that_license_of_flit_is_detected(flit_wheel: Wheel, logger: Logger): 17 | assert flit_wheel.license 18 | assert "WARNING" not in get_logger_output(logger) 19 | -------------------------------------------------------------------------------- /unittests/test_memoize.py: -------------------------------------------------------------------------------- 1 | from pypi2nix.memoize import memoize 2 | 3 | 4 | def test_memoized_method_returns_correct_result(): 5 | class A: 6 | @memoize 7 | def x(self): 8 | return 1 9 | 10 | assert A().x() == 1 11 | 12 | 13 | def test_memoized_method_gets_called_only_once(): 14 | class A: 15 | def __init__(self): 16 | self.times_called = 0 17 | 18 | @memoize 19 | def x(self): 20 | self.times_called += 1 21 | return 22 | 23 | a = A() 24 | a.x() 25 | a.x() 26 | assert a.times_called == 1 27 | -------------------------------------------------------------------------------- /unittests/test_network_file.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import tempfile 3 | 4 | import pytest 5 | 6 | from pypi2nix.logger import Logger 7 | from pypi2nix.network_file import DiskTextFile 8 | from pypi2nix.network_file import GitTextFile 9 | from pypi2nix.network_file import NetworkFile 10 | from pypi2nix.network_file import UrlTextFile 11 | from pypi2nix.nix import Nix 12 | 13 | from .switches import nix 14 | 15 | 16 | @nix 17 | def test_calculate_sha256_for_text_file(logger: Logger): 18 | test_file = UrlTextFile( 19 | url="https://raw.githubusercontent.com/nix-community/pypi2nix/6fe6265b62b53377b4677a39c6ee48550c1f2186/.gitignore", 20 | logger=logger, 21 | name="testname", 22 | ) 23 | assert "*.pyc" in test_file.fetch() 24 | assert "0b2s1lyfr12v83rrb69j1cfcsksisgwyzfl5mix6qz5ldxfww8p0" == test_file.sha256 25 | 26 | 27 | @nix 28 | def test_can_evaluate_expression_of_fetched_file(logger: Logger, nix: Nix) -> None: 29 | test_file = UrlTextFile( 30 | url="https://raw.githubusercontent.com/nix-community/pypi2nix/6fe6265b62b53377b4677a39c6ee48550c1f2186/.gitignore", 31 | logger=logger, 32 | name="testname", 33 | ) 34 | nix.build_expression( 35 | expression=f"let pkgs = import {{}}; in {test_file.nix_expression() }" 36 | ) 37 | 38 | 39 | @nix 40 | def test_can_calculate_hash_for_git_files(logger: Logger): 41 | repository_url = "https://github.com/nix-community/pypi2nix.git" 42 | path = ".gitignore" 43 | revision_name = "e56cbbce0812359e80ced3d860e1f232323b2976" 44 | network_file = GitTextFile( 45 | repository_url=repository_url, 46 | revision_name=revision_name, 47 | path=path, 48 | logger=logger, 49 | ) 50 | 51 | assert network_file.sha256 == "1vhdippb0daszp3a0m3zb9qcb25m6yib4rpggaiimg7yxwwwzyh4" 52 | assert "*.pyc" in network_file.fetch() 53 | 54 | 55 | @nix 56 | def test_can_evaluate_nix_expression(network_file: NetworkFile, nix: Nix): 57 | expression = f"let pkgs = import {{}}; in {network_file.nix_expression()}" 58 | nix.evaluate_expression(expression) 59 | 60 | 61 | @nix 62 | def test_fetch_content_equals_file_content_from_nix_expression( 63 | network_file: NetworkFile, nix: Nix 64 | ): 65 | fetch_content = network_file.fetch() 66 | 67 | nix_expression = "with builtins;" 68 | nix_expression += "let pkgs = import {};" 69 | nix_expression += f"fileContent = readFile ({network_file.nix_expression()});" 70 | nix_expression += " in " 71 | nix_expression += 'pkgs.writeTextFile { name = "test"; text = fileContent; }' 72 | with tempfile.TemporaryDirectory() as target_directory: 73 | target_file = os.path.join(target_directory, "result") 74 | nix.build_expression(nix_expression, out_link=target_file) 75 | with open(target_file) as f: 76 | nix_content = f.read() 77 | assert nix_content == fetch_content 78 | 79 | 80 | @pytest.fixture(params=["url", "git", "disk"]) 81 | def network_file(logger: Logger, request, data_directory): 82 | if request.param == "url": 83 | return UrlTextFile( 84 | url="https://raw.githubusercontent.com/nix-community/pypi2nix/6fe6265b62b53377b4677a39c6ee48550c1f2186/.gitignore", 85 | logger=logger, 86 | name="testname", 87 | ) 88 | elif request.param == "disk": 89 | return DiskTextFile(path=os.path.join(data_directory, "test.txt"),) 90 | else: 91 | repository_url = "https://github.com/nix-community/pypi2nix.git" 92 | path = ".gitignore" 93 | revision_name = "e56cbbce0812359e80ced3d860e1f232323b2976" 94 | return GitTextFile( 95 | repository_url=repository_url, 96 | revision_name=revision_name, 97 | path=path, 98 | logger=logger, 99 | ) 100 | -------------------------------------------------------------------------------- /unittests/test_nix.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | import pytest 4 | 5 | from pypi2nix.nix import EvaluationFailed 6 | from pypi2nix.nix import ExecutableNotFound 7 | from pypi2nix.nix import Nix 8 | 9 | from .switches import nix 10 | 11 | HERE = os.path.dirname(__file__) 12 | 13 | 14 | @pytest.fixture 15 | def nix_instance(tmpdir, logger): 16 | nix_path_addition = tmpdir.mkdir("testpath_exists") 17 | yield Nix( 18 | nix_path=["test_variable={}".format(str(nix_path_addition))], logger=logger 19 | ) 20 | 21 | 22 | @pytest.fixture 23 | def dummy_derivation(): 24 | return os.path.join(HERE, "data/shell_environment.nix") 25 | 26 | 27 | @nix 28 | def test_evaluate_nix_expression_works(nix_instance): 29 | assert nix_instance.evaluate_expression("1 + 1") == "2" 30 | 31 | 32 | @nix 33 | def test_evalulate_nix_expression_respects_additions_to_nix_path(nix_instance): 34 | assert "testpath_exists" in nix_instance.evaluate_expression("") 35 | 36 | 37 | @nix 38 | def test_evaluate_nix_expression_raises_exception_when_executable_not_found(logger): 39 | nix = Nix(executable_directory="/does-not-exist", logger=logger) 40 | with pytest.raises(ExecutableNotFound): 41 | nix.evaluate_expression("true") 42 | 43 | 44 | @nix 45 | def test_shell_accepts_file_path_to_run_shell_script(nix_instance, dummy_derivation): 46 | output = nix_instance.shell("echo $out", derivation_path=dummy_derivation) 47 | assert "hello" in output 48 | 49 | 50 | @nix 51 | def test_shell_accepts_nix_arguments(nix_instance, dummy_derivation): 52 | output = nix_instance.shell( 53 | "echo $out", 54 | derivation_path=dummy_derivation, 55 | nix_arguments={"dummy_argument": "perl"}, 56 | ) 57 | assert "perl" in output 58 | 59 | 60 | @nix 61 | def test_evaluate_expression_throws_on_erroneous_expression(nix_instance): 62 | with pytest.raises(EvaluationFailed): 63 | nix_instance.evaluate_expression("1+") 64 | 65 | 66 | @nix 67 | def test_build_expression_throws_on_syntax_error(nix_instance): 68 | with pytest.raises(EvaluationFailed): 69 | nix_instance.build_expression("with import {}; hello(") 70 | 71 | 72 | @nix 73 | def test_build_expression_creates_proper_out_link(nix_instance, tmpdir): 74 | output_path = tmpdir.join("output-link") 75 | nix_instance.build_expression( 76 | "with import {}; hello", out_link=str(output_path) 77 | ) 78 | assert output_path.exists() 79 | 80 | 81 | @nix 82 | def test_build_respects_boolean_arguments(nix_instance, tmpdir): 83 | source_path = tmpdir.join("test.nix") 84 | with open(source_path, "w") as f: 85 | f.write( 86 | " ".join( 87 | [ 88 | "{ argument }:", 89 | "with import {};" 90 | 'if lib.assertMsg argument "Argument is false" then hello else null', 91 | ] 92 | ) 93 | ) 94 | nix_instance.build(source_file=str(source_path), arguments={"argument": True}) 95 | 96 | 97 | @nix 98 | def test_build_expression_respects_boolean_arguments(nix_instance): 99 | expression = " ".join( 100 | [ 101 | "{ argument }:", 102 | "with import {};" 103 | 'if lib.assertMsg argument "Argument is false" then hello else null', 104 | ] 105 | ) 106 | nix_instance.build_expression(expression, arguments={"argument": True}) 107 | -------------------------------------------------------------------------------- /unittests/test_package_source.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pypi2nix.nix import Nix 4 | from pypi2nix.package_source import GitSource 5 | from pypi2nix.package_source import HgSource 6 | from pypi2nix.package_source import PathSource 7 | from pypi2nix.package_source import UrlSource 8 | 9 | from .switches import nix 10 | 11 | URL_SOURCE_URL = "https://github.com/nix-community/pypi2nix/archive/4e85fe7505dd7e703aacc18d9ef45f7e47947a6a.zip" 12 | URL_SOURCE_HASH = "1x3dzqlnryplmxm3z1lnl40y0i2g8n6iynlngq2kkknxj9knjyhv" 13 | 14 | 15 | @pytest.fixture 16 | def git_source(): 17 | return GitSource( 18 | url="https://github.com/nix-community/pypi2nix.git", 19 | revision="4e85fe7505dd7e703aacc18d9ef45f7e47947a6a", 20 | ) 21 | 22 | 23 | @pytest.fixture 24 | def hg_source(logger): 25 | return HgSource( 26 | url="https://bitbucket.org/tarek/flake8", revision="a209fb69350c", logger=logger 27 | ) 28 | 29 | 30 | @pytest.fixture 31 | def url_source(logger): 32 | return UrlSource(url=URL_SOURCE_URL, logger=logger) 33 | 34 | 35 | @pytest.fixture 36 | def path_source(): 37 | return PathSource("/test/path") 38 | 39 | 40 | @pytest.fixture 41 | def expression_evaluater(logger): 42 | nix_instance = Nix(logger=logger) 43 | return lambda expression: nix_instance.evaluate_expression( 44 | "let pkgs = import {}; in " + expression 45 | ) 46 | 47 | 48 | @nix 49 | def test_git_source_gives_correct_hash_value(git_source): 50 | assert ( 51 | git_source.hash_value() 52 | == "113sngkfi93pdlws1i8kq2rqff10xr1n3z3krn2ilq0fdrddyk96" 53 | ) 54 | 55 | 56 | @nix 57 | def test_git_source_produces_valid_nix_expression(git_source, expression_evaluater): 58 | expression_evaluater(git_source.nix_expression()) 59 | 60 | 61 | @nix 62 | def test_hg_source_gives_correct_hash_value(hg_source): 63 | assert ( 64 | hg_source.hash_value() == "1n0fzlzmfmynnay0n757yh3qwjd9xxcfi7vq4sxqvsv90c441s7v" 65 | ) 66 | 67 | 68 | @nix 69 | def test_hg_source_produces_valid_nix_expression(hg_source, expression_evaluater): 70 | expression_evaluater(hg_source.nix_expression()) 71 | 72 | 73 | @nix 74 | def test_url_source_gives_correct_hash_value(url_source): 75 | assert url_source.hash_value() == URL_SOURCE_HASH 76 | 77 | 78 | @nix 79 | def test_url_source_gives_valid_nix_expression(url_source, expression_evaluater): 80 | expression_evaluater(url_source.nix_expression()) 81 | 82 | 83 | def test_url_source_nix_expression_contains_specified_hash_when_given(logger): 84 | # We specify the wrong hash on purpose to see that UrlSource just 85 | # "accepts" the given hash and puts it into the generated nix 86 | # expression 87 | url_source = UrlSource( 88 | URL_SOURCE_URL, hash_value=URL_SOURCE_HASH + "1", logger=logger 89 | ) 90 | assert URL_SOURCE_HASH + "1" in url_source.nix_expression() 91 | 92 | 93 | @nix 94 | def test_path_source_gives_valid_nix_expression(path_source, expression_evaluater): 95 | expression_evaluater(path_source.nix_expression()) 96 | 97 | 98 | def test_path_source_paths_with_one_segement_get_dot_appended_for_nix(): 99 | source = PathSource("segment") 100 | assert source.nix_expression() == "segment/." 101 | -------------------------------------------------------------------------------- /unittests/test_prefetch_url.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pypi2nix.utils import prefetch_url 4 | 5 | from .switches import nix 6 | 7 | 8 | @nix 9 | def test_prefetch_url_returns_correct_hash(logger): 10 | url = "https://github.com/nix-community/pypi2nix/archive/4e85fe7505dd7e703aacc18d9ef45f7e47947a6a.zip" 11 | expected_hash = "1x3dzqlnryplmxm3z1lnl40y0i2g8n6iynlngq2kkknxj9knjyhv" 12 | assert prefetch_url(url, logger) == expected_hash 13 | 14 | 15 | @nix 16 | def test_prefetch_url_raises_on_invalid_name(logger): 17 | """nix-prefetch-url cannot handle file names with period in them. Here 18 | we test if the code throws a ValueError in that instance. 19 | """ 20 | url = "https://raw.githubusercontent.com/nix-community/pypi2nix/6fe6265b62b53377b4677a39c6ee48550c1f2186/.gitignore" 21 | with pytest.raises(ValueError): 22 | prefetch_url(url, logger) 23 | 24 | 25 | @nix 26 | def test_can_provide_name_so_prefetch_does_not_fail(logger): 27 | url = "https://raw.githubusercontent.com/nix-community/pypi2nix/6fe6265b62b53377b4677a39c6ee48550c1f2186/.gitignore" 28 | sha256 = prefetch_url(url, logger, name="testname") 29 | assert sha256 == "0b2s1lyfr12v83rrb69j1cfcsksisgwyzfl5mix6qz5ldxfww8p0" 30 | -------------------------------------------------------------------------------- /unittests/test_project_directory.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import tempfile 3 | 4 | import pytest 5 | 6 | from pypi2nix.project_directory import PersistentProjectDirectory 7 | from pypi2nix.project_directory import TemporaryProjectDirectory 8 | 9 | 10 | @pytest.fixture(params=("tempfile", "persistent")) 11 | def project_directory(request,): 12 | if request.param == "tempfile": 13 | yield TemporaryProjectDirectory() 14 | elif request.param == "persistent": 15 | with TemporaryProjectDirectory() as directory: 16 | yield PersistentProjectDirectory(path=directory) 17 | 18 | 19 | def test_can_write_to_project_directory(project_directory): 20 | with project_directory as directory: 21 | file_path = os.path.join(directory, "test.txt") 22 | with open(file_path, "w") as f: 23 | f.write("test") 24 | 25 | 26 | def test_tempfile_project_directory_is_deleted_after_exception(): 27 | with pytest.raises(Exception), TemporaryProjectDirectory() as directory: 28 | path = directory 29 | raise Exception() 30 | assert not os.path.exists(path) 31 | 32 | 33 | def test_persistent_project_directory_is_not_deleted_on_exception(): 34 | with tempfile.TemporaryDirectory() as directory: 35 | with pytest.raises(Exception), PersistentProjectDirectory( 36 | path=directory 37 | ) as _project_dir: 38 | project_directory = _project_dir 39 | raise Exception() 40 | assert os.path.exists(project_directory) 41 | -------------------------------------------------------------------------------- /unittests/test_pypi.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pypi2nix.logger import Logger 4 | from pypi2nix.pypi import Pypi 5 | 6 | from .switches import nix 7 | 8 | 9 | @pytest.fixture 10 | def pypi(logger: Logger): 11 | return Pypi(logger) 12 | 13 | 14 | @nix 15 | def test_pypi_get_package_returns_package_with_correct_name(pypi): 16 | assert pypi.get_package("six").name == "six" 17 | 18 | 19 | @nix 20 | def test_pypi_get_package_returns_package_with_releases(pypi): 21 | assert pypi.get_package("six").releases 22 | 23 | 24 | @nix 25 | def test_pypi_gets_correct_source_release_for_package_version_with_only_source_release( 26 | pypi, 27 | ): 28 | release = pypi.get_source_release("six", "0.9.0") 29 | assert ( 30 | release.sha256_digest 31 | == "14fd1ed3dd0e1a46cc53b8fc890b5a3b11737515aeb7f42c3af9f38e8d8975d7" 32 | ) 33 | 34 | 35 | @nix 36 | def test_pypi_gets_correct_source_release_for_package_with_multiple_release_types(pypi): 37 | release = pypi.get_source_release("six", "1.12.0") 38 | assert ( 39 | release.sha256_digest 40 | == "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" 41 | ) 42 | 43 | 44 | @nix 45 | def test_pypi_gets_correct_source_release_for_radiotherm_1_2(pypi): 46 | release = pypi.get_source_release("radiotherm", "1.2") 47 | assert ( 48 | release.sha256_digest 49 | == "e8a70e0cf38f21170a3a43d5de62954aa38032dfff20adcdf79dd6c39734b8cc" 50 | ) 51 | 52 | 53 | @nix 54 | def test_pypi_gets_correct_source_release_for_setuptools_1_6_0(pypi): 55 | release = pypi.get_source_release("setuptools-scm", "1.6.0") 56 | assert ( 57 | release.sha256_digest 58 | == "c4f1b14e4fcc7dd69287a6c0b571c889dd4970559c7fa0512b2311f1513d86f4" 59 | ) 60 | -------------------------------------------------------------------------------- /unittests/test_python_version.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pypi2nix.python_version import PythonVersion 4 | from pypi2nix.python_version import python_version_from_version_string 5 | from pypi2nix.target_platform import PlatformGenerator 6 | 7 | from .switches import nix 8 | 9 | 10 | @pytest.mark.parametrize("python_version", PythonVersion) 11 | @nix 12 | def test_available_python_versions_exist_in_nixpkgs( 13 | python_version: PythonVersion, platform_generator: PlatformGenerator 14 | ): 15 | target_platform = platform_generator.from_python_version(python_version) 16 | assert target_platform is not None 17 | 18 | 19 | @pytest.mark.parametrize( 20 | "version_string, expected_python_version", 21 | [ 22 | ("3.5", PythonVersion.python35), 23 | ("3.6", PythonVersion.python36), 24 | ("3.7", PythonVersion.python37), 25 | ("3.8", PythonVersion.python38), 26 | ], 27 | ) 28 | def test_can_get_python_version_from_version_string( 29 | version_string, expected_python_version 30 | ): 31 | assert python_version_from_version_string(version_string) == expected_python_version 32 | -------------------------------------------------------------------------------- /unittests/test_requirement_collector.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | import pathlib 4 | from contextlib import contextmanager 5 | from typing import Any 6 | from typing import Generator 7 | 8 | import pytest 9 | 10 | from pypi2nix.dependency_graph import DependencyGraph 11 | from pypi2nix.logger import Logger 12 | from pypi2nix.package_source import PathSource 13 | from pypi2nix.requirement_parser import RequirementParser 14 | from pypi2nix.requirements import PathRequirement 15 | from pypi2nix.requirements_collector import RequirementsCollector 16 | from pypi2nix.target_platform import TargetPlatform 17 | 18 | 19 | @contextmanager 20 | def current_working_directory(dir: str) -> Generator[None, None, None]: 21 | current = os.getcwd() 22 | try: 23 | os.chdir(dir) 24 | yield 25 | finally: 26 | os.chdir(current) 27 | 28 | 29 | @pytest.fixture 30 | def collector( 31 | current_platform: TargetPlatform, 32 | requirement_parser: RequirementParser, 33 | logger: Logger, 34 | project_dir: str, 35 | ) -> RequirementsCollector: 36 | return RequirementsCollector( 37 | current_platform, requirement_parser, logger, project_dir, DependencyGraph() 38 | ) 39 | 40 | 41 | def test_that_we_can_generate_an_empty_requirement_set_from_freshly_constructed_collector( 42 | current_platform: TargetPlatform, 43 | requirement_parser: RequirementParser, 44 | logger: Logger, 45 | project_dir: str, 46 | ) -> None: 47 | collector = RequirementsCollector( 48 | current_platform, requirement_parser, logger, project_dir, DependencyGraph() 49 | ) 50 | requirements = collector.requirements() 51 | assert len(requirements) == 0 52 | 53 | 54 | def test_that_we_can_add_command_line_requirements_by_name( 55 | collector: RequirementsCollector, 56 | ) -> None: 57 | collector.add_line("pytest") 58 | requirements = collector.requirements() 59 | assert "pytest" in requirements 60 | 61 | 62 | def test_that_we_can_add_a_requirements_file_path( 63 | collector: RequirementsCollector, tmpdir: pathlib.Path 64 | ) -> None: 65 | requirements_txt = tmpdir / "requirements.txt" 66 | requirements_lines = ["pytest", "flake8"] 67 | with open(requirements_txt, "w") as f: 68 | for requirement in requirements_lines: 69 | print(requirement, file=f) 70 | collector.add_file(str(requirements_txt)) 71 | assert "pytest" in collector.requirements() 72 | assert "flake8" in collector.requirements() 73 | 74 | 75 | def test_that_requirements_with_relative_paths_are_absolute_paths_after_adding( 76 | collector: RequirementsCollector, 77 | ) -> None: 78 | collector.add_line("./path/to/egg#egg=testegg") 79 | requirement = collector.requirements().get("testegg") 80 | assert isinstance(requirement, PathRequirement) 81 | assert os.path.isabs(requirement.path()) 82 | 83 | 84 | def test_that_sources_can_be_extracted_from_a_collector( 85 | collector: RequirementsCollector, 86 | ) -> None: 87 | collector.add_line("path/to/egg#egg=testegg") 88 | assert "testegg" in collector.sources() 89 | 90 | 91 | def test_that_relative_paths_are_preserved_in_sources( 92 | collector: RequirementsCollector, 93 | ) -> None: 94 | collector.add_line("path/to/egg#egg=testegg") 95 | testegg_source = collector.sources()["testegg"] 96 | assert isinstance(testegg_source, PathSource) 97 | assert testegg_source.path == "path/to/egg" 98 | 99 | 100 | def test_that_path_paths_from_requirement_files_are_preserved_in_sources( 101 | collector: RequirementsCollector, tmpdir: Any 102 | ) -> None: 103 | with current_working_directory(str(tmpdir)): 104 | requirements_file_path = tmpdir.join("requirements.txt") 105 | with open(requirements_file_path, "w") as f: 106 | print("path/to/egg#egg=testegg", file=f) 107 | collector.add_file(str(requirements_file_path)) 108 | testegg_source = collector.sources()["testegg"] 109 | assert isinstance(testegg_source, PathSource) 110 | assert testegg_source.path == "path/to/egg" 111 | 112 | 113 | def test_that_path_sources_from_requirement_files_are_preserved_in_sources_relative_to_file( 114 | collector: RequirementsCollector, tmpdir: Any 115 | ) -> None: 116 | with current_working_directory(str(tmpdir)): 117 | requirements_directory = tmpdir.join("directory") 118 | requirements_directory.mkdir() 119 | requirements_file_path = requirements_directory.join("requirements.txt") 120 | with open(requirements_file_path, "w") as f: 121 | print("path/to/egg#egg=testegg", file=f) 122 | collector.add_file(str(requirements_file_path)) 123 | testegg_source = collector.sources()["testegg"] 124 | assert isinstance(testegg_source, PathSource) 125 | assert testegg_source.path == "directory/path/to/egg" 126 | -------------------------------------------------------------------------------- /unittests/test_requirement_dependency_retriever.py: -------------------------------------------------------------------------------- 1 | from pypi2nix.dependency_graph import DependencyGraph 2 | from pypi2nix.external_dependencies import ExternalDependency 3 | from pypi2nix.external_dependency_collector import RequirementDependencyRetriever 4 | from pypi2nix.requirement_parser import RequirementParser 5 | 6 | 7 | def test_no_external_dependency_for_empty_dependency_graph( 8 | requirement_parser: RequirementParser, 9 | ) -> None: 10 | dependency_graph = DependencyGraph() 11 | retriever = RequirementDependencyRetriever(dependency_graph) 12 | requirement = requirement_parser.parse("testpackage") 13 | assert not retriever.get_external_dependency_for_requirement(requirement) 14 | 15 | 16 | def test_external_dependencies_from_graph_are_retrieved( 17 | requirement_parser: RequirementParser, 18 | ) -> None: 19 | dependency_graph = DependencyGraph() 20 | requirement = requirement_parser.parse("testpackage") 21 | external_dependency = ExternalDependency("external") 22 | dependency_graph.set_external_dependency( 23 | dependent=requirement, dependency=external_dependency 24 | ) 25 | retriever = RequirementDependencyRetriever(dependency_graph) 26 | assert external_dependency in retriever.get_external_dependency_for_requirement( 27 | requirement 28 | ) 29 | -------------------------------------------------------------------------------- /unittests/test_requirement_parser.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from parsley import ParseError 3 | 4 | from pypi2nix.logger import Logger 5 | from pypi2nix.requirement_parser import RequirementParser 6 | from pypi2nix.requirements import PathRequirement 7 | 8 | from .logger import get_logger_output 9 | 10 | 11 | def test_parses_pip_style_url(requirement_parser): 12 | requirement_parser.compiled_grammar()( 13 | "git+https://github.com/nix-community/pypi2nix.git" 14 | ).URI_pip_style() 15 | 16 | 17 | def test_parse_pip_style_requirement(requirement_parser): 18 | requirement_parser.compiled_grammar()( 19 | "git+https://github.com/nix-community/pypi2nix.git#egg=pypi2nix" 20 | ).url_req_pip_style() 21 | 22 | 23 | def test_that_python_implemntation_marker_can_be_parsed(requirement_parser): 24 | requirement_parser.compiled_grammar()( 25 | 'testspec; python_implementation == "CPython"' 26 | ) 27 | 28 | 29 | @pytest.mark.parametrize("path", ("/test/path", "./test/path", "test/path", "./.")) 30 | def test_that_file_path_with_leading_slash_can_be_parsed(path, requirement_parser): 31 | assert requirement_parser.compiled_grammar()(path).file_path() == path 32 | 33 | 34 | @pytest.mark.parametrize( 35 | "path", ("#", "/#/", "/test#/", "#/test", "path/test#egg=testegg") 36 | ) 37 | def test_that_path_with_hashpound_is_not_recognized(path, requirement_parser): 38 | with pytest.raises(ParseError): 39 | requirement_parser.compiled_grammar()(path).file_path() 40 | 41 | 42 | def test_that_we_can_parse_pip_style_requirement_with_file_path(requirement_parser): 43 | requirement = requirement_parser.compiled_grammar()( 44 | "path/to/egg#egg=testegg" 45 | ).path_req_pip_style() 46 | assert requirement.name() == "testegg" 47 | assert requirement.path() == "path/to/egg" 48 | 49 | 50 | @pytest.mark.parametrize( 51 | "line", 52 | ( 53 | "cffi>=1.8,!=1.11.3; python_implementation != 'PyPy'", 54 | "cffi>=1.1; python_implementation != 'PyPy'", 55 | "cffi>=1.4.1; python_implementation != 'PyPy'", 56 | ), 57 | ) 58 | def test_regressions_with_cryptography( 59 | requirement_parser: RequirementParser, line: str, logger: Logger 60 | ) -> None: 61 | requirement = requirement_parser.parse(line) 62 | assert requirement.name() == "cffi" 63 | assert "WARNING" in get_logger_output(logger) 64 | assert "PEP 508" in get_logger_output(logger) 65 | 66 | 67 | def test_that_path_is_parsed_to_path_requirement(requirement_parser: RequirementParser): 68 | requirement = requirement_parser.parse("local_path/egg#egg=local-path") 69 | assert isinstance(requirement, PathRequirement) 70 | 71 | 72 | def test_that_requirement_parser_does_not_choke_on_sys_dot_platform( 73 | requirement_parser: RequirementParser, logger: Logger 74 | ): 75 | line = 'macfsevents ; sys.platform == "darwin"' 76 | requirement = requirement_parser.parse(line) 77 | assert requirement.name() == "macfsevents" 78 | assert "WARNING" in get_logger_output(logger) 79 | assert "PEP 508" in get_logger_output(logger) 80 | 81 | 82 | def test_that_comment_is_parsed_correctly(requirement_parser: RequirementParser): 83 | comment_string = "# this is a comment" 84 | result = requirement_parser.compiled_grammar()(comment_string).comment() 85 | assert result == "this is a comment" 86 | 87 | 88 | def test_that_comment_without_string_after_it_is_parsed_correctly( 89 | requirement_parser: RequirementParser, 90 | ): 91 | comment_string = "#" 92 | result = requirement_parser.compiled_grammar()(comment_string).comment() 93 | assert result == "" 94 | 95 | 96 | def test_that_name_requirements_can_have_comments( 97 | requirement_parser: RequirementParser, 98 | ): 99 | line = "requirement # comment" 100 | result = requirement_parser.compiled_grammar()(line).name_req() 101 | assert result.name() == "requirement" 102 | 103 | 104 | def test_that_url_req_can_have_comments(requirement_parser: RequirementParser): 105 | line = "test @ https://test.url # comment" 106 | result = requirement_parser.compiled_grammar()(line).url_req() 107 | assert result.name() == "test" 108 | 109 | 110 | def test_that_url_req_pip_style_can_have_comments( 111 | requirement_parser: RequirementParser, 112 | ): 113 | line = "https://test.url#egg=test # comment" 114 | result = requirement_parser.compiled_grammar()(line).url_req_pip_style() 115 | assert result.name() == "test" 116 | 117 | 118 | def test_that_path_req_pip_style_can_have_comments( 119 | requirement_parser: RequirementParser, 120 | ): 121 | line = "/path/requirement#egg=test # comment" 122 | result = requirement_parser.compiled_grammar()(line).path_req_pip_style() 123 | assert result.name() == "test" 124 | -------------------------------------------------------------------------------- /unittests/test_requirements_file.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pypi2nix.logger import Logger 4 | from pypi2nix.requirement_parser import RequirementParser 5 | from pypi2nix.requirements_file import RequirementsFile 6 | 7 | 8 | @pytest.fixture 9 | def requirements_file_from_lines( 10 | project_dir, tmpdir_factory, requirement_parser, logger: Logger 11 | ): 12 | def factory(lines): 13 | requirements_file = tmpdir_factory.mktemp("test").join("requirements.txt") 14 | requirements_file.write("\n".join(lines)) 15 | return RequirementsFile( 16 | str(requirements_file), project_dir, requirement_parser, logger 17 | ) 18 | 19 | return factory 20 | 21 | 22 | def test_requirements_file_handles_comments(requirements_file_from_lines): 23 | requirements_file = requirements_file_from_lines(["# comment"]) 24 | requirements_file.process() 25 | 26 | 27 | def test_requirements_file_handles_empty_lines(requirements_file_from_lines): 28 | requirements_file = requirements_file_from_lines([""]) 29 | requirements_file.process() 30 | 31 | 32 | def test_requirements_file_can_be_created_from_requirements_lines( 33 | project_dir: str, requirement_parser: RequirementParser, logger: Logger 34 | ): 35 | RequirementsFile.from_lines( 36 | ["pytest"], 37 | requirement_parser=requirement_parser, 38 | project_dir=project_dir, 39 | logger=logger, 40 | ) 41 | 42 | 43 | def test_regular_requirements_stay_in_processed_file( 44 | project_dir: str, requirement_parser: RequirementParser, logger: Logger 45 | ): 46 | requirement_file = RequirementsFile.from_lines( 47 | ["pytest"], 48 | requirement_parser=requirement_parser, 49 | project_dir=project_dir, 50 | logger=logger, 51 | ) 52 | processed_file = requirement_file.processed_requirements_file_path() 53 | with open(processed_file) as f: 54 | assert "pytest" in f.read() 55 | -------------------------------------------------------------------------------- /unittests/test_source_distribution.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | 4 | import pytest 5 | 6 | from pypi2nix.archive import Archive 7 | from pypi2nix.logger import Logger 8 | from pypi2nix.package.exceptions import DistributionNotDetected 9 | from pypi2nix.requirement_parser import RequirementParser 10 | from pypi2nix.source_distribution import SourceDistribution 11 | from pypi2nix.target_platform import TargetPlatform 12 | 13 | from .logger import get_logger_output 14 | from .switches import nix 15 | 16 | 17 | @pytest.fixture 18 | def source_distribution( 19 | six_source_distribution_archive, 20 | logger: Logger, 21 | requirement_parser: RequirementParser, 22 | ): 23 | return SourceDistribution.from_archive( 24 | six_source_distribution_archive, logger, requirement_parser=requirement_parser 25 | ) 26 | 27 | 28 | @pytest.fixture 29 | def flit_distribution( 30 | data_directory, logger: Logger, requirement_parser: RequirementParser 31 | ): 32 | archive = Archive(os.path.join(data_directory, "flit-1.3.tar.gz")) 33 | return SourceDistribution.from_archive( 34 | archive, logger, requirement_parser=requirement_parser 35 | ) 36 | 37 | 38 | @nix 39 | def test_from_archive_picks_up_on_name(source_distribution): 40 | assert source_distribution.name == "six" 41 | 42 | 43 | @nix 44 | def test_that_a_source_distributions_name_is_canonicalized( 45 | logger: Logger, requirement_parser: RequirementParser 46 | ): 47 | distribution = SourceDistribution( 48 | name="NaMe_teSt", logger=logger, requirement_parser=requirement_parser 49 | ) 50 | assert distribution.name == "name-test" 51 | 52 | 53 | @nix 54 | def test_six_package_has_no_pyproject_toml(source_distribution): 55 | assert source_distribution.pyproject_toml is None 56 | 57 | 58 | @nix 59 | def test_that_flit_pyproject_toml_is_recognized(flit_distribution): 60 | assert flit_distribution.pyproject_toml is not None 61 | 62 | 63 | @nix 64 | def test_that_flit_build_dependencies_contains_requests( 65 | flit_distribution: SourceDistribution, current_platform: TargetPlatform 66 | ): 67 | assert "requests" in flit_distribution.build_dependencies(current_platform) 68 | 69 | 70 | @nix 71 | def test_that_we_can_generate_objects_from_source_archives( 72 | source_distribution_archive, logger: Logger, requirement_parser: RequirementParser, 73 | ): 74 | SourceDistribution.from_archive( 75 | source_distribution_archive, logger, requirement_parser=requirement_parser 76 | ) 77 | 78 | 79 | @nix 80 | def test_that_we_can_detect_setup_requirements_for_setup_cfg_projects( 81 | distribution_archive_for_jsonschema, 82 | current_platform, 83 | logger: Logger, 84 | requirement_parser: RequirementParser, 85 | ): 86 | distribution = SourceDistribution.from_archive( 87 | distribution_archive_for_jsonschema, 88 | logger, 89 | requirement_parser=requirement_parser, 90 | ) 91 | assert "setuptools-scm" in distribution.build_dependencies(current_platform) 92 | 93 | 94 | def test_that_trying_to_create_source_distribution_from_random_zip_throws( 95 | test_zip_path, logger: Logger, requirement_parser: RequirementParser 96 | ): 97 | archive = Archive(path=test_zip_path) 98 | with pytest.raises(DistributionNotDetected): 99 | SourceDistribution.from_archive( 100 | archive, logger, requirement_parser=requirement_parser, 101 | ) 102 | 103 | 104 | def test_build_dependencies_for_invalid_deps_logs_warning( 105 | data_directory, 106 | current_platform, 107 | logger: Logger, 108 | requirement_parser: RequirementParser, 109 | ): 110 | spacy_distribution_path = os.path.join(data_directory, "spacy-2.1.0.tar.gz") 111 | archive = Archive(spacy_distribution_path) 112 | 113 | dist = SourceDistribution.from_archive( 114 | archive, logger, requirement_parser=requirement_parser 115 | ) 116 | 117 | assert "WARNING:" not in get_logger_output(logger) 118 | dist.build_dependencies(current_platform) 119 | assert "WARNING:" in get_logger_output(logger) 120 | 121 | 122 | def test_invalid_build_dependencies_for_setupcfg_package_logs_warning( 123 | data_directory, 124 | current_platform, 125 | logger: Logger, 126 | requirement_parser: RequirementParser, 127 | ): 128 | distribution_path = os.path.join( 129 | data_directory, "setupcfg-package", "setupcfg-package.tar.gz" 130 | ) 131 | archive = Archive(distribution_path) 132 | 133 | dist = SourceDistribution.from_archive( 134 | archive, logger, requirement_parser=requirement_parser 135 | ) 136 | 137 | assert "WARNING:" not in get_logger_output(logger) 138 | dist.build_dependencies(current_platform) 139 | assert "WARNING:" in get_logger_output(logger) 140 | -------------------------------------------------------------------------------- /unittests/test_sources.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pypi2nix.package_source import PathSource 4 | from pypi2nix.sources import Sources 5 | 6 | 7 | @pytest.fixture 8 | def sources(): 9 | return Sources() 10 | 11 | 12 | @pytest.fixture 13 | def other_sources(): 14 | return Sources() 15 | 16 | 17 | def test_sources_can_be_added_to(sources): 18 | sources.add("testsource", PathSource("/test/path")) 19 | 20 | assert "testsource" in sources 21 | 22 | 23 | def test_sources_can_be_queried_by_name(sources): 24 | source = PathSource("/test/path") 25 | sources.add("testsource", source) 26 | 27 | assert sources["testsource"] is source 28 | 29 | 30 | def test_sources_can_be_merged(sources, other_sources): 31 | assert "testsource" not in sources 32 | other_sources.add("testsource", PathSource("/test/path")) 33 | sources.update(other_sources) 34 | assert "testsource" in sources 35 | 36 | 37 | def test_items_returns_length_on_tuple_for_one_entry(sources): 38 | sources.add("testitem", PathSource("/test/path")) 39 | assert len(sources.items()) == 1 40 | 41 | 42 | def test_empty_sources_has_length_0(sources): 43 | assert len(sources) == 0 44 | -------------------------------------------------------------------------------- /unittests/test_util_cmd.py: -------------------------------------------------------------------------------- 1 | from pypi2nix.logger import Logger 2 | from pypi2nix.utils import cmd 3 | 4 | 5 | def test_consistent_output(logger: Logger): 6 | exit_code, output = cmd(["seq", "5"], logger=logger) 7 | assert output == "1\n2\n3\n4\n5\n" 8 | -------------------------------------------------------------------------------- /unittests/test_wheel.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import pytest 4 | 5 | from pypi2nix.logger import Logger 6 | from pypi2nix.requirement_parser import RequirementParser 7 | from pypi2nix.requirement_set import RequirementSet 8 | from pypi2nix.target_platform import TargetPlatform 9 | from pypi2nix.wheel import Wheel 10 | 11 | from .switches import nix 12 | 13 | 14 | @pytest.fixture 15 | def wheel(current_platform): 16 | build_dependencies = RequirementSet(current_platform) 17 | dependencies = RequirementSet(current_platform) 18 | return Wheel( 19 | name="test-wheel", 20 | version="1.0", 21 | deps=dependencies, 22 | homepage="https://example.test", 23 | license="", 24 | description="description", 25 | build_dependencies=build_dependencies, 26 | target_platform=current_platform, 27 | ) 28 | 29 | 30 | @nix 31 | def test_can_create_wheel_from_valid_directory( 32 | extracted_six_package, 33 | current_platform, 34 | logger: Logger, 35 | requirement_parser: RequirementParser, 36 | ): 37 | Wheel.from_wheel_directory_path( 38 | extracted_six_package, current_platform, logger, requirement_parser 39 | ) 40 | 41 | 42 | @nix 43 | def test_can_add_build_dependencies_to_wheel( 44 | wheel: Wheel, 45 | current_platform: TargetPlatform, 46 | requirement_parser: RequirementParser, 47 | ): 48 | build_dependencies = RequirementSet(current_platform) 49 | build_dependencies.add(requirement_parser.parse("dep1")) 50 | build_dependencies.add(requirement_parser.parse("dep2")) 51 | wheel.add_build_dependencies(build_dependencies) 52 | assert "dep1" in wheel.build_dependencies(current_platform) 53 | assert "dep2" in wheel.build_dependencies(current_platform) 54 | 55 | 56 | def test_that_to_dict_is_json_serializable(wheel: Wheel): 57 | json.dumps(wheel.to_dict()) 58 | 59 | 60 | def test_that_setupcfg_package_wheel_contains_requests_as_dependency( 61 | setupcfg_package_wheel: Wheel, 62 | ): 63 | assert "requests" in setupcfg_package_wheel.dependencies() 64 | 65 | 66 | def test_that_setupcfg_package_wheel_contains_pytest_as_testing_dependency( 67 | setupcfg_package_wheel: Wheel, 68 | ): 69 | assert "pytest" in setupcfg_package_wheel.dependencies(extras=["testing"]) 70 | 71 | 72 | def test_that_setupcfg_package_wheel_does_not_contain_pytest_as_non_testing_dependency( 73 | setupcfg_package_wheel: Wheel, 74 | ): 75 | assert "pytest" not in setupcfg_package_wheel.dependencies() 76 | -------------------------------------------------------------------------------- /unittests/test_wheel_builder.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import pytest 4 | 5 | from pypi2nix.logger import Logger 6 | from pypi2nix.metadata_fetcher import MetadataFetcher 7 | from pypi2nix.pypi import Pypi 8 | from pypi2nix.requirement_parser import RequirementParser 9 | from pypi2nix.requirement_set import RequirementSet 10 | from pypi2nix.sources import Sources 11 | from pypi2nix.target_platform import TargetPlatform 12 | from pypi2nix.wheel import Wheel 13 | from pypi2nix.wheel_builder import WheelBuilder 14 | 15 | from .switches import nix 16 | 17 | 18 | @pytest.fixture 19 | def build_wheels( 20 | wheel_builder: WheelBuilder, 21 | current_platform: TargetPlatform, 22 | requirement_parser: RequirementParser, 23 | logger: Logger, 24 | sources_for_test_packages: Sources, 25 | pypi: Pypi, 26 | ): 27 | def wrapper(requirement_lines: List[str]) -> List[Wheel]: 28 | requirements = RequirementSet(current_platform) 29 | for line in requirement_lines: 30 | requirements.add(requirement_parser.parse(line)) 31 | wheel_paths = wheel_builder.build(requirements) 32 | metadata_fetcher = MetadataFetcher( 33 | sources_for_test_packages, logger, requirement_parser, pypi 34 | ) 35 | return metadata_fetcher.main( 36 | wheel_paths, current_platform, wheel_builder.source_distributions 37 | ) 38 | 39 | return wrapper 40 | 41 | 42 | @nix 43 | def test_extracts_myextra_dependencies_from_package3(build_wheels,): 44 | wheels = build_wheels(["package3[myextra]"]) 45 | assert [wheel for wheel in wheels if wheel.name == "package1"] 46 | 47 | 48 | @nix 49 | def test_does_not_package_myextra_dependencies_if_no_extras_specified(build_wheels,): 50 | wheels = build_wheels(["package3"]) 51 | assert not [wheel for wheel in wheels if wheel.name == "package1"] 52 | 53 | 54 | @nix 55 | def test_does_detect_extra_requirements_from_requirements(build_wheels): 56 | wheels = build_wheels(["package4"]) 57 | assert [wheel for wheel in wheels if wheel.name == "package1"] 58 | 59 | 60 | @nix 61 | def test_that_we_filter_extra_requirements_that_do_not_apply_to_target_platform( 62 | build_wheels, 63 | ): 64 | wheels = build_wheels(["package3[other_platform]"]) 65 | assert not [wheel for wheel in wheels if wheel.name == "package2"] 66 | --------------------------------------------------------------------------------