├── CHANGELOG.md ├── tests ├── cli │ ├── __init__.py │ ├── test_convert.py │ └── test_install.py ├── packages │ ├── has-build-dep │ │ ├── has_dep │ │ │ └── __init__.py │ │ └── pyproject.toml │ └── has-data-files │ │ ├── pyproject.toml │ │ └── setup.py ├── pypi_local_index │ ├── demo-package │ │ ├── demo_package-0.1.0.tar.gz │ │ ├── demo_package-0.1.0-py3-none-any.whl │ │ └── index.html │ ├── index.html │ └── generate_index.py ├── test_repodata.py ├── test_utils.py ├── conftest.py ├── test_main.py ├── test_build.py ├── test_downloader.py ├── test_installer_data_files.py ├── test_conda_meta_json_filename.py ├── test_plugins.py ├── test_conda_create.py ├── test_validate.py ├── test_editable.py ├── test_convert_tree.py ├── test_benchmarks.py └── test_install.py ├── MANIFEST.in ├── docs ├── changelog.md ├── robots.txt ├── _static │ └── css │ │ └── custom.css ├── reference │ ├── commands │ │ ├── convert.rst │ │ ├── install.rst │ │ └── index.rst │ ├── troubleshooting.md │ └── conda-channels-naming-analysis.md ├── modules.md ├── why │ ├── potential-solutions.md │ ├── existing-strategies.md │ ├── index.md │ └── conda-vs-pypi.md ├── developer │ ├── developer-notes.md │ └── architecture.md ├── index.md ├── quickstart.md ├── conf.py └── features.md ├── .gitattributes ├── conda_pypi ├── cli │ ├── __init__.py │ ├── main.py │ ├── convert.py │ └── install.py ├── post_command │ ├── __init__.py │ └── install.py ├── pre_command │ ├── __init__.py │ ├── extract_whl_or_tarball.py │ └── extract_whl.py ├── __main__.py ├── exceptions.py ├── paths.py ├── dependencies │ ├── __init__.py │ └── pypi.py ├── index.py ├── __init__.py ├── dependencies_subprocess.py ├── data │ └── EXTERNALLY-MANAGED ├── utils.py ├── plugin.py ├── conda_build_utils.py ├── whl.py ├── downloader.py ├── installer.py ├── python_paths.py ├── build.py ├── convert_tree.py └── translate.py ├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ ├── pypi.yml │ ├── docs.yml │ ├── test.yml │ └── build.yml ├── config.yaml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── .pre-commit-config.yaml ├── recipe └── meta.yaml ├── .gitignore └── pyproject.toml /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | conda_pypi/data/* 2 | -------------------------------------------------------------------------------- /tests/packages/has-build-dep/has_dep/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/changelog.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | ```{include} ../CHANGELOG.md 3 | ``` 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # GitHub syntax highlighting 2 | pixi.lock linguist-language=YAML 3 | -------------------------------------------------------------------------------- /conda_pypi/cli/__init__.py: -------------------------------------------------------------------------------- 1 | from conda_pypi.cli import main 2 | 3 | __all__ = ["main"] 4 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Default owners for all files in the repository 2 | * @conda-incubator/conda-pypi 3 | -------------------------------------------------------------------------------- /docs/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | 3 | Sitemap: https://conda-incubator.github.io/conda-pypi/sitemap.xml 4 | -------------------------------------------------------------------------------- /conda_pypi/post_command/__init__.py: -------------------------------------------------------------------------------- 1 | from conda_pypi.post_command import install 2 | 3 | __all__ = ["install"] 4 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | flask: 2 | - "3.0.3" 3 | click: 4 | - "8.1.7" 5 | markupsafe: 6 | - "2.1.3" 7 | werkzeug: 8 | - "3.0.4" 9 | -------------------------------------------------------------------------------- /tests/packages/has-data-files/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /conda_pypi/pre_command/__init__.py: -------------------------------------------------------------------------------- 1 | from conda_pypi.pre_command import extract_whl, extract_whl_or_tarball 2 | 3 | __all__ = ["extract_whl", "extract_whl_or_tarball"] 4 | -------------------------------------------------------------------------------- /conda_pypi/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from conda.cli.main import main_subshell 4 | 5 | if __name__ == "__main__": 6 | sys.exit(main_subshell("pip", *sys.argv[1:])) 7 | -------------------------------------------------------------------------------- /tests/pypi_local_index/demo-package/demo_package-0.1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conda-incubator/conda-pypi/HEAD/tests/pypi_local_index/demo-package/demo_package-0.1.0.tar.gz -------------------------------------------------------------------------------- /conda_pypi/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Errors specific to conda pypi. 3 | """ 4 | 5 | from conda.exceptions import CondaError 6 | 7 | 8 | class CondaPypiError(CondaError): 9 | pass 10 | -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | /** 2 | * This rule is here to avoid the scrollbar appearing when this 3 | * is not hosted on ReadTheDocs 4 | */ 5 | #rtd-footer-container { 6 | display: none; 7 | } 8 | -------------------------------------------------------------------------------- /tests/pypi_local_index/demo-package/demo_package-0.1.0-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conda-incubator/conda-pypi/HEAD/tests/pypi_local_index/demo-package/demo_package-0.1.0-py3-none-any.whl -------------------------------------------------------------------------------- /conda_pypi/paths.py: -------------------------------------------------------------------------------- 1 | """ 2 | Paths we need to know. 3 | """ 4 | 5 | from pathlib import Path 6 | 7 | import conda.common.path 8 | 9 | 10 | def get_python_executable(prefix: Path): 11 | return Path(prefix, conda.common.path.get_python_short_path()) 12 | -------------------------------------------------------------------------------- /docs/reference/commands/convert.rst: -------------------------------------------------------------------------------- 1 | ``conda pypi convert`` 2 | ********************** 3 | 4 | .. argparse:: 5 | :module: conda_pypi.cli.main 6 | :func: generate_parser 7 | :prog: conda pypi 8 | :path: convert 9 | :nodefault: 10 | :nodefaultconst: 11 | -------------------------------------------------------------------------------- /docs/reference/commands/install.rst: -------------------------------------------------------------------------------- 1 | ``conda pypi install`` 2 | ********************** 3 | 4 | .. argparse:: 5 | :module: conda_pypi.cli.main 6 | :func: generate_parser 7 | :prog: conda pypi 8 | :path: install 9 | :nodefault: 10 | :nodefaultconst: 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | commit-message: 8 | prefix: "CI: " 9 | groups: 10 | github-actions: 11 | patterns: 12 | - "*" 13 | -------------------------------------------------------------------------------- /tests/packages/has-build-dep/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | # packaging has no dependencies in recent versions and is available on conda-forge 3 | requires = ["flit_core", "packaging"] 4 | 5 | [project] 6 | name = "has_dep" 7 | version = "1" 8 | description = "has dependency unlikely to exist in test environment" 9 | -------------------------------------------------------------------------------- /conda_pypi/dependencies/__init__.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from conda_pypi.dependencies.pypi import ( 4 | check_dependencies, 5 | ensure_requirements, 6 | MissingDependencyError, 7 | ) 8 | 9 | __all__ = [ 10 | "check_dependencies", 11 | "ensure_requirements", 12 | "MissingDependencyError", 13 | ] 14 | -------------------------------------------------------------------------------- /tests/pypi_local_index/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Simple index 7 | 8 | 9 | demo-package 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/packages/has-data-files/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name="test-package-with-data", 5 | version="1.0.0", 6 | packages=find_packages(where="src"), 7 | package_dir={"": "src"}, 8 | data_files=[ 9 | ("share/test-package-with-data/data", ["src/test_package_with_data/data/test.txt"]), 10 | ("share/test-package-with-data", ["src/test_package_with_data/share/config.json"]), 11 | ], 12 | ) 13 | -------------------------------------------------------------------------------- /docs/reference/commands/index.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Commands 3 | ======== 4 | 5 | Conda-pypi provides commands for better PyPI interoperability with conda. 6 | The links on this page provide help for each command. 7 | You can also access help from the command line with the 8 | ``--help`` flag: 9 | 10 | .. code-block:: bash 11 | 12 | conda pypi install --help 13 | 14 | The following commands are part of conda-pypi: 15 | 16 | .. toctree:: 17 | :maxdepth: 1 18 | 19 | install 20 | convert 21 | -------------------------------------------------------------------------------- /conda_pypi/index.py: -------------------------------------------------------------------------------- 1 | """ 2 | Interface to conda-index. 3 | """ 4 | 5 | from conda_index.index import ChannelIndex 6 | 7 | 8 | def update_index(path): 9 | channel_index = ChannelIndex( 10 | path, 11 | None, 12 | threads=1, 13 | debug=False, 14 | write_bz2=False, 15 | write_zst=True, 16 | write_run_exports=True, 17 | compact_json=True, 18 | write_current_repodata=False, 19 | ) 20 | channel_index.index(patch_generator=None) 21 | channel_index.update_channeldata() 22 | -------------------------------------------------------------------------------- /conda_pypi/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | conda-pypi 3 | """ 4 | 5 | try: 6 | from ._version import __version__ 7 | except ImportError: 8 | # _version.py is only created after running `pip install` 9 | try: 10 | from setuptools_scm import get_version 11 | 12 | __version__ = get_version(root="..", relative_to=__file__) 13 | except (ImportError, OSError, LookupError): 14 | # ImportError: setuptools_scm isn't installed 15 | # OSError: git isn't installed 16 | # LookupError: setuptools_scm unable to detect version 17 | __version__ = "0.0.0.dev0+placeholder" 18 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | You'll need a copy of `pixi` and `git` in your machine. Then: 4 | 5 | 1. Clone this repo to disk. 6 | 2. Configure conda-forge to your channels if you haven't already: 7 | ```bash 8 | conda config --add channels conda-forge 9 | ``` 10 | This ensures tests can find packages from conda-forge 11 | 3. `pixi run test` to run the tests. Choose your desired Python version by picking the adequate environment. 12 | 4. `pixi run docs` to build the docs and `pixi run serve` to serve them in your browser. 13 | 5. `pixi run lint` to run the pre-commit linters and formatters. 14 | -------------------------------------------------------------------------------- /conda_pypi/pre_command/extract_whl_or_tarball.py: -------------------------------------------------------------------------------- 1 | from conda.gateways.disk.create import extract_tarball 2 | from conda_pypi.pre_command import extract_whl 3 | 4 | 5 | def extract_whl_or_tarball( 6 | source_full_path, 7 | target_full_path=None, 8 | progress_update_callback=None, 9 | ): 10 | if source_full_path.endswith(".whl"): 11 | return extract_whl.extract_whl_as_conda_pkg( 12 | source_full_path, 13 | target_full_path, 14 | ) 15 | else: 16 | return extract_tarball( 17 | source_full_path, target_full_path, progress_update_callback=progress_update_callback 18 | ) 19 | -------------------------------------------------------------------------------- /tests/test_repodata.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test functions for transforming repodata. 3 | """ 4 | 5 | from conda_pypi.translate import FileDistribution, MatchSpec, conda_to_requires 6 | 7 | 8 | def test_file_distribution(): 9 | dist = FileDistribution( 10 | """\ 11 | Metadata-Version: 2.1 12 | Name: conda_pypi 13 | Version: 0.0.1 14 | """ 15 | ) 16 | metadata = dist.read_text("METADATA") or "" 17 | assert "conda_pypi" in metadata 18 | assert dist.read_text("missing") is None 19 | assert dist.locate_file("always None") is None 20 | 21 | 22 | def test_translate_twine(): 23 | requirement = conda_to_requires(MatchSpec("twine==6.0.0")) 24 | assert requirement.name == "twine" 25 | -------------------------------------------------------------------------------- /tests/cli/test_convert.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | from conda.cli.main import main_subshell 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "source, editable", 9 | [ 10 | ("tests/packages/has-build-dep", False), 11 | ("tests/packages/has-build-dep", True), 12 | ], 13 | ) 14 | def test_convert_writes_output(tmp_path, source, editable): 15 | out_dir = tmp_path / "out" 16 | out_dir.mkdir() 17 | 18 | args = ["pypi", "convert", "--output-folder", str(out_dir)] 19 | if editable: 20 | args.append("-e") 21 | args.append(source) 22 | main_subshell(*args) 23 | 24 | files = list(out_dir.glob("*.conda")) 25 | assert files, f"No .conda artifacts found in {out_dir}" 26 | 27 | assert files[0].is_file() 28 | assert os.path.getsize(files[0]) > 0 29 | -------------------------------------------------------------------------------- /conda_pypi/dependencies_subprocess.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run under target Python interpreter to get dependencies using pypa/build. 3 | 4 | Alternative implementation would use conda PrefixData plus conda/pypi name 5 | translation; but this one supports extras and the format in pyproject.toml. 6 | """ 7 | 8 | import json 9 | import sys 10 | 11 | import build 12 | 13 | 14 | def check_dependencies(build_system_requires): 15 | missing = [u for d in build_system_requires for u in build.check_dependency(d)] 16 | return missing 17 | 18 | 19 | def main(argv): 20 | name, flag, requirements = argv 21 | assert flag == "-r" 22 | requirements = json.loads(requirements) 23 | stuff = check_dependencies(requirements) 24 | return json.dumps(stuff) 25 | 26 | 27 | if __name__ == "__main__": # pragma: no cover 28 | print(main(sys.argv)) 29 | -------------------------------------------------------------------------------- /conda_pypi/data/EXTERNALLY-MANAGED: -------------------------------------------------------------------------------- 1 | [externally-managed] 2 | Error=You are trying to modify a conda environment! 3 | 4 | If you want to install PyPI packages into this conda environment, 5 | please consider using 'conda pypi install ...' instead. 6 | This will make a few checks to ensure that the PyPI packages 7 | you want to install will not disrupt your conda environment. 8 | 9 | If you still want to proceed with 'pip install ...', you can continue 10 | at your own risk by doing one of the following: 11 | 12 | - Add the '--break-system-packages' flag to your command 13 | - Set the 'PIP_BREAK_SYSTEM_PACKAGES' environment variable to '1' 14 | 15 | Be aware that this might render your conda environment unusable! 16 | Check the following links for more information: 17 | 18 | - https://packaging.python.org/en/latest/specifications/externally-managed-environments/ 19 | - https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-break-system-packages 20 | -------------------------------------------------------------------------------- /tests/pypi_local_index/demo-package/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Links for demo-package 8 | 9 | 10 |

Links for demo-package

11 | demo_package-0.1.0.tar.gz
12 | demo_package-0.1.0-py3-none-any.whl 13 | 14 | 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Jaime Rodríguez-Guerra 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/modules.md: -------------------------------------------------------------------------------- 1 | # Reference 2 | 3 | ## build 4 | 5 | ```{eval-rst} 6 | .. automodule:: conda_pypi.build 7 | :members: 8 | :undoc-members: 9 | :show-inheritance: 10 | ``` 11 | 12 | ## conda_build_utils 13 | 14 | ```{eval-rst} 15 | .. automodule:: conda_pypi.conda_build_utils 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | ``` 20 | 21 | ## translate 22 | 23 | ```{eval-rst} 24 | .. automodule:: conda_pypi.translate 25 | :members: 26 | :undoc-members: 27 | :show-inheritance: 28 | ``` 29 | 30 | ## editable 31 | 32 | ```{eval-rst} 33 | .. automodule:: conda_pypi.convert_tree 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | ``` 38 | 39 | ## index 40 | 41 | ```{eval-rst} 42 | .. automodule:: conda_pypi.index 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | ``` 47 | 48 | ## installer 49 | 50 | ```{eval-rst} 51 | .. automodule:: conda_pypi.installer 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | ``` 56 | 57 | ## downloader 58 | 59 | ```{eval-rst} 60 | .. automodule:: conda_pypi.downloader 61 | :members: 62 | :undoc-members: 63 | :show-inheritance: 64 | ``` 65 | 66 | ```{toctree} 67 | :maxdepth: 4 68 | ``` 69 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # conda-pypi 2 | 3 | Better PyPI interoperability for the conda ecosystem. 4 | 5 | > [!IMPORTANT] 6 | > This project is still in early stages of development. Don't use it in production (yet). 7 | > We do welcome feedback on what the expected behaviour should have been if something doesn't work! 8 | 9 | ## What is this? 10 | 11 | Includes: 12 | 13 | - `conda pypi install`: Converts PyPI packages to `.conda` format for safer installation. 14 | - `conda pypi install -e .`: Converts a path to an editable `.conda` format package. 15 | - `conda pypi convert`: Convert PyPI packages to `.conda` format without installing them. 16 | - Adds `EXTERNALLY-MANAGED` to your environments. 17 | 18 | ## Why? 19 | 20 | Mixing conda and PyPI is often discouraged in the conda ecosystem. 21 | There are only a handful patterns that are safe to run. This tool 22 | aims to provide a safer way of keeping your conda environments functional 23 | while mixing it with PyPI dependencies. Refer to the [documentation](docs/) 24 | for more details. 25 | 26 | ## Attribution 27 | 28 | This project now incorporates [conda-pupa](https://github.com/dholth/conda-pupa) 29 | by Daniel Holth, which provides the core PyPI-to-conda conversion functionality. 30 | 31 | ## Contributing 32 | 33 | Please refer to [`CONTRIBUTING.md`](/CONTRIBUTING.md). 34 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | """Tests for the utils module.""" 2 | 3 | from __future__ import annotations 4 | 5 | import pytest 6 | from conda_pypi.utils import pypi_spec_variants 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "input_spec,expected_count", 11 | [ 12 | ("setuptools-scm", 2), 13 | ("setuptools_scm", 2), 14 | ("numpy", 1), 15 | ("setuptools-scm>=1.0", 3), 16 | ("a-b_c", 3), 17 | ], 18 | ) 19 | def test_pypi_spec_variants_generates_correct_count(input_spec: str, expected_count: int): 20 | """Test that pypi_spec_variants generates the expected number of variants.""" 21 | variants = list(pypi_spec_variants(input_spec)) 22 | assert len(variants) == expected_count 23 | assert len(variants) == len(set(variants)) 24 | 25 | 26 | def test_pypi_spec_variants_preserves_original(): 27 | """Test that the original specification is always the first variant.""" 28 | assert list(pypi_spec_variants("setuptools-scm"))[0] == "setuptools-scm" 29 | assert list(pypi_spec_variants("setuptools_scm"))[0] == "setuptools_scm" 30 | 31 | 32 | def test_pypi_spec_variants_creates_name_variants(): 33 | """Test that pypi_spec_variants creates hyphen/underscore variants.""" 34 | variants = list(pypi_spec_variants("setuptools-scm")) 35 | assert "setuptools-scm" in variants 36 | assert "setuptools_scm" in variants 37 | -------------------------------------------------------------------------------- /conda_pypi/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | import sys 5 | 6 | from logging import getLogger 7 | from pathlib import Path 8 | from typing import Iterator 9 | 10 | from conda.base.context import context, locate_prefix_by_name 11 | from conda.models.match_spec import MatchSpec 12 | 13 | 14 | logger = getLogger(f"conda.{__name__}") 15 | 16 | 17 | def get_prefix(prefix: os.PathLike = None, name: str = None) -> Path: 18 | if prefix: 19 | return Path(prefix) 20 | elif name: 21 | return Path(locate_prefix_by_name(name)) 22 | else: 23 | return Path(context.target_prefix) 24 | 25 | 26 | def pypi_spec_variants(spec_str: str) -> Iterator[str]: 27 | yield spec_str 28 | spec = MatchSpec(spec_str) 29 | seen = {spec_str} 30 | for name_variant in ( 31 | spec.name.replace("-", "_"), 32 | spec.name.replace("_", "-"), 33 | ): 34 | if name_variant not in seen: # only yield if actually different 35 | yield str(MatchSpec(spec, name=name_variant)) 36 | seen.add(name_variant) 37 | 38 | 39 | class SuppressOutput: 40 | def __enter__(self): 41 | self._original_stdout = sys.stdout 42 | sys.stdout = open(os.devnull, "w") 43 | 44 | def __exit__(self, exc_type, exc_val, exc_tb): 45 | sys.stdout.close() 46 | sys.stdout = self._original_stdout 47 | -------------------------------------------------------------------------------- /docs/why/potential-solutions.md: -------------------------------------------------------------------------------- 1 | # Potential solutions 2 | 3 | So far, we have outlined why these packaging ecosystems do not always work together 4 | well and a couple strategies users have used in the past to overcome them. How exactly 5 | does the `conda-pypi` plugin plan on addressing them? Below are a couple of methods 6 | we've discussed to address these issues. 7 | 8 | ## On-the-fly conversion of PyPI wheels to conda packages 9 | 10 | The inspiration for this approach initially started with the [conda-pupa](https://github.com/dholth/conda-pupa) 11 | project. The philosophy used here is that we can simply convert a wheel from PyPI into a conda 12 | package and cache it on the host locally. In conda, it's quite easy to configure multiple channels 13 | to be used when installing packages, and by default, a "local" channel is included. As `conda-pypi` 14 | is run, it will begin transforming and caching wheels from PyPI into the conda pacakges which 15 | are then saved in this local channel. 16 | 17 | This is the approach we currently feel most confident with implementing. 18 | 19 | ## Analyze the dependency tree of your PyPI package 20 | 21 | In this approach, we run `pip` with the `--dry-run` option and analyze the proposed solution. Of those packages, 22 | we see which ones are already available on the configured conda channels and install them with `conda` proper. 23 | For the ones that are not available, we pass them to `pip install --no-deps` and hope for an ABI compatible setting. 24 | 25 | This was an approach we initially tried but then gave up in favor for the "conda-pupa" approach. 26 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | import sys 4 | from pathlib import Path 5 | 6 | from xprocess import ProcessStarter 7 | 8 | pytest_plugins = ( 9 | # Add testing fixtures and internal pytest plugins here 10 | "conda.testing", 11 | "conda.testing.fixtures", 12 | ) 13 | HERE = Path(__file__).parent 14 | 15 | 16 | @pytest.fixture(autouse=True) 17 | def do_not_register_envs(monkeypatch): 18 | """Do not register environments created during tests""" 19 | monkeypatch.setenv("CONDA_REGISTER_ENVS", "false") 20 | 21 | 22 | @pytest.fixture(autouse=True) 23 | def do_not_notify_outdated_conda(monkeypatch): 24 | """Do not notify about outdated conda during tests""" 25 | monkeypatch.setenv("CONDA_NOTIFY_OUTDATED_CONDA", "false") 26 | 27 | 28 | @pytest.fixture(scope="session") 29 | def pypi_demo_package_wheel_path() -> Path: 30 | return HERE / "pypi_local_index" / "demo-package" / "demo_package-0.1.0-py3-none-any.whl" 31 | 32 | 33 | @pytest.fixture(scope="session") 34 | def pypi_local_index(xprocess): 35 | """ 36 | Runs a local PyPI index by serving the folder "tests/pypi_local_index" 37 | """ 38 | port = "8035" 39 | 40 | class Starter(ProcessStarter): 41 | pattern = "Serving HTTP on" 42 | timeout = 10 43 | args = [sys.executable, "-m", "http.server", "-d", HERE / "pypi_local_index", port] 44 | env = os.environ.copy() 45 | env["PYTHONUNBUFFERED"] = "1" 46 | 47 | # ensure process is running and return its logfile 48 | xprocess.ensure("pypi_local_index", Starter) 49 | 50 | yield f"http://localhost:{port}" 51 | 52 | xprocess.getinfo("pypi_local_index").terminate() 53 | -------------------------------------------------------------------------------- /docs/why/existing-strategies.md: -------------------------------------------------------------------------------- 1 | # Existing Strategies 2 | 3 | There are currently only a handful of patterns that are considered safe 4 | when install PyPI packages inside a conda environment. We list these 5 | scenarios below: 6 | 7 | ## Only install Python & pip inside conda environments 8 | 9 | In this scenario, users only install Python and `pip` inside of a clean 10 | conda environment. Here, we simply use conda as an environment manager and 11 | let `pip` managed the project dependencies. 12 | 13 | This is what that typically looks like: 14 | 15 | ```console 16 | $ conda create -n pip-environment python=3.10 pip 17 | $ conda activate pip-environment 18 | $ pip install ... 19 | ``` 20 | 21 | ## Editable installs 22 | 23 | In this scenario, `conda` provides all the dependencies of a given package. 24 | Then that package is installed on top in editable mode, without addressing dependencies 25 | to make sure we don't accidentally overwrite conda files: 26 | 27 | ```console 28 | $ git clone git@github.com:owner/package.git 29 | $ conda create -n editable-install package --deps-only 30 | $ conda activate editable-install 31 | $ pip install -e . --no-deps 32 | ``` 33 | 34 | ## Package your PyPI dependencies as conda packages 35 | 36 | This is the safest option in terms of ensuring maximum stability, but it is 37 | also the most time-consuming. Maintaining a separate conda package can be a cumbersome 38 | process and requires continued attention as the newer versions of the pacakge 39 | are released. 40 | 41 | For those that want to choose this approach, tools like [Grayskull](https://conda.github.io/grayskull/) 42 | exist to make it easier to transform a Python package into a conda package recipe. 43 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | from conda_pypi.main import run_conda_cli, run_conda_install 2 | 3 | 4 | def test_run_conda_cli(mocker): 5 | mock = mocker.patch("conda_pypi.main.main_subshell") 6 | 7 | args = ["install", "--prefix", "/path/to/prefix"] 8 | run_conda_cli(*args) 9 | 10 | # Ensure conda module is called 11 | mock.assert_called_once_with(*args) 12 | 13 | 14 | def test_run_conda_install_basic(mocker): 15 | mock = mocker.patch("conda_pypi.main.run_conda_cli") 16 | 17 | run_conda_install( 18 | prefix="idontexist", 19 | specs=[ 20 | "numpy", 21 | ], 22 | ) 23 | 24 | # Ensure conda module is called 25 | mock.assert_called_once_with("install", "--prefix", "idontexist", "numpy") 26 | 27 | 28 | def test_run_conda_install(mocker): 29 | mock = mocker.patch("conda_pypi.main.run_conda_cli") 30 | 31 | run_conda_install( 32 | prefix="idontexist", 33 | specs=["numpy", "scipy"], 34 | dry_run=True, 35 | quiet=True, 36 | verbosity=2, 37 | force_reinstall=True, 38 | yes=True, 39 | json=True, 40 | channels=["mychannel", "abc"], 41 | override_channels=True, 42 | ) 43 | 44 | # Ensure conda module is called 45 | mock.assert_called_once_with( 46 | "install", 47 | "--prefix", 48 | "idontexist", 49 | "--dry-run", 50 | "--quiet", 51 | "-vv", 52 | "--force-reinstall", 53 | "--yes", 54 | "--json", 55 | "--channel", 56 | "mychannel", 57 | "--channel", 58 | "abc", 59 | "--override-channels", 60 | "numpy", 61 | "scipy", 62 | ) 63 | -------------------------------------------------------------------------------- /docs/why/index.md: -------------------------------------------------------------------------------- 1 | # Motivation and Vision 2 | 3 | Although very common among conda users, using `conda` and PyPI (i.e. `pip`) together 4 | can under certain circumstances cause difficult to debug issues and is currently not 5 | seen as 100% stable. `conda-pypi` exists as a solution for creating a better experience 6 | when using these two packaging ecosystems together. Here, we give a high-level overview 7 | of why this conda plugin exists and finish by outlining our strategies for happily combining 8 | the use of both conda and PyPI. 9 | 10 | ## The vision 11 | 12 | `conda-pypi` aims to make it easier and safer to add PyPI packages to existing conda environments. 13 | We acknowledge that we will not be able to solve all problems, specifically as they relate to 14 | binary distributions of packages, but we believe we can provide users with a way to safely 15 | install pure Python packages in conda environments. 16 | 17 | ## The details 18 | 19 | To provide a thorough explanation of the problem and our proposed solutions, we have organized 20 | this section of the documentaiton into the following pages: 21 | 22 | - [Key Differences between conda and PyPI](conda-vs-pypi.md) 23 | gives you a firm understanding of the problems that occur when usig conda and PyPI together. 24 | - [Existing Strategies](existing-strategies.md) shows how users currently deal with 25 | limitations of using conda and PyPI together. 26 | - [Addressing these Issues with conda-pypi](potential-solutions.md) 27 | explains how this plugin can improve the user experience of mixing these two packaging 28 | ecosystems. 29 | 30 | ```{toctree} 31 | :hidden: 32 | 33 | conda-vs-pypi 34 | existing-strategies 35 | potential-solutions 36 | ``` 37 | -------------------------------------------------------------------------------- /conda_pypi/plugin.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from conda.plugins import hookimpl 4 | from conda.plugins.types import CondaSubcommand, CondaPostCommand 5 | 6 | from conda_pypi import cli 7 | from conda_pypi import post_command 8 | from conda_pypi.main import ensure_target_env_has_externally_managed 9 | from conda_pypi.whl import add_whl_support 10 | 11 | 12 | @hookimpl 13 | def conda_subcommands(): 14 | yield CondaSubcommand( 15 | name="pypi", 16 | action=cli.main.execute, 17 | configure_parser=cli.main.configure_parser, 18 | summary="Install PyPI packages as conda packages", 19 | ) 20 | 21 | 22 | @hookimpl 23 | def conda_post_commands(): 24 | yield CondaPostCommand( 25 | name="conda-pypi-ensure-target-env-has-externally-managed", 26 | action=ensure_target_env_has_externally_managed, 27 | run_for={"install", "create", "update", "remove"}, 28 | ) 29 | yield CondaPostCommand( 30 | name="conda-pypi-post-install-create", 31 | action=post_command.install.post_command, 32 | run_for={"install", "create"}, 33 | ) 34 | 35 | 36 | # @hookimpl 37 | # def conda_pre_commands() -> Generator[plugins.CondaPreCommand, None, None]: 38 | # yield CondaPreCommand( 39 | # name="conda-whl-support", 40 | # action=lambda _ : add_whl_support(), 41 | # run_for={ 42 | # "create", 43 | # "install", 44 | # "remove", 45 | # "rename", 46 | # "update", 47 | # "env_create", 48 | # "env_update", 49 | # "list", 50 | # }, 51 | # ) 52 | 53 | # Commenting out the plugin implementation and directly calling `add_whl_support`. 54 | add_whl_support() 55 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPI 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - main 8 | release: 9 | types: 10 | - published 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 17 | with: 18 | fetch-tags: true 19 | fetch-depth: 0 20 | 21 | - name: Set up Python 22 | uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 23 | with: 24 | python-version: "3.x" 25 | 26 | - name: Install build tools 27 | run: python -m pip install --upgrade pip wheel setuptools build twine 28 | 29 | - name: Build wheels 30 | run: python -m build --sdist --wheel . --outdir dist 31 | 32 | - name: Check wheels 33 | working-directory: dist 34 | run: | 35 | ls -alh 36 | python -m pip install *.whl 37 | python -m twine check * 38 | 39 | - name: Upload release distributions 40 | uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 41 | with: 42 | name: release-dists 43 | path: dist/ 44 | 45 | publish: 46 | runs-on: ubuntu-latest 47 | if: github.event_name == 'release' 48 | needs: [build] 49 | permissions: 50 | id-token: write 51 | 52 | steps: 53 | - name: Retrieve release distributions 54 | uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 55 | with: 56 | name: release-dists 57 | path: dist/ 58 | - name: Publish release distributions to PyPI 59 | uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 60 | -------------------------------------------------------------------------------- /tests/test_build.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import json 3 | 4 | from conda_package_streaming import package_streaming 5 | 6 | from conda.testing.fixtures import TmpEnvFixture 7 | from conda.common.path import get_python_short_path 8 | 9 | from conda_pypi.build import build_conda 10 | 11 | 12 | def test_build_conda_package( 13 | tmp_env: TmpEnvFixture, 14 | pypi_demo_package_wheel_path: Path, 15 | tmp_path: Path, 16 | ): 17 | build_path = tmp_path / "build" 18 | build_path.mkdir() 19 | 20 | repo_path = tmp_path / "repo" 21 | repo_path.mkdir() 22 | 23 | target_package_path = repo_path / "demo-package-0.1.0-pypi_0.conda" 24 | 25 | with tmp_env("python=3.12", "pip") as prefix: 26 | conda_package_path = build_conda( 27 | pypi_demo_package_wheel_path, 28 | build_path, 29 | repo_path, 30 | Path(prefix, get_python_short_path()), 31 | is_editable=False, 32 | ) 33 | assert conda_package_path is not None 34 | 35 | # Get a list of all the files in the package 36 | included_package_paths = [ 37 | mm.name for _, mm in package_streaming.stream_conda_component(target_package_path) 38 | ] 39 | 40 | # Get the list of all the paths listed in the paths.json file 41 | for tar, member in package_streaming.stream_conda_info(target_package_path): 42 | if member.name == "info/paths.json": 43 | paths_json = json.load(tar.extractfile(member)) 44 | paths_json_paths = [path.get("_path") for path in paths_json.get("paths")] 45 | break 46 | 47 | # Ensure that the path.json file matches the packages up paths 48 | for path in paths_json_paths: 49 | assert path in included_package_paths 50 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # disable autofixing PRs, commenting "pre-commit.ci autofix" on a pull request triggers a autofix 2 | ci: 3 | autofix_prs: false 4 | # generally speaking we ignore all vendored code as well as tests data 5 | # TODO: Restore index and solver exclude lines before merge 6 | exclude: | 7 | (?x)^( 8 | tests/data/.* | 9 | conda_pypi/data/.* | 10 | pixi.lock 11 | )$ 12 | repos: 13 | # generic verification and formatting 14 | - repo: https://github.com/pre-commit/pre-commit-hooks 15 | rev: v6.0.0 16 | hooks: 17 | # standard end of line/end of file cleanup 18 | - id: mixed-line-ending 19 | - id: end-of-file-fixer 20 | - id: trailing-whitespace 21 | # ensure syntaxes are valid 22 | - id: check-toml 23 | - id: check-yaml 24 | exclude: | 25 | (?x)^( 26 | (conda\.)?recipe/meta.yaml 27 | ) 28 | # catch git merge/rebase problems 29 | - id: check-merge-conflict 30 | - repo: https://github.com/adamchainz/blacken-docs 31 | rev: 1.20.0 32 | hooks: 33 | # auto format Python codes within docstrings 34 | - id: blacken-docs 35 | - repo: https://github.com/astral-sh/ruff-pre-commit 36 | rev: v0.14.9 37 | hooks: 38 | # lint & attempt to correct failures (e.g. pyupgrade) 39 | - id: ruff 40 | args: [--fix] 41 | # compatible replacement for black 42 | - id: ruff-format 43 | - repo: meta 44 | # see https://pre-commit.com/#meta-hooks 45 | hooks: 46 | - id: check-hooks-apply 47 | - id: check-useless-excludes 48 | - repo: local 49 | hooks: 50 | - id: git-diff 51 | name: git diff 52 | entry: git diff --exit-code 53 | language: system 54 | pass_filenames: false 55 | always_run: true 56 | -------------------------------------------------------------------------------- /tests/test_downloader.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test handling of packages without wheels (source distributions only). 3 | 4 | This test verifies issue #121: when a package has no wheel available (only source 5 | distributions), the conversion should fail early with a meaningful error rather 6 | than looping until max attempts (20). 7 | """ 8 | 9 | import os 10 | from pathlib import Path 11 | 12 | import pytest 13 | from conda.testing.fixtures import TmpEnvFixture 14 | 15 | from conda_pypi.exceptions import CondaPypiError 16 | 17 | 18 | REPO = Path(__file__).parents[1] / "synthetic_repo" 19 | 20 | 21 | def test_downloader_detects_no_wheels(tmp_env: TmpEnvFixture, monkeypatch, tmp_path: Path): 22 | """ 23 | Test that the downloader correctly raises an error when no wheels are available. 24 | """ 25 | from conda_pypi.downloader import get_package_finder, find_and_fetch 26 | import tempfile 27 | 28 | CONDA_PKGS_DIRS = tmp_path / "test-pkgs" 29 | CONDA_PKGS_DIRS.mkdir(exist_ok=True) 30 | monkeypatch.setitem(os.environ, "CONDA_PKGS_DIRS", str(CONDA_PKGS_DIRS)) 31 | 32 | with tmp_env("python=3.12", "pip") as prefix: 33 | finder = get_package_finder(prefix) 34 | 35 | # Package "ach" only has source distributions 36 | with pytest.raises(CondaPypiError) as exc_info: 37 | with tempfile.TemporaryDirectory() as tmpdir: 38 | find_and_fetch(finder, Path(tmpdir), "ach") 39 | 40 | # Verify we get a meaningful error message 41 | error_msg = str(exc_info.value).lower() 42 | assert "wheel" in error_msg, f"Expected error message to mention 'wheel', got: {error_msg}" 43 | assert "source distributions" in error_msg or "only source" in error_msg, ( 44 | f"Expected error message to mention source distributions, got: {error_msg}" 45 | ) 46 | -------------------------------------------------------------------------------- /tests/test_installer_data_files.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for installer data file handling. 3 | 4 | Tests that data files in wheels are properly installed. 5 | """ 6 | 7 | from pathlib import Path 8 | 9 | import pytest 10 | from conda.testing.fixtures import TmpEnvFixture 11 | from conda.common.path import get_python_short_path 12 | from conda.base.context import context 13 | 14 | from conda_pypi import installer 15 | from conda_pypi.build import build_pypa 16 | 17 | 18 | @pytest.fixture(scope="session") 19 | def test_package_wheel_path(tmp_path_factory): 20 | """Build a wheel from the test package with data files.""" 21 | package_path = Path(__file__).parent / "packages" / "has-data-files" 22 | wheel_output = tmp_path_factory.mktemp("wheels") 23 | prefix = Path(context.default_prefix) 24 | 25 | return build_pypa( 26 | package_path, 27 | wheel_output, 28 | prefix=prefix, 29 | distribution="wheel", 30 | ) 31 | 32 | 33 | @pytest.mark.skip(reason="Test has CI-only failures that need investigation") 34 | def test_install_installer_data_files_present( 35 | tmp_env: TmpEnvFixture, 36 | test_package_wheel_path: Path, 37 | tmp_path: Path, 38 | ): 39 | """Test that data files from wheels are installed in build_path.""" 40 | build_path = tmp_path / "build" 41 | build_path.mkdir() 42 | 43 | with tmp_env("python=3.12", "pip") as prefix: 44 | python_executable = Path(prefix, get_python_short_path()) / "python" 45 | 46 | installer.install_installer( 47 | str(python_executable), 48 | test_package_wheel_path, 49 | build_path, 50 | ) 51 | 52 | # Data files should be installed in build_path/share/ (data scheme) 53 | data_file = build_path / "share" / "test-package-with-data" / "data" / "test.txt" 54 | 55 | assert data_file.exists(), f"Data file not found at {data_file}" 56 | -------------------------------------------------------------------------------- /conda_pypi/conda_build_utils.py: -------------------------------------------------------------------------------- 1 | # Copied from and adapted from conda-build 2 | # Copyright (C) 2014 Anaconda, Inc 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | 5 | import hashlib 6 | from enum import Enum 7 | from os import DirEntry 8 | from os.path import isfile, islink 9 | from typing import Optional 10 | 11 | 12 | class PathType(Enum): 13 | """ 14 | Refers to if the file in question is hard linked or soft linked. Originally 15 | designed to be used in paths.json 16 | """ 17 | 18 | hardlink = "hardlink" 19 | softlink = "softlink" 20 | directory = "directory" # rare or unused? 21 | 22 | # these additional types should not be included by conda-build in packages 23 | linked_package_record = "linked_package_record" # a package's .json file in conda-meta 24 | pyc_file = "pyc_file" 25 | unix_python_entry_point = "unix_python_entry_point" 26 | windows_python_entry_point_script = "windows_python_entry_point_script" 27 | windows_python_entry_point_exe = "windows_python_entry_point_exe" 28 | 29 | def __str__(self): 30 | return self.name 31 | 32 | def __json__(self): 33 | return self.name 34 | 35 | 36 | def sha256_checksum(filename, entry: Optional[DirEntry] = None, buffersize=1 << 18): 37 | if not entry: 38 | is_link = islink(filename) 39 | is_file = isfile(filename) 40 | else: 41 | is_link = entry.is_symlink() 42 | is_file = entry.is_file() 43 | if is_link and not is_file: 44 | # symlink to nowhere so an empty file 45 | # this is the sha256 hash of an empty file 46 | return "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" 47 | if not is_file: 48 | return None 49 | sha256 = hashlib.sha256() 50 | with open(filename, "rb") as f: 51 | for block in iter(lambda: f.read(buffersize), b""): 52 | sha256.update(block) 53 | return sha256.hexdigest() 54 | -------------------------------------------------------------------------------- /tests/test_conda_meta_json_filename.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test that wheel files installed via conda-pypi create JSON files in conda-meta/ 3 | with the correct filename format: name-version-build.json 4 | """ 5 | 6 | from __future__ import annotations 7 | 8 | import json 9 | from pathlib import Path 10 | 11 | 12 | def test_extract_whl_sets_fn_correctly( 13 | pypi_demo_package_wheel_path: Path, 14 | tmp_path: Path, 15 | ): 16 | """ 17 | Test that extract_whl_as_conda_pkg sets the fn field correctly in index.json. 18 | This is a unit test that directly tests the metadata creation. 19 | """ 20 | from conda_pypi.pre_command.extract_whl import extract_whl_as_conda_pkg 21 | 22 | extract_whl_as_conda_pkg(str(pypi_demo_package_wheel_path), str(tmp_path)) 23 | 24 | # Check that index.json was created with correct fn field 25 | index_json_path = tmp_path / "info" / "index.json" 26 | assert index_json_path.exists() 27 | 28 | with open(index_json_path) as f: 29 | index_data = json.load(f) 30 | 31 | # Verify fn field is set correctly with build string and .whl extension 32 | # Note: source.distribution returns the Python package name (with underscores), 33 | # so fn will be "demo_package-0.1.0-pypi_0.whl" not "demo-package-0.1.0-pypi_0.whl" 34 | assert "fn" in index_data, "index.json should contain 'fn' field" 35 | # The fn field should include the build string and .whl extension 36 | assert index_data["fn"].endswith("-pypi_0.whl") 37 | # Verify the format is name-version-build.whl 38 | fn_parts = index_data["fn"].replace(".whl", "").rsplit("-", 2) 39 | assert len(fn_parts) == 3 40 | fn_name, fn_version, fn_build = fn_parts 41 | assert fn_build == "pypi_0" 42 | 43 | # Verify other fields 44 | # The name field uses the Python package name (with underscores) 45 | assert index_data["name"] == "demo_package" 46 | assert index_data["version"] == "0.1.0" 47 | assert index_data["build"] == "pypi_0" 48 | assert index_data["build_number"] == 0 49 | -------------------------------------------------------------------------------- /conda_pypi/dependencies/pypi.py: -------------------------------------------------------------------------------- 1 | """ 2 | Check dependencies in a prefix, either by using Conda's functionality or 3 | Python's in a subprocess. 4 | """ 5 | 6 | import importlib.resources 7 | import json 8 | import subprocess 9 | from pathlib import Path 10 | from typing import Iterable, List 11 | 12 | from conda.cli.main import main_subshell 13 | 14 | from conda_pypi import paths 15 | from conda_pypi.translate import requires_to_conda 16 | 17 | 18 | class MissingDependencyError(Exception): 19 | """ 20 | When the dependency subprocess can't run. 21 | """ 22 | 23 | def __init__(self, dependencies: List[str]): 24 | self.dependencies = dependencies 25 | 26 | 27 | def check_dependencies(requirements: Iterable[str], prefix: Path): 28 | python_executable = str(paths.get_python_executable(prefix)) 29 | dependency_getter = ( 30 | importlib.resources.files("conda_pypi").joinpath("dependencies_subprocess.py").read_text() 31 | ) 32 | try: 33 | result = subprocess.run( 34 | [ 35 | python_executable, 36 | "-I", 37 | "-", 38 | "-r", 39 | json.dumps(sorted(requirements)), 40 | ], 41 | encoding="utf-8", 42 | input=dependency_getter, 43 | capture_output=True, 44 | check=True, 45 | ) 46 | missing = json.loads(result.stdout) 47 | except subprocess.CalledProcessError as e: 48 | if ( 49 | "ModuleNotFound" in e.stderr 50 | ): # Missing 'build' dependency aka 'python-build' in conda land 51 | raise MissingDependencyError(["build"]) from e 52 | else: 53 | raise 54 | 55 | return missing 56 | 57 | 58 | def ensure_requirements(requirements: List[str], prefix: Path): 59 | if requirements: 60 | conda_requirements, _ = requires_to_conda(requirements) 61 | # -y may be appropriate during tests only 62 | main_subshell("install", "--prefix", str(prefix), "-y", *conda_requirements) 63 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Docs 2 | on: 3 | push: 4 | branches: 5 | - main 6 | tags: 7 | - "*" 8 | pull_request: 9 | paths: 10 | - ".github/workflows/docs.yml" 11 | - "docs/**" 12 | workflow_dispatch: 13 | 14 | concurrency: 15 | # Concurrency group that uses the workflow name and PR number if available 16 | # or commit SHA as a fallback. If a new build is triggered under that 17 | # concurrency group while a previous build is running it will be canceled. 18 | # Repeated pushes to a PR will cancel all previous builds, while multiple 19 | # merges to main will not cancel. 20 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} 21 | cancel-in-progress: true 22 | 23 | jobs: 24 | docs: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 28 | with: 29 | fetch-depth: 0 30 | - uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3 31 | with: 32 | environments: docs 33 | - name: Setup project 34 | run: | 35 | pixi reinstall --environment docs conda-pypi 36 | pixi run --environment docs dev 37 | - name: Build docs 38 | run: pixi run docs 39 | - name: Upload artifact 40 | uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 41 | with: 42 | path: 'docs/_build/dirhtml' 43 | 44 | pages: 45 | runs-on: ubuntu-latest 46 | if: github.ref == 'refs/heads/main' 47 | needs: [docs] 48 | 49 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 50 | permissions: 51 | contents: read 52 | pages: write 53 | id-token: write 54 | 55 | environment: 56 | name: github-pages 57 | url: ${{ steps.deployment.outputs.page_url }} 58 | 59 | steps: 60 | - name: Deploy to GitHub Pages 61 | id: deployment 62 | uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 63 | -------------------------------------------------------------------------------- /recipe/meta.yaml: -------------------------------------------------------------------------------- 1 | package: 2 | name: conda-pypi 3 | {% if GIT_DESCRIBE_TAG is defined and GIT_BUILD_STR is defined %} 4 | version: {{ GIT_DESCRIBE_TAG }}+{{ GIT_BUILD_STR }} 5 | {% else %} 6 | version: 0.0.0dev0 7 | {% endif %} 8 | 9 | source: 10 | # git_url only captures committed code 11 | git_url: ../ 12 | 13 | build: 14 | # can't be noarch because we can't place EXTERNALLY-MANAGED in stdlib (first level is site-packages) 15 | number: 0 16 | script: 17 | - set -x # [unix] 18 | - "@ECHO ON" # [win] 19 | - {{ PYTHON }} -m pip install . --no-deps --no-build-isolation -vv 20 | - {{ PYTHON }} -c "from conda_pypi.python_paths import ensure_externally_managed; ensure_externally_managed()" 21 | 22 | requirements: 23 | host: 24 | - python 25 | - pip 26 | - hatchling >=1.12.2 27 | - hatch-vcs >=0.2.0 28 | - importlib_resources # [py<39] 29 | run: 30 | - python 31 | - conda >=23.9.0 32 | - pip >=23.0.1 33 | - importlib_resources # [py<39] 34 | - packaging 35 | - unearth 36 | - python-build 37 | - python-installer 38 | - platformdirs 39 | - conda-index 40 | - conda-package-streaming 41 | - conda-rattler-solver 42 | 43 | test: 44 | imports: 45 | - conda_pypi 46 | - conda_pypi.main 47 | commands: 48 | - python -m conda pypi --help 49 | - python -c "from conda_pypi.python_paths import get_env_stdlib; assert (get_env_stdlib() / 'EXTERNALLY-MANAGED').exists()" 50 | - python -c "from conda_pypi.python_paths import get_env_stdlib; content = (get_env_stdlib() / 'EXTERNALLY-MANAGED').read_text(); assert '[externally-managed]' in content" 51 | - python -c "from conda_pypi.python_paths import get_env_stdlib; content = (get_env_stdlib() / 'EXTERNALLY-MANAGED').read_text(); assert 'conda pypi' in content" 52 | - python -m pip install requests && exit 1 || exit 0 53 | 54 | about: 55 | home: https://github.com/conda-incubator/conda-pypi 56 | license: MIT 57 | license_file: LICENSE 58 | summary: Better PyPI interoperability for the conda ecosystem 59 | dev_url: https://github.com/conda-incubator/conda-pypi 60 | -------------------------------------------------------------------------------- /tests/test_plugins.py: -------------------------------------------------------------------------------- 1 | from conda.testing.fixtures import CondaCLIFixture, TmpEnvFixture 2 | from pytest_mock import MockerFixture 3 | from conda_pypi.pre_command import extract_whl_or_tarball 4 | from conda_pypi.pre_command.extract_whl import extract_whl_as_conda_pkg 5 | import pytest 6 | from pathlib import Path 7 | 8 | 9 | WHL_HTTP_URL = "https://files.pythonhosted.org/packages/45/7f/0e961cf3908bc4c1c3e027de2794f867c6c89fb4916fc7dba295a0e80a2d/boltons-25.0.0-py3-none-any.whl" 10 | CONDA_URL = "https://repo.anaconda.com/pkgs/main/osx-arm64/boltons-25.0.0-py314hca03da5_0.conda" 11 | 12 | 13 | @pytest.mark.parametrize( 14 | "package,call_count", 15 | [ 16 | pytest.param(WHL_HTTP_URL, 1, id=".whl url"), 17 | pytest.param("{file}", 1, id=".whl file"), 18 | pytest.param("file:///{file}", 1, id=".whl file url"), 19 | pytest.param(CONDA_URL, 0, id=".conda url"), 20 | ], 21 | ) 22 | def test_extract_whl_as_conda_called( 23 | tmp_env: TmpEnvFixture, 24 | conda_cli: CondaCLIFixture, 25 | mocker: MockerFixture, 26 | pypi_demo_package_wheel_path: Path, 27 | tmp_pkgs_dir: Path, # use empty package cache directory 28 | tmp_path: Path, 29 | package: str, 30 | call_count: int, 31 | ): 32 | package = package.format(file=pypi_demo_package_wheel_path) 33 | with tmp_env() as prefix: 34 | # mock python installed in prefix 35 | mocker.patch( 36 | "conda.core.link.UnlinkLinkTransaction._get_python_info", 37 | return_value=("3.10", str(tmp_path)), 38 | ) 39 | 40 | # spy on monkeypatches 41 | spy_extract_whl_as_conda_pkg = mocker.spy( 42 | extract_whl_or_tarball.extract_whl, "extract_whl_as_conda_pkg" 43 | ) 44 | 45 | # install package 46 | conda_cli("install", f"--prefix={prefix}", package) 47 | 48 | # wheel extraction only happens for .whl 49 | assert spy_extract_whl_as_conda_pkg.call_count == call_count 50 | 51 | 52 | def test_extract_whl_as_conda_pkg( 53 | pypi_demo_package_wheel_path: Path, 54 | tmp_path: Path, 55 | ): 56 | extract_whl_as_conda_pkg(pypi_demo_package_wheel_path, tmp_path) 57 | -------------------------------------------------------------------------------- /conda_pypi/whl.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | 4 | log = logging.getLogger(__name__) 5 | 6 | 7 | def mocked_is_package_file(path): 8 | return path[-6:] == ".conda" or path[-8:] == ".tar.bz2" or path[-4:] == ".whl" 9 | 10 | 11 | mocked_url_pat = re.compile( 12 | r"(?:(?P.+)(?:[/\\]))?" 13 | r"(?P[^/\\#]+(?:\.tar\.bz2|\.conda|\.whl))" 14 | r"(?:#(" 15 | r"(?P[0-9a-f]{32})" 16 | r"|((sha256:)?(?P[0-9a-f]{64}))" 17 | r"))?$" 18 | ) 19 | 20 | 21 | def add_whl_support() -> None: 22 | """Implement support for installing wheels in conda""" 23 | log.debug("Inside add_whl_support") 24 | 25 | # add .whl to KNOWN EXTENSIONS 26 | import conda.common.path 27 | 28 | conda.common.path.KNOWN_EXTENSIONS = ( 29 | ".conda", 30 | ".tar.bz2", 31 | ".json", 32 | ".jlap", 33 | ".json.zst", 34 | ".whl", 35 | ) 36 | 37 | # Patch the extract_tarball function 38 | # Add support for extracting wheels with in-line creation of conda metadata files 39 | import conda.core.path_actions 40 | from conda_pypi.pre_command.extract_whl_or_tarball import extract_whl_or_tarball 41 | 42 | conda.core.path_actions.extract_tarball = extract_whl_or_tarball 43 | 44 | # Allow the creation of prefix record JSON files for .whl files 45 | import conda.core.prefix_data 46 | 47 | conda.core.prefix_data.CONDA_PACKAGE_EXTENSIONS = (".tar.bz2", ".conda", ".whl") 48 | 49 | import conda.models.match_spec 50 | 51 | conda.models.match_spec.is_package_file = mocked_is_package_file 52 | 53 | import conda.misc 54 | 55 | conda.misc.url_pat = mocked_url_pat 56 | 57 | # Conda CLI/argparse uses MatchSpec to validate specs, causing specs to be 58 | # parsed (and cached) before the whl monkeypatching occurs. 59 | # Clear cache after monkeypatching so subsequent MatchSpec usage 60 | # (e.g., in Environment.from_cli) will parse the spec correctly. 61 | conda.models.match_spec._PARSE_CACHE.clear() 62 | 63 | # TODO 64 | # There is some extension handling taking place in `conda.models.match_spec.MatchSpec.from_dist_str`` 65 | # that we might need to patch 66 | return 67 | -------------------------------------------------------------------------------- /conda_pypi/cli/main.py: -------------------------------------------------------------------------------- 1 | """ 2 | Entry point for all conda pypi subcommands 3 | 4 | See `conda_pypi.plugin` to see how these are registered with conda 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | import argparse 10 | from logging import getLogger 11 | 12 | from conda.cli.conda_argparse import ( 13 | add_output_and_prompt_options, 14 | add_parser_prefix, 15 | ) 16 | from conda.exceptions import ArgumentError 17 | 18 | from conda_pypi.cli.install import ( 19 | execute as execute_install, 20 | configure_parser as configure_parser_install, 21 | ) 22 | from conda_pypi.cli.convert import ( 23 | execute as execute_convert, 24 | configure_parser as configure_parser_convert, 25 | ) 26 | 27 | 28 | logger = getLogger(__name__) 29 | 30 | 31 | def generate_parser(): 32 | """ 33 | Generate the main argument parser for conda pypi. 34 | 35 | This function is used by Sphinx's sphinxarg extension to automatically 36 | generate CLI documentation from the argparse configuration. 37 | """ 38 | parser = argparse.ArgumentParser( 39 | prog="conda pypi", 40 | description="Better PyPI interoperability for the conda ecosystem.", 41 | formatter_class=argparse.RawDescriptionHelpFormatter, 42 | ) 43 | configure_parser(parser) 44 | return parser 45 | 46 | 47 | def configure_parser(parser: argparse.ArgumentParser): 48 | """ 49 | Entry point for all argparse configuration 50 | """ 51 | 52 | # This adds --prefix/--name mutually exclusive options 53 | add_parser_prefix(parser) 54 | add_output_and_prompt_options(parser) 55 | 56 | sub_parsers = parser.add_subparsers( 57 | metavar="COMMAND", 58 | title="commands", 59 | description="The following subcommands are available.", 60 | dest="cmd", 61 | required=True, 62 | ) 63 | 64 | configure_parser_install(sub_parsers) 65 | configure_parser_convert(sub_parsers) 66 | 67 | 68 | def execute(args: argparse.Namespace) -> int: 69 | if args.cmd == "install": 70 | return execute_install(args) 71 | elif args.cmd == "convert": 72 | return execute_convert(args) 73 | else: 74 | raise ArgumentError(f"Unknown subcommand: {args.cmd}") 75 | -------------------------------------------------------------------------------- /docs/developer/developer-notes.md: -------------------------------------------------------------------------------- 1 | # Developer Notes 2 | 3 | This section contains implementation notes, technical insights, and development considerations for conda-pypi. 4 | 5 | ## PyPI Package Analysis 6 | 7 | Example: https://pypi.org/project/torch/#files 8 | 9 | 2.5.1 e.g., torch has only wheels, no sdists. Is called pytorch in conda-land. 10 | 11 | ## Conda Integration 12 | 13 | LibMambaSolver (used to have) LibMambaIndexHelper.reload_local_channels() used 14 | for conda-build, reloads all file:// indices. 15 | 16 | (Can't figure out where this is used) See also reload_channel(). 17 | 18 | ``` 19 | for channel in conda_build_channels: 20 | index.reload_channel(channel) 21 | ``` 22 | 23 | If we call the solver ourselves or if we use the post-solve hook, we could 24 | handle "metadata but not package data converted" and generate the final .conda's 25 | at that time. While generating repodata from the METADATA files downloaded 26 | separately. 27 | 28 | We could generate unpacked `/pkgs//` directories at the 29 | post-solve hook and skip the `.conda` archive. Conda should think it has already 30 | cached the new wheel -> conda packages. 31 | 32 | In the twine example we wind up converting two versions of a package from wheel 33 | to conda. One of them might have conflicted with the discovered solution. 34 | 35 | Hash of a regular Python package is something like py312hca03da5_0 36 | 37 | ## Environment Markers 38 | 39 | Grab parameters from target Python; evaluate marker. 40 | 41 | ```python 42 | some_environment = packaging.markers.default_environment() 43 | packaging.markers.Marker("python_full_version=='3.12.4'").evaluate( 44 | environment=some_environment 45 | ) 46 | ``` 47 | 48 | `build`, which we use for tests uses environment markers, and extras. 49 | 50 | corpus of metadata from pypi can be used to test marker evaluation. 51 | 52 | ## Architecture Packages 53 | 54 | "arch" packages should be allowed. 55 | 56 | ## Build System Design 57 | 58 | A little bit like conda-build: 59 | 60 | Build packages from wheels or sdists or checkouts, then keep them in the local 61 | channel for later. (But what if we are in CI?) 62 | 63 | ## Editable Installation 64 | 65 | 'editable' command: 66 | 67 | Modern replacement for conda-build develop; works like pip install -e . --no-build-isolation 68 | -------------------------------------------------------------------------------- /tests/pypi_local_index/generate_index.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from pathlib import Path 3 | 4 | GLOBAL_INDEX = """ 5 | 6 | 7 | 8 | 9 | 10 | Simple index 11 | 12 | 13 | __LIST_OF_DIRECTORIES__ 14 | 15 | 16 | """.lstrip() 17 | 18 | PROJECT_INDEX = """ 19 | 20 | 21 | 22 | 23 | 24 | 25 | Links for __NAME__ 26 | 27 | 28 |

Links for __NAME__

29 | __LIST_OF_ARTIFACTS__ 30 | 31 | 32 | """.lstrip() 33 | 34 | HERE = Path(__file__).parent 35 | GLOBAL_INDEX_PATH = HERE / "index.html" 36 | 37 | directories = [d for d in HERE.iterdir() if d.is_dir()] 38 | list_of_directories = "
\n".join( 39 | [f"{directory.name}" for directory in directories] 40 | ) 41 | GLOBAL_INDEX_PATH.write_text(GLOBAL_INDEX.replace("__LIST_OF_DIRECTORIES__", list_of_directories)) 42 | 43 | 44 | def checksum(path, algorithm="sha256", bufsize=1024 * 1024) -> str: 45 | hasher = hashlib.new(algorithm) 46 | with open(path, "rb") as f: 47 | # Use a for loop to iterate over the file in chunks 48 | for chunk in iter(lambda: f.read(bufsize), b""): 49 | hasher.update(chunk) 50 | return hasher.hexdigest() 51 | 52 | 53 | for directory in directories: 54 | print(directory) 55 | artifacts = [] 56 | for path in directory.iterdir(): 57 | if path.name.endswith((".tar.gz", ".whl")) and path.is_file(): 58 | print(path) 59 | sha = checksum(path, "sha256") 60 | artifacts.append( 61 | f'' 64 | f"{path.name}" 65 | ) 66 | (HERE / directory / "index.html").write_text( 67 | PROJECT_INDEX.replace("__NAME__", directory.name).replace( 68 | "__LIST_OF_ARTIFACTS__", "
\n".join(artifacts) 69 | ) 70 | ) 71 | -------------------------------------------------------------------------------- /tests/test_conda_create.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import json 3 | import os 4 | from pathlib import Path 5 | 6 | from conda.cli.main import main_subshell 7 | from conda_package_streaming.create import conda_builder 8 | 9 | from conda_pypi.build import filter, paths_json 10 | from conda_pypi.conda_build_utils import PathType, sha256_checksum 11 | from conda_pypi.index import update_index 12 | from conda_pypi.translate import PackageRecord 13 | 14 | here = Path(__file__).parent 15 | 16 | 17 | def test_indexable(tmp_path): 18 | """ 19 | Create a .conda from scratch; index and install the package. 20 | """ 21 | noarch = tmp_path / "noarch" 22 | noarch.mkdir() 23 | 24 | NAME = "somepackage" 25 | VERSION = "1.0" 26 | 27 | record = PackageRecord(name=NAME, version=VERSION, subdir="noarch", depends=[], extras={}) 28 | dest = tmp_path / record.stem 29 | dest.mkdir() 30 | 31 | (dest / "packaged.txt").write_text("packaged file") 32 | (dest / "info").mkdir() 33 | (dest / "info" / "index.json").write_text(json.dumps(record.to_index_json())) 34 | 35 | paths = paths_json(dest) 36 | (dest / "info" / "paths.json").write_text(json.dumps(paths)) 37 | 38 | with conda_builder(record.stem, noarch) as tar: 39 | tar.add(dest, "", filter=filter) 40 | 41 | update_index(tmp_path) 42 | 43 | repodata = json.loads((noarch / "repodata.json").read_text()) 44 | assert repodata["packages.conda"] 45 | 46 | # name, version, build = dist_str.rsplit("-", 2) must be named like this 47 | dest_package = noarch / f"{record.stem}.conda" 48 | assert dest_package.exists() 49 | 50 | main_subshell( 51 | "create", 52 | "--prefix", 53 | str(tmp_path / "env"), 54 | "--channel", 55 | tmp_path.as_uri(), 56 | "--override-channels", 57 | "-y", 58 | "somepackage", 59 | ) 60 | 61 | conda_meta = json.loads((tmp_path / "env" / "conda-meta" / f"{record.stem}.json").read_text()) 62 | assert len(conda_meta["files"]) 63 | 64 | 65 | def test_path_type(): 66 | assert PathType.hardlink.__json__() == str(PathType.hardlink) 67 | 68 | 69 | def test_checksum(tmp_path): 70 | (tmp_path / "nowhere").symlink_to(tmp_path / "missing") 71 | assert sha256_checksum(tmp_path / "nowhere") == hashlib.sha256().hexdigest() 72 | 73 | # Test FIFO/named pipe handling (Unix only) 74 | if hasattr(os, "mkfifo"): 75 | os.mkfifo(tmp_path / "fifo") 76 | assert sha256_checksum(tmp_path / "fifo") is None 77 | 78 | paths = paths_json(tmp_path) 79 | assert len(paths["paths"]) 80 | -------------------------------------------------------------------------------- /conda_pypi/downloader.py: -------------------------------------------------------------------------------- 1 | """ 2 | Fetch matching wheels from pypi. 3 | """ 4 | 5 | from pathlib import Path 6 | from collections.abc import Iterable 7 | import logging 8 | 9 | from conda.core.prefix_data import PrefixData 10 | from conda.gateways.connection.download import download 11 | from conda.models.match_spec import MatchSpec 12 | from unearth import PackageFinder, TargetPython 13 | 14 | from conda_pypi.exceptions import CondaPypiError 15 | from conda_pypi.translate import conda_to_requires 16 | 17 | log = logging.getLogger(__name__) 18 | 19 | DEFAULT_INDEX_URLS = ("https://pypi.org/simple/",) 20 | 21 | 22 | def get_package_finder( 23 | prefix: Path, 24 | index_urls: Iterable[str] = DEFAULT_INDEX_URLS, 25 | ) -> PackageFinder: 26 | """ 27 | Finder with prefix's Python, not our Python. 28 | """ 29 | prefix_data = PrefixData(prefix) 30 | python_records = list(prefix_data.query("python")) 31 | if not python_records: 32 | raise CondaPypiError(f"Python not found in {prefix}") 33 | py_ver = python_records[0].version 34 | py_ver = tuple(map(int, py_ver.split("."))) 35 | target_python = TargetPython(py_ver=py_ver) 36 | return PackageFinder( 37 | target_python=target_python, 38 | only_binary=":all:", 39 | index_urls=index_urls, 40 | ) 41 | 42 | 43 | def find_package(finder: PackageFinder, package: str): 44 | """ 45 | Convert :package: to `MatchSpec`; return best `Link`. 46 | """ 47 | spec = MatchSpec(package) # type: ignore # metaclass confuses type checker 48 | requirement = conda_to_requires(spec) 49 | if not requirement: 50 | raise RuntimeError(f"Could not convert {package} to Python Requirement()!") 51 | return finder.find_best_match(requirement) 52 | 53 | 54 | def find_and_fetch(finder: PackageFinder, target: Path, package: str) -> Path: 55 | """ 56 | Find package on PyPI, download best link to target. 57 | """ 58 | result = find_package(finder, package) 59 | link = result.best and result.best.link 60 | if not link: 61 | raise CondaPypiError(f"No PyPI link for {package}") 62 | 63 | # Check if the file is a wheel (.whl) 64 | filename = link.url_without_fragment.rsplit("/", 1)[-1] 65 | if not filename.endswith(".whl"): 66 | raise CondaPypiError( 67 | f"No wheel file available for {package}. " 68 | f"Only source distributions are available. " 69 | f"conda-pypi requires wheel files for conversion." 70 | ) 71 | 72 | log.info(f"Fetch {package} as {filename}") 73 | target_path = target / filename 74 | download(link.url, target_path) 75 | return target_path 76 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | .python-version 88 | 89 | # pipenv 90 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 91 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 92 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 93 | # install all needed dependencies. 94 | #Pipfile.lock 95 | 96 | __pypackages__/ 97 | 98 | # Celery stuff 99 | celerybeat-schedule 100 | celerybeat.pid 101 | 102 | # SageMath parsed files 103 | *.sage.py 104 | 105 | # Environments 106 | .env 107 | .venv 108 | env/ 109 | venv/ 110 | ENV/ 111 | env.bak/ 112 | venv.bak/ 113 | 114 | # Spyder project settings 115 | .spyderproject 116 | .spyproject 117 | 118 | # Rope project settings 119 | .ropeproject 120 | 121 | # mkdocs documentation 122 | /site 123 | 124 | # mypy 125 | .mypy_cache/ 126 | .dmypy.json 127 | dmypy.json 128 | 129 | # Pyre type checker 130 | .pyre/ 131 | 132 | # pixi 133 | .pixi/ 134 | 135 | # Used in debugging 136 | explicit.txt 137 | # pip editable clones stuff here 138 | src/ 139 | 140 | # hatch-vcs 141 | conda_pypi/_version.py 142 | 143 | .vscode/ 144 | 145 | *.db 146 | synthetic_repodata.json 147 | synthetic_repo/ 148 | 149 | .codspeed/ 150 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # conda-pypi 2 | 3 | Welcome to the `conda-pypi` documentation! 4 | 5 | `conda-pypi` provides better PyPI interoperability for the conda ecosystem. 6 | It allows you to safely install PyPI packages in conda environments by 7 | converting them to conda format when possible, while falling back to 8 | traditional pip installation when needed. 9 | 10 | The tool offers two main commands: `conda pypi install` for safer PyPI 11 | package installation with an intelligent hybrid approach, and `conda pypi 12 | convert` for converting PyPI packages to `.conda` format without installing 13 | them. The smart installation strategy ensures that explicitly requested 14 | packages come from PyPI while dependencies are sourced from conda channels 15 | when available. 16 | 17 | `conda-pypi` includes full support for development workflows through 18 | editable installations with the `-e` flag, and can install directly from git 19 | repositories and local directories. To protect your conda environments, it 20 | automatically deploys `EXTERNALLY-MANAGED` files to prevent accidental pip 21 | usage that could break your environment's integrity. 22 | 23 | :::{warning} 24 | This project is still in early stages of development. Don't use it in 25 | production (yet). We do welcome feedback on what the expected behaviour 26 | should have been if something doesn't work! 27 | ::: 28 | 29 | ::::{grid} 2 30 | 31 | :::{grid-item-card} 🏡 Getting started 32 | :link: quickstart 33 | :link-type: doc 34 | New to `conda-pypi`? Start here to learn the essentials 35 | ::: 36 | 37 | :::{grid-item-card} 💡 Motivation and vision 38 | :link: why/index 39 | :link-type: doc 40 | Read about why `conda-pypi` exists and when you should use it 41 | ::: 42 | :::: 43 | 44 | ::::{grid} 2 45 | 46 | :::{grid-item-card} 🍱 Features 47 | :link: features 48 | :link-type: doc 49 | Overview of what `conda-pypi` can do for you 50 | ::: 51 | 52 | :::{grid-item-card} 🏗️ Architecture 53 | :link: developer/architecture 54 | :link-type: doc 55 | Technical architecture and plugin system design 56 | ::: 57 | 58 | :::: 59 | 60 | ::::{grid} 2 61 | 62 | :::{grid-item-card} 📚 Commands 63 | :link: reference/commands/index 64 | :link-type: doc 65 | Complete command-line interface documentation 66 | ::: 67 | 68 | :::{grid-item-card} 🔧 Troubleshooting 69 | :link: reference/troubleshooting 70 | :link-type: doc 71 | Common issues and how to resolve them 72 | ::: 73 | 74 | :::{grid-item-card} 🔧 Developer Notes 75 | :link: developer/developer-notes 76 | :link-type: doc 77 | Implementation details and technical insights 78 | ::: 79 | 80 | :::: 81 | 82 | ```{toctree} 83 | :hidden: 84 | 85 | quickstart 86 | why/index 87 | features 88 | modules 89 | changelog 90 | developer/architecture 91 | developer/developer-notes 92 | reference/commands/index 93 | reference/troubleshooting 94 | reference/conda-channels-naming-analysis 95 | ``` 96 | -------------------------------------------------------------------------------- /tests/test_validate.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from subprocess import run 3 | 4 | import pytest 5 | 6 | from conda.testing.fixtures import CondaCLIFixture, TmpEnvFixture 7 | from pytest_mock import MockerFixture 8 | 9 | from conda_pypi.python_paths import get_env_python, get_current_externally_managed_path 10 | 11 | 12 | @pytest.mark.skip(reason="conda-pypi install needs to do more work to support this test.") 13 | def test_externally_managed( 14 | tmp_env: TmpEnvFixture, conda_cli: CondaCLIFixture, monkeypatch: MockerFixture 15 | ): 16 | """ 17 | conda-pypi places its own EXTERNALLY-MANAGED file when it is installed in an environment. 18 | We also need to place it in _new_ environments created by conda. 19 | """ 20 | monkeypatch.setenv("CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY", "0") 21 | text = get_current_externally_managed_path(sys.prefix).read_text().strip() 22 | assert text.startswith("[externally-managed]") 23 | assert "conda pypi" in text 24 | with tmp_env("python", "pip>=23.0.1") as prefix: 25 | conda_cli("pypi", "-p", prefix, "--yes", "install", "requests") 26 | externally_managed_file = get_current_externally_managed_path(prefix) 27 | 28 | # Check if EXTERNALLY-MANAGED file was created by conda-pip 29 | assert externally_managed_file.exists() 30 | 31 | text = (externally_managed_file).read_text().strip() 32 | assert text.startswith("[externally-managed]") 33 | assert "conda pypi" in text 34 | run( 35 | [get_env_python(prefix), "-m", "pip", "uninstall", "--isolated", "certifi", "-y"], 36 | capture_output=True, 37 | ) 38 | p = run( 39 | [get_env_python(prefix), "-m", "pip", "install", "--isolated", "certifi"], 40 | capture_output=True, 41 | text=True, 42 | ) 43 | print(p.stdout) 44 | print(p.stderr, file=sys.stderr) 45 | assert p.returncode != 0 46 | all_text = p.stderr + p.stdout 47 | assert "externally-managed-environment" in all_text 48 | assert "conda pypi" in all_text 49 | assert "--break-system-packages" in all_text 50 | p = run( 51 | [ 52 | get_env_python(prefix), 53 | "-m", 54 | "pip", 55 | "install", 56 | "--isolated", 57 | "certifi", 58 | "--break-system-packages", 59 | ], 60 | capture_output=True, 61 | text=True, 62 | ) 63 | assert p.returncode == 0 64 | all_text = p.stderr + p.stdout 65 | assert ( 66 | "Requirement already satisfied: certifi" in all_text 67 | or "Successfully installed certifi" in all_text 68 | ) 69 | 70 | # EXTERNALLY-MANAGED is removed when pip is removed 71 | conda_cli("remove", "-p", prefix, "--yes", "pip") 72 | 73 | # EXTERNALLY-MANAGED is automatically added when pip is reinstalled by the plugin hook 74 | conda_cli("install", "-p", prefix, "--yes", "pip") 75 | assert externally_managed_file.exists() 76 | -------------------------------------------------------------------------------- /conda_pypi/cli/convert.py: -------------------------------------------------------------------------------- 1 | from argparse import Namespace, _SubParsersAction 2 | from pathlib import Path 3 | 4 | from conda.auxlib.ish import dals 5 | from conda.base.context import context 6 | from conda.exceptions import ArgumentError 7 | 8 | from conda_pypi import build 9 | 10 | 11 | def configure_parser(parser: _SubParsersAction) -> None: 12 | """ 13 | Configure all subcommand arguments and options via argparse 14 | """ 15 | # convert subcommand 16 | summary = "Build and convert local Python sdists, wheels or projects to conda packages" 17 | description = summary 18 | epilog = dals( 19 | """ 20 | Examples: 21 | 22 | Convert a PyPI package to conda format without installing:: 23 | 24 | conda pypi convert ./requests-2.32.5-py3-none-any.whl 25 | 26 | Convert a local Python project to conda package:: 27 | 28 | conda pypi convert ./my-python-project 29 | 30 | Convert a package and save to a specific output folder:: 31 | 32 | conda pypi convert --output-folder ./conda-packages ./numpy-2.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl 33 | 34 | Convert a local Python project to an editable package:: 35 | 36 | conda pypi convert -e . --output-folder ./conda-packages 37 | 38 | Convert a package from a Git repository:: 39 | 40 | git clone https://github.com/user/repo.git 41 | conda pypi convert ./repo 42 | 43 | """ 44 | ) 45 | 46 | convert = parser.add_parser( 47 | "convert", 48 | help=summary, 49 | description=description, 50 | epilog=epilog, 51 | ) 52 | 53 | convert.add_argument( 54 | "--output-folder", 55 | help="Folder to write output package(s)", 56 | type=Path, 57 | required=False, 58 | default=Path.cwd() / "conda-pypi-output", 59 | ) 60 | convert.add_argument( 61 | "project_path", 62 | metavar="PROJECT", 63 | help="Convert named path/url as wheel converted to conda.", 64 | ) 65 | convert.add_argument( 66 | "-e", 67 | "--editable", 68 | action="store_true", 69 | help="Build PROJECT as an editable package.", 70 | ) 71 | 72 | 73 | def execute(args: Namespace) -> int: 74 | """ 75 | Entry point for the `conda pypi convert` subcommand 76 | """ 77 | prefix_path = Path(context.target_prefix) 78 | if not Path(args.project_path).exists(): 79 | raise ArgumentError("PROJECT must be a local path to a sdist, wheel or directory.") 80 | project_path = Path(args.project_path).expanduser() 81 | output_folder = Path(args.output_folder).expanduser() 82 | output_folder.mkdir(parents=True, exist_ok=True) 83 | 84 | distribution = "editable" if args.editable else "wheel" 85 | package_path = build.pypa_to_conda( 86 | project_path, 87 | distribution=distribution, 88 | output_path=output_folder, 89 | prefix=prefix_path, 90 | ) 91 | print(f"Conda package at {package_path} built successfully. Output folder: {output_folder}.") 92 | return 0 93 | -------------------------------------------------------------------------------- /conda_pypi/installer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Install a wheel / install a conda. 3 | """ 4 | 5 | import os 6 | import subprocess 7 | import tempfile 8 | from pathlib import Path 9 | from unittest.mock import patch 10 | import logging 11 | 12 | from conda.cli.main import main_subshell 13 | from conda.core.package_cache_data import PackageCacheData 14 | from installer import install 15 | from installer.destinations import SchemeDictionaryDestination 16 | from installer.sources import WheelFile 17 | 18 | log = logging.getLogger(__name__) 19 | 20 | 21 | def install_installer(python_executable: str, whl: Path, build_path: Path): 22 | # Handler for installation directories and writing into them. 23 | # Create site-packages directory if it doesn't exist 24 | site_packages = build_path / "site-packages" 25 | site_packages.mkdir(parents=True, exist_ok=True) 26 | 27 | # Use minimal scheme to mimic pip --target: purelib, platlib, and scripts 28 | # See sysconfig documentation for more details on scheme keys. 29 | # https://docs.python.org/3/library/sysconfig.html#installation-paths 30 | scheme = { 31 | "purelib": str(site_packages), # Pure Python packages 32 | "platlib": str(site_packages), # Platform-specific packages 33 | "scripts": str(build_path / "bin"), # Console scripts 34 | "data": str(build_path), # Data files (JS, CSS, templates, etc.) 35 | } 36 | 37 | destination = SchemeDictionaryDestination( 38 | scheme, 39 | interpreter=str(python_executable), 40 | script_kind="posix", 41 | ) 42 | 43 | with WheelFile.open(whl) as source: 44 | install( 45 | source=source, 46 | destination=destination, 47 | # Additional metadata that is generated by the installation tool. 48 | additional_metadata={ 49 | "INSTALLER": b"conda-pypi", 50 | }, 51 | ) 52 | 53 | log.debug(f"Installed to {build_path}") 54 | 55 | 56 | def install_pip(python_executable: str, whl: Path, build_path: Path): 57 | command = [ 58 | python_executable, 59 | "-m", 60 | "pip", 61 | "install", 62 | "--quiet", 63 | "--no-deps", 64 | "--target", 65 | str(build_path / "site-packages"), 66 | whl, 67 | ] 68 | subprocess.run(command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 69 | log.debug(f"Installed to {build_path}") 70 | 71 | 72 | def install_ephemeral_conda(prefix: Path, package: Path): 73 | """ 74 | Install [editable] conda package without adding it to the environment's 75 | package cache, since we don't want to accidentally re-install "a link to a 76 | source checkout" elsewhere. 77 | 78 | Installing packages directly from a file does not resolve dependencies. 79 | Should we automatically install the project's dependencies also? 80 | """ 81 | persistent_pkgs = PackageCacheData.first_writable().pkgs_dir 82 | with ( 83 | tempfile.TemporaryDirectory(dir=persistent_pkgs, prefix="ephemeral") as cache_dir, 84 | patch.dict(os.environ, {"CONDA_PKGS_DIRS": cache_dir}), 85 | ): 86 | main_subshell("install", "--prefix", str(prefix), str(package)) 87 | -------------------------------------------------------------------------------- /docs/developer/architecture.md: -------------------------------------------------------------------------------- 1 | # Architecture 2 | 3 | This page documents the technical architecture of `conda-pypi`, explaining 4 | how it integrates with conda and the internal organization of its components. 5 | 6 | ## Plugin System Integration 7 | 8 | `conda-pypi` is implemented as a conda plugin using `conda`'s official plugin 9 | architecture. This design enables seamless integration with conda's existing 10 | workflows without requiring modifications to conda itself. 11 | 12 | The plugin registers several hooks with `conda`'s plugin system. The 13 | subcommand hook adds the `conda pypi` subcommand to conda through 14 | `conda_pypi.plugin.conda_subcommands()`, providing both `conda pypi install` 15 | for installing PyPI packages with conversion and `conda pypi convert` for 16 | converting PyPI packages without installing them. 17 | 18 | The plugin also registers two post-command hooks that extend conda's 19 | existing commands. The environment protection hook triggers after `install`, 20 | `create`, `update`, and `remove` commands to automatically deploy 21 | `EXTERNALLY-MANAGED` files that prevent accidental `pip` (or any other Python install tool) usage. This is 22 | implemented through `ensure_target_env_has_externally_managed()`. 23 | 24 | ## Data Flow Architecture 25 | 26 | ### Installation Flow 27 | 28 | ``` 29 | conda pypi install package 30 | ↓ 31 | CLI Argument Parsing 32 | ↓ 33 | Environment Validation 34 | ↓ 35 | Package Classification 36 | ↓ 37 | VCS/Editable? ----Yes---→ Direct pip install --> Deploy EXTERNALLY-MANAGED 38 | ↓ No 39 | Dependency Resolution 40 | ↓ 41 | Channel Search for Dependencies 42 | ↓ 43 | Convert Missing from PyPI 44 | ↓ 45 | Install via conda 46 | ↓ 47 | Deploy EXTERNALLY-MANAGED 48 | ``` 49 | 50 | ### Conversion Flow 51 | 52 | ``` 53 | conda pypi convert package 54 | ↓ 55 | Fetch from PyPI 56 | ↓ 57 | Download Wheels 58 | ↓ 59 | Convert to .conda 60 | ↓ 61 | Save to Output Directory 62 | ``` 63 | 64 | ### Plugin Hook Flow 65 | 66 | ``` 67 | conda command executed 68 | ↓ 69 | Post-command hook? 70 | ↓ ↓ 71 | install/ install/create/ 72 | create update/remove 73 | ↓ ↓ 74 | Process Deploy 75 | PyPI EXTERNALLY- 76 | lines MANAGED 77 | ↓ ↓ 78 | Install Create marker 79 | PyPI files 80 | packages 81 | ``` 82 | 83 | ## Key Design Principles 84 | 85 | The architecture of `conda-pypi` is built around several key design 86 | principles that ensure effective integration between conda and PyPI 87 | ecosystems. 88 | 89 | Conda-native integration is achieved by using `conda`'s official plugin 90 | system and leveraging `conda`'s existing infrastructure including solvers, 91 | channels, and metadata systems. This approach maintains full compatibility 92 | with existing conda workflows. 93 | 94 | This hybrid approach ensures that explicit packages always come 95 | from PyPI to respect user intent, while dependencies prefer conda channels 96 | for ecosystem compatibility. The system falls back to PyPI conversion only 97 | when needed. 98 | 99 | This architecture enables conda-pypi to provide a seamless bridge between 100 | the conda and PyPI ecosystems while maintaining the integrity and benefits of 101 | both package management systems. 102 | -------------------------------------------------------------------------------- /conda_pypi/python_paths.py: -------------------------------------------------------------------------------- 1 | """ 2 | Logic to place and find Python paths and EXTERNALLY-MANAGED in target (conda) environments. 3 | 4 | Since functions in this module might be called to facilitate installation of the package, 5 | this module MUST only use the Python stdlib. No 3rd party allowed (except for importlib-resources). 6 | """ 7 | 8 | import os 9 | import sys 10 | import sysconfig 11 | from importlib.resources import files as importlib_files 12 | from logging import getLogger 13 | from pathlib import Path 14 | from subprocess import check_output 15 | from typing import Iterator 16 | 17 | 18 | logger = getLogger(__name__) 19 | 20 | 21 | def get_env_python(prefix: os.PathLike = None) -> Path: 22 | prefix = Path(prefix or sys.prefix) 23 | if os.name == "nt": 24 | return prefix / "python.exe" 25 | return prefix / "bin" / "python" 26 | 27 | 28 | def _get_env_sysconfig_path(key: str, prefix: os.PathLike = None) -> Path: 29 | prefix = Path(prefix or sys.prefix) 30 | if str(prefix) == sys.prefix or prefix.resolve() == Path(sys.prefix).resolve(): 31 | return Path(sysconfig.get_path(key)) 32 | path = check_output( 33 | [get_env_python(prefix), "-c", f"import sysconfig as s; print(s.get_path('{key}'))"], 34 | text=True, 35 | ).strip() 36 | if not path: 37 | raise RuntimeError(f"Could not identify sysconfig path for '{key}' at '{prefix}'") 38 | return Path(path) 39 | 40 | 41 | def get_env_stdlib(prefix: os.PathLike = None) -> Path: 42 | return _get_env_sysconfig_path("stdlib", prefix) 43 | 44 | 45 | def get_env_site_packages(prefix: os.PathLike = None) -> Path: 46 | return _get_env_sysconfig_path("purelib", prefix) 47 | 48 | 49 | def get_current_externally_managed_path(prefix: os.PathLike = None) -> Path: 50 | """ 51 | Returns the path for EXTERNALLY-MANAGED for the given Python installation in 'prefix'. 52 | Not guaranteed to exist. There might be more EXTERNALLY-MANAGED files in 'prefix' for 53 | older Python versions. These are not returned. 54 | 55 | It assumes Python is installed in 'prefix' and will call it with a subprocess if needed. 56 | """ 57 | prefix = Path(prefix or sys.prefix) 58 | return get_env_stdlib(prefix) / "EXTERNALLY-MANAGED" 59 | 60 | 61 | def get_externally_managed_paths(prefix: os.PathLike = None) -> Iterator[Path]: 62 | """ 63 | Returns all the possible EXTERNALLY-MANAGED paths in 'prefix', for all found 64 | Python (former) installations. The paths themselves are not guaranteed to exist. 65 | 66 | This does NOT invoke python's sysconfig because Python might not be installed (anymore). 67 | """ 68 | prefix = Path(prefix or sys.prefix) 69 | if os.name == "nt": 70 | yield prefix / "Lib" / "EXTERNALLY-MANAGED" 71 | else: 72 | for python_dir in sorted(Path(prefix, "lib").glob("python*")): 73 | if python_dir.is_dir(): 74 | yield Path(python_dir, "EXTERNALLY-MANAGED") 75 | 76 | 77 | def ensure_externally_managed(prefix: os.PathLike = None) -> Path: 78 | """ 79 | conda-pypi places its own EXTERNALLY-MANAGED file when it is installed in an environment. 80 | We also need to place it in _new_ environments created by conda. We do this by implementing 81 | some extra plugin hooks. 82 | """ 83 | target_path = get_current_externally_managed_path(prefix) 84 | if not target_path.exists(): 85 | logger.info("Placing EXTERNALLY-MANAGED in %s", target_path.parent) 86 | resource = importlib_files("conda_pypi") / "data" / "EXTERNALLY-MANAGED" 87 | target_path.write_text(resource.read_text()) 88 | return target_path 89 | -------------------------------------------------------------------------------- /docs/quickstart.md: -------------------------------------------------------------------------------- 1 | # Quick start 2 | 3 | ## Installation 4 | 5 | `conda-pypi` is a `conda` plugin that needs to be installed next to 6 | `conda` in the `base` environment: 7 | 8 | ```bash 9 | conda install -n base conda-pypi 10 | ``` 11 | 12 | Once installed, the `conda pypi` subcommand becomes available across all your 13 | conda environments. 14 | 15 | ## Basic usage 16 | 17 | `conda-pypi` provides several {doc}`features`. The main functionality is 18 | accessed through the `conda pypi` command: 19 | 20 | ### Installing PyPI packages 21 | 22 | Assuming you have an activated conda environment named `my-python-env` that 23 | includes `python` and `pip` installed, and a configured conda channel, you can 24 | use `conda pypi install` like this: 25 | 26 | ```bash 27 | conda pypi install niquests 28 | ``` 29 | 30 | This will download and convert `niquests` from PyPI to `.conda` format 31 | (since it was explicitly requested), but install its dependencies from 32 | the conda channel when available. For example, if `niquests` depends on 33 | `urllib3` and `certifi`, and both are available on the conda channel, those 34 | dependencies will be installed from conda rather than PyPI. 35 | 36 | ```bash 37 | conda pypi install build 38 | ``` 39 | 40 | This will download and convert the `build` package from PyPI to `.conda` 41 | format. Even though `python-build` exists on conda, the explicitly requested 42 | package always comes from PyPI to ensure you get exactly what you asked for. 43 | However, its dependencies will preferentially come from conda channels when 44 | available. 45 | 46 | ```bash 47 | conda pypi install some-package-with-many-deps 48 | ``` 49 | 50 | Here's where the hybrid approach really shines: 51 | `some-package-with-many-deps` itself will be converted from PyPI, but 52 | conda-pypi will analyze its dependency tree and: 53 | - Install dependencies like `numpy`, `pandas`, etc. from the conda channel (if 54 | available) 55 | - Convert only the dependencies that aren't available on conda channels from 56 | PyPI 57 | 58 | ```bash 59 | conda pypi install --ignore-channels some-package 60 | ``` 61 | 62 | This forces dependency resolution to use only PyPI, bypassing conda channel 63 | checks for dependencies. The requested package is always converted from PyPI 64 | regardless of this flag. 65 | 66 | ### Converting packages without installing 67 | 68 | You can also convert PyPI packages to `.conda` format without installing 69 | them: 70 | 71 | ```bash 72 | # Convert to current directory 73 | conda pypi convert niquests rope 74 | 75 | # Convert to specific directory 76 | conda pypi convert -d ./my_packages niquests rope 77 | ``` 78 | 79 | This is useful for creating conda packages from PyPI distributions or 80 | preparing packages for offline installation. 81 | 82 | ### Development and editable installations 83 | 84 | `conda-pypi` supports editable installations for development workflows: 85 | 86 | ```bash 87 | # Install local project in editable mode 88 | conda pypi install -e ./my-project/ 89 | 90 | # Install from version control in editable mode 91 | conda pypi install -e git+https://github.com/user/project.git 92 | 93 | # Preview what would be installed 94 | conda pypi install --dry-run niquests pandas 95 | ``` 96 | 97 | ### Environment protection 98 | 99 | `conda-pypi` ships a special file called `EXTERNALLY-MANAGED` that helps 100 | protect your conda environments from accidental pip usage that could break 101 | their integrity. This file is automatically installed in the `base` 102 | environment, all new environments, and existing environments that after running 103 | a `conda pypi` command on them. 104 | 105 | More details about this protection mechanism can be found at 106 | {ref}`externally-managed`. 107 | -------------------------------------------------------------------------------- /tests/cli/test_install.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests that use run `conda pypi install` use `conda_cli` as the primary caller 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | from conda.testing.fixtures import CondaCLIFixture 8 | 9 | import re 10 | import pytest 11 | 12 | 13 | def test_cli(conda_cli): 14 | """ 15 | Test that pypi subcommands exist by checking their help output. 16 | """ 17 | # Test that install subcommand exists and help works 18 | # Help commands raise SystemExit, so we need to handle that 19 | out, err, rc = conda_cli("pypi", "install", "--help", raises=SystemExit) 20 | assert rc.value.code == 0 # SystemExit(0) means success 21 | assert "PyPI packages to install" in out 22 | 23 | # Test that convert subcommand exists and help works 24 | out, err, rc = conda_cli("pypi", "convert", "--help", raises=SystemExit) 25 | assert rc.value.code == 0 26 | assert "Convert named path/url as wheel converted to conda" in out 27 | 28 | 29 | def test_cli_plugin(): 30 | # Test that the plugin can be loaded and the subcommand is registered 31 | from conda_pypi.plugin import conda_subcommands 32 | 33 | subcommands = list(conda_subcommands()) 34 | pypi_subcommand = next((sub for sub in subcommands if sub.name == "pypi"), None) 35 | 36 | assert pypi_subcommand is not None 37 | assert pypi_subcommand.summary == "Install PyPI packages as conda packages" 38 | assert pypi_subcommand.action is not None 39 | assert pypi_subcommand.configure_parser is not None 40 | 41 | 42 | def test_index_urls(tmp_env, conda_cli, pypi_local_index): 43 | with tmp_env("python=3.10") as prefix: 44 | out, err, rc = conda_cli( 45 | "pypi", 46 | "--yes", 47 | "install", 48 | "--ignore-channels", 49 | "--prefix", 50 | prefix, 51 | "--index-url", 52 | pypi_local_index, 53 | "demo-package", 54 | ) 55 | assert "Converted packages\n - demo-package==0.1.0" in out 56 | assert rc == 0 57 | 58 | 59 | def test_install_output(tmp_env, conda_cli): 60 | with tmp_env("python=3.12") as prefix: 61 | out, err, rc = conda_cli( 62 | "pypi", 63 | "--yes", 64 | "install", 65 | "--ignore-channels", 66 | "--prefix", 67 | prefix, 68 | "scipy", 69 | ) 70 | 71 | assert rc == 0 72 | 73 | # strip spinner characters 74 | out = out.replace(" \x08\x08/", "") 75 | out = out.replace(" \x08\x08-", "") 76 | out = out.replace(" \x08\x08\\", "") 77 | out = out.replace(" \x08\x08|", "") 78 | out = out.replace(" \x08\x08", "") 79 | 80 | # Ensure a message about the converted packages is shown 81 | assert "Converted packages" in out 82 | 83 | # Ensure the solver messaging is only showed once when the final solve/install happens 84 | assert len(re.findall(r"Solving environment:", out)) == 1 85 | 86 | 87 | def test_install_jupyterlab_package(tmp_env, conda_cli): 88 | with tmp_env("python=3.10") as prefix: 89 | out, err, rc = conda_cli( 90 | "pypi", 91 | "--yes", 92 | "install", 93 | "--ignore-channels", 94 | "--prefix", 95 | prefix, 96 | "jupyterlab", 97 | ) 98 | assert rc == 0 99 | 100 | 101 | def test_install_requires_package_without_editable(conda_cli: CondaCLIFixture): 102 | with pytest.raises(SystemExit) as exc: 103 | conda_cli("pypi", "install") 104 | assert exc.value.code == 2 105 | 106 | 107 | def test_install_editable_without_packages_succeeds(conda_cli: CondaCLIFixture): 108 | project = "tests/packages/has-build-dep" 109 | out, err, rc = conda_cli("pypi", "install", "-e", project) 110 | assert rc == 0 111 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | push: 4 | branches: 5 | - main 6 | tags: 7 | - "*" 8 | pull_request: 9 | paths: 10 | - ".github/workflows/test.yml" 11 | - "conda_pypi/**" 12 | - "tests/**" 13 | - "pyproject.toml" 14 | - "pixi.toml" 15 | 16 | concurrency: 17 | # Concurrency group that uses the workflow name and PR number if available 18 | # or commit SHA as a fallback. If a new build is triggered under that 19 | # concurrency group while a previous build is running it will be canceled. 20 | # Repeated pushes to a PR will cancel all previous builds, while multiple 21 | # merges to main will not cancel. 22 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} 23 | cancel-in-progress: true 24 | 25 | jobs: 26 | tests: 27 | name: ${{ matrix.os }}, py${{ matrix.python-version }} 28 | runs-on: ${{ matrix.os }} 29 | strategy: 30 | fail-fast: false 31 | matrix: 32 | os: [ubuntu-latest, windows-latest] 33 | python-version: ["310", "311", "312", "313"] 34 | include: 35 | - os: macos-15-intel 36 | python-version: "310" 37 | - os: macos-latest 38 | python-version: "311" 39 | env: 40 | PIXI_ENV_NAME: test-py${{ matrix.python-version }} 41 | steps: 42 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 43 | with: 44 | fetch-depth: 0 45 | - uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3 46 | with: 47 | environments: ${{ env.PIXI_ENV_NAME }} 48 | - name: Fix Ubuntu configuration 49 | if: startswith(matrix.os, 'ubuntu-') 50 | run: 51 | # Remove global pip config that interferes with isolated testing 52 | # See: https://github.com/actions/runner-images/blob/6d025759810a/images/ubuntu/scripts/build/install-python.sh#L15-L21 53 | sudo rm /etc/pip.conf 54 | - name: Setup project 55 | run: | 56 | pixi reinstall --environment ${{ env.PIXI_ENV_NAME }} conda-pypi 57 | pixi run --environment ${{ env.PIXI_ENV_NAME }} dev 58 | echo "channels: [conda-forge]" > .pixi/envs/${{ env.PIXI_ENV_NAME }}/.condarc 59 | pixi run --environment ${{ env.PIXI_ENV_NAME }} conda info 60 | - name: Run tests 61 | run: pixi run --environment ${{ env.PIXI_ENV_NAME }} test --basetemp=${{ runner.os == 'Windows' && 'D:\\temp' || runner.temp }} -vv 62 | 63 | linux-benchmarks: 64 | name: benchmarks py${{ matrix.python-version }} 65 | runs-on: ubuntu-latest 66 | strategy: 67 | fail-fast: false 68 | matrix: 69 | python-version: ["312"] 70 | env: 71 | PIXI_ENV_NAME: test-py${{ matrix.python-version }} 72 | steps: 73 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 74 | with: 75 | fetch-depth: 0 76 | - uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3 77 | with: 78 | environments: ${{ env.PIXI_ENV_NAME }} 79 | - name: Fix Ubuntu configuration 80 | run: 81 | # Remove global pip config that interferes with isolated testing 82 | # See: https://github.com/actions/runner-images/blob/6d025759810a/images/ubuntu/scripts/build/install-python.sh#L15-L21 83 | sudo rm /etc/pip.conf 84 | - name: Setup project 85 | run: | 86 | pixi reinstall --environment ${{ env.PIXI_ENV_NAME }} conda-pypi 87 | pixi run --environment ${{ env.PIXI_ENV_NAME }} dev 88 | echo "channels: [conda-forge]" > .pixi/envs/${{ env.PIXI_ENV_NAME }}/.condarc 89 | pixi run --environment ${{ env.PIXI_ENV_NAME }} conda info 90 | - name: Run benchmarks 91 | uses: CodSpeedHQ/action@346a2d8a8d9d38909abd0bc3d23f773110f076ad # v4.4.1 92 | with: 93 | mode: instrumentation 94 | token: ${{ secrets.CODSPEED_TOKEN }} 95 | run: pixi run --environment ${{ env.PIXI_ENV_NAME }} benchmark 96 | -------------------------------------------------------------------------------- /tests/test_editable.py: -------------------------------------------------------------------------------- 1 | import json 2 | import subprocess 3 | from pathlib import Path 4 | 5 | import pytest 6 | from conda.base.context import context 7 | from conda.cli.main import main_subshell 8 | from conda.core.prefix_data import PrefixData 9 | from packaging.requirements import InvalidRequirement 10 | 11 | import build 12 | import conda_pypi.dependencies_subprocess 13 | from conda_pypi.build import filter, pypa_to_conda 14 | from conda_pypi.dependencies.pypi import ensure_requirements 15 | 16 | 17 | def test_editable(tmp_path): 18 | # Other tests can change context.target_prefix by calling "conda install 19 | # --prefix", so we use default_prefix; could alternatively reset context. 20 | pypa_to_conda( 21 | Path(__file__).parents[1], 22 | output_path=tmp_path, 23 | distribution="editable", 24 | prefix=Path(context.default_prefix), 25 | ) 26 | 27 | 28 | def pypa_build_packages(): 29 | """ 30 | Test packages from pypa/build repository. 31 | 32 | (Clone pypa/build into tests/) 33 | """ 34 | here = Path(__file__).parent 35 | return list(p.name for p in Path(here, "build", "tests", "packages").glob("*")) 36 | 37 | 38 | @pytest.fixture 39 | def package_path(): 40 | here = Path(__file__).parent 41 | return Path(here, "build", "tests", "packages") 42 | 43 | 44 | @pytest.mark.parametrize("package", pypa_build_packages()) 45 | def test_build_wheel(package, package_path, tmp_path): 46 | # Some of these will not contain the editable hook; need to test building 47 | # regular wheels also. Some will require a "yes" for conda install 48 | # dependencies. Some are designed to fail. 49 | xfail = [ 50 | "test-bad-backend", 51 | "test-bad-syntax", 52 | "test-bad-wheel", 53 | "test-cant-build-via-sdist", 54 | "test-invalid-requirements", 55 | "test-metadata", 56 | "test-no-project", 57 | "test-no-requires", 58 | "test-optional-hooks", 59 | ] 60 | 61 | try: 62 | pypa_to_conda( 63 | package_path / package, 64 | output_path=tmp_path, 65 | distribution="wheel", 66 | prefix=Path(context.default_prefix), 67 | ) 68 | except ( 69 | build.BuildException, 70 | build.BuildBackendException, 71 | subprocess.CalledProcessError, 72 | InvalidRequirement, 73 | ) as e: 74 | if package in xfail: 75 | pytest.xfail(reason=str(e)) 76 | 77 | 78 | def test_ensure_requirements(mocker): 79 | mock = mocker.patch("conda_pypi.dependencies.pypi.main_subshell") 80 | ensure_requirements(["flit_core"], prefix=Path()) 81 | # normalizes/converts the underscore flit_core->flit-core 82 | assert mock.call_args.args == ("install", "--prefix", ".", "-y", "flit-core") 83 | 84 | 85 | def test_filter_coverage(): 86 | class tarinfo: 87 | name = ".git" 88 | 89 | assert filter(tarinfo) is None # type: ignore 90 | 91 | 92 | def test_create_build_dir(tmp_path): 93 | # XXX should "create default output_path" logic live in pypa_to_conda? 94 | with pytest.raises(build.BuildException): 95 | pypa_to_conda(tmp_path, prefix=Path(context.default_prefix)) 96 | 97 | 98 | def test_build_in_env(tmp_path): 99 | """Test conda-pypi installed in different environment than editable package.""" 100 | main_subshell( 101 | "create", 102 | "--prefix", 103 | str(tmp_path / "env"), 104 | "-y", 105 | "python=3.11", 106 | "python-build", 107 | ) 108 | 109 | prefix = str(tmp_path / "env") 110 | 111 | main_subshell( 112 | "pypi", 113 | "install", 114 | "--prefix", 115 | prefix, 116 | "-e", 117 | str(Path(__file__).parent / "packages" / "has-build-dep"), 118 | ) 119 | 120 | installed = [ 121 | record.name for record in PrefixData(prefix, pip_interop_enabled=True).iter_records() 122 | ] 123 | 124 | assert "packaging" in installed 125 | 126 | 127 | def test_dependencies_subprocess(): 128 | """ 129 | Normally called in a way that doesn't measure coverage. 130 | """ 131 | dependencies = ["xyzzy", "conda"] 132 | missing_dependencies = conda_pypi.dependencies_subprocess.main( 133 | ["-", "-r", json.dumps(dependencies)] 134 | ) 135 | # A list per checked dependency, if that dependency or its dependencies are 136 | # missing. 137 | assert json.loads(missing_dependencies) == [["xyzzy"]] 138 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling", "hatch-vcs"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "conda-pypi" 7 | description = "Better PyPI interoperability for the conda ecosystem." 8 | readme = "README.md" 9 | authors = [ 10 | { name = "Jaime Rodríguez-Guerra", email = "jrodriguez@quansight.com" }, 11 | ] 12 | license = "MIT" 13 | license-files = [ 14 | "LICENSE", 15 | ] 16 | classifiers = [ 17 | "Programming Language :: Python :: 3", 18 | "Programming Language :: Python :: 3 :: Only", 19 | "Programming Language :: Python :: 3.10", 20 | "Programming Language :: Python :: 3.11", 21 | "Programming Language :: Python :: 3.12", 22 | "Programming Language :: Python :: 3.13", 23 | "Programming Language :: Python :: Implementation :: CPython", 24 | "Programming Language :: Python :: Implementation :: PyPy", 25 | ] 26 | requires-python = ">=3.10" 27 | dependencies = [ 28 | # "conda >=25.11.0", 29 | "pip", 30 | "build", 31 | "conda-index", 32 | "installer", 33 | "packaging", 34 | "platformdirs", 35 | "unearth", 36 | ] 37 | dynamic = ["version"] 38 | 39 | [project.urls] 40 | homepage = "https://github.com/conda-incubator/conda-pypi" 41 | 42 | [project.entry-points.conda] 43 | conda-pypi = "conda_pypi.plugin" 44 | 45 | [tool.pixi.workspace] 46 | channels = ["conda-forge"] 47 | platforms = ["linux-64", "osx-64", "osx-arm64", "win-64", "linux-aarch64"] 48 | 49 | [tool.pixi.dependencies] 50 | # must match project.dependencies (sans python and conda) 51 | python = ">=3.10" 52 | conda = ">=25.11" 53 | pip = "*" 54 | packaging = "*" 55 | conda-index = "*" 56 | pytest-codspeed = ">=3.0.0,<5" 57 | conda-rattler-solver = ">=0.0.3,<0.0.4" 58 | 59 | [tool.pixi.pypi-dependencies] 60 | "conda-pypi" = { path = ".", editable = true } 61 | 62 | [tool.pixi.tasks] 63 | dev = 'python -c "from conda_pypi.python_paths import ensure_externally_managed as e; e()"' 64 | 65 | [tool.pixi.feature.build] 66 | dependencies = { anaconda-client = "*", conda-build = "*" } 67 | tasks = { build = "conda build recipe" } 68 | 69 | [tool.pixi.feature.docs.tasks] 70 | docs = { cmd = "sphinx-build -M dirhtml . _build", cwd = "docs" } 71 | serve = { cmd = "python -m http.server", cwd = "docs/_build/dirhtml" } 72 | clean = { cmd = "rm -rf _build", cwd = "docs" } 73 | 74 | [tool.pixi.feature.docs.dependencies] 75 | python = "3.10.*" 76 | conda-sphinx-theme = "*" 77 | linkify-it-py = "*" 78 | myst-parser = "*" 79 | sphinx = "*" 80 | sphinx-argparse = "*" 81 | sphinx-copybutton = "*" 82 | sphinx-design = "*" 83 | sphinx-reredirects = "*" 84 | sphinx-sitemap = "*" 85 | 86 | [tool.pixi.feature.test.tasks] 87 | test = 'python -c "from conda_pypi.python_paths import ensure_externally_managed as e; e()" && python -mpytest -m "not benchmark"' 88 | benchmark= 'python -c "from conda_pypi.python_paths import ensure_externally_managed as e; e()" && python -mpytest --codspeed' 89 | pre-commit = 'pre-commit' 90 | 91 | [tool.pixi.feature.test.dependencies] 92 | pytest = "7.4.3.*" 93 | fmt = "!=10.2.0" 94 | pytest-mock = "3.12.0.*" 95 | conda-build = "*" 96 | pre-commit = "*" 97 | pytest-xprocess = "*" 98 | # Build system dependencies 99 | conda-package-streaming = ">=0.11" 100 | unearth = ">=0.17.2" 101 | python-build = "*" 102 | python-installer = ">=0.7" 103 | conda-rattler-solver = "*" 104 | packaging = ">=24" 105 | # Dependencies for corpus tests 106 | sqlalchemy = "*" 107 | python-multipart = "*" 108 | pyyaml = "*" 109 | zstandard = "*" # Will be in stdlib as compression.zstd in Python 3.14+ 110 | # Build backend for flit tests 111 | flit = "*" 112 | 113 | [tool.pixi.feature.py310.dependencies] 114 | python = "3.10.*" 115 | 116 | [tool.pixi.feature.py311.dependencies] 117 | python = "3.11.*" 118 | 119 | [tool.pixi.feature.py312.dependencies] 120 | python = "3.12.*" 121 | 122 | [tool.pixi.feature.py313.dependencies] 123 | python = "3.13.*" 124 | 125 | [tool.pixi.environments] 126 | build-py310 = ["build", "py310"] 127 | build-py311 = ["build", "py311"] 128 | build-py312 = ["build", "py312"] 129 | build-py313 = ["build", "py313"] 130 | docs = ["docs"] 131 | test-py310 = ["test", "py310"] 132 | test-py311 = ["test", "py311"] 133 | test-py312 = ["test", "py312"] 134 | test-py313 = ["test", "py313"] 135 | 136 | [tool.hatch.version] 137 | source = "vcs" 138 | 139 | [tool.hatch.build.hooks.vcs] 140 | version-file = "conda_pypi/_version.py" 141 | 142 | [tool.ruff] 143 | line-length = 99 144 | 145 | [tool.coverage.report] 146 | exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] 147 | 148 | [tool.coverage.run] 149 | source = ["conda_pypi/", "tests/"] 150 | omit = ["conda_pypi/__init__.py"] 151 | -------------------------------------------------------------------------------- /docs/reference/troubleshooting.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting 2 | 3 | This guide covers the most common issues you may encounter when using `conda-pypi` and how to resolve them. 4 | 5 | ## Environment Issues 6 | 7 | ### Missing Python or pip requirements 8 | 9 | **Problem**: `conda-pypi` fails with an error about missing Python or pip. 10 | 11 | **Error messages**: 12 | ``` 13 | Target environment at /path/to/env requires python>=3.2 14 | Target environment at /path/to/env requires pip>=23.0.1 15 | ``` 16 | 17 | **Solution**: Ensure your target environment has the required dependencies: 18 | 19 | ```bash 20 | # Install required dependencies in your environment 21 | conda install -n myenv python>=3.9 pip>=23.0.1 22 | 23 | # Or when creating a new environment 24 | conda create -n myenv python>=3.10 pip 25 | ``` 26 | 27 | ### Invalid environment 28 | 29 | **Problem**: Commands fail when specifying a non-existent environment. 30 | 31 | **Error messages**: 32 | - `environment does not exist` 33 | - `python>=3.2 not found` 34 | 35 | **Solution**: 36 | ```bash 37 | # Check if environment exists 38 | conda env list 39 | 40 | # Create the environment if it doesn't exist 41 | conda create -n myenv python=3.10 pip 42 | 43 | # Use correct environment name or path 44 | conda pypi install -n myenv package-name 45 | ``` 46 | 47 | ## Package Resolution Issues 48 | 49 | ### Package not found on PyPI 50 | 51 | **Problem**: Package doesn't exist or has a different name on PyPI. 52 | 53 | **Error messages**: 54 | - `No matching distribution found` 55 | - `Could not find a version that satisfies the requirement` 56 | - `404 Client Error` 57 | 58 | **Solutions**: 59 | ```bash 60 | # Check package name on PyPI 61 | pip search package-name # or visit pypi.org 62 | 63 | # Try common name variations 64 | conda pypi install python-package-name # instead of package-name 65 | ``` 66 | 67 | ### Dependency resolution timeout 68 | 69 | **Problem**: Complex dependency trees exceed the solver's retry limit. 70 | 71 | **Error messages**: 72 | - `Exceeded maximum of 20 attempts` 73 | - `Could not resolve dependencies after 20 attempts` 74 | 75 | **Solutions**: 76 | ```bash 77 | # Use --ignore-channels to simplify resolution 78 | conda pypi install --ignore-channels package-name 79 | 80 | # Install dependencies from conda first 81 | conda install numpy pandas scipy 82 | conda pypi install your-package 83 | 84 | # Try installing packages individually 85 | conda pypi install package1 86 | conda pypi install package2 87 | ``` 88 | 89 | ### Conflicting dependencies 90 | 91 | **Problem**: Package requirements conflict with existing environment. 92 | 93 | **Solutions**: 94 | ```bash 95 | # Preview what would be installed 96 | conda pypi install --dry-run package-name 97 | 98 | # Check existing packages 99 | conda list 100 | 101 | # Create a fresh environment for testing 102 | conda create -n test python=3.10 pip 103 | conda activate test 104 | conda pypi install package-name 105 | ``` 106 | 107 | ## Network and Connectivity Issues 108 | 109 | ### PyPI connectivity problems 110 | 111 | **Problem**: Cannot connect to PyPI servers. 112 | 113 | **Error messages**: 114 | - `Connection timeout` 115 | - `Failed to establish connection` 116 | - `Name resolution failed` 117 | 118 | **Solutions**: 119 | ```bash 120 | # Test basic connectivity 121 | ping pypi.org 122 | 123 | # Try with verbose output to see connection details 124 | conda -v pip install package-name 125 | 126 | # Check conda's network configuration 127 | conda config --show channels 128 | ``` 129 | 130 | ## Getting Help 131 | 132 | ### Enable verbose output 133 | 134 | For detailed debugging information: 135 | 136 | ```bash 137 | # Basic verbose output 138 | conda -v pip install package-name 139 | 140 | # INFO level logging (repeat -v twice) 141 | conda -vv pip install package-name 142 | 143 | # DEBUG level logging (repeat -v three times) - most useful for troubleshooting 144 | conda -vvv pip install package-name 145 | 146 | # TRACE level logging (repeat -v four times) - maximum detail 147 | conda -vvvv pip install package-name 148 | ``` 149 | 150 | ## When to Seek Further Help 151 | 152 | If you encounter issues not covered here: 153 | 154 | 1. **Check the version**: Ensure you're using the latest version of `conda-pypi` 155 | 2. **Search existing issues**: Check the [GitHub repository](https://github.com/conda-incubator/conda-pypi) for similar problems 156 | 3. **Report issues**: When reporting issues please to include all the relevant details 157 | 158 | Remember that `conda-pypi` is still in early development, so feedback about unexpected behavior is valuable for improving the tool. 159 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: 3 | push: 4 | branches: 5 | - main 6 | tags: 7 | - "*" 8 | pull_request: 9 | paths: 10 | - ".github/workflows/test.yml" 11 | - "conda_pypi/**" 12 | - "tests/**" 13 | - "pyproject.toml" 14 | - "pixi.toml" 15 | 16 | concurrency: 17 | # Concurrency group that uses the workflow name and PR number if available 18 | # or commit SHA as a fallback. If a new build is triggered under that 19 | # concurrency group while a previous build is running it will be canceled. 20 | # Repeated pushes to a PR will cancel all previous builds, while multiple 21 | # merges to main will not cancel. 22 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} 23 | cancel-in-progress: true 24 | 25 | jobs: 26 | build-conda: 27 | name: Build conda package (${{ matrix.os }} - ${{matrix.python-version}}) 28 | runs-on: ${{ matrix.os }} 29 | defaults: 30 | run: 31 | shell: bash --noprofile --norc -euo pipefail {0} 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | python-version: ["310", "311", "312", "313"] 36 | os: [ubuntu-latest, windows-latest, macos-latest] 37 | env: 38 | PIXI_ENV_NAME: build-py${{ matrix.python-version }} 39 | PYTHONUNBUFFERED: "1" 40 | CONDA_BLD_PATH: ${{ github.workspace }}/pkgs 41 | PACKAGE_NAME: conda-pypi 42 | ANACONDA_ORG_CHANNEL: conda-canary 43 | ANACONDA_ORG_LABEL: dev 44 | BINSTAR_API_TOKEN: ${{ secrets.ANACONDA_ORG_CONDA_CANARY_TOKEN }} 45 | steps: 46 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 47 | with: 48 | fetch-depth: 0 49 | - uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3 50 | with: 51 | environments: ${{ env.PIXI_ENV_NAME }} 52 | - name: Fix Ubuntu configuration 53 | if: startswith(matrix.os, 'ubuntu-') 54 | run: 55 | # Remove global pip config that interferes with isolated testing 56 | # See: https://github.com/actions/runner-images/blob/6d025759810a/images/ubuntu/scripts/build/install-python.sh#L15-L21 57 | sudo rm /etc/pip.conf 58 | - name: Setup project for building 59 | run: | 60 | pixi run --environment ${{ env.PIXI_ENV_NAME }} dev 61 | pixi run --environment ${{ env.PIXI_ENV_NAME }} conda config --add channels defaults 62 | pixi run --environment ${{ env.PIXI_ENV_NAME }} conda config --add channels conda-forge 63 | - name: Debug conda environment 64 | run: | 65 | pixi run --environment ${{ env.PIXI_ENV_NAME }} conda info 66 | pixi run --environment ${{ env.PIXI_ENV_NAME }} conda config --show-sources 67 | pixi run --environment ${{ env.PIXI_ENV_NAME }} conda list 68 | - name: Clean conda cache 69 | run: pixi run --environment ${{ env.PIXI_ENV_NAME }} conda clean --all --yes 70 | - name: Build recipe 71 | env: 72 | _CONDA_BUILD_ISOLATED_ACTIVATION: "1" 73 | run: pixi run --environment ${{ env.PIXI_ENV_NAME }} build 74 | - name: Upload to conda-canary 75 | # Only publish canary builds after successful pushes to main on the canonical repo 76 | if: >- 77 | github.event_name == 'push' 78 | && !cancelled() 79 | && !github.event.repository.fork 80 | && github.ref_name == 'main' 81 | shell: bash 82 | run: | 83 | SEARCH_ROOT="$CONDA_BLD_PATH/${{ matrix.subdir }}" 84 | if [[ ! -d "$SEARCH_ROOT" ]]; then 85 | echo "Expected build output directory $SEARCH_ROOT not found" >&2 86 | exit 1 87 | fi 88 | 89 | PACKAGES=( 90 | $( 91 | find "$SEARCH_ROOT" -type f \ 92 | \( -name "${PACKAGE_NAME}-*.conda" -o -name "${PACKAGE_NAME}-*.tar.bz2" \) \ 93 | -print | sort 94 | ) 95 | ) 96 | 97 | if [[ ${#PACKAGES[@]} -eq 0 ]]; then 98 | echo "No conda packages found in $SEARCH_ROOT" >&2 99 | exit 1 100 | fi 101 | 102 | printf 'Uploading packages to %s/label/%s:\n' "$ANACONDA_ORG_CHANNEL" "$ANACONDA_ORG_LABEL" 103 | printf ' %s\n' "${PACKAGES[@]}" 104 | pixi run --environment ${{ env.PIXI_ENV_NAME }} \ 105 | anaconda \ 106 | upload \ 107 | --force \ 108 | --register \ 109 | --no-progress \ 110 | --user "$ANACONDA_ORG_CHANNEL" \ 111 | --label "$ANACONDA_ORG_LABEL" \ 112 | "${PACKAGES[@]}" 113 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Configuration file for the Sphinx documentation builder. 3 | # 4 | # For the full list of built-in configuration values, see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | 16 | sys.path.insert(0, os.path.abspath("..")) 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 21 | 22 | project = html_title = "conda-pypi" 23 | copyright = "2024, conda-pypi contributors" 24 | author = "conda-pypi contributors" 25 | 26 | # -- General configuration --------------------------------------------------- 27 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 28 | 29 | # Add any Sphinx extension module names here, as strings. They can be 30 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 31 | # ones. 32 | extensions = [ 33 | "myst_parser", 34 | "sphinx.ext.autodoc", 35 | "sphinx.ext.autosummary", 36 | "sphinx.ext.graphviz", 37 | "sphinx.ext.ifconfig", 38 | "sphinx.ext.inheritance_diagram", 39 | "sphinx.ext.napoleon", 40 | "sphinx.ext.viewcode", 41 | "sphinx_copybutton", 42 | "sphinx_design", 43 | "sphinx_reredirects", 44 | "sphinx_sitemap", 45 | "sphinxarg.ext", 46 | ] 47 | 48 | # Autodoc configuration 49 | autodoc_mock_imports = [] 50 | autodoc_default_options = { 51 | "members": True, 52 | "undoc-members": True, 53 | "show-inheritance": True, 54 | } 55 | 56 | # Handle missing imports gracefully 57 | autodoc_typehints = "description" 58 | 59 | myst_heading_anchors = 3 60 | myst_enable_extensions = [ 61 | "amsmath", 62 | "colon_fence", 63 | "deflist", 64 | "dollarmath", 65 | "html_admonition", 66 | "html_image", 67 | "linkify", 68 | "replacements", 69 | "smartquotes", 70 | "substitution", 71 | "tasklist", 72 | ] 73 | 74 | 75 | # Add any paths that contain templates here, relative to this directory. 76 | templates_path = ["_templates"] 77 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 78 | 79 | # List of patterns, relative to source directory, that match files and 80 | # directories to ignore when looking for source files. 81 | # This pattern also affects html_static_path and html_extra_path. 82 | exclude_patterns = [] 83 | 84 | 85 | # -- Options for HTML output ------------------------------------------------- 86 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 87 | 88 | html_theme = "conda_sphinx_theme" 89 | html_static_path = ["_static"] 90 | 91 | html_css_files = [ 92 | "css/custom.css", 93 | ] 94 | 95 | # Serving the robots.txt since we want to point to the sitemap.xml file 96 | html_extra_path = ["robots.txt"] 97 | 98 | html_theme_options = { 99 | "navigation_depth": -1, 100 | "use_edit_page_button": True, 101 | "navbar_center": ["navbar_center"], 102 | "icon_links": [ 103 | { 104 | "name": "GitHub", 105 | "url": "https://github.com/conda-incubator/conda-pypi", 106 | "icon": "fa-brands fa-square-github", 107 | "type": "fontawesome", 108 | }, 109 | { 110 | "name": "Element", 111 | "url": "https://matrix.to/#/#conda_conda:gitter.im", 112 | "icon": "_static/element_logo.svg", 113 | "type": "local", 114 | }, 115 | { 116 | "name": "Discourse", 117 | "url": "https://conda.discourse.group/", 118 | "icon": "fa-brands fa-discourse", 119 | "type": "fontawesome", 120 | }, 121 | ], 122 | } 123 | 124 | html_context = { 125 | "github_user": "conda-incubator", 126 | "github_repo": "conda-pypi", 127 | "github_version": "main", 128 | "doc_path": "docs", 129 | } 130 | 131 | html_baseurl = "https://conda-incubator.github.io" 132 | 133 | # We don't have a locale set, so we can safely ignore that for the sitemaps. 134 | sitemap_locales = [None] 135 | # We're hard-coding stable here since that's what we want Google to point to. 136 | sitemap_url_scheme = "{link}" 137 | 138 | # -- For sphinx_reredirects ------------------------------------------------ 139 | 140 | redirects = {} 141 | -------------------------------------------------------------------------------- /tests/test_convert_tree.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test converting a dependency tree to conda. 3 | """ 4 | 5 | import os 6 | from pathlib import Path 7 | 8 | from conda.models.match_spec import MatchSpec 9 | from conda.testing.fixtures import TmpEnvFixture 10 | from pytest_mock import MockerFixture 11 | 12 | from conda_pypi.convert_tree import ( 13 | ConvertTree, 14 | parse_libmamba_solver_error, 15 | parse_rattler_solver_error, 16 | ) 17 | from conda_pypi.downloader import get_package_finder 18 | from conda_pypi.exceptions import CondaPypiError 19 | 20 | import pytest 21 | 22 | REPO = Path(__file__).parents[1] / "synthetic_repo" 23 | 24 | 25 | def test_multiple(tmp_env: TmpEnvFixture, tmp_path: Path, monkeypatch: MockerFixture): 26 | """ 27 | Install multiple only-available-from-pypi dependencies into an environment. 28 | """ 29 | CONDA_PKGS_DIRS = tmp_path / "conda-pkgs" 30 | CONDA_PKGS_DIRS.mkdir() 31 | 32 | WHEEL_DIR = tmp_path / "wheels" 33 | WHEEL_DIR.mkdir(exist_ok=True) 34 | 35 | REPO.mkdir(parents=True, exist_ok=True) 36 | 37 | TARGET_DEP = MatchSpec("twine==5.1.1") # type: ignore 38 | 39 | # Defeat package cache for ConvertTree 40 | monkeypatch.setitem(os.environ, "CONDA_PKGS_DIRS", str(CONDA_PKGS_DIRS)) 41 | 42 | with tmp_env("python=3.12", "pip") as prefix: 43 | converter = ConvertTree(prefix, repo=REPO, override_channels=True) 44 | converter.convert_tree([TARGET_DEP]) 45 | 46 | 47 | def test_convert_local_pypi_package( 48 | tmp_env: TmpEnvFixture, 49 | tmp_path: Path, 50 | monkeypatch: MockerFixture, 51 | pypi_local_index: str, 52 | ): 53 | """ 54 | Convert a local pypi package 55 | """ 56 | CONDA_PKGS_DIRS = tmp_path / "conda-pkgs" 57 | CONDA_PKGS_DIRS.mkdir() 58 | 59 | WHEEL_DIR = tmp_path / "wheels" 60 | WHEEL_DIR.mkdir(exist_ok=True) 61 | 62 | REPO.mkdir(parents=True, exist_ok=True) 63 | 64 | TARGET_DEP = MatchSpec("demo-package") # type: ignore 65 | 66 | # Defeat package cache for ConvertTree 67 | monkeypatch.setitem(os.environ, "CONDA_PKGS_DIRS", str(CONDA_PKGS_DIRS)) 68 | 69 | with tmp_env("python=3.12", "pip") as prefix: 70 | finder = get_package_finder(prefix, (pypi_local_index,)) 71 | converter = ConvertTree(prefix, repo=REPO, override_channels=True, finder=finder) 72 | changes = converter.convert_tree([TARGET_DEP]) 73 | 74 | assert len(changes[0]) == 0 75 | assert len(changes[1]) == 1 76 | assert changes[1][0].name == "demo-package" 77 | 78 | 79 | def test_package_without_wheel_should_fail_early( 80 | tmp_env: TmpEnvFixture, tmp_path: Path, monkeypatch 81 | ): 82 | """ 83 | Test that when a package has no wheel available, the convert_tree method 84 | raises CondaPypiError with a meaningful message rather than looping for max_attempts. 85 | 86 | This verifies the fix for issue #121. 87 | """ 88 | CONDA_PKGS_DIRS = tmp_path / "conda-pkgs" 89 | CONDA_PKGS_DIRS.mkdir() 90 | 91 | REPO.mkdir(parents=True, exist_ok=True) 92 | 93 | # "ach" is mentioned in the issue as an example package that only has source distributions 94 | TARGET_PKG = MatchSpec("ach") # type: ignore 95 | 96 | # Defeat package cache for ConvertTree 97 | monkeypatch.setitem(os.environ, "CONDA_PKGS_DIRS", str(CONDA_PKGS_DIRS)) 98 | 99 | with tmp_env("python=3.12", "pip") as prefix: 100 | converter = ConvertTree(prefix, repo=REPO, override_channels=True) 101 | 102 | # Should raise CondaPypiError immediately instead of looping 103 | with pytest.raises(CondaPypiError) as exc_info: 104 | converter.convert_tree([TARGET_PKG], max_attempts=5) 105 | 106 | # Verify we get a meaningful error message 107 | error_msg = str(exc_info.value).lower() 108 | assert "wheel" in error_msg 109 | 110 | 111 | def test_parse_libmamba_solver_error(): 112 | error_message = "'Encountered problems while solving:\n - nothing provides numpy <2.6,>=1.25.2 needed by scipy-1.16.3-pypi_0\n\nCould not solve for environment specs\nThe following package could not be installed\n└─ \x1b[31mscipy =* *\x1b[0m is not installable because it requires\n └─ \x1b[31mnumpy <2.6,>=1.25.2 *\x1b[0m, which does not exist (perhaps a missing channel).'" 113 | assert set(parse_libmamba_solver_error(error_message)) == {"numpy <2.6,>=1.25.2"} 114 | 115 | 116 | def test_parse_rattler_solver_error(): 117 | error_message = "'Cannot solve the request because of: scipy * cannot be installed because there are no viable options:\n└─ scipy 1.16.3 would require\n └─ numpy <2.6,>=1.25.2, for which no candidates were found.\n'" 118 | assert set(parse_rattler_solver_error(error_message)) == {"numpy <2.6,>=1.25.2"} 119 | -------------------------------------------------------------------------------- /conda_pypi/post_command/install.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from logging import getLogger 4 | from typing import TYPE_CHECKING 5 | 6 | from conda.base.context import context 7 | from conda.reporters import get_spinner 8 | from conda.exceptions import CondaVerificationError, CondaFileIOError 9 | 10 | from conda_pypi.main import run_pip_install, compute_record_sum, PyPIDistribution 11 | from conda_pypi.python_paths import get_env_site_packages 12 | 13 | if TYPE_CHECKING: 14 | from typing import Iterable, Literal 15 | 16 | log = getLogger(f"conda.{__name__}") 17 | 18 | 19 | def _prepare_pypi_transaction(lines: Iterable[str]) -> dict[str, dict[str, str]]: 20 | pkgs = {} 21 | for line in lines: 22 | dist = PyPIDistribution.from_lockfile_line(line) 23 | pkgs[(dist.name, dist.version)] = { 24 | "url": dist.find_wheel_url(), 25 | "hashes": dist.record_checksums, 26 | } 27 | return pkgs 28 | 29 | 30 | def _verify_pypi_transaction( 31 | prefix: str, 32 | pkgs: dict[str, dict[str, str]], 33 | on_error: Literal["ignore", "warn", "error"] = "warn", 34 | ): 35 | site_packages = get_env_site_packages(prefix) 36 | errors = [] 37 | dist_infos = [path for path in site_packages.glob("*.dist-info") if path.is_dir()] 38 | for (name, version), pkg in pkgs.items(): 39 | norm_name = name.lower().replace("-", "_").replace(".", "_") 40 | dist_info = next( 41 | ( 42 | d 43 | for d in dist_infos 44 | if d.stem.rsplit("-", 1) in ([name, version], [norm_name, version]) 45 | ), 46 | None, 47 | ) 48 | if not dist_info: 49 | errors.append(f"Could not find installation for {name}=={version}") 50 | continue 51 | 52 | expected_hashes = pkg.get("hashes") 53 | if expected_hashes: 54 | found_hashes = compute_record_sum(dist_info / "RECORD", expected_hashes.keys()) 55 | log.info("Verifying %s==%s with %s", name, version, ", ".join(expected_hashes)) 56 | for algo, expected_hash in expected_hashes.items(): 57 | found_hash = found_hashes.get(algo) 58 | if found_hash and expected_hash != found_hash: 59 | msg = ( 60 | "%s checksum for %s==%s didn't match! Expected=%s, found=%s", 61 | algo, 62 | name, 63 | version, 64 | expected_hash, 65 | found_hash, 66 | ) 67 | if on_error == "warn": 68 | log.warning(*msg) 69 | elif on_error == "error": 70 | errors.append(msg[0] % msg[1:]) 71 | else: 72 | log.debug(*msg) 73 | if errors: 74 | errors = "\n- ".join(errors) 75 | raise CondaVerificationError(f"PyPI packages checksum verification failed:\n- {errors}") 76 | 77 | 78 | def post_command(command: str): 79 | if command not in ("install", "create"): 80 | return 0 81 | 82 | pypi_lines = _pypi_lines_from_paths() 83 | if not pypi_lines: 84 | return 0 85 | 86 | with get_spinner("\nPreparing PyPI transaction"): 87 | pkgs = _prepare_pypi_transaction(pypi_lines) 88 | 89 | with get_spinner("Executing PyPI transaction"): 90 | run_pip_install( 91 | context.target_prefix, 92 | args=[pkg["url"] for pkg in pkgs.values()], 93 | dry_run=context.dry_run, 94 | quiet=context.quiet, 95 | verbosity=context.verbosity, 96 | force_reinstall=context.force_reinstall, 97 | yes=context.always_yes, 98 | check=True, 99 | ) 100 | 101 | with get_spinner("Verifying PyPI transaction"): 102 | on_error_dict = {"disabled": "ignore", "warn": "warn", "enabled": "error"} 103 | on_error = on_error_dict.get(context.safety_checks, "warn") 104 | _verify_pypi_transaction(context.target_prefix, pkgs, on_error=on_error) 105 | 106 | 107 | def _pypi_lines_from_paths(paths: Iterable[str] | None = None) -> list[str]: 108 | if paths is None: 109 | file_arg = context.raw_data["cmd_line"].get("file") 110 | if file_arg is None: 111 | return [] 112 | paths = file_arg.value(None) 113 | lines = [] 114 | line_prefix = PyPIDistribution._line_prefix 115 | for path in paths: 116 | path = path.value(None) 117 | try: 118 | with open(path) as f: 119 | for line in f: 120 | if line.startswith(line_prefix): 121 | lines.append(line[len(line_prefix) :]) 122 | except OSError as exc: 123 | raise CondaFileIOError(f"Could not process {path}") from exc 124 | return lines 125 | -------------------------------------------------------------------------------- /tests/test_benchmarks.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pathlib import Path 3 | 4 | from conda.testing.fixtures import CondaCLIFixture 5 | from conda.models.match_spec import MatchSpec 6 | from conda.common.path import get_python_short_path 7 | 8 | from conda_pypi.convert_tree import ConvertTree 9 | from conda_pypi.downloader import find_and_fetch, get_package_finder 10 | from conda_pypi.build import build_conda 11 | 12 | 13 | # @pytest.mark.benchmark 14 | # @pytest.mark.parametrize( 15 | # "packages", 16 | # [ 17 | # pytest.param(("imagesize",), id="imagesize"), # small package, few dependencies 18 | # pytest.param(("scipy",), id="scipy"), # slightly larger package 19 | # pytest.param(("jupyterlab",), id="jupyterlab"), 20 | # ], 21 | # ) 22 | # def test_conda_pypi_install_basic( 23 | # tmp_path_factory, 24 | # conda_cli: CondaCLIFixture, 25 | # packages: tuple[str], 26 | # benchmark, 27 | # monkeypatch: MonkeyPatch, 28 | # ): 29 | # """Benchmark basic conda pypi install functionality.""" 30 | 31 | # def setup(): 32 | # # Setup function is run every time. So, using benchmarks to run multiple 33 | # # iterations of the test will create new paths for repo_dir and 34 | # # prefix for each iteration. This ensures a clean test without any 35 | # # cached packages and in a clean environment. 36 | # repo_dir = tmp_path_factory.mktemp(f"{'-'.join(packages)}-pkg-repo") 37 | # prefix = str(tmp_path_factory.mktemp(f"{'-'.join(packages)}")) 38 | 39 | # monkeypatch.setattr("platformdirs.user_data_dir", lambda s: str(repo_dir)) 40 | 41 | # conda_cli("create", "--yes", "--prefix", prefix, "python=3.11") 42 | # return (prefix,), {} 43 | 44 | # def target(prefix): 45 | # _, _, rc = conda_cli( 46 | # "pypi", 47 | # "--yes", 48 | # "install", 49 | # "--prefix", 50 | # prefix, 51 | # *packages, 52 | # ) 53 | # return rc 54 | 55 | # result = benchmark.pedantic( 56 | # target, 57 | # setup=setup, 58 | # rounds=2, 59 | # warmup_rounds=0, # no warm up, cleaning the cache every time 60 | # ) 61 | # assert result == 0 62 | 63 | 64 | @pytest.mark.benchmark 65 | @pytest.mark.parametrize( 66 | "packages", 67 | [ 68 | pytest.param(("imagesize",), id="imagesize"), # small package, few dependencies 69 | pytest.param(("jupyterlab",), id="jupyterlab"), # large package 70 | pytest.param(("numpy>2.0",), id="numpy>2.0"), # package with version constraint 71 | ], 72 | ) 73 | def test_convert_tree( 74 | tmp_path_factory, 75 | conda_cli: CondaCLIFixture, 76 | packages: tuple[str], 77 | benchmark, 78 | ): 79 | """Benchmark convert_tree. This test overrides channels so the whole 80 | dependency tree is converted. 81 | """ 82 | 83 | def setup(): 84 | repo_dir = tmp_path_factory.mktemp(f"{'-'.join(packages)}-pkg-repo") 85 | prefix = str(tmp_path_factory.mktemp(f"{'-'.join(packages)}")) 86 | conda_cli("create", "--yes", "--prefix", prefix, "python=3.11") 87 | 88 | tree_converter = ConvertTree(prefix, True, repo_dir) 89 | return (tree_converter,), {} 90 | 91 | def target(tree_converter): 92 | match_specs = [MatchSpec(pkg) for pkg in packages] 93 | tree_converter.convert_tree(match_specs) 94 | 95 | benchmark.pedantic( 96 | target, 97 | setup=setup, 98 | rounds=2, 99 | warmup_rounds=0, # no warm up, cleaning the cache every time 100 | ) 101 | 102 | 103 | @pytest.mark.benchmark 104 | @pytest.mark.parametrize( 105 | "package", 106 | [ 107 | pytest.param("imagesize", id="imagesize"), 108 | pytest.param("jupyterlab", id="jupyterlab"), 109 | ], 110 | ) 111 | def test_build_conda( 112 | tmp_path_factory, 113 | conda_cli: CondaCLIFixture, 114 | package: str, 115 | benchmark, 116 | ): 117 | """Benchmark building the conda package from a wheel.""" 118 | wheel_dir = tmp_path_factory.mktemp("wheel_dir") 119 | 120 | def setup(): 121 | prefix = str(tmp_path_factory.mktemp(f"{package}")) 122 | build_path = tmp_path_factory.mktemp(f"build-{package}") 123 | output_path = tmp_path_factory.mktemp(f"output-{package}") 124 | 125 | conda_cli("create", "--yes", "--prefix", prefix, "python=3.11") 126 | 127 | python_exe = Path(prefix, get_python_short_path()) 128 | finder = get_package_finder(prefix) 129 | wheel_path = find_and_fetch(finder, wheel_dir, package) 130 | 131 | return (wheel_path, python_exe, build_path, output_path), {} 132 | 133 | def target(wheel_path, python_exe, build_path, output_path): 134 | build_conda( 135 | wheel_path, 136 | build_path, 137 | output_path, 138 | python_exe, 139 | is_editable=False, 140 | ) 141 | 142 | benchmark.pedantic( 143 | target, 144 | setup=setup, 145 | rounds=2, 146 | warmup_rounds=0, # no warm up, cleaning the cache every time 147 | ) 148 | -------------------------------------------------------------------------------- /conda_pypi/cli/install.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | from argparse import _SubParsersAction, Namespace 3 | from pathlib import Path 4 | 5 | from conda.auxlib.ish import dals 6 | from conda.models.match_spec import MatchSpec 7 | 8 | from conda_pypi import convert_tree, build, installer 9 | from conda_pypi.downloader import get_package_finder 10 | from conda_pypi.main import run_conda_install 11 | from conda_pypi.utils import get_prefix 12 | 13 | 14 | def configure_parser(parser: _SubParsersAction) -> None: 15 | """ 16 | Configure all subcommand arguments and options via argparse 17 | """ 18 | summary = "Install PyPI packages as conda packages" 19 | description = summary 20 | epilog = dals( 21 | """ 22 | 23 | Install PyPI packages as conda packages. Any dependencies that are 24 | available on the configured conda channels will be installed with `conda`, 25 | while the rest will be converted to conda packages from PyPI. 26 | 27 | Examples: 28 | 29 | Install a single PyPI package into the current conda environment:: 30 | 31 | conda pypi install requests 32 | 33 | Install multiple PyPI packages with specific versions:: 34 | 35 | conda pypi install "numpy>=1.20" "pandas==1.5.0" 36 | 37 | Install packages into a specific conda environment:: 38 | 39 | conda pypi install -n myenv flask django 40 | 41 | Install packages using only PyPI (skip configured conda channels):: 42 | 43 | conda pypi install --ignore-channels fastapi 44 | 45 | Install packages from an alternative package index URL:: 46 | 47 | conda pypi install --index-url https://example.com/simple fastapi 48 | 49 | Install a local project in editable mode:: 50 | 51 | conda pypi install -e ./my-project 52 | 53 | Install the current directory in editable mode:: 54 | 55 | conda pypi install -e . 56 | 57 | """ 58 | ) 59 | install = parser.add_parser( 60 | "install", 61 | help=summary, 62 | description=description, 63 | epilog=epilog, 64 | ) 65 | install.add_argument( 66 | "--ignore-channels", 67 | action="store_true", 68 | help="Do not search default or .condarc channels. Will search PyPI.", 69 | ) 70 | install.add_argument( 71 | "-i", 72 | "--index-url", 73 | dest="index_urls", 74 | action="append", 75 | help="Add a PyPI index URL (can be used multiple times).", 76 | ) 77 | install.add_argument( 78 | "packages", 79 | metavar="PACKAGE", 80 | nargs="*", 81 | help="PyPI packages to install", 82 | ) 83 | install.add_argument( 84 | "-p", 85 | "--prefix", 86 | help="Full path to environment location (i.e. prefix).", 87 | required=False, 88 | ) 89 | install.add_argument( 90 | "-e", 91 | "--editable", 92 | help="Build and install named path as an editable package, linking project into environment.", 93 | ) 94 | 95 | 96 | def execute(args: Namespace) -> int: 97 | """ 98 | Entry point for the `conda pypi install` subcommand. 99 | """ 100 | if not args.editable and not args.packages: 101 | raise SystemExit(2) 102 | 103 | prefix_path = get_prefix() 104 | 105 | if args.editable: 106 | editable_path = Path(args.editable).expanduser() 107 | output_path_manager = tempfile.TemporaryDirectory("conda-pypi") 108 | with output_path_manager as output_path: 109 | package = build.pypa_to_conda( 110 | editable_path, 111 | distribution="editable", 112 | output_path=Path(output_path), 113 | prefix=prefix_path, 114 | ) 115 | installer.install_ephemeral_conda(prefix_path, package) 116 | return 0 117 | 118 | if args.index_urls: 119 | index_urls = tuple(dict.fromkeys(args.index_urls)) 120 | finder = get_package_finder(prefix_path, index_urls) 121 | else: 122 | finder = None 123 | 124 | converter = convert_tree.ConvertTree( 125 | prefix_path, 126 | override_channels=args.ignore_channels, 127 | finder=finder, 128 | ) 129 | 130 | # Convert package strings to MatchSpec objects 131 | match_specs = [MatchSpec(pkg) for pkg in args.packages] 132 | changes = converter.convert_tree(match_specs) 133 | channel_url = converter.repo.as_uri() 134 | 135 | if changes is None: 136 | packages_to_install = () 137 | else: 138 | packages_to_install = changes[1] 139 | converted_packages = [ 140 | str(pkg.to_simple_match_spec()) 141 | for pkg in packages_to_install 142 | if pkg.channel.canonical_name == channel_url 143 | ] 144 | if converted_packages: 145 | converted_packages_dashed = "\n - ".join(converted_packages) 146 | print(f"Converted packages\n - {converted_packages_dashed}\n") 147 | print("Installing environment") 148 | 149 | # Install converted packages to current conda environment 150 | return run_conda_install( 151 | prefix_path, 152 | match_specs, 153 | channels=[channel_url], 154 | override_channels=args.ignore_channels, 155 | yes=args.yes, 156 | quiet=args.quiet, 157 | verbosity=args.verbosity, 158 | dry_run=args.dry_run, 159 | ) 160 | -------------------------------------------------------------------------------- /docs/features.md: -------------------------------------------------------------------------------- 1 | # Features 2 | 3 | `conda-pypi` uses the `conda` plugin system to implement several features 4 | that make `conda` integrate better with the PyPI ecosystem: 5 | 6 | ## The `conda pypi` subcommand 7 | 8 | This subcommand provides a safer way to install PyPI packages in conda 9 | environments by converting them to `.conda` format when possible. It offers two 10 | main subcommands that handle different aspects of PyPI integration. 11 | 12 | ### `conda pypi install` 13 | 14 | The install command takes PyPI packages and converts them to the `.conda` format. 15 | Explicitly requested packages are always installed from PyPI and converted 16 | to `.conda` format to ensure you get exactly what you asked for. For 17 | dependencies, conda-pypi chooses the best source using a 18 | conda-first approach. If a dependency is available on conda channels, it will 19 | be installed with `conda` directly. If not available on conda channels, the 20 | dependency will be converted from PyPI to `.conda` format. 21 | 22 | The system uses multiple sources for package name mapping which are 23 | currently hardcoded. In the future, it will use other means to have 24 | a more active way to get current name mappings. VCS and editable packages 25 | are handled as special cases and installed directly with `pip install --no-deps`. 26 | 27 | You can preview what would be installed without making changes using 28 | `--dry-run`, install packages in editable development mode with `--editable` 29 | or `-e`, and force dependency resolution from PyPI without using conda 30 | channels using `--ignore-channels`. 31 | 32 | ### `conda pypi convert` 33 | 34 | The convert command transforms PyPI packages to `.conda` format without 35 | installing them, which is useful for creating conda packages from PyPI 36 | distributions or preparing packages for offline installation. You can specify 37 | where to save the converted packages using `-d`, `--dest`, or `--output-dir`. 38 | The command supports converting multiple packages at once and can skip conda 39 | channel checks entirely with `--ignore-channels` to convert directly from 40 | PyPI. 41 | 42 | Here are some common usage patterns: 43 | 44 | ```bash 45 | # Convert packages to current directory 46 | conda pypi convert httpx cowsay 47 | 48 | # Convert to specific directory 49 | conda pypi convert -d ./my_packages httpx cowsay 50 | 51 | # Convert without checking conda channels first 52 | conda pypi convert --ignore-channels some-pypi-only-package 53 | ``` 54 | 55 | ## PyPI-to-Conda Conversion Engine 56 | 57 | `conda-pypi` includes a powerful conversion engine that enables direct 58 | conversion of pure Python wheels to `.conda` packages with proper translation of 59 | Python package metadata to conda format. The system includes name 60 | mapping of PyPI dependencies to conda equivalents and provides cross-platform 61 | support for package conversion, ensuring that converted packages work 62 | across different operating systems and architectures. 63 | 64 | (pypi-lines)= 65 | 66 | 67 | ## `conda install` integrations 68 | 69 | The system provides clear error messages if PyPI package installation fails 70 | and uses the same conversion logic as `conda pypi install` for 71 | dependency resolution. This enables full environment reproducibility that 72 | includes both conda and converted PyPI packages, ensuring that environments 73 | can be recreated exactly as they were originally configured. 74 | 75 | ## Editable Package Support 76 | 77 | `conda-pypi` provides comprehensive support for editable (development) 78 | installations, making it ideal for development environments where code is 79 | frequently modified. The system supports both version control system packages 80 | and local packages. 81 | 82 | For VCS packages, you can install directly from git URLs with automatic 83 | cloning. The system caches VCS repositories locally for improved performance 84 | and manages temporary directories and repository clones automatically. Local 85 | package support allows you to install packages from local project directories 86 | in editable mode, which is perfect for active development workflows. 87 | 88 | Here are some common usage patterns for editable installations: 89 | 90 | ```bash 91 | # Install from git repository in editable mode 92 | conda pypi install -e git+https://github.com/user/project.git 93 | 94 | # Install local project in editable mode 95 | conda pypi install -e ./my-project/ 96 | 97 | # Multiple editable packages 98 | conda pypi install -e ./package1/ -e git+https://github.com/user/package2.git 99 | ``` 100 | 101 | ## `conda env` integrations 102 | 103 | :::{admonition} Coming soon 104 | :class: seealso 105 | 106 | `environment.yml` files famously allow a `pip` subsection in their 107 | `dependencies`. This is handled internally by `conda env` via a `pip` 108 | subprocess. We are adding new plugin hooks so `conda-pypi` can handle these 109 | in the same way we do with the `conda pypi` subcommand. 110 | ::: 111 | 112 | (externally-managed)= 113 | 114 | ## Environment marker files 115 | 116 | `conda-pypi` adds support for 117 | [PEP-668](https://peps.python.org/pep-0668/)'s 118 | [`EXTERNALLY-MANAGED`](https://packaging.python.org/en/latest/specifications/externally-managed-environments/) 119 | environment marker files. These files tell `pip` and other PyPI installers 120 | not to install or remove any packages in that environment, guiding users 121 | towards safer alternatives. 122 | 123 | When these marker files are present, they display a message letting users 124 | know that the `conda pypi` subcommand is available as a safer alternative. The 125 | primary goal is to avoid accidental overwrites that could break your conda 126 | environment. If you need to use `pip` directly, you can still do so by adding 127 | the `--break-system-packages` flag, though this is generally not recommended 128 | in conda environments. 129 | -------------------------------------------------------------------------------- /conda_pypi/pre_command/extract_whl.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import json 3 | import os 4 | from os import PathLike 5 | 6 | from installer.records import RecordEntry, Hash 7 | from installer.sources import WheelFile 8 | from installer.destinations import WheelDestination 9 | from installer.utils import Scheme 10 | 11 | import installer.utils 12 | 13 | from typing import Literal, BinaryIO, Iterable, Tuple 14 | 15 | 16 | SUPPORTED_SCEMES: Tuple[Scheme] = ("platlib", "purelib") 17 | 18 | 19 | # inline version of 20 | # from conda.gateways.disk.create import write_as_json_to_file 21 | def write_as_json_to_file(file_path, obj): 22 | with codecs.open(file_path, mode="wb", encoding="utf-8") as fo: 23 | json_str = json.dumps( 24 | obj, 25 | indent=2, 26 | sort_keys=True, 27 | separators=(",", ": "), 28 | ) 29 | fo.write(json_str) 30 | 31 | 32 | class MyWheelDestination(WheelDestination): 33 | def __init__(self, target_full_path: str, source: WheelFile) -> None: 34 | self.target_full_path = target_full_path 35 | self.sp_dir = os.path.join(target_full_path, "site-packages") 36 | self.entry_points = [] 37 | self.source = source 38 | 39 | def write_script( 40 | self, name: str, module: str, attr: str, section: Literal["console"] | Literal["gui"] 41 | ) -> RecordEntry: 42 | # TODO check if console/gui 43 | entry_point = f"{name} = {module}:{attr}" 44 | self.entry_points.append(entry_point) 45 | return RecordEntry( 46 | path=f"../../../bin/{name}", 47 | hash_=None, 48 | size=None, 49 | ) 50 | 51 | def write_file( 52 | self, scheme: Scheme, path: str | PathLike[str], stream: BinaryIO, is_executable: bool 53 | ) -> RecordEntry: 54 | if scheme not in SUPPORTED_SCEMES: 55 | raise ValueError(f"Unsupported scheme: {scheme}") 56 | 57 | path = os.fspath(path) 58 | dest_path = os.path.join(self.sp_dir, path) 59 | 60 | parent_folder = os.path.dirname(dest_path) 61 | if not os.path.exists(parent_folder): 62 | os.makedirs(parent_folder) 63 | 64 | # print(f"Writing {dest_path} from {source}") 65 | with open(dest_path, "wb") as dest: 66 | hash_, size = installer.utils.copyfileobj_with_hashing( 67 | source=stream, 68 | dest=dest, 69 | hash_algorithm="sha256", 70 | ) 71 | 72 | if is_executable: 73 | installer.utils.make_file_executable(dest_path) 74 | 75 | return RecordEntry( 76 | path=path, 77 | hash_=Hash("sha256", hash_), 78 | size=size, 79 | ) 80 | 81 | def _create_conda_metadata( 82 | self, records: Iterable[Tuple[Scheme, RecordEntry]], source: WheelFile 83 | ) -> None: 84 | os.makedirs(os.path.join(self.target_full_path, "info"), exist_ok=True) 85 | # link.json 86 | link_json_data = { 87 | "noarch": { 88 | "type": "python", 89 | }, 90 | "package_metadata_version": 1, 91 | } 92 | if self.entry_points: 93 | link_json_data["noarch"]["entry_points"] = self.entry_points 94 | link_json_path = os.path.join(self.target_full_path, "info", "link.json") 95 | write_as_json_to_file(link_json_path, link_json_data) 96 | 97 | # paths.json 98 | paths = [] 99 | for _, record in records: 100 | if record.path.startswith(".."): 101 | # entry point 102 | continue 103 | path = { 104 | "_path": f"site-packages/{record.path}", 105 | "path_type": "hardlink", 106 | "sha256": record.hash_.value, 107 | "size_in_bytes": record.size, 108 | } 109 | paths.append(path) 110 | paths_json_data = { 111 | "paths": paths, 112 | "paths_version": 1, 113 | } 114 | paths_json_path = os.path.join(self.target_full_path, "info", "paths.json") 115 | write_as_json_to_file(paths_json_path, paths_json_data) 116 | 117 | # index.json 118 | # Set fn to include the build string AND extension so _get_json_fn() works correctly 119 | # Format: name-version-build.whl (e.g., "requests-2.28.0-pypi_0.whl") 120 | # The extension is required because _get_json_fn() uses endswith() to detect package type 121 | package_name = str(source.distribution) 122 | package_version = str(source.version) 123 | build_string = "pypi_0" 124 | fn = f"{package_name}-{package_version}-{build_string}.whl" 125 | 126 | index_json_data = { 127 | "name": package_name, 128 | "version": package_version, 129 | "build": build_string, 130 | "build_number": 0, 131 | "fn": fn, 132 | } 133 | index_json_path = os.path.join(self.target_full_path, "info", "index.json") 134 | write_as_json_to_file(index_json_path, index_json_data) 135 | 136 | def finalize_installation( 137 | self, 138 | scheme: Scheme, 139 | record_file_path: str, 140 | records: Iterable[Tuple[Scheme, RecordEntry]], 141 | ) -> None: 142 | record_list = list(records) 143 | with installer.utils.construct_record_file(record_list, lambda x: None) as record_stream: 144 | dest_path = os.path.join(self.sp_dir, record_file_path) 145 | with open(dest_path, "wb") as dest: 146 | hash_, size = installer.utils.copyfileobj_with_hashing( 147 | record_stream, dest, "sha256" 148 | ) 149 | record_file_record = RecordEntry( 150 | path=record_file_path, 151 | hash_=Hash("sha256", hash_), 152 | size=size, 153 | ) 154 | record_list[-1] = ("purelib", record_file_record) 155 | self._create_conda_metadata(record_list, self.source) 156 | return 157 | 158 | 159 | def extract_whl_as_conda_pkg(whl_full_path: str, target_full_path: str): 160 | with WheelFile.open(whl_full_path) as source: 161 | installer.install( 162 | source=source, 163 | destination=MyWheelDestination(target_full_path, source), 164 | additional_metadata={"INSTALLER": b"conda-via-whl"}, 165 | ) 166 | -------------------------------------------------------------------------------- /tests/test_install.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | import sys 5 | from pathlib import Path 6 | 7 | import pytest 8 | from conda.testing.fixtures import TmpEnvFixture, CondaCLIFixture 9 | 10 | from conda_pypi.python_paths import get_env_site_packages 11 | 12 | 13 | def test_conda_pypi_install_basic(tmp_env: TmpEnvFixture, conda_cli: CondaCLIFixture): 14 | """Test basic conda pypi install functionality.""" 15 | with tmp_env("python=3.11") as prefix: 16 | out, err, rc = conda_cli( 17 | "pypi", 18 | "-p", 19 | prefix, 20 | "--yes", 21 | "install", 22 | "numpy", 23 | ) 24 | assert rc == 0 25 | 26 | 27 | @pytest.mark.parametrize( 28 | "pypi_spec,expected_in_output", 29 | [ 30 | ("certifi", "certifi"), 31 | ("tomli==2.0.1", "tomli"), 32 | ], 33 | ) 34 | def test_conda_pypi_install_package_conversion( 35 | tmp_env: TmpEnvFixture, 36 | conda_cli: CondaCLIFixture, 37 | pypi_spec: str, 38 | expected_in_output: str, 39 | ): 40 | """Test that PyPI packages are correctly converted and installed.""" 41 | with tmp_env("python=3.11") as prefix: 42 | out, err, rc = conda_cli( 43 | "pypi", 44 | "-p", 45 | prefix, 46 | "--yes", 47 | "install", 48 | pypi_spec, 49 | ) 50 | assert rc == 0 51 | assert expected_in_output in out or "All requested packages already installed" in out 52 | 53 | 54 | def test_conda_pypi_install_matchspec_parsing(tmp_env: TmpEnvFixture, conda_cli: CondaCLIFixture): 55 | """Test that MatchSpec parsing works correctly for various package specifications.""" 56 | with tmp_env("python=3.11") as prefix: 57 | test_specs = [ 58 | "numpy", 59 | "numpy>=1.20", 60 | ] 61 | 62 | for spec in test_specs: 63 | out, err, rc = conda_cli( 64 | "pypi", 65 | "-p", 66 | prefix, 67 | "--yes", 68 | "--dry-run", 69 | "install", 70 | spec, 71 | ) 72 | assert rc == 0, f"Failed to parse spec '{spec}'" 73 | 74 | 75 | def test_conda_pypi_install_requires_package_without_editable( 76 | tmp_env: TmpEnvFixture, conda_cli: CondaCLIFixture 77 | ): 78 | """Test that conda pypi install requires a package when not in editable mode.""" 79 | with tmp_env("python=3.11") as prefix: 80 | with pytest.raises(SystemExit) as exc: 81 | conda_cli( 82 | "pypi", 83 | "-p", 84 | prefix, 85 | "install", 86 | ) 87 | assert exc.value.code == 2 88 | 89 | 90 | def test_conda_pypi_install_editable_without_packages_succeeds( 91 | tmp_env: TmpEnvFixture, conda_cli: CondaCLIFixture 92 | ): 93 | """Test that conda pypi install -e succeeds without additional packages.""" 94 | with tmp_env("python=3.11") as prefix: 95 | out, err, rc = conda_cli( 96 | "pypi", 97 | "-p", 98 | prefix, 99 | "--yes", 100 | "install", 101 | "-e", 102 | str(Path(__file__).parent / "packages" / "has-build-dep"), 103 | ) 104 | assert rc == 0 105 | 106 | 107 | @pytest.mark.skip(reason="Migrating to alternative install method using conda pupa") 108 | def test_spec_normalization( 109 | tmp_env: TmpEnvFixture, 110 | conda_cli: CondaCLIFixture, 111 | ): 112 | with tmp_env("python=3.9", "pip", "pytest-cov") as prefix: 113 | for spec in ("pytest-cov", "pytest_cov", "PyTest-Cov"): 114 | out, err, rc = conda_cli("pypi", "--dry-run", "-p", prefix, "--yes", "install", spec) 115 | print(out) 116 | print(err, file=sys.stderr) 117 | assert rc == 0 118 | assert "All requested packages already installed." in out 119 | 120 | 121 | @pytest.mark.skip(reason="Migrating to alternative install method using conda pupa") 122 | @pytest.mark.parametrize( 123 | "pypi_spec,requested_conda_spec,installed_conda_specs", 124 | [ 125 | ("PyQt5", "pyqt[version='>=5.0.0,<6.0.0.0dev0']", ("pyqt-5", "qt-main-5")), 126 | ], 127 | ) 128 | def test_pyqt( 129 | tmp_env: TmpEnvFixture, 130 | conda_cli: CondaCLIFixture, 131 | pypi_spec: str, 132 | requested_conda_spec: str, 133 | installed_conda_specs: tuple[str], 134 | ): 135 | with tmp_env("python=3.9", "pip") as prefix: 136 | out, err, rc = conda_cli("pypi", "-p", prefix, "--yes", "--dry-run", "install", pypi_spec) 137 | print(out) 138 | print(err, file=sys.stderr) 139 | assert rc == 0 140 | assert requested_conda_spec in out 141 | for conda_spec in installed_conda_specs: 142 | assert conda_spec in out 143 | 144 | 145 | @pytest.mark.skip(reason="Migrating to alternative install method using conda pupa") 146 | @pytest.mark.parametrize( 147 | "requirement,name", 148 | [ 149 | pytest.param( 150 | # pure Python 151 | "git+https://github.com/dateutil/dateutil.git@2.9.0.post0", 152 | "python_dateutil", 153 | marks=pytest.mark.skip(reason="Fragile test with git repo state issues"), 154 | ), 155 | pytest.param( 156 | # compiled bits 157 | "git+https://github.com/yaml/pyyaml.git@6.0.1", 158 | "PyYAML", 159 | marks=pytest.mark.skip(reason="Editable install path detection issues"), 160 | ), 161 | pytest.param( 162 | # has conda dependencies 163 | "git+https://github.com/python-poetry/cleo.git@2.1.0", 164 | "cleo", 165 | ), 166 | ], 167 | ) 168 | def test_editable_installs( 169 | tmp_path: Path, tmp_env: TmpEnvFixture, conda_cli: CondaCLIFixture, requirement, name 170 | ): 171 | os.chdir(tmp_path) 172 | with tmp_env("python=3.9", "pip") as prefix: 173 | out, err, rc = conda_cli( 174 | "pypi", 175 | "-p", 176 | prefix, 177 | "--yes", 178 | "install", 179 | "-e", 180 | f"{requirement}#egg={name}", 181 | ) 182 | assert rc == 0 183 | sp = get_env_site_packages(prefix) 184 | editable_pth = list(sp.glob(f"__editable__.{name}-*.pth")) # Modern pip format 185 | if not editable_pth: 186 | editable_pth = list(sp.glob(f"{name}.pth")) # Older format 187 | 188 | assert len(editable_pth) == 1, ( 189 | f"Expected 1 editable .pth file for {name}, found: {editable_pth}" 190 | ) 191 | pth_contents = editable_pth[0].read_text().strip() 192 | src_path = tmp_path / "src" 193 | 194 | if not pth_contents.startswith(f"import __editable___{name}"): 195 | pth_path = Path(pth_contents) 196 | assert ( 197 | src_path in pth_path.parents 198 | or src_path == pth_path 199 | or pth_path.is_relative_to(src_path) 200 | ), f"Expected {src_path} to be a parent of or equal to {pth_path}" 201 | -------------------------------------------------------------------------------- /conda_pypi/build.py: -------------------------------------------------------------------------------- 1 | """ 2 | Create .conda packages from wheels. 3 | 4 | Create wheels from pypa projects. 5 | """ 6 | 7 | import base64 8 | import csv 9 | import hashlib 10 | import itertools 11 | import json 12 | import os 13 | import sys 14 | import tempfile 15 | from importlib.metadata import PathDistribution 16 | from pathlib import Path 17 | from typing import Union, Optional 18 | import logging 19 | 20 | from conda_package_streaming.create import conda_builder 21 | from conda.common.path.windows import win_path_to_unix 22 | from conda.common.compat import on_win 23 | 24 | from build import ProjectBuilder 25 | 26 | from conda_pypi import dependencies, installer, paths 27 | from conda_pypi.conda_build_utils import PathType, sha256_checksum 28 | from conda_pypi.translate import CondaMetadata 29 | 30 | 31 | log = logging.getLogger(__name__) 32 | 33 | 34 | def filter(tarinfo): 35 | """ 36 | Anonymize uid/gid; exclude .git directories. 37 | """ 38 | if tarinfo.name.endswith(".git"): 39 | return None 40 | tarinfo.uid = tarinfo.gid = 0 41 | tarinfo.uname = tarinfo.gname = "" 42 | return tarinfo 43 | 44 | 45 | # see conda_build.build.build_info_files_json_v1 46 | def paths_json(base: Union[Path, str]): 47 | """ 48 | Build simple paths.json with only 'hardlink' or 'symlink' types. 49 | """ 50 | base = str(base) 51 | 52 | if not base.endswith(os.sep): 53 | base = base + os.sep 54 | 55 | return { 56 | "paths": sorted(_paths(base, base), key=lambda entry: entry["_path"]), 57 | "paths_version": 1, 58 | } 59 | 60 | 61 | def _paths(base, path, filter=lambda x: x.name != ".git"): 62 | for entry in os.scandir(path): 63 | relative_path = entry.path[len(base) :] 64 | if on_win: 65 | relative_path = win_path_to_unix(relative_path) 66 | if relative_path == "info" or not filter(entry): 67 | continue 68 | if entry.is_dir(): 69 | yield from _paths(base, entry.path, filter=filter) 70 | elif entry.is_file() or entry.is_symlink(): 71 | try: 72 | st_size = entry.stat().st_size 73 | except FileNotFoundError: 74 | st_size = 0 # symlink to nowhere 75 | yield { 76 | "_path": relative_path, 77 | "path_type": str(PathType.softlink if entry.is_symlink() else PathType.hardlink), 78 | "sha256": sha256_checksum(entry.path, entry), 79 | "size_in_bytes": st_size, 80 | } 81 | else: 82 | log.debug(f"Not regular file '{entry}'") 83 | # will Python's tarfile add pipes, device nodes to the archive? 84 | 85 | 86 | def json_dumps(object): 87 | """ 88 | Consistent json formatting. 89 | """ 90 | return json.dumps(object, indent=2, sort_keys=True) 91 | 92 | 93 | def flatten(iterable): 94 | return [*itertools.chain(*iterable)] 95 | 96 | 97 | def build_pypa( 98 | path: Path, 99 | output_path, 100 | prefix: Path, 101 | distribution="editable", 102 | ): 103 | """ 104 | Args: 105 | distribution: "editable" or "wheel" 106 | """ 107 | python_executable = str(paths.get_python_executable(prefix)) 108 | 109 | builder = ProjectBuilder(path, python_executable=python_executable) 110 | 111 | build_system_requires = builder.build_system_requires 112 | for _retry in range(2): 113 | try: 114 | missing = dependencies.check_dependencies(build_system_requires, prefix=prefix) 115 | break 116 | except dependencies.MissingDependencyError as e: 117 | dependencies.ensure_requirements(e.dependencies, prefix=prefix) 118 | 119 | log.debug(f"Installing requirements for build system: {missing}") 120 | # does flatten() work for a deeper dependency chain? 121 | dependencies.ensure_requirements(flatten(missing), prefix=prefix) 122 | 123 | requirements = builder.check_dependencies(distribution) 124 | log.debug(f"Additional requirements for {distribution}: {requirements}") 125 | dependencies.ensure_requirements(flatten(requirements), prefix=prefix) 126 | 127 | editable_file = builder.build(distribution, output_path) 128 | log.debug(f"The wheel is at {editable_file}") 129 | 130 | return editable_file 131 | 132 | 133 | def build_conda( 134 | whl, 135 | build_path: Path, 136 | output_path: Path, 137 | python_executable, 138 | project_path: Optional[Path] = None, 139 | is_editable=False, 140 | ) -> Path: 141 | if not build_path.exists(): 142 | build_path.mkdir() 143 | 144 | installer.install_installer(python_executable, whl, build_path) 145 | 146 | site_packages = build_path / "site-packages" 147 | dist_info = next(site_packages.glob("*.dist-info")) 148 | metadata = CondaMetadata.from_distribution(PathDistribution(dist_info)) 149 | record = metadata.package_record.to_index_json() 150 | # XXX set build string as hash of pypa metadata so that conda can re-install 151 | # when project gains new entry-points, dependencies? 152 | 153 | file_id = f"{record['name']}-{record['version']}-{record['build']}" 154 | 155 | (build_path / "info").mkdir() 156 | (build_path / "info" / "index.json").write_text(json_dumps(record)) 157 | (build_path / "info" / "about.json").write_text(json_dumps(metadata.about)) 158 | 159 | # used especially for console_scripts 160 | if link_json := metadata.link_json(): 161 | (build_path / "info" / "link.json").write_text(json_dumps(link_json)) 162 | 163 | # Allow pip to list us as editable or show the path to our project. 164 | # XXX leaks path 165 | if project_path: 166 | direct_url = project_path.absolute().as_uri() 167 | direct_url_path = dist_info / "direct_url.json" 168 | direct_url_path.write_text( 169 | json.dumps({"dir_info": {"editable": is_editable}, "url": direct_url}) 170 | ) 171 | record_path = dist_info / "RECORD" 172 | # Rewrite RECORD for any changed files 173 | update_RECORD(record_path, site_packages, direct_url_path) 174 | 175 | # Write conda's paths after all other changes 176 | paths = paths_json(build_path) 177 | 178 | (build_path / "info" / "paths.json").write_text(json_dumps(paths)) 179 | 180 | with conda_builder(file_id, output_path) as tar: 181 | tar.add(build_path, "", filter=filter) 182 | 183 | return output_path / f"{file_id}.conda" 184 | 185 | 186 | def update_RECORD(record_path: Path, base_path: Path, changed_path: Path): 187 | """ 188 | Rewrite RECORD with new size, checksum for updated_file. 189 | """ 190 | # note `installer` also has code to handle RECORD 191 | record_text = record_path.read_text() 192 | record_rows = list(csv.reader(record_text.splitlines())) 193 | 194 | relpath = str(changed_path.relative_to(base_path)).replace(os.sep, "/") 195 | for row in record_rows: 196 | if row[0] == relpath: 197 | data = changed_path.read_bytes() 198 | size = len(data) 199 | checksum = ( 200 | base64.urlsafe_b64encode(hashlib.sha256(data).digest()) 201 | .rstrip(b"=") 202 | .decode("utf-8") 203 | ) 204 | row[1] = f"sha256={checksum}" 205 | row[2] = str(size) 206 | 207 | with record_path.open(mode="w", newline="", encoding="utf-8") as record_file: 208 | writer = csv.writer(record_file) 209 | writer.writerows(record_rows) 210 | 211 | 212 | def pypa_to_conda( 213 | project, 214 | prefix: Path, 215 | distribution="editable", 216 | output_path: Optional[Path] = None, 217 | ): 218 | project = Path(project) 219 | 220 | # Should this logic be moved to the caller? 221 | if not output_path: 222 | output_path = project / "build" 223 | if not output_path.exists(): 224 | output_path.mkdir() 225 | 226 | with tempfile.TemporaryDirectory(prefix="conda") as tmp_path: 227 | tmp_path = Path(tmp_path) 228 | 229 | normal_wheel = build_pypa( 230 | Path(project), tmp_path, prefix=prefix, distribution=distribution 231 | ) 232 | 233 | build_path = tmp_path / "build" 234 | 235 | package_conda = build_conda( 236 | normal_wheel, 237 | build_path, 238 | output_path or tmp_path, 239 | sys.executable, 240 | project_path=project, 241 | is_editable=distribution == "editable", 242 | ) 243 | 244 | return package_conda 245 | -------------------------------------------------------------------------------- /conda_pypi/convert_tree.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert a dependency tree from pypi into .conda packages 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | import logging 8 | import pathlib 9 | import re 10 | import tempfile 11 | from pathlib import Path 12 | from typing import Union, Optional, List 13 | 14 | from conda_rattler_solver.solver import RattlerSolver 15 | 16 | import conda.exceptions 17 | import platformdirs 18 | from conda.base.context import context, fresh_context 19 | from conda.common.path import get_python_short_path 20 | from conda.models.channel import Channel 21 | from conda.models.match_spec import MatchSpec 22 | from conda.models.records import PrefixRecord 23 | from conda.reporters import get_spinner 24 | from conda.core.solve import Solver 25 | from conda.exceptions import UnsatisfiableError 26 | 27 | from unearth import PackageFinder 28 | 29 | from conda_pypi.build import build_conda 30 | from conda_pypi.downloader import find_and_fetch, get_package_finder 31 | from conda_pypi.index import update_index 32 | from conda_pypi.utils import SuppressOutput 33 | 34 | log = logging.getLogger(__name__) 35 | 36 | NOTHING_PROVIDES_RE = re.compile(r"nothing provides (.*) needed by") 37 | RATTLER_NOTHING_PROVIDES_RE = re.compile(r"\b(.*), (.)* (n|N)o candidates were found(.*)") 38 | 39 | 40 | def parse_libmamba_solver_error(message: str): 41 | """ 42 | Parse missing packages out of UnsatisfiableError message. 43 | """ 44 | for line in message.splitlines(): 45 | if match := NOTHING_PROVIDES_RE.search(line): 46 | yield match.group(1) 47 | 48 | 49 | def parse_rattler_solver_error(message: str): 50 | """ 51 | Parse missing packages out of UnsatisfiableError message. 52 | """ 53 | for line in message.splitlines(): 54 | if match := RATTLER_NOTHING_PROVIDES_RE.search(line): 55 | yield match.group(1) 56 | 57 | 58 | # import / pupate / transmogrify / ... 59 | class ConvertTree: 60 | def __init__( 61 | self, 62 | prefix: Optional[Union[pathlib.Path, str]], 63 | override_channels=False, 64 | repo: Optional[pathlib.Path] = None, 65 | finder: Optional[PackageFinder] = None, # to change index_urls e.g. 66 | ): 67 | # platformdirs location has a space in it; ok? 68 | # will be expanded to %20 in "as uri" output, conda understands that. 69 | self.repo = repo or Path(platformdirs.user_data_dir("conda-pypi")) 70 | prefix = prefix or context.active_prefix 71 | if not prefix: 72 | raise ValueError("prefix is required") 73 | self.prefix = Path(prefix) 74 | self.override_channels = override_channels 75 | self.python_exe = Path(self.prefix, get_python_short_path()) 76 | 77 | if not finder: 78 | finder = self.default_package_finder() 79 | self.finder = finder 80 | 81 | def _convert_loop( 82 | self, 83 | max_attempts: int, 84 | solver: Solver, 85 | tmp_path: Path, 86 | ) -> tuple[tuple[PrefixRecord, ...], tuple[PrefixRecord, ...]] | None: 87 | converted = set() 88 | fetched_packages = set() 89 | missing_packages = set() 90 | attempts = 0 91 | 92 | repo = self.repo 93 | wheel_dir = tmp_path / "wheels" 94 | wheel_dir.mkdir(exist_ok=True) 95 | 96 | while len(fetched_packages) < max_attempts and attempts < max_attempts: 97 | attempts += 1 98 | try: 99 | # suppress messages coming from the solver 100 | with SuppressOutput(): 101 | changes = solver.solve_for_diff() 102 | break 103 | except conda.exceptions.PackagesNotFoundError as e: 104 | missing_packages = set(e._kwargs["packages"]) 105 | log.debug(f"Missing packages: {missing_packages}") 106 | except UnsatisfiableError as e: 107 | # parse message 108 | log.debug("Unsatisfiable: %r", e) 109 | missing_packages.update(set(parse_rattler_solver_error(e.message))) 110 | 111 | for package in sorted(missing_packages - fetched_packages): 112 | find_and_fetch(self.finder, wheel_dir, package) 113 | fetched_packages.add(package) 114 | 115 | for normal_wheel in wheel_dir.glob("*.whl"): 116 | if normal_wheel in converted: 117 | continue 118 | 119 | log.debug(f"Converting '{normal_wheel}'") 120 | 121 | build_path = tmp_path / normal_wheel.stem 122 | build_path.mkdir() 123 | 124 | try: 125 | package_conda = build_conda( 126 | normal_wheel, 127 | build_path, 128 | repo / "noarch", # XXX could be arch 129 | self.python_exe, 130 | is_editable=False, 131 | ) 132 | log.debug("Conda at", package_conda) 133 | except FileExistsError: 134 | log.debug( 135 | f"Tried to convert wheel that is already conda-ized: {normal_wheel}", 136 | exc_info=True, 137 | ) 138 | 139 | converted.add(normal_wheel) 140 | 141 | update_index(repo) 142 | else: 143 | log.debug(f"Exceeded maximum of {max_attempts} attempts") 144 | return None 145 | return changes 146 | 147 | def default_package_finder(self): 148 | return get_package_finder(self.prefix) 149 | 150 | def _get_converting_spinner_message(self, channels) -> str: 151 | pypi_index_names_dashed = "\n - ".join( 152 | s.get("url") for s in self.finder.sources if s.get("type") == "index" 153 | ) 154 | 155 | canonical_names = list(dict.fromkeys([Channel(c).canonical_name for c in channels])) 156 | canonical_names_dashed = "\n - ".join(canonical_names) 157 | return ( 158 | "Inspecting pypi and conda dependencies\n" 159 | "PYPI index channels:\n" 160 | f" - {pypi_index_names_dashed}\n" 161 | "Conda channels:\n" 162 | f" - {canonical_names_dashed}\n" 163 | "Converting required pypi packages" 164 | ) 165 | 166 | def convert_tree( 167 | self, requested: List[MatchSpec], max_attempts: int = 80 168 | ) -> tuple[tuple[PrefixRecord, ...], tuple[PrefixRecord, ...]] | None: 169 | """ 170 | Preform a solve on the list of requested packages and converts the full dependency 171 | tree to conda packages if required. The converted packages will be stored in the 172 | local conda-pypi channel. 173 | 174 | Args: 175 | requested: The list of requested packages. 176 | max_attempts: max number of times to try to execute the solve. 177 | 178 | Returns: 179 | A two-tuple of PackageRef sequences. The first is the group of packages to 180 | remove from the environment, in sorted dependency order from leaves to roots. 181 | The second is the group of packages to add to the environment, in sorted 182 | dependency order from roots to leaves. 183 | 184 | """ 185 | (self.repo / "noarch").mkdir(parents=True, exist_ok=True) 186 | if not (self.repo / "noarch" / "repodata.json").exists(): 187 | update_index(self.repo) 188 | 189 | with tempfile.TemporaryDirectory() as tmp_path: 190 | tmp_path = pathlib.Path(tmp_path) 191 | 192 | WHEEL_DIR = tmp_path / "wheels" 193 | WHEEL_DIR.mkdir(exist_ok=True) 194 | 195 | prefix = pathlib.Path(self.prefix) 196 | assert prefix.exists() 197 | 198 | local_channel = Channel(self.repo.as_uri()) 199 | 200 | if not self.override_channels: 201 | channels = [local_channel, *context.channels] 202 | else: # more wheels for us to convert 203 | channels = [local_channel] 204 | 205 | solver = RattlerSolver( 206 | prefix=str(prefix), 207 | channels=channels, 208 | subdirs=context.subdirs, 209 | specs_to_add=requested, 210 | command="install", 211 | ) 212 | 213 | context_env = { 214 | "CONDA_AGGRESSIVE_UPDATE_PACKAGES": "", 215 | "CONDA_AUTO_UPDATE_CONDA": "false", 216 | } 217 | 218 | with get_spinner(self._get_converting_spinner_message(channels)): 219 | with fresh_context(env=context_env): 220 | changes = self._convert_loop( 221 | max_attempts=max_attempts, solver=solver, tmp_path=tmp_path 222 | ) 223 | 224 | return changes 225 | -------------------------------------------------------------------------------- /docs/reference/conda-channels-naming-analysis.md: -------------------------------------------------------------------------------- 1 | # Conda Channel Naming Discrepancies Analysis Report 2 | 3 | ## Executive Summary 4 | 5 | This report analyzes naming differences between the main conda channel and conda-forge channel for Python packages. The analysis was done by comparing the PyPI mappings between main and conda-forge channel packages 6 | and then identifying when they, the conda package names, are different. This analysis was done mid August 2025. 7 | 8 | ## Study Overview 9 | 10 | Channel data was downloaded from cf-graph-countyfair (grayskull_pypi_mapping.json) for conda-forge 11 | channel and an internal Anaconda metadata source for the main channel. With both sources, containing 12 | the conda package name and the pypi name, the following analysis was done: 13 | 14 | ```python 15 | # main_df: Pandas Dataframe from main channel 16 | # cf_df: Pandas Dataframe from conda-forge channel 17 | 18 | # Merging data by PyPi Name 19 | # main_df stores the package name as name and cf_df as conda_name so there is no collision 20 | mdf = pd.merge(main_df, cf_df, left_on="pypi_name", right_on="pypi_name") 21 | 22 | not_found_df = mdf[mdf.name != mdf.conda_name].sort_values(by="name") 23 | 24 | # Because there could be other names (aliases or older names) we should check 25 | # if each of the discrepancies also exists. 26 | 27 | conda_search_results = {} 28 | for x in not_found_df.conda_name: 29 | if x in conda_search_results: 30 | continue 31 | print(f"Looking up {x} on main...") 32 | # Note: In the original jupyter notebook of the analysis 33 | # this is what was run: query = !conda search {x} 34 | query = subprocess.run(["conda", "search", x], capture_output=True, text=True) 35 | conda_search_results[x] = query 36 | 37 | # Find any than are also found in main (these were noted belov) 38 | found_on_main = [ 39 | k 40 | for k, v in conda_search_results.items() 41 | if not any("PackagesNotFoundError" in l for l in v) 42 | ] 43 | ``` 44 | 45 | ## Data Overview 46 | 47 | **72 packages** were found with discrepancies between main conda channel and conda-forge channel conda 48 | package names. This was done by collecting information main's packages (from an internal to Anaconda 49 | data store) and comparing this against conda-forge's cf-graph-countyfair. The cf-graph-county-fair 50 | is the data repository for conda-forge's automation. This repository stores the dependency graph 51 | and its introspection. Prefix's parselmouth repository (which is used to store similar mapping data) 52 | was not used in this comparison. 53 | 54 | A few of these were not listed below as the data mapping was incorrect. 55 | 56 | ## Key Findings 57 | 58 | Discrepancies fall in several categories. There are cases where the name of the package was changed 59 | (This, of course, has it's own challenges by having to select the 'correct' conda package). 60 | 61 | ### Naming Pattern Categories 62 | 63 | #### **Prefix/Suffix Standardization Differences** 64 | 65 | |#| Main Channel Name | Conda-Forge Name | PyPI Name | Notes | 66 | |-|-------------------|-------------------|-----------|-------| 67 | |1| `astropy` | `astropy-base` | `astropy` | | 68 | |2| `avro-python3` | `python-avro` | `avro-python3` | | 69 | |3| `cufflinks-py` | `python-cufflinks` | `cufflinks` | | 70 | |4| `duckdb` | `python-duckdb` | `duckdb` | | 71 | |5| `pyct-core` | `pyct` | `pyct` | main also has `pyct` | 72 | |6| `pandera` | `pandera-core` | `pandera` | main also has `pandera-core`| 73 | |7| `qtconsole` | `qtconsole-base` | `qtconsole` | | 74 | |8| `seaborn` | `seaborn-base` | `seaborn` | | 75 | |9| `spyder` | `spyder-base` | `spyder` | | 76 | |10| `tables` | `pytables` | `tables` | main also has `pytables` | 77 | 78 | #### **Vendor/Project Name Clarification** 79 | 80 | |#| Main Channel Name | Conda-Forge Name | PyPI Name | Notes | 81 | |-|-------------------|-------------------|-----------|-------| 82 | |1| `analytics-python` | `segment-analytics-python` | `segment-analytics-python` | | 83 | |2| `authzed` | `authzed-py` | `authzed` | main also has `authzed-py`| 84 | |3| `jupyterlab-variableinspector` | `lckr_jupyterlab_variableinspector` | `lckr-jupyterlab-variableinspector` | | 85 | |4| `performance` | `pyperformance` | `pyperformance` | main also has `pyperformance` | 86 | |5| `prince` | `prince-factor-analysis` | `prince` | | 87 | |6| `pywget` | `python-wget` | `wget` | | 88 | |7| `lit` | `lit-nlp` | `lit` || 89 | 90 | #### **Hyphen vs Underscore Standardization** 91 | 92 | |#| Main Channel Name | Conda-Forge Name | PyPI Name | Notes | 93 | |-|-------------------|-------------------|-----------|-------| 94 | |1| `argon2_cffi` | `argon2-cffi` | `argon2-cffi` | main also has `argon2-cffi` | 95 | |2| `cached-property` | `cached_property` | `cached-property` | | 96 | |3| `et_xmlfile` | `et-xmlfile` | `et-xmlfile` | | 97 | |4| `eval-type-backport` | `eval_type_backport` | `eval-type-backport` | | 98 | |5| `flask-json` | `flask_json` | `flask-json` | | 99 | |6| `importlib-resources` | `importlib_resources` | `importlib-resources` | main also has `importlib_resources` | 100 | |7| `lazy_loader` | `lazy-loader` | `lazy-loader` | | 101 | |8| `sarif_om` | `sarif-om` | `sarif-om` | | 102 | |9| `service_identity` | `service-identity` | `service-identity` | | 103 | |10| `setuptools-scm-git-archive` | `setuptools_scm_git_archive` | `setuptools-scm-git-archive` | main also has `setuptools_scm_git_archive` | 104 | |11| `streamlit-option-menu` | `streamlit_option_menu` | `streamlit-option-menu` | | 105 | |12| `typing-extensions` | `typing_extensions` | `typing-extensions` | | 106 | 107 | #### **Package Family Consolidation** 108 | 109 | |#| Main Channel Name | Conda-Forge Name | PyPI Name | Notes | 110 | |-|-------------------|-------------------|-----------|-------| 111 | |1| `diffusers-base` | `diffusers` | `diffusers` | main also has `diffusers` | 112 | |2| `diffusers-torch` | `diffusers` | `diffusers` | | 113 | |3| `gql-with-aiohttp` | `gql` | `gql` | | 114 | |4| `gql-with-all` | `gql` | `gql` | | 115 | |5| `gql-with-botocore` | `gql` | `gql` | | 116 | |6| `gql-with-httpx` | `gql` | `gql` | | 117 | |7| `gql-with-requests` | `gql` | `gql` | | 118 | |8| `gql-with-websockets` | `gql` | `gql` | | 119 | |9| `keras-base` | `keras` | `keras` | main also has `keras` | 120 | |10| `keras-gpu` | `keras` | `keras` | main also has `keras` | 121 | |11| `pandera-base` | `pandera-core` | `pandera` | This is weird as conda-forge maintains both pandera and pandera-core which point to the same PyPI project pandera. | 122 | |12| `pandera-dask` | `pandera-core` | `pandera` | | 123 | |13| `pandera-fastapi` | `pandera-core` | `pandera` | | 124 | |14| `pandera-geopandas` | `pandera-core` | `pandera` | | 125 | |15| `pandera-hypotheses` | `pandera-core` | `pandera` | | 126 | |16| `pandera-io` | `pandera-core` | `pandera` | | 127 | |17| `pandera-modin` | `pandera-core` | `pandera` | | 128 | |18| `pandera-modin-dask` | `pandera-core` | `pandera` | | 129 | |19| `pandera-modin-ray` | `pandera-core` | `pandera` | | 130 | |20| `pandera-mypy` | `pandera-core` | `pandera` | | 131 | |21| `pandera-pyspark` | `pandera-core` | `pandera` | | 132 | |22| `pandera-strategies` | `pandera-core` | `pandera` | | 133 | |23| `pytorch-cpu` | `pytorch` | `torch` | Old main variants, main now uses pytorch. | 134 | |24| `pytorch-gpu` | `pytorch` | `torch` | Old main variants, main now uses pytorch. | 135 | |25| `uvicorn-standard` | `uvicorn` | `uvicorn` | main also has `uvicorn` | 136 | 137 | 138 | ### Impact Analysis 139 | 140 | - There are discrepancies even within channels (pandera and pandera-core) 141 | - There are very few discrepancies between the two channels. Main has more than 2k different PyPI projects and there are only about 20 real differences. 142 | - Because of main's longevity, there are older packages that are no longer maintained as main seems to be moving to use conda-forge package names whenever possible. 143 | 144 | ## Research Questions for Further Investigation 145 | 146 | - Is there disagreement among cf-graph-countyfair and parselmouth? 147 | - Would we expect it to be very significant if there was? 148 | - Should there be an effort to identify packages that are no longer current? 149 | - This is for cases where the conda package name has changed within a channel. 150 | 151 | ## Recommendations 152 | 153 | - Because of the few discrepancies, conda-pypi could get away with using conda-forge mapping with little impact. 154 | - As we move forward to a sustainable solution (a continually updated ecosystem mapping), they should be on a channel by channel basis. 155 | - For the short term MVP, conda-pypi should hard code as a long-term sustainable solution is decided on and implemented. 156 | 157 | ## Conclusion 158 | 159 | There are very few instances of name differences between main and conda-forge. Though it would be optimal to have an index by channel, in the short term 160 | conda-pypi could just use conda-forge mappings. For extra coverage, we could hard code, the small list of exceptions found by this report. 161 | -------------------------------------------------------------------------------- /docs/why/conda-vs-pypi.md: -------------------------------------------------------------------------------- 1 | # Key differences between conda and PyPI 2 | 3 | Below, we'll go over the two key differences between conda and PyPI packaging and why 4 | this leads to issues for users. The first problem is related to how binary distributions 5 | are packaged and distributed and the second problem is related to the package index 6 | and how each tool tracks what is currently installed. 7 | 8 | ## Summary 9 | 10 | - Conda and PyPI use different strategies for building binary distributions; when 11 | using these packaging formats together, it can lead to hard debug issues. 12 | - PyPI tools are aware of what is installed in a conda environment but when these 13 | tools make changes to the environment conda looses track of what is installed. 14 | - conda relies on all packaging metadata (available packages, their dependencies, etc) 15 | being available upfront. PyPI only lists the available packages, but their dependencies 16 | need to be fetched on a package-per-package basis. This means that the solvers are 17 | designed to work differently; a conda solver won't easily take the PyPI metadata 18 | because it is not designed to work iteratively. 19 | - PyPI names are not always the same in a conda channel. They might have a different name, 20 | or use a different packaging approach altogether. 21 | 22 | ## Differences in binary distributions 23 | 24 | Conda and PyPI are separate packaging ecosystems with different packaging 25 | formats and philosophies. Conda distributes packages as .conda and .tar.bz2 26 | files, which can include Python libraries and pre-compiled binaries with dynamic 27 | links to other dependencies. In contrast, PyPI provides .whl files (as defined 28 | in [PEP 427](https://peps.python.org/pep-0427/)), which typically bundle all 29 | required binaries or rely on system-level dependencies, as it lacks support for 30 | non-Python dependency declarations [^1]. PyPI also supports source distributions, 31 | though these require building during installation. 32 | 33 | With that in mind, what are some potential ways this could break when combining the two 34 | ecosystems together? Because wheels typically include all of their pre-compiled binaries inside 35 | the wheel itself, this can lead to incompatibilities when used with conda packages containing 36 | pre-compiled binaries. In the conda ecosystem, these dependencies are normally tested with 37 | each other before being published during the build process, but the PyPI ecosystem does not test 38 | its wheels with conda packages and therefore users are typically the first one to run into these 39 | errors. 40 | 41 | Some examples of these incompatibilities include symbol errors, segfaults and other hard to debug 42 | issues. Refer to the excellent [pypackaging-native key issues](https://pypackaging-native.github.io/#key-issues) 43 | for even more information on this topic and specific examples. 44 | 45 | ## Differences in metadata concerning installed packages 46 | 47 | The second relevant difference regarding how these two packaging ecosystems interact with 48 | each other deals with how they track which packages are installed into an environment. 49 | Inside conda environments, package metadata is saved in JSON files in a folder called 50 | `conda-meta`. This serves as a way to easily identify everything that is currently installed 51 | because each JSON file represents a package installed in that environment. 52 | 53 | Tools such as `pip` that follow the Database of Installed Python Distributions 54 | standard ([PEP 376](https://peps.python.org/pep-0376/)) do not store metadata 55 | about installed packages in a single central database like `conda-meta`. 56 | Instead, each installed distribution has its own `.dist-info` directory located 57 | in `lib//site-packages/`, containing metadata files like 58 | METADATA, RECORD, and INSTALLER that track the distribution's files and 59 | installation details. 60 | 61 | The good thing is that conda Python packages will normally install this directory 62 | when the package is installed. This means that `pip` installations on top of an existing 63 | conda environment will be able to tell what is already installed and resolve its dependencies 64 | relatively well. But, after you have installed something with `pip` in that environment, 65 | conda no longer knows exactly what is installed because `pip` did not update the contents 66 | of the `conda-meta` folder. 67 | 68 | This ultimately means that conda begins to lose track of what is installed in a given environment. 69 | Not only that, `pip` and `conda` can begin to overwrite what each has placed in the 70 | `lib//site-packages` folder. The more you run each tool independently, 71 | the more likely it is that this will happen, and the more this happens, the more unstable 72 | and prone to errors this environment becomes. 73 | 74 | ## Package metadata differences 75 | 76 | PyPI and conda expose their packaging metadata in different ways, which results in their 77 | solvers working differently too: 78 | 79 | In the conda ecosystem, packages are published to a *channel*. The metadata in 80 | each package is extracted and aggregated into a per-platform JSON file 81 | (`repodata.json`) upon package publication. `repodata.json` contains all the 82 | packaging metadata needed for the solver to operate, and it's typically fetched 83 | and updated every time the user tries to install something. 84 | 85 | In PyPI, packages are published to an *index* following the Simple Repository 86 | API standard ([PEP 503](https://peps.python.org/pep-0503/)). The index provides 87 | a list of all the available wheel files, with their filenames encoding some 88 | packaging metadata (like Python version and platform compatibility). Other 89 | metadata like the dependencies for that package need to be fetched on a 90 | per-wheel basis. As a result, the solver fetches metadata as it goes. Modern 91 | PyPI implementations can serve this index data in either HTML (the original PEP 92 | 503 format) or JSON ([PEP 691](https://peps.python.org/pep-0691/)) using HTTP 93 | content negotiation. 94 | 95 | In a nutshell: 96 | 97 | - Conda's metadata is aggregated upfront in `repodata.json` files, providing 98 | solvers with comprehensive dependency information before package resolution 99 | begins. 100 | - PyPI's approach has traditionally required per-package metadata fetching 101 | during solving, though the JSON-based Simple API for Python Package Indexes 102 | ([PEP 691](https://peps.python.org/pep-0691/)) now provides more structured 103 | metadata access that reduces this need. 104 | - The conda solvers can work entirely from the aggregated metadata, while 105 | PyPI-focused solvers have typically needed to fetch additional metadata as 106 | solutions are explored, though this pattern is evolving with newer API 107 | capabilities. 108 | 109 | So, if we wanted to integrate PyPI with conda, this represents an architectural 110 | difference: how to adapt PyPI's per-package metadata model to conda's 111 | expectation of comprehensive upfront metadata aggregation. 112 | 113 | ```{note} 114 | Some solver backends (e.g. [`resolvo`](https://github.com/prefix-dev/resolvo)) do support iterative solving like in the PyPI model, but they have not been adapted for conda+PyPI interoperability. 115 | ``` 116 | 117 | ## Mappings and names 118 | 119 | Even if we had all the necessary metadata available upfront, we would face one more problem: given a Python project, the name in PyPI does not necessarily match its names in a conda channel. It's true that in a good percentage of cases they would match, but the problem arises in the edge cases. Some examples: 120 | 121 | - A package being published with different names: [`pypa/build`](https://github.com/pypa/build) is published to PyPI as `build` but it's `python-build` in conda-forge. 122 | - A package with names encoding versions: `PyQt` v5 is published in PyPI as `pyqt5`, but in conda-forge is simply `pyqt` with version `5` (`pyqt=5`). 123 | - Different packages with the same name: `art` is a [popular ASCII art package](https://github.com/sepandhaghighi/art/) in PyPI but a [genomics project](https://www.niehs.nih.gov/research/resources/software/biostatistics/art) in `bioconda`. 124 | 125 | Additionally there are other challenges like name normalization: in PyPI dashes and underscores are treated in the same way, but conda packaging considers them separate. This leads to efforts like publishing two conda packages for a given PyPI project if it contains any of this separators: PyPI's `typing-extensions` is available as both `typing-extensions` and `typing_extensions`. However these alias packages are not always published, and the separator flavor you get on the conda side is not always consistent. 126 | 127 | ## More on this topic 128 | 129 | For an excellent overview of how Python packaging works: 130 | 131 | - [Python Packaging - packaging.python.org](https://packaging.python.org/en/latest/overview/) 132 | 133 | For an excellent overview of what a conda package actually is: 134 | 135 | - [What is a conda package? - prefix.dev](https://prefix.dev/blog/what-is-a-conda-package) 136 | 137 | [^1]: At least as of August 2025. Check [PEP 725](https://peps.python.org/pep-0725/) for a proposal external dependency metadata. 138 | -------------------------------------------------------------------------------- /conda_pypi/translate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert Python `*.dist-info/METADATA` to conda `info/index.json` 3 | """ 4 | 5 | import dataclasses 6 | import json 7 | import logging 8 | import pkgutil 9 | import sys 10 | import time 11 | from importlib.metadata import Distribution, PathDistribution 12 | 13 | try: 14 | from importlib.metadata import PackageMetadata 15 | except ImportError: 16 | # Python < 3.10 compatibility 17 | PackageMetadata = Distribution 18 | from pathlib import Path 19 | from typing import Any, Optional, List, Dict 20 | 21 | from conda.models.match_spec import MatchSpec 22 | from packaging.requirements import InvalidRequirement, Requirement 23 | from packaging.utils import canonicalize_name 24 | 25 | log = logging.getLogger(__name__) 26 | 27 | 28 | class FileDistribution(Distribution): 29 | """ 30 | From a file e.g. a single `.metadata` fetched from pypi instead of a 31 | `*.dist-info` folder. 32 | """ 33 | 34 | def __init__(self, raw_text): 35 | self.raw_text = raw_text 36 | 37 | def read_text(self, filename: str) -> Optional[str]: 38 | if filename == "METADATA": 39 | return self.raw_text 40 | else: 41 | return None 42 | 43 | def locate_file(self, path): 44 | """ 45 | Given a path to a file in this distribution, return a path 46 | to it. 47 | """ 48 | return None 49 | 50 | 51 | @dataclasses.dataclass 52 | class PackageRecord: 53 | # what goes in info/index.json 54 | name: str 55 | version: str 56 | subdir: str 57 | depends: List[str] 58 | extras: Dict[str, List[str]] 59 | build_number: int = 0 60 | build_text: str = "pypi" # e.g. hash 61 | license_family: str = "" 62 | license: str = "" 63 | noarch: str = "" 64 | timestamp: int = 0 65 | 66 | def to_index_json(self): 67 | return { 68 | "build_number": self.build_number, 69 | "build": self.build, 70 | "depends": self.depends, 71 | "extras": self.extras, 72 | "license_family": self.license_family, 73 | "license": self.license, 74 | "name": self.name, 75 | "noarch": self.noarch, 76 | "subdir": self.subdir, 77 | "timestamp": self.timestamp, 78 | "version": self.version, 79 | } 80 | 81 | @property 82 | def build(self): 83 | return f"{self.build_text}_{self.build_number}" 84 | 85 | @property 86 | def stem(self): 87 | return f"{self.name}-{self.version}-{self.build}" 88 | 89 | 90 | @dataclasses.dataclass 91 | class CondaMetadata: 92 | metadata: PackageMetadata 93 | console_scripts: List[str] 94 | package_record: PackageRecord 95 | about: Dict[str, Any] 96 | 97 | def link_json(self) -> Optional[dict]: 98 | """ 99 | info/link.json used for console scripts; None if empty. 100 | 101 | Note the METADATA file aka PackageRecord does not list console scripts. 102 | """ 103 | # XXX gui scripts? 104 | return { 105 | "noarch": {"entry_points": self.console_scripts, "type": "python"}, 106 | "package_metadata_version": 1, 107 | } 108 | 109 | @classmethod 110 | def from_distribution(cls, distribution: Distribution): 111 | metadata = distribution.metadata 112 | 113 | python_version = metadata["requires-python"] 114 | requires_python = "python" 115 | if python_version: 116 | requires_python = f"python {python_version}" 117 | 118 | requirements, extras = requires_to_conda(distribution.requires) 119 | 120 | # conda does support ~=3.0.0 "compatibility release" matches 121 | depends = [requires_python] + requirements 122 | 123 | console_scripts = [ 124 | f"{ep.name} = {ep.value}" 125 | for ep in distribution.entry_points 126 | if ep.group == "console_scripts" 127 | ] 128 | 129 | noarch = "python" 130 | 131 | # Common "about" keys 132 | # ['channels', 'conda_build_version', 'conda_version', 'description', 133 | # 'dev_url', 'doc_url', 'env_vars', 'extra', 'home', 'identifiers', 134 | # 'keywords', 'license', 'license_family', 'license_file', 'root_pkgs', 135 | # 'summary', 'tags', 'conda_private', 'doc_source_url', 'license_url'] 136 | 137 | about = { 138 | "summary": metadata.get("summary"), 139 | "description": metadata.get("description"), 140 | # https://packaging.python.org/en/latest/specifications/core-metadata/#license-expression 141 | "license": metadata.get("license_expression") or metadata.get("license"), 142 | } 143 | 144 | if project_urls := metadata.get_all("project-url"): 145 | urls = dict(url.split(", ", 1) for url in project_urls) 146 | for py_name, conda_name in ( 147 | ("Home", "home"), 148 | ("Development", "dev_url"), 149 | ("Documentation", "doc_url"), 150 | ): 151 | if py_name in urls: 152 | about[conda_name] = urls[py_name] 153 | 154 | name = pypi_to_conda_name( 155 | getattr(distribution, "name", None) or distribution.metadata["name"] 156 | ) 157 | version = getattr(distribution, "version", None) or distribution.metadata["version"] 158 | 159 | package_record = PackageRecord( 160 | build_number=0, 161 | depends=depends, 162 | extras=extras, 163 | license=about["license"] or "", 164 | license_family="", 165 | name=name, 166 | version=version, 167 | subdir="noarch", 168 | noarch=noarch, 169 | timestamp=time.time_ns() // 1000000, 170 | ) 171 | 172 | return cls( 173 | metadata=metadata, 174 | package_record=package_record, 175 | console_scripts=console_scripts, 176 | about=about, 177 | ) 178 | 179 | 180 | # The keys are pypi names 181 | # conda_pypi.dist_repodata.grayskull_pypi_mapping['zope-hookable'] 182 | # { 183 | # "pypi_name": "zope-hookable", 184 | # "conda_name": "zope.hookable", 185 | # "import_name": "zope.hookable", 186 | # "mapping_source": "regro-bot", 187 | # } 188 | grayskull_pypi_mapping = json.loads( 189 | pkgutil.get_data("conda_pypi", "grayskull_pypi_mapping.json") or "{}" 190 | ) 191 | 192 | 193 | def requires_to_conda(requires: Optional[List[str]]): 194 | from collections import defaultdict 195 | 196 | extras: Dict[str, List[str]] = defaultdict(list) 197 | requirements = [] 198 | for requirement in [Requirement(dep) for dep in requires or []]: 199 | # requirement.marker.evaluate 200 | 201 | # if requirement.marker and not requirement.marker.evaluate(): 202 | # # excluded by environment marker 203 | # # see also marker evaluation according to given sys.executable 204 | # continue 205 | 206 | name = canonicalize_name(requirement.name) 207 | requirement.name = pypi_to_conda_name(name) 208 | as_conda = f"{requirement.name} {requirement.specifier}" 209 | 210 | if (marker := requirement.marker) is not None: 211 | # for var, _, value in marker._markers: 212 | for mark in marker._markers: 213 | if isinstance(mark, tuple): 214 | var, _, value = mark 215 | if str(var) == "extra": 216 | extras[str(value)].append(as_conda) 217 | else: 218 | requirements.append(f"{requirement.name} {requirement.specifier}".strip()) 219 | 220 | return requirements, dict(extras) 221 | 222 | # if there is a url or extras= here we have extra work, may need to 223 | # yield Requirement not str 224 | # sorted(packaging.requirements.SpecifierSet("<5,>3")._specs, key=lambda x: x.version) 225 | # or just sorted lexicographically in str(SpecifierSet) 226 | # yield f"{requirement.name} {requirement.specifier}" 227 | 228 | 229 | def conda_to_requires(matchspec: MatchSpec): 230 | name = matchspec.name 231 | if isinstance(name, str): 232 | pypi_name = conda_to_pypi_name(name) 233 | # XXX ugly 'omits = for exact version' 234 | # .spec omits package[version='>=1.0'] bracket format when possible 235 | best_format = str(matchspec) 236 | if "version=" in best_format: 237 | best_format = matchspec.spec 238 | try: 239 | return Requirement(best_format.replace(name, pypi_name)) 240 | except InvalidRequirement: 241 | # attempt to catch 'httpcore 1.*' style conda requirement 242 | best_format = "==".join(matchspec.spec.split()) 243 | return Requirement(best_format.replace(name, pypi_name)) 244 | 245 | 246 | def pypi_to_conda_name(pypi_name: str): 247 | pypi_name = canonicalize_name(pypi_name) 248 | return grayskull_pypi_mapping.get( 249 | pypi_name, 250 | { 251 | "pypi_name": pypi_name, 252 | "conda_name": pypi_name, 253 | "import_name": None, 254 | "mapping_source": None, 255 | }, 256 | )["conda_name"] 257 | 258 | 259 | _to_pypi_name_map = {} 260 | 261 | 262 | def conda_to_pypi_name(name: str): 263 | if not _to_pypi_name_map: 264 | for value in grayskull_pypi_mapping.values(): 265 | conda_name = value["conda_name"] 266 | # XXX sometimes conda:pypi is n:1 267 | _to_pypi_name_map[conda_name] = value 268 | 269 | found = _to_pypi_name_map.get(name) 270 | if found: 271 | name = found["pypi_name"] 272 | return canonicalize_name(name) 273 | 274 | 275 | if __name__ == "__main__": # pragma: no cover 276 | base = sys.argv[1] 277 | for path in Path(base).glob("*.dist-info"): 278 | print(CondaMetadata.from_distribution(PathDistribution(path))) 279 | --------------------------------------------------------------------------------