├── docs ├── dev │ ├── changelog.md │ ├── code-of-conduct.md │ ├── performance-testing.md │ ├── index.md │ ├── future-work.md │ ├── workflows.md │ ├── shards_python.mmd │ ├── setup.md │ └── sharded.md ├── robots.txt ├── _static │ └── css │ │ └── custom.css ├── user-guide │ ├── subcommands.md │ ├── more-resources.md │ ├── performance.md │ ├── configuration.md │ ├── index.md │ └── faq.md ├── Makefile ├── _templates │ └── navbar_center.html ├── index.md ├── environment.yml └── conf.py ├── .devcontainer ├── apt-deps.txt ├── post_create.sh ├── defaults │ └── devcontainer.json ├── conda-forge │ └── devcontainer.json └── post_start.sh ├── tests ├── data │ ├── lock_this_env.yml │ ├── mamba_repo │ │ ├── noarch │ │ │ ├── repodata.json.bz2 │ │ │ ├── current_repodata.json.bz2 │ │ │ ├── test-package-0.1-0.tar.bz2 │ │ │ ├── repodata_from_packages.json.bz2 │ │ │ ├── repodata.json │ │ │ ├── current_repodata.json │ │ │ ├── repodata_from_packages.json │ │ │ └── index.html │ │ ├── recipes │ │ │ └── test-package │ │ │ │ └── meta.yaml │ │ ├── channeldata.json │ │ ├── LICENSE │ │ └── index.html │ ├── conda_build_recipes │ │ ├── baddeps │ │ │ └── meta.yaml │ │ ├── multioutput │ │ │ └── meta.yaml │ │ ├── jedi │ │ │ └── meta.yaml │ │ ├── LICENSE │ │ └── stackvana │ │ │ └── meta.yaml │ └── mambaforge.linux-64.lock ├── __init__.py ├── requirements.txt ├── conftest.py ├── run_in_profiler.py ├── test_user_agent.py ├── utils.py ├── test_plugin.py ├── test_downstream.py ├── http_test_server.py ├── test_experimental.py ├── test_repoquery.py ├── test_workarounds.py ├── test_performance.py ├── channel_testing │ └── helpers.py ├── repodata_time_machine.py └── test_index.py ├── .github ├── labels.yml ├── disclaimer.txt ├── CODEOWNERS ├── dependabot.yml ├── TEST_FAILURE_REPORT_TEMPLATE.md ├── workflows │ ├── project.yml │ ├── cla.yml │ ├── issues.yml │ ├── docs.yml │ ├── labels.yml │ ├── lock.yml │ ├── builds-review.yaml │ ├── stale.yml │ ├── update.yml │ └── performance.yml ├── template-files │ ├── templates │ │ ├── pull_request_template_details.md │ │ └── issue_template_details.yml │ └── config.yml ├── ISSUE_TEMPLATE │ ├── 2_documentation.yml │ ├── 1_feature.yml │ ├── 0_bug.yml │ └── epic.yml └── PULL_REQUEST_TEMPLATE.md ├── .flake8 ├── news ├── TEMPLATE ├── 744-remove-tar-bz2.md ├── 710-offline-shards-support.md ├── 805-track-features-list └── cuda-cpuonly-issue-returns ├── dev ├── requirements.txt └── scripts │ ├── example.xonshrc │ ├── fix_xonshrc.py │ ├── requests-fetch-all-shards.py │ ├── httpx-fetch-all-shards.py │ └── BenchmarkChart.ipynb ├── AUTHORS.md ├── conda_libmamba_solver ├── exceptions.py ├── __init__.py ├── plugin.py ├── conda_build_exceptions.py ├── shards_typing.py ├── utils.py ├── shards_cache.py └── mamba_utils.py ├── rever.xsh ├── .mailmap ├── recipe └── meta.yaml ├── LICENSE ├── .pre-commit-config.yaml ├── README.md ├── .authors.yml ├── .gitignore └── pyproject.toml /docs/dev/changelog.md: -------------------------------------------------------------------------------- 1 | ../../CHANGELOG.md -------------------------------------------------------------------------------- /docs/dev/code-of-conduct.md: -------------------------------------------------------------------------------- 1 | ../../CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /.devcontainer/apt-deps.txt: -------------------------------------------------------------------------------- 1 | git 2 | less 3 | htop 4 | nano 5 | ssh 6 | -------------------------------------------------------------------------------- /tests/data/lock_this_env.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | - conda-forge 3 | dependencies: 4 | - zlib 5 | -------------------------------------------------------------------------------- /docs/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | 3 | Sitemap: https://conda.github.io/conda-libmamba-solver/sitemap.xml 4 | -------------------------------------------------------------------------------- /.github/labels.yml: -------------------------------------------------------------------------------- 1 | # Builds 2 | - name: build::review 3 | description: trigger a build for this PR 4 | color: "7B4052" 5 | -------------------------------------------------------------------------------- /.github/disclaimer.txt: -------------------------------------------------------------------------------- 1 | Copyright (C) 2022 Anaconda, Inc 2 | Copyright (C) 2023 conda 3 | SPDX-License-Identifier: BSD-3-Clause 4 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Syntax for this file at https://help.github.com/articles/about-codeowners/ 2 | 3 | * @conda/conda-maintainers 4 | -------------------------------------------------------------------------------- /tests/data/mamba_repo/noarch/repodata.json.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conda/conda-libmamba-solver/HEAD/tests/data/mamba_repo/noarch/repodata.json.bz2 -------------------------------------------------------------------------------- /tests/data/mamba_repo/noarch/current_repodata.json.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conda/conda-libmamba-solver/HEAD/tests/data/mamba_repo/noarch/current_repodata.json.bz2 -------------------------------------------------------------------------------- /tests/data/mamba_repo/noarch/test-package-0.1-0.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conda/conda-libmamba-solver/HEAD/tests/data/mamba_repo/noarch/test-package-0.1-0.tar.bz2 -------------------------------------------------------------------------------- /tests/data/mamba_repo/noarch/repodata_from_packages.json.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conda/conda-libmamba-solver/HEAD/tests/data/mamba_repo/noarch/repodata_from_packages.json.bz2 -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 99 3 | ignore = E126,E133,E226,E241,E242,E302,E704,E731,E722,W503,E402,W504,F821,E203 4 | exclude = tests/*,conda_libmamba_solver/mamba_utils.py 5 | -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | /** 2 | * This rule is here to avoid the scrollbar appearing when this 3 | * is not hosted on ReadTheDocs 4 | */ 5 | #rtd-footer-container { 6 | display: none; 7 | } 8 | -------------------------------------------------------------------------------- /tests/requirements.txt: -------------------------------------------------------------------------------- 1 | conda-build 2 | conda-forge::pytest-xprocess 3 | conda-index 4 | conda-lock 5 | conda-forge::pre-commit 6 | pytest-codspeed >=4 7 | conda-forge::pytest-mock 8 | # needed for many conda tests 9 | flask 10 | -------------------------------------------------------------------------------- /news/TEMPLATE: -------------------------------------------------------------------------------- 1 | ### Enhancements 2 | 3 | * 4 | 5 | ### Bug fixes 6 | 7 | * 8 | 9 | ### Deprecations 10 | 11 | * 12 | 13 | ### Docs 14 | 15 | * 16 | 17 | ### Other 18 | 19 | * 20 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | groups: 8 | github-actions: 9 | patterns: 10 | - '*' 11 | -------------------------------------------------------------------------------- /news/744-remove-tar-bz2.md: -------------------------------------------------------------------------------- 1 | ### Bug fixes 2 | 3 | * Remove `.tar.bz2` with matching `.conda`-format packages during shard 4 | traversal if `conda` is not in "use_only_tar_bz2" mode; needed as shards 5 | directly adds individual packages to the solver. (#710) 6 | -------------------------------------------------------------------------------- /dev/requirements.txt: -------------------------------------------------------------------------------- 1 | # build-time 2 | pip 3 | # run-time 4 | boltons>=23.0.0 5 | conda>=25.9 6 | conda-forge::libmamba>=2.0.0 7 | conda-forge::libmambapy>=2.0.0 8 | msgpack-python>=1.1.1 9 | # needed for 'pip install -e conda-libmamba-solver' 10 | hatchling 11 | hatch-vcs 12 | editables 13 | -------------------------------------------------------------------------------- /news/710-offline-shards-support.md: -------------------------------------------------------------------------------- 1 | ### Enhancements 2 | 3 | * Add offline mode support for sharded repodata. When offline mode is enabled, the solver will use cached shards even if they are expired, and gracefully fall back to non-sharded repodata if no cache exists. Missing shards in offline mode return empty shards rather than failing. (#710) 4 | -------------------------------------------------------------------------------- /news/805-track-features-list: -------------------------------------------------------------------------------- 1 | ### Enhancements 2 | 3 | * 4 | 5 | ### Bug fixes 6 | 7 | * Ensure `track_features` fields are recorded properly in `conda-meta/*.json`. (#804 via #805) 8 | 9 | ### Deprecations 10 | 11 | * 12 | 13 | ### Docs 14 | 15 | * 16 | 17 | ### Other 18 | 19 | * 20 | -------------------------------------------------------------------------------- /tests/data/mamba_repo/recipes/test-package/meta.yaml: -------------------------------------------------------------------------------- 1 | package: 2 | name: test-package 3 | version: 0.1 4 | 5 | build: 6 | number: 0 7 | script: echo Hello world 8 | noarch: generic 9 | 10 | about: 11 | home: https://github.com/mamba-org/mamba 12 | license: BSD 13 | license_family: BSD 14 | summary: I am just a test package! 15 | -------------------------------------------------------------------------------- /.github/TEST_FAILURE_REPORT_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "{{ env.TITLE }}" 3 | labels: [type::bug, type::testing] 4 | --- 5 | 6 | The {{ workflow }} workflow failed on {{ date | date("YYYY-MM-DD HH:mm") }} UTC 7 | 8 | Full run: https://github.com/conda/conda-libmamba-solver/actions/runs/{{ env.RUN_ID }} 9 | 10 | (This post will be updated if another test fails, as long as this issue remains open.) 11 | -------------------------------------------------------------------------------- /news/cuda-cpuonly-issue-returns: -------------------------------------------------------------------------------- 1 | ### Docs 2 | 3 | * Updated documentation to reflect that the cudatoolkit/cpuonly issue has returned. 4 | 5 | ### Other 6 | 7 | * The `cpuonly` mutex no longer correctly prevents CUDA packages from being 8 | installed. For a time it appeared to work when non-cuda systems showed a 9 | virtual package `__cuda=0=0`, but with no `__cuda` package the mutex appears 10 | to no longer work. Reversing (#131 via #741) 11 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | 5 | pytest_plugins = ( 6 | # Add testing fixtures and internal pytest plugins here 7 | "conda.testing", 8 | "conda.testing.fixtures", 9 | ) 10 | 11 | # Allow fixtures from test_shards to be available globally, specifically in 12 | # test_shards_subset: 13 | from .test_shards import ( # noqa: F401 14 | http_server_shards, 15 | prepare_shards_test, 16 | shard_cache_with_data, 17 | ) 18 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | All of the people who have made at least one contribution to conda-libmamba-solver. 2 | Authors are sorted alphabetically. 3 | 4 | * Agriya Khetarpal 5 | * Albert DeFusco 6 | * Christopher Ostrouchov 7 | * Daniel Holth 8 | * Jaime Rodríguez-Guerra 9 | * Jannis Leidel 10 | * John Kirkham 11 | * Jonathan J. Helmus 12 | * Julien Jerphanion 13 | * Ken Odegard 14 | * Kevin Markham 15 | * Klaus Zimmermann 16 | * Matthew R. Becker 17 | * Stacy Noland 18 | * Thomas Lam 19 | * Travis Hathaway 20 | * conda-bot 21 | * dependabot[bot] 22 | * pre-commit-ci[bot] 23 | -------------------------------------------------------------------------------- /conda_libmamba_solver/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | Exceptions used in conda-libmamba-solver 6 | """ 7 | 8 | from conda.exceptions import UnsatisfiableError 9 | 10 | 11 | class LibMambaUnsatisfiableError(UnsatisfiableError): 12 | """An exception to report unsatisfiable dependencies. 13 | The error message is passed directly as a str. 14 | """ 15 | 16 | def __init__(self, message: str, **kwargs): 17 | super(UnsatisfiableError, self).__init__(str(message)) 18 | -------------------------------------------------------------------------------- /docs/user-guide/subcommands.md: -------------------------------------------------------------------------------- 1 | # Subcommands 2 | 3 | The conda-libmamba-solver package also provides conda subcommand plugins in addition to the solver plugin. 4 | 5 | ## `conda repoquery` 6 | 7 | A conda subcommand plugin that offers the same functionality as `mamba repoquery` and 8 | `micromamba repoquery`. It provides three actions: 9 | 10 | - `conda repoquery search`: Query repodata for packages matching a pattern. 11 | - `conda repoquery depends`: Show the dependencies of the requested package. 12 | - `conda repoquery whoneeds`: Show the packages that depend on the requested package. 13 | 14 | Check the `--help` messages for each task for more information. 15 | -------------------------------------------------------------------------------- /conda_libmamba_solver/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | try: 5 | from ._version import version as __version__ 6 | except ImportError: 7 | try: 8 | from importlib.metadata import version 9 | 10 | __version__ = version("conda_libmamba_solver") 11 | del version 12 | except ImportError: 13 | __version__ = "0.0.0.unknown" 14 | 15 | 16 | def get_solver_class(key: str = "libmamba"): 17 | if key == "libmamba": 18 | from .solver import LibMambaSolver 19 | 20 | return LibMambaSolver 21 | raise ValueError("Key must be 'libmamba'") 22 | -------------------------------------------------------------------------------- /docs/dev/performance-testing.md: -------------------------------------------------------------------------------- 1 | # Performance testing 2 | 3 | ## Codspeed 4 | 5 | Performance testing for this project can be found on codspeed here: 6 | 7 | - https://codspeed.io/conda/conda-libmamba-solver 8 | 9 | To run performance tests locally, run the following command in the root of the project: 10 | 11 | ```shell 12 | pytest --codspeed 13 | ``` 14 | 15 | To profile tests, add the following decorator above it: 16 | 17 | ```python 18 | import pytest 19 | 20 | 21 | @pytest.mark.benchmark 22 | def test_new_feature(): 23 | """Ensure feature performs well""" 24 | ``` 25 | 26 | See the official [codspeed documentation](https://codspeed.io/docs) for more information. 27 | -------------------------------------------------------------------------------- /docs/dev/index.md: -------------------------------------------------------------------------------- 1 | # Developer Guide 2 | 3 | To get started with developing conda-libmamba-solver, please check out 4 | the following pages: 5 | 6 | - [How to set up your environment](setup) 7 | - [Typical development workflows](workflows) 8 | - [Implementation details](implementation) 9 | - [Performance testing](performance-testing) 10 | - [Sharded repodata implementation](sharded) 11 | - [Future work](future-work) 12 | - [Code of conduct](code-of-conduct) 13 | - [CHANGELOG](changelog) 14 | 15 | ```{toctree} 16 | :maxdepth: 0 17 | :hidden: 18 | setup 19 | workflows 20 | implementation 21 | performance-testing 22 | sharded 23 | future-work 24 | code-of-conduct 25 | changelog 26 | ``` 27 | -------------------------------------------------------------------------------- /.github/workflows/project.yml: -------------------------------------------------------------------------------- 1 | name: Add to Project 2 | 3 | on: 4 | issues: 5 | types: 6 | - opened 7 | pull_request_target: 8 | types: 9 | - opened 10 | 11 | jobs: 12 | add_to_project: 13 | if: '!github.event.repository.fork' 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2 17 | with: 18 | # issues are added to the Planning project 19 | # PRs are added to the Review project 20 | project-url: https://github.com/orgs/conda/projects/${{ github.event_name == 'issues' && 2 || 16 }} 21 | github-token: ${{ secrets.PROJECT_TOKEN }} 22 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /tests/data/mamba_repo/noarch/repodata.json: -------------------------------------------------------------------------------- 1 | { 2 | "info": { 3 | "subdir": "noarch" 4 | }, 5 | "packages": { 6 | "test-package-0.1-0.tar.bz2": { 7 | "build": "0", 8 | "build_number": 0, 9 | "depends": [], 10 | "license": "BSD", 11 | "license_family": "BSD", 12 | "md5": "2a8595f37faa2950e1b433acbe91d481", 13 | "name": "test-package", 14 | "noarch": "generic", 15 | "sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988", 16 | "size": 5719, 17 | "subdir": "noarch", 18 | "timestamp": 1613117294885, 19 | "version": "0.1" 20 | } 21 | }, 22 | "packages.conda": {}, 23 | "removed": [], 24 | "repodata_version": 1 25 | } 26 | -------------------------------------------------------------------------------- /docs/dev/future-work.md: -------------------------------------------------------------------------------- 1 | # Future work 2 | 3 | ```{warning} WIP 4 | These will/should be expanded into issues. 5 | ``` 6 | 7 | * `MatchSpec` preparation logic contains too many exceptions for `libmamba` (e.g. differences in `update` vs `install`) 8 | * Clean-up `MatchSpec` exchange fixes and workarounds 9 | * Channel names and URLs need pre- and post- treatment (e.g. URL escaping) to workaround escaping issues, etc. 10 | * Investigate better usage of `libsolv` and `libmamba` flags 11 | * Better condense retry logic and conflict handling; we lack the "Optional" feature `classic` has 12 | * Investigate ignored tests in `pyproject.toml` 13 | * Conversely, investigate which other tests we should ignore in the name of simplifying the logic 14 | -------------------------------------------------------------------------------- /tests/data/mamba_repo/noarch/current_repodata.json: -------------------------------------------------------------------------------- 1 | { 2 | "info": { 3 | "subdir": "noarch" 4 | }, 5 | "packages": { 6 | "test-package-0.1-0.tar.bz2": { 7 | "build": "0", 8 | "build_number": 0, 9 | "depends": [], 10 | "license": "BSD", 11 | "license_family": "BSD", 12 | "md5": "2a8595f37faa2950e1b433acbe91d481", 13 | "name": "test-package", 14 | "noarch": "generic", 15 | "sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988", 16 | "size": 5719, 17 | "subdir": "noarch", 18 | "timestamp": 1613117294885, 19 | "version": "0.1" 20 | } 21 | }, 22 | "packages.conda": {}, 23 | "removed": [], 24 | "repodata_version": 1 25 | } 26 | -------------------------------------------------------------------------------- /tests/data/conda_build_recipes/baddeps/meta.yaml: -------------------------------------------------------------------------------- 1 | {% set name = "stackvana-core" %} 2 | {% set version = "0.2021.43" %} 3 | 4 | package: 5 | name: {{ name|lower }} 6 | version: {{ version }} 7 | 8 | build: 9 | number: 0 10 | 11 | outputs: 12 | - name: stackvana-core-impl 13 | version: {{ version }} 14 | build: 15 | script: 16 | - echo "BUILDING IMPL" >> $PREFIX/stackvana-core-impl # [unix] 17 | - echo "BUILDING IMPL" >> %PREFIX%/stackvana-core-impl # [win] 18 | - name: stackvana-core 19 | version: {{ version }} 20 | run_exports: 21 | - {{ pin_subpackage('stackvana-core-impl', exact=True) }} 22 | 23 | requirements: 24 | run: 25 | - thispackagedoesnotexist >=100000000 26 | -------------------------------------------------------------------------------- /tests/data/mamba_repo/noarch/repodata_from_packages.json: -------------------------------------------------------------------------------- 1 | { 2 | "info": { 3 | "subdir": "noarch" 4 | }, 5 | "packages": { 6 | "test-package-0.1-0.tar.bz2": { 7 | "build": "0", 8 | "build_number": 0, 9 | "depends": [], 10 | "license": "BSD", 11 | "license_family": "BSD", 12 | "md5": "2a8595f37faa2950e1b433acbe91d481", 13 | "name": "test-package", 14 | "noarch": "generic", 15 | "sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988", 16 | "size": 5719, 17 | "subdir": "noarch", 18 | "timestamp": 1613117294885, 19 | "version": "0.1" 20 | } 21 | }, 22 | "packages.conda": {}, 23 | "removed": [], 24 | "repodata_version": 1 25 | } 26 | -------------------------------------------------------------------------------- /docs/user-guide/more-resources.md: -------------------------------------------------------------------------------- 1 | # More resources 2 | 3 | If you want to learn more about `conda-libmamba-solver` and `conda`, we have compiled a list of additional material you can check: 4 | 5 | * ["A Faster Solver for Conda: Libmamba"](https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community) (Blog post) 6 | * Anaconda Webinars "Scaling Conda" series: 7 | * [A Faster conda for a Growing Community](https://anaconda.cloud/a-faster-conda-for-a-growing-community) 8 | * [A Closer Look at conda’s New libmamba Solver](https://anaconda.cloud/closer-look-at-conda-s-new-libmamba-solver) 9 | * [Scaling conda: Extending conda with a New Plugin Ecosystem](https://anaconda.cloud/scaling-conda-extending-conda-a-new-plugin-ecosystem) 10 | 11 | -------------------------------------------------------------------------------- /.github/template-files/templates/pull_request_template_details.md: -------------------------------------------------------------------------------- 1 | ### Checklist - did you ... 2 | 3 | 6 | 7 | - [ ] Add a file to the `news` directory ([using the template](https://github.com/conda/conda-libmamba-solver/blob/main/news/TEMPLATE)) for the next release's release notes? 8 | 14 | - [ ] Add / update necessary tests? 15 | - [ ] Add / update outdated documentation? 16 | -------------------------------------------------------------------------------- /tests/data/conda_build_recipes/multioutput/meta.yaml: -------------------------------------------------------------------------------- 1 | {% set name = "ocp" %} 2 | {% set version = "7.5.2beta" %} 3 | {% set occt_version = "=7.5.2" %} 4 | 5 | package: 6 | name: {{ name }}-split 7 | version: {{ version }} 8 | 9 | build: 10 | number: 0 11 | 12 | outputs: 13 | - name: ocp-devel 14 | build: 15 | script: 16 | - echo "BUILDING IMPL" >> $PREFIX/stackvana-core-impl # [unix] 17 | - echo "BUILDING IMPL" >> %PREFIX%/stackvana-core-impl # [win] 18 | 19 | - name: ocp 20 | build: 21 | script: 22 | - echo "BUILDING IMPL" >> $PREFIX/ocp # [unix] 23 | - echo "BUILDING IMPL" >> %PREFIX%/ocp # [win] 24 | requirements: 25 | host: 26 | - "{{ pin_subpackage('ocp-devel', exact=True) }}" 27 | run: 28 | - "{{ pin_subpackage('ocp-devel', exact=True) }}" 29 | -------------------------------------------------------------------------------- /docs/_templates/navbar_center.html: -------------------------------------------------------------------------------- 1 | 15 | -------------------------------------------------------------------------------- /docs/dev/workflows.md: -------------------------------------------------------------------------------- 1 | # Typical development workflows 2 | 3 | ```{note} 4 | Check out ["How to set up your development environment"](setup.md) if you haven't yet! 5 | ``` 6 | 7 | ## Testing 8 | 9 | The solver is a critical part of `conda` as a tool. 10 | In addition to unit tests for `conda_libmamba_solver`, 11 | our CI also runs the full `conda/conda` integration suite. 12 | 13 | ### conda-libmamba-solver tests 14 | 15 | From the properly mounted devcontainer (see ["Development environment setup"](setup.md)): 16 | 17 | ```bash 18 | $ cd /workspaces/conda-libmamba-solver 19 | $ pytest 20 | ``` 21 | 22 | Or just use the PyTest integrations in VS Code (flask icon). 23 | 24 | ### Upstream tests 25 | 26 | From the properly mounted devcontainer (see ["Development environment setup"](setup.md)): 27 | 28 | ```bash 29 | $ cd /workspaces/conda 30 | $ CONDA_SOLVER=libmamba pytest 31 | ``` 32 | -------------------------------------------------------------------------------- /docs/dev/shards_python.mmd: -------------------------------------------------------------------------------- 1 | flowchart LR 2 | python --> bzip2 3 | python --> libffi 4 | python --> libzlib 5 | python --> ncurses 6 | python --> openssl 7 | python --> readline 8 | python --> sqlite 9 | python --> tk 10 | python --> tzdata 11 | python --> xz 12 | python --> libsqlite 13 | python --> libcxx 14 | python --> zlib 15 | python --> __osx 16 | python --> liblzma 17 | python --> libexpat 18 | python --> libmpdec 19 | python --> python_abi 20 | python --> zstd 21 | python --> _python_rc 22 | python --> expat 23 | python --> libiconv 24 | libsqlite --> icu 25 | openssl --> ca-certificates 26 | python_abi --> pypy3.6 27 | python_abi --> pypy3.7 28 | python_abi --> pypy3.8 29 | python_abi --> pypy3.9 30 | xz --> liblzma-devel 31 | xz --> xz-gpl-tools 32 | xz --> xz-tools 33 | zstd --> lz4-c 34 | ca-certificates --> __win 35 | ca-certificates --> __unix 36 | -------------------------------------------------------------------------------- /tests/data/conda_build_recipes/jedi/meta.yaml: -------------------------------------------------------------------------------- 1 | {% set name = "jedi" %} 2 | {% set version = "0.19.2" %} 3 | 4 | package: 5 | name: {{ name }} 6 | version: {{ version }} 7 | 8 | source: 9 | url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz 10 | sha256: 4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0 11 | 12 | build: 13 | number: 0 14 | script: {{ PYTHON }} -m pip install . --no-deps --no-build-isolation --ignore-installed --no-cache-dir -vv 15 | # the minimal version of python supported is 3.6 16 | skip: true # [py<36] 17 | 18 | requirements: 19 | host: 20 | - python 21 | - pip 22 | - setuptools 23 | - wheel 24 | run: 25 | - python 26 | - parso >=0.8.4,<0.9.0 27 | 28 | test: 29 | imports: 30 | - jedi 31 | - jedi.api 32 | - jedi.common 33 | - jedi.inference 34 | - jedi.inference.compiled 35 | - jedi.inference.compiled.subprocess 36 | - jedi.inference.gradual 37 | - jedi.inference.value 38 | - jedi.plugins 39 | -------------------------------------------------------------------------------- /tests/run_in_profiler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (C) 2022 Anaconda, Inc 3 | # Copyright (C) 2023 conda 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | """ 6 | To be run with python -m scalene 7 | """ 8 | 9 | import logging 10 | import os 11 | import pathlib 12 | 13 | from conda.base.context import reset_context 14 | 15 | import tests.test_shards_subset 16 | from conda_libmamba_solver import shards, shards_cache, shards_subset 17 | 18 | os.environ["CONDA_TOKEN"] = "" 19 | os.environ["CONDA_PLUGINS_USE_SHARDED_REPODATA"] = "1" 20 | os.environ["CONDA_REPODATA_THREADS"] = ( 21 | "10" # shave a half second off our time by avoiding CondaSession() creation 22 | ) 23 | reset_context() 24 | pathlib.Path("/tmp/shards").mkdir(exist_ok=True) 25 | 26 | tmp_path = pathlib.Path("/tmp/shards") 27 | 28 | logging.basicConfig(level=logging.INFO) 29 | for module in (shards, shards_cache, shards_subset): 30 | module.log.setLevel(logging.DEBUG) 31 | 32 | tests.test_shards_subset.test_build_repodata_subset_pipelined(None, tmp_path) 33 | -------------------------------------------------------------------------------- /tests/data/mamba_repo/channeldata.json: -------------------------------------------------------------------------------- 1 | { 2 | "channeldata_version": 1, 3 | "packages": { 4 | "test-package": { 5 | "activate.d": false, 6 | "binary_prefix": false, 7 | "deactivate.d": false, 8 | "description": null, 9 | "dev_url": null, 10 | "doc_source_url": null, 11 | "doc_url": null, 12 | "home": "https://github.com/mamba-org/mamba", 13 | "icon_hash": null, 14 | "icon_url": null, 15 | "identifiers": null, 16 | "keywords": null, 17 | "license": "BSD", 18 | "post_link": false, 19 | "pre_link": false, 20 | "pre_unlink": false, 21 | "recipe_origin": null, 22 | "run_exports": {}, 23 | "source_git_url": null, 24 | "source_url": null, 25 | "subdirs": [ 26 | "noarch" 27 | ], 28 | "summary": "I am just a test package!", 29 | "tags": null, 30 | "text_prefix": false, 31 | "timestamp": 1613117294, 32 | "version": "0.1" 33 | } 34 | }, 35 | "subdirs": [ 36 | "noarch" 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /rever.xsh: -------------------------------------------------------------------------------- 1 | # edit this in https://github.com/conda/infrastructure 2 | 3 | $ACTIVITIES = ["authors", "changelog"] 4 | 5 | # Basic settings 6 | $PROJECT = $GITHUB_REPO = $(basename $(git remote get-url origin)).split('.')[0].strip() 7 | $GITHUB_ORG = "conda" 8 | 9 | # Authors settings 10 | $AUTHORS_FILENAME = "AUTHORS.md" 11 | $AUTHORS_SORTBY = "alpha" 12 | 13 | # Changelog settings 14 | $CHANGELOG_FILENAME = "CHANGELOG.md" 15 | $CHANGELOG_PATTERN = r"\[//\]: # \(current developments\)" 16 | $CHANGELOG_HEADER = """[//]: # (current developments) 17 | 18 | ## $VERSION ($RELEASE_DATE) 19 | 20 | """ 21 | $CHANGELOG_CATEGORIES = [ 22 | "Enhancements", 23 | "Bug fixes", 24 | "Deprecations", 25 | "Docs", 26 | "Other", 27 | ] 28 | $CHANGELOG_CATEGORY_TITLE_FORMAT = "### {category}\n\n" 29 | $CHANGELOG_AUTHORS_TITLE = "Contributors" 30 | $CHANGELOG_AUTHORS_FORMAT = "* @{github}\n" 31 | 32 | try: 33 | # allow repository to customize synchronized-from-infa rever config 34 | from rever_overrides import * 35 | except ImportError: 36 | pass 37 | -------------------------------------------------------------------------------- /dev/scripts/example.xonshrc: -------------------------------------------------------------------------------- 1 | 2 | 3 | # >>> conda initialize >>> 4 | # !! Contents within this block are managed by 'conda init' !! 5 | if !(test -f "/usr/share/miniconda/envs/test/bin/conda"): 6 | import sys as _sys 7 | from types import ModuleType as _ModuleType 8 | _mod = _ModuleType("xontrib.conda", 9 | "Autogenerated from $(/usr/share/miniconda/envs/test/bin/conda shell.xonsh hook)") 10 | __xonsh__.execer.exec($("/usr/share/miniconda/envs/test/bin/conda" "shell.xonsh" "hook"), 11 | glbs=_mod.__dict__, 12 | filename="$(/usr/share/miniconda/envs/test/bin/conda shell.xonsh hook)") 13 | _sys.modules["xontrib.conda"] = _mod 14 | del _sys, _mod, _ModuleType 15 | # <<< conda initialize <<< 16 | 17 | 18 | # ---------------------------------------------------------------------------- 19 | # Conda Setup Action: Basic configuration 20 | set -eo pipefail 21 | # Conda Setup Action: Custom activation 22 | conda activate "test" 23 | # ---------------------------------------------------------------------------- 24 | -------------------------------------------------------------------------------- /tests/test_user_agent.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | Ensure the right User-Agent headers are set by `conda.context`. 6 | These will be sent to the server on each request. 7 | """ 8 | 9 | import json 10 | import os 11 | from importlib.metadata import version 12 | from subprocess import check_output 13 | 14 | import pytest 15 | 16 | 17 | @pytest.mark.parametrize("solver", ("classic", "libmamba")) 18 | def test_user_agent_conda_info(solver): 19 | env = os.environ.copy() 20 | env["CONDA_SOLVER"] = solver 21 | out = check_output(["conda", "info", "--json"], env=env) 22 | info = json.loads(out) 23 | assert "conda/" in info["user_agent"] 24 | if solver == "classic": 25 | assert "solver/" not in info["user_agent"] 26 | elif solver == "libmamba": 27 | assert "solver/libmamba" in info["user_agent"] 28 | assert f"conda-libmamba-solver/{version('conda-libmamba-solver')}" in info["user_agent"] 29 | assert f"libmambapy/{version('libmambapy')}" in info["user_agent"] 30 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # conda-libmamba-solver 2 | 3 | Welcome to the conda-libmamba-solver documentation! The conda-libmamba-solver 4 | was written to make conda faster and is now the default solver. On this site, 5 | you will find information about how to configure and use this solver. 6 | 7 | ## Learn 8 | 9 | ::::{grid} 2 10 | 11 | :::{grid-item-card} Getting started 12 | :link: user-guide/index 13 | :link-type: doc 14 | New to conda-libmamba-solver? Start here to learn the essentials 15 | ::: 16 | 17 | :::{grid-item-card} Configuration 18 | :link: user-guide/configuration 19 | :link-type: doc 20 | Learn about all available configuration options 21 | 22 | :::: 23 | 24 | ## Development 25 | 26 | ::::{grid} 2 27 | 28 | :::{grid-item-card} Development environment 29 | :link: dev/setup 30 | :link-type: doc 31 | Learn how to set up your development environment 32 | ::: 33 | 34 | :::{grid-item-card} Changelog 35 | :link: dev/changelog 36 | :link-type: doc 37 | Recent changes and udpates to the project 38 | :::: 39 | 40 | 41 | ```{seealso} 42 | Found a bug? [File an issue here](https://github.com/conda/conda-libmamba-solver/issues/new/choose) 43 | ``` 44 | 45 | ```{toctree} 46 | :hidden: 47 | 48 | user-guide/index 49 | dev/index 50 | ``` 51 | -------------------------------------------------------------------------------- /.github/workflows/cla.yml: -------------------------------------------------------------------------------- 1 | name: CLA 2 | 3 | on: 4 | issue_comment: 5 | types: 6 | - created 7 | pull_request_target: 8 | 9 | jobs: 10 | check: 11 | if: >- 12 | !github.event.repository.fork 13 | && ( 14 | github.event.issue.pull_request 15 | && github.event.comment.body == '@conda-bot check' 16 | || github.event_name == 'pull_request_target' 17 | ) 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Check CLA 21 | uses: conda/actions/check-cla@f05161c6e6e37a49b17c8e0b436197b53830318a # v25.9.2 22 | with: 23 | # [required] 24 | # A token with ability to comment, label, and modify the commit status 25 | # (`pull_request: write` and `statuses: write` for fine-grained PAT; `repo` for classic PAT) 26 | # (default: secrets.GITHUB_TOKEN) 27 | token: ${{ secrets.CLA_ACTION_TOKEN }} 28 | # [required] 29 | # Label to apply to contributor's PR once CLA is signed 30 | label: cla-signed 31 | 32 | # [required] 33 | # Token for opening signee PR in the provided `cla_repo` 34 | # (`pull_request: write` for fine-grained PAT; `repo` and `workflow` for classic PAT) 35 | cla_token: ${{ secrets.CLA_FORK_TOKEN }} 36 | -------------------------------------------------------------------------------- /dev/scripts/fix_xonshrc.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | Fix .xonshrc to work around an indentation bug related to 6 | https://github.com/conda-incubator/setup-miniconda/blame/f77e237980e9445f93e2033f0c4a695a697a967a/src/conda.ts#L387-L397 7 | and possibly "conda init". 8 | """ 9 | 10 | import re 11 | from pathlib import Path 12 | 13 | # Looking for Conda Setup including odd indentation. 14 | PATTERN = re.compile(""" # ---------------------------------------------------------------------------- 15 | # Conda Setup Action: Basic configuration""") 16 | 17 | 18 | def fix_xonshrc(xonshrc): 19 | if not xonshrc.exists(): 20 | print("No .xonshrc") 21 | return 22 | 23 | split = PATTERN.split(xonshrc.read_text()) 24 | if len(split) != 2: 25 | print("Broken .xonshrc not found") 26 | return # already fixed or not as expected 27 | 28 | pattern_lines = PATTERN.pattern.splitlines() 29 | pattern_lines[1:1] = ["if True:"] 30 | 31 | split[1:1] = pattern_lines 32 | 33 | fixed = "\n".join(split) 34 | xonshrc.write_text(fixed) 35 | 36 | print(".xonshrc is now", fixed) 37 | 38 | 39 | if __name__ == "__main__": 40 | fix_xonshrc(Path.home() / ".xonshrc") 41 | -------------------------------------------------------------------------------- /.github/template-files/templates/issue_template_details.yml: -------------------------------------------------------------------------------- 1 | - type: textarea 2 | id: info 3 | attributes: 4 | label: Conda Info 5 | description: | 6 | Let's collect some basic information about your conda install. 7 | 8 | Please run the following command from your command line and paste the output below. 9 | 10 | ```bash 11 | conda info 12 | ``` 13 | render: shell 14 | - type: textarea 15 | id: config 16 | attributes: 17 | label: Conda Config 18 | description: | 19 | Let's collect any customizations you may have for your conda install. 20 | 21 | Please run the following command from your command line and paste the output below. 22 | 23 | ```bash 24 | conda config --show-sources 25 | ``` 26 | render: shell 27 | - type: textarea 28 | id: list 29 | attributes: 30 | label: Conda list 31 | description: | 32 | The packages installed into your environment can offer clues as to the problem you are facing. 33 | 34 | Please activate the environment within which you are encountering this bug, run the following command from your command line, and paste the output below. 35 | 36 | ```bash 37 | conda list --show-channel-urls 38 | ``` 39 | render: shell 40 | -------------------------------------------------------------------------------- /conda_libmamba_solver/plugin.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | Entry points for the conda plugin system 6 | """ 7 | 8 | from conda.common.configuration import PrimitiveParameter 9 | from conda.plugins import hookimpl 10 | from conda.plugins.types import CondaSetting, CondaSolver, CondaSubcommand 11 | 12 | from .repoquery import configure_parser, repoquery 13 | from .solver import LibMambaSolver 14 | 15 | 16 | @hookimpl 17 | def conda_solvers(): 18 | """ 19 | The conda plugin hook implementation to load the solver into conda. 20 | """ 21 | yield CondaSolver( 22 | name="libmamba", 23 | backend=LibMambaSolver, 24 | ) 25 | 26 | 27 | @hookimpl 28 | def conda_subcommands(): 29 | yield CondaSubcommand( 30 | name="repoquery", 31 | summary="Advanced search for repodata.", 32 | action=repoquery, 33 | configure_parser=configure_parser, 34 | ) 35 | 36 | 37 | @hookimpl 38 | def conda_settings(): 39 | """ 40 | Define all settings specific to the conda-libmamba-solver plugin. 41 | """ 42 | yield CondaSetting( 43 | name="use_sharded_repodata", 44 | description="Enable use of sharded repodata when available.", 45 | parameter=PrimitiveParameter(False, element_type=bool), 46 | ) 47 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | import sys 5 | from pathlib import Path 6 | from subprocess import CompletedProcess, run 7 | 8 | from libmambapy.specs import PackageInfo 9 | from ruamel.yaml import YAML 10 | 11 | python_site_packages_path_support = getattr(PackageInfo, "python_site_packages_path", False) 12 | 13 | 14 | def conda_subprocess(*args, explain=False, capture_output=True, **kwargs) -> CompletedProcess: 15 | cmd = [sys.executable, "-m", "conda", *[str(a) for a in args]] 16 | check = kwargs.pop("check", True) 17 | if explain: 18 | print("+", " ".join(cmd)) 19 | p = run( 20 | cmd, 21 | capture_output=capture_output, 22 | text=kwargs.pop("text", capture_output), 23 | check=False, 24 | **kwargs, 25 | ) 26 | if capture_output and (explain or p.returncode): 27 | print(p.stdout) 28 | print(p.stderr, file=sys.stderr) 29 | if check: 30 | p.check_returncode() 31 | return p 32 | 33 | 34 | def write_env_config(prefix, force=False, **kwargs): 35 | condarc = Path(prefix) / ".condarc" 36 | if condarc.is_file() and not force: 37 | raise RuntimeError(f"File {condarc} already exists. Use force=True to overwrite.") 38 | yaml = YAML(typ="full", pure=True) 39 | with open(condarc, "w") as f: 40 | yaml.dump(kwargs, f) 41 | -------------------------------------------------------------------------------- /.github/workflows/issues.yml: -------------------------------------------------------------------------------- 1 | name: Automate Issues 2 | 3 | on: 4 | # NOTE: github.event is issue_comment payload: 5 | # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#issue_comment 6 | issue_comment: 7 | types: [created] 8 | 9 | env: 10 | FEEDBACK_LBL: pending::feedback 11 | SUPPORT_LBL: pending::support 12 | 13 | jobs: 14 | # NOTE: will update label if anyone responds, not just the author/reporter 15 | # TODO: create conda-issue-sorting team and modify this to toggle label based on 16 | # whether a non-issue-sorting engineer commented 17 | pending_support: 18 | # if [pending::feedback] and anyone responds 19 | if: >- 20 | !github.event.repository.fork 21 | && !github.event.issue.pull_request 22 | && contains(github.event.issue.labels.*.name, 'pending::feedback') 23 | runs-on: ubuntu-latest 24 | steps: 25 | # remove [pending::feedback] 26 | - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1.3.0 27 | with: 28 | labels: ${{ env.FEEDBACK_LBL }} 29 | github_token: ${{ secrets.PROJECT_TOKEN }} 30 | # add [pending::support], if still open 31 | - uses: actions-ecosystem/action-add-labels@18f1af5e3544586314bbe15c0273249c770b2daf # v1.1.3 32 | if: github.event.issue.state == 'open' 33 | with: 34 | labels: ${{ env.SUPPORT_LBL }} 35 | github_token: ${{ secrets.PROJECT_TOKEN }} 36 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | # This file was autogenerated by rever: https://regro.github.io/rever-docs/ 2 | # This prevent git from showing duplicates with various logging commands. 3 | # See the git documentation for more details. The syntax is: 4 | # 5 | # good-name bad-name 6 | # 7 | # You can skip bad-name if it is the same as good-name and is unique in the repo. 8 | # 9 | # This file is up-to-date if the command git log --format="%aN <%aE>" | sort -u 10 | # gives no duplicates. 11 | 12 | Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> 13 | Albert DeFusco 14 | Christopher Ostrouchov 15 | Daniel Holth 16 | Jaime Rodríguez-Guerra jaimergp 17 | Jannis Leidel 18 | John Kirkham 19 | Jonathan J. Helmus 20 | Julien Jerphanion 21 | Ken Odegard 22 | Kevin Markham 23 | Klaus Zimmermann 24 | Matthew R. Becker 25 | Stacy Noland <46572585+stacynoland@users.noreply.github.com> 26 | Thomas Lam <79589038+tl-hbk@users.noreply.github.com> 27 | Travis Hathaway 28 | conda-bot <18747875+conda-bot@users.noreply.github.com> Conda Bot <18747875+conda-bot@users.noreply.github.com> 29 | dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 30 | pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> 31 | -------------------------------------------------------------------------------- /conda_libmamba_solver/conda_build_exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # Copyright (C) 2023 conda 5 | # SPDX-License-Identifier: BSD-3-Clause 6 | """ 7 | This exception is only used in conda-build, so we can't import it directly. 8 | conda_build is not a dependency, but we only import this when conda-build is calling the 9 | solver, so it's fine to import it here. 10 | """ 11 | 12 | from __future__ import annotations 13 | 14 | from typing import TYPE_CHECKING 15 | 16 | if TYPE_CHECKING: 17 | from collections.abc import Iterable 18 | 19 | from conda.models.match_spec import MatchSpec 20 | 21 | from conda_build.exceptions import DependencyNeedsBuildingError 22 | 23 | 24 | class ExplainedDependencyNeedsBuildingError(DependencyNeedsBuildingError): 25 | """ 26 | We need to subclass this to add the explanation to the error message. 27 | We also add a couple of attributes to make it easier to set up. 28 | """ 29 | 30 | def __init__( 31 | self, 32 | matchspecs: Iterable[MatchSpec] | None = None, 33 | explanation: str | None = None, 34 | *args, 35 | **kwargs, 36 | ): 37 | super().__init__(*args, **kwargs) 38 | self.matchspecs = self.matchspecs or matchspecs or [] 39 | self.explanation = explanation 40 | 41 | def __str__(self) -> str: 42 | msg = self.message 43 | if not self.explanation: 44 | # print simple message in log.warning() calls 45 | return msg 46 | return "\n".join([msg, self.explanation]) 47 | -------------------------------------------------------------------------------- /docs/user-guide/performance.md: -------------------------------------------------------------------------------- 1 | # Performance tips and tricks 2 | 3 | `conda-libmamba-solver` is much faster than classic for [many reasons](./libmamba-vs-classic.md), 4 | but there are certain tricks you can use to make it even faster! 5 | 6 | These tips apply to both solvers: 7 | 8 | - **Explicit is better**. 9 | Instead of letting the solver do all the work, specify target versions for your packages. 10 | `conda install python=3.11 numpy` is way better than `conda install python numpy`. 11 | * Use `--strict-channel-priority`. 12 | Strict channel priority drastically reduces the solver search space when you are mixing channels. 13 | Make this decision permanent with `conda config --set channel_priority strict`. 14 | * Use `--update-specs`. 15 | For existing environments, do not attempt to freeze installed packages by default. 16 | 17 | ## For `conda-libmamba-solver` 18 | 19 | * Experimental: `CONDA_LIBMAMBA_SOLVER_MAX_ATTEMPTS=0`. 20 | Setting this environment variable will disable the retry loop, making it behave more like `micromamba`. 21 | 22 | ## For conda `classic` 23 | 24 | The above tips also apply to `classic`, but you can supplement them with: 25 | 26 | * `--repodata-fn=repodata.json` to skip using `current_repodata.json`. 27 | * `CONDA_UNSATISFIABLE_HINTS_CHECK_DEPTH=1` won't help solves get any faster, but failures will be reported more quickly. 28 | 29 | ## References 30 | 31 | - [Understanding and Improving Conda's performance](https://www.anaconda.com/blog/understanding-and-improving-condas-performance) 32 | - [How We Made Conda Faster in 4.7](https://www.anaconda.com/blog/how-we-made-conda-faster-4-7) 33 | -------------------------------------------------------------------------------- /tests/test_plugin.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | Ensure configuration plugin functions as expected 6 | """ 7 | 8 | import pytest 9 | from conda.auxlib.ish import dals 10 | from conda.base.context import context, reset_context 11 | 12 | 13 | @pytest.fixture(scope="function", autouse=True) 14 | def always_reset_context(): 15 | reset_context() 16 | 17 | 18 | def test_enabled_sharded_repodata(): 19 | """ 20 | Ensure that the setting `plugins.enable_sharded_repodata` exists and is set 21 | to the correct default value. 22 | """ 23 | assert not context.plugins.use_sharded_repodata # type: ignore 24 | 25 | 26 | def test_enabled_sharded_repodata_environment_variable(monkeypatch): 27 | """ 28 | Ensure that the sharded repodata flag can be controlled by its corresponding environment variable. 29 | """ 30 | monkeypatch.setenv("CONDA_PLUGINS_USE_SHARDED_REPODATA", "true") 31 | context.__init__() 32 | 33 | assert context.plugins.use_sharded_repodata # type: ignore 34 | 35 | 36 | def test_enabled_sharded_repodata_condarc(tmp_path): 37 | """ 38 | Ensure that the sharded repodata flag can be controlled by a condarc file. 39 | """ 40 | condarc_file = tmp_path / "conda.yml" 41 | with condarc_file.open("w") as f: 42 | condarc_yml = dals(""" 43 | plugins: 44 | use_sharded_repodata: true 45 | """) 46 | f.write(condarc_yml) 47 | 48 | context.__init__(search_path=(str(condarc_file),)) 49 | 50 | assert context.plugins.use_sharded_repodata # type: ignore 51 | -------------------------------------------------------------------------------- /tests/data/mamba_repo/LICENSE: -------------------------------------------------------------------------------- 1 | This directory was taken from mamba-org/mamba, commit bff16c2bdc4103ba74c23ab4fdbf58849a55981c, on Mar 17 2022 2 | 3 | --- 4 | 5 | Copyright 2019 QuantStack and the Mamba contributors. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 12 | 13 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 16 | -------------------------------------------------------------------------------- /tests/data/conda_build_recipes/LICENSE: -------------------------------------------------------------------------------- 1 | This directory was taken from mamba-org/boa, commit 3213180564e51b72a27efed5183d21f97f630692, on May 12 2022 2 | 3 | --- 4 | 5 | Copyright 2020 QuantStack and the Boa contributors. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 12 | 13 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 16 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2_documentation.yml: -------------------------------------------------------------------------------- 1 | # edit this in https://github.com/conda/infrastructure 2 | 3 | name: Documentation 4 | description: Create a documentation related issue. 5 | labels: 6 | - type::documentation 7 | body: 8 | - type: markdown 9 | attributes: 10 | value: | 11 | > [!NOTE] 12 | > Documentation requests that are incomplete or missing information may be closed as inactionable. 13 | 14 | Since there are already a lot of open issues, please also take a moment to search existing ones to see if your bug has already been reported. If you find something related, please upvote that issue and provide additional details as necessary. 15 | 16 | 💐 Thank you for helping to make conda better. We would be unable to improve conda without our community! 17 | - type: checkboxes 18 | id: checks 19 | attributes: 20 | label: Checklist 21 | description: Please confirm and check all of the following options. 22 | options: 23 | - label: I added a descriptive title 24 | required: true 25 | - label: I searched open reports and couldn't find a duplicate 26 | required: true 27 | - type: textarea 28 | id: what 29 | attributes: 30 | label: What happened? 31 | description: Mention here any typos, broken links, or missing, incomplete, or outdated information, etc. that you have noticed in the conda docs or CLI help. 32 | validations: 33 | required: true 34 | - type: textarea 35 | id: context 36 | attributes: 37 | label: Additional Context 38 | description: Include any additional information (or screenshots) that you think would be valuable. 39 | -------------------------------------------------------------------------------- /tests/test_downstream.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | import os 5 | from pathlib import Path 6 | from subprocess import CalledProcessError, check_call 7 | 8 | import pytest 9 | from conda.base.context import context 10 | 11 | DATA = Path(__file__).parent / "data" 12 | 13 | 14 | @pytest.mark.parametrize( 15 | "recipe", 16 | [ 17 | pytest.param(x, id=x.name) 18 | for x in sorted((DATA / "conda_build_recipes").iterdir()) 19 | if (x / "meta.yaml").is_file() 20 | ], 21 | ) 22 | def test_build_recipe(recipe): 23 | """ 24 | Adapted from 25 | https://github.com/mamba-org/boa/blob/3213180564/tests/test_mambabuild.py#L6 26 | 27 | See /tests/data/conda_build_recipes/LICENSE for more details 28 | """ 29 | expected_fail_recipes = ["baddeps"] 30 | env = os.environ.copy() 31 | env["CONDA_SOLVER"] = "libmamba" 32 | recipe_name = Path(recipe).name 33 | if recipe_name in expected_fail_recipes: 34 | with pytest.raises(CalledProcessError): 35 | check_call(["conda-build", recipe], env=env) 36 | else: 37 | check_call(["conda-build", recipe], env=env) 38 | 39 | 40 | def test_conda_lock(tmp_path): 41 | conda_exe_path = "Scripts/conda.exe" if os.name == "nt" else "bin/conda" 42 | check_call( 43 | [ 44 | "conda-lock", 45 | "lock", 46 | "--platform", 47 | context.subdir, 48 | "--file", 49 | DATA / "lock_this_env.yml", 50 | "--conda", 51 | Path(context.conda_prefix) / conda_exe_path, 52 | ], 53 | cwd=tmp_path, 54 | ) 55 | -------------------------------------------------------------------------------- /.devcontainer/post_create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script assumes we are running in a Miniconda container where: 4 | # - /opt/conda is the Miniconda or Miniforge installation directory 5 | # - https://github.com/conda/conda is mounted at /workspaces/conda 6 | # - https://github.com/conda/conda-libmamba-solver is mounted at 7 | # /workspaces/conda-libmamba-solver 8 | # - https://github.com/mamba-org/mamba is (optionally) mounted at 9 | # /workspaces/mamba 10 | 11 | set -euo pipefail 12 | 13 | HERE=$(dirname $0) 14 | BASE_CONDA=${BASE_CONDA:-/opt/conda} 15 | SRC_CONDA=${SRC_CONDA:-/workspaces/conda} 16 | SRC_CONDA_LIBMAMBA_SOLVER=${SRC_CONDA_LIBMAMBA_SOLVER:-/workspaces/conda-libmamba-solver} 17 | 18 | if which apt-get > /dev/null; then 19 | echo "Installing system dependencies" 20 | apt-get update 21 | DEBIAN_FRONTEND=noninteractive xargs -a "$HERE/apt-deps.txt" apt-get install -y 22 | fi 23 | 24 | 25 | if [ ! -f "$SRC_CONDA/pyproject.toml" ]; then 26 | echo "https://github.com/conda/conda not found! Please clone or mount to $SRC_CONDA" 27 | exit 1 28 | fi 29 | 30 | # Clear history to avoid unneeded conflicts 31 | echo "Clearing base history..." 32 | echo '' > "$BASE_CONDA/conda-meta/history" 33 | 34 | echo "Installing dev & test dependencies..." 35 | "$BASE_CONDA/bin/conda" install -n base --yes --quiet \ 36 | --file="$SRC_CONDA/tests/requirements.txt" \ 37 | --file="$SRC_CONDA/tests/requirements-ci.txt" \ 38 | --file="$SRC_CONDA/tests/requirements-Linux.txt" \ 39 | --file="$SRC_CONDA/tests/requirements-s3.txt" \ 40 | --file="$SRC_CONDA_LIBMAMBA_SOLVER/dev/requirements.txt" \ 41 | --file="$SRC_CONDA_LIBMAMBA_SOLVER/tests/requirements.txt"\ 42 | pre-commit 43 | -------------------------------------------------------------------------------- /recipe/meta.yaml: -------------------------------------------------------------------------------- 1 | package: 2 | name: "conda-libmamba-solver" 3 | version: "{{ GIT_DESCRIBE_TAG }}.{{ GIT_BUILD_STR }}" 4 | 5 | source: 6 | # git_url is nice in that it won't capture devenv stuff. However, it only 7 | # captures committed code, so pay attention. 8 | git_url: ../ 9 | folder: src/ 10 | 11 | build: 12 | noarch: python 13 | number: 0 14 | script: "{{ PYTHON }} -m pip install src/ -vv --no-deps --no-build-isolation" 15 | 16 | requirements: 17 | host: 18 | - python >=3.10 19 | - pip 20 | - hatchling 21 | - hatch-vcs 22 | run: 23 | - python >=3.10 24 | - conda >=25.9 25 | - libmambapy >=2.0.0 26 | - boltons >=23.0.0 27 | - msgpack-python >=1.1.1 28 | - requests >=2.28.0,<3 29 | - zstandard >=0.15 30 | 31 | test: 32 | imports: 33 | - conda_libmamba_solver 34 | commands: 35 | - conda create -n test --dry-run scipy --solver=libmamba 36 | - >- 37 | python -c 38 | "import conda_libmamba_solver as cls; 39 | from importlib.metadata import version; 40 | assert '{{ PKG_VERSION }}' == cls.__version__ == version('conda_libmamba_solver'), 41 | '{{ PKG_VERSION }}' + f', {cls.__version__}, ' + version('conda_libmamba_solver') 42 | " || true 43 | # we accept errors here because GIT_DESCRIBE_TAG has some accuracy issues. 44 | # Shouldn't happen in production recipes; aka do not add '|| true' in the feedstock 45 | 46 | about: 47 | home: https://github.com/conda/conda-libmamba-solver 48 | license: BSD-3-Clause 49 | license_family: BSD 50 | license_file: src/LICENSE 51 | summary: 'The fast mamba solver, now in conda!' 52 | 53 | extra: 54 | recipe-maintainers: 55 | - jaimergp 56 | - jezdez 57 | - wolfv 58 | -------------------------------------------------------------------------------- /docs/user-guide/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | ## Basic options 4 | 5 | conda-libmamba-solver is a _solver plugin_ and can be configured with the same settings as the classic solver. This is usually done via the `conda config` subcommand. Read the [conda configuration docs](https://docs.conda.io/projects/conda/en/stable/configuration.html) and check for the general solver options there. 6 | 7 | ## Advanced options 8 | 9 | Additionally, conda-libmamba-solver can be further configured via special environment variables. 10 | We do not recomment using these options in production environments. Their behavior might change in the future, or they can be entirely removed without prior notice. 11 | 12 | * `CONDA_LIBMAMBA_SOLVER_MAX_ATTEMPTS`: Maximum number of attempts to find a solution. By default, this is set to the number of installed packages in the environment. In commands that involve a large number of changes in a large environment, it can take a bit to relax the constraints enough to find a solution. This option can be used to reduce the number of attempts and "give up" earlier. 13 | * `CONDA_LIBMAMBA_SOLVER_DEBUG_LIBSOLV`: Enable verbose logging from `libsolv`. Only has an effect if combined with `-vvv` in the CLI. Note that this will incur a big performance overhead. Only use when debugging solver issues. 14 | 15 | ## Sharded Repodata 16 | 17 | Sharded repodata is a feature that enables faster repodata fetching. It is currently 18 | in beta testing. To enable this feature, run the following command: 19 | 20 | ```shell 21 | conda config --set plugins.use_sharded_repodata true 22 | ``` 23 | 24 | Alternatively, you can enable this via an environment variable: 25 | 26 | ```shell 27 | export CONDA_PLUGINS_USE_SHARDED_REPODATA=true 28 | ``` 29 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 3 | 4 | ### Description 5 | 6 | 10 | 11 | ### Checklist - did you ... 12 | 13 | 16 | 17 | - [ ] Add a file to the `news` directory ([using the template](https://github.com/conda/conda-libmamba-solver/blob/main/news/TEMPLATE)) for the next release's release notes? 18 | 24 | - [ ] Add / update necessary tests? 25 | - [ ] Add / update outdated documentation? 26 | 27 | 28 | 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2022, Anaconda, Inc. 4 | Copyright (c) 2023, conda 5 | All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in the 14 | documentation and/or other materials provided with the distribution. 15 | * Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products 17 | derived from this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 23 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 26 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | --- 31 | 32 | This work also includes code borrowed from mamba.utils v0.19, licensed as BSD 3-Clause 33 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Docs 2 | 3 | on: 4 | # Runs on pushes targeting the default branch 5 | push: 6 | branches: ["main"] 7 | pull_request: 8 | paths: 9 | - "docs/**" 10 | # Allows you to run this workflow manually from the Actions tab 11 | workflow_dispatch: 12 | 13 | # Allow one concurrent deployment 14 | concurrency: 15 | group: "pages" 16 | cancel-in-progress: true 17 | 18 | jobs: 19 | docs: 20 | runs-on: ubuntu-latest 21 | defaults: 22 | run: 23 | shell: bash -el {0} 24 | steps: 25 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 26 | 27 | - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0 28 | with: 29 | activate-environment: conda-libmamba-solver 30 | environment-file: docs/environment.yml 31 | python-version: '3.11' 32 | auto-activate-base: false 33 | 34 | - name: Build Documentation 35 | run: | 36 | cd docs 37 | make dirhtml 38 | 39 | - name: Upload artifact 40 | uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 41 | with: 42 | # Upload entire repository 43 | path: 'docs/_build/dirhtml' 44 | 45 | pages: 46 | runs-on: ubuntu-latest 47 | if: github.ref == 'refs/heads/main' 48 | needs: [docs] 49 | 50 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 51 | permissions: 52 | contents: read 53 | pages: write 54 | id-token: write 55 | 56 | environment: 57 | name: github-pages 58 | url: ${{ steps.deployment.outputs.page_url }} 59 | 60 | steps: 61 | - name: Deploy to GitHub Pages 62 | id: deployment 63 | uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 64 | -------------------------------------------------------------------------------- /conda_libmamba_solver/shards_typing.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | TypedDict declarations for shards. 6 | 7 | These are helpful for auto-complete, but do not validate at runtime and are not 8 | normative. They are intentionally not shared with another project (conda) to 9 | reduce coupling. 10 | """ 11 | 12 | from __future__ import annotations 13 | 14 | from typing import TYPE_CHECKING, TypedDict 15 | 16 | if TYPE_CHECKING: 17 | from typing import NotRequired 18 | 19 | 20 | class PackageRecordDict(TypedDict): 21 | """ 22 | Basic package attributes that this module cares about. 23 | """ 24 | 25 | name: str 26 | version: str 27 | build: str 28 | build_number: int 29 | sha256: NotRequired[str | bytes] 30 | md5: NotRequired[str | bytes] 31 | depends: list[str] 32 | constrains: NotRequired[list[str]] 33 | noarch: NotRequired[str] 34 | 35 | 36 | # in this style because "packages.conda" is not a Python identifier 37 | ShardDict = TypedDict( 38 | "ShardDict", 39 | { 40 | "packages": dict[str, PackageRecordDict], 41 | "packages.conda": dict[str, PackageRecordDict], 42 | }, 43 | ) 44 | 45 | 46 | class RepodataInfoDict(TypedDict): # noqa: F811 47 | base_url: str # where packages are stored 48 | shards_base_url: str # where shards are stored 49 | subdir: str 50 | 51 | 52 | class RepodataDict(ShardDict): 53 | """ 54 | Packages plus info. 55 | """ 56 | 57 | info: RepodataInfoDict 58 | repodata_version: int 59 | 60 | 61 | class ShardsIndexDict(TypedDict): 62 | """ 63 | Shards index as deserialized from repodata_shards.msgpack.zst 64 | """ 65 | 66 | info: RepodataInfoDict 67 | version: int # TODO conda-index currently uses 'repodata_version' here 68 | shards: dict[str, bytes] 69 | -------------------------------------------------------------------------------- /.github/workflows/labels.yml: -------------------------------------------------------------------------------- 1 | name: Sync Labels 2 | 3 | on: 4 | # NOTE: github.event is workflow_dispatch payload: 5 | # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch 6 | workflow_dispatch: 7 | inputs: 8 | delete-unmapped: 9 | description: Delete labels not mapped in either global or local label configurations. 10 | default: false 11 | type: boolean 12 | 13 | dry-run: 14 | description: Run label synchronization workflow without making any changes. 15 | default: false 16 | type: boolean 17 | 18 | jobs: 19 | sync: 20 | if: '!github.event.repository.fork' 21 | runs-on: ubuntu-latest 22 | env: 23 | GLOBAL: https://raw.githubusercontent.com/conda/infra/main/.github/global.yml 24 | LOCAL: .github/labels.yml 25 | steps: 26 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 27 | 28 | - id: has_local 29 | uses: andstor/file-existence-action@076e0072799f4942c8bc574a82233e1e4d13e9d6 # v3.0.0 30 | with: 31 | files: ${{ env.LOCAL }} 32 | 33 | - name: Global Only 34 | uses: EndBug/label-sync@52074158190acb45f3077f9099fea818aa43f97a # v2.3.3 35 | if: steps.has_local.outputs.files_exists == 'false' 36 | with: 37 | config-file: ${{ env.GLOBAL }} 38 | delete-other-labels: ${{ inputs.delete-unmapped }} 39 | dry-run: ${{ inputs.dry-run }} 40 | 41 | - name: Global & Local 42 | uses: EndBug/label-sync@52074158190acb45f3077f9099fea818aa43f97a # v2.3.3 43 | if: steps.has_local.outputs.files_exists == 'true' 44 | with: 45 | config-file: | 46 | ${{ env.GLOBAL }} 47 | ${{ env.LOCAL }} 48 | delete-other-labels: ${{ inputs.delete-unmapped }} 49 | dry-run: ${{ inputs.dry-run }} 50 | -------------------------------------------------------------------------------- /.github/template-files/config.yml: -------------------------------------------------------------------------------- 1 | conda/governance: 2 | # [required] community files 3 | - CODE_OF_CONDUCT.md 4 | 5 | conda/conda: 6 | # issue template details for templates/issues/bug.yml template 7 | - .github/template-files/templates/issue_template_details.yml 8 | 9 | conda/infrastructure: 10 | # [required] general workflows 11 | - .github/workflows/cla.yml 12 | - .github/workflows/update.yml 13 | 14 | # [optional] to include repo in https://github.com/orgs/conda/projects/2 15 | - .github/workflows/issues.yml 16 | - .github/workflows/labels.yml 17 | - .github/workflows/project.yml 18 | 19 | # [optional] stale bot workflows 20 | - .github/workflows/stale.yml 21 | - .github/workflows/lock.yml 22 | 23 | # [optional] general processes for the conda org 24 | - src: templates/HOW_WE_USE_GITHUB.md 25 | dst: HOW_WE_USE_GITHUB.md 26 | 27 | # [optional] standard issue templates 28 | - src: templates/issues/bug.yml 29 | dst: .github/ISSUE_TEMPLATE/0_bug.yml 30 | 31 | - src: templates/issues/feature.yml 32 | dst: .github/ISSUE_TEMPLATE/1_feature.yml 33 | 34 | - src: templates/issues/documentation.yml 35 | dst: .github/ISSUE_TEMPLATE/2_documentation.yml 36 | 37 | - src: templates/issues/epic.yml 38 | dst: .github/ISSUE_TEMPLATE/epic.yml 39 | 40 | # [optional] standard PR template 41 | - src: templates/pull_requests/news_tests_docs.md 42 | dst: .github/template-files/templates/pull_request_template_details.md 43 | - src: templates/pull_requests/base.md 44 | dst: .github/PULL_REQUEST_TEMPLATE.md 45 | 46 | # [optional] rever release files 47 | - src: templates/releases/RELEASE.md 48 | dst: RELEASE.md 49 | with: 50 | canary_channel: https://anaconda.org/conda-canary 51 | placeholder: YY.MM.MICRO 52 | placeholder_x: YY.MM.x 53 | - src: templates/releases/rever.xsh 54 | dst: rever.xsh 55 | - src: templates/releases/TEMPLATE 56 | dst: news/TEMPLATE 57 | -------------------------------------------------------------------------------- /docs/environment.yml: -------------------------------------------------------------------------------- 1 | name: conda-libmamba-solver-docs 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - accessible-pygments=0.0.4 7 | - alabaster=0.7.12 8 | - babel=2.11.0 9 | - beautifulsoup4=4.11.1 10 | - brotlipy=0.7.0 11 | - bzip2=1.0.8 12 | - ca-certificates=2022.9.24 13 | - certifi=2022.9.24 14 | - cffi=1.15.1 15 | - charset-normalizer=2.1.1 16 | - colorama=0.4.6 17 | - conda-sphinx-theme==0.1.3 18 | - cryptography=38.0.3 19 | - docutils=0.19 20 | - idna=3.4 21 | - imagesize=1.4.1 22 | - importlib-metadata=5.1.0 23 | - jinja2=3.1.2 24 | - libffi=3.4.2 25 | - libsqlite=3.40.0 26 | - libzlib=1.2.13 27 | - linkify-it-py=2.0.0 28 | - markdown-it-py=3.0.0 29 | - markupsafe=2.1.1 30 | - mdit-py-plugins=0.4.0 31 | - mdurl=0.1.0 32 | - myst-parser=2.0.0 33 | - ncurses=6.3 34 | - openssl=3.0.7 35 | - packaging=21.3 36 | - pip=22.3.1 37 | - pycparser=2.21 38 | - pydata-sphinx-theme=0.14.3 39 | - pygments=2.16.1 40 | - pyopenssl=22.1.0 41 | - pyparsing=3.0.9 42 | - pysocks=1.7.1 43 | - python=3.11.0 44 | - python_abi=3.11 45 | - pytz=2022.6 46 | - pyyaml=6.0.1 47 | - readline=8.1.2 48 | - requests=2.28.1 49 | - setuptools=65.5.1 50 | - six=1.16.0 51 | - snowballstemmer=2.2.0 52 | - soupsieve=2.3.2.post1 53 | - sphinx=7.2.6 54 | - sphinx-copybutton=0.5.0 55 | - sphinx-design=0.5.0 56 | - sphinx-sitemap=2.2.1 57 | - sphinx-reredirects=0.1.2 58 | - sphinxcontrib-applehelp=1.0.2 59 | - sphinxcontrib-devhelp=1.0.2 60 | - sphinxcontrib-htmlhelp=2.0.0 61 | - sphinxcontrib-jsmath=1.0.1 62 | - sphinxcontrib-mermaid=0.7.1 63 | - sphinxcontrib-qthelp=1.0.3 64 | - sphinxcontrib-serializinghtml=1.1.9 65 | - tk=8.6.12 66 | - typing-extensions=4.4.0 67 | - typing_extensions=4.4.0 68 | - tzdata=2022f 69 | - uc-micro-py=1.0.1 70 | - urllib3=1.26.13 71 | - wheel=0.38.4 72 | - xz=5.2.6 73 | - yaml=0.2.5 74 | - zipp=3.10.0 75 | -------------------------------------------------------------------------------- /tests/data/conda_build_recipes/stackvana/meta.yaml: -------------------------------------------------------------------------------- 1 | {% set name = "stackvana-split" %} 2 | {% set version = "0.2021.43" %} 3 | {% set eups_product = "lsst_distrib" %} 4 | 5 | package: 6 | name: {{ name|lower }} 7 | version: {{ version }} 8 | 9 | build: 10 | number: 0 11 | 12 | outputs: 13 | - name: stackvana-core-impl 14 | version: {{ version }} 15 | build: 16 | script: 17 | - echo "BUILDING IMPL" >> $PREFIX/stackvana-core-impl # [unix] 18 | - echo "BUILDING IMPL" >> %PREFIX%/stackvana-core-impl # [win] 19 | test: 20 | commands: 21 | - echo OK 22 | - name: stackvana-core 23 | version: {{ version }} 24 | build: 25 | script: 26 | - echo "BUILDING CORE" >> $PREFIX/stackvana-core # [unix] 27 | - echo "BUILDING CORE" >> %PREFIX%/stackvana-core # [win] 28 | run_exports: 29 | - {{ pin_subpackage('stackvana-core-impl', exact=True) }} 30 | requirements: 31 | run: 32 | - {{ pin_subpackage('stackvana-core-impl', exact=True) }} 33 | test: 34 | commands: 35 | - echo OK 36 | - name: stackvana-{{ eups_product }} 37 | version: {{ version }} 38 | build: 39 | script: 40 | - echo "BUILDING {{ eups_product }}" >> $PREFIX/stackvana-{{ eups_product }} # [unix] 41 | - echo "BUILDING {{ eups_product }}" >> %PREFIX%/stackvana-{{ eups_product }} # [win] 42 | requirements: 43 | host: 44 | - stackvana-core =={{ version }} 45 | run: 46 | - stackvana-core =={{ version }} 47 | test: 48 | commands: 49 | - echo OK 50 | - name: stackvana 51 | version: {{ version }} 52 | build: 53 | script: 54 | - echo "BUILDING STACKVANA" >> $PREFIX/stackvana # [unix] 55 | - echo "BUILDING STACKVANA" >> %PREFIX%/stackvana # [win] 56 | requirements: 57 | - {{ pin_subpackage("stackvana-" ~ eups_product, max_pin="x.x.x") }} 58 | test: 59 | commands: 60 | - echo OK 61 | -------------------------------------------------------------------------------- /.devcontainer/defaults/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json 2 | { 3 | "name": "Miniconda (default-channel=defaults)", 4 | "image": "continuumio/miniconda3:latest", 5 | 6 | // Uncomment to force x64 instead of native (slower!) 7 | // "runArgs": ["--platform=linux/amd64"], 8 | 9 | // Features to add to the dev container. More info: https://containers.dev/features. 10 | // "features": {}, 11 | 12 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 13 | // "forwardPorts": [], 14 | "mounts": [ 15 | "source=${localWorkspaceFolder}/../conda,target=/workspaces/conda,type=bind,consistency=cached", 16 | "source=${localWorkspaceFolder}/../mamba,target=/workspaces/mamba,type=bind,consistency=cached" 17 | ], 18 | // Use 'postCreateCommand' to run commands after the container is created. 19 | "postCreateCommand": "bash /workspaces/conda-libmamba-solver/.devcontainer/post_create.sh", 20 | // Use 'postStartCommand' to run commands after the container is started. 21 | "postStartCommand": "bash /workspaces/conda-libmamba-solver/.devcontainer/post_start.sh", 22 | 23 | // Configure tool-specific properties. 24 | "customizations": { 25 | "vscode": { 26 | "settings": { 27 | "python.defaultInterpreterPath": "/opt/conda/bin/python", 28 | "python.testing.pytestArgs": [ 29 | "tests" 30 | ], 31 | "python.testing.unittestEnabled": false, 32 | "python.testing.pytestEnabled": true 33 | }, 34 | "extensions": [ 35 | "charliermarsh.ruff", 36 | "eamodio.gitlens", 37 | "ms-toolsai.jupyter", 38 | "be5invis.toml" 39 | ] 40 | } 41 | } 42 | 43 | // Adjust to connect as non-root instead. More info: https://aka.ms/dev-containers-non-root. 44 | // "remoteUser": "root", 45 | 46 | } 47 | -------------------------------------------------------------------------------- /.devcontainer/conda-forge/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the 2 | { 3 | "name": "Miniforge (default-channel=conda-forge)", 4 | "image": "condaforge/miniforge3:latest", 5 | 6 | // Uncomment to force x64 instead of native (slower!) 7 | // "runArgs": ["--platform=linux/amd64"], 8 | 9 | // Features to add to the dev container. More info: https://containers.dev/features. 10 | // "features": {}, 11 | 12 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 13 | // "forwardPorts": [], 14 | "mounts": [ 15 | "source=${localWorkspaceFolder}/../conda,target=/workspaces/conda,type=bind,consistency=cached", 16 | "source=${localWorkspaceFolder}/../mamba,target=/workspaces/mamba,type=bind,consistency=cached" 17 | ], 18 | // Use 'postCreateCommand' to run commands after the container is created. 19 | "postCreateCommand": "bash /workspaces/conda-libmamba-solver/.devcontainer/post_create.sh", 20 | // Use 'postStartCommand' to run commands after the container is started. 21 | "postStartCommand": "bash /workspaces/conda-libmamba-solver/.devcontainer/post_start.sh", 22 | 23 | // Configure tool-specific properties. 24 | "customizations": { 25 | "vscode": { 26 | "settings": { 27 | "python.defaultInterpreterPath": "/opt/conda/bin/python", 28 | "python.testing.pytestArgs": [ 29 | "tests" 30 | ], 31 | "python.testing.unittestEnabled": false, 32 | "python.testing.pytestEnabled": true 33 | }, 34 | "extensions": [ 35 | "charliermarsh.ruff", 36 | "eamodio.gitlens", 37 | "ms-toolsai.jupyter", 38 | "be5invis.toml" 39 | ] 40 | } 41 | } 42 | 43 | // Adjust to connect as non-root instead. More info: https://aka.ms/dev-containers-non-root. 44 | // "remoteUser": "root", 45 | 46 | } 47 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # disable autofixing PRs, commenting "pre-commit.ci autofix" on a pull request triggers a autofix 2 | ci: 3 | autofix_prs: false 4 | # generally speaking we ignore all vendored code as well as tests data 5 | exclude: | 6 | (?x)^( 7 | tests/data/.* | 8 | conda_libmamba_solver/mamba_utils\.py 9 | )$ 10 | repos: 11 | # generic verification and formatting 12 | - repo: https://github.com/pre-commit/pre-commit-hooks 13 | rev: v6.0.0 14 | hooks: 15 | # standard end of line/end of file cleanup 16 | - id: mixed-line-ending 17 | - id: end-of-file-fixer 18 | - id: trailing-whitespace 19 | # ensure syntaxes are valid 20 | - id: check-toml 21 | - id: check-yaml 22 | exclude: | 23 | (?x)^( 24 | (conda\.)?recipe/meta.yaml 25 | ) 26 | # catch git merge/rebase problems 27 | - id: check-merge-conflict 28 | - repo: https://github.com/asottile/blacken-docs 29 | rev: 1.20.0 30 | hooks: 31 | # auto format Python codes within docstrings 32 | - id: blacken-docs 33 | additional_dependencies: [black] 34 | - repo: https://github.com/astral-sh/ruff-pre-commit 35 | rev: v0.14.9 36 | hooks: 37 | # lint & attempt to correct failures (e.g. pyupgrade) 38 | - id: ruff-check 39 | args: [--fix] 40 | # compatible replacement for black 41 | - id: ruff-format 42 | - repo: meta 43 | # see https://pre-commit.com/#meta-hooks 44 | hooks: 45 | - id: check-hooks-apply 46 | - id: check-useless-excludes 47 | - repo: https://github.com/PyCQA/bandit 48 | rev: 1.9.2 49 | hooks: 50 | - id: bandit 51 | args: [--exit-zero] 52 | exclude: ^(tests/) 53 | - repo: https://github.com/Lucas-C/pre-commit-hooks 54 | rev: v1.5.5 55 | hooks: 56 | - id: insert-license 57 | files: \.py$ 58 | args: [--license-filepath, .github/disclaimer.txt, --no-extra-eol] 59 | exclude: ^(tests/repodata_time_machine.py|mamba_utils\.py|tests/channel_testing/helpers\.py|tests/channel_testing/reposerver\.py) # extend global exclude 60 | -------------------------------------------------------------------------------- /tests/http_test_server.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | Local test server based on http.server 6 | """ 7 | # From conda/tests; data/reposerver.py was refusing connections on Windows for shards tests. 8 | 9 | import contextlib 10 | import http.server 11 | import queue 12 | import socket 13 | import threading 14 | 15 | 16 | def run_test_server(directory: str) -> http.server.ThreadingHTTPServer: 17 | """ 18 | Run a test server on a random port. Inspect returned server to get port, 19 | shutdown etc. 20 | """ 21 | 22 | class DualStackServer(http.server.ThreadingHTTPServer): 23 | daemon_threads = False # These are per-request threads 24 | allow_reuse_address = True # Good for tests 25 | request_queue_size = 64 # Should be more than the number of test packages 26 | 27 | def server_bind(self): 28 | # suppress exception when protocol is IPv4 29 | with contextlib.suppress(Exception): 30 | self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) 31 | return super().server_bind() 32 | 33 | def finish_request(self, request, client_address): 34 | self.RequestHandlerClass(request, client_address, self, directory=directory) 35 | 36 | def start_server(queue): 37 | with DualStackServer(("127.0.0.1", 0), http.server.SimpleHTTPRequestHandler) as httpd: 38 | host, port = httpd.socket.getsockname()[:2] 39 | queue.put(httpd) 40 | url_host = f"[{host}]" if ":" in host else host 41 | print(f"Serving HTTP on {host} port {port} (http://{url_host}:{port}/) ...") 42 | try: 43 | httpd.serve_forever() 44 | except KeyboardInterrupt: 45 | print("\nKeyboard interrupt received, exiting.") 46 | 47 | started = queue.Queue() 48 | 49 | threading.Thread(target=start_server, args=(started,), daemon=True).start() 50 | 51 | return started.get(timeout=1) 52 | 53 | 54 | if __name__ == "__main__": 55 | server = run_test_server(directory=".") 56 | print(server) 57 | -------------------------------------------------------------------------------- /.github/workflows/lock.yml: -------------------------------------------------------------------------------- 1 | name: Lock 2 | 3 | on: 4 | # NOTE: github.event is workflow_dispatch payload: 5 | # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch 6 | workflow_dispatch: 7 | 8 | schedule: 9 | - cron: 0 6 * * * 10 | 11 | permissions: 12 | issues: write 13 | pull-requests: write 14 | 15 | jobs: 16 | lock: 17 | if: '!github.event.repository.fork' 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771 # v5.0.1 21 | with: 22 | # Number of days of inactivity before a closed issue is locked 23 | issue-inactive-days: 180 24 | # Do not lock issues created before a given timestamp, value must follow ISO 8601 25 | exclude-issue-created-before: '' 26 | # Do not lock issues with these labels, value must be a comma separated list of labels or '' 27 | exclude-any-issue-labels: '' 28 | # Labels to add before locking an issue, value must be a comma separated list of labels or '' 29 | add-issue-labels: locked 30 | # Reason for locking an issue, value must be one of resolved, off-topic, too heated, spam or '' 31 | issue-lock-reason: resolved 32 | # Number of days of inactivity before a closed pull request is locked 33 | pr-inactive-days: 365 34 | # Do not lock pull requests created before a given timestamp, value must follow ISO 8601 35 | exclude-pr-created-before: '' 36 | # Do not lock pull requests with these labels, value must be a comma separated list of labels or '' 37 | exclude-any-pr-labels: '' 38 | # Labels to add before locking a pull request, value must be a comma separated list of labels or '' 39 | add-pr-labels: locked 40 | # Reason for locking a pull request, value must be one of resolved, off-topic, too heated, spam or '' 41 | pr-lock-reason: resolved 42 | 43 | # Limit locking to issues, pull requests or discussions, value must be a comma separated list of issues, prs, discussions or '' 44 | process-only: issues, prs 45 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/1_feature.yml: -------------------------------------------------------------------------------- 1 | # edit this in https://github.com/conda/infrastructure 2 | 3 | name: Feature Request 4 | description: Create a feature request. 5 | labels: 6 | - type::feature 7 | body: 8 | - type: markdown 9 | attributes: 10 | value: | 11 | Because processing new feature requests is time-consuming, we would like to ask you to fill out the following form to the best of your ability and as completely as possible. 12 | 13 | > [!NOTE] 14 | > Feature requests that are incomplete or missing information may be closed as inactionable. 15 | 16 | Since there are already a lot of open issues, please also take a moment to search existing ones to see if your feature request has already been submitted. If you find something related, please upvote that issue and provide additional details as necessary. 17 | 18 | 💐 Thank you for helping to make `conda/conda-libmamba-solver` better. We would be unable to improve `conda/conda-libmamba-solver` without our community! 19 | - type: checkboxes 20 | id: checks 21 | attributes: 22 | label: Checklist 23 | description: Please confirm and check all of the following options. 24 | options: 25 | - label: I added a descriptive title 26 | required: true 27 | - label: I searched open requests and couldn't find a duplicate 28 | required: true 29 | - type: textarea 30 | id: idea 31 | attributes: 32 | label: What is the idea? 33 | description: Describe what the feature is and the desired state. 34 | validations: 35 | required: true 36 | - type: textarea 37 | id: why 38 | attributes: 39 | label: Why is this needed? 40 | description: Who would benefit from this feature? Why would this add value to them? What problem does this solve? 41 | - type: textarea 42 | id: what 43 | attributes: 44 | label: What should happen? 45 | description: What should be the user experience with the feature? Describe from a user perspective what they would do and see. 46 | - type: textarea 47 | id: context 48 | attributes: 49 | label: Additional Context 50 | description: Include any additional information that you think would be valuable. 51 | -------------------------------------------------------------------------------- /tests/test_experimental.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | Ensure experimental features work accordingly. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import sys 11 | from subprocess import run 12 | from typing import TYPE_CHECKING 13 | 14 | import pytest 15 | from conda.base.context import context, fresh_context 16 | from conda.exceptions import CondaEnvironmentError 17 | 18 | if TYPE_CHECKING: 19 | from conda.testing.fixtures import CondaCLIFixture 20 | from pytest import MonkeyPatch 21 | 22 | 23 | def print_and_check_output(*args, **kwargs): 24 | kwargs.setdefault("capture_output", True) 25 | kwargs.setdefault("universal_newlines", True) 26 | process = run(*args, **kwargs) 27 | print("stdout", process.stdout, "---", "stderr", process.stderr, sep="\n") 28 | process.check_returncode() 29 | return process 30 | 31 | 32 | @pytest.mark.xfail(reason="base protections not enabled anymore") 33 | def test_protection_for_base_env(monkeypatch: MonkeyPatch, conda_cli: CondaCLIFixture) -> None: 34 | with pytest.raises(CondaEnvironmentError), fresh_context(CONDA_SOLVER="libmamba"): 35 | monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False) 36 | conda_cli( 37 | "install", 38 | f"--prefix={context.root_prefix}", 39 | "--dry-run", 40 | "scipy", 41 | "--solver=libmamba", 42 | ) 43 | 44 | 45 | def test_cli_flag_in_help(): 46 | commands_with_flag = ( 47 | ["install"], 48 | ["update"], 49 | ["remove"], 50 | ["create"], 51 | ["env", "create"], 52 | ["env", "update"], 53 | ["env", "remove"], 54 | ) 55 | for command in commands_with_flag: 56 | process = print_and_check_output([sys.executable, "-m", "conda"] + command + ["--help"]) 57 | assert "--solver" in process.stdout 58 | 59 | commands_without_flag = ( 60 | ["config"], 61 | ["list"], 62 | ["info"], 63 | ["run"], 64 | ["env", "list"], 65 | ) 66 | for command in commands_without_flag: 67 | process = print_and_check_output([sys.executable, "-m", "conda"] + command + ["--help"]) 68 | assert "--solver" not in process.stdout 69 | -------------------------------------------------------------------------------- /.github/workflows/builds-review.yaml: -------------------------------------------------------------------------------- 1 | name: Review builds 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - labeled 7 | 8 | jobs: 9 | build: 10 | if: | 11 | github.event_name == 'pull_request' && 12 | github.event.label.name == 'build::review' 13 | strategy: 14 | matrix: 15 | include: 16 | - runner: ubuntu-latest 17 | subdir: noarch 18 | runs-on: ${{ matrix.runner }} 19 | steps: 20 | 21 | - name: Remove build label 22 | uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 23 | with: 24 | github-token: ${{ secrets.CANARY_ACTION_TOKEN }} 25 | script: | 26 | const { data: pullRequest } = await github.rest.pulls.get({ 27 | ...context.repo, 28 | pull_number: context.issue.number, 29 | }) 30 | const buildLabel = '${{ github.event.label.name }}' 31 | const labels = pullRequest.labels.map(label => label.name) 32 | const hasBuildLabel = labels.includes(buildLabel) 33 | if (hasBuildLabel) { 34 | await github.rest.issues.removeLabel({ 35 | ...context.repo, 36 | issue_number: context.issue.number, 37 | name: buildLabel, 38 | }) 39 | } 40 | # Clean checkout of specific git ref needed for package metadata version 41 | # which needs env vars GIT_DESCRIBE_TAG and GIT_BUILD_STR: 42 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 43 | with: 44 | ref: ${{ github.ref }} 45 | clean: true 46 | fetch-depth: 0 47 | 48 | - name: Create and upload review build 49 | uses: conda/actions/canary-release@f05161c6e6e37a49b17c8e0b436197b53830318a # v25.9.2 50 | with: 51 | package-name: conda-libmamba-solver 52 | subdir: ${{ matrix.subdir }} 53 | anaconda-org-channel: conda-canary 54 | anaconda-org-label: '${{ github.event.repository.name }}-pr-${{ github.event.number }}' 55 | anaconda-org-token: ${{ secrets.ANACONDA_ORG_CONDA_CANARY_TOKEN }} 56 | comment-headline: 'Review build status' 57 | comment-token: ${{ secrets.CANARY_ACTION_TOKEN }} 58 | conda-build-arguments: '--override-channels -c conda-forge/label/mamba_dev -c conda-forge' 59 | -------------------------------------------------------------------------------- /tests/data/mamba_repo/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | repo 4 | 67 | 68 | 69 |

repo

70 |

RSS Feed   channeldata.json

71 | noarch    72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 |
PackageLatest VersionDocDevLicensenoarch Summary
test-package0.1BSDX I am just a test package!
88 |
Updated: 2021-02-12 09:02:37 +0000 - Files: 1
89 | 90 | 91 | -------------------------------------------------------------------------------- /tests/test_repoquery.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | import json 5 | 6 | from conda.models.channel import Channel 7 | 8 | from conda_libmamba_solver.index import LibMambaIndexHelper 9 | 10 | from .utils import conda_subprocess, python_site_packages_path_support 11 | 12 | 13 | def test_repoquery(): 14 | p = conda_subprocess("repoquery", "--help") 15 | assert "whoneeds" in p.stdout 16 | assert "depends" in p.stdout 17 | assert "search" in p.stdout 18 | 19 | p = conda_subprocess("repoquery", "depends", "conda", "--json") 20 | print(p.stdout) 21 | data = json.loads(p.stdout) 22 | assert data["result"]["status"] == "OK" 23 | assert len(data["result"]["pkgs"]) > 0 24 | assert len([p for p in data["result"]["pkgs"] if p["name"] == "python"]) == 1 25 | 26 | 27 | def test_query_search(): 28 | index = LibMambaIndexHelper(channels=[Channel("conda-forge")]) 29 | for query in ( 30 | "ca-certificates", 31 | "ca-certificates =2022.9.24", 32 | "ca-certificates >=2022.9.24", 33 | "ca-certificates >2022.9.24", 34 | "ca-certificates<2022.9.24,>2020", 35 | "ca-certificates<=2022.9.24,>2020", 36 | "ca-certificates !=2022.9.24,>2020", 37 | "ca-certificates=*=*_0", 38 | # TODO: channel specs are accepted but they seem to be ignored by libmambapy.Query! 39 | # "defaults::ca-certificates", 40 | # "defaults::ca-certificates=2022.9.24", 41 | # "defaults::ca-certificates[version='>=2022.9.24']", 42 | # "defaults::ca-certificates[build='*_0']", 43 | ): 44 | results = index.search(query) 45 | assert len(results) > 0, query 46 | 47 | assert index.search("ca-certificates=*=*_0") == index.search("ca-certificates[build='*_0']") 48 | assert index.search("ca-certificates >=2022.9.24") == index.search( 49 | "ca-certificates[version='>=2022.9.24']" 50 | ) 51 | 52 | 53 | def test_query_search_includes_python_site_packages_path(): 54 | index = LibMambaIndexHelper(channels=[Channel("conda-forge")], subdirs=("linux-64", "noarch")) 55 | results = index.search("python=3.13.2=h4724d56_1_cp313t") 56 | assert len(results) == 1 57 | prec = results[0] 58 | assert prec.name == "python" 59 | assert prec.version == "3.13.2" 60 | if python_site_packages_path_support: 61 | assert prec.python_site_packages_path == "lib/python3.13t/site-packages" 62 | else: 63 | assert prec.python_site_packages_path is None 64 | -------------------------------------------------------------------------------- /docs/user-guide/index.md: -------------------------------------------------------------------------------- 1 | # User Guide 2 | 3 | The `conda-libmamba-solver` plugin allows you to use `libmamba`, the same `libsolv`-powered solver used by `mamba` and `micromamba`, directly in `conda`. 4 | 5 | ## How to install 6 | 7 | If you have a recent `conda` (23.10 or later), you don't have to do anything. `conda-libmamba-solver` is already preconfigured as default. 8 | For older versions `conda`, we simply recommend updating `conda` to a more recent version: 9 | 10 | ```console 11 | $ conda update -n base conda 12 | ``` 13 | 14 | If this command fails, check this entry in the FAQ section: {ref}`install-older-conda`. 15 | 16 | ```{admonition} Update from the experimental versions 17 | :class: note 18 | 19 | Please refer to the [v22.12.0 release notes](https://github.com/conda/conda-libmamba-solver/releases/tag/22.12.0) for more details on how to update from a previous version if you were already using the experimental builds (conda-libmamba-solver 22.9 and below). 20 | ``` 21 | 22 | ## Usage 23 | 24 | From `conda` 23.10, `conda-libmamba-solver` is the default solver. You don't have to do anything else. It will just work. 25 | 26 | 27 | ````{admonition} Usage with conda 23.9 and below 28 | `conda <23.10` won't use `conda-libmamba-solver` by default. 29 | It will still rely on the `classic` solver. 30 | 31 |
32 | 33 | Sporadic use 34 | 35 | To enable it for one operation, you can use the `--solver` flag, available for `conda create|install|remove|update` commands. 36 | 37 | ``` 38 | $ conda install tensorflow --solver=libmamba 39 | ``` 40 | 41 | Note: The `--solver` flag is also exposed as an environment variable, `CONDA_SOLVER`, 42 | in case you need that. 43 | 44 |
45 | 46 |
47 | 48 | Set as default 49 | 50 | To enable it permanently, you can add `solver: libmamba` to your `.condarc` file, either manually, or with this command: 51 | 52 | ``` 53 | $ conda config --set solver libmamba 54 | ``` 55 | 56 |
57 | ```` 58 | 59 | ## Revert to `classic` 60 | 61 | If you ever need to use the classic solver temporarily, use `--solver` flag: 62 | 63 | ``` 64 | $ conda install numpy --solver=classic 65 | ``` 66 | 67 | Finally, if you need to revert the default configuration back to `classic`, you can: 68 | 69 | * Run `conda config --set solver classic` (to make your choice explicit). 70 | 71 | ```{admonition} Tip 72 | If you are unsure what configuration is being used by conda, you can inspect 73 | it with `conda config --show-sources`. 74 | ``` 75 | 76 | ```{toctree} 77 | :hidden: 78 | 79 | subcommands 80 | configuration 81 | faq 82 | libmamba-vs-classic 83 | performance 84 | more-resources 85 | ``` 86 | -------------------------------------------------------------------------------- /dev/scripts/requests-fetch-all-shards.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (C) 2022 Anaconda, Inc 3 | # Copyright (C) 2023 conda 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | """ 6 | Collect all repodata shards from fast.prefix.dev/conda-forge. 7 | 8 | Takes about 23s on a hot CDN cache or 5 minutes on a cold remote cache. 9 | """ 10 | 11 | import concurrent.futures 12 | import sqlite3 13 | import time 14 | 15 | import msgpack 16 | import requests 17 | import zstandard 18 | 19 | 20 | def connect(dburi="cache.db"): 21 | """ 22 | Get database connection. 23 | 24 | dburi: uri-style sqlite database filename; accepts certain ?= parameters. 25 | """ 26 | conn = sqlite3.connect(dburi, uri=True) 27 | conn.row_factory = sqlite3.Row 28 | conn.execute("PRAGMA journal_mode = WAL") 29 | conn.execute("PRAGMA synchronous = 1") # less fsync, good for WAL mode 30 | conn.execute("PRAGMA foreign_keys = ON") 31 | return conn 32 | 33 | 34 | def to_url(shard): 35 | return f"https://fast.prefix.dev/conda-forge/linux-64/shards/{shard.hex()}.msgpack.zst" 36 | 37 | 38 | base_url = "https://fast.prefix.dev/conda-forge/linux-64/repodata_shards.msgpack.zst" 39 | 40 | s = requests.Session() 41 | 42 | index = msgpack.loads(zstandard.decompress(s.get(base_url).content)) 43 | 44 | conn = connect("conda-forge-shards.db") 45 | conn.execute("CREATE TABLE IF NOT EXISTS shards (url TEXT PRIMARY KEY, package TEXT, shard BLOB)") 46 | 47 | 48 | def shard_urls(): 49 | for package, hash in index["shards"].items(): 50 | yield package, to_url(hash) 51 | 52 | 53 | def fetch(s, package, url): 54 | b1 = time.time_ns() 55 | data = s.get(url).content 56 | e1 = time.time_ns() 57 | print(f"{(e1 - b1) / 1e9}s", package, url) 58 | return data 59 | 60 | 61 | begin = time.time_ns() 62 | 63 | # beneficial to have thread pool larger than requests' default 10 max 64 | # connections per session 65 | with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: 66 | futures = { 67 | executor.submit(fetch, s, package, url): (url, package) for package, url in shard_urls() 68 | } 69 | # May be inconvenient to cancel a large number of futures. Also, ctrl-C doesn't work reliably out of the box. 70 | for future in concurrent.futures.as_completed(futures): 71 | print(".", futures[future]) 72 | data = future.result() 73 | url, package = futures[future] 74 | with conn as c: 75 | c.execute( 76 | "INSERT OR IGNORE INTO SHARDS (url, package, shard) VALUES (?, ?, ?)", 77 | (url, package, data), 78 | ) 79 | 80 | end = time.time_ns() 81 | 82 | conn.close() 83 | 84 | print(f"Took {(end - begin) / 1e9} seconds") 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # conda-libmamba-solver 2 | 3 | The fast mamba solver, now in conda! 4 | 5 | ## What is this exactly? 6 | 7 | conda-libmamba-solver is a new solver for the 8 | [conda package manager](https://docs.conda.io/) which uses the solver from the 9 | [mamba project](https://mamba.readthedocs.io/) behind the scenes, while 10 | carefully implementing conda's functionality and expected behaviors on top. 11 | The library used by mamba to do the heavy-lifting is called [libsolv](https://github.com/openSUSE/libsolv). 12 | 13 | Additional information about the project can be found in the blog post on Anaconda's weblog: 14 | [A Faster Solver for Conda: Libmamba](https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community). 15 | 16 | ## Documentation 17 | 18 | Check the [documentation](https://conda.github.io/conda-libmamba-solver/) for 19 | instructions on how to install, use and make the most out the new conda solver! 20 | 21 | 22 | ## Build status 23 | 24 | | [![Build status](https://github.com/conda/conda-libmamba-solver/actions/workflows/tests.yml/badge.svg)](https://github.com/conda/conda-libmamba-solver/actions/workflows/tests.yml?query=branch%3Amain) [![Docs status](https://github.com/conda/conda-libmamba-solver/actions/workflows/docs.yml/badge.svg)](https://github.com/conda/conda-libmamba-solver/actions/workflows/docs.yml?query=branch%3Amain) [![codecov](https://codecov.io/gh/conda/conda-libmamba-solver/branch/main/graph/badge.svg)](https://codecov.io/gh/conda/conda-libmamba-solver) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/conda/conda-libmamba-solver/main.svg)](https://results.pre-commit.ci/latest/github/conda/conda-libmamba-solver/main) [![CalVer version used YY.MM.MICRO](https://img.shields.io/badge/calver-YY.MM.MICRO-22bfda.svg?style=flat-square)](https://calver.org) | [![Anaconda-Server Badge](https://anaconda.org/conda-canary/conda-libmamba-solver/badges/latest_release_date.svg)](https://anaconda.org/conda-canary/conda-libmamba-solver) | 25 | | --- | :-: | 26 | | [`conda install defaults::conda-libmamba-solver`](https://anaconda.org/anaconda/conda-libmamba-solver) | [![Anaconda-Server Badge](https://anaconda.org/anaconda/conda-libmamba-solver/badges/version.svg)](https://anaconda.org/anaconda/conda-libmamba-solver) | 27 | | [`conda install conda-forge::conda-libmamba-solver`](https://anaconda.org/conda-forge/conda-libmamba-solver) | [![Anaconda-Server Badge](https://anaconda.org/conda-forge/conda-libmamba-solver/badges/version.svg)](https://anaconda.org/conda-forge/conda-libmamba-solver) | 28 | | [`conda install conda-canary/label/dev::conda-libmamba-solver`](https://anaconda.org/conda-canary/conda-libmamba-solver) | [![Anaconda-Server Badge](https://anaconda.org/conda-canary/conda-libmamba-solver/badges/version.svg)](https://anaconda.org/conda-canary/conda-libmamba-solver) | 29 | -------------------------------------------------------------------------------- /.authors.yml: -------------------------------------------------------------------------------- 1 | - name: Jaime Rodríguez-Guerra 2 | email: jaimergp@users.noreply.github.com 3 | aliases: 4 | - jaimergp 5 | num_commits: 169 6 | first_commit: 2022-01-31 17:24:37 7 | github: jaimergp 8 | - name: Jannis Leidel 9 | email: jannis@leidel.info 10 | num_commits: 40 11 | first_commit: 2022-02-17 14:48:48 12 | github: jezdez 13 | - name: pre-commit-ci[bot] 14 | email: 66853113+pre-commit-ci[bot]@users.noreply.github.com 15 | num_commits: 93 16 | first_commit: 2022-11-22 08:39:31 17 | github: pre-commit-ci[bot] 18 | - name: Christopher Ostrouchov 19 | email: chris.ostrouchov@gmail.com 20 | num_commits: 10 21 | first_commit: 2023-01-24 15:12:29 22 | github: costrouc 23 | - name: Daniel Holth 24 | email: dholth@anaconda.com 25 | num_commits: 15 26 | first_commit: 2022-10-19 21:11:39 27 | github: dholth 28 | - name: Matthew R. Becker 29 | email: beckermr@users.noreply.github.com 30 | num_commits: 1 31 | first_commit: 2022-01-19 18:26:25 32 | github: beckermr 33 | - name: conda-bot 34 | email: 18747875+conda-bot@users.noreply.github.com 35 | aliases: 36 | - Conda Bot 37 | num_commits: 81 38 | first_commit: 2022-11-15 16:45:31 39 | github: conda-bot 40 | - name: Ken Odegard 41 | email: kodegard@anaconda.com 42 | num_commits: 10 43 | first_commit: 2022-03-04 23:25:33 44 | github: kenodegard 45 | - name: Albert DeFusco 46 | email: albert.defusco+gh@me.com 47 | num_commits: 1 48 | first_commit: 2023-03-01 13:29:51 49 | github: AlbertDeFusco 50 | - name: John Kirkham 51 | github: jakirkham 52 | email: jakirkham@gmail.com 53 | num_commits: 1 54 | first_commit: 2023-04-18 12:42:06 55 | - name: Travis Hathaway 56 | email: travis.j.hathaway@gmail.com 57 | num_commits: 8 58 | first_commit: 2023-11-10 15:58:32 59 | github: travishathaway 60 | - name: Klaus Zimmermann 61 | email: klaus.zimmermann@quansight.com 62 | num_commits: 1 63 | first_commit: 2024-06-11 10:52:46 64 | - name: Thomas Lam 65 | email: 79589038+tl-hbk@users.noreply.github.com 66 | num_commits: 1 67 | first_commit: 2024-03-19 14:05:09 68 | github: tl-hbk 69 | - name: dependabot[bot] 70 | email: 49699333+dependabot[bot]@users.noreply.github.com 71 | num_commits: 20 72 | first_commit: 2024-07-22 19:11:59 73 | github: dependabot[bot] 74 | - name: Jonathan J. Helmus 75 | email: jjhelmus@gmail.com 76 | num_commits: 4 77 | first_commit: 2024-09-20 12:11:32 78 | github: jjhelmus 79 | - name: Kevin Markham 80 | email: justmarkham@users.noreply.github.com 81 | num_commits: 1 82 | first_commit: 2024-08-12 11:49:20 83 | github: justmarkham 84 | - name: Julien Jerphanion 85 | email: git@jjerphan.xyz 86 | github: jjerphan 87 | num_commits: 1 88 | first_commit: 2025-03-05 00:42:28 89 | - name: Agriya Khetarpal 90 | email: 74401230+agriyakhetarpal@users.noreply.github.com 91 | github: agriyakhetarpal 92 | num_commits: 1 93 | first_commit: 2025-10-16 17:26:43 94 | - name: Stacy Noland 95 | email: 46572585+stacynoland@users.noreply.github.com 96 | github: stacynoland 97 | num_commits: 1 98 | first_commit: 2025-11-24 12:33:24 99 | -------------------------------------------------------------------------------- /tests/data/mamba_repo/noarch/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | repo/noarch 4 | 44 | 45 | 46 |

repo/noarch

47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 |
FilenameSizeLast ModifiedSHA256MD5
repodata.json586 B2021-02-12 09:01:48 +0000cc5f72aaa8d3f508c8adca196fe05cf4b19e1ca1006cfcbb3892d73160bd3b047501ec77771889b42a39c615158cb9c4
repodata.json.bz2351 B2021-02-12 09:01:48 +00009a0288ca48c6b8caa348d7cafefd0981c2d25dcb4a5837a5187ab200b8b9fb450c926155642f0e894d97dc8a5af7007b
repodata_from_packages.json586 B2021-02-12 09:01:48 +0000cc5f72aaa8d3f508c8adca196fe05cf4b19e1ca1006cfcbb3892d73160bd3b047501ec77771889b42a39c615158cb9c4
repodata_from_packages.json.bz2351 B2021-02-12 09:01:48 +00009a0288ca48c6b8caa348d7cafefd0981c2d25dcb4a5837a5187ab200b8b9fb450c926155642f0e894d97dc8a5af7007b
test-package-0.1-0.tar.bz26 KB2021-02-12 08:08:14 +0000b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f69882a8595f37faa2950e1b433acbe91d481
86 |
Updated: 2021-02-12 09:02:37 +0000 - Files: 1
87 | 88 | 89 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/0_bug.yml: -------------------------------------------------------------------------------- 1 | # edit this in https://github.com/conda/infrastructure 2 | 3 | name: Bug Report 4 | description: Create a bug report. 5 | labels: 6 | - type::bug 7 | body: 8 | - type: markdown 9 | attributes: 10 | value: | 11 | Because processing new bug reports is time-consuming, we would like to ask you to fill out the following form to the best of your ability and as completely as possible. 12 | 13 | > [!NOTE] 14 | > Bug reports that are incomplete or missing information may be closed as inactionable. 15 | 16 | Since there are already a lot of open issues, please also take a moment to search existing ones to see if your bug has already been reported. If you find something related, please upvote that issue and provide additional details as necessary. 17 | 18 | 💐 Thank you for helping to make `conda/conda-libmamba-solver` better. We would be unable to improve `conda/conda-libmamba-solver` without our community! 19 | - type: checkboxes 20 | id: checks 21 | attributes: 22 | label: Checklist 23 | description: Please confirm and check all of the following options. 24 | options: 25 | - label: I added a descriptive title 26 | required: true 27 | - label: I searched open reports and couldn't find a duplicate 28 | required: true 29 | - type: textarea 30 | id: what 31 | attributes: 32 | label: What happened? 33 | description: What should have happened instead? Please provide as many details as possible. The more information provided, the more likely we are able to replicate your problem and offer a solution. 34 | validations: 35 | required: true 36 | - type: textarea 37 | id: info 38 | attributes: 39 | label: Conda Info 40 | description: | 41 | Let's collect some basic information about your conda install. 42 | 43 | Please run the following command from your command line and paste the output below. 44 | 45 | ```bash 46 | conda info 47 | ``` 48 | render: shell 49 | - type: textarea 50 | id: config 51 | attributes: 52 | label: Conda Config 53 | description: | 54 | Let's collect any customizations you may have for your conda install. 55 | 56 | Please run the following command from your command line and paste the output below. 57 | 58 | ```bash 59 | conda config --show-sources 60 | ``` 61 | render: shell 62 | - type: textarea 63 | id: list 64 | attributes: 65 | label: Conda list 66 | description: | 67 | The packages installed into your environment can offer clues as to the problem you are facing. 68 | 69 | Please activate the environment within which you are encountering this bug, run the following command from your command line, and paste the output below. 70 | 71 | ```bash 72 | conda list --show-channel-urls 73 | ``` 74 | render: shell 75 | 76 | - type: textarea 77 | id: context 78 | attributes: 79 | label: Additional Context 80 | description: Include any additional information (or screenshots) that you think would be valuable. 81 | -------------------------------------------------------------------------------- /dev/scripts/httpx-fetch-all-shards.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (C) 2022 Anaconda, Inc 3 | # Copyright (C) 2023 conda 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | """ 6 | Collect all repodata shards from fast.prefix.dev/conda-forge. 7 | 8 | Takes about 12.7 seconds, a few requests fail and would need to be re-tried. 9 | """ 10 | 11 | import asyncio 12 | import sqlite3 13 | import time 14 | 15 | import httpx 16 | import msgpack 17 | import zstandard 18 | 19 | 20 | def connect(dburi="cache.db"): 21 | """ 22 | Get database connection. 23 | 24 | dburi: uri-style sqlite database filename; accepts certain ?= parameters. 25 | """ 26 | conn = sqlite3.connect(dburi, uri=True) 27 | conn.row_factory = sqlite3.Row 28 | # conn.execute("PRAGMA journal_mode = WAL") 29 | # conn.execute("PRAGMA synchronous = 1") # less fsync, good for WAL mode 30 | conn.execute("PRAGMA foreign_keys = ON") 31 | return conn 32 | 33 | 34 | def to_url(shard): 35 | return f"https://fast.prefix.dev/conda-forge/linux-64/shards/{shard.hex()}.msgpack.zst" 36 | 37 | 38 | base_url = "https://fast.prefix.dev/conda-forge/linux-64/repodata_shards.msgpack.zst" 39 | 40 | 41 | conn = connect("conda-forge-shards-httpx.db") 42 | conn.execute( 43 | "CREATE TABLE IF NOT EXISTS shards (url TEXT PRIMARY KEY, package TEXT, shard BLOB)" 44 | ) # also last-used? 45 | 46 | 47 | def shard_urls(index): 48 | for package, hash in index["shards"].items(): 49 | yield package, to_url(hash) 50 | 51 | 52 | async def get_one(sem, client, url, package): 53 | try: 54 | async with sem: 55 | response = await client.get(url) 56 | return package, url, response 57 | except httpx.RemoteProtocolError as err: 58 | return package, url, err 59 | 60 | 61 | async def main(): 62 | async with httpx.AsyncClient(http2=True, timeout=60) as client: 63 | # client._transport._pool._max_connections # is 100, does this have 64 | # anything to do with http/2? 65 | # See https://github.com/encode/httpx/issues/1171 66 | # Default 5s timeout is bigger issue in request failures. 67 | print("Max in-flight", client._transport._pool._max_connections) 68 | sem = asyncio.Semaphore(client._transport._pool._max_connections) 69 | response = await client.get(base_url) 70 | print(response.status_code, response.headers) 71 | index = msgpack.loads(zstandard.decompress(response.content)) 72 | awaitables = [ 73 | get_one(sem, client, shard_url, package) for package, shard_url in shard_urls(index) 74 | ] 75 | for awaitable in asyncio.as_completed(awaitables): 76 | package, url, response = await awaitable 77 | if isinstance(response, Exception): 78 | print("x", end="") 79 | # add (package, url) to retry list? 80 | continue 81 | data = response.content 82 | with conn as c: 83 | c.execute( 84 | "INSERT OR IGNORE INTO SHARDS (url, package, shard) VALUES (?, ?, ?)", 85 | (url, package, data), 86 | ) 87 | print(".", end="") 88 | 89 | 90 | if __name__ == "__main__": 91 | begin = time.time_ns() 92 | asyncio.run(main()) 93 | end = time.time_ns() 94 | print(f"Took {(end - begin) / 1e9} seconds") 95 | conn.close() 96 | -------------------------------------------------------------------------------- /tests/test_workarounds.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | import ctypes 5 | import json 6 | import signal 7 | import subprocess as sp 8 | import sys 9 | import time 10 | 11 | import pytest 12 | from conda.common.compat import on_win 13 | 14 | 15 | def test_matchspec_star_version(): 16 | """ 17 | Specs like `libblas=*=*mkl` choked on `MatchSpec.conda_build_form()`. 18 | We work around that with `.utils.safe_conda_build_form()`. 19 | Reported in https://github.com/conda/conda/issues/11347 20 | """ 21 | sp.check_call( 22 | [ 23 | sys.executable, 24 | "-m", 25 | "conda", 26 | "create", 27 | "-p", 28 | "UNUSED", 29 | "--dry-run", 30 | "--override-channels", 31 | "-c", 32 | "conda-test", 33 | "--solver=libmamba", 34 | "activate_deactivate_package=*=*0", 35 | ] 36 | ) 37 | 38 | 39 | def test_build_string_filters(): 40 | process = sp.run( 41 | [ 42 | sys.executable, 43 | "-m", 44 | "conda", 45 | "create", 46 | "-p", 47 | "UNUSED", 48 | "--dry-run", 49 | "--solver=libmamba", 50 | "numpy=*=*py38*", 51 | "--json", 52 | ], 53 | stdout=sp.PIPE, 54 | text=True, 55 | ) 56 | print(process.stdout) 57 | process.check_returncode() 58 | data = json.loads(process.stdout) 59 | assert data["success"] 60 | for pkg in data["actions"]["LINK"]: 61 | if pkg["name"] == "python": 62 | assert pkg["version"].startswith("3.8") 63 | if pkg["name"] == "numpy": 64 | assert "py38" in pkg["build_string"] 65 | 66 | 67 | @pytest.mark.parametrize("stage", ["Collecting package metadata", "Solving environment"]) 68 | def test_ctrl_c(stage): 69 | TIMEOUT = 20 # Used twice in total, so account for double the amount 70 | p = sp.Popen( 71 | [ 72 | sys.executable, 73 | "-m", 74 | "conda", 75 | "create", 76 | "-p", 77 | "UNUSED", 78 | "--dry-run", 79 | "--solver=libmamba", 80 | "--override-channels", 81 | "--channel=conda-forge", 82 | "--quiet", 83 | "vaex", 84 | ], 85 | text=True, 86 | stdout=sp.PIPE, 87 | stderr=sp.PIPE, 88 | ) 89 | t0 = time.time() 90 | while stage not in p.stdout.readline(): 91 | time.sleep(0.1) 92 | if time.time() - t0 > TIMEOUT: 93 | raise RuntimeError("Timeout") 94 | 95 | # works around Windows' awkward CTRL-C signal handling 96 | # https://stackoverflow.com/a/64357453 97 | if on_win: 98 | try: 99 | kernel = ctypes.windll.kernel32 100 | kernel.FreeConsole() 101 | kernel.AttachConsole(p.pid) 102 | kernel.SetConsoleCtrlHandler(None, 1) 103 | kernel.GenerateConsoleCtrlEvent(0, 0) 104 | p.wait(timeout=TIMEOUT) 105 | finally: 106 | kernel.SetConsoleCtrlHandler(None, 0) 107 | else: 108 | p.send_signal(signal.SIGINT) 109 | p.wait(timeout=TIMEOUT) 110 | 111 | assert p.returncode != 0 112 | assert "KeyboardInterrupt" in p.stdout.read() + p.stderr.read() 113 | -------------------------------------------------------------------------------- /.devcontainer/post_start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script assumes we are running in a Miniconda container where: 4 | # - /opt/conda is the Miniconda or Miniforge installation directory 5 | # - https://github.com/conda/conda is mounted at /workspaces/conda 6 | # - https://github.com/conda/conda-libmamba-solver is mounted at 7 | # /workspaces/conda-libmamba-solver 8 | # - https://github.com/mamba-org/mamba is (optionally) mounted at 9 | # /workspaces/mamba 10 | 11 | set -euo pipefail 12 | 13 | BASE_CONDA=${BASE_CONDA:-/opt/conda} 14 | SRC_CONDA=${SRC_CONDA:-/workspaces/conda} 15 | SRC_CONDA_LIBMAMBA_SOLVER=${SRC_CONDA_LIBMAMBA_SOLVER:-/workspaces/conda-libmamba-solver} 16 | SRC_MAMBA=${SRC_MAMBA:-/workspaces/mamba} 17 | 18 | cat >> ~/.bashrc < /tmp/mamba-environment-dev.yml 44 | # Environment.yml is missing make 45 | echo " - make" >> /tmp/mamba-environment-dev.yml 46 | "$BASE_CONDA/condabin/conda" env update \ 47 | --quiet \ 48 | --prefix "$BASE_CONDA" \ 49 | --file /tmp/mamba-environment-dev.yml 50 | # Clean build directory to avoid issues with stale build files 51 | test -f "$SRC_MAMBA/build/CMakeCache.txt" && rm -rf "$SRC_MAMBA/build" 52 | fi 53 | # Compile 54 | cd "$SRC_MAMBA" 55 | "$BASE_CONDA/bin/cmake" -B build/ \ 56 | -DBUILD_LIBMAMBA=ON \ 57 | -DBUILD_SHARED=ON \ 58 | -DCMAKE_INSTALL_PREFIX="$BASE_CONDA" \ 59 | -DCMAKE_PREFIX_PATH="$BASE_CONDA" \ 60 | -DBUILD_LIBMAMBAPY=ON 61 | "$BASE_CONDA/bin/cmake" --build build/ -j\${NPROC:-2} 62 | if [ ! -f ~/.mamba-develop-installed ]; then 63 | "$BASE_CONDA/condabin/conda" remove -p "$BASE_CONDA" -yq --force libmambapy libmamba 64 | fi 65 | make install -C build/ 66 | cd - 67 | "$BASE_CONDA/bin/pip" install -e "$SRC_MAMBA/libmambapy/" --no-deps 68 | if [ "\$mamba_version" == "1" ]; then 69 | test -f "$BASE_CONDA/conda-meta/mamba-"*".json" && "$BASE_CONDA/bin/pip" install -e "$SRC_MAMBA/mamba/" --no-deps 70 | else 71 | echo "Mamba binary installation not supported yet" 72 | fi 73 | touch ~/.mamba-develop-installed || true 74 | ) 75 | EOF 76 | 77 | cd "$SRC_CONDA" 78 | echo "Initializing conda in dev mode..." 79 | "$BASE_CONDA/bin/python" -m conda init --dev bash 80 | cd - 81 | 82 | echo "Installing conda-libmamba-solver in dev mode..." 83 | "$BASE_CONDA/bin/python" -m pip install -e "$SRC_CONDA_LIBMAMBA_SOLVER" --no-deps 84 | 85 | set -x 86 | conda list -p "$BASE_CONDA" 87 | conda info 88 | conda config --show-sources 89 | set +x 90 | test -f "$SRC_MAMBA/mamba/setup.py" \ 91 | && echo "Mamba mounted at $SRC_MAMBA; source ~/.bashrc and run develop-mamba() for dev-install" 92 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 105 | __pypackages__/ 106 | 107 | # Celery stuff 108 | celerybeat-schedule 109 | celerybeat.pid 110 | 111 | # SageMath parsed files 112 | *.sage.py 113 | 114 | # Environments 115 | .env 116 | .venv 117 | env/ 118 | venv/ 119 | ENV/ 120 | env.bak/ 121 | venv.bak/ 122 | 123 | # Spyder project settings 124 | .spyderproject 125 | .spyproject 126 | 127 | # Rope project settings 128 | .ropeproject 129 | 130 | # mkdocs documentation 131 | /site 132 | 133 | # mypy 134 | .mypy_cache/ 135 | .dmypy.json 136 | dmypy.json 137 | 138 | # Pyre type checker 139 | .pyre/ 140 | 141 | # pytype static type analyzer 142 | .pytype/ 143 | 144 | # Cython debug symbols 145 | cython_debug/ 146 | 147 | # PyCharm 148 | # JetBrains specific template is maintainted in a separate JetBrains.gitignore that can 149 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 150 | # and can be added to the global gitignore or merged into this file. For a more nuclear 151 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 152 | #.idea/ 153 | .vscode/ 154 | 155 | # rever 156 | rever/ 157 | 158 | # setuptools_scm 159 | _version.py 160 | 161 | # codspeed 162 | .codspeed/ 163 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/epic.yml: -------------------------------------------------------------------------------- 1 | # edit this in https://github.com/conda/infrastructure 2 | 3 | name: Epic 4 | description: A collection of related tickets. 5 | labels: 6 | - epic 7 | body: 8 | - type: markdown 9 | attributes: 10 | value: | 11 | This form is intended for grouping and collecting together related tickets to better gauge the scope of a problem/feature. 12 | 13 | If you are attempting to report a bug, propose a new feature, or some other code change please use one of the other forms available. 14 | 15 | > [!NOTE] 16 | > Epics that are incomplete or missing information may be closed as inactionable. 17 | 18 | Since there are already a lot of open issues, please also take a moment to search existing ones to see if a similar epic has already been opened. If you find something related, please upvote that issue and provide additional details as necessary. 19 | 20 | 💐 Thank you for helping to make `conda/conda-libmamba-solver` better. We would be unable to improve `conda/conda-libmamba-solver` without our community! 21 | 22 | - type: checkboxes 23 | id: checks 24 | attributes: 25 | label: Checklist 26 | description: Please confirm and check all of the following options. 27 | options: 28 | - label: I added a descriptive title 29 | required: true 30 | - label: I searched open issues and couldn't find a duplicate 31 | required: true 32 | 33 | - type: textarea 34 | id: what 35 | attributes: 36 | label: What? 37 | description: >- 38 | What feature or problem will be addressed in this epic? 39 | placeholder: Please describe here. 40 | validations: 41 | required: true 42 | 43 | - type: textarea 44 | id: why 45 | attributes: 46 | label: Why? 47 | description: >- 48 | Why is the reported issue(s) a problem, or why is the proposed feature needed? 49 | (Research and spike issues can be linked here.) 50 | value: | 51 | - [ ] 52 | placeholder: Please describe here and/or link to relevant supporting issues. 53 | validations: 54 | required: true 55 | 56 | - type: textarea 57 | id: user_impact 58 | attributes: 59 | label: User impact 60 | description: >- 61 | In what specific way(s) will users benefit from this change? (e.g. use cases or performance improvements) 62 | placeholder: Please describe here. 63 | validations: 64 | required: true 65 | 66 | - type: textarea 67 | id: goals 68 | attributes: 69 | label: Goals 70 | description: >- 71 | What goal(s) should this epic accomplish? 72 | value: | 73 | - [ ] 74 | validations: 75 | required: true 76 | 77 | - type: textarea 78 | id: tasks 79 | attributes: 80 | label: Tasks 81 | description: >- 82 | What needs to be done to implement this change? 83 | value: | 84 | - [ ] 85 | validations: 86 | required: false 87 | 88 | - type: textarea 89 | id: blocked_by 90 | attributes: 91 | label: 'This epic is blocked by:' 92 | description: >- 93 | Epics and issues that block this epic. 94 | value: | 95 | - [ ] 96 | validations: 97 | required: false 98 | 99 | - type: textarea 100 | id: blocks 101 | attributes: 102 | label: 'This epic blocks:' 103 | description: >- 104 | Epics and issues that are blocked by this epic. 105 | value: | 106 | - [ ] 107 | validations: 108 | required: false 109 | -------------------------------------------------------------------------------- /docs/user-guide/faq.md: -------------------------------------------------------------------------------- 1 | # Frequently Asked Questions 2 | 3 | ## What's the difference between the available solvers in `conda`? 4 | 5 | Please refer to the section "Technical differences between `libmamba` and `classic`" in 6 | the [`libmamba-vs-classic`](./libmamba-vs-classic.md#technical-differences-between-libmamba-and-classic) docs. 7 | 8 | ## How do I uninstall it? 9 | 10 | If you don't want to use the solver anymore, follow these instructions: 11 | 12 | ```{warning} 13 | Please make sure you __follow all steps below__ to uninstall the solver! 14 | ``` 15 | 16 | 1. If you configured it as the default solver, make sure you revert it with: 17 | 18 | ```bash 19 | $ conda config --remove-key solver 20 | # You might also need this: 21 | $ conda config --remove-key experimental_solver 22 | ``` 23 | 24 | 2. Then, remove the package from `base` with: 25 | 26 | ```bash 27 | $ conda remove -n base conda-libmamba-solver 28 | ``` 29 | 30 | ## How do I configure conda to use the solver permanently? 31 | 32 | Use the following command to always use `libmamba` as your default solver: 33 | 34 | ```bash 35 | $ conda config --set solver libmamba 36 | ``` 37 | 38 | To undo this change permanently, run: 39 | 40 | ```bash 41 | $ conda config --remove-key solver 42 | # You might also need this: 43 | $ conda config --remove-key experimental_solver 44 | ``` 45 | 46 | ## I get an error when I try to use the `--solver` flag 47 | 48 | If you are seeing this error: 49 | 50 | ``` 51 | CondaValueError: Key 'solver' is not a known primitive parameter. 52 | ``` 53 | 54 | It might mean you are using an old version of the conda-libmamba-solver package. 55 | You can check which version is installed with `conda list -n base conda-libmamba-solver`. 56 | 57 | Before version 22.12, the CLI flag was `--experimental-solver`. 58 | We recommend you upgrade to `conda` 22.11 or above, and then `conda-libmamba-solver` 22.12 or above. 59 | 60 | See the [22.12.0 announcement post](https://github.com/conda/conda-libmamba-solver/releases/tag/22.12.0) for more details on how to upgrade. 61 | 62 | (install-older-conda)= 63 | 64 | ## I have an older `conda` and I can't install `conda-libmamba-solver` 65 | 66 | Since older `conda` versions only supported the `classic` solver, you might run into solver conflicts or too long installations if your `base` environment is too constrained. This becomes a "chicken-and-egg" problem where you'd need `conda-libmamba-solver` to update to a more recent `conda` with `conda-libmamba-solver`. 67 | 68 | Fortunately, there's a workaround thanks to the `conda-standalone` project. This is a single binary that bundles recent `conda` versions, with `conda-libmamba-solver` included. It's not a substitute for the full `conda` user experience but it can help bootstrap and rescue conda installations. 69 | 70 | 1. Download the most recent `conda-standalone` from its [Github Releases page](https://github.com/conda/conda-standalone/releases/latest). Make sure to pick the one for your operating system and platform. Once downloaded, rename it as `conda.exe` on Windows and `conda` on Linux / macOS. 71 | 2. Write down the location of your `base` environment: `conda info --root`. 72 | 3. Write down the main preconfigured channel in your installation: `conda config --show channels`. This is usually `defaults` or `conda-forge`. 73 | 4. Go to the Downloads directory and run this command from your terminal: 74 | 75 | On Windows: 76 | 77 | ```console 78 | .\conda.exe install -p "[path given by step 2]" -c [channel from step 3] "conda>=23.10" conda-libmamba-solver 79 | ``` 80 | 81 | On Linux or macOS: 82 | 83 | ```console 84 | ./conda install -p "[path given by step 2]" -c [channel from step 3] "conda>=23.10" conda-libmamba-solver 85 | ``` 86 | 87 | Once the command succeeds, you'll have `conda-libmamba-solver` installed in your base environment and will be ready to use it as normal. You can delete the conda-standalone binaries. 88 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # Configuration file for the Sphinx documentation builder. 5 | # 6 | # For the full list of built-in configuration values, see the documentation: 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 8 | 9 | # -- Project information ----------------------------------------------------- 10 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 11 | 12 | project = html_title = "conda-libmamba-solver" 13 | copyright = "2022, conda-libmamba-solver contributors" 14 | author = "conda-libmamba-solver contributors" 15 | 16 | # -- General configuration --------------------------------------------------- 17 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 18 | 19 | extensions = [ 20 | "myst_parser", 21 | "sphinx.ext.napoleon", 22 | "sphinx.ext.autosummary", 23 | "sphinx.ext.graphviz", 24 | "sphinx.ext.ifconfig", 25 | "sphinx.ext.inheritance_diagram", 26 | "sphinx.ext.viewcode", 27 | "sphinxcontrib.mermaid", 28 | "sphinx_sitemap", 29 | "sphinx_design", 30 | "sphinx_copybutton", 31 | "sphinx_reredirects", 32 | ] 33 | 34 | myst_heading_anchors = 3 35 | myst_enable_extensions = [ 36 | "amsmath", 37 | "colon_fence", 38 | "deflist", 39 | "dollarmath", 40 | "html_admonition", 41 | "html_image", 42 | "linkify", 43 | "replacements", 44 | "smartquotes", 45 | "substitution", 46 | "tasklist", 47 | ] 48 | 49 | 50 | templates_path = ["_templates"] 51 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 52 | 53 | # -- Options for HTML output ------------------------------------------------- 54 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 55 | 56 | html_theme = "conda_sphinx_theme" 57 | html_static_path = ["_static"] 58 | 59 | html_css_files = [ 60 | "css/custom.css", 61 | ] 62 | 63 | # Serving the robots.txt since we want to point to the sitemap.xml file 64 | html_extra_path = ["robots.txt"] 65 | 66 | html_theme_options = { 67 | "navigation_depth": -1, 68 | "use_edit_page_button": True, 69 | "navbar_center": ["navbar_center"], 70 | "icon_links": [ 71 | { 72 | "name": "GitHub", 73 | "url": "https://github.com/conda/conda-libmamba-solver", 74 | "icon": "fa-brands fa-square-github", 75 | "type": "fontawesome", 76 | }, 77 | { 78 | "name": "Element", 79 | "url": "https://matrix.to/#/#conda-libmamba-solver:matrix.org", 80 | "icon": "_static/element_logo.svg", 81 | "type": "local", 82 | }, 83 | { 84 | "name": "Discourse", 85 | "url": "https://conda.discourse.group/", 86 | "icon": "fa-brands fa-discourse", 87 | "type": "fontawesome", 88 | }, 89 | ], 90 | } 91 | 92 | html_context = { 93 | "github_user": "conda", 94 | "github_repo": "conda-libmamba-solver", 95 | "github_version": "main", 96 | "doc_path": "docs", 97 | } 98 | 99 | # We don't have a locale set, so we can safely ignore that for the sitemaps. 100 | sitemap_locales = [None] 101 | # We're hard-coding stable here since that's what we want Google to point to. 102 | sitemap_url_scheme = "{link}" 103 | 104 | # -- For sphinx_reredirects ------------------------------------------------ 105 | 106 | redirects = { 107 | "getting-started": "../user-guide/", 108 | "faq": "../user-guide/faq/", 109 | "configuration": "../user-guide/configuration/", 110 | "libmamba-vs-classic": "../user-guide/libmamba-vs-classic/", 111 | "more-resources": "../user-guide/more-resources/", 112 | "performance": "../user-guide/performance/", 113 | "subcommands": "../user-guide/subcommands/", 114 | } 115 | -------------------------------------------------------------------------------- /tests/test_performance.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | Measure the speed and memory usage of the different backend solvers 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | from pathlib import Path 11 | from typing import TYPE_CHECKING 12 | 13 | import pytest 14 | from conda.base.context import context 15 | from conda.exceptions import DryRunExit 16 | 17 | if TYPE_CHECKING: 18 | from collections.abc import Iterable 19 | 20 | from conda.testing.fixtures import CondaCLIFixture, TmpEnvFixture 21 | from pytest import FixtureRequest 22 | 23 | pytestmark = [ 24 | pytest.mark.slow, 25 | pytest.mark.usefixtures("parametrized_solver_fixture"), 26 | pytest.mark.skip, 27 | ] 28 | 29 | TEST_DATA_DIR = Path(__file__).parent / "data" 30 | PLATFORM = context.subdir 31 | 32 | 33 | def _get_channels_from_lockfile(path: Path) -> tuple[str, ...]: 34 | """Parse `# channels: conda-forge,defaults` comments""" 35 | for line in path.read_text().splitlines(): 36 | if line.startswith("# channels:"): 37 | return tuple(line.split(":")[1].strip().split(",")) 38 | return () 39 | 40 | 41 | def _channels_as_args(channels: Iterable[str]) -> tuple[str, ...]: 42 | if not channels: 43 | return () 44 | return ("--override-channels", *(f"--channel={channel}" for channel in channels)) 45 | 46 | 47 | @pytest.fixture( 48 | scope="session", 49 | params=TEST_DATA_DIR.glob("*.lock"), 50 | ) 51 | def prefix_and_channels( 52 | request: FixtureRequest, 53 | session_tmp_env: TmpEnvFixture, 54 | ) -> Iterable[tuple[Path, tuple[str, ...]]]: 55 | lockfile = Path(request.param) 56 | lock_platform = lockfile.suffixes[-2] 57 | if not lock_platform.endswith(PLATFORM): 58 | pytest.skip(f"Running platform {PLATFORM} does not match file platform {lock_platform}") 59 | 60 | with pytest.MonkeyPatch.context() as monkeypatch: 61 | monkeypatch.setenv("CONDA_TEST_SAVE_TEMPS", "1") 62 | 63 | with session_tmp_env("--file", lockfile) as prefix: 64 | channels = _get_channels_from_lockfile(lockfile) 65 | yield prefix, channels 66 | 67 | 68 | def test_update_python( 69 | prefix_and_channels: tuple[Path, tuple[str, ...]], 70 | conda_cli: CondaCLIFixture, 71 | ) -> None: 72 | prefix, channels = prefix_and_channels 73 | try: 74 | conda_cli( 75 | "update", 76 | f"--prefix={prefix}", 77 | "--dry-run", 78 | *_channels_as_args(channels), 79 | "python", 80 | ) 81 | except DryRunExit: 82 | assert True 83 | else: 84 | # this can happen if "all requirements are satisfied" 85 | assert True 86 | 87 | 88 | def test_install_python_update_deps( 89 | prefix_and_channels: tuple[Path, tuple[str, ...]], 90 | conda_cli: CondaCLIFixture, 91 | ) -> None: 92 | prefix, channels = prefix_and_channels 93 | conda_cli( 94 | "install", 95 | f"--prefix={prefix}", 96 | "--dry-run", 97 | *_channels_as_args(channels), 98 | "python", 99 | "--update-deps", 100 | raises=DryRunExit, 101 | ) 102 | 103 | 104 | def test_update_all( 105 | prefix_and_channels: tuple[Path, tuple[str, ...]], 106 | conda_cli: CondaCLIFixture, 107 | ) -> None: 108 | prefix, channels = prefix_and_channels 109 | conda_cli( 110 | "update", 111 | f"--prefix={prefix}", 112 | "--dry-run", 113 | *_channels_as_args(channels), 114 | "--all", 115 | raises=DryRunExit, 116 | ) 117 | 118 | 119 | def test_install_vaex_from_conda_forge_and_defaults(conda_cli: CondaCLIFixture) -> None: 120 | conda_cli( 121 | "create", 122 | "--dry-run", 123 | *_channels_as_args(["conda-forge", "defaults"]), 124 | "python=3.9", 125 | "vaex", 126 | raises=DryRunExit, 127 | ) 128 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Stale 2 | 3 | on: 4 | # NOTE: github.event is workflow_dispatch payload: 5 | # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch 6 | workflow_dispatch: 7 | inputs: 8 | dryrun: 9 | description: 'dryrun: Preview stale issues/prs without marking them (true|false)' 10 | required: true 11 | type: boolean 12 | default: true 13 | 14 | schedule: 15 | - cron: 0 4 * * * 16 | 17 | permissions: 18 | issues: write 19 | pull-requests: write 20 | 21 | jobs: 22 | stale: 23 | if: '!github.event.repository.fork' 24 | runs-on: ubuntu-latest 25 | strategy: 26 | matrix: 27 | include: 28 | - only-issue-labels: '' 29 | days-before-issue-stale: 365 30 | days-before-issue-close: 30 31 | # [type::support] issues have a more aggressive stale/close timeline 32 | - only-issue-labels: type::support 33 | days-before-issue-stale: 90 34 | days-before-issue-close: 21 35 | steps: 36 | - uses: conda/actions/read-yaml@f05161c6e6e37a49b17c8e0b436197b53830318a # v25.9.2 37 | id: read_yaml 38 | with: 39 | path: https://raw.githubusercontent.com/conda/infra/main/.github/messages.yml 40 | 41 | - uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v10.1.1 42 | id: stale 43 | with: 44 | # Only issues with these labels are checked whether they are stale 45 | only-issue-labels: ${{ matrix.only-issue-labels }} 46 | 47 | # Idle number of days before marking issues stale 48 | days-before-issue-stale: ${{ matrix.days-before-issue-stale }} 49 | # Idle number of days before closing stale issues/PRs 50 | days-before-issue-close: ${{ matrix.days-before-issue-close }} 51 | # Idle number of days before marking PRs stale 52 | days-before-pr-stale: 365 53 | # Idle number of days before closing stale PRs 54 | days-before-pr-close: 30 55 | 56 | # Comment on the staled issues 57 | stale-issue-message: ${{ fromJSON(steps.read_yaml.outputs.value)['stale-issue'] }} 58 | # Label to apply on staled issues 59 | stale-issue-label: stale 60 | # Label to apply on closed issues 61 | close-issue-label: stale::closed 62 | # Reason to use when closing issues 63 | close-issue-reason: not_planned 64 | 65 | # Comment on the staled PRs 66 | stale-pr-message: ${{ fromJSON(steps.read_yaml.outputs.value)['stale-pr'] }} 67 | # Label to apply on staled PRs 68 | stale-pr-label: stale 69 | # Label to apply on closed PRs 70 | close-pr-label: stale::closed 71 | 72 | # Remove stale label from issues/PRs on updates/comments 73 | remove-stale-when-updated: true 74 | # Add specified labels to issues/PRs when they become unstale 75 | labels-to-add-when-unstale: stale::recovered 76 | # Remove specified labels to issues/PRs when they become unstale 77 | labels-to-remove-when-unstale: stale,stale::closed 78 | 79 | # Max number of operations per run 80 | operations-per-run: ${{ secrets.STALE_OPERATIONS_PER_RUN || 100 }} 81 | # Dry-run 82 | debug-only: ${{ github.event.inputs.dryrun || false }} 83 | # Order to get issues/PRs 84 | ascending: true 85 | # Delete branch after closing a stale PR 86 | delete-branch: false 87 | 88 | # Issues with these labels will never be considered stale 89 | exempt-issue-labels: stale::recovered,epic 90 | # Issues with these labels will never be considered stale 91 | exempt-pr-labels: stale::recovered,epic 92 | # Exempt all issues/PRs with milestones from stale 93 | exempt-all-milestones: true 94 | # Assignees on issues/PRs exempted from stale 95 | exempt-assignees: mingwandroid 96 | 97 | - name: Print outputs 98 | run: echo ${{ join(steps.stale.outputs.*, ',') }} 99 | -------------------------------------------------------------------------------- /conda_libmamba_solver/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | Miscellaneous utilities 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | from logging import getLogger 11 | from typing import TYPE_CHECKING 12 | from urllib.parse import quote 13 | 14 | from conda.common.compat import on_win 15 | from conda.common.url import urlparse 16 | from conda.exceptions import PackagesNotFoundError 17 | 18 | if TYPE_CHECKING: 19 | from collections.abc import Iterable 20 | from enum import Enum 21 | 22 | from conda.models.match_spec import MatchSpec 23 | 24 | from .index import LibMambaIndexHelper 25 | 26 | log = getLogger(f"conda.{__name__}") 27 | 28 | 29 | def escape_channel_url(channel: str) -> str: 30 | if channel.startswith("file:"): 31 | if "%" in channel: # it's escaped already 32 | return channel 33 | if on_win: 34 | channel = channel.replace("\\", "/") 35 | parts = urlparse(channel) 36 | if parts.scheme: 37 | components = parts.path.split("/") 38 | if on_win: 39 | if parts.netloc and len(parts.netloc) == 2 and parts.netloc[1] == ":": 40 | # with absolute paths (e.g. C:/something), C:, D:, etc might get parsed as netloc 41 | path = "/".join([parts.netloc] + [quote(p) for p in components]) 42 | parts = parts.replace(netloc="") 43 | else: 44 | path = "/".join(components[:2] + [quote(p) for p in components[2:]]) 45 | else: 46 | path = "/".join([quote(p) for p in components]) 47 | parts = parts.replace(path=path) 48 | return str(parts) 49 | return channel 50 | 51 | 52 | def compatible_specs( 53 | index: LibMambaIndexHelper, specs: Iterable[MatchSpec], raise_not_found: bool = True 54 | ) -> bool: 55 | """ 56 | Assess whether the given specs are compatible with each other. 57 | This is done by querying the index for each spec and taking the 58 | intersection of the results. If the intersection is empty, the 59 | specs are incompatible. 60 | 61 | If raise_not_found is True, a PackagesNotFoundError will be raised 62 | when one of the specs is not found. Otherwise, False will be returned 63 | because the intersection will be empty. 64 | """ 65 | if not len(specs) >= 2: 66 | raise ValueError("Must specify at least two specs") 67 | 68 | matched = None 69 | for spec in specs: 70 | results = set(index.search(spec)) 71 | if not results: 72 | if raise_not_found: 73 | exc = PackagesNotFoundError([spec], index.channels) 74 | exc.allow_retry = False 75 | raise exc 76 | return False 77 | if matched is None: 78 | # First spec, just set matched to the results 79 | matched = results 80 | continue 81 | # Take the intersection of the results 82 | matched &= results 83 | if not matched: 84 | return False 85 | 86 | return bool(matched) 87 | 88 | 89 | class EnumAsBools: 90 | """ 91 | Allows an Enum to be bool-evaluated with attribute access. 92 | 93 | >>> update_modifier = UpdateModifier("update_deps") 94 | >>> update_modifier_as_bools = EnumAsBools(update_modifier) 95 | >>> update_modifier == UpdateModifier.UPDATE_DEPS # from this 96 | True 97 | >>> update_modidier_as_bools.UPDATE_DEPS # to this 98 | True 99 | >>> update_modifier_as_bools.UPDATE_ALL 100 | False 101 | """ 102 | 103 | def __init__(self, enum: Enum): 104 | self._enum = enum 105 | self._names = {v.name for v in self._enum.__class__.__members__.values()} 106 | 107 | def __getattr__(self, name: str) -> bool: 108 | if name in ("name", "value"): 109 | return getattr(self._enum, name) 110 | if name in self._names: 111 | return self._enum.name == name 112 | raise AttributeError(f"'{name}' is not a valid name for {self._enum.__class__.__name__}") 113 | 114 | def __eq__(self, obj: object) -> bool: 115 | return self._enum.__eq__(obj) 116 | 117 | def _dict(self) -> dict[str, bool]: 118 | return {name: self._enum.name == name for name in self._names} 119 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling", "hatch-vcs"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "conda-libmamba-solver" 7 | description = "The fast mamba solver, now in conda" 8 | readme = "README.md" 9 | authors = [ 10 | {name = "Anaconda, Inc.", email = "conda@continuum.io"} 11 | ] 12 | license = {file = "LICENSE"} 13 | classifiers = [ 14 | "License :: OSI Approved :: BSD License", 15 | "Programming Language :: Python :: 3", 16 | "Programming Language :: Python :: 3 :: Only", 17 | "Programming Language :: Python :: 3.10", 18 | "Programming Language :: Python :: 3.11", 19 | "Programming Language :: Python :: 3.12", 20 | "Programming Language :: Python :: 3.13", 21 | "Programming Language :: Python :: Implementation :: CPython", 22 | "Programming Language :: Python :: Implementation :: PyPy" 23 | ] 24 | requires-python = ">=3.10" 25 | dependencies = [ 26 | "conda >=25.9", 27 | # "libmambapy >=2", 28 | "boltons >=23.0.0", 29 | "msgpack >=1.1.1", 30 | "requests >=2.28.0,<3", 31 | "zstandard >=0.15" 32 | ] 33 | dynamic = [ 34 | "version" 35 | ] 36 | 37 | [project.urls] 38 | homepage = "https://github.com/conda/conda-libmamba-solver" 39 | 40 | [project.entry-points.conda] 41 | conda-libmamba-solver = "conda_libmamba_solver.plugin" 42 | 43 | [tool.hatch.version] 44 | source = "vcs" 45 | 46 | [tool.black] 47 | line-length = 99 48 | target-version = ['py310', 'py311', 'py312', 'py313'] 49 | exclude = ''' 50 | ( 51 | ^/conda_libmamba_solver/_libmamba\.py 52 | | ^/tests/_reposerver\.py 53 | ) 54 | ''' 55 | 56 | [tool.pytest.ini_options] 57 | minversion = "6.0" 58 | addopts = [ 59 | "--color=yes", 60 | "--tb=native", 61 | "--strict-markers" 62 | ] 63 | filterwarnings = [ 64 | # elevate all deprecation warnings to errors 65 | "error::PendingDeprecationWarning", 66 | "error::DeprecationWarning", 67 | "error::FutureWarning", 68 | # temporary ignores 69 | "ignore:conda.cli.install.print_activate:PendingDeprecationWarning:conda.env.env", 70 | "ignore:`remote_definition`:FutureWarning:argparse", 71 | "ignore:conda.core.prefix_data.python_record_for_prefix:PendingDeprecationWarning:conda.core.link", 72 | "ignore:conda.core.prefix_data.PrefixDataType.__call__:PendingDeprecationWarning:conda_libmamba_solver.state", 73 | "ignore:conda.env.specs.binstar:DeprecationWarning:conda", 74 | "ignore:conda.env.specs.binstar:PendingDeprecationWarning:conda", 75 | "ignore:.*conda.plugins.types.*:DeprecationWarning", 76 | "ignore:.*conda.plugins.types.*:PendingDeprecationWarning", 77 | "ignore:conda.trust:DeprecationWarning:conda", 78 | "ignore:conda.trust:PendingDeprecationWarning:conda", 79 | "ignore:conda.core.link.PrefixActions:DeprecationWarning:conda", 80 | "ignore:conda.core.link.PrefixActions:PendingDeprecationWarning:conda", 81 | ] 82 | markers = [ 83 | "integration: integration tests that usually require an internet connect", 84 | "slow: slow running tests", 85 | ] 86 | 87 | [tool.ruff] 88 | exclude = ["conda_libmamba_solver/mamba_utils.py", "tests/data/"] 89 | line-length = 99 90 | show-fixes = true 91 | target-version = "py310" 92 | 93 | [tool.ruff.lint] 94 | # E, W = pycodestyle errors and warnings 95 | # F = pyflakes 96 | # I = isort 97 | # D = pydocstyle 98 | # UP = pyupgrade 99 | # ISC = flake8-implicit-str-concat 100 | # TCH = flake8-type-checking 101 | # T10 = flake8-debugger 102 | # FA = flake8-future-annotations 103 | # see also https://docs.astral.sh/ruff/rules/ 104 | select = ["E", "W", "F", "I", "D1", "UP", "ISC", "TCH", "T10", "FA"] 105 | # E402 module level import not at top of file 106 | # E501 line too long 107 | # E722 do not use bare 'except' 108 | # E731 do not assign a lambda expression, use a def 109 | # D101 Missing docstring in public class 110 | # D102 Missing docstring in public method 111 | # D103 Missing docstring in public function 112 | # D104 Missing docstring in public package 113 | # D105 Missing docstring in magic method 114 | # D107 Missing docstring in `__init__` 115 | ignore = ["E402", "E501", "E722", "E731", "D101", "D102", "D103", "D104", "D105", "D107"] 116 | extend-per-file-ignores = {"docs/*" = ["D1"], "tests/*" = ["D1"]} 117 | pydocstyle = {convention = "pep257"} 118 | flake8-type-checking = {exempt-modules = [], strict = true} 119 | 120 | # [tool.pyright] 121 | # include = ["conda_libmamba_solver"] 122 | # ignore = ["conda_libmamba_solver/*_v1.py"] 123 | # strict = ["**/"] 124 | 125 | [tool.coverage.report] 126 | exclude_also = [ 127 | "pragma: no cover", 128 | "if TYPE_CHECKING:", 129 | "\\.\\.\\.", 130 | ] 131 | -------------------------------------------------------------------------------- /tests/channel_testing/helpers.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2019 QuantStack and the Mamba contributors. 2 | # Copyright (C) 2022 Anaconda, Inc 3 | # Copyright (C) 2023 conda 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | from __future__ import annotations 6 | 7 | import os 8 | import pathlib 9 | import re 10 | import socket 11 | import sys 12 | from typing import TYPE_CHECKING 13 | 14 | import pytest 15 | from xprocess import ProcessStarter 16 | 17 | if TYPE_CHECKING: 18 | from xprocess import XProcess 19 | 20 | 21 | def _dummy_http_server( 22 | xprocess: XProcess, 23 | name, 24 | port, 25 | auth="none", 26 | user=None, 27 | password=None, 28 | token=None, 29 | path=None, 30 | ): 31 | """ 32 | Adapted from 33 | https://github.com/mamba-org/powerloader/blob/effe2b7e1/test/helpers.py#L11 34 | """ 35 | curdir = pathlib.Path(__file__).parent 36 | print("Starting dummy_http_server") 37 | 38 | if not path: 39 | path = curdir / ".." / "data" / "mamba_repo" 40 | 41 | class Starter(ProcessStarter): 42 | pattern = r"Server started at localhost:(\d+)" 43 | terminate_on_interrupt = True 44 | timeout = 10 45 | args = [ 46 | sys.executable, 47 | "-u", # unbuffered 48 | str(curdir / "reposerver.py"), 49 | "-d", 50 | str(path), 51 | "--port", 52 | str(port), 53 | ] 54 | if auth == "token": 55 | assert token 56 | args += ["--token", token] 57 | elif auth: 58 | args += ["--auth", auth] 59 | env = os.environ.copy() 60 | env["PYTHONUNBUFFERED"] = "1" 61 | if user and password: 62 | env["TESTPWD"] = f"{user}:{password}" 63 | 64 | def startup_check(self): # type: ignore 65 | nonlocal xprocess, port 66 | info = xprocess.getinfo(name) 67 | loglines: str = info.logpath.read_text(encoding="utf-8") 68 | for line in reversed(loglines.splitlines()): 69 | match = re.search(self.pattern, line) # type: ignore 70 | if match: 71 | port = int(match.group(1)) 72 | break 73 | s = socket.socket() 74 | address = "localhost" 75 | error = False 76 | try: 77 | s.connect((address, port)) 78 | except Exception as e: 79 | print(f"something's wrong with {address}:{port}. Exception is {e}") 80 | error = True 81 | finally: 82 | s.close() 83 | 84 | return not error 85 | 86 | logfile = xprocess.ensure(name, Starter) 87 | print("Logfile at", logfile) 88 | 89 | if user and password: 90 | yield f"http://{user}:{password}@localhost:{port}" 91 | elif token: 92 | yield f"http://localhost:{port}/t/{token}" 93 | else: 94 | yield f"http://localhost:{port}" 95 | 96 | xprocess.getinfo(name).terminate() 97 | 98 | 99 | @pytest.fixture 100 | def http_server_auth_none(xprocess): 101 | yield from _dummy_http_server(xprocess, name="http_server_auth_none", port=8000, auth="none") 102 | 103 | 104 | @pytest.fixture 105 | def http_server_auth_none_debug_repodata(xprocess): 106 | yield from _dummy_http_server( 107 | xprocess, 108 | name="http_server_auth_none_debug_repodata", 109 | port=8000, 110 | auth="none-debug-repodata", 111 | ) 112 | 113 | 114 | @pytest.fixture 115 | def http_server_auth_none_debug_packages(xprocess): 116 | yield from _dummy_http_server( 117 | xprocess, 118 | name="http_server_auth_none_debug_packages", 119 | port=8000, 120 | auth="none-debug-packages", 121 | ) 122 | 123 | 124 | @pytest.fixture 125 | def http_server_auth_basic(xprocess): 126 | yield from _dummy_http_server( 127 | xprocess, 128 | name="http_server_auth_basic", 129 | port=8000, 130 | auth="basic", 131 | user="user", 132 | password="test", 133 | ) 134 | 135 | 136 | @pytest.fixture 137 | def http_server_auth_basic_email(xprocess): 138 | yield from _dummy_http_server( 139 | xprocess, 140 | name="http_server_auth_basic_email", 141 | port=8000, 142 | auth="basic", 143 | user="user@email.com", 144 | password="test", 145 | ) 146 | 147 | 148 | @pytest.fixture 149 | def http_server_auth_token(xprocess): 150 | yield from _dummy_http_server( 151 | xprocess, 152 | name="http_server_auth_token", 153 | port=8000, 154 | auth="token", 155 | token="xy-12345678-1234-1234-1234-123456789012", 156 | ) 157 | -------------------------------------------------------------------------------- /tests/data/mambaforge.linux-64.lock: -------------------------------------------------------------------------------- 1 | # This file may be used to create an environment using: 2 | # $ conda create --name --file 3 | # platform: linux-64 4 | # channels: conda-forge 5 | @EXPLICIT 6 | https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 7 | https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2020.12.5-ha878542_0.tar.bz2 8 | https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.35.1-hea4e1c9_2.tar.bz2 9 | https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-9.3.0-h6de172a_18.tar.bz2 10 | https://conda.anaconda.org/conda-forge/linux-64/libgomp-9.3.0-h2828fa1_18.tar.bz2 11 | https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-1_gnu.tar.bz2 12 | https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-9.3.0-h2828fa1_18.tar.bz2 13 | https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2 14 | https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.17.1-h7f98852_1.tar.bz2 15 | https://conda.anaconda.org/conda-forge/linux-64/icu-68.1-h58526e2_0.tar.bz2 16 | https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2 17 | https://conda.anaconda.org/conda-forge/linux-64/libffi-3.3-h58526e2_2.tar.bz2 18 | https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.16-h516909a_0.tar.bz2 19 | https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.3-h9c3ff4c_0.tar.bz2 20 | https://conda.anaconda.org/conda-forge/linux-64/lzo-2.10-h516909a_1000.tar.bz2 21 | https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.2-h58526e2_4.tar.bz2 22 | https://conda.anaconda.org/conda-forge/linux-64/openssl-1.1.1j-h7f98852_0.tar.bz2 23 | https://conda.anaconda.org/conda-forge/linux-64/reproc-14.2.1-h36c2ea0_0.tar.bz2 24 | https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.5-h516909a_1.tar.bz2 25 | https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h516909a_0.tar.bz2 26 | https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.11-h516909a_1010.tar.bz2 27 | https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2 28 | https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.43.0-h812cca2_0.tar.bz2 29 | https://conda.anaconda.org/conda-forge/linux-64/libsolv-0.7.17-h780b84a_0.tar.bz2 30 | https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.9.0-ha56f1ee_6.tar.bz2 31 | https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.9.10-h72842e0_3.tar.bz2 32 | https://conda.anaconda.org/conda-forge/linux-64/readline-8.0-he28a2e2_2.tar.bz2 33 | https://conda.anaconda.org/conda-forge/linux-64/reproc-cpp-14.2.1-h58526e2_0.tar.bz2 34 | https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.10-h21135ba_1.tar.bz2 35 | https://conda.anaconda.org/conda-forge/linux-64/zstd-1.4.9-ha95c52a_0.tar.bz2 36 | https://conda.anaconda.org/conda-forge/linux-64/krb5-1.17.2-h926e7f8_0.tar.bz2 37 | https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.5.1-h3f442fb_1.tar.bz2 38 | https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.34.0-h74cdb3f_0.tar.bz2 39 | https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.75.0-hc4aaa36_0.tar.bz2 40 | https://conda.anaconda.org/conda-forge/linux-64/python-3.7.10-hffdb5ce_100_cpython.tar.bz2 41 | https://conda.anaconda.org/conda-forge/noarch/idna-2.10-pyh9f0ad1d_0.tar.bz2 42 | https://conda.anaconda.org/conda-forge/noarch/pycparser-2.20-pyh9f0ad1d_2.tar.bz2 43 | https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.7-1_cp37m.tar.bz2 44 | https://conda.anaconda.org/conda-forge/noarch/six-1.15.0-pyh9f0ad1d_0.tar.bz2 45 | https://conda.anaconda.org/conda-forge/noarch/tqdm-4.59.0-pyhd8ed1ab_0.tar.bz2 46 | https://conda.anaconda.org/conda-forge/noarch/wheel-0.36.2-pyhd3deb0d_0.tar.bz2 47 | https://conda.anaconda.org/conda-forge/linux-64/certifi-2020.12.5-py37h89c1867_1.tar.bz2 48 | https://conda.anaconda.org/conda-forge/linux-64/cffi-1.14.5-py37hc58025e_0.tar.bz2 49 | https://conda.anaconda.org/conda-forge/linux-64/chardet-4.0.0-py37h89c1867_1.tar.bz2 50 | https://conda.anaconda.org/conda-forge/linux-64/conda-package-handling-1.7.2-py37hb5d75c8_0.tar.bz2 51 | https://conda.anaconda.org/conda-forge/linux-64/pycosat-0.6.3-py37h5e8e339_1006.tar.bz2 52 | https://conda.anaconda.org/conda-forge/linux-64/pysocks-1.7.1-py37h89c1867_3.tar.bz2 53 | https://conda.anaconda.org/conda-forge/linux-64/ruamel_yaml-0.15.80-py37h5e8e339_1004.tar.bz2 54 | https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py37h5e8e339_1001.tar.bz2 55 | https://conda.anaconda.org/conda-forge/linux-64/cryptography-3.4.5-py37h5d9358c_1.tar.bz2 56 | https://conda.anaconda.org/conda-forge/linux-64/setuptools-49.6.0-py37h89c1867_3.tar.bz2 57 | https://conda.anaconda.org/conda-forge/noarch/pip-21.0.1-pyhd8ed1ab_0.tar.bz2 58 | https://conda.anaconda.org/conda-forge/noarch/pyopenssl-20.0.1-pyhd8ed1ab_0.tar.bz2 59 | https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.3-pyhd8ed1ab_0.tar.bz2 60 | https://conda.anaconda.org/conda-forge/noarch/requests-2.25.1-pyhd3deb0d_0.tar.bz2 61 | https://conda.anaconda.org/conda-forge/linux-64/conda-4.9.2-py37h89c1867_0.tar.bz2 62 | https://conda.anaconda.org/conda-forge/linux-64/mamba-0.8.0-py37h7f483ca_0.tar.bz2 63 | -------------------------------------------------------------------------------- /.github/workflows/update.yml: -------------------------------------------------------------------------------- 1 | name: Update Repository 2 | 3 | on: 4 | # every Sunday at 00:36 UTC 5 | # https://crontab.guru/#36_2_*_*_0 6 | schedule: 7 | - cron: 36 2 * * 0 8 | 9 | workflow_dispatch: 10 | 11 | issue_comment: 12 | types: 13 | - created 14 | 15 | jobs: 16 | update: 17 | if: >- 18 | !github.event.repository.fork 19 | && ( 20 | github.event_name == 'schedule' 21 | || github.event_name == 'workflow_dispatch' 22 | || ( 23 | github.event_name == 'issue_comment' 24 | && github.event.issue.pull_request 25 | && ( 26 | github.event.comment.body == '@conda-bot render' 27 | || github.event.comment.body == '@conda-bot recreate' 28 | ) 29 | ) 30 | ) 31 | runs-on: ubuntu-latest 32 | steps: 33 | - if: github.event_name == 'issue_comment' 34 | uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 35 | with: 36 | comment-id: ${{ github.event.comment.id }} 37 | reactions: eyes 38 | reactions-edit-mode: replace 39 | token: ${{ secrets.SYNC_TOKEN }} 40 | 41 | - if: github.event.comment.body == '@conda-bot render' 42 | name: Configure git origin 43 | run: | 44 | echo REPOSITORY=$(curl --silent ${{ github.event.issue.pull_request.url }} | jq --raw-output '.head.repo.full_name') >> $GITHUB_ENV 45 | echo REF=$(curl --silent ${{ github.event.issue.pull_request.url }} | jq --raw-output '.head.ref') >> $GITHUB_ENV 46 | 47 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 48 | with: 49 | repository: ${{ env.REPOSITORY || github.repository }} 50 | ref: ${{ env.REF || '' }} 51 | token: ${{ secrets.SYNC_TOKEN }} 52 | 53 | - name: Configure git user 54 | run: | 55 | git config --global user.name 'Conda Bot' 56 | git config --global user.email '18747875+conda-bot@users.noreply.github.com' 57 | 58 | - uses: conda/actions/combine-durations@f05161c6e6e37a49b17c8e0b436197b53830318a # v25.9.2 59 | id: durations 60 | continue-on-error: true 61 | 62 | - uses: conda/actions/template-files@f05161c6e6e37a49b17c8e0b436197b53830318a # v25.9.2 63 | id: templates 64 | continue-on-error: true 65 | 66 | - name: Commit changes 67 | # no-op if there are no updates 68 | continue-on-error: true 69 | run: | 70 | git add . 71 | git commit --message "🤖 updated file(s)" 72 | 73 | - if: github.event.comment.body != '@conda-bot render' 74 | name: Create fork 75 | # no-op if the repository is already forked 76 | run: echo FORK=$(gh repo fork --clone=false --default-branch-only 2>&1 | awk '{print $1}') >> $GITHUB_ENV 77 | env: 78 | GH_TOKEN: ${{ secrets.SYNC_TOKEN }} 79 | 80 | - if: github.event.comment.body != '@conda-bot render' 81 | id: create 82 | # no-op if no commits were made 83 | uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7.0.11 84 | with: 85 | push-to-fork: ${{ env.FORK }} 86 | token: ${{ secrets.SYNC_TOKEN }} 87 | branch: update 88 | delete-branch: true 89 | title: 🤖 Update infrastructure file(s) 90 | body: | 91 | [update.yml]: ${{ github.server_url }}/${{ github.repository }}/blob/main/.github/workflows/update.yml 92 | 93 | Your friendly repository updater. 94 | 95 | ${{ steps.durations.outputs.summary }} 96 | 97 | ${{ steps.templates.outputs.summary }} 98 | 99 | This PR was triggered by @${{ github.triggering_actor }} via ${{ github.event_name }}. 100 | 101 |
102 | Commands 103 | 104 | Trigger actions by commenting on this PR: 105 | 106 | - `@conda-bot render` will run rendering workflows and commit and push any changes to this PR 107 | - `@conda-bot recreate` will recreate this PR, overwriting any edits that have been made to it 108 | 109 |
110 | 111 | ###### Auto-generated by the [`update.yml`][update.yml] workflow, see ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. 112 | 113 | - if: github.event.comment.body == '@conda-bot render' 114 | id: update 115 | name: Push changes 116 | run: git push --force-with-lease 117 | 118 | - if: always() && github.event_name == 'issue_comment' 119 | uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 120 | with: 121 | comment-id: ${{ github.event.comment.id }} 122 | reactions: ${{ (steps.create.conclusion == 'success' || steps.update.conclusion == 'success') && 'hooray' || 'confused' }} 123 | reactions-edit-mode: replace 124 | token: ${{ secrets.SYNC_TOKEN }} 125 | -------------------------------------------------------------------------------- /dev/scripts/BenchmarkChart.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "e28dc53c-5b9c-453d-a9ef-14dd93b85b46", 6 | "metadata": {}, 7 | "source": [ 8 | "# Benchmark Chart\n", 9 | "This notebook reads .codspeed benchmarks generated in the root of conda-libmamba-solver with `pytest -k benchmark --codspeed` e.g. and renders `test_traversal_algorithm_benchmarks` as a plotly chart.\n", 10 | "\n", 11 | "`jupyter nbconvert --clear-output --inplace .ipynb` clears output from the file for better revision control." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "id": "f50f3e70-fc44-44db-b770-033b3cac9576", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "import json\n", 22 | "import os\n", 23 | "from pathlib import Path\n", 24 | "\n", 25 | "import pandas as pd\n", 26 | "import plotly.express as px" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "id": "f410cf7e-344e-4e0b-939c-a7c3836c1c9e", 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "runs = Path(\".codspeed\").glob(\"*.json\")\n", 37 | "most_recent = sorted(runs, key=lambda f: os.stat(f).st_mtime)[-1]\n", 38 | "print(most_recent)" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "id": "3961cb90-b345-43c1-9e13-d3f5d3cecba5", 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "SCENARIOS = [\n", 49 | " \"python\",\n", 50 | " \"data_science_basic\",\n", 51 | " \"data_science_ml\",\n", 52 | " \"web_development\",\n", 53 | " \"scientific_computing\",\n", 54 | " \"devops_automation\",\n", 55 | " \"vaex\",\n", 56 | "]\n", 57 | "\n", 58 | "benchmarks = json.loads(most_recent.read_text())[\"benchmarks\"]\n", 59 | "traversal_benchmarks = []\n", 60 | "for benchmark in benchmarks:\n", 61 | " name = benchmark[\"name\"]\n", 62 | " for scenario in SCENARIOS:\n", 63 | " name = name.replace(scenario.replace(\"_\", \"-\"), scenario.replace(\"-\", \"_\"))\n", 64 | " if \"test_traversal_algorithm_benchmarks\" not in name:\n", 65 | " continue\n", 66 | " stats = benchmark[\"stats\"]\n", 67 | " assert len(name.split(\"[\")[-1].strip(\"]\").split(\"-\")) == 4\n", 68 | " facets = dict(\n", 69 | " zip([\"main\", \"scenario\", \"alg\", \"cache\"], name.split(\"[\")[-1].strip(\"]\").split(\"-\"))\n", 70 | " )\n", 71 | " stats[\"min_s\"] = stats[\"min_ns\"] / 1e9\n", 72 | " traversal_benchmarks.append(stats | facets)" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "id": "109c7077-b54c-4a0d-b202-e3aba18af10f", 79 | "metadata": {}, 80 | "outputs": [], 81 | "source": [ 82 | "df = pd.DataFrame(traversal_benchmarks)" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "id": "d9f25b8e-9a3d-4ed7-a73b-7d4c042799a5", 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "fig = px.bar(\n", 93 | " df,\n", 94 | " x=\"main\",\n", 95 | " y=\"min_s\",\n", 96 | " color=\"alg\",\n", 97 | " barmode=\"group\",\n", 98 | " facet_row=\"scenario\",\n", 99 | " facet_col=\"cache\",\n", 100 | " category_orders={\n", 101 | " \"cache\": [\"cold\", \"warm\", \"lukewarm\"],\n", 102 | " \"alg\": [\"bfs\", \"pipelined\", \"httpx\"],\n", 103 | " \"main\": [\"main\"],\n", 104 | " },\n", 105 | " width=1000,\n", 106 | " height=1600,\n", 107 | ")\n", 108 | "fig.show()" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "id": "a4d02d95-a8c1-46bf-881c-7b6215fc9626", 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "df[df[\"alg\"] == \"pipelined\"].sum()[\"min_ns\"] / df[df[\"alg\"] == \"httpx\"].sum()[\"min_ns\"]" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "id": "e5ed7f9f-eec2-4a66-9f0a-2ac4ed49642d", 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "df[df[\"alg\"] == \"bfs\"].sum()[\"min_ns\"] / df[df[\"alg\"] == \"httpx\"].sum()[\"min_ns\"]" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "id": "f5611550-ca78-407c-9680-1ec27eb7db80", 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [] 138 | } 139 | ], 140 | "metadata": { 141 | "kernelspec": { 142 | "display_name": "Python 3 (ipykernel)", 143 | "language": "python", 144 | "name": "python3" 145 | }, 146 | "language_info": { 147 | "codemirror_mode": { 148 | "name": "ipython", 149 | "version": 3 150 | }, 151 | "file_extension": ".py", 152 | "mimetype": "text/x-python", 153 | "name": "python", 154 | "nbconvert_exporter": "python", 155 | "pygments_lexer": "ipython3", 156 | "version": "3.13.9" 157 | } 158 | }, 159 | "nbformat": 4, 160 | "nbformat_minor": 5 161 | } 162 | -------------------------------------------------------------------------------- /conda_libmamba_solver/shards_cache.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | """ 5 | Cache suitable for shards, not allowed to change because they are named 6 | after their own sha256 hash. 7 | """ 8 | 9 | from __future__ import annotations 10 | 11 | import logging 12 | import sqlite3 13 | from dataclasses import dataclass 14 | from typing import TYPE_CHECKING 15 | 16 | import msgpack 17 | import zstandard 18 | from conda.gateways.disk.delete import unlink_or_rename_to_trash 19 | 20 | if TYPE_CHECKING: 21 | from pathlib import Path 22 | 23 | from .shards_typing import ShardDict 24 | 25 | log = logging.getLogger(__name__) 26 | 27 | SHARD_CACHE_NAME = "repodata_shards.db" 28 | ZSTD_MAX_SHARD_SIZE = 2**20 * 16 # maximum size necessary when compresed data has no size header 29 | 30 | 31 | @dataclass 32 | class AnnotatedRawShard: 33 | def __init__(self, url: str, package: str, compressed_shard: bytes): 34 | # prevent easy mistake of swapping url, package 35 | assert "://" in url 36 | assert "://" not in package 37 | 38 | self.url = url 39 | self.package = package # remove this field to avoid confusion? 40 | self.compressed_shard = compressed_shard 41 | 42 | url: str 43 | package: str 44 | compressed_shard: bytes 45 | 46 | 47 | def connect(dburi="cache.db"): 48 | """ 49 | Get database connection. 50 | 51 | dburi: uri-style sqlite database filename; accepts certain ?= parameters. 52 | """ 53 | conn = sqlite3.connect(dburi, uri=True) 54 | conn.row_factory = sqlite3.Row 55 | conn.execute("PRAGMA foreign_keys = ON") 56 | return conn 57 | 58 | 59 | class ShardCache: 60 | """ 61 | Handle caching for individual shards (not the index of shards). 62 | """ 63 | 64 | def __init__(self, base: Path, create=True): 65 | """ 66 | base: directory and filename prefix for cache. 67 | """ 68 | self.base = base 69 | self.connect() 70 | 71 | def copy(self): 72 | """ 73 | Copy cache with new connection. Useful for threads. 74 | """ 75 | return ShardCache(self.base, create=False) 76 | 77 | def connect(self, create=True): 78 | """ 79 | Args: 80 | create: if True, create table if not exists. 81 | """ 82 | dburi = (self.base / SHARD_CACHE_NAME).as_uri() 83 | self.conn = connect(dburi) 84 | if not create: 85 | return 86 | # this schema will also get confused if we merge packages into a single 87 | # shard, but the package name should be advisory. 88 | self.conn.execute( 89 | "CREATE TABLE IF NOT EXISTS shards (" 90 | "url TEXT PRIMARY KEY, package TEXT, shard BLOB, " 91 | "timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP)" 92 | ) 93 | 94 | def insert(self, raw_shard: AnnotatedRawShard): 95 | """ 96 | Args: 97 | url: of shard 98 | package: package name 99 | raw_shard: msgpack.zst compressed shard data 100 | """ 101 | # decompress and return shard for convenience, also to validate? unless 102 | # caller would rather retrieve the shard from another thread. 103 | with self.conn as c: 104 | c.execute( 105 | "INSERT OR IGNORE INTO SHARDS (url, package, shard) VALUES (?, ?, ?)", 106 | (raw_shard.url, raw_shard.package, raw_shard.compressed_shard), 107 | ) 108 | 109 | def retrieve(self, url) -> ShardDict | None: 110 | with self.conn as c: 111 | row = c.execute("SELECT shard FROM shards WHERE url = ?", (url,)).fetchone() 112 | return ( 113 | msgpack.loads( 114 | zstandard.decompress(row["shard"], max_output_size=ZSTD_MAX_SHARD_SIZE) 115 | ) 116 | if row 117 | else None 118 | ) # type: ignore 119 | 120 | def retrieve_multiple(self, urls: list[str]) -> dict[str, ShardDict | None]: 121 | """ 122 | Query database for cached shard urls. 123 | 124 | Return a dict of urls in cache mapping to the Shard or None if not present. 125 | """ 126 | if not urls: 127 | return {} # this optimization does not save a noticeable amount of time. 128 | 129 | # In one test reusing the context saves difference between .006s and .01s 130 | # We could make this a threadlocal. 131 | dctx = zstandard.ZstdDecompressor() 132 | 133 | query = f"SELECT url, shard FROM shards WHERE url IN ({','.join(('?',) * len(urls))}) ORDER BY url" 134 | with self.conn as c: 135 | result: dict[str, ShardDict | None] = { 136 | row["url"]: msgpack.loads( 137 | dctx.decompress(row["shard"], max_output_size=ZSTD_MAX_SHARD_SIZE) 138 | ) 139 | if row 140 | else None 141 | for row in c.execute(query, urls) # type: ignore 142 | } 143 | return result 144 | 145 | def clear_cache(self): 146 | """ 147 | Truncate the database by removing all rows from tables 148 | """ 149 | with self.conn as c: 150 | c.execute("DELETE FROM shards") 151 | 152 | def remove_cache(self): 153 | """ 154 | Remove the sharded cache database. 155 | """ 156 | # This function appears to support `Path()` except on Windows 157 | # `os.rename(path, path + ".conda_trash")` fails: 158 | self.conn.close() 159 | unlink_or_rename_to_trash(str(self.base / SHARD_CACHE_NAME)) 160 | -------------------------------------------------------------------------------- /docs/dev/setup.md: -------------------------------------------------------------------------------- 1 | # How to set up your development environment 2 | 3 | ## With `devcontainer` in VS Code 4 | 5 | The development workflow is streamlined for Linux thanks to the `devcontainer` configuration 6 | bundled in this repository. You'll need Docker and VS Code with the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers): 7 | 8 | 1. Clone `conda/conda`, `mamba-org/mamba` and `conda/conda-libmamba-solver` to your preferred locations 9 | (e.g. `~/devel/conda`, `~/devel/mamba` and `~/devel/conda-libmamba-solver`, respectively). 10 | The location does not matter as long as all repositories have the same parent directory. 11 | 2. Open your `conda-libmamba-solver` clone with VS Code. 12 | 3. Connect to the DevContainer image via the bottom-left menu (❱❰) and 13 | click on "Reopen in Container". Pick one of the suggested configurations: 14 | conda-forge or defaults. The only difference is the base installation (Miniforge and Miniconda, 15 | respectively). 16 | 4. The image will be built and after a couple minutes, you'll be dropped into a Bash shell. Enjoy! 17 | Since the local repositories are mounted, you can make modifications to the source live, 18 | and they will be reflected in the Docker instance automatically. 19 | Run `pytest` or `conda` as needed, no need to reload Docker! 20 | 5. If the development environment breaks, click again on ❱❰ and, this time, choose 21 | "Rebuild container". You might need to Retry a couple times. 22 | 23 | ```{note} Developing libmamba 24 | The devcontainer configuration also supports libmamba 1.x development. You just need to have the 25 | `mamba-org/mamba` repository (branch `1.x`) cloned next to `conda` and `conda-libmamba-solver`. 26 | Once the container has started, run `develop-mamba` to set it up. 27 | If you are modifying C++ sources, re-run `develop-mamba` to rebuild the libraries. 28 | ``` 29 | 30 | ## With regular Docker 31 | 32 | You can reuse the devcontainer scripts with regular Docker too. 33 | 34 | 1. Clone `conda/conda`, `mamba-org/mamba` and `conda/conda-libmamba-solver` to your preferred locations 35 | (e.g. `~/devel/conda`, `~/devel/mamba` and `~/devel/conda-libmamba-solver`, respectively). 36 | The location does not matter as long as all repositories have the same parent directory. 37 | 2. Start a new Docker instance with this command. Adjust the local mounts as necessary. 38 | 39 | ```bash 40 | # For defaults-based images, use: 41 | docker run -it --rm \ 42 | -v ~/devel/conda:/workspaces/conda \ 43 | -v ~/devel/mamba:/workspaces/mamba \ 44 | -v ~/devel/conda-libmamba-solver:/workspaces/conda-libmamba-solver \ 45 | continuumio/miniconda3:latest \ 46 | bash 47 | # For conda-forge-based images, use the following instead: 48 | docker run -it --rm \ 49 | -v ~/devel/conda:/workspaces/conda \ 50 | -v ~/devel/mamba:/workspaces/mamba \ 51 | -v ~/devel/conda-libmamba-solver:/workspaces/conda-libmamba-solver \ 52 | condaforge/miniforge3:latest \ 53 | bash 54 | ``` 55 | 3. Run the `post_create` and `post_start` scripts: 56 | ```bash 57 | bash /workspaces/conda-libmamba-solver/.devcontainer/post_create.sh 58 | bash /workspaces/conda-libmamba-solver/.devcontainer/post_start.sh 59 | ``` 60 | 4. If you want to develop with mamba in editable mode, run: 61 | ```bash 62 | source ~/.bashrc 63 | develop-mamba 64 | ``` 65 | 66 | ## General workflow 67 | 68 | We strongly suggest you start with the Docker-based workflow above. 69 | It is a better development experience with a fully disposable environment. 70 | However, sometimes you might need to debug issues for non-Linux installations. 71 | In that case, you can follow these general instructions, 72 | but be careful with overwriting your existing `conda` installations, 73 | especially when it comes to `shell` initialization! 74 | 75 | 1. Get yourself familiar with the ["Development environment" guide for `conda` itself][conda_dev]. 76 | 77 | 2. Fork and clone the `conda-libmamba-solver` repository to your preferred location: 78 | 79 | ```bash 80 | git clone "git@github.com:$YOUR_USERNAME/conda-libmamba-solver" "$REPO_LOCATION" 81 | cd "$REPO_LOCATION" 82 | ``` 83 | 84 | 3. Install the required dependencies for `conda-libmamba-solver`: 85 | 86 | ```bash 87 | conda install \ 88 | --file "$REPO_LOCATION"/dev/requirements.txt \ 89 | --file "$REPO_LOCATION"/tests/requirements.txt 90 | ``` 91 | 92 | 4. Install `conda-libmamba-solver` with `pip`: 93 | 94 | ```bash 95 | cd $REPO_LOCATION 96 | python -m pip install --no-deps -e . 97 | ``` 98 | 99 | ## Debugging `conda` and `conda-libmamba-solver` 100 | 101 | Once you have followed the steps described in the general workflow 102 | above you may need to investigate the state in a particular 103 | point. Insert a 104 | [`breakpoint()`](https://docs.python.org/3/library/pdb.html) within 105 | the code and run a test or conda directly to hit the breakpoint. 106 | 107 | ## Debugging Mamba 108 | 109 | While debugging the conda workflows only requires modifying python 110 | code and running conda. Debugging the mamba code requires 111 | recompilation and is not as easy to jump into a debugger to 112 | investigate state. 113 | 114 | 1. Get familiar with the ["Local development" guide for `mamba` itself][mamba_dev]. 115 | 116 | 2. Fork and clone the `mamba` repository to your preferred location: 117 | 118 | ```bash 119 | git clone "git@github.com:$YOUR_USERNAME/mamba" "$REPO_LOCATION" 120 | cd $REPO_LOCATION 121 | ``` 122 | 123 | 3. Use the Docker image for development suggested above and re-run 124 | `develop-mamba` whenever you make change to `mamba` in 125 | `$REPO_LOCATION`. This should take less than a minute. 126 | 127 | We recommend debugging via either breakpoints and using `gdb` or `print` 128 | statements via `std::cout << ... << std::endl`. The 129 | [following](https://github.com/costrouc/mamba/commit/99ac04ee9ca26c9579c67816cfba25bf310c30fb) 130 | shows an example of inserting print statements into the `libmamba` 131 | source in order to debug the [libsolv](https://github.com/openSUSE/libsolv) state. 132 | 133 | 134 | 135 | [conda_dev]: https://docs.conda.io/projects/conda/en/latest/dev-guide/development-environment.html 136 | [mamba_dev]: https://mamba.readthedocs.io/en/latest/developer_zone/build_locally.html 137 | -------------------------------------------------------------------------------- /tests/repodata_time_machine.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2023 conda 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | """ 4 | conda repodata time machine 5 | 6 | Given a date and a channel, this script will: 7 | 8 | - Download a local copy of the (unpatched) repodata 9 | - Trim to the closest timestamp 10 | - Download the closest repodata patches for that channel 11 | - Apply the patches 12 | - Generate a ready-to-use local channel 13 | """ 14 | 15 | import bz2 16 | import json 17 | import os 18 | import urllib.request 19 | from argparse import ArgumentParser 20 | from datetime import datetime 21 | 22 | import requests 23 | from conda.base.context import context 24 | from conda.models.channel import Channel 25 | from conda_index.index import _apply_instructions 26 | from conda_package_handling.api import extract as cph_extract 27 | 28 | PATCHED_CHANNELS = {"defaults", "main", "conda-forge"} 29 | 30 | 31 | def cli(): 32 | p = ArgumentParser() 33 | p.add_argument("channels", nargs="+", metavar="channel") 34 | p.add_argument("-t", "--timestamp", required=True, help="YYYY-MM-DD HH:MM:SS. Assumes UTC.") 35 | p.add_argument( 36 | "-s", 37 | "--subdirs", 38 | default=f"{context.subdir},noarch", 39 | help="Comma-separated list of subdirs to download. Include 'noarch' explicitly if needed.", 40 | ) 41 | return p.parse_args() 42 | 43 | 44 | def download_repodata(channel, subdirs=None): 45 | "Download remote repodata JSON payload to a temporary location in disk" 46 | c = Channel(channel) 47 | if c.canonical_name in PATCHED_CHANNELS: 48 | repodata_fn = "repodata_from_packages" 49 | else: 50 | repodata_fn = "repodata" 51 | subdirs = subdirs or context.subdirs 52 | for url in c.urls(with_credentials=True, subdirs=subdirs): 53 | subdir = url.strip("/").split("/")[-1] 54 | urllib.request.urlretrieve(f"{url}/{repodata_fn}.json.bz2", f"{repodata_fn}.json.bz2") 55 | 56 | with open(f"{repodata_fn}.json.bz2", "rb") as f: 57 | with open(f"{repodata_fn}.json", "wb") as g: 58 | g.write(bz2.decompress(f.read())) 59 | 60 | yield f"{repodata_fn}.json", subdir 61 | 62 | 63 | def trim_to_timestamp(repodata, timestamp: float): 64 | trimmed_tar_pkgs = {} 65 | trimmed_conda_pkgs = {} 66 | with open(repodata) as f: 67 | data = json.load(f) 68 | for name, pkg in data["packages"].items(): 69 | if pkg.get("timestamp", 0) <= timestamp: 70 | trimmed_tar_pkgs[name] = pkg 71 | for name, pkg in data["packages.conda"].items(): 72 | if pkg.get("timestamp", 0) <= timestamp: 73 | trimmed_conda_pkgs[name] = pkg 74 | data["packages"] = trimmed_tar_pkgs 75 | data["packages.conda"] = trimmed_conda_pkgs 76 | fn = f"trimmed.{os.path.basename(repodata)}" 77 | with open(fn, "w") as f: 78 | json.dump(data, f) 79 | return fn 80 | 81 | 82 | def download_patches(channel, timestamp: float): 83 | name = Channel(channel).canonical_name 84 | if name != "conda-forge": 85 | raise NotImplementedError("Only conda-forge is supported for now") 86 | 87 | url = "https://api.anaconda.org/package/conda-forge/conda-forge-repodata-patches/files" 88 | r = requests.get(url) 89 | r.raise_for_status() 90 | pkgs = r.json() 91 | closest_older = None 92 | for pkg in sorted(pkgs, key=lambda pkg: pkg["attrs"]["timestamp"]): 93 | if pkg["attrs"]["timestamp"] <= timestamp: 94 | closest_older = pkg 95 | else: 96 | break 97 | if closest_older is None: 98 | raise ValueError(f"No patch found for timestamp {timestamp}") 99 | 100 | fn = closest_older["basename"].split("/")[-1] 101 | urllib.request.urlretrieve(f"https:{closest_older['download_url']}", fn) 102 | 103 | extract_path = f"conda-forge-repodata-patches-{closest_older['version']}" 104 | cph_extract(fn, dest_dir=extract_path) 105 | return extract_path 106 | 107 | 108 | def apply_patch(repodata_file, patch): 109 | with open(repodata_file) as f, open(patch) as g: 110 | repodata = json.load(f) 111 | instructions = json.load(g) 112 | fn = f"patched.{os.path.basename(repodata_file)}" 113 | with open(fn, "w") as f: 114 | patched = _apply_instructions(None, repodata, instructions) 115 | json.dump(patched, f, indent=2) 116 | return fn 117 | 118 | 119 | def repodata_time_machine(channels, timestamp_str, subdirs=None): 120 | horizon = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") 121 | timestamp = horizon.timestamp() * 1000 122 | original_dir = os.getcwd() 123 | try: 124 | workdir = f"repodata-{timestamp_str.replace(' ', '-').replace(':', '-').replace('.', '-')}" 125 | os.makedirs(workdir, exist_ok=True) 126 | os.chdir(workdir) 127 | # Download repodata 128 | for channel in channels: 129 | print("Rolling back", channel, "to", horizon) 130 | channel_name = Channel(channel).canonical_name 131 | os.makedirs(channel_name, exist_ok=True) 132 | os.chdir(channel_name) 133 | must_patch = channel_name in PATCHED_CHANNELS 134 | if must_patch: 135 | print(" Getting patches") 136 | patch_dir = os.path.abspath(download_patches(channel, timestamp)) 137 | for repodata, subdir in download_repodata(channel, subdirs=subdirs): 138 | print(" Downloaded", repodata, "for", subdir) 139 | print(" Trimming...") 140 | abs_repodata = os.path.abspath(repodata) 141 | os.makedirs(subdir, exist_ok=True) 142 | os.chdir(subdir) 143 | trimmed = trim_to_timestamp(abs_repodata, timestamp) 144 | if must_patch: 145 | print(" Patching...") 146 | instructions = f"{patch_dir}/{subdir}/patch_instructions.json" 147 | patched = apply_patch(trimmed, instructions) 148 | if not os.path.exists("repodata.json"): 149 | os.symlink(patched, "repodata.json") 150 | else: 151 | if not os.path.exists("repodata.json"): 152 | os.symlink(trimmed, "repodata.json") 153 | os.chdir("..") 154 | os.chdir("..") 155 | return workdir 156 | finally: 157 | os.chdir(original_dir) 158 | 159 | 160 | def main(): 161 | args = cli() 162 | return repodata_time_machine(args.channels, args.timestamp, args.subdirs.split(",")) 163 | 164 | 165 | if __name__ == "__main__": 166 | main() 167 | print("Done!") 168 | -------------------------------------------------------------------------------- /docs/dev/sharded.md: -------------------------------------------------------------------------------- 1 | # Sharded repodata implementation 2 | 3 | This document provides an overview on how `conda-libmamba-solver` implements 4 | [CEP-16 Sharded Repodata](https://conda.org/learn/ceps/cep-0016). 5 | 6 | Sharded repodata splits `repodata.json` into an index mapping package names to 7 | shard hashes in `repodata_shards.msgpack.zst`. A shard contains repodata for 8 | every package with a given name. Since shards are named after a hash of their 9 | contents, they can be cached without having to check the server for freshness. 10 | Individual shards only need to change when an individual package has changed, so 11 | only the much smaller index has to be re-fetched often. 12 | 13 | ## Sharded Repodata in conda-libmamba-solver 14 | 15 | For `conda-libmamba-solver`, we wanted a way to implement sharded repodata in 16 | Python without having to touch the C++ `libmamba`. 17 | 18 | We do this by treating all repodata as if it was sharded repodata. Starting with 19 | a list of installed packages and to-be-installed packages, we gather all 20 | repodata for those packages and look for all package names listed in their 21 | dependencies. We repeat the process for every discovered package name that we 22 | have not already visited, fetching repodata shards or examining all artifacts 23 | with that package name as found in monolithic `repodata.json`. This process 24 | gathers all versions of all packages that we might depend on. We do not consider 25 | package versions at this stage; that's the solver's job. 26 | 27 | As of this writing, `conda create -c conda-forge --dry-run python` finds 35 28 | package names; `conda` 137 package names, and `vaex`, a dataframe library with a 29 | complex dependency tree, 678 package names. That's a lot less than the 31k 30 | packages total according to https://conda-forge.org/, and a manageable number to 31 | pre-process in Python before doing a solve with `libmamba`. As long as we can 32 | fetch those packages quickly enough, from cache or from the network, we will 33 | save RAM, disk space, bandwidth and time compared to parsing every package on 34 | the channel every time. 35 | 36 | ### Threading and concurrency 37 | 38 | In order to achieve concurrency, our sharded repodata implementation uses 39 | the Python [threading module](https://docs.python.org/3/library/threading.html). 40 | We have two separate thread workers for fetching cache and network data. These 41 | threads communicate to each other via the following queues: 42 | 43 | - **cache_in_queue** every requested shard goes here first where the cache 44 | worker sees if we have a valid cache record. 45 | - **cache_miss_queue** for every shard not in cache, we send it this queue where 46 | the network worker thread downloads it. 47 | - **shard_out_queue** once a shard has been fetched from either the cache or 48 | network worker threads, it is placed here so we can gather all needed 49 | shards at the end to build our repodata subset. 50 | 51 | :::{mermaid} 52 | 53 | sequenceDiagram 54 | loop 55 | Main ->> Main: "Fetch" in-memory shard 56 | Main ->> Cache: Fetch shard 57 | Cache ->> Network: Cache miss 58 | Cache ->> Main: Cache hit 59 | Network ->> Main: Network result 60 | Main ->> Main: Find new (channel, package) from shard data 61 | end 62 | 63 | ::: 64 | 65 | ## Source code 66 | 67 | The shard handling code is split into `shards.py`, `shards_cache.py`, 68 | `shards_subset.py` and `shards_typing.py` in `conda_libmamba_solver/`. 69 | Additional code in `conda_libmamba_solver/index.py` calls 70 | `build_repodata_subset()` and converts the resulting repodata to `libmamba` 71 | objects. 72 | 73 | ### `shards.py` 74 | 75 | `shards.py` provides an interface to treat sharded repodata and monolithic 76 | `repodata.json` in the same way. It checks a channel for sharded repodata, 77 | returning an object that implements the `ShardLike` interface. 78 | 79 | ### `shards_subset.py` 80 | 81 | `shards_subset.py` accepts a list of `ShardLike` instances and a list of initial 82 | packages to compute a repodata subset. The traversal is simplified thanks to the 83 | `ShardLike` interface, so the algorithm doesn't have to worry too much about the 84 | type of each channel. 85 | 86 | ### `shards_cache.py` 87 | 88 | `shards_cache.py` implements a sqlite3 cache used to store individual shards. 89 | When traversing shards, the cache is checked before making a network request. 90 | The shards cache is a single database for all channels in 91 | `$CONDA_PREFIX/pkgs/cache/repodata_shards.db`. 92 | 93 | The shards index `repodata_shards.msgpack.zst` is cached in the same way as 94 | `repodata.json`, in individual files in `$CONDA_PREFIX/pkgs/cache/` named after 95 | URL hashes. A `has_` remembers if a channel has shards, or not. If 96 | `has_shards` is `false` then we wait 7 days after `last_checked` to make another 97 | request looking for `repodata_shards.msgpack.zst`. The same system remembers 98 | whether a channel provides `repodata.json.zst`, and stores `ETag` and 99 | `Last-Modified` used to refresh the cache. 100 | 101 | ``` 102 | ... 103 | "has_shards": { 104 | "last_checked": "2025-10-15T17:19:44.408989Z", 105 | "value": true 106 | }, 107 | ``` 108 | 109 | ### `shards_typing.py` 110 | 111 | `shards_typing.py` provides type hints for data structures used in sharded 112 | repodata, but it is not normative; it only includes fields used by the sharded 113 | repodata system. 114 | 115 | ### `tests/test_shards.py` 116 | 117 | The sharded repodata tests maintain 100% code coverage of the shards-related code 118 | `shards*.py`. 119 | 120 | ## Example dependency graph for Python 121 | 122 | This is what Python's dependencies look like on `conda-forge` as of this writing. 123 | 124 | If sharded repodata is asked to install Python, we look for `python` in every 125 | active channel. The `python` shard(s) tells us we can fetch `bzip2`, `libffi`, 126 | `...` in parallel, discovering a third layer including `icu`, `ca-certificates`, 127 | and others. `ca-certificates` also depends on some virtual packages, but the 128 | traversal quickly determine that these packages don't appear in any channel by 129 | checking the `repodata_shards.msgpack.zst` index. The solver will let us know if 130 | these missing packages are a problem, virtual or no. 131 | 132 | The first draft of sharded repodata in `conda-libmamba-solver` literally 133 | generated classic `repodata.json` with package subsets to load into the solver, 134 | but now we convert each record into `libmamba` `PackageInfo` objects in memory. 135 | 136 | By giving `libmamba` every possible dependency for a specific request, it has 137 | enough information to produce a solution. 138 | 139 | :::{mermaid} shards_python.mmd 140 | -------------------------------------------------------------------------------- /tests/test_index.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022 Anaconda, Inc 2 | # Copyright (C) 2023 conda 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | from __future__ import annotations 5 | 6 | import json 7 | import shutil 8 | import time 9 | from pathlib import Path 10 | from typing import TYPE_CHECKING 11 | 12 | import pytest 13 | from conda.base.context import context, reset_context 14 | from conda.common.compat import on_win 15 | from conda.core.subdir_data import SubdirData 16 | from conda.gateways.logging import initialize_logging 17 | from conda.models.channel import Channel 18 | 19 | from conda_libmamba_solver.index import LibMambaIndexHelper, _is_sharded_repodata_enabled 20 | from conda_libmamba_solver.state import SolverInputState 21 | 22 | from .test_shards import CONDA_FORGE_WITH_SHARDS 23 | 24 | if TYPE_CHECKING: 25 | import os 26 | 27 | from conda.testing.fixtures import CondaCLIFixture 28 | from pytest_benchmark.plugin import BenchmarkFixture 29 | 30 | 31 | initialize_logging() 32 | DATA = Path(__file__).parent / "data" 33 | 34 | 35 | def test_given_channels(monkeypatch: pytest.MonkeyPatch, tmp_path: os.PathLike): 36 | monkeypatch.setenv("CONDA_PKGS_DIRS", str(tmp_path)) 37 | reset_context() 38 | libmamba_index = LibMambaIndexHelper.from_platform_aware_channel( 39 | channel=Channel("conda-test/noarch") 40 | ) 41 | assert libmamba_index.db.repo_count() == 1 42 | 43 | conda_index = SubdirData(Channel("conda-test/noarch")) 44 | conda_index.load() 45 | 46 | assert libmamba_index.db.package_count() == len(tuple(conda_index.iter_records())) 47 | 48 | 49 | @pytest.mark.parametrize( 50 | "only_tar_bz2", 51 | ( 52 | pytest.param("1", id="CONDA_USE_ONLY_TAR_BZ2=true"), 53 | pytest.param("", id="CONDA_USE_ONLY_TAR_BZ2=false"), 54 | ), 55 | ) 56 | def test_defaults_use_only_tar_bz2(monkeypatch: pytest.MonkeyPatch, only_tar_bz2: bool): 57 | """ 58 | Defaults is particular in the sense that it offers both .tar.bz2 and .conda for LOTS 59 | of packages. SubdirData ignores .tar.bz2 entries if they have a .conda counterpart. 60 | So if we count all the packages in each implementation, libmamba's has way more. 61 | To remain accurate, we test this with `use_only_tar_bz2`: 62 | - When true, we only count .tar.bz2 63 | - When false, we only count .conda 64 | """ 65 | monkeypatch.setenv("CONDA_USE_ONLY_TAR_BZ2", only_tar_bz2) 66 | reset_context() 67 | libmamba_index = LibMambaIndexHelper( 68 | channels=[Channel("defaults")], 69 | subdirs=("noarch",), 70 | installed_records=(), # do not load installed 71 | pkgs_dirs=(), # do not load local cache as a channel 72 | ) 73 | n_repos = 3 if on_win else 2 74 | assert len(libmamba_index.repos) == n_repos 75 | 76 | libmamba_dot_conda_total = libmamba_index.n_packages( 77 | filter_=lambda pkg: pkg.package_url.endswith(".conda") 78 | ) 79 | libmamba_tar_bz2_total = libmamba_index.n_packages( 80 | filter_=lambda pkg: pkg.package_url.endswith(".tar.bz2") 81 | ) 82 | 83 | conda_dot_conda_total = 0 84 | conda_tar_bz2_total = 0 85 | for channel_url in Channel("defaults/noarch").urls(subdirs=("noarch",)): 86 | conda_index = SubdirData(Channel(channel_url)) 87 | conda_index.load() 88 | for pkg in conda_index.iter_records(): 89 | if pkg["url"].endswith(".conda"): 90 | conda_dot_conda_total += 1 91 | elif pkg["url"].endswith(".tar.bz2"): 92 | conda_tar_bz2_total += 1 93 | else: 94 | raise RuntimeError(f"Unrecognized package URL: {pkg['url']}") 95 | 96 | if only_tar_bz2: 97 | assert conda_tar_bz2_total == libmamba_tar_bz2_total 98 | assert libmamba_dot_conda_total == conda_dot_conda_total == 0 99 | else: 100 | assert conda_dot_conda_total == libmamba_dot_conda_total 101 | assert conda_tar_bz2_total == libmamba_tar_bz2_total 102 | 103 | 104 | def test_reload_channels(tmp_path: Path): 105 | (tmp_path / "noarch").mkdir(parents=True, exist_ok=True) 106 | shutil.copy(DATA / "mamba_repo" / "noarch" / "repodata.json", tmp_path / "noarch") 107 | initial_repodata = (tmp_path / "noarch" / "repodata.json").read_text() 108 | index = LibMambaIndexHelper(channels=[Channel(str(tmp_path))]) 109 | initial_count = index.n_packages() 110 | SubdirData._cache_.clear() 111 | 112 | data = json.loads(initial_repodata) 113 | package = data["packages"]["test-package-0.1-0.tar.bz2"] 114 | data["packages"]["test-package-copy-0.1-0.tar.bz2"] = {**package, "name": "test-package-copy"} 115 | modified_repodata = json.dumps(data) 116 | (tmp_path / "noarch" / "repodata.json").write_text(modified_repodata) 117 | 118 | assert initial_repodata != modified_repodata 119 | # TODO: Remove this sleep after addressing 120 | # https://github.com/conda/conda/issues/13783 121 | time.sleep(1) 122 | index.reload_channel(Channel(str(tmp_path))) 123 | assert index.n_packages() == initial_count + 1 124 | 125 | 126 | @pytest.mark.parametrize( 127 | "load_type,requested", 128 | [ 129 | ("shard", ("python",)), 130 | ("shard", ("django", "celery")), 131 | ("shard", ("vaex",)), 132 | ("repodata", ("vaex",)), 133 | ("main", ()), 134 | ], 135 | ids=["shard-small", "shard-medium", "shard-large", "noshard", "main"], 136 | ) 137 | def test_load_channel_repo_info_shards( 138 | load_type: str, 139 | requested: tuple[str, ...], 140 | tmp_path: Path, 141 | conda_cli: CondaCLIFixture, 142 | monkeypatch: pytest.MonkeyPatch, 143 | benchmark: BenchmarkFixture, 144 | ): 145 | """ 146 | Benchmark shards/not-shards under different dependency tree sizes. 147 | """ 148 | load_channel = "defaults" if load_type == "main" else CONDA_FORGE_WITH_SHARDS 149 | 150 | monkeypatch.setattr(context.plugins, "use_sharded_repodata", load_type == "shard") 151 | assert _is_sharded_repodata_enabled() == (load_type == "shard") 152 | 153 | in_state = SolverInputState(str(tmp_path / "env"), requested=requested) 154 | 155 | def index(): 156 | return LibMambaIndexHelper( 157 | # this is expanded to noarch, linux-64 for shards. 158 | channels=[Channel(f"{load_channel}/linux-64")], 159 | subdirs=( 160 | "noarch", 161 | "linux-64", 162 | ), 163 | installed_records=(), # do not load installed 164 | pkgs_dirs=(), # do not load local cache as a channel 165 | in_state=in_state, 166 | ) 167 | 168 | index_helper = benchmark.pedantic(index, rounds=1) 169 | 170 | assert len(index_helper.repos) > 0 171 | -------------------------------------------------------------------------------- /conda_libmamba_solver/mamba_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2019 QuantStack and the Mamba contributors. 2 | # Copyright (C) 2022 Anaconda, Inc 3 | # Copyright (C) 2023 conda 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | # TODO: Temporarily vendored from mamba.utils v0.19 on 2021.12.02 7 | # Decide what to do with it when we split into a plugin 8 | # 2022.02.15: updated vendored parts to v0.21.2 9 | # 2022.11.14: only keeping channel prioritization and context initialization logic now 10 | # 2024.09.24: parameterize init_api_context 11 | 12 | from __future__ import annotations 13 | 14 | import os 15 | import logging 16 | import sys 17 | from collections.abc import Iterable 18 | from functools import lru_cache 19 | from importlib.metadata import version 20 | from pathlib import Path 21 | from typing import TYPE_CHECKING, Iterable 22 | 23 | import libmambapy 24 | from conda.base.constants import ChannelPriority 25 | from conda.base.context import context 26 | from conda.common.compat import on_win 27 | 28 | if TYPE_CHECKING: 29 | from .index import _ChannelRepoInfo 30 | 31 | 32 | 33 | log = logging.getLogger(f"conda.{__name__}") 34 | _db_log = logging.getLogger("conda.libmamba.db") 35 | _libmamba_context = None 36 | 37 | @lru_cache(maxsize=1) 38 | def mamba_version() -> str: 39 | return version("libmambapy") 40 | 41 | 42 | def _get_base_url(url: str, name: str | None = None) -> str: 43 | tmp = url.rsplit("/", 1)[0] 44 | if name: 45 | if tmp.endswith(name): 46 | return tmp.rsplit("/", 1)[0] 47 | return tmp 48 | 49 | 50 | def init_libmamba_context( 51 | channels: Iterable[str] | None = None, 52 | platform: str | None = None, 53 | target_prefix: str | None = None, 54 | ) -> libmambapy.Context: 55 | global _libmamba_context 56 | if _libmamba_context is None: 57 | # This function has to be called BEFORE 1st initialization of the context 58 | _libmamba_context = libmambapy.Context( 59 | libmambapy.ContextOptions( 60 | enable_signal_handling=False, 61 | enable_logging=True, 62 | ) 63 | ) 64 | libmamba_context = _libmamba_context 65 | 66 | # Output params 67 | libmamba_context.output_params.json = context.json 68 | if libmamba_context.output_params.json: 69 | libmambapy.cancel_json_output(libmamba_context) 70 | libmamba_context.output_params.quiet = context.quiet 71 | libmamba_context.output_params.verbosity = context.verbosity 72 | libmamba_context.set_log_level( 73 | { 74 | 4: libmambapy.LogLevel.TRACE, 75 | 3: libmambapy.LogLevel.DEBUG, 76 | 2: libmambapy.LogLevel.INFO, 77 | 1: libmambapy.LogLevel.WARNING, 78 | 0: libmambapy.LogLevel.ERROR, 79 | }[context.verbosity] 80 | ) 81 | 82 | # Prefix params 83 | libmamba_context.prefix_params.conda_prefix = context.conda_prefix 84 | libmamba_context.prefix_params.root_prefix = context.root_prefix 85 | libmamba_context.prefix_params.target_prefix = str( 86 | target_prefix if target_prefix is not None else context.target_prefix 87 | ) 88 | 89 | # Networking params -- we always operate offline from libmamba's perspective 90 | libmamba_context.remote_fetch_params.user_agent = context.user_agent 91 | libmamba_context.local_repodata_ttl = context.local_repodata_ttl 92 | libmamba_context.offline = True 93 | libmamba_context.use_index_cache = True 94 | 95 | # General params 96 | libmamba_context.add_pip_as_python_dependency = context.add_pip_as_python_dependency 97 | libmamba_context.always_yes = context.always_yes 98 | libmamba_context.dry_run = context.dry_run 99 | libmamba_context.envs_dirs = context.envs_dirs 100 | libmamba_context.pkgs_dirs = context.pkgs_dirs 101 | libmamba_context.use_lockfiles = False 102 | libmamba_context.use_only_tar_bz2 = context.use_only_tar_bz2 103 | 104 | # Channels and platforms 105 | libmamba_context.platform = platform if platform is not None else context.subdir 106 | libmamba_context.channels = list(channels) if channels is not None else context.channels 107 | libmamba_context.channel_alias = str( 108 | _get_base_url(context.channel_alias.url(with_credentials=True)) 109 | ) 110 | 111 | RESERVED_NAMES = {"local", "defaults"} 112 | additional_custom_channels = {} 113 | for el in context.custom_channels: 114 | if context.custom_channels[el].canonical_name not in RESERVED_NAMES: 115 | additional_custom_channels[el] = _get_base_url( 116 | context.custom_channels[el].url(with_credentials=True), el 117 | ) 118 | libmamba_context.custom_channels = additional_custom_channels 119 | 120 | additional_custom_multichannels = { 121 | "local": list(context.conda_build_local_paths), 122 | "defaults": [channel.url(with_credentials=True) for channel in context.default_channels], 123 | } 124 | for el in context.custom_multichannels: 125 | if el not in RESERVED_NAMES: 126 | additional_custom_multichannels[el] = [] 127 | for c in context.custom_multichannels[el]: 128 | additional_custom_multichannels[el].append( 129 | _get_base_url(c.url(with_credentials=True)) 130 | ) 131 | libmamba_context.custom_multichannels = additional_custom_multichannels 132 | 133 | libmamba_context.default_channels = [ 134 | _get_base_url(x.url(with_credentials=True)) for x in context.default_channels 135 | ] 136 | 137 | if context.channel_priority is ChannelPriority.STRICT: 138 | libmamba_context.channel_priority = libmambapy.ChannelPriority.Strict 139 | elif context.channel_priority is ChannelPriority.FLEXIBLE: 140 | libmamba_context.channel_priority = libmambapy.ChannelPriority.Flexible 141 | elif context.channel_priority is ChannelPriority.DISABLED: 142 | libmamba_context.channel_priority = libmambapy.ChannelPriority.Disabled 143 | 144 | return libmamba_context 145 | 146 | 147 | def logger_callback(level: libmambapy.solver.libsolv.LogLevel, msg: str, logger: logging.Logger =_db_log) -> None: 148 | # from libmambapy.solver.libsolv import LogLevel 149 | # levels = { 150 | # LogLevel.Debug: logging.DEBUG, # 0 -> 10 151 | # LogLevel.Warning: logging.WARNING, # 1 -> 30 152 | # LogLevel.Error: logging.ERROR, # 2 -> 40 153 | # LogLevel.Fatal: logging.FATAL, # 3 -> 50 154 | # } 155 | if level.value == 0: 156 | # This incurs a large performance hit! 157 | logger.debug(msg) 158 | else: 159 | logger.log((level.value + 2) * 10, msg) 160 | 161 | 162 | def palettes_and_formats() -> tuple[libmambapy.solver.ProblemsMessageFormat, libmambapy.solver.ProblemsMessageFormat]: 163 | # _indents = ["│ ", " ", "├─ ", "└─ "] 164 | if os.getenv("NO_COLOR"): 165 | use_color = False 166 | elif os.getenv("FORCE_COLOR"): 167 | use_color = True 168 | else: 169 | use_color = all([sys.stdout.isatty(), sys.stdin.isatty()]) 170 | palette_no_color = libmambapy.Palette.no_color() 171 | problems_format_nocolor = libmambapy.solver.ProblemsMessageFormat() 172 | problems_format_nocolor.unavailable = palette_no_color.failure 173 | problems_format_nocolor.available = palette_no_color.success 174 | problems_format_auto = ( 175 | libmambapy.solver.ProblemsMessageFormat() 176 | if use_color 177 | else problems_format_nocolor 178 | ) 179 | 180 | return problems_format_auto, problems_format_nocolor 181 | 182 | 183 | problems_format_auto, problems_format_nocolor = palettes_and_formats() 184 | -------------------------------------------------------------------------------- /.github/workflows/performance.yml: -------------------------------------------------------------------------------- 1 | name: Performance 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - labeled 7 | workflow_run: 8 | workflows: 9 | - CI 10 | branches: 11 | - main 12 | types: 13 | - completed 14 | 15 | concurrency: 16 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} 17 | cancel-in-progress: true 18 | 19 | env: 20 | CONDA_SOLVER: libmamba 21 | 22 | jobs: 23 | linux: 24 | if: | 25 | (github.event_name == 'pull_request' && 26 | github.event.label.name == 'performance::run') || 27 | (github.event.workflow_run.conclusion == 'success') 28 | name: Linux, Python ${{ matrix.python-version }} 29 | runs-on: ubuntu-latest 30 | strategy: 31 | fail-fast: false 32 | matrix: 33 | python-version: ["3.9"] 34 | 35 | env: 36 | OS: "linux" 37 | PYTHON: ${{ matrix.python-version }} 38 | 39 | steps: 40 | - name: Checkout our source 41 | uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 42 | with: 43 | path: conda-libmamba-solver 44 | fetch-depth: 0 45 | 46 | - name: Checkout conda 47 | uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 48 | with: 49 | fetch-depth: 0 50 | repository: conda/conda 51 | path: conda 52 | 53 | - name: Run performance tests 54 | run: > 55 | docker run 56 | --rm 57 | -v ${GITHUB_WORKSPACE}/conda:/opt/conda-src 58 | -v ${GITHUB_WORKSPACE}/conda-libmamba-solver:/opt/conda-libmamba-solver-src 59 | -e TEST_SPLITS 60 | -e TEST_GROUP 61 | -e CONDA_SOLVER 62 | ghcr.io/conda/conda-ci:main-linux-python${{ matrix.python-version }} 63 | bash -c "sudo /opt/conda/condabin/conda install -p /opt/conda \ 64 | --file /opt/conda-libmamba-solver-src/dev/requirements.txt && 65 | /opt/conda/bin/python -m pip install /opt/conda-libmamba-solver-src \ 66 | --no-deps -vvv && 67 | source /opt/conda-src/dev/linux/bashrc.sh && 68 | /opt/conda/bin/python -m pytest /opt/conda-libmamba-solver-src \ 69 | -vv --durations=0 --timeout=1800 -m 'slow'" 70 | 71 | macos: 72 | if: | 73 | (github.event_name == 'pull_request' && 74 | github.event.label.name == 'performance::run') || 75 | (github.event.workflow_run.conclusion == 'success') 76 | name: MacOS, Python ${{ matrix.python-version }} 77 | runs-on: macos-latest 78 | strategy: 79 | fail-fast: false 80 | matrix: 81 | python-version: ["3.9"] 82 | 83 | env: 84 | OS: "macos" 85 | PYTHON: ${{ matrix.python-version }} 86 | 87 | steps: 88 | - name: Checkout our source 89 | uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 90 | with: 91 | path: conda-libmamba-solver 92 | fetch-depth: 0 93 | 94 | - name: Checkout conda 95 | uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 96 | with: 97 | fetch-depth: 0 98 | repository: conda/conda 99 | ref: libmamba 100 | path: conda 101 | 102 | - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0 103 | with: 104 | activate-environment: conda-test-env 105 | use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! 106 | 107 | - name: Setup environment 108 | shell: bash -l {0} 109 | working-directory: conda 110 | run: | 111 | set -euxo pipefail 112 | # restoring the default for changeps1 to have parity with dev 113 | conda config --set changeps1 true 114 | # make sure the caching works correctly 115 | conda config --set use_only_tar_bz2 true 116 | # install all test requirements 117 | conda install --yes \ 118 | --file tests/requirements.txt \ 119 | --file ../conda-libmamba-solver/dev/requirements.txt \ 120 | python=${{ matrix.python-version }} 121 | conda update openssl ca-certificates certifi 122 | conda info 123 | python -c "from mamba import __version__; print('mamba', __version__)" 124 | 125 | - name: Install conda-libmamba-solver 126 | shell: bash -l {0} 127 | working-directory: conda-libmamba-solver 128 | run: | 129 | python -m pip install . -vv --no-deps 130 | 131 | - name: Run performance tests 132 | shell: bash -l {0} 133 | working-directory: conda 134 | run: | 135 | eval "$(sudo ${CONDA_PREFIX}/bin/python -m conda init bash --dev)" 136 | python -m pytest ${GITHUB_WORKSPACE}/conda-libmamba-solver -vv --durations=0 --timeout=1800 -m "slow" 137 | 138 | windows: 139 | if: | 140 | (github.event_name == 'pull_request' && 141 | github.event.label.name == 'performance::run') || 142 | (github.event.workflow_run.conclusion == 'success') 143 | name: Windows, Python ${{ matrix.python-version }} 144 | runs-on: windows-latest 145 | strategy: 146 | fail-fast: false 147 | matrix: 148 | python-version: ["3.9"] 149 | 150 | env: 151 | OS: "windows" 152 | PYTHON: ${{ matrix.python-version }} 153 | 154 | steps: 155 | - name: Checkout our source 156 | uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 157 | with: 158 | path: conda-libmamba-solver 159 | fetch-depth: 0 160 | 161 | - name: Checkout conda 162 | uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 163 | with: 164 | fetch-depth: 0 165 | repository: conda/conda 166 | ref: libmamba 167 | path: conda 168 | 169 | - name: Set temp dirs correctly 170 | # https://github.com/actions/virtual-environments/issues/712 171 | run: | 172 | echo "TMPDIR=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV 173 | echo "TEMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV 174 | echo "TMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV 175 | 176 | - name: Setup environment 177 | shell: cmd 178 | working-directory: conda 179 | run: | 180 | :: add mamba to requirements 181 | type ..\conda-libmamba-solver\dev\requirements.txt >> .\tests\requirements.txt 182 | if errorlevel 1 exit 1 183 | :: initialize conda dev 184 | call .\dev\windows\setup.bat 185 | if errorlevel 1 exit 1 186 | call .\dev-init.bat 187 | if errorlevel 1 exit 1 188 | python -c "from mamba import __version__; print('mamba', __version__)" 189 | if errorlevel 1 exit 1 190 | conda info -a 191 | if errorlevel 1 exit 1 192 | 193 | - name: Install conda-libmamba-solver 194 | shell: cmd 195 | working-directory: conda 196 | run: | 197 | call .\dev-init.bat 198 | if errorlevel 1 exit 1 199 | python -m pip install --no-deps -vv "%GITHUB_WORKSPACE%\conda-libmamba-solver" 200 | if errorlevel 1 exit 1 201 | 202 | - name: Run performance tests 203 | shell: cmd 204 | working-directory: conda 205 | run: | 206 | CALL dev-init.bat 207 | if errorlevel 1 exit 1 208 | python -m pytest "%GITHUB_WORKSPACE%\conda-libmamba-solver" -vv --durations=0 --timeout=1800 -m "slow" 209 | if errorlevel 1 exit 1 210 | --------------------------------------------------------------------------------