├── requests
    └── .keep
├── .github
    ├── ISSUE_TEMPLATE.md
    ├── CODEOWNERS
    ├── dependabot.yml
    ├── workflows
    │   ├── pr.yml
    │   ├── repodata_patching.yml
    │   ├── create_feedstocks.yml
    │   └── main.yml
    └── PULL_REQUEST_TEMPLATE.md
├── examples
    ├── example-travis.yml
    ├── example-archive.yml
    ├── example-unarchive.yml
    ├── example-not-broken.yml
    ├── example-broken.yml
    ├── example-add-feedstock-output.yml
    ├── example-cfep-3.yml
    ├── example-token-reset.yml
    └── example-open-gpu-server.yml
├── pyproject.toml
├── environment.yml
├── .access_control.yml
├── conda_forge_admin_requests
    ├── __init__.py
    ├── utils.py
    ├── archive_feedstock.py
    ├── __main__.py
    ├── cfep3_copy.py
    ├── mark_broken.py
    ├── feedstock_outputs.py
    ├── token_reset.py
    └── access_control.py
├── scripts
    ├── create_feedstocks
    ├── update_repodata_patches.py
    └── create_feedstocks.py
├── .gitignore
└── README.md


/requests/.keep:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE.md:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/.github/CODEOWNERS:
--------------------------------------------------------------------------------
1 | * @conda-forge/core
2 | 


--------------------------------------------------------------------------------
/examples/example-travis.yml:
--------------------------------------------------------------------------------
1 | action: travis
2 | feedstocks:
3 | - conda-smithy
4 | 


--------------------------------------------------------------------------------
/examples/example-archive.yml:
--------------------------------------------------------------------------------
1 | action: archive
2 | feedstocks:
3 |   - cf-autotick-bot-test-package
4 | 


--------------------------------------------------------------------------------
/examples/example-unarchive.yml:
--------------------------------------------------------------------------------
1 | action: unarchive
2 | feedstocks:
3 |   - cf-autotick-bot-test-package
4 | 


--------------------------------------------------------------------------------
/examples/example-not-broken.yml:
--------------------------------------------------------------------------------
1 | action: not_broken
2 | packages:
3 | - linux-64/cf-autotick-bot-test-package-0.10-py36h5fab9bb_3.tar.bz2
4 | 


--------------------------------------------------------------------------------
/examples/example-broken.yml:
--------------------------------------------------------------------------------
1 | action: broken
2 | packages:
3 | - linux-aarch64/cf-autotick-bot-test-package-0.13-py38h2063c64_0.tar.bz2
4 | - linux-aarch64/cf-autotick-bot-test-package-0.13-py37hd9ded2f_0.tar.bz2
5 | 


--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
 1 | version: 2
 2 | updates:
 3 |   - package-ecosystem: "github-actions"
 4 |     directory: "/"
 5 |     schedule:
 6 |       interval: "monthly"
 7 |     groups:
 8 |       github-actions:
 9 |         patterns:
10 |           - '*'
11 | 


--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
 1 | [build-system]
 2 | requires = ["setuptools >= 61.0"]
 3 | build-backend = "setuptools.build_meta"
 4 | 
 5 | [tool.setuptools]
 6 | packages = ["conda_forge_admin_requests"]
 7 | 
 8 | [project]
 9 | name = "conda_forge_admin_requests"
10 | version = "2024.01.08"
11 | 


--------------------------------------------------------------------------------
/environment.yml:
--------------------------------------------------------------------------------
 1 | name: cf
 2 | channels:
 3 |   - conda-forge
 4 |   - nodefaults
 5 | dependencies:
 6 |   - anaconda-client
 7 |   - python =3.10
 8 |   - requests
 9 |   - pygithub
10 |   - pydantic
11 |   - cirun
12 |   - conda-smithy >=3.30.1
13 |   - conda-forge-metadata >=0.9.1
14 |   - ruamel.yaml
15 |   - conda-build >=25.3.1
16 | 


--------------------------------------------------------------------------------
/examples/example-add-feedstock-output.yml:
--------------------------------------------------------------------------------
 1 | action: add_feedstock_output
 2 | feedstock_to_output_mapping:
 3 |   # this entry adds a package name
 4 |   clang-compiler-activation:
 5 |     - clang_impl_osx-64
 6 |     - clang_impl_osx-arm64
 7 |   # While fully specified package names are preferred, you can also use
 8 |   # this entry to add an allowed glob pattern
 9 |   llvmdev:
10 |     - "libllvm*"
11 | 


--------------------------------------------------------------------------------
/examples/example-cfep-3.yml:
--------------------------------------------------------------------------------
1 | action: cfep3_copy
2 | anaconda_org_packages:
3 |   # list of {package: channel[/label/<label>]/subdir/filename.extension, sha256: SHA256}
4 |   - package: jaimergp/label/conda-standalone-24.7.1/osx-arm64/conda-standalone-24.7.1-hce30654_0.conda
5 |     sha256: cd11f1f0fbe8c203212d002ca4e01795d0b1c886ce47d8b08e6bca3366a7c8e6
6 | to_anaconda_org_label: main  # optional, default=main
7 | 


--------------------------------------------------------------------------------
/examples/example-token-reset.yml:
--------------------------------------------------------------------------------
1 | action: token_reset
2 | feedstocks:
3 | - !!!PUT THE NAME OF YOUR FEEDSTOCK HERE!!!
4 | # do not use this unless you have special requirements for your feedstock
5 | # can be a list with azure, travis, circle, drone, appveyor, github_actions
6 | # skip_providers: []
7 | # we can expire tokens earlier if desired - default is null which is immediately
8 | # existing_tokens_time_to_expiration: null  # set to an int in seconds to use
9 | 


--------------------------------------------------------------------------------
/.access_control.yml:
--------------------------------------------------------------------------------
 1 | # This file is exclusively handled by the existing automation or the core team.
 2 | # Please DO NOT submit modifications to this document.
 3 | version: 1
 4 | access_control:
 5 |   travis:
 6 |     - feedstock: example
 7 |   cirun-gpu-small:
 8 |     - feedstock: example
 9 |   cirun-openstack-gpu-large:
10 |     - feedstock: cf-autotick-bot-test-package-feedstock
11 |     - feedstock: conda-smithy-feedstock
12 |     - feedstock: conda-forge-ci-setup-feedstock
13 |     - feedstock: pytorch-cpu-feedstock
14 |     - feedstock: libmagma-feedstock
15 | 


--------------------------------------------------------------------------------
/examples/example-open-gpu-server.yml:
--------------------------------------------------------------------------------
 1 | action: cirun
 2 | feedstocks:
 3 |   # list of feedstock names (sans `-feedstock` suffix)
 4 |   - pytorch-cpu
 5 | resources:
 6 |   # list of resource names you are applying to
 7 |   # see https://github.com/conda-forge/.cirun for available options
 8 |   - cirun-openstack-gpu-large
 9 |   # - cirun-openstack-cpu-large
10 | # switch to true if you need it on pull-requests too
11 | pull_request: false  
12 | # revoke access to the gpu runner
13 | revoke: false
14 | # Send an automatic PR to the feedstock to enable cirun defaults
15 | send_pr: true
16 | 


--------------------------------------------------------------------------------
/.github/workflows/pr.yml:
--------------------------------------------------------------------------------
 1 | name: check
 2 | on:
 3 |   pull_request:
 4 |     branches:
 5 |       - main
 6 | 
 7 | env:
 8 |   PYTHONUNBUFFERED: 1
 9 | 
10 | jobs:
11 |   check:
12 |     runs-on: ubuntu-latest
13 |     steps:
14 |       - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
15 | 
16 |       - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0
17 |         with:
18 |           activate-environment: cf
19 |           environment-file: environment.yml
20 |           auto-activate-base: true
21 |           miniforge-version: latest
22 | 
23 |       - name: Check request YAML files
24 |         shell: bash -el {0}
25 |         env:
26 |           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
27 |         run: |
28 |           conda activate cf
29 |           python -m conda_forge_admin_requests check
30 | 


--------------------------------------------------------------------------------
/conda_forge_admin_requests/__init__.py:
--------------------------------------------------------------------------------
 1 | import importlib
 2 | import pkgutil
 3 | from . import archive_feedstock, mark_broken, token_reset, access_control, cfep3_copy, feedstock_outputs
 4 | 
 5 | actions = {}
 6 | 
 7 | def get_actions():
 8 |     return actions.copy()
 9 | 
10 | 
11 | def register_action(name, module):
12 |     assert name not in actions
13 |     actions[name] = module
14 | 
15 | 
16 | def register_actions():
17 |     register_action("archive", archive_feedstock)
18 |     register_action("unarchive", archive_feedstock)
19 |     register_action("broken", mark_broken)
20 |     register_action("not_broken", mark_broken)
21 |     register_action("token_reset", token_reset)
22 |     register_action("travis", access_control)
23 |     register_action("cirun", access_control)
24 |     register_action("cfep3_copy", cfep3_copy)
25 |     register_action("add_feedstock_output", feedstock_outputs)
26 |     for pkg in pkgutil.iter_modules():
27 |         if pkg.name.startswith("conda_forge_admin_requests_"):
28 |             spec = importlib.util.find_spec(pkg.name)
29 |             module = importlib.util.module_from_spec(spec)
30 |             spec.loader.exec_module(module)
31 |             getattr(module, "register_actions")()
32 | 
33 | 


--------------------------------------------------------------------------------
/conda_forge_admin_requests/utils.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | 
 3 | from conda_build.utils import create_file_with_permissions
 4 | 
 5 | SMITHY_CONF = os.path.expanduser('~/.conda-smithy')
 6 | 
 7 | 
 8 | def _write_token(name, token):
 9 |     path = os.path.join(SMITHY_CONF, name + '.token')
10 |     with create_file_with_permissions(path, 0o600) as fh:
11 |         fh.write(token)
12 | 
13 | 
14 | def write_secrets_to_files():
15 |     if not os.path.exists(SMITHY_CONF):
16 |         os.makedirs(SMITHY_CONF, exist_ok=True)
17 | 
18 |     for token_fname, token_name in [
19 |         ("circle", "CIRCLE_TOKEN"),
20 |         ("azure", "AZURE_TOKEN"),
21 |         ("drone", "DRONE_TOKEN"),
22 |         ("travis", "TRAVIS_TOKEN"),
23 |         ("github", "GITHUB_TOKEN"),
24 |         ("anaconda", "STAGING_BINSTAR_TOKEN"),
25 |     ]:
26 |         if token_name in os.environ:
27 |             _write_token(token_fname, os.environ[token_name])
28 | 
29 | 
30 | def split_label_from_channel(channel: str) -> tuple[str, str]:
31 |     if "/label/" in channel:
32 |         return channel.split("/label/", 1)
33 |     return channel, "main"
34 | 
35 | 
36 | def parse_filename(filename: str) -> tuple[str, str, str, str]:  
37 |     if filename.endswith(".tar.bz2"):
38 |         basename = filename[:-len(".tar.bz2")]
39 |         extension = "tar.bz2"
40 |     elif filename.endswith(".conda"):
41 |         basename = filename[:-len(".conda")]
42 |         extension = "conda"
43 |     else:
44 |         raise ValueError(f"Unknown extension for {filename}")
45 |     pkg_name, version, build = basename.rsplit("-", 2)
46 |     return pkg_name, version, build, extension
47 | 


--------------------------------------------------------------------------------
/scripts/create_feedstocks:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | set -e
 4 | set -x
 5 | 
 6 | # Ensure we are on the latest commit
 7 | # of the branch where we are converting
 8 | # recipes from. Currently this is `main`.
 9 | export CF_CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
10 | git checkout "${CF_CURRENT_BRANCH}"
11 | 
12 | # 2 core available on Travis CI Linux workers: https://docs.travis-ci.com/user/ci-environment/#Virtualization-environments
13 | # CPU_COUNT is passed through conda build: https://github.com/conda/conda-build/pull/1149
14 | export CPU_COUNT=2
15 | 
16 | export PYTHONUNBUFFERED=1
17 | 
18 | # Install Miniforge3.
19 | echo ""
20 | echo "Installing a fresh version of Miniforge3."
21 | curl -L "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" > ~/Miniforge3.sh
22 | chmod +x ~/Miniforge3.sh
23 | bash ~/Miniforge3.sh -b -p ~/Miniforge3
24 | touch ~/Miniforge3/conda-meta/pinned
25 | (
26 |     source ~/Miniforge3/bin/activate
27 | 
28 |     # Configure conda.
29 |     echo ""
30 |     echo "Configuring conda."
31 |     conda config --set show_channel_urls true
32 |     conda config --set auto_update_conda false
33 |     conda config --set add_pip_as_python_dependency false
34 |     conda config --set solver libmamba
35 | 
36 |     conda update -n base --yes --quiet conda mamba conda-libmamba-solver
37 | )
38 | source ~/Miniforge3/bin/activate
39 | 
40 | conda install --yes --quiet \
41 |   conda-forge-ci-setup=4.* \
42 |   "conda-smithy>=3.38.0,<4.0.0a0" \
43 |   conda-forge-pinning \
44 |   "conda-build>=24.3" \
45 |   "gitpython>=3.0.8,<3.1.20" \
46 |   requests \
47 |   ruamel.yaml \
48 |   "pygithub>=2.1.1" \
49 |   "rattler-build-conda-compat>=1.2.0,<2.0.0a0" \
50 |   "conda-forge-feedstock-ops>=0.5.0" \
51 |   "conda-forge-metadata>=0.8.1"
52 | 
53 | conda info
54 | mamba info
55 | conda config --get
56 | git config --global init.defaultBranch main
57 | 


--------------------------------------------------------------------------------
/.github/workflows/repodata_patching.yml:
--------------------------------------------------------------------------------
 1 | name: repodata_patching
 2 | 
 3 | on:
 4 |   schedule:
 5 |     - cron: '0 0 * * 0'
 6 |   workflow_dispatch: null
 7 | 
 8 | jobs:
 9 |   repodata_patching:
10 |     runs-on: ubuntu-latest
11 |     steps:
12 |       - name: Prevent multiple jobs running in parallel
13 |         id: conversion_lock
14 |         uses: beckermr/turnstyle-python@8f1ceb87dabbbbebe42257b85c368f6110bb9170 # v2
15 |         with:
16 |           abort-after-seconds: 3
17 |           poll-interval-seconds: 2
18 |           github-token: ${{ secrets.GITHUB_TOKEN }}
19 |         continue-on-error: true
20 | 
21 |       - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
22 |         # outcome is evaluated before continue-on-error above
23 |         if: steps.conversion_lock.outcome == 'success'
24 | 
25 |       - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0
26 |         if: steps.conversion_lock.outcome == 'success'
27 |         with:
28 |           activate-environment: cf
29 |           environment-file: environment.yml
30 |           auto-activate-base: true
31 |           miniforge-version: latest
32 | 
33 |       - name: Generate token
34 |         if: steps.conversion_lock.outcome == 'success'
35 |         id: generate_token
36 |         uses: actions/create-github-app-token@7e473efe3cb98aa54f8d4bac15400b15fad77d94 # v1
37 |         with:
38 |           app-id: ${{ secrets.CF_CURATOR_APP_ID }}
39 |           private-key: ${{ secrets.CF_CURATOR_PRIVATE_KEY }}
40 |           owner: ${{ github.repository_owner }}
41 | 
42 |       - name: patch repodata
43 |         if: steps.conversion_lock.outcome == 'success'
44 |         shell: bash -l {0}
45 |         run: |
46 |           conda activate cf
47 |           git config --global user.email "79913779+conda-forge-curator[bot]@users.noreply.github.com"
48 |           git config --global user.name "conda-forge-curator[bot]"
49 |           git config --global pull.rebase false
50 |           python scripts/update_repodata_patches.py
51 |         env:
52 |           GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
53 | 


--------------------------------------------------------------------------------
/conda_forge_admin_requests/archive_feedstock.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import requests
 3 | import subprocess
 4 | 
 5 | 
 6 | def raise_json_for_status(request):
 7 |     try:
 8 |         request.raise_for_status()
 9 |     except Exception as exc:
10 |         exc.args = exc.args + (request.json(), )
11 |         raise exc.with_traceback(exc.__traceback__)
12 | 
13 | 
14 | def process_repo(repo, task):
15 |     owner = "conda-forge"
16 |     headers = {
17 |         "X-GitHub-Api-Version": "2022-11-28",
18 |         "Accept": "application/vnd.github+json",
19 |         "User-Agent": "conda-forge/admin-requests",
20 |         "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}",
21 |     }
22 | 
23 |     r = requests.get(
24 |         f"https://api.github.com/repos/{owner}/{repo}",
25 |         headers=headers,
26 |     )
27 |     raise_json_for_status(r)
28 | 
29 |     if task == "archive":
30 |         target_status = "archived"
31 |     else:
32 |         target_status = "unarchived"
33 | 
34 |     data = r.json()
35 |     if task == "archive" and data["archived"]:
36 |         print("feedstock %s is already %s" % (repo, target_status), flush=True)
37 |         return
38 | 
39 |     if task == "unarchive" and not data["archived"]:
40 |         print("feedstock %s is already %s" % (repo, target_status), flush=True)
41 |         return
42 | 
43 |     r = requests.patch(
44 |         f"https://api.github.com/repos/{owner}/{repo}",
45 |         headers=headers,
46 |         json={"archived": task == "archive"}
47 |     )
48 |     raise_json_for_status(r)
49 | 
50 |     print("feedstock %s was %s" % (repo, target_status), flush=True)
51 | 
52 | 
53 | def run(request):
54 |     feedstocks = request["feedstocks"]
55 |     task = request["action"]
56 | 
57 |     pkgs_to_do_again = []
58 |     for feedstock in feedstocks:
59 |         try:
60 |             process_repo(f"{feedstock}-feedstock", task)
61 |         except Exception as e:
62 |             print(
63 |                 "failed to %s '%s': %s" % (task, feedstock, repr(e)),
64 |                 flush=True,
65 |             )
66 |             pkgs_to_do_again.append(feedstock)
67 | 
68 |     if pkgs_to_do_again:
69 |         request["feedstocks"] = pkgs_to_do_again
70 | 
71 |     subprocess.check_call(["git", "show"])
72 | 
73 | 
74 | def check(request):
75 |     assert "feedstocks" in request
76 | 
77 |     missing_feedstocks = []
78 | 
79 |     for feedstock in request["feedstocks"]:
80 |         r = requests.get(f"https://github.com/conda-forge/{feedstock}-feedstock")
81 |         if r.status_code != 200:
82 |             missing_feedstocks.append(feedstock)
83 | 
84 |     if missing_feedstocks:
85 |         raise RuntimeError(
86 |             "feedstocks %s could not be found!" % list(set(missing_feedstocks))
87 |         )
88 | 


--------------------------------------------------------------------------------
/.github/workflows/create_feedstocks.yml:
--------------------------------------------------------------------------------
 1 | name: create_feedstocks
 2 | 
 3 | on:
 4 |   schedule:
 5 |     - cron: '*/10 * * * *'
 6 |   workflow_dispatch: null
 7 | 
 8 | jobs:
 9 |   create_feedstocks:
10 |     name: create feedstocks
11 |     runs-on: ubuntu-latest
12 | 
13 |     steps:
14 |       - name: checkout staged-recipes
15 |         uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
16 |         with:
17 |           repository: conda-forge/staged-recipes
18 |           token: ${{ secrets.CF_ADMIN_GITHUB_TOKEN }}
19 |           path: staged-recipes
20 | 
21 |       - name: checkout admin-requests
22 |         uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
23 |         with:
24 |           repository: conda-forge/admin-requests
25 |           path: admin-requests
26 | 
27 |       - name: prevent multiple jobs running in parallel
28 |         id: conversion_lock
29 |         uses: beckermr/turnstyle-python@8f1ceb87dabbbbebe42257b85c368f6110bb9170 # v2
30 |         with:
31 |           abort-after-seconds: 3
32 |           poll-interval-seconds: 2
33 |           github-token: ${{ secrets.CF_ADMIN_GITHUB_TOKEN }}
34 |         continue-on-error: true
35 | 
36 |       - name: commit any changes upon checkout
37 |         run: |
38 |           pushd staged-recipes
39 | 
40 |           git config --global user.email "chrisburr73+conda-forge-admin@gmail.com"
41 |           git config --global user.name "conda-forge-admin"
42 |           git config --global pull.rebase false
43 |           git add *
44 |           git commit -am "make sure we have no windows line endings" || exit 0
45 |           for i in `seq 1 5`; do
46 |             git pull
47 |             git push
48 |           done
49 | 
50 |           popd
51 | 
52 |       - name: Run feedstock creation
53 |         # outcome is evaluated before continue-on-error above
54 |         if: ${{ steps.conversion_lock.outcome == 'success' }}
55 |         run: |
56 |           pushd staged-recipes
57 | 
58 |           # Avoid wasting CI time if there are no recipes ready for conversion
59 |           # Checks for both `meta.yaml` and `recipe.yaml` files in the recipes directory
60 |           # Excludes the example recipe(s)
61 |           if [ "$(ls recipes/*/meta.yaml recipes/*/recipe.yaml 2>/dev/null | grep -vE 'recipes/example/meta.yaml|recipes/example/recipe.yaml' --count)" -eq 0 ]; then
62 |             echo "No new recipes found, exiting..."
63 |             exit 0
64 |           fi
65 | 
66 |           echo "Creating feedstocks from the recipe(s)."
67 | 
68 |           source ../admin-requests/scripts/create_feedstocks
69 | 
70 |           python ../admin-requests/scripts/create_feedstocks.py
71 | 
72 |           popd
73 |         env:
74 |           STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }}
75 |           GH_TOKEN: ${{ secrets.CF_ADMIN_GITHUB_TOKEN }}
76 |           TRAVIS_TOKEN: ${{ secrets.CF_ADMIN_TRAVIS_TOKEN }}
77 |           AZURE_TOKEN: ${{ secrets.AZURE_TOKEN }}
78 | 


--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
 1 | <!--
 2 | Hi!
 3 | 
 4 | Thank you for making an admin request on this repo. We strive to make a decision
 5 | on these requests within 24 hours.
 6 | 
 7 | Please use the text below to add context about this PR, especially if:
 8 | - You want to mark packages as broken
 9 | - You want to archive a feedstock
10 | - You want to request access to opt-in CI resources
11 | 
12 | Cheers and thank you for contributing to conda-forge!
13 | -->
14 | 
15 | ## Guidelines for marking packages as broken:
16 | 
17 | * We prefer to patch the repo data (see [here](https://github.com/conda-forge/conda-forge-repodata-patches-feedstock))
18 |   instead of marking packages as broken. This alternative workflow makes environments more reproducible.
19 | * Packages with requirements/metadata that are too strict but otherwise work are
20 |   not technically broken and should not be marked as such.
21 | * Packages with missing metadata can be marked as broken on a temporary basis
22 |   but should be patched in the repo data and be marked unbroken later.
23 | * In some cases where the number of users of a package is small or it is used by
24 |   the maintainers only, we can allow packages to be marked broken more liberally.
25 | * We (`conda-forge/core`) try to make a decision on these requests within 24 hours.
26 | 
27 | What will happen when a package is marked broken?
28 | 
29 | * Our bots will add the `broken` label to the package. The `main` label will remain on the package and this is normal.
30 | * Our bots will rebuild our repodata patches to remove this package from the repodata.
31 | * In a few hours after the `anaconda.org` CDN picks up the new patches, you will no longer be able to install the package from the `main` channel.
32 | 
33 | 
34 | ## Checklist:
35 | 
36 | * [ ] I want to mark a package as broken (or not broken):
37 |   * [ ] Added a description of the problem with the package in the PR description.
38 |   * [ ] Pinged the team for the package for their input.
39 | 
40 | * [ ] I want to archive a feedstock:
41 |   * [ ] Pinged the team for that feedstock for their input.
42 |   * [ ] Make sure you have opened an issue on the feedstock explaining why it was archived.
43 |   * [ ] Linked that issue in this PR description.
44 |   * [ ] Added links to any other relevant issues/PRs in the PR description.
45 | 
46 | * [ ] I want to request (or revoke) access to an opt-in CI resource:
47 |   * [ ] Pinged the relevant feedstock team(s)
48 |   * [ ] Added a small description explaining why access is needed
49 | 
50 | * [ ] I want to copy an artifact following [CFEP-3](https://github.com/conda-forge/cfep/blob/main/cfep-03.md):
51 |   * [ ] Pinged the relevant feedstock team(s)
52 |   * [ ] Added a reference to the original PR
53 |   * [ ] Posted a link to the conda artifacts
54 |   * [ ] Posted a link to the build logs
55 | 
56 | * [ ] I want to add a package output to a feedstock:
57 |   * [ ] Pinged the relevant feedstock team(s)
58 |   * [ ] Added a small description of why the output is being added.
59 | 
60 | <!--
61 | For example if you are trying to mark a `foo` conda package as broken.
62 | 
63 |   ping @conda-forge/foo
64 | 
65 | -->
66 | 


--------------------------------------------------------------------------------
/conda_forge_admin_requests/__main__.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import glob
  3 | import yaml
  4 | import sys
  5 | import subprocess
  6 | from conda_forge_admin_requests import get_actions, register_actions
  7 | 
  8 | def _get_task_files():
  9 |     return (
 10 |         list(glob.glob(os.path.join("requests", "*.yml")))
 11 |         + list(glob.glob(os.path.join("requests", "*.yaml")))
 12 |     )
 13 | 
 14 | 
 15 | def check():
 16 |     # error if people put thinks in old places
 17 |     old_files = glob.glob("broken/*")
 18 |     if old_files:
 19 |         assert False, (
 20 |             f"Found old files ({old_files}) in wrong location. "
 21 |             "Please put YAML-formatted requests in the `requests` directory."
 22 |         )
 23 | 
 24 |     if not all(
 25 |         fname.endswith(".yaml") or fname.endswith(".yml")
 26 |         for fname in glob.glob("requests/*")
 27 |     ):
 28 |         assert False, (
 29 |             "Found non-YAML files in the `requests` directory. Please "
 30 |             "use only YAML-formatted requests with filename extensions "
 31 |             "`.yml` or `.yaml`."
 32 |         )
 33 | 
 34 |     filenames = _get_task_files()
 35 | 
 36 |     for filename in filenames:
 37 |         with open(filename) as f:
 38 |             request = yaml.safe_load(f)
 39 | 
 40 |         assert "action" in request, f"Invalid request: {request}"
 41 | 
 42 |         action = request["action"]
 43 |         actions = get_actions()
 44 | 
 45 |         if action not in actions:
 46 |             assert False, f"Unknown action: {action}"
 47 | 
 48 |         getattr(actions[action], "check")(request)
 49 | 
 50 | 
 51 | def run():
 52 |     filenames = _get_task_files()
 53 | 
 54 |     for filename in filenames:
 55 |         with open(filename) as f:
 56 |             request = yaml.safe_load(f)
 57 | 
 58 |         assert "action" in request, f"Invalid request: {request}"
 59 | 
 60 |         action = request["action"]
 61 |         actions = get_actions()
 62 | 
 63 |         if action not in actions:
 64 |             assert False, f"Unknown action: {action}"
 65 | 
 66 |         try_again = getattr(actions[action], "run")(request)
 67 | 
 68 |         if try_again:
 69 |             with open(filename, "w") as fp:
 70 |                 yaml.dump(try_again, fp)
 71 |             subprocess.check_call(["git", "add", filename])
 72 |             subprocess.check_call(
 73 |                 [
 74 |                     "git",
 75 |                     "commit",
 76 |                     "--allow-empty",
 77 |                     "-m",
 78 |                     f"Keeping {filename} after failed {action}",
 79 |                 ]
 80 |             )
 81 |         else:
 82 |             subprocess.check_call(["git", "rm", filename])
 83 |             subprocess.check_call(
 84 |                 ["git", "commit", "-m", f"Remove {filename} after {action}"]
 85 |             )
 86 | 
 87 | 
 88 | if __name__ == "__main__":
 89 |     if len(sys.argv) != 2:
 90 |         sys.exit("Usage: python -m conda_forge_admin_requests [check | run]")
 91 | 
 92 |     register_actions()
 93 |     check_only = sys.argv[1] == "check"
 94 | 
 95 |     if check_only:
 96 |         check()
 97 |     else:
 98 |         run()
 99 | 
100 | 


--------------------------------------------------------------------------------
/.github/workflows/main.yml:
--------------------------------------------------------------------------------
 1 | name: run
 2 | on:
 3 |   push:
 4 |     branches:
 5 |       - main
 6 |   schedule:
 7 |     - cron: "0,15,30,45 * * * *"
 8 |   workflow_dispatch: null
 9 | 
10 | jobs:
11 |   run:
12 |     runs-on: ubuntu-latest
13 |     steps:
14 |       - name: Prevent multiple jobs running in parallel
15 |         id: conversion_lock
16 |         uses: beckermr/turnstyle-python@8f1ceb87dabbbbebe42257b85c368f6110bb9170 # v2
17 |         with:
18 |           abort-after-seconds: 3
19 |           poll-interval-seconds: 2
20 |           github-token: ${{ secrets.GITHUB_TOKEN }}
21 |         continue-on-error: true
22 | 
23 |       - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
24 |         # outcome is evaluated before continue-on-error above
25 |         if: steps.conversion_lock.outcome == 'success'
26 |         with:
27 |           token: ${{ secrets.CF_ADMIN_GITHUB_TOKEN }}
28 | 
29 |       - name: fast finish
30 |         if: ${{ steps.conversion_lock.outcome == 'success' }}
31 |         run: |
32 |           set -x
33 |           # avoid wasting CI time if there is nothing to do
34 |           count_yml=$(ls requests/*.yml | wc -l)
35 |           count_yaml=$(ls requests/*.yaml | wc -l)
36 |           count=$(($count_yml + $count_yaml))
37 |           if [[ "${count}" == "0" ]]; then
38 |             echo "nothing to do, setting ci skip!"
39 |             echo "CI_SKIP=true" >> $GITHUB_ENV
40 |           fi
41 | 
42 |       - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0
43 |         if: steps.conversion_lock.outcome == 'success' && ! env.CI_SKIP
44 |         with:
45 |           activate-environment: cf
46 |           environment-file: environment.yml
47 |           auto-activate-base: true
48 |           miniforge-version: latest
49 | 
50 |       - name: Generate token
51 |         if: steps.conversion_lock.outcome == 'success' && ! env.CI_SKIP
52 |         id: generate_token
53 |         uses: actions/create-github-app-token@7e473efe3cb98aa54f8d4bac15400b15fad77d94 # v2.2.0
54 |         with:
55 |           app-id: ${{ secrets.CF_CURATOR_APP_ID }}
56 |           private-key: ${{ secrets.CF_CURATOR_PRIVATE_KEY }}
57 |           owner: ${{ github.repository_owner }}
58 | 
59 |       - name: Process requests based on YAML files
60 |         if: steps.conversion_lock.outcome == 'success' && ! env.CI_SKIP
61 |         shell: bash -el {0}
62 |         run: |
63 |           git config --global user.email "79913779+conda-forge-curator[bot]@users.noreply.github.com"
64 |           git config --global user.name "conda-forge-curator[bot]"
65 |           git config --global pull.rebase false
66 |           python -m conda_forge_admin_requests run
67 |         env:
68 |           PROD_BINSTAR_TOKEN: ${{ secrets.PROD_BINSTAR_TOKEN }}
69 |           GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
70 |           CIRCLE_TOKEN: ${{ secrets.CIRCLE_TOKEN }}
71 |           TRAVIS_TOKEN: ${{ secrets.CF_ADMIN_TRAVIS_TOKEN }}
72 |           AZURE_TOKEN: ${{ secrets.AZURE_TOKEN }}
73 |           DRONE_TOKEN: ${{ secrets.DRONE_TOKEN }}
74 |           CIRUN_API_KEY: ${{ secrets.CIRUN_API_KEY }}
75 |           GITHUB_ADMIN_TOKEN: ${{ secrets.CF_ADMIN_GITHUB_TOKEN }}
76 |           CIRUN_INSTALLATION_ID: ${{ secrets.CIRUN_INSTALLATION_ID }}
77 |           STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }}
78 | 
79 |       - name: pull and push changes
80 |         if: steps.conversion_lock.outcome == 'success' && ! env.CI_SKIP
81 |         shell: bash -el {0}
82 |         run: |
83 |           git pull
84 |           git push
85 | 


--------------------------------------------------------------------------------
/scripts/update_repodata_patches.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import subprocess
  3 | import os
  4 | import tempfile
  5 | import github
  6 | import datetime
  7 | 
  8 | 
  9 | def _commit_to_patches(tmpdir):
 10 |     subprocess.check_call(
 11 |         ["git", "reset", "--hard", "HEAD"],
 12 |         cwd=os.path.join(tmpdir, "conda-forge-repodata-patches-feedstock"),
 13 |     )
 14 | 
 15 |     subprocess.check_call(
 16 |         ["git", "commit", "--allow-empty", "-am", "resync repo data for weekly cron-job"],
 17 |         cwd=os.path.join(tmpdir, "conda-forge-repodata-patches-feedstock"),
 18 |     )
 19 | 
 20 |     subprocess.check_call(
 21 |         ["git", "push"],
 22 |         cwd=os.path.join(tmpdir, "conda-forge-repodata-patches-feedstock"),
 23 |     )
 24 | 
 25 | 
 26 | def _post_issue_with_diff(diff):
 27 |     msg = f"""\
 28 | Hi! Our weekly job found a non-zero repodata patch diff:
 29 | 
 30 | <details>
 31 | 
 32 | ```
 33 | {diff}
 34 | ```
 35 | 
 36 | </details>
 37 | """
 38 | 
 39 |     today = datetime.date.today().strftime("%Y-%m-%d")
 40 |     gh = github.Github(os.environ['GITHUB_TOKEN'])
 41 |     repo = gh.get_repo("conda-forge/conda-forge-repodata-patches-feedstock")
 42 |     repo.create_issue(
 43 |         f"[{today}] non-zero repodata patch diff",
 44 |         body=msg,
 45 |     )
 46 | 
 47 | 
 48 | def update_repodata_patches(dry_run):
 49 | 
 50 |     skipme = [
 51 |         "================================================================================",
 52 |         "linux-armv7l",
 53 |         "linux-ppc64le",
 54 |         "linux-aarch64",
 55 |         "noarch",
 56 |         "win-32",
 57 |         "osx-arm64",
 58 |         "osx-64",
 59 |         "linux-64",
 60 |         "win-64",
 61 |     ]
 62 |     with tempfile.TemporaryDirectory() as tmpdir:
 63 |         subprocess.check_call(
 64 |             [
 65 |                 "git",
 66 |                 "clone",
 67 |                 "https://github.com/conda-forge/conda-forge-repodata-patches-feedstock.git",
 68 |             ],
 69 |             cwd=tmpdir,
 70 |         )
 71 | 
 72 |         origin_url = (
 73 |             f"https://x-access-token:{os.environ['GITHUB_TOKEN']}@github.com/"
 74 |             "conda-forge/conda-forge-repodata-patches-feedstock.git"
 75 |         )
 76 |         subprocess.check_call(
 77 |             [
 78 |                 "git",
 79 |                 "remote",
 80 |                 "set-url",
 81 |                 "--push",
 82 |                 "origin",
 83 |                 origin_url,
 84 |             ],
 85 |             cwd=os.path.join(tmpdir, "conda-forge-repodata-patches-feedstock"),
 86 |         )
 87 | 
 88 |         d = subprocess.check_output(
 89 |             ["python", "show_diff.py"],
 90 |             cwd=os.path.join(
 91 |                 tmpdir,
 92 |                 "conda-forge-repodata-patches-feedstock",
 93 |                 "recipe",
 94 |             ),
 95 |             text=True,
 96 |         )
 97 | 
 98 |         empty = True
 99 |         for line in d.splitlines():
100 |             line = line.strip()
101 |             if len(line) > 0 and not (
102 |                 line.startswith("Downloading")
103 |                 or line in skipme
104 |             ):
105 |                 empty = False
106 | 
107 |         print("diff:\n" + d, flush=True)
108 |         print("is empty:", empty, flush=True)
109 | 
110 |         if len(d) > 0 and not empty:
111 |             if not dry_run:
112 |                 _post_issue_with_diff(d)
113 |                 _commit_to_patches(tmpdir)
114 | 
115 | 
116 | if __name__ == "__main__":
117 |     if len(sys.argv) > 2:
118 |         raise RuntimeError("Need 0 or 1 arguments")
119 |     if len(sys.argv) == 2 and sys.argv[1] == '--dry-run':
120 |         dry_run = True
121 |     else:
122 |         dry_run = False
123 | 
124 |     update_repodata_patches(dry_run)
125 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | share/python-wheels/
 24 | *.egg-info/
 25 | .installed.cfg
 26 | *.egg
 27 | MANIFEST
 28 | 
 29 | # PyInstaller
 30 | #  Usually these files are written by a python script from a template
 31 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 32 | *.manifest
 33 | *.spec
 34 | 
 35 | # Installer logs
 36 | pip-log.txt
 37 | pip-delete-this-directory.txt
 38 | 
 39 | # Unit test / coverage reports
 40 | htmlcov/
 41 | .tox/
 42 | .nox/
 43 | .coverage
 44 | .coverage.*
 45 | .cache
 46 | nosetests.xml
 47 | coverage.xml
 48 | *.cover
 49 | *.py,cover
 50 | .hypothesis/
 51 | .pytest_cache/
 52 | cover/
 53 | 
 54 | # Translations
 55 | *.mo
 56 | *.pot
 57 | 
 58 | # Django stuff:
 59 | *.log
 60 | local_settings.py
 61 | db.sqlite3
 62 | db.sqlite3-journal
 63 | 
 64 | # Flask stuff:
 65 | instance/
 66 | .webassets-cache
 67 | 
 68 | # Scrapy stuff:
 69 | .scrapy
 70 | 
 71 | # Sphinx documentation
 72 | docs/_build/
 73 | 
 74 | # PyBuilder
 75 | .pybuilder/
 76 | target/
 77 | 
 78 | # Jupyter Notebook
 79 | .ipynb_checkpoints
 80 | 
 81 | # IPython
 82 | profile_default/
 83 | ipython_config.py
 84 | 
 85 | # pyenv
 86 | #   For a library or package, you might want to ignore these files since the code is
 87 | #   intended to run in multiple environments; otherwise, check them in:
 88 | # .python-version
 89 | 
 90 | # pipenv
 91 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 92 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 93 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 94 | #   install all needed dependencies.
 95 | #Pipfile.lock
 96 | 
 97 | # poetry
 98 | #   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
 99 | #   This is especially recommended for binary packages to ensure reproducibility, and is more
100 | #   commonly ignored for libraries.
101 | #   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102 | #poetry.lock
103 | 
104 | # pdm
105 | #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106 | #pdm.lock
107 | #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108 | #   in version control.
109 | #   https://pdm.fming.dev/#use-with-ide
110 | .pdm.toml
111 | 
112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113 | __pypackages__/
114 | 
115 | # Celery stuff
116 | celerybeat-schedule
117 | celerybeat.pid
118 | 
119 | # SageMath parsed files
120 | *.sage.py
121 | 
122 | # Environments
123 | .env
124 | .venv
125 | env/
126 | venv/
127 | ENV/
128 | env.bak/
129 | venv.bak/
130 | 
131 | # Spyder project settings
132 | .spyderproject
133 | .spyproject
134 | 
135 | # Rope project settings
136 | .ropeproject
137 | 
138 | # mkdocs documentation
139 | /site
140 | 
141 | # mypy
142 | .mypy_cache/
143 | .dmypy.json
144 | dmypy.json
145 | 
146 | # Pyre type checker
147 | .pyre/
148 | 
149 | # pytype static type analyzer
150 | .pytype/
151 | 
152 | # Cython debug symbols
153 | cython_debug/
154 | 
155 | # PyCharm
156 | #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157 | #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158 | #  and can be added to the global gitignore or merged into this file.  For a more nuclear
159 | #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
160 | #.idea/
161 | 
162 | .DS_Store
163 | 


--------------------------------------------------------------------------------
/conda_forge_admin_requests/cfep3_copy.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Copy approved artifacts from an external channel to production conda-forge.
  3 | """
  4 | 
  5 | import copy
  6 | import hmac
  7 | import os
  8 | import subprocess
  9 | from typing import Dict, Any
 10 | 
 11 | import requests
 12 | 
 13 | from .utils import split_label_from_channel, parse_filename
 14 | 
 15 | 
 16 | def check_one(package: str, sha256: str):
 17 |     if not isinstance(sha256, str) or len(sha256) != 64:
 18 |         raise ValueError(
 19 |             f"Key '{sha256}' must be SHA256 for the artifact (64 hexadecimal characters)"
 20 |         )
 21 | 
 22 |     channel_and_maybe_label, subdir, artifact = package.rsplit("/", 2)
 23 |     channel, _ = split_label_from_channel(channel_and_maybe_label)
 24 |     pkg_name, version, _, _ = parse_filename(artifact)
 25 | 
 26 |     # Check existence
 27 |     url = (
 28 |         f"https://conda-web.anaconda.org/{channel_and_maybe_label}/{subdir}/{artifact}"
 29 |     )
 30 |     r = requests.head(url)
 31 |     if not r.ok:
 32 |         raise ValueError(f"Package '{package}' at {channel} does not seem to exist")
 33 | 
 34 |     # Check SHA256
 35 |     r = requests.get(
 36 |         f"https://api.anaconda.org/dist/{channel}/{pkg_name}/{version}/{subdir}/{artifact}",
 37 |         timeout=10,
 38 |     )
 39 |     r.raise_for_status()
 40 |     api_sha256 = r.json()["sha256"]
 41 |     if not hmac.compare_digest(sha256, api_sha256):
 42 |         raise ValueError(
 43 |             f"User-provided SHA256 {sha256} does not match expected value {api_sha256}"
 44 |         )
 45 | 
 46 | 
 47 | def check(request: Dict[str, Any]) -> None:
 48 |     packages = request.get("anaconda_org_packages")
 49 |     if not packages or not isinstance(packages[0], dict):
 50 |         raise ValueError(
 51 |             "Must define 'anaconda_org_packages' as a list of dicts with keys "
 52 |             "{'package': channel/subdir/artifact, 'sha256': SHA256}"
 53 |         )
 54 |     for item in packages:
 55 |         if not item.get("package") or not item.get("sha256"):
 56 |             raise ValueError(
 57 |                 "Each 'anaconda_org_packages' entry must be a dict with keys "
 58 |                 "{'package': channel/subdir/artifact, 'sha256': SHA256}"
 59 |             )
 60 |         check_one(item["package"], item["sha256"])
 61 | 
 62 | 
 63 | def run(request: Dict[str, Any]) -> Dict[str, Any] | None:
 64 |     if "PROD_BINSTAR_TOKEN" not in os.environ:
 65 |         return copy.deepcopy(request)
 66 | 
 67 |     to_label = request.get("to_anaconda_org_label") or ()
 68 |     if to_label:
 69 |         to_label = ("--to-label", to_label)
 70 |     packages_to_try_again = []
 71 |     for item in request["anaconda_org_packages"]:
 72 |         check_one(item["package"], item["sha256"])
 73 |         channel_and_maybe_label, subdir, artifact = item["package"].rsplit("/", 2)
 74 |         channel, label = split_label_from_channel(channel_and_maybe_label)
 75 |         from_label = ("--from-label", label) if label != "main" else ()
 76 |         pkg_name, version, _, _ = parse_filename(artifact)
 77 |         spec = f"{channel}/{pkg_name}/{version}/{subdir}/{artifact}"
 78 |         cmd = [
 79 |             "anaconda",
 80 |             "--token",
 81 |             os.environ["PROD_BINSTAR_TOKEN"],
 82 |             "copy",
 83 |             "--to-owner",
 84 |             "conda-forge",
 85 |             *from_label,
 86 |             *to_label,
 87 |             spec,
 88 |         ]
 89 |         print("Copying", item["package"], "...")
 90 |         p = subprocess.run(cmd)
 91 |         if p.returncode == 0:
 92 |             print("... OK!")
 93 |         else:
 94 |             print("... failed!")
 95 |             packages_to_try_again.append(item)
 96 | 
 97 |     if packages_to_try_again:
 98 |         request = copy.deepcopy(request)
 99 |         request["anaconda_org_packages"] = packages_to_try_again
100 |         return request
101 |     else:
102 |         return None
103 | 


--------------------------------------------------------------------------------
/conda_forge_admin_requests/mark_broken.py:
--------------------------------------------------------------------------------
  1 | import subprocess
  2 | import os
  3 | import tempfile
  4 | import requests
  5 | import copy
  6 | 
  7 | 
  8 | def split_pkg(pkg):
  9 |     if pkg.endswith(".tar.bz2"):
 10 |         pkg = pkg[:-len(".tar.bz2")]
 11 |     elif pkg.endswith(".conda"):
 12 |         pkg = pkg[:-len(".conda")]
 13 |     else:
 14 |         raise RuntimeError("Can only process packages that end in .tar.bz2 or .conda!")
 15 |     plat, pkg_name = pkg.split("/")
 16 |     name_ver, build = pkg_name.rsplit('-', 1)
 17 |     name, ver = name_ver.rsplit('-', 1)
 18 |     return plat, name, ver, build
 19 | 
 20 | 
 21 | def check(request):
 22 |     action = request["action"]
 23 |     assert action in ("broken", "not_broken")
 24 | 
 25 |     if action == "broken":
 26 |         channel = "conda-forge"
 27 |     else:
 28 |         channel = "conda-forge/label/broken"
 29 | 
 30 |     assert "packages" in request
 31 |     pkgs = request["packages"]
 32 | 
 33 |     for pkg in pkgs:
 34 |         # check to ensure the artifact exists
 35 |         r = requests.head(f"https://conda.anaconda.org/conda-forge/{pkg}")
 36 |         r.raise_for_status()
 37 | 
 38 |         # check it is on the right channel
 39 |         plat, name, ver, build = split_pkg(pkg)
 40 |         env = os.environ.copy()
 41 |         env["CONDA_SUBDIR"] = plat
 42 |         subprocess.check_call(
 43 |             ["conda", "search", f"{name}={ver}={build}", "-c", channel, "--override-channels"],
 44 |             env=env,
 45 |         )
 46 | 
 47 | 
 48 | def mark_broken_pkg(pkg, action):
 49 |     plat, name, ver, build = split_pkg(pkg)
 50 | 
 51 |     if action == "broken":
 52 |         func = requests.post
 53 |     else:
 54 |         func = requests.delete
 55 | 
 56 |     r = func(
 57 |         "https://api.anaconda.org/channels/conda-forge/broken",
 58 |         headers={'Authorization': 'token {}'.format(os.environ["PROD_BINSTAR_TOKEN"])},
 59 |         json={
 60 |             "basename": pkg,
 61 |             "package": name,
 62 |             "version": ver,
 63 |         }
 64 |     )
 65 |     if r.status_code != 201:
 66 |         print(f"        could not mark {action}", flush=True)
 67 |         return False
 68 |     else:
 69 |         print(f"        marked {action}", flush=True)
 70 |         return True
 71 | 
 72 | 
 73 | def run(request):
 74 |     if "PROD_BINSTAR_TOKEN" not in os.environ:
 75 |         return copy.deepcopy(request)
 76 | 
 77 |     packages = request["packages"]
 78 |     action = request["action"]
 79 | 
 80 |     pkgs_to_try_again = []
 81 |     did_any = False
 82 |     for package in packages:
 83 |         print(f"working on package {package}", flush=True)
 84 |         success = mark_broken_pkg(package, action)
 85 |         if success:
 86 |             did_any = True
 87 |         else:
 88 |             pkgs_to_try_again.append(package)
 89 | 
 90 |     if did_any:
 91 |         with tempfile.TemporaryDirectory() as tmpdir:
 92 |             subprocess.check_call(
 93 |                 [
 94 |                     "git",
 95 |                     "clone", 
 96 |                     "https://github.com/conda-forge/conda-forge-repodata-patches-feedstock.git",
 97 |                 ],
 98 |                 cwd=tmpdir,
 99 |             )
100 | 
101 |             origin_url = (
102 |                 f"https://x-access-token:{os.environ['GITHUB_TOKEN']}@github.com/conda-forge/"
103 |                 "conda-forge-repodata-patches-feedstock.git"
104 |             )
105 |             subprocess.check_call(
106 |                 [
107 |                     "git",
108 |                     "remote",
109 |                     "set-url",
110 |                     "--push",
111 |                     "origin",
112 |                     origin_url,
113 |                 ],
114 |                 cwd=os.path.join(tmpdir, "conda-forge-repodata-patches-feedstock"),
115 |             )
116 | 
117 |             success_pkgs = set(packages) - set(pkgs_to_try_again)
118 |             fstr = " ".join(f for f in success_pkgs)
119 |             subprocess.check_call(
120 |                 [
121 |                     "git",
122 |                     "commit",
123 |                     "--allow-empty",
124 |                     "-am",
125 |                     f"resync repo data for broken/not-broken packages {fstr}",
126 |                 ],
127 |                 cwd=os.path.join(tmpdir, "conda-forge-repodata-patches-feedstock"),
128 |             )
129 | 
130 |             subprocess.check_call(
131 |                 ["git", "push"],
132 |                 cwd=os.path.join(tmpdir, "conda-forge-repodata-patches-feedstock"),
133 |             )
134 | 
135 |     if pkgs_to_try_again:
136 |         request = copy.deepcopy(request)
137 |         request["packages"] = pkgs_to_try_again
138 |         return request
139 |     else:
140 |         return None
141 | 


--------------------------------------------------------------------------------
/conda_forge_admin_requests/feedstock_outputs.py:
--------------------------------------------------------------------------------
  1 | import io
  2 | import json
  3 | import os
  4 | import requests
  5 | 
  6 | import ruamel.yaml
  7 | from conda_forge_metadata.feedstock_outputs import sharded_path as _get_sharded_path
  8 | import github
  9 | 
 10 | 
 11 | def _test_and_raise_besides_file_not_exists(e: github.GithubException):
 12 |     if isinstance(e, github.UnknownObjectException):
 13 |         return
 14 |     if e.status == 404 and "No object found" in e.data["message"]:
 15 |         return
 16 |     raise e
 17 | 
 18 | 
 19 | def _add_feedstock_output(
 20 |     feedstock,
 21 |     pkg_name,
 22 | ):
 23 |     gh_token = os.environ['GITHUB_TOKEN']
 24 |     gh = github.Github(auth=github.Auth.Token(gh_token))
 25 |     repo = gh.get_repo("conda-forge/feedstock-outputs")
 26 |     try:
 27 |         contents = repo.get_contents(_get_sharded_path(pkg_name))
 28 |     except github.GithubException as e:
 29 |         _test_and_raise_besides_file_not_exists(e)
 30 |         contents = None
 31 | 
 32 |     if contents is None:
 33 |         data = {"feedstocks": [feedstock]}
 34 |         repo.create_file(
 35 |             _get_sharded_path(pkg_name),
 36 |             f"[cf admin skip] ***NO_CI*** add output {pkg_name} for conda-forge/{feedstock}-feedstock",
 37 |             json.dumps(data),
 38 |         )
 39 |         print(f"    output {pkg_name} added for feedstock conda-forge/{feedstock}-feedstock", flush=True)
 40 |     else:
 41 |         data = json.loads(contents.decoded_content.decode("utf-8"))
 42 |         if feedstock not in data["feedstocks"]:
 43 |             data["feedstocks"].append(feedstock)
 44 |             repo.update_file(
 45 |                 contents.path,
 46 |                 f"[cf admin skip] ***NO_CI*** add output {pkg_name} for conda-forge/{feedstock}-feedstock",
 47 |                 json.dumps(data),
 48 |                 contents.sha,
 49 |             )
 50 |             print(f"    output {pkg_name} added for feedstock conda-forge/{feedstock}-feedstock", flush=True)
 51 |         else:
 52 |             print(f"    output {pkg_name} already exists for feedstock conda-forge/{feedstock}-feedstock", flush=True)
 53 | 
 54 | 
 55 | def _add_feedstock_output_glob(
 56 |     feedstock,
 57 |     glob_str,
 58 | ):
 59 |     gh_token = os.environ['GITHUB_TOKEN']
 60 |     gh = github.Github(auth=github.Auth.Token(gh_token))
 61 |     repo = gh.get_repo("conda-forge/feedstock-outputs")
 62 |     contents = repo.get_contents("feedstock_outputs_autoreg_allowlist.yml")
 63 | 
 64 |     yaml = ruamel.yaml.YAML(typ="rt")  # use round-trip to preserve comments
 65 |     data = yaml.load(contents.decoded_content.decode("utf-8"))
 66 |     current_globs = data.get(feedstock, [])
 67 |     if glob_str not in current_globs:
 68 |         current_globs.append(glob_str)
 69 |     data[feedstock] = current_globs
 70 |     fp = io.StringIO()
 71 |     yaml.dump(data, fp)
 72 |     repo.update_file(
 73 |         contents.path,
 74 |         f"[cf admin skip] ***NO_CI*** add glob {glob_str} for conda-forge/{feedstock}-feedstock",
 75 |         fp.getvalue(),
 76 |         contents.sha,
 77 |     )
 78 |     print(f"    glob {glob_str} added for feedstock conda-forge/{feedstock}-feedstock", flush=True)
 79 | 
 80 | 
 81 | def check(request):
 82 |     action = request["action"]
 83 |     assert action == "add_feedstock_output"
 84 | 
 85 |     assert request.get("feedstock_to_output_mapping")
 86 |     msg = "feedstock to output mapping syntax has changed and requires a dictionary now. See example."
 87 |     assert isinstance(request["feedstock_to_output_mapping"], dict), msg
 88 | 
 89 |     for feedstock, pkgs in request["feedstock_to_output_mapping"].items():
 90 |         if feedstock.endswith("-feedstock"):
 91 |             feedstock = feedstock[:-10]
 92 | 
 93 |         r = requests.head(
 94 |             f"https://github.com/conda-forge/{feedstock}-feedstock"
 95 |         )
 96 |         r.raise_for_status()
 97 |         for pkg_name in pkgs:
 98 |             if not isinstance(pkg_name, str):
 99 |                 raise ValueError(
100 |                     f"Value for '{feedstock}' entry must be a str (output name, or a glob), "
101 |                     f"but you provided {pkg_name:!r}."
102 |                 )
103 | 
104 | 
105 | def run(request):
106 |     action = request["action"]
107 |     assert action == "add_feedstock_output"
108 | 
109 |     assert request.get("feedstock_to_output_mapping")
110 |     items_to_keep = {}
111 |     for feedstock, pkgs in request["feedstock_to_output_mapping"].items():
112 |         if feedstock.endswith("-feedstock"):
113 |             feedstock = feedstock[:-10]
114 |         pkgs_to_keep = []
115 |         for pkg_name in pkgs:
116 |             try:
117 |                 if any(_c in pkg_name for _c in ["*", "?", "[", "]", "!"]):
118 |                     _add_feedstock_output_glob(feedstock, pkg_name)
119 |                 else:
120 |                     _add_feedstock_output(feedstock, pkg_name)
121 |             except Exception as e:
122 |                 print(f"    could not add output {pkg_name} for feedstock conda-forge/{feedstock}-feedstock: {e}", flush=True)
123 |                 pkgs_to_keep.append(pkg_name)
124 |         if pkgs_to_keep:
125 |             items_to_keep[feedstock] = pkgs_to_keep
126 | 
127 |     if items_to_keep:
128 |         request["feedstock_to_output_mapping"] = items_to_keep
129 |         return request
130 |     else:
131 |         return None
132 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # admin requests
 2 | 
 3 | [![repodata_patching](https://github.com/conda-forge/admin-requests/actions/workflows/repodata_patching.yml/badge.svg)](https://github.com/conda-forge/admin-requests/actions/workflows/repodata_patching.yml) [![run](https://github.com/conda-forge/admin-requests/actions/workflows/main.yml/badge.svg)](https://github.com/conda-forge/admin-requests/actions/workflows/main.yml) [![create_feedstocks](https://github.com/conda-forge/admin-requests/actions/workflows/create_feedstocks.yml/badge.svg)](https://github.com/conda-forge/admin-requests/actions/workflows/create_feedstocks.yml)
 4 | 
 5 | This repo is for making requests to `conda-forge/core` for various administrative
 6 | tasks.
 7 | 
 8 | 
 9 | ## Mark packages as broken on conda-forge
10 | 
11 | If you want to mark a package as broken on `conda-forge`, send a Pull Request
12 | adding a new `.yml` file in `requests` folder with a list of the full names of the packages
13 | for which the `broken` label will be added. See `examples/example-broken.yml` for an example.
14 | 
15 | Guidelines for marking packages as broken:
16 | 
17 | * If the package is functional but with incorrect metadata (e.g. missing dependencies), then
18 |   we prefer to patch the repo data (see [here](https://github.com/conda-forge/conda-forge-repodata-patches-feedstock))
19 |   instead of marking packages as broken. This alternative workflow makes environments more reproducible.
20 | * Packages with requirements/metadata that are too strict but otherwise work are
21 |   not technically broken and should not be marked as such.
22 | * Packages with missing metadata can be marked as broken on a temporary basis
23 |   but should be patched in the repo data and be marked unbroken later.
24 | * In some cases where the number of users of a package is small or it is used by
25 |   the maintainers only, we can allow packages to be marked broken more liberally.
26 | * We (`conda-forge/core`) try to make a decision on these requests within 24 hours.
27 | 
28 | 
29 | ## Mark packages as not broken on conda-forge
30 | 
31 | If you want to remove the broken label from packages on `conda-forge`, send a Pull Request
32 | adding a new `.yml` file in `requests` folder with a list of the full names of the packages
33 | for which the label `broken` will be removed. See `examples/example-not-broken.yml` for an example.
34 | 
35 | 
36 | ## Reset your Feedstock Token
37 | 
38 | If you want to reset your feedstock token to fix issues with uploads, send a Pull Request
39 | adding a new `.yml` file in `requests` folder with a list of the feedstock names
40 | without `-feedstock`. See `examples/example-token-reset.yml` for an example.
41 | (e.g., for `python-feedstock`, the feedstocks list must contain `python`).
42 | 
43 | 
44 | ## Archive or unarchive a feedstock
45 | 
46 | If you want to request a feedstock to be archived, send a Pull Request
47 | adding a new `.yml` file in `requests` folder with a list of the feedstock names
48 | without `-feedstock`. See `examples/example-archive.yml` for an example.
49 | (e.g., for `python-feedstock`, the feedstocks list must contain `python`).
50 | For unarchiving, see `examples/example-unarchive.yml` for an example.
51 | 
52 | For feedstocks that need to be archived, please leave an open issue with some details about
53 | why that decision was taken (e.g. it has been deprecated by a new feedstock),
54 | and link it in your PR description.
55 | 
56 | 
57 | ## Request / revoke access to CI resources
58 | 
59 | Certain CI resources are opt-in only. If you want to request access to these resources, please
60 | submit a PR adding your feedstock name to a new `.yml` file in `requests` folder.
61 | 
62 | Available opt-in resources:
63 | 
64 | - Travis CI: See `examples/example-travis.yml`
65 | - [`open-gpu-server`](https://github.com/Quansight/open-gpu-server) (includes GPU CI and long-running builds): See `examples/example-open-gpu-server.yml`.
66 | 
67 | ## Request a CFEP-3 copy to conda-forge
68 | 
69 | CFEP-3 specifies the process to add foreign builds to conda-forge. [Read the CFEP](https://github.com/conda-forge/cfep/blob/main/cfep-03.md) for more details.
70 | This workflow allows users to request a copy once the manual review has been passed.
71 | 
72 | To do so, please create a new `.yml` file in the `requests` folder. Check `examples/example-cfep-3.yml` for the required metadata.
73 | 
74 | For provenance and transparency, the PR description must include a link to the original PR and the logs, along with the artifact(s) to be reviewed.
75 | 
76 | ## Add a package output to a feedstock
77 | 
78 | By default, `conda-forge` feedstocks cannot push packages to our channel that another feedstock makes. If you encountered an error
79 | when building your package indicating that the given package was not allowed for your feedstock (e.g., you moved a package
80 | build from one feedstock to another), you should request the output be added to the new feedstock via this repository. An example request
81 | is located in [examples/example-add-feedstock-output.yml](examples/example-add-feedstock-output.yml). You can add both glob patterns
82 | and package names.
83 | 
84 | While glob patterns are support, they should be used with care as they
85 | essentially "squat" on all future matched. If you are requesting a specific
86 | package output, please use the full name of the package output. We support the
87 | glob syntax of the Python `fnmatch` module. Make a PR putting your `.yml`
88 | request file in the `requests` directory and the `conda-forge/core` team will
89 | review it.
90 | 


--------------------------------------------------------------------------------
/conda_forge_admin_requests/token_reset.py:
--------------------------------------------------------------------------------
  1 | import copy
  2 | import os
  3 | import requests
  4 | import subprocess
  5 | import tempfile
  6 | import github
  7 | 
  8 | from .utils import write_secrets_to_files
  9 | 
 10 | 
 11 | FEEDSTOCK_TOKENS_REPO = None
 12 | 
 13 | 
 14 | def feedstock_token_exists(feedstock_name):
 15 |     r = requests.get(
 16 |         "https://api.github.com/repos/conda-forge/"
 17 |         "feedstock-tokens/contents/tokens/%s.json" % (feedstock_name),
 18 |         headers={"Authorization": "token %s" % os.environ["GITHUB_TOKEN"]},
 19 |     )
 20 |     if r.status_code != 200:
 21 |         return False
 22 |     else:
 23 |         return True
 24 | 
 25 | 
 26 | def get_feedstock_token_repo():
 27 |     global FEEDSTOCK_TOKENS_REPO
 28 | 
 29 |     if "GITHUB_TOKEN" not in os.environ:
 30 |         raise RuntimeError(
 31 |             "Cannot delete feedstock token since "
 32 |             "we do not have a github token!"
 33 |         )
 34 | 
 35 |     if FEEDSTOCK_TOKENS_REPO is None:
 36 |         FEEDSTOCK_TOKENS_REPO = (
 37 |             github
 38 |             .Github(os.environ["GITHUB_TOKEN"])
 39 |             .get_repo("conda-forge/feedstock-tokens")
 40 |         )
 41 | 
 42 |     return FEEDSTOCK_TOKENS_REPO
 43 | 
 44 | 
 45 | def delete_feedstock_token(feedstock_name):
 46 |     feedstock_tokens_repo = get_feedstock_token_repo()
 47 | 
 48 |     token_file = "tokens/%s.json" % feedstock_name
 49 |     fn = feedstock_tokens_repo.get_contents(token_file)
 50 |     feedstock_tokens_repo.delete_file(
 51 |         token_file,
 52 |         "[ci skip] [skip ci] [cf admin skip] ***NO_CI*** removing "
 53 |         "token for %s" % feedstock_name,
 54 |         fn.sha,
 55 |     )
 56 | 
 57 | 
 58 | def reset_feedstock_token(
 59 |     name,
 60 |     skips=None,
 61 |     unique_token_per_provider=False,
 62 |     existing_tokens_time_to_expiration=None,
 63 | ):
 64 |     from conda_smithy.ci_register import travis_get_repo_info
 65 |     skips = skips or []
 66 | 
 67 |     if "travis" not in skips:
 68 |         # test to make sure travis ci api is working
 69 |         # if not skip migration
 70 |         repo_info = travis_get_repo_info("conda-forge", name + "-feedstock")
 71 |         if not repo_info:
 72 |             raise RuntimeError("Travis-CI API token is not working!")
 73 | 
 74 |     owner_info = ['--organization', 'conda-forge']
 75 |     token_repo = (
 76 |         f"https://x-access-token:{os.environ['GITHUB_TOKEN']}@github.com/"
 77 |         "conda-forge/feedstock-tokens"
 78 |     )
 79 |     if unique_token_per_provider:
 80 |         uniq_args = ['--unique-token-per-provider']
 81 |     else:
 82 |         uniq_args = []
 83 |     if existing_tokens_time_to_expiration:
 84 |         expire_args = [
 85 |             '--existing-tokens-time-to-expiration',
 86 |             str(existing_tokens_time_to_expiration),
 87 |         ]
 88 |     else:
 89 |         expire_args = []
 90 | 
 91 |     with tempfile.TemporaryDirectory() as tmpdir:
 92 |         feedstock_dir = os.path.join(tmpdir, name + "-feedstock")
 93 |         os.makedirs(feedstock_dir)
 94 | 
 95 |         if (
 96 |             feedstock_token_exists(name + "-feedstock")
 97 |             and (
 98 |                 existing_tokens_time_to_expiration is None
 99 |                 or int(existing_tokens_time_to_expiration) <= 0
100 |             )
101 |         ):
102 |             delete_feedstock_token(name + "-feedstock")
103 | 
104 |         subprocess.check_call(
105 |             [
106 |                 'conda', 'smithy', 'generate-feedstock-token',
107 |                 '--feedstock_directory', feedstock_dir
108 |             ]
109 |             + owner_info
110 |             + uniq_args
111 |         )
112 |         subprocess.check_call(
113 |             [
114 |                 'conda', 'smithy', 'register-feedstock-token',
115 |                 '--without-circle', '--without-drone',
116 |             ]
117 |             + [
118 |                 f"--without-{s.replace('_', '-')}" for s in skips
119 |                 if s not in [
120 |                     "circle",
121 |                     "drone",
122 |                 ]
123 |             ]
124 |             + [
125 |                 '--feedstock_directory', feedstock_dir,
126 |             ]
127 |             + owner_info
128 |             + ['--token_repo', token_repo]
129 |             + uniq_args
130 |             + expire_args
131 |         )
132 | 
133 |         subprocess.check_call(
134 |             [
135 |                 'conda', 'smithy', 'rotate-binstar-token',
136 |                 '--without-appveyor', '--without-azure',
137 |                 '--without-circle', '--without-drone',
138 |                 '--without-github-actions',
139 |             ]
140 |             + [
141 |                 f"--without-{s.replace('_', '-')}" for s in skips
142 |                 if s not in [
143 |                     "circle",
144 |                     "drone",
145 |                     "appveyor",
146 |                     "azure",
147 |                     "github_actions",
148 |                 ]
149 |             ]
150 |             + [
151 |                 '--token_name', 'STAGING_BINSTAR_TOKEN'
152 |             ],
153 |             cwd=feedstock_dir)
154 | 
155 | 
156 | def run(request):
157 |     check(request)
158 |     write_secrets_to_files()
159 | 
160 |     feedstocks = request["feedstocks"]
161 | 
162 |     skips = request.get("skip_providers", [])
163 |     unique_token_per_provider = True
164 |     existing_tokens_time_to_expiration = request.get(
165 |         "existing_tokens_time_to_expiration", None
166 |     )
167 | 
168 |     feedstocks_to_do_again = []
169 | 
170 |     for feedstock in feedstocks:
171 |         try:
172 |             reset_feedstock_token(
173 |                 feedstock,
174 |                 skips=skips,
175 |                 existing_tokens_time_to_expiration=existing_tokens_time_to_expiration,
176 |                 unique_token_per_provider=unique_token_per_provider,
177 |             )
178 |         except Exception as e:
179 |             print(
180 |                 "failed to reset token for '%s': %s" % (feedstock, repr(e)),
181 |                 flush=True,
182 |             )
183 |             feedstocks_to_do_again.append(feedstock)
184 | 
185 |     if feedstocks_to_do_again:
186 |         request = copy.deepcopy(request)
187 |         request["feedstocks"] = feedstocks_to_do_again
188 |         return request
189 |     else:
190 |         return None
191 | 
192 | 
193 | def check(request):
194 |     assert "feedstocks" in request
195 |     feedstocks = request["feedstocks"]
196 |     missing_feedstocks = []
197 | 
198 |     for feedstock in feedstocks:
199 |         r = requests.get(
200 |             f"https://github.com/conda-forge/{feedstock}-feedstock"
201 |         )
202 |         if r.status_code != 200:
203 |             missing_feedstocks.append(feedstock)
204 | 
205 |     if missing_feedstocks:
206 |         raise RuntimeError(
207 |             f"feedstocks {missing_feedstocks} could not be found!"
208 |         )
209 | 


--------------------------------------------------------------------------------
/conda_forge_admin_requests/access_control.py:
--------------------------------------------------------------------------------
  1 | """
  2 | This script will process the `travis` and `cirun` requests.
  3 | 
  4 | Main logic lives in conda-smithy. This is just a wrapper for admin-requests infra.
  5 | """
  6 | import os
  7 | import subprocess
  8 | import tempfile
  9 | import time
 10 | from typing import Dict, List, Any
 11 | import textwrap
 12 | import copy
 13 | from unittest import mock
 14 | 
 15 | import requests
 16 | 
 17 | from .utils import write_secrets_to_files
 18 | 
 19 | from conda_smithy.utils import update_conda_forge_config
 20 | from conda_smithy.github import Github, gh_token
 21 | 
 22 | 
 23 | GH_ORG = os.environ.get("GH_ORG", "conda-forge")
 24 | 
 25 | DEFAULT_CIRUN_OPENSTACK_VALUES = {
 26 |     "cirun_roles": ["admin", "maintain", "write"],
 27 |     "cirun_users_from_json": ["https://raw.githubusercontent.com/Quansight/open-gpu-server/main/access/conda-forge-users.json"]
 28 | }
 29 | 
 30 | 
 31 | def send_pr_cirun(
 32 |     feedstock: str,
 33 |     feedstock_dir: str,
 34 |     resources: List[str],
 35 |     pull_request: bool,
 36 | ) -> None:
 37 |     """
 38 |     Send PR to feedstock to enable Github Actions with Cirun
 39 | 
 40 |     Parameters:
 41 |     feedstock (str): The name of the feedstock.
 42 |     feedstock_dir (str): Path to a git checkout of the feedstock.
 43 |     resources (List[str]): The names of the resources for access control.
 44 |     pull_request (bool): Whether to allow Pull Requests.
 45 |     """
 46 | 
 47 |     with update_conda_forge_config(
 48 |             os.path.join(
 49 |                 feedstock_dir,
 50 |                 "recipe",
 51 |                 "conda_build_config.yaml")) as cbc, \
 52 |             update_conda_forge_config(
 53 |                 os.path.join(feedstock_dir, "conda-forge.yml")) as cfg:
 54 |         if any(label.startswith("cirun-") for label in cbc.get(
 55 |                 "github_actions_labels", [])):
 56 |             return
 57 |         cfg["github_actions"] = {"self_hosted": True}
 58 |         if pull_request:
 59 |             cfg["github_actions"]["triggers"] = ["push", "pull_request"]
 60 |         if "provider" not in cfg:
 61 |             cfg["provider"] = {}
 62 |         cfg["provider"]["linux_64"] = "github_actions"
 63 |         cbc["github_actions_labels"] = resources
 64 | 
 65 |     gh = Github(os.environ['GITHUB_TOKEN'])
 66 |     user = gh.get_user()
 67 | 
 68 |     repo = gh.get_repo(f"{GH_ORG}/{feedstock}")
 69 |     repo.create_fork()
 70 | 
 71 |     base_branch = f"cirun-{int(time.time())}"
 72 | 
 73 |     resource_str = ", ".join(resources)
 74 | 
 75 |     git_cmds = [
 76 |         ["git", "add", "recipe/conda_build_config.yaml", "conda-forge.yml"],
 77 |         ["git", "remote", "add", user.login, 
 78 |          f"https://x-access-token:{os.environ['GITHUB_TOKEN']}@github.com/{user.login}/{feedstock}.git"
 79 |         ],
 80 |         ["git", "commit", "-m", f"Enable {resource_str} using Cirun", "--author", f"{user.name} <{user.email}>"],
 81 |         ["conda-smithy", "rerender", "-c", "auto", "--no-check-uptodate"],
 82 |         ["git", "push", user.login, f"HEAD:{base_branch}"],
 83 |     ]
 84 |     for git_cmd in git_cmds:
 85 |         print("Running:", git_cmd, "in", feedstock_dir)
 86 |         subprocess.check_call(git_cmd, cwd=feedstock_dir)
 87 | 
 88 |     print("Creating PR:")
 89 |     repo.create_pull(
 90 |         base="main",
 91 |         head=f"{user.login}:{base_branch}",
 92 |         title=f"Update feedstock to use {resource_str} with Cirun",
 93 |         body=textwrap.dedent("""
 94 |         Note that only builds triggered by maintainers of the feedstock (and core)
 95 |         who have accepted the terms of service and privacy policy will run
 96 |         on Github actions via Cirun.
 97 |         - [ ] Maintainers have accepted the terms of service and privacy policy
 98 |           at https://github.com/Quansight/open-gpu-server
 99 |         """),
100 |     )
101 | 
102 | 
103 | def _process_request_for_feedstock(
104 |     feedstock: str,
105 |     action: str,
106 |     resources: List[str] = None,
107 |     revoke: bool = False,
108 |     pull_request: bool = False,
109 |     send_pr: bool = True,
110 | ) -> None:
111 |     """
112 |     Process the access control request for a single feedstock.
113 | 
114 |     Parameters:
115 |     feedstock (str): The name of the feedstock.
116 |     resources (List[str]): The names of the resources for access control.
117 |     revoke (bool): Whether to remove the access control.
118 |     pull_request (bool): Whether to allow PRs for resource.
119 |     """
120 | 
121 |     # We need a token with admin permissions for Cirun
122 |     with tempfile.TemporaryDirectory() as tmp_dir, \
123 |             mock.patch.dict('os.environ', {'GITHUB_TOKEN': os.environ['GITHUB_ADMIN_TOKEN']}):
124 |         feedstock_dir = os.path.join(tmp_dir, feedstock)
125 |         assert GH_ORG
126 |         clone_cmd = [
127 |             "git",
128 |             "clone",
129 |             "--depth",
130 |             "1",
131 |             f"https://github.com/{GH_ORG}/{feedstock}.git",
132 |             feedstock_dir,
133 |         ]
134 |         print("Cloning:", *clone_cmd)
135 |         subprocess.check_call(clone_cmd)
136 | 
137 |         owner_info = ["--organization", GH_ORG]
138 |         token_repo = (
139 |             f"https://x-access-token:{os.environ['GITHUB_TOKEN']}@github.com/"
140 |             f"{GH_ORG}/feedstock-tokens"
141 |         )
142 | 
143 |         register_ci_cmd = [
144 |             "conda-smithy",
145 |             "register-ci",
146 |             "--feedstock_dir",
147 |             feedstock_dir,
148 |             "--without-all",
149 |             "--without-anaconda-token",
150 |             *owner_info,
151 |         ]
152 |         if action == "travis":
153 |             register_ci_cmd.append("--with-travis")
154 | 
155 |         elif action == "cirun":
156 |             register_ci_cmd.append("--with-cirun")
157 | 
158 |             for resource in resources:
159 |                 register_ci_cmd.extend(["--cirun-resources", resource])
160 |                 assert resource.startswith("cirun-"), f"Unknown resource {resource}"
161 | 
162 |             # this part is specific to github.com/Quansight/open-gpu-server
163 |             if all(resource.startswith("cirun-openstack") for resource in resources):
164 |                 for key, value in DEFAULT_CIRUN_OPENSTACK_VALUES.items():
165 |                     for arg in value:
166 |                         register_ci_cmd.extend((f"--{key.replace('_', '-')}", arg))
167 |             elif all(resource.startswith("cirun-") for resource in resources):
168 |                 pass
169 |             else:
170 |                 assert False, f"Unknown resources {resources}"
171 | 
172 |             if pull_request:
173 |                 register_ci_cmd.extend(("--cirun-policy-args", "pull_request"))
174 | 
175 |             if revoke:
176 |                 register_ci_cmd.append("--remove")
177 | 
178 |         print("Register-CI command:", *register_ci_cmd)
179 |         subprocess.check_call(register_ci_cmd)
180 | 
181 |         if not revoke:
182 |             if action == "travis":
183 |                 with_cmd = "--with-travis"
184 |             elif action == "cirun":
185 |                 with_cmd = "--with-github-actions"
186 | 
187 |             print("Generating a new feedstock token")
188 |             subprocess.check_call(
189 |                 [
190 |                     'conda', 'smithy', 'generate-feedstock-token',
191 |                     '--unique-token-per-provider',
192 |                     '--feedstock_directory', feedstock_dir,
193 |                     *owner_info,
194 |                 ]
195 |             )
196 | 
197 |             print("Register new feedstock token with provider and feedstock-tokens repo.")
198 |             subprocess.check_call(
199 |                 [
200 |                     'conda', 'smithy', 'register-feedstock-token',
201 |                     '--unique-token-per-provider',
202 |                     '--feedstock_directory', feedstock_dir,
203 |                     '--without-all', with_cmd,
204 |                     *owner_info,
205 |                     '--token_repo', token_repo,
206 |                 ]
207 |             )
208 | 
209 |             if action == "travis":
210 |                 print("Add STAGING_BINSTAR_TOKEN to travis")
211 |                 subprocess.check_call(
212 |                     [
213 |                         'conda', 'smithy', 'rotate-binstar-token',
214 |                         '--feedstock_directory', feedstock_dir,
215 |                         '--without-all', with_cmd,
216 |                         *owner_info,
217 |                         '--token_name', 'STAGING_BINSTAR_TOKEN',
218 |                     ]
219 |                 )
220 | 
221 |             if action == "cirun" and send_pr:
222 |                 print("Sending PR to feedstock")
223 |                 send_pr_cirun(feedstock, feedstock_dir, resources, pull_request)
224 | 
225 | 
226 | def check_if_repo_exists(feedstock_name: str) -> None:
227 |     """
228 |     Check if a repository exists on GitHub.
229 | 
230 |     Parameters:
231 |     feedstock_name (str): The name of the feedstock to check.
232 | 
233 |     Raises:
234 |     ValueError: If the repository does not exist on GitHub.
235 |     """
236 |     repo = f"{feedstock_name}-feedstock"
237 |     owner_repo = f"{GH_ORG}/{repo}"
238 |     print(f"Checking if {owner_repo} exists")
239 |     response = requests.get(f"https://api.github.com/repos/{owner_repo}")
240 |     if response.status_code != 200:
241 |         raise ValueError(f"Repository: {owner_repo} not found!")
242 | 
243 | 
244 | def check(request: Dict[str, Any]) -> None:
245 |     """Check if the access control requests in both 'grant_access'
246 |     and 'revoke_access' directories are valid."""
247 |     print("Checking access control request")
248 |     assert "feedstocks" in request
249 |     feedstocks = request["feedstocks"]
250 |     for feedstock in feedstocks:
251 |         check_if_repo_exists(feedstock)
252 | 
253 |     action = request["action"]
254 |     assert action in ("travis", "cirun"), f"Unknown action {action}"
255 | 
256 |     if action == "cirun":
257 |         assert "resources" in request, "No resources field in request"
258 |         resources = request["resources"]
259 |         assert resources, "Empty resources"
260 |         for resource in resources:
261 |             assert resource.startswith("cirun-")
262 | 
263 |     if action == "travis":
264 |         assert not request.get("revoke", False)
265 | 
266 | 
267 | def run(request: Dict[str, Any]) -> None:
268 |     """
269 |     The main function to process the access control requests. It performs the following steps:
270 |     1. Check if the requests are valid.
271 |     2. Process the access control requests.
272 |     """
273 |     check(request)
274 |     write_secrets_to_files()
275 | 
276 |     feedstocks = request["feedstocks"]
277 |     for feedstock in feedstocks:
278 |         request_copy = copy.deepcopy(request)
279 |         del request_copy["feedstocks"]
280 |         _process_request_for_feedstock(f"{feedstock}-feedstock", **request_copy)
281 | 


--------------------------------------------------------------------------------
/scripts/create_feedstocks.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | """
  3 | Convert all recipes into feedstocks.
  4 | 
  5 | This script is to be run in a TravisCI context, with all secret environment
  6 | variables defined (STAGING_BINSTAR_TOKEN, GH_TOKEN)
  7 | 
  8 | Such as:
  9 | 
 10 |     export GH_TOKEN=$(cat ~/.conda-smithy/github.token)
 11 | 
 12 | """
 13 | 
 14 | from __future__ import print_function
 15 | from __future__ import annotations
 16 | 
 17 | import json
 18 | from pathlib import Path
 19 | from typing import Iterator
 20 | from conda_build.metadata import MetaData
 21 | from rattler_build_conda_compat.render import MetaData as RattlerBuildMetaData
 22 | from conda_smithy.utils import get_feedstock_name_from_meta
 23 | from contextlib import contextmanager
 24 | from datetime import datetime, timezone
 25 | from github import Github, GithubException
 26 | import os.path
 27 | import shutil
 28 | import subprocess
 29 | import sys
 30 | import tempfile
 31 | import traceback
 32 | import time
 33 | 
 34 | import github
 35 | import requests
 36 | from ruamel.yaml import YAML
 37 | from conda_forge_feedstock_ops.parse_package_and_feedstock_names import (
 38 |     parse_package_and_feedstock_names,
 39 | )
 40 | from conda_forge_metadata.feedstock_outputs import sharded_path as _get_sharded_path
 41 | from conda_build.utils import create_file_with_permissions
 42 | 
 43 | # Enable DEBUG to run the diagnostics, without actually creating new feedstocks.
 44 | DEBUG = False
 45 | 
 46 | REPO_SKIP_LIST = ["core", "bot", "staged-recipes", "arm-arch", "systems", "ctx"]
 47 | 
 48 | recipe_directory_name = "recipes"
 49 | 
 50 | 
 51 | def _test_and_raise_besides_file_not_exists(e: github.GithubException):
 52 |     if isinstance(e, github.UnknownObjectException):
 53 |         return
 54 |     if e.status == 404 and "No object found" in e.data["message"]:
 55 |         return
 56 |     raise e
 57 | 
 58 | 
 59 | def _register_package_for_feedstock(feedstock, pkg_name, gh):
 60 |     repo = gh.get_repo("conda-forge/feedstock-outputs")
 61 |     try:
 62 |         contents = repo.get_contents(_get_sharded_path(pkg_name))
 63 |     except github.GithubException as e:
 64 |         _test_and_raise_besides_file_not_exists(e)
 65 |         contents = None
 66 | 
 67 |     if contents is None:
 68 |         data = {"feedstocks": [feedstock]}
 69 |         repo.create_file(
 70 |             _get_sharded_path(pkg_name),
 71 |             f"[cf admin skip] ***NO_CI*** add output {pkg_name} for conda-forge/{feedstock}-feedstock",
 72 |             json.dumps(data),
 73 |         )
 74 |         print(f"    output {pkg_name} added for feedstock {feedstock}", flush=True)
 75 |     else:
 76 |         # we proceed anyways and do not raise since it could be a rerun of staged recipes
 77 |         # print a warning for the users
 78 |         data = json.loads(contents.decoded_content.decode("utf-8"))
 79 |         print(
 80 |             f"    WARNING: output {pkg_name} already exists from feedstock(s) {data['feedstocks']}",
 81 |             flush=True,
 82 |         )
 83 | 
 84 | 
 85 | def list_recipes() -> Iterator[tuple[str, str]]:
 86 |     """
 87 |     Locates all the recipes in the `recipes/` folder at the root of the repository.
 88 | 
 89 |     For each found recipe this function returns a tuple consisting of
 90 |     * the path to the recipe directory
 91 |     * the name of the feedstock
 92 |     """
 93 |     repository_root = Path(__file__).parent.parent.parent.parent.absolute()
 94 |     repository_recipe_dir = repository_root / recipe_directory_name
 95 | 
 96 |     # Ignore if the recipe directory does not exist.
 97 |     if not repository_recipe_dir.is_dir:
 98 |         return
 99 | 
100 |     for recipe_dir in repository_recipe_dir.iterdir():
101 |         # We don't list the "example" feedstock. It is an example, and is there
102 |         # to be helpful.
103 |         # .DS_Store is created by macOS to store custom attributes of its
104 |         # containing folder.
105 |         if recipe_dir.name in ["example", "example-v1", ".DS_Store"]:
106 |             continue
107 | 
108 |         # Try to look for a conda-build recipe.
109 |         absolute_feedstock_path = repository_recipe_dir / recipe_dir
110 |         try:
111 |             yield (
112 |                 str(absolute_feedstock_path),
113 |                 get_feedstock_name_from_meta(MetaData(absolute_feedstock_path)),
114 |             )
115 |             continue
116 |         except OSError:
117 |             pass
118 | 
119 |         # If no conda-build recipe was found, try to load a rattler-build recipe.
120 |         yield (
121 |             str(absolute_feedstock_path),
122 |             get_feedstock_name_from_meta(RattlerBuildMetaData(absolute_feedstock_path)),
123 |         )
124 | 
125 | 
126 | @contextmanager
127 | def tmp_dir(*args, **kwargs):
128 |     temp_dir = tempfile.mkdtemp(*args, **kwargs)
129 |     try:
130 |         yield temp_dir
131 |     finally:
132 |         shutil.rmtree(temp_dir)
133 | 
134 | 
135 | def repo_exists(gh, organization, name):
136 |     # Use the organization provided.
137 |     org = gh.get_organization(organization)
138 |     try:
139 |         org.get_repo(name)
140 |         return True
141 |     except GithubException as e:
142 |         if e.status == 404:
143 |             return False
144 |         raise
145 | 
146 | 
147 | def repo_default_branch(gh, organization, name):
148 |     # Use the organization provided.
149 |     org = gh.get_organization(organization)
150 |     try:
151 |         repo = org.get_repo(name)
152 |         return repo.default_branch
153 |     except GithubException as e:
154 |         if e.status == 404:
155 |             return "main"
156 |         raise
157 | 
158 | 
159 | def _set_default_branch(feedstock_dir, default_branch):
160 |     yaml = YAML()
161 |     with open(os.path.join(feedstock_dir, "conda-forge.yml"), "r") as fp:
162 |         cfg = yaml.load(fp.read())
163 | 
164 |     if "github" not in cfg:
165 |         cfg["github"] = {}
166 |     cfg["github"]["branch_name"] = default_branch
167 |     cfg["github"]["tooling_branch_name"] = "main"
168 | 
169 |     if (
170 |         "upload_on_branch" in cfg
171 |         and cfg["upload_on_branch"] != default_branch
172 |         and cfg["upload_on_branch"] in ["master", "main"]
173 |     ):
174 |         cfg["upload_on_branch"] = default_branch
175 | 
176 |     if "conda_build" not in cfg:
177 |         cfg["conda_build"] = {}
178 | 
179 |     if "error_overlinking" not in cfg["conda_build"]:
180 |         cfg["conda_build"]["error_overlinking"] = True
181 | 
182 |     with open(os.path.join(feedstock_dir, "conda-forge.yml"), "w") as fp:
183 |         yaml.dump(cfg, fp)
184 | 
185 | 
186 | def feedstock_token_exists(organization, name):
187 |     r = requests.get(
188 |         "https://api.github.com/repos/%s/"
189 |         "feedstock-tokens/contents/tokens/%s.json" % (organization, name),
190 |         headers={"Authorization": "token %s" % os.environ["GH_TOKEN"]},
191 |     )
192 |     if r.status_code != 200:
193 |         return False
194 |     else:
195 |         return True
196 | 
197 | 
198 | def get_rate_limit(gh):
199 |     # Get GitHub API Rate Limit usage and total
200 |     rate_limit = gh.get_rate_limit()
201 | 
202 |     # unfortunately, PyGithub doesn't have a __version__
203 |     # so we have to use hasattr/getattr tricks to
204 |     # ensure compatibility with older versions
205 | 
206 |     # PyGithub 2.7.0 and later
207 |     # get_rate_limit return RateLimitOverview
208 |     # https://github.com/PyGithub/PyGithub/pull/3205
209 |     # If the attribute "resources" exists, we return it (2.7.0 and later).
210 |     # if it doesn't, we return the rate_limit object itself (older versions).
211 |     return getattr(rate_limit, "resources", rate_limit)
212 | 
213 | 
214 | def print_rate_limiting_info(gh, user):
215 |     # Compute some info about our GitHub API Rate Limit.
216 |     # Note that it doesn't count against our limit to
217 |     # get this info. So, we should be doing this regularly
218 |     # to better know when it is going to run out. Also,
219 |     # this will help us better understand where we are
220 |     # spending it and how to better optimize it.
221 | 
222 |     rate_limit = get_rate_limit(gh)
223 |     gh_api_remaining = rate_limit.core.remaining
224 |     gh_api_total = rate_limit.core.limit
225 | 
226 |     # Compute time until GitHub API Rate Limit reset
227 |     gh_api_reset_time = rate_limit.core.reset
228 |     gh_api_reset_time -= datetime.now(timezone.utc)
229 | 
230 |     print("")
231 |     print("GitHub API Rate Limit Info:")
232 |     print("---------------------------")
233 |     print("token: ", user)
234 |     print(
235 |         "Currently remaining {remaining} out of {total}.".format(
236 |             remaining=gh_api_remaining, total=gh_api_total
237 |         )
238 |     )
239 |     print("Will reset in {time}.".format(time=gh_api_reset_time))
240 |     print("")
241 |     return gh_api_remaining
242 | 
243 | 
244 | def sleep_until_reset(gh):
245 |     # sleep the job with printing every minute if we are out
246 |     # of github api requests
247 | 
248 |     rate_limit = get_rate_limit(gh)
249 |     gh_api_remaining = rate_limit.core.remaining
250 | 
251 |     if gh_api_remaining == 0:
252 |         gh_api_reset_time = rate_limit.core.reset
253 |         gh_api_reset_time -= datetime.now(timezone.utc)
254 | 
255 |         mins_to_sleep = int(gh_api_reset_time.total_seconds() / 60)
256 |         mins_to_sleep += 2
257 | 
258 |         print("Sleeping until GitHub API resets.")
259 |         for i in range(mins_to_sleep):
260 |             time.sleep(60)
261 |             print(
262 |                 "slept for minute {curr} out of {tot}.".format(
263 |                     curr=i + 1, tot=mins_to_sleep
264 |                 )
265 |             )
266 |         return True
267 |     else:
268 |         return False
269 | 
270 | 
271 | if __name__ == "__main__":
272 |     exit_code = 0
273 | 
274 |     is_merged_pr = os.environ.get("CF_CURRENT_BRANCH") == "main"
275 | 
276 |     smithy_conf = os.path.expanduser("~/.conda-smithy")
277 |     if not os.path.exists(smithy_conf):
278 |         os.mkdir(smithy_conf)
279 | 
280 |     def write_token(name, token):
281 |         path = os.path.join(smithy_conf, name + ".token")
282 |         with create_file_with_permissions(path, 0o600) as fh:
283 |             fh.write(token)
284 | 
285 |     if "APPVEYOR_TOKEN" in os.environ:
286 |         write_token("appveyor", os.environ["APPVEYOR_TOKEN"])
287 |     if "CIRCLE_TOKEN" in os.environ:
288 |         write_token("circle", os.environ["CIRCLE_TOKEN"])
289 |     if "AZURE_TOKEN" in os.environ:
290 |         write_token("azure", os.environ["AZURE_TOKEN"])
291 |     if "DRONE_TOKEN" in os.environ:
292 |         write_token("drone", os.environ["DRONE_TOKEN"])
293 |     if "TRAVIS_TOKEN" in os.environ:
294 |         write_token("travis", os.environ["TRAVIS_TOKEN"])
295 |     if "STAGING_BINSTAR_TOKEN" in os.environ:
296 |         write_token("anaconda", os.environ["STAGING_BINSTAR_TOKEN"])
297 | 
298 |     # gh_drone = Github(os.environ['GH_DRONE_TOKEN'])
299 |     # gh_drone_remaining = print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
300 | 
301 |     # gh_travis = Github(os.environ['GH_TRAVIS_TOKEN'])
302 |     gh_travis = None
303 | 
304 |     gh = None
305 |     if "GH_TOKEN" in os.environ:
306 |         write_token("github", os.environ["GH_TOKEN"])
307 |         gh = Github(os.environ["GH_TOKEN"])
308 | 
309 |         # Get our initial rate limit info.
310 |         gh_remaining = print_rate_limiting_info(gh, "GH_TOKEN")
311 | 
312 |         # if we are out, exit early
313 |         # if sleep_until_reset(gh):
314 |         #     sys.exit(1)
315 | 
316 |         # try the other token maybe?
317 |         # if gh_remaining < gh_drone_remaining and gh_remaining < 100:
318 |         #     write_token('github', os.environ['GH_DRONE_TOKEN'])
319 |         #     gh = Github(os.environ['GH_DRONE_TOKEN'])
320 | 
321 |     owner_info = ["--organization", "conda-forge"]
322 | 
323 |     print("Calculating the recipes which need to be turned into feedstocks.")
324 |     with tmp_dir("__feedstocks") as feedstocks_dir:
325 |         feedstock_dirs = []
326 |         for recipe_dir, name in list_recipes():
327 |             if name.lower() in REPO_SKIP_LIST:
328 |                 continue
329 |             if name.lower() == "ctx":
330 |                 sys.exit(1)
331 | 
332 |             feedstock_dir = os.path.join(feedstocks_dir, name + "-feedstock")
333 |             print("Making feedstock for {}".format(name))
334 |             try:
335 |                 subprocess.check_call(
336 |                     [
337 |                         "conda",
338 |                         "smithy",
339 |                         "init",
340 |                         recipe_dir,
341 |                         "--feedstock-directory",
342 |                         feedstock_dir,
343 |                     ]
344 |                 )
345 |             except subprocess.CalledProcessError:
346 |                 traceback.print_exception(*sys.exc_info())
347 |                 continue
348 | 
349 |             if not is_merged_pr:
350 |                 # We just want to check that conda-smithy is doing its
351 |                 # thing without having any metadata issues.
352 |                 continue
353 | 
354 |             subprocess.check_call(
355 |                 [
356 |                     "git",
357 |                     "remote",
358 |                     "add",
359 |                     "upstream_with_token",
360 |                     "https://conda-forge-manager:{}@github.com/"
361 |                     "conda-forge/{}-feedstock".format(os.environ["GH_TOKEN"], name),
362 |                 ],
363 |                 cwd=feedstock_dir,
364 |             )
365 |             # print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
366 | 
367 |             # Sometimes we already have the feedstock created. We need to
368 |             # deal with that case.
369 |             if repo_exists(gh, "conda-forge", name + "-feedstock"):
370 |                 default_branch = repo_default_branch(
371 |                     gh, "conda-forge", name + "-feedstock"
372 |                 )
373 |                 subprocess.check_call(
374 |                     ["git", "fetch", "upstream_with_token"], cwd=feedstock_dir
375 |                 )
376 |                 subprocess.check_call(
377 |                     ["git", "branch", "-m", default_branch, "old"], cwd=feedstock_dir
378 |                 )
379 |                 try:
380 |                     subprocess.check_call(
381 |                         [
382 |                             "git",
383 |                             "checkout",
384 |                             "-b",
385 |                             default_branch,
386 |                             "upstream_with_token/%s" % default_branch,
387 |                         ],
388 |                         cwd=feedstock_dir,
389 |                     )
390 |                 except subprocess.CalledProcessError:
391 |                     # Sometimes, we have a repo, but there are no commits on
392 |                     # it! Just catch that case.
393 |                     subprocess.check_call(
394 |                         ["git", "checkout", "-b", default_branch], cwd=feedstock_dir
395 |                     )
396 |             else:
397 |                 default_branch = "main"
398 | 
399 |             feedstock_dirs.append([feedstock_dir, name, recipe_dir, default_branch])
400 | 
401 |             # print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
402 | 
403 |             # set the default branch in the conda-forge.yml
404 |             _set_default_branch(feedstock_dir, default_branch)
405 | 
406 |             # now register with github
407 |             subprocess.check_call(
408 |                 ["conda", "smithy", "register-github", feedstock_dir] + owner_info
409 |                 # hack to help travis work
410 |                 # + ['--extra-admin-users', gh_travis.get_user().login]
411 |                 # end of hack
412 |             )
413 |             # print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
414 | 
415 |             if gh:
416 |                 # Get our final rate limit info.
417 |                 print_rate_limiting_info(gh, "GH_TOKEN")
418 | 
419 |         # drone doesn't run our jobs any more so no reason to do this
420 |         # from conda_smithy.ci_register import drone_sync
421 |         # print("Running drone sync (can take ~100s)", flush=True)
422 |         # print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
423 |         # drone_sync()
424 |         # for _drone_i in range(10):
425 |         #     print(
426 |         #         "syncing drone - %d seconds left" % (10*(10 - _drone_i)),
427 |         #         flush=True,
428 |         #     )
429 |         #     time.sleep(10)  # actually wait
430 |         # print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
431 | 
432 |         # Break the previous loop to allow the TravisCI registering
433 |         # to take place only once per function call.
434 |         # Without this, intermittent failures to synch the TravisCI repos ensue.
435 |         # Hang on to any CI registration errors that occur and raise them at the end.
436 |         for num, (feedstock_dir, name, recipe_dir, default_branch) in enumerate(
437 |             feedstock_dirs
438 |         ):
439 |             if name.lower() in REPO_SKIP_LIST:
440 |                 continue
441 |             print("\n\nregistering CI services for %s..." % name)
442 |             if num >= 10:
443 |                 exit_code = 0
444 |                 break
445 |             # Try to register each feedstock with CI.
446 |             # However sometimes their APIs have issues for whatever reason.
447 |             # In order to bank our progress, we note the error and handle it.
448 |             # After going through all the recipes and removing the converted ones,
449 |             # we fail the build so that people are aware that things did not clear.
450 | 
451 |             # hack to help travis work
452 |             # from conda_smithy.ci_register import add_project_to_travis
453 |             # add_project_to_travis("conda-forge", name + "-feedstock")
454 |             # print_rate_limiting_info(gh_travis, 'GH_TRAVIS_TOKEN')
455 |             # end of hack
456 | 
457 |             try:
458 |                 subprocess.check_call(
459 |                     [
460 |                         "conda",
461 |                         "smithy",
462 |                         "register-ci",
463 |                         "--without-appveyor",
464 |                         "--without-circle",
465 |                         "--without-drone",
466 |                         "--without-cirun",
467 |                         "--without-webservice",
468 |                         "--feedstock_directory",
469 |                         feedstock_dir,
470 |                     ]
471 |                     + owner_info
472 |                 )
473 |                 subprocess.check_call(
474 |                     ["conda", "smithy", "rerender", "--no-check-uptodate"],
475 |                     cwd=feedstock_dir,
476 |                 )
477 |             except subprocess.CalledProcessError:
478 |                 exit_code = 0
479 |                 traceback.print_exception(*sys.exc_info())
480 |                 continue
481 | 
482 |             # slow down so we make sure we are registered
483 |             for i in range(1, 13):
484 |                 time.sleep(10)
485 |                 print("Waiting for registration: {i} s".format(i=i * 10))
486 | 
487 |             # if we get here, now we make the feedstock token and add the staging token
488 |             print("making the feedstock token and adding the staging binstar token")
489 |             try:
490 |                 if not feedstock_token_exists("conda-forge", name + "-feedstock"):
491 |                     subprocess.check_call(
492 |                         [
493 |                             "conda",
494 |                             "smithy",
495 |                             "generate-feedstock-token",
496 |                             "--unique-token-per-provider",
497 |                             "--feedstock_directory",
498 |                             feedstock_dir,
499 |                         ]
500 |                         + owner_info
501 |                     )
502 |                     subprocess.check_call(
503 |                         [
504 |                             "conda",
505 |                             "smithy",
506 |                             "register-feedstock-token",
507 |                             "--unique-token-per-provider",
508 |                             "--without-circle",
509 |                             "--without-drone",
510 |                             "--feedstock_directory",
511 |                             feedstock_dir,
512 |                         ]
513 |                         + owner_info
514 |                     )
515 | 
516 |                 # add staging token env var to all CI probiders except appveyor
517 |                 # and azure
518 |                 # azure has it by default and appveyor is not used
519 |                 subprocess.check_call(
520 |                     [
521 |                         "conda",
522 |                         "smithy",
523 |                         "rotate-binstar-token",
524 |                         "--without-appveyor",
525 |                         "--without-azure",
526 |                         "--without-github-actions",
527 |                         "--without-circle",
528 |                         "--without-drone",
529 |                         "--token_name",
530 |                         "STAGING_BINSTAR_TOKEN",
531 |                     ],
532 |                     cwd=feedstock_dir,
533 |                 )
534 | 
535 |                 yaml = YAML()
536 |                 with open(os.path.join(feedstock_dir, "conda-forge.yml"), "r") as fp:
537 |                     _cfg = yaml.load(fp.read())
538 |                 _cfg["conda_forge_output_validation"] = True
539 |                 with open(os.path.join(feedstock_dir, "conda-forge.yml"), "w") as fp:
540 |                     yaml.dump(_cfg, fp)
541 |                 subprocess.check_call(
542 |                     ["git", "add", "conda-forge.yml"], cwd=feedstock_dir
543 |                 )
544 |                 subprocess.check_call(
545 |                     ["conda", "smithy", "rerender", "--no-check-uptodate"],
546 |                     cwd=feedstock_dir,
547 |                 )
548 | 
549 |                 # pre-register outputs
550 |                 print("registering outputs...")
551 |                 _, pkg_names, _ = parse_package_and_feedstock_names(
552 |                     feedstock_dir, use_container=False
553 |                 )
554 |                 for pkg_name in pkg_names:
555 |                     _register_package_for_feedstock(name, pkg_name, gh)
556 |             except subprocess.CalledProcessError:
557 |                 exit_code = 0
558 |                 traceback.print_exception(*sys.exc_info())
559 |                 continue
560 | 
561 |             print("making a commit and pushing...")
562 |             subprocess.check_call(
563 |                 [
564 |                     "git",
565 |                     "commit",
566 |                     "--allow-empty",
567 |                     "-am",
568 |                     "Re-render the feedstock after CI registration.",
569 |                 ],
570 |                 cwd=feedstock_dir,
571 |             )
572 |             for i in range(5):
573 |                 try:
574 |                     # Capture the output, as it may contain the GH_TOKEN.
575 |                     out = subprocess.check_output(
576 |                         [
577 |                             "git",
578 |                             "push",
579 |                             "upstream_with_token",
580 |                             "HEAD:%s" % default_branch,
581 |                         ],
582 |                         cwd=feedstock_dir,
583 |                         stderr=subprocess.STDOUT,
584 |                     )
585 |                     break
586 |                 except subprocess.CalledProcessError:
587 |                     pass
588 | 
589 |                 # Likely another job has already pushed to this repo.
590 |                 # Place our changes on top of theirs and try again.
591 |                 out = subprocess.check_output(
592 |                     ["git", "fetch", "upstream_with_token", default_branch],
593 |                     cwd=feedstock_dir,
594 |                     stderr=subprocess.STDOUT,
595 |                 )
596 |                 try:
597 |                     subprocess.check_call(
598 |                         [
599 |                             "git",
600 |                             "rebase",
601 |                             "upstream_with_token/%s" % default_branch,
602 |                             default_branch,
603 |                         ],
604 |                         cwd=feedstock_dir,
605 |                     )
606 |                 except subprocess.CalledProcessError:
607 |                     # Handle rebase failure by choosing the changes in default_branch.
608 |                     subprocess.check_call(
609 |                         ["git", "checkout", default_branch, "--", "."],
610 |                         cwd=feedstock_dir,
611 |                     )
612 |                     subprocess.check_call(
613 |                         ["git", "rebase", "--continue"], cwd=feedstock_dir
614 |                     )
615 | 
616 |             # Remove this recipe from the repo.
617 |             if is_merged_pr:
618 |                 subprocess.check_call(["git", "rm", "-rf", recipe_dir])
619 |                 # hack to help travis work
620 |                 # from conda_smithy.ci_register import travis_cleanup
621 |                 # travis_cleanup("conda-forge", name + "-feedstock")
622 |                 # end of hack
623 | 
624 |             if gh:
625 |                 # Get our final rate limit info.
626 |                 print_rate_limiting_info(gh, "GH_TOKEN")
627 | 
628 |     # Update status based on the remote.
629 |     subprocess.check_call(["git", "stash", "--keep-index", "--include-untracked"])
630 |     subprocess.check_call(["git", "fetch"])
631 |     # CBURR: Debugging
632 |     subprocess.check_call(["git", "status"])
633 |     subprocess.check_call(["git", "rebase", "--autostash"])
634 |     subprocess.check_call(["git", "add", "."])
635 |     try:
636 |         subprocess.check_call(["git", "stash", "pop"])
637 |     except subprocess.CalledProcessError:
638 |         # In case there was nothing to stash.
639 |         # Finish quietly.
640 |         pass
641 | 
642 |     # Parse `git status --porcelain` to handle some merge conflicts and
643 |     # generate the removed recipe list.
644 |     changed_files = subprocess.check_output(
645 |         ["git", "status", "--porcelain", recipe_directory_name], universal_newlines=True
646 |     )
647 |     changed_files = changed_files.splitlines()
648 | 
649 |     # Add all files from AU conflicts. They are new files that we
650 |     # weren't tracking previously.
651 |     # Adding them resolves the conflict and doesn't actually add anything to the index.
652 |     new_file_conflicts = filter(lambda _: _.startswith("AU "), changed_files)
653 |     new_file_conflicts = map(
654 |         lambda _: _.replace("AU", "", 1).lstrip(), new_file_conflicts
655 |     )
656 |     for each_new_file in new_file_conflicts:
657 |         subprocess.check_call(["git", "add", each_new_file])
658 | 
659 |     # Generate a fresh listing of recipes removed.
660 |     #
661 |     # * Each line we get back is a change to a file in the recipe directory.
662 |     # * We narrow the list down to recipes that are staged for deletion
663 |     #   (ignores examples).
664 |     # * Then we clean up the list so that it only has the recipe names.
665 |     removed_recipes = filter(lambda _: _.startswith("D "), changed_files)
666 |     removed_recipes = map(lambda _: _.replace("D", "", 1).lstrip(), removed_recipes)
667 |     removed_recipes = map(
668 |         lambda _: os.path.relpath(_, recipe_directory_name), removed_recipes
669 |     )
670 |     removed_recipes = map(lambda _: _.split(os.path.sep)[0], removed_recipes)
671 |     removed_recipes = sorted(set(removed_recipes))
672 | 
673 |     # Commit any removed packages.
674 |     subprocess.check_call(["git", "status"])
675 |     if removed_recipes:
676 |         msg = "Removed recipe{s} ({}) after converting into feedstock{s}.".format(
677 |             ", ".join(removed_recipes), s=("s" if len(removed_recipes) > 1 else "")
678 |         )
679 |         msg += " [ci skip]"
680 |         if is_merged_pr:
681 |             # Capture the output, as it may contain the GH_TOKEN.
682 |             out = subprocess.check_output(
683 |                 [
684 |                     "git",
685 |                     "remote",
686 |                     "add",
687 |                     "upstream_with_token",
688 |                     "https://x-access-token:{}@github.com/"
689 |                     "conda-forge/staged-recipes".format(os.environ["GH_TOKEN"]),
690 |                 ],
691 |                 stderr=subprocess.STDOUT,
692 |             )
693 |             subprocess.check_call(["git", "commit", "-m", msg])
694 |             # Capture the output, as it may contain the GH_TOKEN.
695 |             branch = os.environ.get("CF_CURRENT_BRANCH")
696 |             out = subprocess.check_output(
697 |                 ["git", "push", "upstream_with_token", "HEAD:%s" % branch],
698 |                 stderr=subprocess.STDOUT,
699 |             )
700 |         else:
701 |             print("Would git commit, with the following message: \n   {}".format(msg))
702 | 
703 |     if gh:
704 |         # Get our final rate limit info.
705 |         print_rate_limiting_info(gh, "GH_TOKEN")
706 |     # if gh_drone:
707 |     #     print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
708 |     # if gh_travis:
709 |     #     print_rate_limiting_info(gh_travis, 'GH_TRAVIS_TOKEN')
710 | 
711 |     sys.exit(exit_code)
712 | 


--------------------------------------------------------------------------------